SSL configuration for cloudere manager and services manually is tedious process, since there are many services cloudere Manager/Hue/hive/Imapala/solr/oozie/hdfs and their properties prviatekey/publickey/keypassword/truststorekey/ we need to update.
To make it simple using python api , we can configure all the setup without taking much time.
Steps:
1) first install and configure the anconda python
2) install the cm-api python module offered by cloudera
pip install cm-api
Then create below script and run
#!/usr/bin/env python
import socket
from cm_api.api_client import ApiResource
from cm_api.api_client import ApiException
from cm_api.endpoints.cms import ClouderaManager
from cm_api.endpoints.services import ApiService
import ssl
import json
import sys
CM_HOST = "cm.tanu.com"
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
cxt = ssl.create_default_context(cafile="ca_trust_store.pem")
#api = ApiResource(CM_HOST,version=12, username="admin", password="admin",use_tls=True,ssl_context=cxt) ### IF CM already configured with SSL
api = ApiResource(CM_HOST,version=12, username="admin", password="admin") #For Non-ssl CM
############ Update the keystore and trustore and pem file location for each service ###########
hdfs_ssl_enable = { 'ssl_client_truststore_location':'/opt/pki/etc/tca/test123.jks','ssl_client_truststore_password':'test123','hdfs_hadoop_ssl_enabled' : 'true','ssl_server_keystore_location' : '/app/opt/cloudera/certs/jks/javakeystore.jks','ssl_server_keystore_password':'test123','ssl_server_keystore_keypassword':'test123' }
hdfs_httpfs_ssl_enable = { 'httpfs_https_truststore_file':'/opt/pki/etc/tca/test123.jks','httpfs_https_truststore_password':'test123','httpfs_use_ssl' : 'true','httpfs_https_keystore_file' : '/app/opt/cloudera/certs/jks/javakeystore.jks','httpfs_https_keystore_password':'test123' }
yarn_ssl_enable = { 'ssl_server_keystore_location' : '/app/opt/cloudera/certs/jks/javakeystore.jks','ssl_server_keystore_password':'test123','ssl_server_keystore_keypassword':'test123' }
cm_ssl_conf = {'WEB_TLS':'true','KEYSTORE_PATH':'/opt/cloudera-manager/ssl/jks/javakeystore.jks','KEYSTORE_PASSWORD':'test123','TRUSTSTORE_PATH':'/opt/pki/etc/tca/test123.jks','TRUSTSTORE_PASSWORD':'test123'}
cm_managed_service = {'ssl_client_truststore_location':'/opt/pki/etc/tca/test123.jks','ssl_client_truststore_password':'test123'}
hbase_ssl_enable = { 'hbase_hadoop_ssl_enabled' : 'true','ssl_server_keystore_location' : '/app/opt/cloudera/certs/jks/javakeystore.jks','ssl_server_keystore_password':'test123','ssl_server_keystore_keypassword':'test123' }
oozie_role_ssl_enable = { 'oozie_https_truststore_file':'/opt/pki/etc/tca/test123.jks','oozie_https_truststore_password':'test123','oozie_https_keystore_file' : '/app/opt/cloudera/certs/jks/javakeystore.jks','oozie_https_keystore_password':'test123' }
oozie_ssl_enable = { 'oozie_use_ssl' : 'true'}
hive_ssl_enable={'hiveserver2_keystore_path':'/app/opt/cloudera/certs/jks/javakeystore.jks','hiveserver2_keystore_password':'test123','hiveserver2_truststore_file':'/opt/pki/etc/tca/test123.jks','hiveserver2_truststore_password':'test123','hiveserver2_enable_ssl':'true'}
solr_ssl_enable={'solr_https_keystore_file':'/app/opt/cloudera/certs/jks/javakeystore.jks','solr_https_keystore_password':'test123','solr_https_truststore_file':'/opt/pki/etc/tca/test123.jks','solr_https_truststore_password':'test123','solr_use_ssl':'true'}
impala_ssl_enable={"client_services_ssl_enabled": "true", "ssl_server_certificate": "/app/opt/cloudera/certs/pem/cert.pem","ssl_private_key_password": "test123", "ssl_client_ca_certificate": "/opt/pki/etc/tca/test123.pem", "ssl_private_key": "/app/opt/cloudera/certs/pem/key.pem"}
impala_BASE_ssl_enable={"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "webserver_private_key_password_cmd": "test123"}
impala_STATESTORE_ssl_enable={"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem","webserver_private_key_password_cmd": "test123"}
impala_CATALOGSERVER_ssl_enable={"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem","webserver_private_key_password_cmd": "test123"}
hue_SERVER_role_enable_ssl={"ssl_certificate": "/app/opt/cloudera/certs/pem/cert.pem", "ssl_private_key_password": "test123", "ssl_cacerts": "/opt/pki/etc/tca/test123.pem", "ssl_enable": "true", "ssl_private_key": "/app/opt/cloudera/certs/pem/key.pem"}
### UPDATE CLOUDERA MANAGER ##
cm=ClouderaManager(api)
cm.update_config(cm_ssl_conf)
#### UPDATE HDFS SSL CONFIG ###
hdfs=clu.get_service('hdfs')
hdfs.update_config(hdfs_ssl_enable)
### UPDATE HTTPFS SSL CONFIG ###
httpfs_role_group=hdfs.get_role_config_group("hdfs-HTTPFS-BASE")
httpfs_role_group.update_config(hdfs_httpfs_ssl_enable)
### UPDATE YARN SSL CONFIG ###
yarn=clu.get_service('yarn')
yarn.update_config(yarn_ssl_enable)
#### UPDATE HBASE SSL CONFIG ###
print("Updating Hbase SSL Config")
hbase=clu.get_service('hbase')
hbase.update_config(hbase_ssl_enable)
print(hbase.get_config())
#### UPDATE OOZIE SSL CONFIG ###
oozie=clu.get_service('oozie')
oozie.update_config(oozie_ssl_enable)
oozie_role_group=oozie.get_role_config_group("oozie-OOZIE_SERVER-BASE")
oozie_role_group.update_config(oozie_role_ssl_enable)
#### UPDATE HIVE SSL CONFIG ###
hive=clu.get_service('hive')
hive.update_config(hive_ssl_enable)
#### UPDATE solr SSL CONFIG ###
solr=clu.get_service('solr')
solr.update_config(solr_ssl_enable)
### IMPALA SSL UPDATE ###
impala=clu.get_service('impala')
impala.update_config(impala_ssl_enable)
impala_IMPALAD_role_group=impala.get_role_config_group("impala-IMPALAD-BASE")
impala_STATESTORE_role_group=impala.get_role_config_group("impala-STATESTORE-BASE")
impala_CATALOGSERVER_role_group=impala.get_role_config_group("impala-CATALOGSERVER-BASE")
impala_IMPALAD_role_group.update_config(impala_BASE_ssl_enable)
impala_STATESTORE_role_group.update_config(impala_STATESTORE_ssl_enable)
impala_CATALOGSERVER_role_group.update_config(impala_CATALOGSERVER_ssl_enable)
### UPDATE HUE SSL ###
hue=clu.get_service('hue')
hue_SERVER_role_group=hue.get_role_config_group("hue-HUE_SERVER-BASE")
hue_SERVER_role_group.update_config(hue_SERVER_role_enable_ssl)
#############################
you can also use this script to update other configurations as well. you can use below line to dump the existing configurations and update with your own value and run it.
to dump the existing configuration :
For Impala :
impala=clu.get_service("impala")
y=impala.get_config(view="summary")
json.dump(y, sys.stdout)
for role in impala.get_all_role_config_groups():
print(role)
print("--------------------------------")
x=role.get_config(view="summary")
json.dump(x, sys.stdout)
output
[{"client_services_ssl_enabled": "true", "ssl_server_certificate": "/app/opt/cloudera/certs/pem/cert.pem", "admission_control_enabled": "true", "hbase_service": "hbase", "hive_service": "hive", "hdfs_service": "hdfs", "ssl_private_key_password": "test123", "ssl_client_ca_certificate": "/opt/pki/etc/tca/test123.pem", "ssl_private_key": "/app/opt/cloudera/certs/pem/key.pem", "all_admission_control_enabled": "true"}, {}]<ApiRoleConfigGroup>: impala-IMPALAD-BASE (cluster: cluster; service: impala)
--------------------------------
{"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "impalad_memory_limit": "17179869184", "enable_audit_event_log": "true", "scratch_dirs": "/app/hadoop/impala/impalad", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "log_dir": "/app/var/log/impalad", "lineage_event_log_dir": "/app/var/log/impalad/lineage", "webserver_private_key_password_cmd": "test123"}<ApiRoleConfigGroup>: impala-STATESTORE-BASE (cluster: cluster; service: impala)
--------------------------------
{"webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "log_threshold": "DEBUG", "webserver_private_key_password_cmd": "test123", "log_dir": "/app/var/log/statestore", "webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem"}<ApiRoleConfigGroup>: impala-CATALOGSERVER-BASE (cluster: cluster; service: impala)
--------------------------------
{"catalogd_embedded_jvm_heapsize": "34359738368", "webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "load_catalog_in_background": "true", "log_dir": "/app/var/log/catalogd", "oom_heap_dump_enabled": "false", "webserver_private_key_password_cmd": "test123"}<ApiRoleConfigGroup>: impala-LLAMA-BASE (cluster: cluster; service: impala)
For HDFS :
hdfs=clu.get_service("hdfs")
y=hdfs.get_config(view="summary")
json.dump(y, sys.stdout)
for role in hdfs.get_all_role_config_groups():
print(role)
print("--------------------------------")
x=role.get_config(view="summary")
json.dump(x, sys.stdout)
Output
[{"hdfs_hadoop_ssl_enabled": "true", "core_site_safety_valve": "<property> <name>hadoop.user.group.static.mapping.overrides</name> <value>dr.who=;mapred=mapred,hadoop;impala=impala,hive,yarn,hadoop;</value>\r\n</property>\r\n", "ssl_server_keystore_password": "test123", "kms_service": "kms", "dfs_namenode_acls_enabled": "true", "ssl_server_keystore_keypassword": "test123", "dfs_block_local_path_access_user": "impala", "ssl_server_keystore_location": "/app/opt/cloudera/certs/jks/javakeystore.jks", "ssl_client_truststore_password": "test123", "audit_event_log_dir": "/app/var/log/hadoop-hdfs/audit", "dfs_replication": "1", "ssl_client_truststore_location": "/opt/pki/etc/tca/test123.jks"}, {}]<ApiRoleConfigGroup>: hdfs-DATANODE-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"dfs_datanode_max_xcievers": "8192", "oom_heap_dump_enabled": "false", "dfs_datanode_data_dir_perm": "755", "datanode_log_dir": "/app/var/log/hadoop-hdfs", "dfs_datanode_volume_choosing_policy": "org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy", "dfs_data_dir_list": "/app/hadoop/data/d01/dfs/dn,/app/hadoop/data/d02/dfs/dn,/app/hadoop/data/d03/dfs/dn,/app/hadoop/data/d04/dfs/dn,/app/hadoop/data/d05/dfs/dn,/app/hadoop/data/d06/dfs/dn,/app/hadoop/data/d07/dfs/dn,/app/hadoop/data/d08/dfs/dn,/app/hadoop/data/d09/dfs/dn,/app/hadoop/data/d10/dfs/dn,/app/hadoop/data/d11/dfs/dn,/app/hadoop/data/d12/dfs/dn,/app/hadoop/data/d13/dfs/dn,/app/hadoop/data/d14/dfs/dn,/app/hadoop/data/d15/dfs/dn,/app/hadoop/data/d16/dfs/dn,/app/hadoop/data/d17/dfs/dn,/app/hadoop/data/d18/dfs/dn,/app/hadoop/data/d19/dfs/dn,/app/hadoop/data/d20/dfs/dn", "dfs_datanode_failed_volumes_tolerated": "10"}<ApiRoleConfigGroup>: hdfs-NAMENODE-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"dfs_name_dir_list": "/app/nn", "dfs_namenode_servicerpc_address": "8022", "namenode_log_dir": "/app/var/log/hadoop-hdfs", "fs_trash_interval": "60", "oom_heap_dump_enabled": "false", "dfs_safemode_min_datanodes": "0"}<ApiRoleConfigGroup>: hdfs-FAILOVERCONTROLLER-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"oom_heap_dump_enabled": "false"}<ApiRoleConfigGroup>: hdfs-BALANCER-BASE (cluster: cluster; service: hdfs)
--------------------------------
{}<ApiRoleConfigGroup>: hdfs-GATEWAY-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"hdfs_client_config_safety_valve": "<property>\r\n<name>dfs.client.block.write.replace-datanode-on-failure.enable</name>\r\n<value>NEVER</value>\r\n</property>", "dfs_client_use_trash": "true"}<ApiRoleConfigGroup>: hdfs-SECONDARYNAMENODE-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"secondarynamenode_log_dir": "/app/var/log/hadoop-hdfs", "oom_heap_dump_enabled": "false", "fs_checkpoint_dir_list": "/app/nn/dfs/snn"}<ApiRoleConfigGroup>: hdfs-JOURNALNODE-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"oom_heap_dump_enabled": "false"}<ApiRoleConfigGroup>: hdfs-HTTPFS-BASE (cluster: cluster; service: hdfs)
--------------------------------
{"httpfs_https_keystore_file": "/app/opt/cloudera/certs/jks/javakeystore.jks", "httpfs_https_keystore_password": "test123", "httpfs_https_truststore_password": "test123", "httpfs_https_truststore_file": "/opt/pki/etc/tca/test123.jks", "oom_heap_dump_enabled": "false", "httpfs_use_ssl": "true"}<ApiRoleConfigGroup>: hdfs-NFSGATEWAY-BASE (cluster: cluster; service: hdfs)
