configure cloudera services/ssl using python scripting.

SSL configuration for  cloudere manager and services manually is tedious process, since there are many services cloudere Manager/Hue/hive/Imapala/solr/oozie/hdfs and their properties prviatekey/publickey/keypassword/truststorekey/ we need to update.

To make it simple using python api , we can configure all the setup without taking much time.

Steps:

1) first install and configure the anconda python
2) install the cm-api python module offered by cloudera
     pip install cm-api

Then create below script and run


#!/usr/bin/env python

import socket
from cm_api.api_client import ApiResource
from cm_api.api_client import ApiException
from cm_api.endpoints.cms import ClouderaManager
from cm_api.endpoints.services import ApiService
import ssl
import json
import sys

CM_HOST = "cm.tanu.com"
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
cxt = ssl.create_default_context(cafile="ca_trust_store.pem")

#api = ApiResource(CM_HOST,version=12, username="admin", password="admin",use_tls=True,ssl_context=cxt) ### IF CM already configured with SSL
api = ApiResource(CM_HOST,version=12, username="admin", password="admin") #For Non-ssl CM

############ Update the keystore and trustore and pem file location for each service ###########

hdfs_ssl_enable = { 'ssl_client_truststore_location':'/opt/pki/etc/tca/test123.jks','ssl_client_truststore_password':'test123','hdfs_hadoop_ssl_enabled' : 'true','ssl_server_keystore_location' : '/app/opt/cloudera/certs/jks/javakeystore.jks','ssl_server_keystore_password':'test123','ssl_server_keystore_keypassword':'test123' }
hdfs_httpfs_ssl_enable = { 'httpfs_https_truststore_file':'/opt/pki/etc/tca/test123.jks','httpfs_https_truststore_password':'test123','httpfs_use_ssl' : 'true','httpfs_https_keystore_file' : '/app/opt/cloudera/certs/jks/javakeystore.jks','httpfs_https_keystore_password':'test123' }
yarn_ssl_enable = { 'ssl_server_keystore_location' : '/app/opt/cloudera/certs/jks/javakeystore.jks','ssl_server_keystore_password':'test123','ssl_server_keystore_keypassword':'test123' }
cm_ssl_conf = {'WEB_TLS':'true','KEYSTORE_PATH':'/opt/cloudera-manager/ssl/jks/javakeystore.jks','KEYSTORE_PASSWORD':'test123','TRUSTSTORE_PATH':'/opt/pki/etc/tca/test123.jks','TRUSTSTORE_PASSWORD':'test123'}
cm_managed_service = {'ssl_client_truststore_location':'/opt/pki/etc/tca/test123.jks','ssl_client_truststore_password':'test123'}
hbase_ssl_enable = { 'hbase_hadoop_ssl_enabled' : 'true','ssl_server_keystore_location' : '/app/opt/cloudera/certs/jks/javakeystore.jks','ssl_server_keystore_password':'test123','ssl_server_keystore_keypassword':'test123' }
oozie_role_ssl_enable = { 'oozie_https_truststore_file':'/opt/pki/etc/tca/test123.jks','oozie_https_truststore_password':'test123','oozie_https_keystore_file' : '/app/opt/cloudera/certs/jks/javakeystore.jks','oozie_https_keystore_password':'test123' }
oozie_ssl_enable = { 'oozie_use_ssl' : 'true'}
hive_ssl_enable={'hiveserver2_keystore_path':'/app/opt/cloudera/certs/jks/javakeystore.jks','hiveserver2_keystore_password':'test123','hiveserver2_truststore_file':'/opt/pki/etc/tca/test123.jks','hiveserver2_truststore_password':'test123','hiveserver2_enable_ssl':'true'}
solr_ssl_enable={'solr_https_keystore_file':'/app/opt/cloudera/certs/jks/javakeystore.jks','solr_https_keystore_password':'test123','solr_https_truststore_file':'/opt/pki/etc/tca/test123.jks','solr_https_truststore_password':'test123','solr_use_ssl':'true'}

impala_ssl_enable={"client_services_ssl_enabled": "true", "ssl_server_certificate": "/app/opt/cloudera/certs/pem/cert.pem","ssl_private_key_password": "test123", "ssl_client_ca_certificate": "/opt/pki/etc/tca/test123.pem", "ssl_private_key": "/app/opt/cloudera/certs/pem/key.pem"}
impala_BASE_ssl_enable={"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "webserver_private_key_password_cmd": "test123"}
impala_STATESTORE_ssl_enable={"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem","webserver_private_key_password_cmd": "test123"}
impala_CATALOGSERVER_ssl_enable={"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem","webserver_private_key_password_cmd": "test123"}

hue_SERVER_role_enable_ssl={"ssl_certificate": "/app/opt/cloudera/certs/pem/cert.pem", "ssl_private_key_password": "test123", "ssl_cacerts": "/opt/pki/etc/tca/test123.pem", "ssl_enable": "true", "ssl_private_key": "/app/opt/cloudera/certs/pem/key.pem"}


### UPDATE CLOUDERA MANAGER ##
cm=ClouderaManager(api)
cm.update_config(cm_ssl_conf)

#### UPDATE HDFS SSL CONFIG ###
hdfs=clu.get_service('hdfs')
hdfs.update_config(hdfs_ssl_enable)

###  UPDATE HTTPFS SSL CONFIG ###
httpfs_role_group=hdfs.get_role_config_group("hdfs-HTTPFS-BASE")
httpfs_role_group.update_config(hdfs_httpfs_ssl_enable)

###  UPDATE YARN SSL CONFIG ###
yarn=clu.get_service('yarn')
yarn.update_config(yarn_ssl_enable)

#### UPDATE HBASE SSL CONFIG ###
print("Updating Hbase SSL Config")
hbase=clu.get_service('hbase')
hbase.update_config(hbase_ssl_enable)
print(hbase.get_config())

#### UPDATE OOZIE SSL CONFIG ###
oozie=clu.get_service('oozie')
oozie.update_config(oozie_ssl_enable)
oozie_role_group=oozie.get_role_config_group("oozie-OOZIE_SERVER-BASE")
oozie_role_group.update_config(oozie_role_ssl_enable)


#### UPDATE HIVE SSL CONFIG ###
hive=clu.get_service('hive')
hive.update_config(hive_ssl_enable)

#### UPDATE solr SSL CONFIG ###
solr=clu.get_service('solr')
solr.update_config(solr_ssl_enable)

### IMPALA SSL UPDATE ###
impala=clu.get_service('impala')
impala.update_config(impala_ssl_enable)
impala_IMPALAD_role_group=impala.get_role_config_group("impala-IMPALAD-BASE")
impala_STATESTORE_role_group=impala.get_role_config_group("impala-STATESTORE-BASE")
impala_CATALOGSERVER_role_group=impala.get_role_config_group("impala-CATALOGSERVER-BASE")
impala_IMPALAD_role_group.update_config(impala_BASE_ssl_enable)
impala_STATESTORE_role_group.update_config(impala_STATESTORE_ssl_enable)
impala_CATALOGSERVER_role_group.update_config(impala_CATALOGSERVER_ssl_enable)


### UPDATE HUE SSL ###
hue=clu.get_service('hue')
hue_SERVER_role_group=hue.get_role_config_group("hue-HUE_SERVER-BASE")
hue_SERVER_role_group.update_config(hue_SERVER_role_enable_ssl)

#############################

you can also use this script to update other configurations as well. you can use below line to dump the existing configurations and update with your own value and run it.

to dump the existing configuration :

For Impala : 

impala=clu.get_service("impala")

y=impala.get_config(view="summary")

json.dump(y, sys.stdout)

for role in impala.get_all_role_config_groups():

        print(role)

        print("--------------------------------")

        x=role.get_config(view="summary")

        json.dump(x, sys.stdout)

output

[{"client_services_ssl_enabled": "true", "ssl_server_certificate": "/app/opt/cloudera/certs/pem/cert.pem", "admission_control_enabled": "true", "hbase_service": "hbase", "hive_service": "hive", "hdfs_service": "hdfs", "ssl_private_key_password": "test123", "ssl_client_ca_certificate": "/opt/pki/etc/tca/test123.pem", "ssl_private_key": "/app/opt/cloudera/certs/pem/key.pem", "all_admission_control_enabled": "true"}, {}]<ApiRoleConfigGroup>: impala-IMPALAD-BASE (cluster: cluster; service: impala)

--------------------------------

{"webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "impalad_memory_limit": "17179869184", "enable_audit_event_log": "true", "scratch_dirs": "/app/hadoop/impala/impalad", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "log_dir": "/app/var/log/impalad", "lineage_event_log_dir": "/app/var/log/impalad/lineage", "webserver_private_key_password_cmd": "test123"}<ApiRoleConfigGroup>: impala-STATESTORE-BASE (cluster: cluster; service: impala)

--------------------------------

{"webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "log_threshold": "DEBUG", "webserver_private_key_password_cmd": "test123", "log_dir": "/app/var/log/statestore", "webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem"}<ApiRoleConfigGroup>: impala-CATALOGSERVER-BASE (cluster: cluster; service: impala)

--------------------------------

{"catalogd_embedded_jvm_heapsize": "34359738368", "webserver_private_key_file": "/app/opt/cloudera/certs/pem/key.pem", "webserver_certificate_file": "/app/opt/cloudera/certs/pem/cert.pem", "load_catalog_in_background": "true", "log_dir": "/app/var/log/catalogd", "oom_heap_dump_enabled": "false", "webserver_private_key_password_cmd": "test123"}<ApiRoleConfigGroup>: impala-LLAMA-BASE (cluster: cluster; service: impala)

For HDFS :

hdfs=clu.get_service("hdfs")

y=hdfs.get_config(view="summary")

json.dump(y, sys.stdout)

for role in hdfs.get_all_role_config_groups():

        print(role)

        print("--------------------------------")

        x=role.get_config(view="summary")

        json.dump(x, sys.stdout)

Output

[{"hdfs_hadoop_ssl_enabled": "true", "core_site_safety_valve": "<property>        <name>hadoop.user.group.static.mapping.overrides</name>        <value>dr.who=;mapred=mapred,hadoop;impala=impala,hive,yarn,hadoop;</value>\r\n</property>\r\n", "ssl_server_keystore_password": "test123", "kms_service": "kms", "dfs_namenode_acls_enabled": "true", "ssl_server_keystore_keypassword": "test123", "dfs_block_local_path_access_user": "impala", "ssl_server_keystore_location": "/app/opt/cloudera/certs/jks/javakeystore.jks", "ssl_client_truststore_password": "test123", "audit_event_log_dir": "/app/var/log/hadoop-hdfs/audit", "dfs_replication": "1", "ssl_client_truststore_location": "/opt/pki/etc/tca/test123.jks"}, {}]<ApiRoleConfigGroup>: hdfs-DATANODE-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"dfs_datanode_max_xcievers": "8192", "oom_heap_dump_enabled": "false", "dfs_datanode_data_dir_perm": "755", "datanode_log_dir": "/app/var/log/hadoop-hdfs", "dfs_datanode_volume_choosing_policy": "org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy", "dfs_data_dir_list": "/app/hadoop/data/d01/dfs/dn,/app/hadoop/data/d02/dfs/dn,/app/hadoop/data/d03/dfs/dn,/app/hadoop/data/d04/dfs/dn,/app/hadoop/data/d05/dfs/dn,/app/hadoop/data/d06/dfs/dn,/app/hadoop/data/d07/dfs/dn,/app/hadoop/data/d08/dfs/dn,/app/hadoop/data/d09/dfs/dn,/app/hadoop/data/d10/dfs/dn,/app/hadoop/data/d11/dfs/dn,/app/hadoop/data/d12/dfs/dn,/app/hadoop/data/d13/dfs/dn,/app/hadoop/data/d14/dfs/dn,/app/hadoop/data/d15/dfs/dn,/app/hadoop/data/d16/dfs/dn,/app/hadoop/data/d17/dfs/dn,/app/hadoop/data/d18/dfs/dn,/app/hadoop/data/d19/dfs/dn,/app/hadoop/data/d20/dfs/dn", "dfs_datanode_failed_volumes_tolerated": "10"}<ApiRoleConfigGroup>: hdfs-NAMENODE-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"dfs_name_dir_list": "/app/nn", "dfs_namenode_servicerpc_address": "8022", "namenode_log_dir": "/app/var/log/hadoop-hdfs", "fs_trash_interval": "60", "oom_heap_dump_enabled": "false", "dfs_safemode_min_datanodes": "0"}<ApiRoleConfigGroup>: hdfs-FAILOVERCONTROLLER-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"oom_heap_dump_enabled": "false"}<ApiRoleConfigGroup>: hdfs-BALANCER-BASE (cluster: cluster; service: hdfs)

--------------------------------

{}<ApiRoleConfigGroup>: hdfs-GATEWAY-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"hdfs_client_config_safety_valve": "<property>\r\n<name>dfs.client.block.write.replace-datanode-on-failure.enable</name>\r\n<value>NEVER</value>\r\n</property>", "dfs_client_use_trash": "true"}<ApiRoleConfigGroup>: hdfs-SECONDARYNAMENODE-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"secondarynamenode_log_dir": "/app/var/log/hadoop-hdfs", "oom_heap_dump_enabled": "false", "fs_checkpoint_dir_list": "/app/nn/dfs/snn"}<ApiRoleConfigGroup>: hdfs-JOURNALNODE-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"oom_heap_dump_enabled": "false"}<ApiRoleConfigGroup>: hdfs-HTTPFS-BASE (cluster: cluster; service: hdfs)

--------------------------------

{"httpfs_https_keystore_file": "/app/opt/cloudera/certs/jks/javakeystore.jks", "httpfs_https_keystore_password": "test123", "httpfs_https_truststore_password": "test123", "httpfs_https_truststore_file": "/opt/pki/etc/tca/test123.jks", "oom_heap_dump_enabled": "false", "httpfs_use_ssl": "true"}<ApiRoleConfigGroup>: hdfs-NFSGATEWAY-BASE (cluster: cluster; service: hdfs)

convert p12 into pem and jks format

Convert p12 into pem

openssl pkcs12 -in certificate.p12 -nocerts -out key.pem
openssl pkcs12 -in certificate.p12 -clcerts -nokeys -out cert.pem


convert p12 into jks

keytool -v -importkeystore -srckeystore ../certificate.p12 -srcstoretype PKCS12 -destkeystore keystore.ks -deststoretype JKS

OpenSSL CA Authority setup with SAN certificate for Cloudera

As i mentioned in my previous blog Secure hadoop/cloudera cluster using PKI implementation setting up PKI will be good practice, In that blog i didn't mentioned about SAN certificate which will be more important while setting up fronted load balancer for hive/impala services in a multi node cluster.

I have created simple shell script with input file, this will create Single ROOT CA keystore and certificate and it will create jks/pem formate keystore and truststore for hive/impala/cm/hue

Certificate input file

#LBNAME:NODES:FORMAT(PEM|JKS)
hive.tanu.com:nm1.tanu.com,node1.tanu.com,node2.tanu.com,hive.tanu.com:jks
hue.tanu.com:nm1.tanu.com,node1.tanu.com,node2.tanu.com,hue.tanu.com:pem
impala.tanu.com:nm1.tanu.com,node1.tanu.com,node2.tanu.com,impala.tanu.com:jks
cm.tanu.com:nm1.tanu.com,node1.tanu.com,node2.tanu.com,cm.tanu.com:jks

ROOT CA creation script:

root@ubuntu:/mnt/pool/usb-disk/X509CA/Project-cloudera1# cat Create_ROOTCA.sh
openssl genrsa -des3 -out MyRootCA.key 2048
openssl req -x509 -new -nodes -key MyRootCA.key -sha256 -days 5000 -out MyRootCA.pem -subj "/C=US/ST=NY/L=NYC/O=Global Security/OU=IT Department/CN=Hadoop CA Authority"


SAN Certificate Creating script JKS FORMAT

nodes=`cat certificates.txt|grep -i ":jks" |grep -v "^#" |cut -d":" -f2`
#array=( $san )
#echo "Number of elements: ${#array[@]}"
#count=1
export PATH=$PATH:/opt/jdk1.8.0_144/bin
for cnname in `cat certificates.txt|grep -i ":jks"|grep -v "^#"`
        do
                unset san
                cn=$(echo $cnname|cut -d":" -f1)
                count=1
                echo "[ req ]">openssl-ext.cnf
                echo "req_extensions   = v3_req" >>openssl-ext.cnf
                echo "[ v3_req ]" >>openssl-ext.cnf
                echo "subjectAltName = @alt_names" >>openssl-ext.cnf
                echo "[alt_names]" >>openssl-ext.cnf
                for x in $(echo $cnname|cut -d":" -f2|tr "," "\n")
                        do
                        san+="dns:$x,"
                        echo "DNS.$count = $x">>openssl-ext.cnf
                        count=$((count + 1 ))
                done
                echo $san
                mkdir $cn
                keytool -genkey -alias $cn -keyalg RSA -keystore $cn/$cn.jks -keysize 2048 -dname "CN=$cn,OU=IT,O=IT,L=NYC,S=NY,C=US" -storepass cloud123 -keypass cloud123
                keytool -certreq -alias $cn -keystore $cn/$cn.jks -file $cn/$cn.csr -ext SAN="$san" -storepass cloud123 -keypass cloud123
                openssl x509 -req -in $cn/$cn.csr -CA MyRootCA.pem -CAkey MyRootCA.key -CAcreateserial -out $cn/$cn.pem -days 5000  -sha256 -extfile openssl-ext.cnf -extensions v3_req -passin pass:sathish123
                echo "Importing CA Cetificate into clinet key"
                keytool -importcert -trustcacerts -keystore $cn/$cn.jks -alias MyRootCA.pem -file MyRootCA.pem -storepass cloud123 -keypass cloud123 -noprompt
                echo "Importing Certiicate into client keystore"
                keytool -importcert -trustcacerts -keystore $cn/$cn.jks -alias $cn -file $cn/$cn.pem -storepass cloud123 -keypass cloud123 -noprompt
                echo "Creating Truststore and import CA ceertificate"
                keytool -importcert -keystore $cn/$cn.truststore -alias MyRootCA -file MyRootCA.pem -storepass cloud123 -keypass cloud123 -noprompt



        done

SAN Certificate Creating script PEM FORMAT

root@ubuntu:/mnt/pool/usb-disk/X509CA/Project-cloudera1# cat pem__cert_create.sh

export PATH=$PATH:/opt/jdk1.8.0_144/bin

for cnname in `cat certificates.txt|grep -v "^#"`

        do

                unset san

                cn=$(echo $cnname|cut -d":" -f1)

                count=1

                echo "[ req ]" >openssl-ext.cnf

                echo "distinguished_name = req_distinguished_name" >>openssl-ext.cnf

                echo "req_extensions   = v3_req" >>openssl-ext.cnf

                echo "prompt = no" >>openssl-ext.cnf

                echo "[req_distinguished_name]" >>openssl-ext.cnf

                echo "C = US" >>openssl-ext.cnf

                echo "ST = NY" >>openssl-ext.cnf

                echo "L = NYC" >>openssl-ext.cnf

                echo "O = Global Security" >>openssl-ext.cnf

                echo "OU = IT Department" >>openssl-ext.cnf

                echo "CN = $cn" >>openssl-ext.cnf

                echo "[ v3_req ]" >>openssl-ext.cnf

                echo "subjectAltName = @alt_names" >>openssl-ext.cnf

                echo "[alt_names]" >>openssl-ext.cnf

                for x in $(echo $cnname|cut -d":" -f2|tr "," "\n")

                        do

                        san+="dns:$x,"

                        echo "DNS.$count = $x">>openssl-ext.cnf

                        count=$((count + 1 ))

                        done

                echo $san

                mkdir $cn

                openssl genrsa -des3 -out $cn/$cn.key -passout pass:cloud123 2048

                openssl req -new -out $cn/$cn.csr -key  $cn/$cn.key  -passin pass:cloud123 -config openssl-ext.cnf

                openssl x509 -req -in $cn/$cn.csr -CA MyRootCA.pem -CAkey MyRootCA.key -CAcreateserial -out $cn/$cn.pem -days 5000  -sha256 -extfile openssl-ext.cnf -extensions v3_req -passin pass:sathish123

        done

Secure hadoop/cloudera cluster using PKI implementation

Setting Up SSL/TLS for cloudera Manager and the components using self signed certificate is little painful.

Since  cloudera or  hadoop infrastructure will have more nodes and creating self signed certificate and maintaining trust store across the nodes will be more complex.


In my experience implementing single root CA Authority for entire cluster would be good practice.  So that we need not to have all the certificates in trust store other than CA certificate for TLS communication.

Here is the simple shell script to create

ROOT CA certificate
Create/issue certificate and sign PEM format certificate for nodes
Create/issue certificate and sign JKS format certificate for cloudera manger and hive

--------------------------------- Create root ca --------------------------------------

root@ubuntu:/mnt/pool/usb-disk/X509CA# cat Create_ROOTCA.sh
openssl genrsa -des3 -out MyRootCA.key 2048
openssl req -x509 -new -nodes -key MyRootCA.key -sha256 -days 5000 -out MyRootCA.pem -subj "/C=US/ST=NY/L=NYC/O=Global Security/OU=IT Department/CN=Hadoop CA Authority"

--------------- Create Certificate in PEM Format --------------------------

root@ubuntu:/mnt/pool/usb-disk/X509CA# cat Createcert_pem_format.sh
echo "Enter the client comman Name : "
read cn
mkdir $cn
openssl genrsa -des3 -out $cn/$cn.key 2048
openssl req -new -key $cn/$cn.key -out $cn/$cn.csr -subj "/C=US/ST=NY/L=NYC/O=Global Security/OU=IT Department/CN=$cn"
openssl x509 -req -in $cn/$cn.csr -CA MyRootCA.pem -CAkey MyRootCA.key -CAcreateserial -out $cn/$cn.pem -days 5000  -sha256
cp MyRootCA.pem $cn/
#openssl req -x509 -new -nodes -key MyRootCA.key -sha256 -days 5000 -out MyRootCA.pem -subj "/C=US/ST=NY/L=NYC/O=Global Security/OU=IT Department/CN=Hadoop CA Authority"

---------------- Create keystore and Truststore Certificate in JKS Format ---------------------------------------

root@ubuntu:/mnt/pool/usb-disk/X509CA# cat Createcert_jks_format.sh 

export PATH=$PATH:/opt/jdk1.8.0_144/bin

echo "Enter th Comman Name :"

read cn

mkdir $cn

keytool -genkey -alias $cn -keyalg RSA -keystore $cn/$cn.jks -keysize 2048 -dname "CN=$cn,OU=IT,O=IT,L=NYC,S=NY,C=US" 

keytool -certreq -alias $cn -keystore $cn/$cn.jks -file $cn/$cn.csr

openssl x509 -req -in $cn/$cn.csr -CA MyRootCA.pem -CAkey MyRootCA.key -CAcreateserial -out $cn/$cn.pem -days 5000  -sha256

keytool -importcert -trustcacerts -keystore $cn/$cn.jks -alias MyRootCA.pem -file MyRootCA.pem

keytool -importcert -trustcacerts -keystore $cn/$cn.jks -alias $cn -file $cn/$cn.pem

keytool -importcert -keystore $cn/$cn.truststore -alias MyRootCA -file MyRootCA.pem