6, install and cofigure etcd cluster, on all master nodes
yum install -y etcd
configure etcd, comment out all default configuration entries, and add contents below.
vi /etc/etcd/etcd.conf
ETCD_NAME=master1.openshift.qyos.com ETCD_LISTEN_PEER_URLS=https://192.168.2.206:2380 ETCD_DATA_DIR=/var/lib/etcd/ #ETCD_SNAPSHOT_COUNTER=10000 ETCD_HEARTBEAT_INTERVAL=500 ETCD_ELECTION_TIMEOUT=2500 ETCD_LISTEN_CLIENT_URLS=https://192.168.2.206:2379 #ETCD_MAX_SNAPSHOTS=5 #ETCD_MAX_WALS=5 #ETCD_CORS= #[cluster] ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.2.206:2380 ETCD_INITIAL_CLUSTER=master1.openshift.qyos.com=https://192.168.2.206:2380,master2.openshift.qyos.com=https://192.168.2.207:2380,master3.openshift.qyos.com=https://192.168.2.208:2380 ETCD_INITIAL_CLUSTER_STATE=new ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster-1 #ETCD_DISCOVERY= #ETCD_DISCOVERY_SRV= #ETCD_DISCOVERY_FALLBACK=proxy #ETCD_DISCOVERY_PROXY= ETCD_ADVERTISE_CLIENT_URLS=https://192.168.2.206:2379 #[proxy] #ETCD_PROXY=off #[security] ETCD_CA_FILE=/etc/etcd/ca.crt ETCD_CERT_FILE=/etc/etcd/server.crt ETCD_KEY_FILE=/etc/etcd/server.key ETCD_PEER_CA_FILE=/etc/etcd/ca.crt ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt ETCD_PEER_KEY_FILE=/etc/etcd/peer.key
NOTE: ETCD_NAME and ip address should be updated to different valuses for different master nodes.
generate etcd certificate files
on master1:
mkdir /etc/etcd/ca mkdir /etc/etcd/ca/crl mkdir /etc/etcd/ca/fragments mkdir /etc/etcd/ca/certs
copy system openssl.cnf file to /etc/etcd/ca/ and append the following contents:
cp /etc/pki/tls/openssl.cnf /etc/etcd/ca/
vi /etc/etcd/ca/openssl.cnf
[ etcd_v3_req ] basicConstraints = critical,CA:FALSE keyUsage = digitalSignature,keyEncipherment subjectAltName = ${ENV::SAN} [ etcd_ca ] dir = /etc/etcd/ca crl_dir = /etc/etcd/ca/crl database = /etc/etcd/ca/index.txt new_certs_dir = /etc/etcd/ca/certs certificate = /etc/etcd/ca/ca.crt serial = /etc/etcd/ca/serial private_key = /etc/etcd/ca/ca.key crl_number = /etc/etcd/ca/crlnumber x509_extensions = etcd_v3_ca_client default_days = 365 default_md = sha256 preserve = no name_opt = ca_default cert_opt = ca_default policy = policy_anything unique_subject = no copy_extensions = copy [ etcd_v3_ca_self ] authorityKeyIdentifier = keyid,issuer basicConstraints = critical,CA:TRUE,pathlen:0 keyUsage = critical,digitalSignature,keyEncipherment,keyCertSign subjectKeyIdentifier = hash [ etcd_v3_ca_peer ] authorityKeyIdentifier = keyid,issuer:always basicConstraints = critical,CA:FALSE extendedKeyUsage = clientAuth,serverAuth keyUsage = digitalSignature,keyEncipherment subjectKeyIdentifier = hash [ etcd_v3_ca_server ] authorityKeyIdentifier = keyid,issuer:always basicConstraints = critical,CA:FALSE extendedKeyUsage = serverAuth keyUsage = digitalSignature,keyEncipherment subjectKeyIdentifier = hash [ etcd_v3_ca_client ] authorityKeyIdentifier = keyid,issuer:always basicConstraints = critical,CA:FALSE extendedKeyUsage = clientAuth keyUsage = digitalSignature,keyEncipherment subjectKeyIdentifier = hash
touch /etc/etcd/ca/index.txt echo 01 >/etc/etcd/ca/serial date +%s
export SAN=’etcd-signer’
generate etcd ca
openssl req -config /etc/etcd/ca/openssl.cnf -newkey rsa:4096 -keyout /etc/etcd/ca/ca.key -new -out /etc/etcd/ca/ca.crt -x509 -extensions etcd_v3_ca_self -batch -nodes -days 365 -subj /CN=etcd-signer@1488960808
generate etcd server certs
mkdir -p /etc/etcd/generated_certs/etcd-master1.openshift.qyos.com cd /etc/etcd/generated_certs/etcd-master1.openshift.qyos.com export SAN="IP:192.168.2.206" openssl req -new -keyout server.key \ -config /etc/etcd/ca/openssl.cnf \ -out server.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master1.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out server.crt \ -in server.csr \ -extensions etcd_v3_ca_server -batch openssl req -new -keyout peer.key \ -config /etc/etcd/ca/openssl.cnf \ -out peer.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master1.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out peer.crt \ -in peer.csr \ -extensions etcd_v3_ca_peer -batch cp ./* /etc/etcd/ cp /etc/etcd/ca/ca.crt /etc/etcd/ cd .. mkdir etcd-master2.openshift.qyos.com cd /etc/etcd/generated_certs/etcd-master2.openshift.qyos.com export SAN="IP:192.168.2.207" openssl req -new -keyout server.key \ -config /etc/etcd/ca/openssl.cnf \ -out server.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master2.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out server.crt \ -in server.csr \ -extensions etcd_v3_ca_server -batch openssl req -new -keyout peer.key \ -config /etc/etcd/ca/openssl.cnf \ -out peer.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master2.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out peer.crt \ -in peer.csr \ -extensions etcd_v3_ca_peer -batch cp /etc/etcd/ca/ca.crt . scp ./* root@192.168.2.207:/etc/etcd/ cd .. mkdir etcd-master3.openshift.qyos.com cd /etc/etcd/generated_certs/etcd-master3.openshift.qyos.com export SAN="IP:192.168.2.208" openssl req -new -keyout server.key \ -config /etc/etcd/ca/openssl.cnf \ -out server.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master3.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out server.crt \ -in server.csr \ -extensions etcd_v3_ca_server -batch openssl req -new -keyout peer.key \ -config /etc/etcd/ca/openssl.cnf \ -out peer.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master3.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out peer.crt \ -in peer.csr \ -extensions etcd_v3_ca_peer -batch cp /etc/etcd/ca/ca.crt ./ scp ./* root@192.168.2.208:/etc/etcd/ cd ..
generate etcd client key file
mkdir openshift-master-master1.openshift.qyos.com cd openshift-master-master1.openshift.qyos.com/ export SAN="IP:192.168.2.206" openssl req -new -keyout master.etcd-client.key \ -config /etc/etcd/ca/openssl.cnf \ -out master.etcd-client.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master1.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out master.etcd-client.crt \ -in master.etcd-client.csr \ -batch cp /etc/etcd/ca/ca.crt ./master.etcd-ca.crt cp ./* /etc/origin/master/ cd .. mkdir openshift-master-master2.openshift.qyos.com cd openshift-master-master2.openshift.qyos.com/ export SAN="IP:192.168.2.207" openssl req -new -keyout master.etcd-client.key \ -config /etc/etcd/ca/openssl.cnf \ -out master.etcd-client.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master2.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out master.etcd-client.crt \ -in master.etcd-client.csr \ -batch cp /etc/etcd/ca/ca.crt ./master.etcd-ca.crt scp ./* root@192.168.2.207:/etc/origin/master/ cd .. mkdir openshift-master-master3.openshift.qyos.com cd openshift-master-master3.openshift.qyos.com/ export SAN="IP:192.168.2.208" openssl req -new -keyout master.etcd-client.key \ -config /etc/etcd/ca/openssl.cnf \ -out master.etcd-client.csr \ -reqexts etcd_v3_req -batch -nodes \ -subj /CN=master3.openshift.qyos.com openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \ -out master.etcd-client.crt \ -in master.etcd-client.csr \ -batch cp /etc/etcd/ca/ca.crt ./master.etcd-ca.crt scp ./* root@192.168.2.208:/etc/origin/master/
install iptables service on all nodes
yum install -y iptables iptables-services systemctl daemon-reload systemctl unmask iptables systemctl unmask ip6tables systemctl enable iptables systemctl start iptables iptables -N OS_FIREWALL_ALLOW iptables -I INPUT 5 -j OS_FIREWALL_ALLOW iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 10250 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 10250 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 10255 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 80 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 443 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 4789 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 10255 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 2380 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 2379 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 8443 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 8444 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 4001 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 24224 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 24224 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 2224 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 5404 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 5405 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 8053 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 8053 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 53 -j ACCEPT iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 53 -j ACCEPT
now, we can start all services we have intalled.
systemctl enable docker systemctl start docker systemctl enable etcd systemctl start etcd systemctl enable origin-master-api systemctl enable origin-master-controllers systemctl start origin-master-api systemctl start origin-master-controllers