openshift origin multi-master manually deployment part-3

6, install and cofigure etcd cluster, on all master nodes

yum install -y etcd

configure etcd, comment out all default configuration entries, and add contents below.
vi /etc/etcd/etcd.conf

ETCD_NAME=master1.openshift.qyos.com
ETCD_LISTEN_PEER_URLS=https://192.168.2.206:2380
ETCD_DATA_DIR=/var/lib/etcd/
#ETCD_SNAPSHOT_COUNTER=10000
ETCD_HEARTBEAT_INTERVAL=500
ETCD_ELECTION_TIMEOUT=2500
ETCD_LISTEN_CLIENT_URLS=https://192.168.2.206:2379
#ETCD_MAX_SNAPSHOTS=5
#ETCD_MAX_WALS=5
#ETCD_CORS=



#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.2.206:2380
ETCD_INITIAL_CLUSTER=master1.openshift.qyos.com=https://192.168.2.206:2380,master2.openshift.qyos.com=https://192.168.2.207:2380,master3.openshift.qyos.com=https://192.168.2.208:2380
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster-1
#ETCD_DISCOVERY=
#ETCD_DISCOVERY_SRV=
#ETCD_DISCOVERY_FALLBACK=proxy
#ETCD_DISCOVERY_PROXY=
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.2.206:2379

#[proxy]
#ETCD_PROXY=off

#[security]
ETCD_CA_FILE=/etc/etcd/ca.crt
ETCD_CERT_FILE=/etc/etcd/server.crt
ETCD_KEY_FILE=/etc/etcd/server.key
ETCD_PEER_CA_FILE=/etc/etcd/ca.crt
ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt
ETCD_PEER_KEY_FILE=/etc/etcd/peer.key

NOTE: ETCD_NAME and ip address should be updated to different valuses for different master nodes.
generate etcd certificate files
on master1:

mkdir /etc/etcd/ca
mkdir /etc/etcd/ca/crl
mkdir /etc/etcd/ca/fragments
mkdir /etc/etcd/ca/certs

copy system openssl.cnf file to /etc/etcd/ca/ and append the following contents:

cp /etc/pki/tls/openssl.cnf /etc/etcd/ca/

vi /etc/etcd/ca/openssl.cnf

[ etcd_v3_req ]
basicConstraints = critical,CA:FALSE
keyUsage         = digitalSignature,keyEncipherment
subjectAltName   = ${ENV::SAN}

[ etcd_ca ]
dir             = /etc/etcd/ca
crl_dir         = /etc/etcd/ca/crl
database        = /etc/etcd/ca/index.txt
new_certs_dir   = /etc/etcd/ca/certs
certificate     = /etc/etcd/ca/ca.crt
serial          = /etc/etcd/ca/serial
private_key     = /etc/etcd/ca/ca.key
crl_number      = /etc/etcd/ca/crlnumber
x509_extensions = etcd_v3_ca_client
default_days    = 365
default_md      = sha256
preserve        = no
name_opt        = ca_default
cert_opt        = ca_default
policy          = policy_anything
unique_subject  = no
copy_extensions = copy

[ etcd_v3_ca_self ]
authorityKeyIdentifier = keyid,issuer
basicConstraints       = critical,CA:TRUE,pathlen:0
keyUsage               = critical,digitalSignature,keyEncipherment,keyCertSign
subjectKeyIdentifier   = hash

[ etcd_v3_ca_peer ]
authorityKeyIdentifier = keyid,issuer:always
basicConstraints       = critical,CA:FALSE
extendedKeyUsage       = clientAuth,serverAuth
keyUsage               = digitalSignature,keyEncipherment
subjectKeyIdentifier   = hash

[ etcd_v3_ca_server ]
authorityKeyIdentifier = keyid,issuer:always
basicConstraints       = critical,CA:FALSE
extendedKeyUsage       = serverAuth
keyUsage               = digitalSignature,keyEncipherment
subjectKeyIdentifier   = hash

[ etcd_v3_ca_client ]
authorityKeyIdentifier = keyid,issuer:always
basicConstraints       = critical,CA:FALSE
extendedKeyUsage       = clientAuth
keyUsage               = digitalSignature,keyEncipherment
subjectKeyIdentifier   = hash

touch /etc/etcd/ca/index.txt
echo 01 >/etc/etcd/ca/serial
date  +%s

export SAN=’etcd-signer’
generate etcd ca

openssl req -config /etc/etcd/ca/openssl.cnf -newkey rsa:4096 -keyout /etc/etcd/ca/ca.key -new -out /etc/etcd/ca/ca.crt -x509 -extensions etcd_v3_ca_self -batch -nodes -days 365 -subj /CN=etcd-signer@1488960808

generate etcd server certs

mkdir -p /etc/etcd/generated_certs/etcd-master1.openshift.qyos.com
cd /etc/etcd/generated_certs/etcd-master1.openshift.qyos.com
export SAN="IP:192.168.2.206"
openssl req -new -keyout server.key 	\
    -config /etc/etcd/ca/openssl.cnf 	\
    -out server.csr 	\
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master1.openshift.qyos.com

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out server.crt \
      -in server.csr \
      -extensions etcd_v3_ca_server -batch 

openssl req -new -keyout peer.key \
    -config /etc/etcd/ca/openssl.cnf \
    -out peer.csr \
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master1.openshift.qyos.com 

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out peer.crt \
      -in peer.csr \
      -extensions etcd_v3_ca_peer -batch

cp ./* /etc/etcd/
cp /etc/etcd/ca/ca.crt /etc/etcd/
cd ..
mkdir etcd-master2.openshift.qyos.com
cd /etc/etcd/generated_certs/etcd-master2.openshift.qyos.com

export SAN="IP:192.168.2.207"
openssl req -new -keyout server.key 	\
    -config /etc/etcd/ca/openssl.cnf 	\
    -out server.csr 	\
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master2.openshift.qyos.com

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out server.crt \
      -in server.csr \
      -extensions etcd_v3_ca_server -batch 

openssl req -new -keyout peer.key \
    -config /etc/etcd/ca/openssl.cnf \
    -out peer.csr \
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master2.openshift.qyos.com 

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out peer.crt \
      -in peer.csr \
      -extensions etcd_v3_ca_peer -batch

cp /etc/etcd/ca/ca.crt .
scp ./* root@192.168.2.207:/etc/etcd/
cd ..

mkdir etcd-master3.openshift.qyos.com
cd /etc/etcd/generated_certs/etcd-master3.openshift.qyos.com

export SAN="IP:192.168.2.208"
openssl req -new -keyout server.key 	\
    -config /etc/etcd/ca/openssl.cnf 	\
    -out server.csr 	\
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master3.openshift.qyos.com

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out server.crt \
      -in server.csr \
      -extensions etcd_v3_ca_server -batch 

openssl req -new -keyout peer.key \
    -config /etc/etcd/ca/openssl.cnf \
    -out peer.csr \
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master3.openshift.qyos.com 

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out peer.crt \
      -in peer.csr \
      -extensions etcd_v3_ca_peer -batch

cp /etc/etcd/ca/ca.crt ./
scp ./* root@192.168.2.208:/etc/etcd/
cd ..

generate etcd client key file

mkdir openshift-master-master1.openshift.qyos.com
cd openshift-master-master1.openshift.qyos.com/

export SAN="IP:192.168.2.206"
openssl req -new -keyout master.etcd-client.key \
    -config /etc/etcd/ca/openssl.cnf \
    -out master.etcd-client.csr \
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master1.openshift.qyos.com

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out master.etcd-client.crt \
      -in master.etcd-client.csr \
      -batch

cp /etc/etcd/ca/ca.crt ./master.etcd-ca.crt
cp ./* /etc/origin/master/
cd ..
mkdir openshift-master-master2.openshift.qyos.com
cd openshift-master-master2.openshift.qyos.com/
export SAN="IP:192.168.2.207"

openssl req -new -keyout master.etcd-client.key \
    -config /etc/etcd/ca/openssl.cnf \
    -out master.etcd-client.csr \
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master2.openshift.qyos.com

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out master.etcd-client.crt \
      -in master.etcd-client.csr \
      -batch
cp /etc/etcd/ca/ca.crt ./master.etcd-ca.crt
scp ./* root@192.168.2.207:/etc/origin/master/

cd ..

mkdir openshift-master-master3.openshift.qyos.com
cd openshift-master-master3.openshift.qyos.com/
export SAN="IP:192.168.2.208"

openssl req -new -keyout master.etcd-client.key \
    -config /etc/etcd/ca/openssl.cnf \
    -out master.etcd-client.csr \
    -reqexts etcd_v3_req -batch -nodes \
    -subj /CN=master3.openshift.qyos.com

openssl ca -name etcd_ca -config /etc/etcd/ca/openssl.cnf \
      -out master.etcd-client.crt \
      -in master.etcd-client.csr \
      -batch
cp /etc/etcd/ca/ca.crt ./master.etcd-ca.crt
scp ./* root@192.168.2.208:/etc/origin/master/

install iptables service on all nodes

yum install -y iptables iptables-services
systemctl daemon-reload
systemctl unmask iptables 
systemctl unmask ip6tables

systemctl enable iptables
systemctl start iptables

iptables -N OS_FIREWALL_ALLOW

iptables -I INPUT 5 -j OS_FIREWALL_ALLOW
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 10250 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 10250 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 10255 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 80 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 443 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 4789 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 10255 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 2380 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 2379 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 8443 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 8444 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 4001 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 24224 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 24224 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 2224 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 5404 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 5405 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 8053 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 8053 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p udp -m udp --dport 53 -j ACCEPT
iptables -I OS_FIREWALL_ALLOW -p tcp -m tcp --dport 53 -j ACCEPT

now, we can start all services we have intalled.

systemctl enable docker
systemctl start docker

systemctl enable etcd
systemctl start etcd

systemctl enable origin-master-api
systemctl enable origin-master-controllers

systemctl start origin-master-api
systemctl start origin-master-controllers

Leave a Reply

Your email address will not be published. Required fields are marked *