openshift origin multi-master manually deployment part-2

5, install origin master service, on all master

yum install -y centos-release-openshift-origin
yum install -y origin-master bash-completion

create origin-master-api systemd file
vi /usr/lib/systemd/system/origin-master-api.service

[Unit]
Description=Atomic OpenShift Master API
Documentation=https://github.com/openshift/origin
After=network-online.target
After=etcd.service
Before=origin-node.service
Requires=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/origin-master-api
Environment=GOTRACEBACK=crash
ExecStart=/usr/bin/openshift start master api --config=${CONFIG_FILE} $OPTIONS
LimitNOFILE=131072
LimitCORE=infinity
WorkingDirectory=/var/lib/origin
SyslogIdentifier=atomic-openshift-master-api
RestartSec=5s



[Install]
WantedBy=multi-user.target
WantedBy=origin-node.service

create origin-master-controllers systemd startup file
vi /usr/lib/systemd/system/origin-master-controllers.service

[Unit]
Description=Atomic OpenShift Master Controllers
Documentation=https://github.com/openshift/origin
After=network-online.target
After=origin-master-api.service
Wants=origin-master-api.service
Requires=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/origin-master-controllers
Environment=GOTRACEBACK=crash
ExecStart=/usr/bin/openshift start master controllers --config=${CONFIG_FILE} $OPTIONS
LimitNOFILE=131072
LimitCORE=infinity
WorkingDirectory=/var/lib/origin
SyslogIdentifier=origin-master-controllers
Restart=always
RestartSec=5s

[Install]
WantedBy=multi-user.target

create master-api environment file
vi /etc/sysconfig/origin-master-api

OPTIONS=--loglevel=2 --listen=https://0.0.0.0:8443 --master=https://master1.openshift.qyos.com:8443
CONFIG_FILE=/etc/origin/master/master-config.yaml

NOTE: master1 should be updated to master2 or master3 when you are setting up service on master2 or master3.
create master-controllers environment file
vi /etc/sysconfig/origin-master-controllers

OPTIONS=--loglevel=2 --listen=https://0.0.0.0:8444
CONFIG_FILE=/etc/origin/master/master-config.yaml

mask out origin-master service and firewalld service

systemctl mask origin-master
systemctl mask firewalld

when the rpm packages’ installation is done, some default certificates will be generated, and these files are not suitable
for our cluster, we must regenerated theme and store our cluster informatsions in it. backup the old files, and let’s regenerate new
files.

mkdir /etc/origin/master-old
mv /etc/origin/master/* /etc/origin/master-old/

now, on master1,

cd ~
mkdir -p openshift.local.config/master
cd openshift.local.config/master
oc adm create-master-certs --cert-dir=. \
            --master=https://openshift.qyos.com:8443 \
            --public-master=https://openshift.qyos.com:8443 \
            --hostnames=openshift.qyos.com,localhost,127.0.0.1,172.17.42.1,kubernetes.default.local,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,master1.openshift.qyos.com,master2.openshift.qyos.com,master3.openshift.qyos.com,openshift,openshift.default,openshift.default.svc,openshift.default.svc.cluster.local,172.30.0.1,192.168.2.206,192.168.2.207,192.168.2.208 \
            --overwrite=false 

NOTE: change openshift.qyos.com, three hostnames of master node, and IP address of masters to yours.
create master-config.yaml
vi master-config.yaml

admissionConfig:
apiLevels:
- v1
apiVersion: v1
assetConfig:
  logoutURL: ""
  masterPublicURL: https://openshift.qyos.com:8443
  publicURL: https://openshift.qyos.com:8443/console/
  servingInfo:
    bindAddress: 0.0.0.0:8443
    bindNetwork: tcp4
    certFile: master.server.crt
    clientCA: ""
    keyFile: master.server.key
    maxRequestsInFlight: 0
    requestTimeoutSeconds: 0
controllerLeaseTTL: 30
controllerConfig:
  serviceServingCert:
    signer:
      certFile: service-signer.crt
      keyFile: service-signer.key
controllers: '*'
corsAllowedOrigins:
  - 127.0.0.1
  - localhost
  - 192.168.2.206
  - lb.openshift.qyos.com
  - kubernetes.default
  - kubernetes.default.svc.cluster.local
  - kubernetes
  - openshift.default
  - openshift.default.svc
  - 172.30.0.1
  - openshift.qyos.com
  - master1.openshift.qyos.com
  - openshift.default.svc.cluster.local
  - kubernetes.default.svc
  - openshift
dnsConfig:
  bindAddress: 0.0.0.0:8053
  bindNetwork: tcp4
etcdClientInfo:
  ca: master.etcd-ca.crt
  certFile: master.etcd-client.crt
  keyFile: master.etcd-client.key
  urls:
    - https://master1.openshift.qyos.com:2379
    - https://master2.openshift.qyos.com:2379
    - https://master3.openshift.qyos.com:2379
etcdStorageConfig:
  kubernetesStoragePrefix: kubernetes.io
  kubernetesStorageVersion: v1
  openShiftStoragePrefix: openshift.io
  openShiftStorageVersion: v1
imageConfig:
  format: openshift/origin-${component}:${version}
  latest: false
kind: MasterConfig
kubeletClientInfo:
  ca: ca.crt
  certFile: master.kubelet-client.crt
  keyFile: master.kubelet-client.key
  port: 10250
kubernetesMasterConfig:
  admissionConfig:
    pluginConfig:
      {}
  apiServerArguments: 
  controllerArguments: 
  masterCount: 3
  masterIP: 192.168.2.206
  podEvictionTimeout: 
  proxyClientInfo:
    certFile: master.proxy-client.crt
    keyFile: master.proxy-client.key
  schedulerConfigFile: /etc/origin/master/scheduler.json
  servicesNodePortRange: ""
  servicesSubnet: 172.30.0.0/16
  staticNodeNames: []
masterClients:
  externalKubernetesClientConnectionOverrides:
    acceptContentTypes: application/vnd.kubernetes.protobuf,application/json
    contentType: application/vnd.kubernetes.protobuf
    burst: 400
    qps: 200
  externalKubernetesKubeConfig: ""
  openshiftLoopbackClientConnectionOverrides:
    acceptContentTypes: application/vnd.kubernetes.protobuf,application/json
    contentType: application/vnd.kubernetes.protobuf
    burst: 600
    qps: 300
  openshiftLoopbackKubeConfig: openshift-master.kubeconfig
masterPublicURL: https://openshift.qyos.com:8443
networkConfig:
  clusterNetworkCIDR: 10.128.0.0/14
  hostSubnetLength: 9
  networkPluginName: redhat/openshift-ovs-multitenant
# serviceNetworkCIDR must match kubernetesMasterConfig.servicesSubnet
  serviceNetworkCIDR: 172.30.0.0/16
  externalIPNetworkCIDRs: 
  - 0.0.0.0/0
oauthConfig:
  assetPublicURL: https://openshift.qyos.com:8443/console/
  grantConfig:
    method: auto
  identityProviders:
  - challenge: true
    login: true
    mappingMethod: claim
    name: allow_all
    provider:
      apiVersion: v1
      kind: AllowAllPasswordIdentityProvider
  masterCA: ca-bundle.crt
  masterPublicURL: https://openshift.qyos.com:8443
  masterURL: https://lb.openshift.qyos.com:8443
  sessionConfig:
    sessionMaxAgeSeconds: 3600
    sessionName: ssn
    sessionSecretsFile: /etc/origin/master/session-secrets.yaml
  tokenConfig:
    accessTokenMaxAgeSeconds: 86400
    authorizeTokenMaxAgeSeconds: 500
pauseControllers: false
policyConfig:
  bootstrapPolicyFile: /etc/origin/master/policy.json
  openshiftInfrastructureNamespace: openshift-infra
  openshiftSharedResourcesNamespace: openshift
projectConfig:
  defaultNodeSelector: ""
  projectRequestMessage: ""
  projectRequestTemplate: ""
  securityAllocator:
    mcsAllocatorRange: "s0:/2"
    mcsLabelsPerProject: 5
    uidAllocatorRange: "1000000000-1999999999/10000"
routingConfig:
  subdomain:  "openshift.qyos.com"
serviceAccountConfig:
  limitSecretReferences: false
  managedNames:
  - default
  - builder
  - deployer
  masterCA: ca-bundle.crt
  privateKeyFile: serviceaccounts.private.key
  publicKeyFiles:
  - serviceaccounts.public.key
servingInfo:
  bindAddress: 0.0.0.0:8443
  bindNetwork: tcp4
  certFile: master.server.crt
  clientCA: ca.crt
  keyFile: master.server.key
  maxRequestsInFlight: 500
  requestTimeoutSeconds: 3600
volumeConfig:
  dynamicProvisioningEnabled: True

NOTE: change openshift.qyos.com, three hostnames of master node, and IP address of masters to yours.
create scheduler.json file
vi scheduler.json

{
    "apiVersion": "v1",
    "kind": "Policy",
    "predicates": [
        {
            "name": "MatchNodeSelector"
        },
        {
            "name": "PodFitsResources"
        },
        {
            "name": "PodFitsPorts"
        },
        {
            "name": "NoDiskConflict"
        },
        {
            "name": "NoVolumeZoneConflict"
        },
        {
            "name": "MaxEBSVolumeCount"
        },
        {
            "name": "MaxGCEPDVolumeCount"
        },
        {
            "argument": {
                "serviceAffinity": {
                    "labels": [
                        "region"
                    ]
                }
            },
            "name": "Region"
        }
    ],
    "priorities": [
        {
            "name": "LeastRequestedPriority",
            "weight": 1
        },
        {
            "name": "SelectorSpreadPriority",
            "weight": 1
        },
        {
            "argument": {
                "serviceAntiAffinity": {
                    "label": "zone"
                }
            },
            "name": "Zone",
            "weight": 2
        }
    ]
}

create session-secrets file
vi session-secrets.yaml

apiVersion: v1
kind: SessionSecrets
secrets:
- authentication: "E81/4lsFxuANlG3RDXRnElNfOENAlmPf"
  encryption: "E81/4lsFxuANlG3RDXRnElNfOENAlmPf"

cp /etc/origin/master-old/policy.json .

generate certificates file for master2 and master3, the hostnames and ip address should be updated to yours.

mkdir /etc/origin/generated-configs
mkdir /etc/origin/generated-configs/master-master2.openshift.qyos.com
mkdir /etc/origin/generated-configs/master-master3.openshift.qyos.com
chmod 0700 /etc/origin/generated-configs/* -R

cp ./* /etc/origin/generated-configs/master-master2.openshift.qyos.com/
cp ./* /etc/origin/generated-configs/master-master3.openshift.qyos.com/
cp ./* /etc/origin/master/

cd /etc/origin/generated-configs/master-master2.openshift.qyos.com/
rm -rf master.server.crt openshift-master.crt
oc adm ca create-master-certs --cert-dir=./ \
            --master=https://openshift.qyos.com:8443 \
            --public-master=https://openshift.qyos.com:8443 \
            --hostnames=openshift.qyos.com,localhost,127.0.0.1,172.17.42.1,kubernetes.default.local,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,master1.openshift.qyos.com,master2.openshift.qyos.com,master3.openshift.qyos.com,openshift,openshift.default,openshift.default.svc,openshift.default.svc.cluster.local,172.30.0.1,192.168.2.206,192.168.2.207,192.168.2.208 \
            --overwrite=false 

cd /etc/origin/generated-configs/master-master3.openshift.qyos.com/
rm -rf master.server.crt openshift-master.crt
oc adm create-master-certs --cert-dir=. \
            --master=https://openshift.qyos.com:8443 \
            --public-master=https://openshift.qyos.com:8443 \
            --hostnames=openshift.qyos.com,localhost,127.0.0.1,172.17.42.1,kubernetes.default.local,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,master1.openshift.qyos.com,master2.openshift.qyos.com,master3.openshift.qyos.com,openshift,openshift.default,openshift.default.svc,openshift.default.svc.cluster.local,172.30.0.1,192.168.2.206,192.168.2.207,192.168.2.208 \
            --overwrite=false 

cd ../
tar -czvf master-master2.openshift.qyos.com.tar.gz -C master-master2.openshift.qyos.com/ .
tar -czvf master-master3.openshift.qyos.com.tar.gz -C master-master3.openshift.qyos.com/ .
scp master-master2.openshift.qyos.com.tar.gz root@master2.openshift.qyos.com:~
scp master-master3.openshift.qyos.com.tar.gz root@master3.openshift.qyos.com:~

on master2:

tar xzf master-master2.openshift.qyos.com.tar.gz -C /etc/origin/master/ .

on master3:

tar xzf master-master3.openshift.qyos.com.tar.gz -C /etc/origin/master/ .

on all masters, create oc client config file.

cd ~
mkdir .kube
cp /etc/origin/master/admin.kubeconfig .kube/config

do not start master service at this time.

Leave a Reply

Your email address will not be published. Required fields are marked *