5, install origin master service, on all master
yum install -y centos-release-openshift-origin yum install -y origin-master bash-completion
create origin-master-api systemd file
vi /usr/lib/systemd/system/origin-master-api.service
[Unit] Description=Atomic OpenShift Master API Documentation=https://github.com/openshift/origin After=network-online.target After=etcd.service Before=origin-node.service Requires=network-online.target [Service] Type=notify EnvironmentFile=/etc/sysconfig/origin-master-api Environment=GOTRACEBACK=crash ExecStart=/usr/bin/openshift start master api --config=${CONFIG_FILE} $OPTIONS LimitNOFILE=131072 LimitCORE=infinity WorkingDirectory=/var/lib/origin SyslogIdentifier=atomic-openshift-master-api RestartSec=5s [Install] WantedBy=multi-user.target WantedBy=origin-node.service
create origin-master-controllers systemd startup file
vi /usr/lib/systemd/system/origin-master-controllers.service
[Unit] Description=Atomic OpenShift Master Controllers Documentation=https://github.com/openshift/origin After=network-online.target After=origin-master-api.service Wants=origin-master-api.service Requires=network-online.target [Service] Type=notify EnvironmentFile=/etc/sysconfig/origin-master-controllers Environment=GOTRACEBACK=crash ExecStart=/usr/bin/openshift start master controllers --config=${CONFIG_FILE} $OPTIONS LimitNOFILE=131072 LimitCORE=infinity WorkingDirectory=/var/lib/origin SyslogIdentifier=origin-master-controllers Restart=always RestartSec=5s [Install] WantedBy=multi-user.target
create master-api environment file
vi /etc/sysconfig/origin-master-api
OPTIONS=--loglevel=2 --listen=https://0.0.0.0:8443 --master=https://master1.openshift.qyos.com:8443 CONFIG_FILE=/etc/origin/master/master-config.yaml
NOTE: master1 should be updated to master2 or master3 when you are setting up service on master2 or master3.
create master-controllers environment file
vi /etc/sysconfig/origin-master-controllers
OPTIONS=--loglevel=2 --listen=https://0.0.0.0:8444 CONFIG_FILE=/etc/origin/master/master-config.yaml
mask out origin-master service and firewalld service
systemctl mask origin-master systemctl mask firewalld
when the rpm packages’ installation is done, some default certificates will be generated, and these files are not suitable
for our cluster, we must regenerated theme and store our cluster informatsions in it. backup the old files, and let’s regenerate new
files.
mkdir /etc/origin/master-old mv /etc/origin/master/* /etc/origin/master-old/
now, on master1,
cd ~ mkdir -p openshift.local.config/master cd openshift.local.config/master oc adm create-master-certs --cert-dir=. \ --master=https://openshift.qyos.com:8443 \ --public-master=https://openshift.qyos.com:8443 \ --hostnames=openshift.qyos.com,localhost,127.0.0.1,172.17.42.1,kubernetes.default.local,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,master1.openshift.qyos.com,master2.openshift.qyos.com,master3.openshift.qyos.com,openshift,openshift.default,openshift.default.svc,openshift.default.svc.cluster.local,172.30.0.1,192.168.2.206,192.168.2.207,192.168.2.208 \ --overwrite=false
NOTE: change openshift.qyos.com, three hostnames of master node, and IP address of masters to yours.
create master-config.yaml
vi master-config.yaml
admissionConfig: apiLevels: - v1 apiVersion: v1 assetConfig: logoutURL: "" masterPublicURL: https://openshift.qyos.com:8443 publicURL: https://openshift.qyos.com:8443/console/ servingInfo: bindAddress: 0.0.0.0:8443 bindNetwork: tcp4 certFile: master.server.crt clientCA: "" keyFile: master.server.key maxRequestsInFlight: 0 requestTimeoutSeconds: 0 controllerLeaseTTL: 30 controllerConfig: serviceServingCert: signer: certFile: service-signer.crt keyFile: service-signer.key controllers: '*' corsAllowedOrigins: - 127.0.0.1 - localhost - 192.168.2.206 - lb.openshift.qyos.com - kubernetes.default - kubernetes.default.svc.cluster.local - kubernetes - openshift.default - openshift.default.svc - 172.30.0.1 - openshift.qyos.com - master1.openshift.qyos.com - openshift.default.svc.cluster.local - kubernetes.default.svc - openshift dnsConfig: bindAddress: 0.0.0.0:8053 bindNetwork: tcp4 etcdClientInfo: ca: master.etcd-ca.crt certFile: master.etcd-client.crt keyFile: master.etcd-client.key urls: - https://master1.openshift.qyos.com:2379 - https://master2.openshift.qyos.com:2379 - https://master3.openshift.qyos.com:2379 etcdStorageConfig: kubernetesStoragePrefix: kubernetes.io kubernetesStorageVersion: v1 openShiftStoragePrefix: openshift.io openShiftStorageVersion: v1 imageConfig: format: openshift/origin-${component}:${version} latest: false kind: MasterConfig kubeletClientInfo: ca: ca.crt certFile: master.kubelet-client.crt keyFile: master.kubelet-client.key port: 10250 kubernetesMasterConfig: admissionConfig: pluginConfig: {} apiServerArguments: controllerArguments: masterCount: 3 masterIP: 192.168.2.206 podEvictionTimeout: proxyClientInfo: certFile: master.proxy-client.crt keyFile: master.proxy-client.key schedulerConfigFile: /etc/origin/master/scheduler.json servicesNodePortRange: "" servicesSubnet: 172.30.0.0/16 staticNodeNames: [] masterClients: externalKubernetesClientConnectionOverrides: acceptContentTypes: application/vnd.kubernetes.protobuf,application/json contentType: application/vnd.kubernetes.protobuf burst: 400 qps: 200 externalKubernetesKubeConfig: "" openshiftLoopbackClientConnectionOverrides: acceptContentTypes: application/vnd.kubernetes.protobuf,application/json contentType: application/vnd.kubernetes.protobuf burst: 600 qps: 300 openshiftLoopbackKubeConfig: openshift-master.kubeconfig masterPublicURL: https://openshift.qyos.com:8443 networkConfig: clusterNetworkCIDR: 10.128.0.0/14 hostSubnetLength: 9 networkPluginName: redhat/openshift-ovs-multitenant # serviceNetworkCIDR must match kubernetesMasterConfig.servicesSubnet serviceNetworkCIDR: 172.30.0.0/16 externalIPNetworkCIDRs: - 0.0.0.0/0 oauthConfig: assetPublicURL: https://openshift.qyos.com:8443/console/ grantConfig: method: auto identityProviders: - challenge: true login: true mappingMethod: claim name: allow_all provider: apiVersion: v1 kind: AllowAllPasswordIdentityProvider masterCA: ca-bundle.crt masterPublicURL: https://openshift.qyos.com:8443 masterURL: https://lb.openshift.qyos.com:8443 sessionConfig: sessionMaxAgeSeconds: 3600 sessionName: ssn sessionSecretsFile: /etc/origin/master/session-secrets.yaml tokenConfig: accessTokenMaxAgeSeconds: 86400 authorizeTokenMaxAgeSeconds: 500 pauseControllers: false policyConfig: bootstrapPolicyFile: /etc/origin/master/policy.json openshiftInfrastructureNamespace: openshift-infra openshiftSharedResourcesNamespace: openshift projectConfig: defaultNodeSelector: "" projectRequestMessage: "" projectRequestTemplate: "" securityAllocator: mcsAllocatorRange: "s0:/2" mcsLabelsPerProject: 5 uidAllocatorRange: "1000000000-1999999999/10000" routingConfig: subdomain: "openshift.qyos.com" serviceAccountConfig: limitSecretReferences: false managedNames: - default - builder - deployer masterCA: ca-bundle.crt privateKeyFile: serviceaccounts.private.key publicKeyFiles: - serviceaccounts.public.key servingInfo: bindAddress: 0.0.0.0:8443 bindNetwork: tcp4 certFile: master.server.crt clientCA: ca.crt keyFile: master.server.key maxRequestsInFlight: 500 requestTimeoutSeconds: 3600 volumeConfig: dynamicProvisioningEnabled: True
NOTE: change openshift.qyos.com, three hostnames of master node, and IP address of masters to yours.
create scheduler.json file
vi scheduler.json
{ "apiVersion": "v1", "kind": "Policy", "predicates": [ { "name": "MatchNodeSelector" }, { "name": "PodFitsResources" }, { "name": "PodFitsPorts" }, { "name": "NoDiskConflict" }, { "name": "NoVolumeZoneConflict" }, { "name": "MaxEBSVolumeCount" }, { "name": "MaxGCEPDVolumeCount" }, { "argument": { "serviceAffinity": { "labels": [ "region" ] } }, "name": "Region" } ], "priorities": [ { "name": "LeastRequestedPriority", "weight": 1 }, { "name": "SelectorSpreadPriority", "weight": 1 }, { "argument": { "serviceAntiAffinity": { "label": "zone" } }, "name": "Zone", "weight": 2 } ] }
create session-secrets file
vi session-secrets.yaml
apiVersion: v1 kind: SessionSecrets secrets: - authentication: "E81/4lsFxuANlG3RDXRnElNfOENAlmPf" encryption: "E81/4lsFxuANlG3RDXRnElNfOENAlmPf" cp /etc/origin/master-old/policy.json .
generate certificates file for master2 and master3, the hostnames and ip address should be updated to yours.
mkdir /etc/origin/generated-configs mkdir /etc/origin/generated-configs/master-master2.openshift.qyos.com mkdir /etc/origin/generated-configs/master-master3.openshift.qyos.com chmod 0700 /etc/origin/generated-configs/* -R cp ./* /etc/origin/generated-configs/master-master2.openshift.qyos.com/ cp ./* /etc/origin/generated-configs/master-master3.openshift.qyos.com/ cp ./* /etc/origin/master/ cd /etc/origin/generated-configs/master-master2.openshift.qyos.com/ rm -rf master.server.crt openshift-master.crt oc adm ca create-master-certs --cert-dir=./ \ --master=https://openshift.qyos.com:8443 \ --public-master=https://openshift.qyos.com:8443 \ --hostnames=openshift.qyos.com,localhost,127.0.0.1,172.17.42.1,kubernetes.default.local,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,master1.openshift.qyos.com,master2.openshift.qyos.com,master3.openshift.qyos.com,openshift,openshift.default,openshift.default.svc,openshift.default.svc.cluster.local,172.30.0.1,192.168.2.206,192.168.2.207,192.168.2.208 \ --overwrite=false cd /etc/origin/generated-configs/master-master3.openshift.qyos.com/ rm -rf master.server.crt openshift-master.crt oc adm create-master-certs --cert-dir=. \ --master=https://openshift.qyos.com:8443 \ --public-master=https://openshift.qyos.com:8443 \ --hostnames=openshift.qyos.com,localhost,127.0.0.1,172.17.42.1,kubernetes.default.local,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,master1.openshift.qyos.com,master2.openshift.qyos.com,master3.openshift.qyos.com,openshift,openshift.default,openshift.default.svc,openshift.default.svc.cluster.local,172.30.0.1,192.168.2.206,192.168.2.207,192.168.2.208 \ --overwrite=false cd ../ tar -czvf master-master2.openshift.qyos.com.tar.gz -C master-master2.openshift.qyos.com/ . tar -czvf master-master3.openshift.qyos.com.tar.gz -C master-master3.openshift.qyos.com/ . scp master-master2.openshift.qyos.com.tar.gz root@master2.openshift.qyos.com:~ scp master-master3.openshift.qyos.com.tar.gz root@master3.openshift.qyos.com:~
on master2:
tar xzf master-master2.openshift.qyos.com.tar.gz -C /etc/origin/master/ .
on master3:
tar xzf master-master3.openshift.qyos.com.tar.gz -C /etc/origin/master/ .
on all masters, create oc client config file.
cd ~ mkdir .kube cp /etc/origin/master/admin.kubeconfig .kube/config
do not start master service at this time.