# OpenShift Agent install disconnected
## Prepare
### Install Mirror Registry
[download](https://console.redhat.com/openshift/downloads#tool-mirror-registry)
找一台RHEL 8以上(2cores,8G,500G)的主機，安裝podman後，將下載的 mirror-registry.tar.gz 放置在主機上，並解壓縮
(範例:/quay/registry.tar.gz , /quay/quayroot空間為主要放置image的路徑 )
```
# ./mirror-registry install
```
裝完後會得到如下的資訊,提供credentials
```
INFO[2023-03-31 23:01:24] Quay installed successfully, config data is stored in ~/quay-install
INFO[2023-03-31 23:01:24] Quay is available at https://mirror-registry.ocp4.lab.local:8443 with credentials (init, 3lQso4O0Jx5zP976q8rHReuVIMk1DLi2)
```
嘗試login
```
[root@mirror-registry quay]# podman login -u init -p 3lQso4O0Jx5zP976q8rHReuVIMk1DLi2 mirror-registry.ocp4.lab.local:8443 --tls-verify=false
Login Succeeded!
```
下載oc ,oc-mirror
https://console.redhat.com/openshift/downloads
解壓縮到/usr/local/bin
```
chmod +x oc-mirror
oc mirror help #verify install success
```
將pull-secret 輸出成json格式，放置到/root/pull-secret
```
[root@mirror-registry tmp]# cat ./pull-secret |jq . >/root/pull-secret.json
將 quay secret ($XDG_RUNTIME_DIR/containers/auth.json)添加到/root/pull-secret中，並且置換掉$XDG_RUNTIME_DIR/containers/auth.json
```
結果範例如下
```
{
"auths": {
"mirror-registry.ocp4.lab.local:8443": {
"auth": "aW5pdDozbFFzbzRPMEp4NXpQOTc2cThySFJldVZJTWsxRExpMg==",
"email": "bchen@redhat.com"
},
"cloud.openshift.com": {
"auth": "b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfZWZlM2Q5ZjQ",
"email": "bchen@redhat.com"
},
"quay.io": {
"auth": "b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfZWZlM2Q5ZjQ",
"email": "bchen@redhat.com"
},
"registry.connect.redhat.com": {
"auth": "fHVoYy1wb29sLTFkYTBkZTRmLTQ0MzUtNGM0Ny1hN2FiLTRjMmY4ZTY3ZGE0MjpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSXlOekEyWkRNek5UVXlNMkkwT0dWa09UTTVME9XSm1OakZsWVNKOS5ETDNWZFV0cV91ZENGMFVjNUNscnFrVnhJYk5yQWFiMF",
"email": "bchen@redhat.com"
},
"registry.redhat.io": {
"auth": "fHVoYy1wb29sLTFkYTBkZTRmLTQ0MzUtNGM0Ny1hN2FiLTRjMmY4ZTY3ZGE0MjpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSXlOekEyWkRNek5UVXlNMkkwT0dWa09UTTVORE13TkRBME9XSm1OakZsWVNKOS5ETDNWZFV0cV91ZENGMFVjNUNscnFrVnhJYk5yQWFiMF9EcFo0bFoteGswa",
"email": "bchen@redhat.com"
}
}
}
[root@mirror-registry ~]# oc-mirror list releases --channels --version=4.12
Listing channels for version 4.12.
candidate-4.13
candidate-4.12
eus-4.12
fast-4.12
stable-4.12
[root@mirror-registry ~]# oc-mirror list releases --channel=eus-4.12
Channel: eus-4.12
Architecture: amd64
4.12.0
4.12.1
4.12.2
4.12.3
4.12.4
4.12.5
4.12.6
4.12.7
4.12.8
oc mirror init --registry mirror-registry.ocp4.lab.local/quay/oc-mirror-metadata >imageset-config.yaml
Edit imageset-config.yaml
example:
kind: ImageSetConfiguration
apiVersion: mirror.openshift.io/v1alpha2
storageConfig:
registry:
imageURL: mirror-registry.ocp4.lab.local:8443/mirror/oc-mirror-metadata
skipTLS: true
mirror:
platform:
channels:
- name: stable-4.12
type: ocp
operators:
- catalog: registry.redhat.io/redhat/redhat-operator-index:v4.13
packages:
- name: serverless-operator
channels:
- name: stable
additionalImages:
- name: registry.redhat.io/ubi8/ubi:latest
helm: {}
mirror from mirror
oc mirror --config=./imageset-config.yaml docker://mirror-registry.ocp4.lab.local:8443
note:
oc adm catalog mirror file://redhat/redhat-operator-index:v4.13 REGISTRY/REPOSITORY
source:
https://blog.csdn.net/weixin_43902588/article/details/124121691
```
### DNS 設定～～略
### 準備 install-config.yaml and agent-config.yaml
```bash=
dnf install /usr/bin/nmstatectl -y
```
install-config.yaml 範例
```
apiVersion: v1
baseDomain: lab.local
compute:
- architecture: amd64
hyperthreading: Enabled
name: worker
replicas: 3
controlPlane:
architecture: amd64
hyperthreading: Enabled
name: master
replicas: 3
metadata:
name: ocp4
networking:
clusterNetwork:
- cidr: 10.128.0.0/14
hostPrefix: 23
machineNetwork:
- cidr: 192.168.8.0/24
networkType: OVNKubernetes
serviceNetwork:
- 172.30.0.0/16
platform:
baremetal:
apiVIPs:
- 192.168.8.30
ingressVIPs:
- 192.168.8.29
hosts:
- name: master-1.ocp4.lab.local
role: master
bootMACAddress: 00:50:56:b0:4d:d2
- name: master-2.ocp4.lab.local
role: master
bootMACAddress: 00:50:56:b0:e9:03
- name: master-3.ocp4.lab.local
role: master
bootMACAddress: 00:50:56:b0:57:58
- name: worker-1.ocp4.lab.local
role: worker
bootMACAddress: 00:50:56:b0:2d:9a
- name: worker-2.ocp4.lab.local
role: worker
bootMACAddress: 00:50:56:b0:da:32
- name: worker-3.ocp4.lab.local
role: worker
bootMACAddress: 00:50:56:b0:42:c6
pullSecret: '{"auths":{"mirror-registry.ocp4.lab.local:8443":{"auth":"aW5pdDozbFFzbzRPMEp4NXpQOTc2cThySFJldVZJTWsxRExpMg==","email":"bchen@redhat.com"}}}'
sshKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfGpEDh5Cu7Z+jBzGZSK1ihaLGWBhsnFsbbzuC8XwpIhndh6D6bOMKgslDeqSpF5G2cv+29FDzl5XuTO8wBe+p+VrJFvElltDT9UxziRVaJxSsq+hDrJKCEiFmzynaRuLBz59SiO72hto7zBITP6Ou5lohHony48R7Rk1ywS5p76e3c7YWvoZVaqGb341/JGgk0+Yay3I4LbblgoUbcHzfsWdJV2k3V7JAs/pms1wAXoc7KU9gizTK/iJxnSv4dXhb6IqW6dm1039rFO0AAF2HbwJXB1sDjQIRU61CuJ4ROnWcaYtE0TXeHgC26R4HssJVEmcWb3NrE3iPqDKkKnW+CwPlRrnhZw891VNtxqXEeevb9N5iG1uWuj/4v28cmGnySp4WHgFCiHAjq/YOSjlAOEwGwDKbf9PxyDoXDcgHriK9RAxYvI3FC/xn65x/kTcYC7CbiqRfzgRB1iACE6008IgmZrB5bAC5ZLXLCpnYbIOASV9S1oM= root@bastion.ocp4.lab.com'
imageContentSources:
- mirrors:
- mirror-registry.ocp4.lab.local:8443/openshift/release
source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
- mirrors:
- mirror-registry.ocp4.lab.local:8443/openshift/release-images
source: quay.io/openshift-release-dev/ocp-release
additionalTrustBundle: |
-----BEGIN CERTIFICATE-----
MIID4TCCAsmgAwIBAgIUGwOx5MhBUy/1lbZYd0CjwAlLuJowDQYJKoZIhvcNAQEL
BQAweDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMREwDwYDVQQHDAhOZXcgWW9y
azENMAsGA1UECgwEUXVheTERMA8GA1UECwwIRGl2aXNpb24xJzAlBgNVBAMMHm1p
cnJvci1yZWdpc3RyeS5vY3A0LmxhYi5sb2NhbDAeFw0yMzA0MDUxNDQ0MTNaFw0y
NjAxMjMxNDQ0MTNaMHgxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJWQTERMA8GA1UE
BwwITmV3IFlvcmsxDTALBgNVBAoMBFF1YXkxETAPBgNVBAsMCERpdmlzaW9uMScw
JQYDVQQDDB5taXJyb3ItcmVnaXN0cnkub2NwNC5sYWIubG9jYWwwggEiMA0GCSqG
SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDXzfUu8D+fhROius5Rg6eVlyaLfoL+qnV6
kVUfHS5YSfGlGBFF0mlT9l5uzi7d7hEjH91QcH8lGHTH6KWdCG4exo93+WxuvAI3
oqClo0o0yY2QD0fGbu8xuWg1Qxo2mUnvaEel7pP2O41raMez/kQ1ZbYdq/+be6fW
lY4Kiw7uoZbCJmGfcGT+XRr2qSsl12FIujeH7FfXZ+XAtJbe6H+v6WozrRw08Hfz
caLUr2qA2Ks/K2JvKyDzJgXbrSool3jdA9unS42/ntX7DoZPNF+d81wf6yIb71He
Hj7/T+5a0IrL/2GcKP/CeBnm1iH8IvIKaWq03kPJU8QWxn3HjltxAgMBAAGjYzBh
MAsGA1Ud
aXJyb3ItcmVnaXN0cnkub2NwNC5sYWIubG9jYWwwEgYDVR0TAQH/BAgwBgEB/wIB
ATANBgkqhkiG9w0BAQsFAAOCAQEAvtYPscbh0lRKPADoEcolKV8YUZgAiH14G/9H
HVJRj4voFWqtwnGfSKDq9cLC8hqAJTf/lMfIv4DS7yuzExr27DleZPskkif5CBNv
ACZB9OYbUbNO8IdKq0Y3NAhkxTPBv481mA87apXmWKNdSAIG6DzOnR7VXTtHg20u
eTYA4PNwSZMJpnMkig7CKkQsnUbnkUC1/34nEDL8nBPYPsbLhhpp2HwS0ovGCfdf
rUbCEA67Th+Sxm8FZw+MqzbdXr7mLAYsEFafj28xhKD+28qxR3zGOwq59T+rJJfc
0VDLm/wx9MpQbcxXhHmE3KCZi9DsuVEVfCuHVxNtX9C50cVC/A==
-----END CERTIFICATE-----
```
agent-config.yaml 範例
```
apiVersion: v1alpha1
kind: AgentConfig
metadata:
name: ocp4
rendezvousIP: 192.168.8.31
hosts:
- hostname: master-1.ocp4.lab.local
role: master
interfaces:
- name: enp1s0
macAddress: 00:50:56:b0:4d:d2
networkConfig:
interfaces:
- name: enp1s0
type: ethernet
state: up
mac-address: 00:50:56:b0:4d:d2
ipv4:
enabled: true
address:
- ip: 192.168.8.31
prefix-length: 24
dhcp: false
dns-resolver:
config:
server:
- 192.168.8.254
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.8.1
next-hop-interface: enp1s0
table-id: 254
- hostname: master-2.ocp4.lab.local
role: master
interfaces:
- name: enp1s0
macAddress: 00:50:56:b0:e9:03
networkConfig:
interfaces:
- name: enp1s0
type: ethernet
state: up
mac-address: 00:50:56:b0:e9:03
ipv4:
enabled: true
address:
- ip: 192.168.8.32
prefix-length: 24
dhcp: false
dns-resolver:
config:
server:
- 192.168.8.254
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.8.1
next-hop-interface: enp1s0
table-id: 254
- hostname: master-3.ocp4.lab.local
role: master
interfaces:
- name: enp1s0
macAddress: 00:50:56:b0:57:58
networkConfig:
interfaces:
- name: enp1s0
type: ethernet
state: up
mac-address: 00:50:56:b0:57:58
ipv4:
enabled: true
address:
- ip: 192.168.8.33
prefix-length: 24
dhcp: false
dns-resolver:
config:
server:
- 192.168.8.254
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.8.1
next-hop-interface: enp1s0
table-id: 254
- hostname: worker-1.ocp4.lab.local
role: worker
interfaces:
- name: enp1s0
macAddress: 00:50:56:b0:2d:9a
networkConfig:
interfaces:
- name: enp1s0
type: ethernet
state: up
mac-address: 00:50:56:b0:2d:9a
ipv4:
enabled: true
address:
- ip: 192.168.8.51
prefix-length: 24
dhcp: false
dns-resolver:
config:
server:
- 192.168.8.254
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.8.1
next-hop-interface: enp1s0
table-id: 254
- hostname: worker-2.ocp4.lab.local
role: worker
interfaces:
- name: enp1s0
macAddress: 00:50:56:b0:da:32
networkConfig:
interfaces:
- name: enp1s0
type: ethernet
state: up
mac-address: 00:50:56:b0:da:32
ipv4:
enabled: true
address:
- ip: 192.168.8.52
prefix-length: 24
dhcp: false
dns-resolver:
config:
server:
- 192.168.8.254
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.8.1
next-hop-interface: enp1s0
table-id: 254
- hostname: worker-3.ocp4.lab.local
role: worker
interfaces:
- name: enp1s0
macAddress: 00:50:56:b0:42:c6
networkConfig:
interfaces:
- name: enp1s0
type: ethernet
state: up
mac-address: 00:50:56:b0:42:c6
ipv4:
enabled: true
address:
- ip: 192.168.8.53
prefix-length: 24
dhcp: false
dns-resolver:
config:
server:
- 192.168.8.254
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.8.1
next-hop-interface: enp1s0
table-id: 254
```
將範例兩個檔案放到某目錄（例如 ~/ocp4)
建立iso
```bash=
# openshift-install --dir ocp4 agent create image
WARNING Platform.Baremetal.Hosts[0].Name: master-1.ocp4.lab.local is ignored
WARNING Platform.Baremetal.Hosts[0].Role: master is ignored
WARNING Platform.Baremetal.Hosts[0].BootMACAddress: 00:50:56:b0:4d:d2 is ignored
WARNING Platform.Baremetal.Hosts[0].HardwareProfile: default is ignored
WARNING Platform.Baremetal.Hosts[1].Name: master-2.ocp4.lab.local is ignored
WARNING Platform.Baremetal.Hosts[1].Role: master is ignored
WARNING Platform.Baremetal.Hosts[1].BootMACAddress: 00:50:56:b0:e9:03 is ignored
WARNING Platform.Baremetal.Hosts[1].HardwareProfile: default is ignored
WARNING Platform.Baremetal.Hosts[2].Name: master-3.ocp4.lab.local is ignored
WARNING Platform.Baremetal.Hosts[2].Role: master is ignored
WARNING Platform.Baremetal.Hosts[2].BootMACAddress: 00:50:56:b0:57:58 is ignored
WARNING Platform.Baremetal.Hosts[2].HardwareProfile: default is ignored
WARNING Platform.Baremetal.Hosts[3].Name: worker-1.ocp4.lab.local is ignored
WARNING Platform.Baremetal.Hosts[3].Role: worker is ignored
WARNING Platform.Baremetal.Hosts[3].BootMACAddress: 00:50:56:b0:2d:9a is ignored
WARNING Platform.Baremetal.Hosts[3].HardwareProfile: default is ignored
WARNING Platform.Baremetal.Hosts[4].Name: worker-2.ocp4.lab.local is ignored
WARNING Platform.Baremetal.Hosts[4].Role: worker is ignored
WARNING Platform.Baremetal.Hosts[4].BootMACAddress: 00:50:56:b0:da:32 is ignored
WARNING Platform.Baremetal.Hosts[4].HardwareProfile: default is ignored
WARNING Platform.Baremetal.Hosts[5].Name: worker-3.ocp4.lab.local is ignored
WARNING Platform.Baremetal.Hosts[5].Role: worker is ignored
WARNING Platform.Baremetal.Hosts[5].BootMACAddress: 00:50:56:b0:42:c6 is ignored
WARNING Platform.Baremetal.Hosts[5].HardwareProfile: default is ignored
WARNING The ImageContentSources configuration in install-config.yaml should have at-least one source field matching the releaseImage value quay.io/openshift-release-dev/ocp-release@sha256
INFO The rendezvous host IP (node0 IP) is 192.168.8.31
INFO Extracting base ISO from release payload
INFO Base ISO obtained from release and cached at /root/.cache/agent/image_cache/coreos-x86_64.iso
INFO Consuming Agent Config from target directory
INFO Consuming Install Config from target directory
```
將 /root/ocp4/agent.x86_64.iso下載，並放入master-1開機
PS.如果是VMware 虛擬機，記得disk.EnableUUID 設定為TRUE
依序放入 master-2 , master-3 , worker-1 ,worker-2 ,worker-3開機
```bash=
[root@bastion ~]# openshift-install --dir ocp4 agent wait-for bootstrap-complete --log-level=info
INFO Cluster is not ready for install. Check validations
WARNING Cluster validation: The cluster has hosts that are not ready to install.
WARNING Cluster validation: Clusters must have at most 3 dedicated masters. Please check your configuration and add or remove hosts as needed to meet the requirement.
WARNING Host master-1.ocp4.lab.local validation: Host couldn't synchronize with any NTP server
INFO Host 4dca3042-eaad-efdd-1481-e466ab7c2922: Successfully registered
WARNING Host master-2.ocp4.lab.local validation: Host couldn't synchronize with any NTP server
INFO Host master-2.ocp4.lab.local: calculated role is master
INFO Host a90c3042-0d23-c6d9-8b5d-ecd6c594c726: Successfully registered
WARNING Host master-3.ocp4.lab.local validation: No connectivity to the majority of hosts in the cluster
WARNING Host master-3.ocp4.lab.local validation: Host couldn't synchronize with any NTP server
WARNING Host master-2.ocp4.lab.local validation: No connectivity to the majority of hosts in the cluster
WARNING Host master-1.ocp4.lab.local validation: No connectivity to the majority of hosts in the cluster
WARNING Host master-1.ocp4.lab.local: validation 'belongs-to-majority-group' failed
INFO Cluster validation: The cluster has a sufficient number of master candidates.
INFO Host master-1.ocp4.lab.local validation: Host NTP is synced
INFO Host master-1.ocp4.lab.local: validation 'ntp-synced' is now fixed
INFO Host master-2.ocp4.lab.local validation: Host NTP is synced
INFO Host master-2.ocp4.lab.local: validation 'ntp-synced' is now fixed
INFO Host master-3.ocp4.lab.local validation: Host NTP is synced
INFO Host master-3.ocp4.lab.local: validation 'ntp-synced' is now fixed
INFO Host master-3.ocp4.lab.local validation: Host has connectivity to the majority of hosts in the cluster
INFO Host master-1.ocp4.lab.local validation: Host has connectivity to the majority of hosts in the cluster
INFO Host master-2.ocp4.lab.local validation: Host has connectivity to the majority of hosts in the cluster
INFO Host master-2.ocp4.lab.local: updated status from insufficient to known (Host is ready to be installed)
INFO Cluster is ready for install
INFO Cluster validation: All hosts in the cluster are ready to install.
INFO Preparing cluster for installation
INFO Cluster validation: All hosts in the cluster are ready to install.
INFO Host worker-3.ocp4.lab.local: updated status from known to preparing-for-installation (Host finished successfully to prepare for installation)
INFO Host worker-2.ocp4.lab.local: New image status quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e4ec37845ebf2d8d772a18ade2d439d77b8b23e0546f73e6fdc973822083766a. result: success. time: 2.47 seconds; size: 400.15 Megabytes; download rate: 169.57 MBps
INFO Host master-1.ocp4.lab.local: updated status from preparing-for-installation to preparing-successful (Host finished successfully to prepare for installation)
INFO Host master-2.ocp4.lab.local: New image status quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e4ec37845ebf2d8d772a18ade2d439d77b8b23e0546f73e6fdc973822083766a. result: success. time: 2.28 seconds; size: 400.15 Megabytes; download rate: 184.13 MBps
INFO Host master-2.ocp4.lab.local: updated status from preparing-for-installation to preparing-successful (Host finished successfully to prepare for installation)
INFO Host worker-3.ocp4.lab.local: New image status quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e4ec37845ebf2d8d772a18ade2d439d77b8b23e0546f73e6fdc973822083766a. result: success. time: 2.47 seconds; size: 400.15 Megabytes; download rate: 169.54 MBps
INFO Host worker-3.ocp4.lab.local: updated status from preparing-for-installation to preparing-successful (Host finished successfully to prepare for installation)
...
....
.....
.....
INFO Host: master-3.ocp4.lab.local, reached installation stage Writing image to disk: 94%
INFO Host: worker-3.ocp4.lab.local, reached installation stage Writing image to disk: 96%
INFO Host: worker-1.ocp4.lab.local, reached installation stage Writing image to disk: 100%
INFO Uploaded logs for host worker-1.ocp4.lab.local cluster afed1832-03ef-495d-bdf2-81770bfd3dfe
INFO Host: master-3.ocp4.lab.local, reached installation stage Waiting for control plane
INFO Host: master-2.ocp4.lab.local, reached installation stage Waiting for control plane
INFO Host: worker-3.ocp4.lab.local, reached installation stage Rebooting
INFO Host: worker-3.ocp4.lab.local, reached installation stage Configuring
INFO Host: worker-1.ocp4.lab.local, reached installation stage Configuring
INFO Host: master-1.ocp4.lab.local, reached installation stage Waiting for bootkube
INFO Host: worker-3.ocp4.lab.local, reached installation stage Done
INFO Uploaded logs for host master-2.ocp4.lab.local cluster afed1832-03ef-495d-bdf2-81770bfd3dfe
INFO Host: master-2.ocp4.lab.local, reached installation stage Rebooting
INFO Host: master-2.ocp4.lab.local, reached installation stage Waiting for ignition
INFO Host: master-3.ocp4.lab.local, reached installation stage Waiting for ignition
INFO Host: worker-2.ocp4.lab.local, reached installation stage Configuring
INFO Host: master-1.ocp4.lab.local, reached installation stage Waiting for controller: waiting for controller pod ready event
INFO Bootstrap configMap status is complete
INFO cluster bootstrap is complete
```
第一次裝
一樣的iso第二次裝
問題：
1. ROLE看起來是隨機的，在install-config.yaml裡的role設定似乎沒有生效(HA mode)，如果真要對應role，workaround是不是建立compact mode OCP再加入worker node ?
2. Baremetal install 不需要外部的Load balancer，How to add worker node after installation complete ?
```
Sign in with Wallet
Connect another wallet