# rke2 offline install
## 1. 下載1.24.6離線安裝所需image
```shell=
rancher@rms1:~> sudo mkdir /root/rke2-artifacts && cd /root/rke2-artifacts/
mkdir: cannot create directory ‘/root/rke2-artifacts’: File exists
rancher@rms1:~> sudo su
rms1:/home/rancher # cd /root/rke2-artifacts/
rms1:~/rke2-artifacts # ll
total 0
rms1:~/rke2-artifacts # curl -OLs https://github.com/rancher/rke2/releases/download/v1.24.6%2Brke2r1/rke2-images.linux-amd64.tar.zst
rms1:~/rke2-artifacts # curl -OLs https://github.com/rancher/rke2/releases/download/v1.24.6%2Brke2r1/rke2.linux-amd64.tar.gz
rms1:~/rke2-artifacts # curl -OLs https://github.com/rancher/rke2/releases/download/v1.24.6%2Brke2r1/sha256sum-amd64.txt
rms1:~/rke2-artifacts # curl -sfL https://get.rke2.io --output install.sh
rms1:~/rke2-artifacts # ll
total 823052
-rw-r--r-- 1 root root 21438 Oct 11 20:19 install.sh
-rw-r--r-- 1 root root 794531974 Oct 11 20:17 rke2-images.linux-amd64.tar.zst
-rw-r--r-- 1 root root 48238609 Oct 11 20:18 rke2.linux-amd64.tar.gz
-rw-r--r-- 1 root root 3626 Oct 11 20:18 sha256sum-amd64.txt
```
## 2. 解壓縮與基本設定
```shell=
rms1:~/rke2-artifacts # INSTALL_RKE2_ARTIFACT_PATH=/root/rke2-artifacts sh install.sh
[WARN] /usr/local is read-only or a mount point; installing to /opt/rke2
[INFO] staging local checksums from /root/rke2-artifacts/sha256sum-amd64.txt
[INFO] staging zst airgap image tarball from /root/rke2-artifacts/rke2-images.linux-amd64.tar.zst
[INFO] staging tarball from /root/rke2-artifacts/rke2.linux-amd64.tar.gz
[INFO] verifying airgap tarball
grep: /tmp/rke2-install.XRvs61dJ7e/rke2-images.checksums: No such file or directory
[INFO] installing airgap tarball to /var/lib/rancher/rke2/agent/images
[INFO] verifying tarball
[INFO] unpacking tarball file to /opt/rke2
[INFO] updating tarball contents to reflect install path
[INFO] moving systemd units to /etc/systemd/system
[INFO] install complete; you may want to run: export PATH=$PATH:/opt/rke2/bin
rms1:~/rke2-artifacts # export PATH=$PATH:/opt/rke2/bin
```
## 3. 叢集基礎組態
```shell=
rms1:~/rke2-artifacts # mkdir -p /etc/rancher/rke2/
[sudo] root 的密碼：
rms1:~/rke2-artifacts # vim /etc/rancher/rke2/config.yaml
rms1:~/rke2-artifacts # cat /etc/rancher/rke2/config.yaml
node-name:
- "rms1"
token: my-shared-secret
```
## 4. 啟用RKE2服務
```shell=
rms1:~/rke2-artifacts # systemctl enable --now rke2-server
Created symlink /etc/systemd/system/multi-user.target.wants/rke2-server.service → /etc/systemd/system/rke2-server.service.
```
## 5. 設定一般帳號使用kubectl
```shell=
rms1:~/rke2-artifacts # exit
exit
rancher@rms1:~> mkdir .kube
rancher@rms1:~> sudo cp /etc/rancher/rke2/rke2.yaml .kube/config
[sudo] password for root:
rancher@rms1:~> sudo chown rancher .kube/config
rancher@rms1:~> sudo cp /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/
rancher@rms1:~> kubectl get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system cloud-controller-manager-rms1 1/1 Running 0 3m5s
kube-system etcd-rms1 1/1 Running 0 2m46s
kube-system helm-install-rke2-canal-lpv22 0/1 Completed 0 3m
kube-system helm-install-rke2-coredns-xndpj 0/1 Completed 0 3m
kube-system helm-install-rke2-ingress-nginx-5r5sq 0/1 Completed 0 3m
kube-system helm-install-rke2-metrics-server-wkz6c 0/1 Completed 0 3m
kube-system kube-apiserver-rms1 1/1 Running 0 2m37s
kube-system kube-controller-manager-rms1 1/1 Running 0 2m30s
kube-system kube-proxy-rms1 1/1 Running 0 2m57s
kube-system kube-scheduler-rms1 1/1 Running 0 2m37s
kube-system rke2-canal-clqp4 2/2 Running 0 2m41s
kube-system rke2-coredns-rke2-coredns-76cb76d66-xpnqg 1/1 Running 0 2m42s
kube-system rke2-coredns-rke2-coredns-autoscaler-58867f8fc5-hzz2l 1/1 Running 0 2m42s
kube-system rke2-ingress-nginx-controller-zfbvx 1/1 Running 0 93s
kube-system rke2-metrics-server-6979d95f95-kmbcv 1/1 Running 0 109s
```
## 6. 加入其他節點
```shell=
```
## 7. 安裝helm
```shell=
rancher@rms1:~> wget https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
--2022-10-11 20:33:57-- https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
Resolving get.helm.sh (get.helm.sh)... 152.199.39.108, 2606:2800:247:1cb7:261b:1f9c:2074:3c
Connecting to get.helm.sh (get.helm.sh)|152.199.39.108|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 14026634 (13M) [application/x-tar]
Saving to: ‘helm-v3.9.4-linux-amd64.tar.gz’
helm-v3.9.4-linux-amd64.tar.gz 100%[======================================================>] 13.38M 11.0MB/s in 1.2s
2022-10-11 20:34:00 (11.0 MB/s) - ‘helm-v3.9.4-linux-amd64.tar.gz’ saved [14026634/14026634]
rancher@rms1:~> tar zxvf helm-v3.9.4-linux-amd64.tar.gz
linux-amd64/
linux-amd64/helm
linux-amd64/LICENSE
linux-amd64/README.md
rancher@rms1:~> sudo cp linux-amd64/helm /usr/local/bin/
```
## 8. 啟用cert-manager(optional)
```shell=
rancher@rms1:~> kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.7.1/cert-manager.crds.yaml
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created
rancher@rms1:~> helm repo add jetstack https://charts.jetstack.io
"jetstack" has been added to your repositories
rancher@rms1:~> helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "jetstack" chart repository
Update Complete. ⎈Happy Helming!⎈
rancher@rms1:~> helm install cert-manager jetstack/cert-manager \
> --namespace cert-manager \
> --create-namespace \
> --version v1.7.1
NAME: cert-manager
LAST DEPLOYED: Tue Oct 11 20:35:43 2022
NAMESPACE: cert-manager
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
cert-manager v1.7.1 has been deployed successfully!
In order to begin issuing certificates, you will need to set up a ClusterIssuer
or Issuer resource (for example, by creating a 'letsencrypt-staging' issuer).
More information on the different types of issuers and how to configure them
can be found in our documentation:
https://cert-manager.io/docs/configuration/
For information on how to configure cert-manager to automatically provision
Certificates for Ingress resources, take a look at the `ingress-shim`
documentation:
https://cert-manager.io/docs/usage/ingress/
rancher@rms1:~> kubectl -n cert-manager get po
NAME READY STATUS RESTARTS AGE
cert-manager-646c67487-p9w77 1/1 Running 0 72s
cert-manager-cainjector-7cb8669d6b-cwghz 1/1 Running 0 72s
cert-manager-webhook-696c5db7ff-sjbw4 1/1 Running 0 72s
```
## 9. 啟用Rancher 2.6.8
```shell=
rancher@rms1:~> helm repo add rancher-stable https://releases.rancher.com/server-charts/stable
"rancher-stable" has been added to your repositories
rancher@rms1:~> kubectl create namespace cattle-system
namespace/cattle-system created
rancher@rms1:~> helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "jetstack" chart repository
...Successfully got an update from the "rancher-stable" chart repository
Update Complete. ⎈Happy Helming!⎈
rancher@rms1:~> helm install rancher rancher-stable/rancher --namespace cattle-system --set hostname=rancher.example.com
NAME: rancher
LAST DEPLOYED: Tue Oct 11 20:37:49 2022
NAMESPACE: cattle-system
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Rancher Server has been installed.
NOTE: Rancher may take several minutes to fully initialize. Please standby while Certificates are being issued, Containers are started and the Ingress rule comes up.
Check out our docs at https://rancher.com/docs/
If you provided your own bootstrap password during installation, browse to https://rancher.example.com to get started.
If this is the first time you installed Rancher, get started by running this command and clicking the URL it generates:
echo https://rancher.example.com/dashboard/?setup=$(kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}')
To get just the bootstrap password on its own, run:
kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{ "\n" }}'
Happy Containering!
rancher@rms1:~> kubectl -n cattle-system get po
NAME READY STATUS RESTARTS AGE
rancher-69595dc9c4-mrhwt 0/1 ContainerCreating 0 2m17s
rancher-69595dc9c4-nhl5m 0/1 ContainerCreating 0 2m17s
rancher-69595dc9c4-tswd8 0/1 ContainerCreating 0 2m17s
rancher@rms1:~> kubectl -n cattle-system get po
NAME READY STATUS RESTARTS AGE
rancher-69595dc9c4-mrhwt 0/1 Running 0 2m27s
rancher-69595dc9c4-nhl5m 0/1 Running 0 2m27s
rancher-69595dc9c4-tswd8 0/1 ContainerCreating 0 2m27s
rancher@rms1:~> kubectl -n cattle-system get po
NAME READY STATUS RESTARTS AGE
rancher-69595dc9c4-mrhwt 1/1 Running 2 (77s ago) 7m12s
rancher-69595dc9c4-nhl5m 1/1 Running 1 (2m39s ago) 7m12s
rancher-69595dc9c4-tswd8 1/1 Running 2 (77s ago) 7m12s
```
### 2.1. harbor須先取得安裝套件
1. docker compose離線安裝腳本、binary檔。
2. harbor 2.5.3離線安裝包。
:::warning
01. 共計三個檔案，需存放同一個目錄。
02. 先執行init.sh，會自動匯入docker-compose指令與基本組態。
03. harbor安裝需要先解壓縮2.5.3安裝包。
04. harbor安裝包解壓縮後，會建立harbor資料夾，請把server相關的檔案全部放入harbor資料夾。
05. 複製一份harbor.yml.sample為harbor.yml，請參考文件sample設定。
06. 安裝harbor可指定scanner backend(--with-XXXX)，注意clair已被移除。
07. rancher image列表過多，官方匯入使用docker load指令，初期解壓縮階段不會有任何系統訊息，後續才有匯入訊息。
08. 請先匯入cert-manager相關image做為測試。
09. harbor預設使用80、443 port。
10. 預設公開的repository為library，image push請push到192.168.99.9/library
11. 請關閉VM防火牆、設定固定IP。
:::
### 2.2. 可用image scanner backend一攬：
```shell=
rancher@harbor:~/harbor> ./install.sh --help
Note: Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https.
Please set --with-trivy if needs enable Trivy in Harbor
Please set --with-chartmuseum if needs enable Chartmuseum in Harbor
```
### 2.3. harbor node
VM重點組態
:::info
1. SLES 15 SP3 or SP4與docker module。
2. 需要設定hosts解析。
3. 硬碟需要2顆硬碟，一顆作為OS安裝使用，需要100GB，另一顆需要200G，作為rancher相關image存放使用。
＊. Rancher image list非常的長，但不包含cert-manager。
:::
**停止docker服務：**
```shell=
sudo systemctl stop docker
```
**編輯 /etc/docker/daemon.json**
```
{
"log-level": "warn",
"log-driver": "json-file",
"log-opts": {
"max-size": "10m",
"max-file": "5"
}, "data-root": "/dockerimages",
"insecure-registries": ["192.168.98.9"]
}
```
**啟動docker**
```shell=
sudo systemctl start docker
```
**copy docker-compose相關檔案到192.168.99.9:/home/rancher/**
```shell=
sam@sam:~> cd docker-compose/
sam@sam:~/docker-compose> ls
docker-compose harbor-offline-installer-v2.5.3.tgz init.sh
sam@sam:~/docker-compose> scp * rancher@192.168.99.9:/home/rancher/
Password:
docker-compose 100% 25MB 52.7MB/s 00:00
harbor-offline-installer-v2.5.3.tgz 100% 629MB 124.2MB/s 00:05
init.sh 100% 577 32.6KB/s 00:00
```
登入到harbor節點，執行init.sh安裝docker-compose、建立harbor所使用的憑證。
```shell=
rancher@harbor:~> ll
total 669696
drwxr-xr-x 2 rancher users 6 Jul 28 16:15 bin
-rwxr-xr-x 1 rancher users 25747456 Aug 2 11:23 docker-compose
-rw-r--r-- 1 rancher users 660014621 Aug 2 11:23 harbor-offline-installer-v2.5.3.tgz
-rwxr-xr-x 1 rancher users 577 Aug 2 11:23 init.sh
drwxr-xr-x 2 rancher users 24 Jul 28 16:15 public_html
rancher@harbor:~> sudo ./init.sh
basic init
Docker Compose version v2.6.1
Docker Compose version v2.6.1
self-sigh key
Generating RSA private key, 2048 bit long modulus (2 primes)
....................................+++++
...................................................+++++
e is 65537 (0x010001)
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [AU]:
State or Province Name (full name) [Some-State]:
Locality Name (eg, city) []:
Organization Name (eg, company) [Internet Widgits Pty Ltd]:
Organizational Unit Name (eg, section) []:
Common Name (e.g. server FQDN or YOUR name) []:
Email Address []:
Please enter the following 'extra' attributes
to be sent with your certificate request
A challenge password []:
An optional company name []:
Signature ok
subject=C = AU, ST = Some-State, O = Internet Widgits Pty Ltd
Getting Private key
```
解壓縮harbor-offline-installer-v2.5.3.tgz
```shell=
rancher@harbor:~> tar -zxvf harbor-offline-installer-v2.5.3.tgz
harbor/harbor.v2.5.3.tar.gz
harbor/prepare
harbor/LICENSE
harbor/install.sh
harbor/common.sh
harbor/harbor.yml.tmpl
```
copy server相關檔案
```shell=
rancher@harbor:~> sudo cp server.* harbor
```
複製harbor.yml.sample為harbor.yml
```shell=
rancher@harbor:~> cd harbor/
rancher@harbor:~/harbor> cp harbor.yml.tmpl harbor.yml
```
修改harbor.yml
```yaml=
Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: 192.168.99.9
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /home/rancher/harbor/server.crt
private_key: /home/rancher/harbor/server.key
...
...
...
```
匯入與啟動harbor
```shell=
rancher@harbor:~/harbor> sudo ./install.sh --with-trivy
[Step 0]: checking if docker is installed ...
Note: docker version: 19.03.15
[Step 1]: checking docker-compose is installed ...
Note: docker-compose version: 2.6.1
[Step 2]: loading Harbor images ...
ed825a5a3f34: Loading layer [==================================================>] 37.6MB/37.6MB
eb50d8bbd990: Loading layer [==================================================>] 7.668MB/7.668MB
04e75300c772: Loading layer [==================================================>] 7.362MB/7.362MB
e6830bb442bf: Loading layer [==================================================>] 1MB/1MB
Loaded image: goharbor/harbor-portal:v2.5.3
7e761f0c6325: Loading layer [==================================================>] 8.898MB/8.898MB
...
...
...
enerated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /config/trivy-adapter/env
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[Step 5]: starting Harbor ...
[+] Running 11/11
⠿ Network harbor_harbor Created 0.2s
⠿ Container harbor-log Started 5.8s
⠿ Container harbor-db Started 8.3s
⠿ Container registry Started 8.1s
⠿ Container redis Started 7.2s
⠿ Container registryctl Started 7.8s
⠿ Container harbor-portal Started 8.2s
⠿ Container trivy-adapter Started 8.2s
⠿ Container harbor-core Started 8.9s
⠿ Container nginx Started 10.3s
⠿ Container harbor-jobservice Started 10.0s
✔ ----Harbor has been installed and started successfully.----
```
確認harbor是否運作正常，服務啟動會需要約一分鐘時間，待STATUS變成health即可。
```shell=
rancher@harbor:~/harbor> sudo docker container list
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
0526a1244e18 goharbor/nginx-photon:v2.5.3 "nginx -g 'daemon of…" About a minute ago Up About a minute (healthy) 0.0.0.0:80->8080/tcp, 0.0.0.0:443->8443/tcp nginx
c565e76829a4 goharbor/harbor-jobservice:v2.5.3 "/harbor/entrypoint.…" About a minute ago Up About a minute (healthy) harbor-jobservice
13a9a6d40e9c goharbor/harbor-core:v2.5.3 "/harbor/entrypoint.…" About a minute ago Up About a minute (healthy) harbor-core
49d5c2be1c8a goharbor/trivy-adapter-photon:v2.5.3 "/home/scanner/entry…" About a minute ago Up About a minute (healthy) trivy-adapter
36dcdf1b22b4 goharbor/redis-photon:v2.5.3 "redis-server /etc/r…" About a minute ago Up About a minute (healthy) redis
607713f312e2 goharbor/harbor-db:v2.5.3 "/docker-entrypoint.…" About a minute ago Up About a minute (healthy) harbor-db
d9d5cc8e43fc goharbor/registry-photon:v2.5.3 "/home/harbor/entryp…" About a minute ago Up About a minute (healthy) registry
c3cec663307d goharbor/harbor-portal:v2.5.3 "nginx -g 'daemon of…" About a minute ago Up About a minute (healthy) harbor-portal
d561baed3217 goharbor/harbor-registryctl:v2.5.3 "/home/harbor/start.…" About a minute ago Up About a minute (healthy) registryctl
7591e44fb642 goharbor/harbor-log:v2.5.3 "/bin/sh -c /usr/loc…" About a minute ago Up About a minute (healthy) 127.0.0.1:1514->10514/tcp harbor-log
```
關閉harbor指令。
```shell=
rancher@harbor:~/harbor> sudo docker-compose down -v
[+] Running 11/11
⠿ Container harbor-jobservice Removed 1.2s
⠿ Container trivy-adapter Removed 1.8s
⠿ Container registryctl Removed 10.9s
⠿ Container nginx Removed 1.8s
⠿ Container harbor-portal Removed 1.4s
⠿ Container harbor-core Removed 1.5s
⠿ Container harbor-db Removed 4.9s
⠿ Container registry Removed 1.5s
⠿ Container redis Removed 1.9s
⠿ Container harbor-log Removed 10.8s
⠿ Network harbor_harbor Removed 0.2s
```
重新啟動harbor指令。
```shell=
rancher@harbor:~/harbor> sudo docker-compose up -d
[+] Running 11/11
⠿ Network harbor_harbor Created 0.1s
⠿ Container harbor-log Started 3.9s
⠿ Container registryctl Started 4.7s
⠿ Container harbor-portal Started 4.5s
⠿ Container redis Started 5.1s
⠿ Container registry Started 5.5s
⠿ Container harbor-db Started 5.6s
⠿ Container trivy-adapter Started 5.1s
⠿ Container harbor-core Started 5.5s
⠿ Container harbor-jobservice Started 6.0s
⠿ Container nginx Started 6.2s
```
匯入cert-manager相關image到harbor/library中。
```shell=
sam@sam:~/airgap/cert-manager> cd ..
sam@sam:~/airgap> scp cert-manager/* rancher@192.168.98.9:/home/rancher/cert-manager/
Password:
cert-mag.sh 100% 368 20.0KB/s 00:00
cert-manager-cainjector.tar.gz 100% 44MB 111.0MB/s 00:00
cert-manager-controller.tar.gz 100% 64MB 94.5MB/s 00:00
cert-manager.crds.yaml 100% 350KB 3.3MB/s 00:00
cert-manager-ctl.tar.gz 100% 56MB 80.7MB/s 00:00
cert-manager-v1.7.1.tgz 100% 58KB 1.8MB/s 00:00
cert-manager-webhook.tar.gz 100% 51MB 98.6MB/s 00:00
rancher@harbor:~/cert-manager> sudo docker login -u admin 192.168.98.9
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
sudo docker load -i cert-manager-cainjector.tar.gz
sudo docker load -i cert-manager-controller.tar.gz
sudo docker load -i cert-manager-webhook.tar.gz
sudo docker load -i cert-manager-ctl.tar.gz
sudo docker tag quay.io/jetstack/cert-manager-controller:v1.7.1 88.249.100.226/library/cert-manager-controller:v1.7.1
sudo docker tag quay.io/jetstack/cert-manager-webhook:v1.7.1 88.249.100.226/library/cert-manager-webhook:v1.7.1
sudo docker tag quay.io/jetstack/cert-manager-cainjector:v1.7.1 88.249.100.226/library/cert-manager-cainjector:v1.7.1
sudo docker tag quay.io/jetstack/cert-manager-ctl:v1.7.1 88.249.100.226/library/cert-manager-ctl:v1.7.1
XX sudo docker push 88.249.100.226/library/cert-manager:v1.7.1
sudo docker push 88.249.100.226/library/cert-manager-controller:v1.7.1
sudo docker push 88.249.100.226/library/cert-manager-webhook:v1.7.1
sudo docker push 88.249.100.226/library/cert-manager-cainjector:v1.7.1
sudo docker push 88.249.100.226/library/cert-manager-ctl:v1.7.1
```
匯入busybox待K8S環境啟動後測試使用。
```shell=
rancher@harbor:~> sudo docker load -i busybox.tar
[sudo] password for root:
7ad00cd55506: Loading layer [==================================================>] 1.463MB/1.463MB
Loaded image: busybox:latest
rancher@harbor:~> sudo docker image list |grep busybox
busybox latest 62aedd01bd85 7 weeks ago 1.24MB
rancher@harbor:~> sudo docker tag busybox:latest 192.168.98.9/library/busybox:latest
rancher@harbor:~> sudo docker push 192.168.98.9/library/busybox:latest
The push refers to repository [192.168.98.9/library/busybox]
7ad00cd55506: Pushed
latest: digest: sha256:dcdf379c574e1773d703f0c0d56d67594e7a91d6b84d11ff46799f60fb081c52 size: 527
```
上傳Rancher image壓縮包，預計需要兩小時。
```shell=
rancher@harbor:~/rancher> sudo ./rancher-load-images.sh --image-list ./rancher-images.txt --registry 192.168.98.9/library |tee result.log
Loaded image: rancher/k3s-upgrade:v1.22.4-k3s1
Loaded image: rancher/system-agent-installer-rke2:v1.22.9-rke2r1
Loaded image: rancher/hardened-ib-sriov-cni:v1.0.0-build20210310
Loaded image: rancher/rke2-runtime:v1.23.6-rke2r1
Loaded image: rancher/system-agent-installer-rke2:v1.21.12-rke2r1
Loaded image: rancher/hyperkube:v1.22.9-rancher1
Loaded image: rancher/mirrored-calico-pod2daemon-flexvol:v3.21.4
Loaded image: rancher/backup-restore-operator:v2.1.2
Loaded image: rancher/local-path-provisioner:v0.0.19
...
...
...
7b35f2def65d: Pushed
f93fff1ab6f7: Pushed
2: digest: sha256:6f86956ed3802764cf98f5f20ed549a649ffd1f24fb273aef48748da5531b576 size: 1363
```
hosts must add harbor ip
```shell=
192.168.98.21 rke21
192.168.98.9 harbor
```
:::info
1. 到這邊Harbor就設定完成，接下來設定RKE2 airgap安裝模式。
2. RKE2在airgap安裝模式不需要harbor，但是需要設定registories.yaml，在pull image時可以轉向到harbor。
3. RKE2環境建議在匯入rancher image時進行，會省下很多時間。
:::
# RKE2 v1.26+ & Harbor v2.5.3 (in SLES 15sp4)
## Harbor
### 1. 安裝 Docker
```shell=
sudo zypper in docker
```
### 2. 下載 Docker-Compose binary file (v2.15.0)
```shell=
cd /usr/bin
sudo wget https://github.com/docker/compose/releases/download/v2.15.0/docker-compose-linux-x86_64
sudo mv docker-compose-Linux-x86_64 docker-compose
sudo chmod 755 docker-compose
sudo groupadd docker
sudo usermod -aG docker $USER
```
### 3. 下載 Harbor v2.5.3 offline版本
```shell=
wget https://github.com/goharbor/harbor/releases/download/v2.5.3/harbor-offline-installer-v2.5.3.tgz
tar zxvf harbor-offline-installer-v2.5.3.tgz
```
### 4. 編輯 Docker daemon file
檔案位置 /etc/docker/daemon.json 。
```shell=
{
"log-level": "warn",
"log-driver": "json-file",
"insecure-registries": ["192.168.11.90","harbor.example.com"],
"log-opts": {
"max-size": "10m",
"max-file": "5"
}
}
```
將 insecure-registries 新增進 docker daemon.json。
```shell=
"insecure-registries": ["ip","harbor_dns"],
```
並將 docker.service重啟。
```shell=
sudo systemctl stop docker.service
sudo systemctl start docker.service
```
### 5. 生成SSL
```shell=
mkdir ssl
vim mk
chmod +x mk
./mk create $DNS $IP
./mk test
```
:::spoiler cert shell script named "mk"
```shell=!
$ cat mk
#!/bin/bash
dns=$2
ip=$3
help()
{
cat <<EOF
Usage: mk [OPTIONS]
Available options:
create create [DNS] [IP]
delete delete cert
test test
EOF
exit
}
ssl()
{
openssl genrsa -aes256 -passout pass:password -out ca-key.pem 4096
openssl req -new -x509 -sha256 -days 365 -subj "/C=TW/ST=Taipei/L=Taipei/O=test/OU=lab/CN=example" -passin pass:password -key ca-key.pem -out ca.pem
openssl genrsa -out cert-key.pem 4096
openssl req -new -sha256 -subj "/CN=example" -key cert-key.pem -out cert.csr
echo -e "subjectAltName=DNS:${dns},IP:${ip}\nextendedKeyUsage = serverAuth" > extfile.cnf
openssl x509 -req -sha256 -days 365 -passin pass:password -in cert.csr -CA ca.pem -CAkey ca-key.pem -out cert.pem -extfile extfile.cnf -CAcreateserial
}
de()
{
rm ca-key.pem ca.pem ca.srl cert.csr cert-key.pem cert.pem extfile.cnf &>/dev/null
if [ "$?" == "0" ];then
echo "delete all cert ok!"
else
echo "delete cert fail,please check!"
fi
}
ts()
{
openssl verify -CAfile ca.pem -verbose cert.pem
}
case $1 in
create)
if [ "$#" == "3" ];then
ssl
else
help
fi
;;
delete)
de
;;
test)
ts
;;
*)
help
;;
esac
```
:::
### 6. 編輯 harbor.yml
```shell=
# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: harbor.example.com
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /home/rancher/ssl/cert.pem
private_key: /home/rancher/ssl/cert-key.pem
# # Uncomment following will enable tls communication between all harbor components
# internal_tls:
# # set enabled to true means internal tls is enabled
# enabled: true
# # put your cert and key files on dir
# dir: /etc/harbor/tls/internal
# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
# external_url: https://reg.mydomain.com:8433
# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: Harbor12345
```
在運行harbor腳本先，必須先調整 ssl檔案位置及hostname。
### 7. 執行harbor安裝腳本
欲安裝harbor掃描工具 使用 "--with-trivy"，如不使用空白即可。
```shell=
sudo ./install.sh --with-trivy
```
在安裝執行腳本結束後會有 Harbor Successful的字樣，並使用 docker 指令確認所有 container都正確運行。
```shell=
sudo docker container list
```
Docker-compose 指令。
```shell=
# docker-compose 啟動 Harbor
docker-compose up -d
# docker-compose 關閉 Harbor
docker-compose down -v
```
### 8. 登入 harbor
```shell=
sudo docker login -u admin https://harbor.example.com
```
### 9. 測試推送 image 至 harbor
推送 image。
```shell=
docker pull nginx
```
重新命名 image。
```shell=
docker tag nginx harbor.example.com/library/nginx
```
推送 image 至 harbor。
```shell=
docker push harbor.example.com/library/nginx
```
## RKE2
### 1. 下載1.26.1離線安裝所需image
```!
$ sudo su
$ mkdir /root/rke2-artifacts && cd /root/rke2-artifacts/
$ curl -OLs https://github.com/rancher/rke2/releases/download/v1.26.1%2Brke2r1/rke2-images.linux-amd64.tar.zst
$ curl -OLs https://github.com/rancher/rke2/releases/download/v1.26.1%2Brke2r1/rke2.linux-amd64.tar.gz
$ curl -OLs https://github.com/rancher/rke2/releases/download/v1.26.1%2Brke2r1/sha256sum-amd64.txt
$ curl -sfL https://get.rke2.io --output install.sh
$ chmod +x install.sh
# 要把以下這些檔案都放在 /root/rke2-artifacts
$ ll
total 896640
-rwxr-xr-x 1 root root 22292 Feb 16 10:03 install.sh
-rw-r--r-- 1 root root 891547439 Feb 16 10:03 rke2-images.linux-amd64.tar.zst
-rw-r--r-- 1 root root 26582042 Feb 16 10:03 rke2.linux-amd64.tar.gz
-rw-r--r-- 1 root root 3626 Feb 16 10:03 sha256sum-amd64.txt
```
### 2. 解壓縮與基本設定
```shell=!
rms1:~/rke2-artifacts # INSTALL_RKE2_ARTIFACT_PATH=/root/rke2-artifacts sh install.sh
[WARN] /usr/local is read-only or a mount point; installing to /opt/rke2
[INFO] staging local checksums from /root/rke2-artifacts/sha256sum-amd64.txt
[INFO] staging zst airgap image tarball from /root/rke2-artifacts/rke2-images.linux-amd64.tar.zst
[INFO] staging tarball from /root/rke2-artifacts/rke2.linux-amd64.tar.gz
[INFO] verifying airgap tarball
grep: /tmp/rke2-install.XRvs61dJ7e/rke2-images.checksums: No such file or directory
[INFO] installing airgap tarball to /var/lib/rancher/rke2/agent/images
[INFO] verifying tarball
[INFO] unpacking tarball file to /opt/rke2
[INFO] updating tarball contents to reflect install path
[INFO] moving systemd units to /etc/systemd/system
[INFO] install complete; you may want to run: export PATH=$PATH:/opt/rke2/bin
rms1:~/rke2-artifacts # export PATH=$PATH:/opt/rke2/bin
```
### 3. 叢集基礎組態
```shell=
rms1:~/rke2-artifacts # mkdir -p /etc/rancher/rke2/
[sudo] root 的密碼：
rms1:~/rke2-artifacts # vim /etc/rancher/rke2/config.yaml
rms1:~/rke2-artifacts # cat /etc/rancher/rke2/config.yaml
node-name:
- "rms1"
token: my-shared-secret
node-taint:
- "CriticalAddonsOnly=true:NoExecute"
```
* 設定從我們自建的 harbor 拉 image
```!
# 每一台 node 都要設定
# 並且都要可以解析的到這個位置
$ vim /etc/rancher/rke2/registries.yaml
mirrors:
docker.io:
endpoint:
- "https://harbor.example.com"
configs:
"harbor.example.com":
auth:
username: admin
password: Harbor12345
tls:
insecure_skip_verify: true
```
### 4. 啟用RKE2服務
```shell=!
$ systemctl enable --now rke2-server
Created symlink /etc/systemd/system/multi-user.target.wants/rke2-server.service → /etc/systemd/system/rke2-server.service.
```
### 5. 設定一般帳號使用kubectl
```shell=!
rms1:~/rke2-artifacts # exit
exit
$ mkdir .kube
$ sudo cp /etc/rancher/rke2/rke2.yaml .kube/config
[sudo] password for root:
$ sudo chown rancher .kube/config
$ sudo cp /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/
$ kubectl get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system cloud-controller-manager-rms1 1/1 Running 0 3m5s
kube-system etcd-rms1 1/1 Running 0 2m46s
kube-system helm-install-rke2-canal-lpv22 0/1 Completed 0 3m
kube-system helm-install-rke2-coredns-xndpj 0/1 Completed 0 3m
kube-system helm-install-rke2-ingress-nginx-5r5sq 0/1 Completed 0 3m
kube-system helm-install-rke2-metrics-server-wkz6c 0/1 Completed 0 3m
kube-system kube-apiserver-rms1 1/1 Running 0 2m37s
kube-system kube-controller-manager-rms1 1/1 Running 0 2m30s
kube-system kube-proxy-rms1 1/1 Running 0 2m57s
kube-system kube-scheduler-rms1 1/1 Running 0 2m37s
kube-system rke2-canal-clqp4 2/2 Running 0 2m41s
kube-system rke2-coredns-rke2-coredns-76cb76d66-xpnqg 1/1 Running 0 2m42s
kube-system rke2-coredns-rke2-coredns-autoscaler-58867f8fc5-hzz2l 1/1 Running 0 2m42s
kube-system rke2-ingress-nginx-controller-zfbvx 1/1 Running 0 93s
kube-system rke2-metrics-server-6979d95f95-kmbcv 1/1 Running 0 109s
```
### 6. install worker
```shell=!
$ sudo su
$ mkdir -p /etc/rancher/rke2/
$ vim /etc/rancher/rke2/config.yaml
$ cat /etc/rancher/rke2/config.yaml
server: https://192.168.11.116:9345
node-name:
- "w1"
token: my-shared-secret
$ vim /etc/rancher/rke2/registries.yaml
mirrors:
docker.io:
endpoint:
- "https://harbor.example.com"
configs:
"harbor.example.com":
auth:
username: admin
password: Harbor12345
tls:
insecure_skip_verify: true
```
```shell=!
$ INSTALL_RKE2_ARTIFACT_PATH=/root/rke2-artifacts INSTALL_RKE2_TYPE="agent" sh install.sh
[WARN] /usr/local is read-only or a mount point; installing to /opt/rke2
[INFO] staging local checksums from /root/rke2-artifacts/sha256sum-amd64.txt
[INFO] staging zst airgap image tarball from /root/rke2-artifacts/rke2-images.linux-amd64.tar.zst
[INFO] staging tarball from /root/rke2-artifacts/rke2.linux-amd64.tar.gz
[INFO] verifying airgap tarball
grep: /tmp/rke2-install.r4ItaeDDRu/rke2-images.checksums: No such file or directory
[INFO] installing airgap tarball to /var/lib/rancher/rke2/agent/images
[INFO] verifying tarball
[INFO] unpacking tarball file to /opt/rke2
[INFO] updating tarball contents to reflect install path
[INFO] moving systemd units to /etc/systemd/system
[INFO] install complete; you may want to run: export PATH=$PATH:/opt/rke2/bin
$ export PATH=$PATH:/opt/rke2/bin
```
```!
$ systemctl enable rke2-agent.service
Created symlink /etc/systemd/system/multi-user.target.wants/rke2-agent.service → /etc/systemd/system/rke2-agent.service.
$ systemctl start rke2-agent.service
```
```
# 在 master 貼上 worker 的 label
rancher@m1:~> kubectl label node w1 node-role.kubernetes.io/worker=
```
```
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
m1 Ready control-plane,etcd,master 159m v1.26.1+rke2r1
w1 Ready worker 120m v1.26.1+rke2r1
```
Sign in with Wallet
Connect another wallet