# Upstream Cluster(RKE2) Rotate Certificate

## 下載 rke2 檢查憑證腳本

$ git clone https://github.com/cooloo9871/k8s-cert_check.git; cd k8s-cert_check

$ kubectl get no
NAME   STATUS   ROLES                       AGE   VERSION
rms    Ready    control-plane,etcd,master   53d   v1.27.10+rke2r1
rms2   Ready    control-plane,etcd,master   53d   v1.27.10+rke2r1
rms3   Ready    control-plane,etcd,master   53d   v1.27.10+rke2r1

* 檢查 rke2 rms master 憑證時效

$ sudo ./cert_check.sh
[sudo] password for root:
client-admin.crt:              Feb 20 07:38:00 2025 GMT
client-auth-proxy.crt:         Feb 20 07:38:00 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
client-controller.crt:         Feb 20 07:38:00 2025 GMT
client-kube-apiserver.crt:     Feb 20 07:38:00 2025 GMT
client-kube-proxy.crt:         Feb 20 07:38:00 2025 GMT
client-rke2-cloud-controller.crt: Feb 20 07:38:00 2025 GMT
client-rke2-controller.crt:    Feb 20 07:38:00 2025 GMT
client-scheduler.crt:          Feb 20 07:38:00 2025 GMT
client-supervisor.crt:         Feb 20 07:38:00 2025 GMT
request-header-ca.crt:         Feb 18 07:38:00 2034 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
server-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
serving-kube-apiserver.crt:    Feb 20 07:38:00 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-kubelet.crt:            Apr  8 03:21:02 2025 GMT
client-kube-proxy.crt:         Feb 20 07:38:00 2025 GMT
client-rke2-controller.crt:    Feb 20 07:38:00 2025 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
serving-kubelet.crt:           Apr  8 03:21:02 2025 GMT

* 檢查 rke2 rms2 master 憑證時效

$ sudo ./cert_check.sh
client-admin.crt:              Feb 20 07:42:48 2025 GMT
client-auth-proxy.crt:         Feb 20 07:42:48 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
client-controller.crt:         Feb 20 07:42:48 2025 GMT
client-kube-apiserver.crt:     Feb 20 07:42:48 2025 GMT
client-kube-proxy.crt:         Feb 20 07:42:48 2025 GMT
client-rke2-cloud-controller.crt: Feb 20 07:42:48 2025 GMT
client-rke2-controller.crt:    Feb 20 07:42:48 2025 GMT
client-scheduler.crt:          Feb 20 07:42:48 2025 GMT
client-supervisor.crt:         Feb 20 07:42:48 2025 GMT
request-header-ca.crt:         Feb 18 07:38:00 2034 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
server-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
serving-kube-apiserver.crt:    Feb 20 07:42:48 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-kubelet.crt:            Apr  8 03:26:32 2025 GMT
client-kube-proxy.crt:         Feb 20 07:42:48 2025 GMT
client-rke2-controller.crt:    Feb 20 07:42:48 2025 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
serving-kubelet.crt:           Apr  8 03:26:32 2025 GMT

* 檢查 rke2 rms3 master 憑證時效

$ sudo ./cert_check.sh
client-admin.crt:              Feb 20 07:43:11 2025 GMT
client-auth-proxy.crt:         Feb 20 07:43:11 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
client-controller.crt:         Feb 20 07:43:11 2025 GMT
client-kube-apiserver.crt:     Feb 20 07:43:11 2025 GMT
client-kube-proxy.crt:         Feb 20 07:43:11 2025 GMT
client-rke2-cloud-controller.crt: Feb 20 07:43:11 2025 GMT
client-rke2-controller.crt:    Feb 20 07:43:11 2025 GMT
client-scheduler.crt:          Feb 20 07:43:11 2025 GMT
client-supervisor.crt:         Feb 20 07:43:11 2025 GMT
request-header-ca.crt:         Feb 18 07:38:00 2034 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
server-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
serving-kube-apiserver.crt:    Feb 20 07:43:11 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-kubelet.crt:            Apr  8 03:28:36 2025 GMT
client-kube-proxy.crt:         Feb 20 07:43:11 2025 GMT
client-rke2-controller.crt:    Feb 20 07:43:11 2025 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
serving-kubelet.crt:           Apr  8 03:28:36 2025 GMT

## 更換憑證
* 更換 rms master 憑證

# Change user to root
$ sudo su

# Stop RKE2
$ systemctl stop rke2-server.service

# Rotate certificates
$ rke2 certificate rotate

# Start RKE2
$ systemctl start rke2-server.service

* 更換 rms2 master 憑證

# Change user to root
$ sudo su

# Stop RKE2
$ systemctl stop rke2-server.service

# Rotate certificates
$ rke2 certificate rotate

# Start RKE2
$ systemctl start rke2-server.service

* 更換 rms3 master 憑證

# Change user to root
$ sudo su

# Stop RKE2
$ systemctl stop rke2-server.service

# Rotate certificates
$ rke2 certificate rotate

# Start RKE2
$ systemctl start rke2-server.service

## 確認憑證是否更新完成

* 檢查 rms master 憑證時效

$ sudo ./cert_check.sh
client-admin.crt:              Apr 15 01:55:20 2025 GMT
client-auth-proxy.crt:         Apr 15 01:55:20 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
client-controller.crt:         Apr 15 01:55:20 2025 GMT
client-kube-apiserver.crt:     Apr 15 01:55:20 2025 GMT
client-kube-proxy.crt:         Apr 15 01:55:20 2025 GMT
client-rke2-cloud-controller.crt: Apr 15 01:55:20 2025 GMT
client-rke2-controller.crt:    Apr 15 01:55:20 2025 GMT
client-scheduler.crt:          Apr 15 01:55:20 2025 GMT
client-supervisor.crt:         Feb 20 07:38:00 2025 GMT
request-header-ca.crt:         Feb 18 07:38:00 2034 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
server-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
serving-kube-apiserver.crt:    Apr 15 01:55:20 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-kube-proxy.crt:         Apr 15 01:55:20 2025 GMT
client-kubelet.crt:            Apr 15 01:55:22 2025 GMT
client-rke2-controller.crt:    Apr 15 01:55:20 2025 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
serving-kubelet.crt:           Apr 15 01:55:22 2025 GMT

* 檢查 rms2 master 憑證時效

$ sudo ./cert_check.sh
client-admin.crt:              Apr 15 01:58:37 2025 GMT
client-auth-proxy.crt:         Apr 15 01:58:37 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
client-controller.crt:         Apr 15 01:58:37 2025 GMT
client-kube-apiserver.crt:     Apr 15 01:58:37 2025 GMT
client-kube-proxy.crt:         Apr 15 01:58:37 2025 GMT
client-rke2-cloud-controller.crt: Apr 15 01:58:37 2025 GMT
client-rke2-controller.crt:    Apr 15 01:58:37 2025 GMT
client-scheduler.crt:          Apr 15 01:58:37 2025 GMT
client-supervisor.crt:         Feb 20 07:42:48 2025 GMT
request-header-ca.crt:         Feb 18 07:38:00 2034 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
server-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
serving-kube-apiserver.crt:    Apr 15 01:58:37 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-kube-proxy.crt:         Apr 15 01:58:37 2025 GMT
client-kubelet.crt:            Apr 15 01:58:42 2025 GMT
client-rke2-controller.crt:    Apr 15 01:58:37 2025 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
serving-kubelet.crt:           Apr 15 01:58:41 2025 GMT

* 檢查 rms3 master 憑證時效

$ sudo ./cert_check.sh
client-admin.crt:              Apr 15 02:00:37 2025 GMT
client-auth-proxy.crt:         Apr 15 02:00:37 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
client-controller.crt:         Apr 15 02:00:37 2025 GMT
client-kube-apiserver.crt:     Apr 15 02:00:37 2025 GMT
client-kube-proxy.crt:         Apr 15 02:00:37 2025 GMT
client-rke2-cloud-controller.crt: Apr 15 02:00:37 2025 GMT
client-rke2-controller.crt:    Apr 15 02:00:37 2025 GMT
client-scheduler.crt:          Apr 15 02:00:37 2025 GMT
client-supervisor.crt:         Feb 20 07:43:11 2025 GMT
request-header-ca.crt:         Feb 18 07:38:00 2034 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
server-ca.nochain.crt:         Feb 18 07:38:00 2034 GMT
serving-kube-apiserver.crt:    Apr 15 02:00:37 2025 GMT
client-ca.crt:                 Feb 18 07:38:00 2034 GMT
client-kube-proxy.crt:         Apr 15 02:00:37 2025 GMT
client-kubelet.crt:            Apr 15 02:00:41 2025 GMT
client-rke2-controller.crt:    Apr 15 02:00:37 2025 GMT
server-ca.crt:                 Feb 18 07:38:00 2034 GMT
serving-kubelet.crt:           Apr 15 02:00:41 2025 GMT

## 遇到 client-supervisor.crt 憑證沒有 rotate 問題

* 在 rke2-1.27.13 版本以下會遇到這個問題，具體可參考 issue https://github.com/rancher/rke2/issues/5652
* 對於 k8s 1.27，會在 rke2-1.27.13 版本上修復，https://github.com/rancher/rke2/issues/5792
* k8s 1.28 對應 v1.28.9+rke2r1 ，https://github.com/rancher/rke2/issues/5791
* k8s 1.29 對應 v1.29.4+rke2r1 ，https://github.com/rancher/rke2/issues/5652

* `client-supervisor.crt` 他是 rke2 內部溝通會需要用的憑證，像是 worker 的 kubelet 或是 rke2 agent，目的是用來連接 rke2 server 和 apiserver。
* `client-supervisor.crt` 這個憑證是給 lb-server-port 這個服務用的，lb-server-port 就是 Supervisor 用來讓 `worker、control-plane、etcd` 加入到叢集的 port。
- server port: 9345
- client port: 6444

### 目前解決辦法

# mkdir /root/backup-cert

# mv /var/lib/rancher/rke2/server/tls/client-supervisor.crt /root/backup-cert/client-supervisor.crt

# mv /var/lib/rancher/rke2/server/tls/client-supervisor.key /root/backup-cert/client-supervisor.key

# systemctl stop rke2-server.service

# rke2 certificate rotate

# systemctl start rke2-server.service

## 參考連結

https://docs.rke2.io/security/certificates

https://github.com/rancher/rke2/security/advisories/GHSA-p45j-vfv5-wprq