# Install K3s in Containers without k3d
## 1. 環境準備
### 1.1. 準備 Podman Network
```bash!
$ sudo podman network create --subnet 172.22.1.0/24 --gateway 172.22.1.254 k3s-network
$ sudo podman network ls
NETWORK ID NAME DRIVER
78b8668c5acd k3s-network bridge
2f259bab93aa podman bridge
```
### 1.2. 設定必要核心參數
```
$ cat <<EOF | sudo tee -a /etc/sysctl.conf
net.bridge.bridge-nf-call-iptables = 1
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=524288
net.netfilter.nf_conntrack_max = 131072
kernel.dmesg_restrict = 0
EOF
$ sudo reboot
```
### 1.3. 建立工作目錄
```bash!
$ mkdir -p "$HOME"/k3s/config/c31 &&
cd "$HOME"/k3s/config/c31
```
### 1.4. 產生設定 containerd 的基礎模板
```bash!
$ cat <<EOF > "$HOME"/k3s/config/c31/config.toml.tmpl
{{ template "base" . }}
# RuntimeClass Handler
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
runtime_type = "io.containerd.runsc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc.options]
BinaryName = "/usr/bin/runsc"
EOF
```
### 1.5. 產生設定私有 Container images Registry 設定檔
```bash!
$ cat <<EOF > "$HOME"/k3s/config/c31/registries.yaml
mirrors:
dkreg.kube-system:5000:
endpoint:
- "http://dkreg.kube-system:5000"
configs:
"dkreg.kube-system:5000":
auth:
username: bigred
password: bigred
EOF
```
### 1.6. 產生 runsc runtimeclass 設定檔
```bash!
$ cat <<EOF > "$HOME"/k3s/config/c31/runtimeclass.yaml
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
name: gvisor
handler: runsc
EOF
```
### 1.7. 設計客製化 K3s image
```
$ nano "$HOME"/k3s/config/c31/Containerfile
```
內容如下 :
```dockerfile=
# 由於官方提供的 K3s Container images 中缺少套件管理命令，
# 因此我們需要重新製作 Container image，並添加 apk 套件管理工具。
# 同時，我們也將設定好 Containerd，以確保能夠下載來自 dkreg 的 Container images。
ARG K3S_TAG="v1.31.1-k3s1"
ARG ALPINE_TAG="3.20"
FROM docker.io/rancher/k3s:$K3S_TAG as k3s
FROM docker.io/library/alpine:$ALPINE_TAG as collect
# 安裝 OCI runtime spec (包含 crun 和 runsc)
RUN apk -U add crun bash nftables mount && \
ARCH=$(uname -m); URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH} && \
wget ${URL}/runsc ${URL}/runsc.sha512 ${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512 && \
rm -f *.sha512 && \
chmod a+rx runsc containerd-shim-runsc-v1; mv runsc containerd-shim-runsc-v1 /bin
FROM scratch
COPY --from=k3s / /
RUN mkdir /usr/bin && \
mkdir /usr/lib
# Setup musl libc
## musl is an implementation of the C standard library built on top of the Linux system call API
COPY --from=collect /lib/ld-musl-x86_64.so.1 /lib
# Setup Bash Shell
COPY --from=collect /bin/bash /bin
COPY --from=collect /usr/lib/libreadline.so.8 /usr/lib
COPY --from=collect /usr/lib/libncursesw.so.6 /usr/lib
# Setup runsc
COPY --from=collect /bin/runsc /bin
COPY --from=collect /bin/containerd-shim-runsc-v1 /bin
# Setup crun
COPY --from=collect /usr/bin/crun /usr/bin
COPY --from=collect /usr/lib/libseccomp.so.2 /usr/lib
COPY --from=collect /usr/lib/libcap.so.2 /usr/lib
COPY --from=collect /usr/lib/libyajl.so.2 /usr/lib
# Setup nft
COPY --from=collect /usr/sbin/nft /usr/bin
COPY --from=collect /usr/lib/libnftables.so.1 /usr/lib
COPY --from=collect /usr/lib/libreadline.so.8 /usr/lib
COPY --from=collect /usr/lib/libmnl.so.0 /usr/lib
COPY --from=collect /usr/lib/libnftnl.so.11 /usr/lib
COPY --from=collect /usr/lib/libjansson.so.4 /usr/lib
COPY --from=collect /usr/lib/libgmp.so.10 /usr/lib
# Setup mount
COPY --from=collect /bin/mount /bin
COPY --from=collect /lib/libmount.so.1 /lib
COPY --from=collect /lib/libblkid.so.1 /lib
COPY --from=collect /usr/lib/libeconf.so.0 /usr/lib
# 設定 containerd 的基礎模板
## 將 runSC OCI runtimes 的相關設定新增進 containerd
COPY config.toml.tmpl /var/lib/rancher/k3s/agent/etc/containerd/
## 設定私有 Container images Registry，確保可以拉取來自 dkreg 的 Container images
## 參考文章 : https://docs.k3s.io/installation/private-registry#configs
COPY registries.yaml /etc/rancher/k3s/
# 設定 runsc RuntimeClass
COPY runtimeclass.yaml /var/lib/rancher/k3s/server/manifests/
VOLUME /var/lib/kubelet
VOLUME /var/lib/rancher/k3s
VOLUME /var/lib/cni
VOLUME /var/log
ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"
ENV CRI_CONFIG_FILE="/var/lib/rancher/k3s/agent/etc/crictl.yaml"
ENTRYPOINT ["/bin/k3s"]
CMD ["agent"]
```
### 1.8. 建立客製化 K3s image
```bash!
$ K3S_TAG=v1.31.1-k3s1
$ sudo podman build --squash-all --no-cache \
--build-arg K3S_TAG="$K3S_TAG" \
-t k3s-add-runtime:"$K3S_TAG" \
"$HOME"/k3s/config/c${K3S_TAG:3:2}/
```
## 2. 安裝 K3s
### 2.1. 建立 server node
```bash!
# 2.1. 建立 podman volume
$ sudo podman volume create --label server=cni && \
sudo podman volume create --label server=kubelet && \
sudo podman volume create --label server=k3s && \
sudo podman volume create --label server=log
$ server_cni=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=cni" ) && \
server_kubelet=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=kubelet") && \
server_k3s=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=k3s") && \
server_log=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=log")
# 2.2. 建立 server node container
$ sudo podman run -d \
--name k3s-c31-server-0 \
--privileged \
-e CRI_CONFIG_FILE=/var/lib/rancher/k3s/agent/etc/crictl.yaml \
-e HOSTNAME=k3d-c31-server-0 \
-e K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml \
-e K3S_TOKEN=c31Token \
-e container=podman \
--network k3s-network \
--ip=172.22.1.11 \
--hostname k3s-c31-server-0 \
--ulimit nproc=131072:131072 \
--ulimit nofile=26677:26677 \
--volume /home/bigred/k3d/wulin:/opt/kadm:rw,rprivate,rbind \
--volume ${server_cni}:/var/lib/cni:rprivate,rw,nodev,exec,nosuid,rbind \
--volume ${server_kubelet}:/var/lib/kubelet:rprivate,rw,nodev,exec,nosuid,rbind \
--volume ${server_k3s}:/var/lib/rancher/k3s:rprivate,rw,nodev,exec,nosuid,rbind \
--volume ${server_log}:/var/log:rprivate,rw,nodev,exec,nosuid,rbind \
-p 80:80/tcp \
-p 443:443/tcp \
-p 5000:5000/tcp \
-p 5900-5905:5900-5905/tcp \
-p 6080:6080/tcp \
-p 8080:8080/tcp \
-p 9000:9000/tcp \
-p 22001-22002:22001-22002/tcp \
-p 22100-22102:22100-22102/tcp \
-p 51820:51820/tcp \
-p 6443:30001/tcp \
localhost/k3s-add-runtime:v1.31.1-k3s1 \
server --cluster-domain=k1.org \
--bind-address=172.22.1.11 \
--cluster-cidr=10.244.0.0/16 \
--service-cidr=10.98.0.0/16 \
--kubelet-arg=container-log-max-files=3 \
--kubelet-arg=container-log-max-size=10Mi \
--egress-selector-mode=disabled \
--disable=traefik \
--disable=servicelb \
--node-label app=taroko \
--tls-san k1.org \
--tls-san 172.22.1.11
```
### 2.2. 檢查 K3s server node 狀態
```bash!
$ sudo podman exec k3s-c31-server-0 kubectl get nodes
```
執行結果 :
```bash!
NAME STATUS ROLES AGE VERSION
k3s-c31-server-0 Ready control-plane,master 7s v1.31.1+k3s1
```
### 2.3. 檢查 K3s 所有 pods 狀態
```bash!
$ sudo podman exec k3s-c31-server-0 kubectl get pods -A
```
執行結果 :
```bash
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-56f6fc8fd7-t9hlb 1/1 Running 0 99s
kube-system local-path-provisioner-846b9dcb6c-xddbs 1/1 Running 0 99s
kube-system metrics-server-5985cbc9d7-889p8 1/1 Running 0 99s
```
### 2.4. 建立第一台 agent node
```bash!
# 3.1. 建立 podman volume
$ sudo podman volume create --label agent0=cni && \
sudo podman volume create --label agent0=kubelet && \
sudo podman volume create --label agent0=k3s && \
sudo podman volume create --label agent0=log
$ agent_0_cni=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=cni") && \
agent_0_kubelet=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=kubelet") && \
agent_0_k3s=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=k3s") && \
agent_0_log=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=log") && \
server_ip=$(sudo podman exec k3s-c31-server-0 ip a s eth0 | grep "inet " | awk '{print $2}')
# 3.2. 建立 agent node container
$ sudo podman run -d\
--name k3s-c31-agent-0 \
--hostname k3s-c31-agent-0 \
--privileged \
--env K3S_TOKEN=c31Token \
--env K3S_URL=https://${server_ip%/*}:6443 \
--env K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml \
--ulimit nproc=131072:131072 \
--ulimit nofile=26677:26677 \
--network k3s-network \
-p 22011:22011 \
-p 33891:33891 \
-v ${agent_0_cni}:/var/lib/cni:rw,rprivate,nodev,exec,nosuid,rbind \
-v ${agent_0_kubelet}:/var/lib/kubelet:rw,rprivate,nodev,exec,nosuid,rbind \
-v ${agent_0_k3s}:/var/lib/rancher/k3s:rw,rprivate,nodev,exec,nosuid,rbind \
-v ${agent_0_log}:/var/log:rw,rprivate,nodev,exec,nosuid,rbind \
localhost/k3s-add-runtime:v1.31.1-k3s1 \
agent
```
### 2.5. 檢查 K3s agent node 狀態
```bash!
$ sudo podman exec k3s-c31-server-0 kubectl get nodes
```
執行結果 :
```bash!
NAME STATUS ROLES AGE VERSION
k3s-c31-agent-0 Ready <none> 3s v1.31.1+k3s1
k3s-c31-server-0 Ready control-plane,master 2m51s v1.31.1+k3s1
```
### 2.6. 檢查 K3s 所有 pods 狀態
```bash!
$ sudo podman exec k3s-c31-server-0 kubectl get pods -A
```
執行結果 :
```bash
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-56f6fc8fd7-t9hlb 1/1 Running 0 3m39s
kube-system local-path-provisioner-846b9dcb6c-xddbs 1/1 Running 0 3m39s
kube-system metrics-server-5985cbc9d7-889p8 1/1 Running 0 3m39s
```
### 2.7. 建立第二台 agent node
```bash!
# 3.1. 建立 podman volume
$ sudo podman volume create --label agent1=cni && \
sudo podman volume create --label agent1=kubelet && \
sudo podman volume create --label agent1=k3s && \
sudo podman volume create --label agent1=log
$ agent_2_cni=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=cni") && \
agent_2_kubelet=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=kubelet") && \
agent_2_k3s=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=k3s") && \
agent_2_log=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=log") && \
server_ip=$(sudo podman exec k3s-c31-server-0 ip a s eth0 | grep "inet " | awk '{print $2}')
# 3.2. 建立 server node container
$ sudo podman run -d\
--name k3s-c31-agent-1 \
--hostname k3s-c31-agent-1 \
--privileged \
--env K3S_TOKEN=c31Token \
--env K3S_URL=https://${server_ip%/*}:6443 \
--env K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml \
--ulimit nproc=131072:131072 \
--ulimit nofile=26677:26677 \
--network k3s-network \
-p 22012:22012 \
-p 33892:33892 \
-v ${agent_2_cni}:/var/lib/cni:rw,rprivate,nodev,exec,nosuid,rbind \
-v ${agent_2_kubelet}:/var/lib/kubelet:rw,rprivate,nodev,exec,nosuid,rbind \
-v ${agent_2_k3s}:/var/lib/rancher/k3s:rw,rprivate,nodev,exec,nosuid,rbind \
-v ${agent_2_log}:/var/log:rw,rprivate,nodev,exec,nosuid,rbind \
localhost/k3s-add-runtime:v1.31.1-k3s1 \
agent
```
### 2.8. 檢查 K3s agent node 狀態
```bash!
$ sudo podman exec k3s-c31-server-0 kubectl get nodes
```
執行結果 :
```bash!
NAME STATUS ROLES AGE VERSION
k3s-c31-agent-0 Ready <none> 65s v1.31.1+k3s1
k3s-c31-agent-1 Ready <none> 18s v1.31.1+k3s1
k3s-c31-server-0 Ready control-plane,master 3m53s v1.31.1+k3s1
```
### 2.9. 檢查 K3s 所有 pods 狀態
```bash!
$ sudo podman exec k3s-c31-server-0 kubectl get pods -A
```
執行結果 :
```bash
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-56f6fc8fd7-t9hlb 1/1 Running 0 4m3s
kube-system local-path-provisioner-846b9dcb6c-xddbs 1/1 Running 0 4m3s
kube-system metrics-server-5985cbc9d7-889p8 1/1 Running 0 4m3s
```
### 2.10. 設定 KubeConfig
```bash!
$ mkdir -p "$HOME"/.kube && \
sudo podman cp k3s-c31-server-0:/output/kubeconfig.yaml "$HOME"/.kube/config && \
sudo chown -R $(id -u):$(id -g) "$HOME"/.kube
```
### 2.11. 確認可直接執行 kubectl
```bash!
$ kubectl get nodes -o wide
```
> 安裝 `kubectl` 命令
> ```!
> $ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
> chmod +x kubectl; sudo mv kubectl /usr/local/bin/
> ```
執行結果 :
```
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE
KERNEL-VERSION CONTAINER-RUNTIME
k3s-c31-agent-0 Ready <none> 2m48s v1.31.1+k3s1 172.22.1.4 <none> K3s v1.31.1+k3s1 6.6.50-0-lts containerd://1.7.21-k3s2
k3s-c31-agent-1 Ready <none> 2m30s v1.31.1+k3s1 172.22.1.5 <none> K3s v1.31.1+k3s1 6.6.50-0-lts containerd://1.7.21-k3s2
k3s-c31-server-0 Ready control-plane,master 4m24s v1.31.1+k3s1 172.22.1.11 <none> K3s v1.31.1+k3s1 6.6.50-0-lts containerd://1.7.21-k3s2
```
## 3. 完全清除環境
```bash!
$ sudo podman rm -f k3s-c31-server-0 k3s-c31-agent-0 k3s-c31-agent-1 \
k3s-working-container alpine-working-container && \
volume=$(for i in server agent0 agent1; do \
sudo podman volume ls --format "{{.Name}}" --filter label=${i}; \
done) && \
sudo podman volume rm $volume && \
sudo podman rmi localhost/k3s-add-runtime:v1.31.1-k3s1 \
docker.io/rancher/k3s:v1.31.1-k3s1 && \
rm -rf "$HOME"/.kube "$HOME"/k3s
```
Sign in with Wallet
Connect another wallet