# kubectl debug
## debug node
* kubectl debug node 會在你指定的節點產生一個 pod ，並且會掛載 hostpath 在跟目錄上，因此可以使用 chroot 進到這個節點
* 但是進入後會有所有的 capabilities 可以直接對本機關機，因此要慎用
* 可以使用 neuvector 去限制，使用這條規則 `PSP Best Practice.`

$ kubectl debug node/rms -it --image=docker.io/taiwanese/debug.alp

/ # ls -l
total 0
drwxr-xr-x    1 root     root           838 Apr 14  2021 bin
drwxr-xr-x    5 root     root           380 Jul 13 05:20 dev
drwxr-xr-x    1 root     root            22 Jul 13 05:20 etc
drwxr-xr-x    1 root     root             0 Apr 14  2021 home
drwxr-xr-x    1 root     root           156 Jun 30 02:12 host
drwxr-xr-x    1 root     root           290 Apr 14  2021 lib
drwxr-xr-x    1 root     root            28 Apr 14  2021 media
drwxr-xr-x    1 root     root             0 Apr 14  2021 mnt
drwxr-xr-x    1 root     root             0 Apr 14  2021 opt
dr-xr-xr-x  295 root     root             0 Jul 13 05:20 proc
drwx------    1 root     root            24 Jul 13 05:21 root
drwxr-xr-x    1 root     root            14 Jul 13 05:20 run
drwxr-xr-x    1 root     root           800 Apr 14  2021 sbin
drwxr-xr-x    1 root     root             0 Apr 14  2021 srv
dr-xr-xr-x   13 root     root             0 Jun 30 02:31 sys
drwxrwxrwt    1 root     root             0 Apr 14  2021 tmp
drwxr-xr-x    1 root     root            40 Apr 14  2021 usr
drwxr-xr-x    1 root     root            86 Apr 14  2021 var

/ # chroot /host

rms:/ # ls -l
total 0
drwxr-x---   1 root root   46 Jun 30 11:27 .snapshots
drwxr-xr-x   1 root root 1778 Jun 30 10:19 bin
drwxr-xr-x   1 root root  696 Jun 30 10:24 boot
drwxr-xr-x  19 root root 3900 Jun 30 10:32 dev
drwxr-xr-x   1 root root 4340 Jun 30 11:27 etc
drwxr-xr-x   1 root root   14 Jun 30 10:22 home
drwxr-xr-x   1 root root  100 Jun 30 10:19 lib
drwxr-xr-x   1 root root 2728 Jun 30 10:19 lib64
drwxr-xr-x   1 root root    0 Mar 15  2022 mnt
drwxr-xr-x   1 root root   14 Jun 30 10:43 opt
dr-xr-xr-x 295 root root    0 Jun 30 10:31 proc
drwx------   1 root root  182 Jul  7 13:51 root
drwxr-xr-x  35 root root  920 Jun 30 11:27 run
drwxr-xr-x   1 root root 3440 Jun 30 10:19 sbin
drwxr-xr-x   1 root root    0 Mar 15  2022 selinux
drwxr-xr-x   1 root root    6 Jun 30 10:12 srv
dr-xr-xr-x  13 root root    0 Jun 30 10:31 sys
drwxrwxrwt   1 root root  310 Jul 13 13:21 tmp
drwxr-xr-x   1 root root  104 Jun 30 10:12 usr
drwxr-xr-x   1 root root  110 Jun 30 10:23 var

* 檢查 yaml 看到是掛 hostpath 到根目錄

$ kubectl get pod node-debugger-rms-65tf2 -o yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: "2023-07-13T05:20:43Z"
  name: node-debugger-rms-65tf2
  namespace: default
  resourceVersion: "8904684"
  uid: ebb676d1-9149-4afa-9f11-9612b74437ce
spec:
  containers:
  - image: quay.io/cooloo9871/alpine
    imagePullPolicy: Always
    name: debugger
    resources: {}
    stdin: true
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    tty: true
    volumeMounts:
    - mountPath: /host
      name: host-root
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-wvb5c
      readOnly: true
......
  volumes:
  - hostPath:
      path: /
      type: ""
    name: host-root

## debug pod

* 針對 pod 有問題，並且是 Distroless images 的話無法透過 shell 進入到 pod 去排除問題。
* 透過 debug 會在要 debug 的 pod 長一個 container ，透過 share process 的方式來查看原本的 container 的問題。
* `profile=sysadmin` 讓這個 debug container 有最大權限

$ kubectl get po
NAME                        READY   STATUS    RESTARTS      AGE
mt-0                        1/1     Running   1 (13d ago)   13d
mt-1                        1/1     Running   1 (13d ago)   13d
nvtest123-87bf6679d-g6v2t   1/1     Running   0             6d18h
webpolicy-5c486fdc8-876h9   1/1     Running   0             7d22h

# --target 代表目標 container 是誰，因此需要知道 container 的名字
$ kubectl debug webpolicy-5c486fdc8-876h9 -it --image=docker.io/taiwanese/debug.alp --share-processes --target=nginx --profile=sysadmin

/ # ps aux
PID   USER     TIME  COMMAND
    1 root      0:00 nginx: master process nginx -g daemon off;
   29 101       0:00 nginx: worker process
   30 101       0:00 nginx: worker process
   31 101       0:00 nginx: worker process
   32 101       0:00 nginx: worker process
   33 root      0:00 /bin/bash
   42 root      0:00 /bin/sh
   48 root      0:00 ps aux
 
# 查看目標 container 的檔案目錄區結構
/ # ls -l /proc/1/root/
total 28
lrwxrwxrwx    1 root     root             7 Nov 20 00:00 bin -> usr/bin
drwxr-xr-x    1 root     root             0 Sep 29 20:04 boot
drwxr-xr-x    5 root     root           360 Nov 29 02:58 dev
drwxr-xr-x    1 root     root            54 Nov 21 09:05 docker-entrypoint.d
-rwxrwxr-x    1 root     root          1620 Nov 21 09:05 docker-entrypoint.sh
drwxr-xr-x    1 root     root            20 Nov 29 02:58 etc
drwxr-xr-x    1 root     root             0 Sep 29 20:04 home
lrwxrwxrwx    1 root     root             7 Nov 20 00:00 lib -> usr/lib
lrwxrwxrwx    1 root     root             9 Nov 20 00:00 lib32 -> usr/lib32
lrwxrwxrwx    1 root     root             9 Nov 20 00:00 lib64 -> usr/lib64
lrwxrwxrwx    1 root     root            10 Nov 20 00:00 libx32 -> usr/libx32
drwxr-xr-x    1 root     root             0 Nov 20 00:00 media
drwxr-xr-x    1 root     root             0 Nov 20 00:00 mnt
drwxr-xr-x    1 root     root             0 Nov 20 00:00 opt
dr-xr-xr-x  265 root     root             0 Nov 29 02:58 proc
drwx------    1 root     root            30 Nov 20 00:00 root
drwxr-xr-x    1 root     root            32 Nov 29 02:58 run
lrwxrwxrwx    1 root     root             8 Nov 20 00:00 sbin -> usr/sbin
drwxr-xr-x    1 root     root             0 Nov 20 00:00 srv
dr-xr-xr-x   13 root     root             0 Nov 29 02:58 sys
drwxrwxrwt    1 root     root             0 Nov 21 09:05 tmp
drwxr-xr-x    1 root     root            10 Nov 20 00:00 usr
drwxr-xr-x    1 root     root            10 Nov 20 00:00 var
   
# 查看一些特殊資訊
/ # cat /proc/1/root/etc/os-release
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
NAME="Debian GNU/Linux"
VERSION_ID="12"
VERSION="12 (bookworm)"
VERSION_CODENAME=bookworm
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"


# 不過離開後查看後並不會看到有兩個 container
$ kubectl get pod
NAME                        READY   STATUS    RESTARTS      AGE
mt-0                        1/1     Running   1 (13d ago)   13d
mt-1                        1/1     Running   1 (13d ago)   13d
nvtest123-87bf6679d-g6v2t   1/1     Running   0             6d18h
webpolicy-5c486fdc8-876h9   1/1     Running   0             7d23h

# 但是使用 describe 可以看得到產生 container 的紀錄
$ kubectl describe pod webpolicy-5c486fdc8-876h9
.......
Events:
  Type    Reason   Age    From     Message
  ----    ------   ----   ----     -------
  Normal  Pulling  2m50s  kubelet  Pulling image "docker.io/taiwanese/debug.alp"
  Normal  Pulled   2m41s  kubelet  Successfully pulled image "docker.io/taiwanese/debug.alp" in 9.006165257s (9.006199762s including waiting)
  Normal  Created  2m40s  kubelet  Created container debugger-sgjj8
  Normal  Started  2m40s  kubelet  Started container debugger-sgjj8

* 此方式會複製原本的 pod ，並且再長出 container 來查看問題。
* 因此不會直接修改到原本的 pod。

# --copy-to 代表複製出的 pod 要取甚麼名字
$ kubectl debug myweb-5bc6d9974f-6scrp -it --image=docker.io/taiwanese/debug.alp --share-processes --copy-to=fixpod

/ # ps aux
PID   USER     TIME  COMMAND
    1 65535     0:00 /pause
    7 root      0:00 busybox httpd -p 80 -h /opt/www -f
   13 root      0:00 /bin/sh
   20 root      0:00 ps aux
   
# 可以看到複製出的 fixpod 這個 pod 裡面有兩個 container
$ kubectl get pod
NAME                           READY   STATUS    RESTARTS        AGE
fixpod                         2/2     Running   1 (6m13s ago)   9m13s
harbor-test-6f84d4d5fd-f6whs   1/1     Running   0               10d
mybird-fd874d9c9-7q2jf         1/1     Running   0               9d
myweb-5bc6d9974f-6scrp         1/1     Running   0               10d