# CKS D3C1.1 Kata Container
###### tags: `CKS Day3`
  Kata Containers is a novel implementation of a lightweight virtual machine that seamlessly integrates within the container ecosystem. Kata Containers are as light and fast as containers and integrate with the container management layers, while also delivering the security advantages of VMs.
  Kata Containers is a merger of two existing open source projects: Intel Clear Containers and Hyper runV. The new project brings together the best of both technologies with a common vision of retooling virtualization to fit container-native applications, in order to deliver the speed of containers, and the security of VMs.
  Kata Containers benefits from each project’s strengths. Intel Clear Containers are focused on performance (<100ms boot time) and enhanced security, while Hyper runV prioritized being technology-agnostic by supporting many different CPU architectures and hypervisors. By merging the projects, Kata Containers delivers a superior end user experience in both performance and compatibility, unifies the developer communities, and accelerates feature development to tackle future use cases.
## Architecture
参考链接:
[Katacontainers.io](https://katacontainers.io/learn/)
[K8s & KataContainer 实践](https://segmentfault.com/a/1190000021845002)
## 前置条件
1. 硬件依赖环境
> 首先节点需要支持以下四种任意一种cpu虚拟化技术
> Intel VT-x technology
> ARM Hyp mode
> IBM Power Systems
> IBM Z manframes
2. 软件依赖环境
> 正常运行的 K8s 环境
> containerd
## Install Kata runtime (Ubuntu)
方法1:
```
$ ARCH=$(arch)
$ BRANCH="${BRANCH:-master}"
$ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/ /' > /etc/apt/sources.list.d/kata-containers.list"
$ curl -sL http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/Release.key | sudo apt-key add -
$ sudo -E apt-get update
$ sudo -E apt-get -y install kata-runtime kata-proxy kata-shim
```
方法2: 使用Kata Manager脚本安装
```
$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) install-packages"
```
方法1&2都是自动安装, 版本都为Kata 1.x版本, 目前Ubuntu不支持自动安装2.x版本.
## 安装确认
安装完成之后，执行命令 kata-runtime kata-check 检查系统是否支持运行 kata runtime
```
root@ubuk8s-vm04:~# kata-runtime kata-check
WARN[0000] Not running network checks as super user arch=amd64 name=kata-runtime pid=94154 source=runtime
WARN[0000] modprobe insert module failed arch=amd64 error="exit status 1" module=vhost_vsock name=kata-runtime output="modprobe: ERROR: could not insert 'vhost_vsock': Device or resource busy\n" pid=94154 source=runtime
ERRO[0000] kernel property not found arch=amd64 description="Host Support for Linux VM Sockets" name=vhost_vsock pid=94154 source=runtime type=module
System is capable of running Kata Containers
System can currently create Kata Containers
```
System is capable of running Kata Containers
System can currently create Kata Containers
有这两行显示表示Kata Runtime可用. 部分报错是由于在VMware虚拟机中占用了vHost, 暂且忽略.
## 设置Containerd
在Containerd的配置文件中(/etc/containerd/config.toml)加入
```
plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
```
注意加入的位置, 以下方代码块为例
```
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0
[grpc]
address = "/run/containerd/containerd.sock"
tcp_address = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
[ttrpc]
address = ""
uid = 0
gid = 0
[debug]
address = ""
uid = 0
gid = 0
level = ""
[metrics]
address = ""
grpc_histogram = false
[cgroup]
path = ""
[timeouts]
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
pause_threshold = 0.02
deletion_threshold = 0
mutation_threshold = 100
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
disable_tcp_service = true
stream_server_address = "127.0.0.1"
stream_server_port = "0"
stream_idle_timeout = "4h0m0s"
enable_selinux = false
selinux_category_range = 1024
sandbox_image = "k8s.gcr.io/pause:3.2"
stats_collect_period = 10
systemd_cgroup = false
enable_tls_streaming = false
max_container_log_line_size = 16384
disable_cgroup = false
disable_apparmor = false
restrict_oom_score_adj = false
max_concurrent_downloads = 3
disable_proc_mount = false
unset_seccomp_profile = ""
tolerate_missing_hugetlb_controller = true
disable_hugetlb_controller = true
ignore_image_defined_volumes = false
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "overlayfs"
default_runtime_name = "runc"
no_pivot = false
disable_snapshot_annotations = true
discard_unpacked_layers = false
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
runtime_type = ""
runtime_engine = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
runtime_type = ""
runtime_engine = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
runtime_type = "io.containerd.runsc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
runtime_engine = ""
runtime_root = ""
privileged_without_host_devices = false
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
max_conf_num = 1
conf_template = ""
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://registry-1.docker.io"]
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = ""
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false
[plugins."io.containerd.runtime.v1.linux"]
shim = "containerd-shim"
runtime = "runc"
runtime_root = ""
no_shim = false
shim_debug = false
[plugins."io.containerd.runtime.v2.task"]
platforms = ["linux/amd64"]
[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]
[plugins."io.containerd.snapshotter.v1.devmapper"]
root_path = ""
pool_name = ""
base_image_size = ""
async_remove = false
```
<br>
重启 containerd 和 kubelet
```
systemctl daemon-reload
systemctl start containerd
systemctl restart kubelet
```
Node上尝试启动Containerd看是否有报错, 如下输出则为正常:
```
root@ubuk8s-vm04:/etc/containerd# containerd
INFO[2021-04-12T03:17:44.577508831-04:00] starting containerd revision=05f951a3781f4f2c1911b05e61c160e9c30eaa8e version=v1.4.4
INFO[2021-04-12T03:17:44.610873130-04:00] loading plugin "io.containerd.content.v1.content"... type=io.containerd.content.v1
INFO[2021-04-12T03:17:44.610972745-04:00] loading plugin "io.containerd.snapshotter.v1.aufs"... type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.613266713-04:00] loading plugin "io.containerd.snapshotter.v1.btrfs"... type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.613924390-04:00] skip loading plugin "io.containerd.snapshotter.v1.btrfs"... error="path /var/lib/containerd/io.containerd.snapshotter.v1.btrfs (ext4) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.613959760-04:00] loading plugin "io.containerd.snapshotter.v1.devmapper"... type=io.containerd.snapshotter.v1
WARN[2021-04-12T03:17:44.614026510-04:00] failed to load plugin io.containerd.snapshotter.v1.devmapper error="devmapper not configured"
INFO[2021-04-12T03:17:44.614045917-04:00] loading plugin "io.containerd.snapshotter.v1.native"... type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.614089401-04:00] loading plugin "io.containerd.snapshotter.v1.overlayfs"... type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.614300951-04:00] loading plugin "io.containerd.snapshotter.v1.zfs"... type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.614865003-04:00] skip loading plugin "io.containerd.snapshotter.v1.zfs"... error="path /var/lib/containerd/io.containerd.snapshotter.v1.zfs must be a zfs filesystem to be used with the zfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
INFO[2021-04-12T03:17:44.614895364-04:00] loading plugin "io.containerd.metadata.v1.bolt"... type=io.containerd.metadata.v1
WARN[2021-04-12T03:17:44.614930461-04:00] could not use snapshotter devmapper in metadata plugin error="devmapper not configured"
INFO[2021-04-12T03:17:44.614948636-04:00] metadata content store policy set policy=shared
```
在K8s Master上确认Node是Ready的.
## 创建RuntimeClass
```
kind: RuntimeClass
apiVersion: node.k8s.io/v1beta1
metadata:
name: kata-containers
handler: kata
```
## 调用Kata创建Pod
```
apiVersion: v1
kind: Pod
metadata:
name: nettool-kata
spec:
nodeName: ubuk8s-vm04
runtimeClassName: kata-containers
containers:
- name: nettool
image: rock981119/net-tools:v2
```
## 校验
宿主机中查看内核版本:
```
uname -a
Linux ubuk8s-vm04 4.15.0-140-generic #144-Ubuntu SMP Fri Mar 19 14:12:35 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
root@ubuk8s-vm04:/etc/containerd# uname -r
4.15.0-140-generic
```
进入Pod内查看内核版本
```
kubectl exec -it nettool-kata bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@nettool-kata:/# uname -a
Linux nettool-kata 5.4.60-52.container #1 SMP Sat Jan 16 05:49:34 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
root@nettool-kata:/# uname -r
5.4.60-52.container
```
宿主机查看Kata Runtime进程:
```
kata-runtime list
ID PID STATUS BUNDLE CREATED OWNER
02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3 2021-04-12T00:00:04.159191984Z #0
8b5b9a0b6d350169b0870b0296f9e017031f1261fcecef08a04ad6ddf5e9a358 -1 running /run/containerd/io.containerd.runtime.v2.task/k8s.io/8b5b9a0b6d350169b0870b0296f9e017031f1261fcecef08a04ad6ddf5e9a358 2021-04-12T00:00:04.728608154Z #0
```
```
ps -aux |grep qemu
root 10463 0.5 2.4 2861188 199992 ? Sl Apr11 2:41 /usr/bin/qemu-vanilla-system-x86_64 -name sandbox-02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3 -uuid 75e13e6c-f231-4c05-ac1c-61d928531dcf -machine pc,accel=kvm,kernel_irqchip,nvdimm -cpu host,pmu=off -qmp unix:/run/vc/vm/02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3/qmp.sock,server,nowait -m 2048M,slots=10,maxmem=8977M -device pci-bridge,bus=pci.0,id=pci-bridge-0,chassis_nr=1,shpc=on,addr=2,romfile= -device virtio-serial-pci,disable-modern=true,id=serial0,romfile=,max_ports=2 -device virtconsole,chardev=charconsole0,id=console0 -chardev socket,id=charconsole0,path=/run/vc/vm/02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3/console.sock,server,nowait -device nvdimm,id=nv0,memdev=mem0 -object memory-backend-file,id=mem0,mem-path=/usr/share/kata-containers/kata-containers-image_clearlinux_1.13.0-alpha0_agent_27b90c2690.img,size=268435456 -device virtio-scsi-pci,id=scsi0,disable-modern=true,romfile= -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0,romfile= -device virtserialport,chardev=charch0,id=channel0,name=agent.channel.0 -chardev socket,id=charch0,path=/run/vc/vm/02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3/kata.sock,server,nowait -device virtio-9p-pci,disable-modern=true,fsdev=extra-9p-kataShared,mount_tag=kataShared,romfile= -fsdev local,id=extra-9p-kataShared,path=/run/kata-containers/shared/sandboxes/02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3/shared,security_model=none,multidevs=remap -netdev tap,id=network-0,vhost=on,vhostfds=3,fds=4 -device driver=virtio-net-pci,netdev=network-0,mac=f6:fd:1d:19:f4:ea,disable-modern=true,mq=on,vectors=4,romfile= -rtc base=utc,driftfix=slew,clock=host -global kvm-pit.lost_tick_policy=discard -vga none -no-user-config -nodefaults -nographic --no-reboot -daemonize -object memory-backend-ram,id=dimm1,size=2048M -numa node,memdev=dimm1 -kernel /usr/share/kata-containers/vmlinuz-5.4.60.91-52.container -append tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k console=hvc0 console=hvc1 cryptomgr.notests net.ifnames=0 pci=lastbus=0 iommu=off root=/dev/pmem0p1 rootflags=dax,data=ordered,errors=remount-ro ro rootfstype=ext4 quiet systemd.show_status=false panic=1 nr_cpus=8 agent.use_vsock=false systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none -pidfile /run/vc/vm/02a9277cd7cddba3cb25a3074e91a43ca975abbc36de3911add6922f7fae11f3/pid -smp 1,cores=1,threads=1,sockets=8,maxcpus=8
root 100155 0.0 0.0 13144 1096 pts/0 S+ 03:31 0:00 grep --color=auto qemu
```
Sign in with Wallet
Connect another wallet