---
# System prepended metadata

title: Scheduling Workloads
tags: [' CKA', k8s-fundamentals]

---

---
tags: k8s-fundamentals, CKA
---
# Scheduling Workloads
This page contains some experiments on K8S workload scheduling.
* NodeAffinity
* PodAffinity
* PodAntiAffinity
* Pod topologySpreadConstraints

## Experiments
### Setup kind cluster
```
// kind.yaml
kind: Cluster
apiVersion: kind.sigs.k8s.io/v1alpha3
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
```
```
kind create cluster --config ./kind.yaml --image kindest/node:v1.18.0
```
### Label a node and Unlabel a node
```
kubectl label node kind-worker node-role.kubernetes.io/worker=
~/tech-explore/pod-scheduling  kubectl get nodes
NAME                 STATUS   ROLES    AGE    VERSION
kind-control-plane   Ready    master   109m   v1.18.0
kind-worker          Ready    worker   108m   v1.18.0
kind-worker2         Ready    <none>   108m   v1.18.0
kind-worker3         Ready    <none>   108m   v1.18.0

kubectl label node kind-worker node-role.kubernetes.io/worker=
~/tech-explore/pod-scheduling  kubectl get nodes
NAME                 STATUS   ROLES    AGE    VERSION
kind-control-plane   Ready    master   109m   v1.18.0
kind-worker          Ready    <none>   108m   v1.18.0
kind-worker2         Ready    <none>   108m   v1.18.0
kind-worker3         Ready    <none>   108m   v1.18.0
```

### Create a pod on a random node
```
kubectl run nginx --image=nginx --restart=Never -o yaml --dry-run=client > nginx-pod.yaml

 ~/tech-explore/pod-scheduling  kubectl get pod -o wide
NAME    READY   STATUS    RESTARTS   AGE   IP           NODE           NOMINATED NODE   READINESS GATES
nginx   1/1     Running   0          11s   10.244.3.2   kind-worker3   <none>           <none>
```

### Create a pod on a particular node with nodeName
```
// For testing purpose, I am going to shutdown the controller manager, sheduler in kind and, the pod still can be created

docker exec -it kind-control-plane bash

root@kind-control-plane:/etc/kubernetes/manifests# crictl ps
CONTAINER           IMAGE               CREATED             STATE               NAME                      ATTEMPT             POD ID
a7f4b8a0562e3       db10073a6f829       2 hours ago         Running             local-path-provisioner    0                   65923e0030098
d77d86c094e02       67da37a9a360e       2 hours ago         Running             coredns                   0                   c95ed97ccbcaa
06383b940ee74       67da37a9a360e       2 hours ago         Running             coredns                   0                   a2b961215cdf7
e00a7f3c4405d       8aff47b2a8d8d       2 hours ago         Running             kube-proxy                0                   84136aa6c7724
3862a3d86abae       2186a1a396deb       2 hours ago         Running             kindnet-cni               0                   8c37f628331c6
0c598f608ea02       303ce5db0e90d       2 hours ago         Running             etcd                      0                   80871c79ca7a9
390dac1bf7f26       06898432911e4       2 hours ago         Running             kube-apiserver            0                   ea71ce42bd71b
f2b9bce63b40d       0086f582d6f07       2 hours ago         Running             kube-controller-manager   0                   e79df33546948
4a73613bd6f64       e7ef4082901cb       2 hours ago         Running             kube-scheduler            0                   0fb43feda0941

cd /etc/kubernetes/manifests
root@kind-control-plane:/etc/kubernetes/manifests# mv kube-controller-manager.yaml ../
root@kind-control-plane:/etc/kubernetes/manifests# mv kube-scheduler.yaml ../

root@kind-control-plane:/etc/kubernetes/manifests# crictl ps
CONTAINER           IMAGE               CREATED             STATE               NAME                     ATTEMPT             POD ID
a7f4b8a0562e3       db10073a6f829       2 hours ago         Running             local-path-provisioner   0                   65923e0030098
d77d86c094e02       67da37a9a360e       2 hours ago         Running             coredns                  0                   c95ed97ccbcaa
06383b940ee74       67da37a9a360e       2 hours ago         Running             coredns                  0                   a2b961215cdf7
e00a7f3c4405d       8aff47b2a8d8d       2 hours ago         Running             kube-proxy               0                   84136aa6c7724
3862a3d86abae       2186a1a396deb       2 hours ago         Running             kindnet-cni              0                   8c37f628331c6
0c598f608ea02       303ce5db0e90d       2 hours ago         Running             etcd                     0                   80871c79ca7a9
390dac1bf7f26       06898432911e4       2 hours ago         Running             kube-apiserver           0                   ea71ce42bd71b

cat > nginx-pod.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
  labels:
    run: nginx
  name: nginx
spec:
  nodeName: kind-worker
  containers:
  - image: nginx
    name: nginx
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Never
EOF
kubectl apply -f nginx-pod.yaml

 ~/tech-explore/pod-scheduling  kubectl get pods -o wide
NAME    READY   STATUS    RESTARTS   AGE   IP           NODE          NOMINATED NODE   READINESS GATES
nginx   1/1     Running   0          6s    10.244.1.3   kind-worker   <none>           <none>
```

### Create a deployment with nodeName
In order to make it work, you might want to enable the kube-controller-manager and kube-scheduler in kind
```
cat > nginx-deploy.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  creationTimestamp: null
  labels:
    app: nginx
  name: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  strategy: {}
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: nginx
    spec:
      nodeName: kind-worker3
      containers:
      - image: nginx
        name: nginx
        resources: {}
status: {}
EOF


kubectl apply -f nginx-deploy.yaml
```
### Create a deployment with nodeSelector
```
apiVersion: apps/v1
kind: Deployment
metadata:
  creationTimestamp: null
  labels:
    app: nginx
  name: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  strategy: {}
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: nginx
    spec:
      nodeSelector:
        pickme: "" // You have to have one of the node labeled with pickme first
      containers:
      - image: nginx
        name: nginx
        resources: {}
status: {}


kubectl apply -f nginx-deploy.yaml
```

### Pod Topology Spread Constraints
* Label nodes: two nodes are labeled with zone=A, one node is labeled with zone=B
```
 ~/tech-explore/pod-scheduling  kubectl get nodes -o wide --show-labels
NAME                 STATUS   ROLES    AGE     VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE       KERNEL-VERSION     CONTAINER-RUNTIME                 LABELS
kind-control-plane   Ready    master   5h27m   v1.18.0   172.17.0.5    <none>        Ubuntu 19.10   4.19.76-linuxkit   containerd://1.3.3-14-g449e9269   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-control-plane,kubernetes.io/os=linux,node-role.kubernetes.io/master=
kind-worker          Ready    <none>   5h26m   v1.18.0   172.17.0.4    <none>        Ubuntu 19.10   4.19.76-linuxkit   containerd://1.3.3-14-g449e9269   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-worker,kubernetes.io/os=linux,zone=A
kind-worker2         Ready    <none>   5h26m   v1.18.0   172.17.0.2    <none>        Ubuntu 19.10   4.19.76-linuxkit   containerd://1.3.3-14-g449e9269   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-worker2,kubernetes.io/os=linux,zone=A
kind-worker3         Ready    <none>   5h26m   v1.18.0   172.17.0.3    <none>        Ubuntu 19.10   4.19.76-linuxkit   containerd://1.3.3-14-g449e9269   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-worker3,kubernetes.io/os=linux,zone=B
```
* Create a deployment
```
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx
  name: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  strategy: {}
  template:
    metadata:
      labels:
        app: nginx
    spec:
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: zone
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: nginx
      containers:
      - image: nginx
        name: nginx
        resources: {}
status: {}

kubectl apply -f <filename.yaml>
```
### Scale replicas to 3
```
kubectl scale deploy nginx --replicas=3

 ~/tech-explore/pod-scheduling  kubectl get pods -o wide --show-labels
NAME                     READY   STATUS    RESTARTS   AGE     IP           NODE           NOMINATED NODE   READINESS GATES   LABELS
nginx-7b8dd6c565-6tc6g   1/1     Running   0          3m34s   10.244.3.6   kind-worker3   <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-8z4rc   1/1     Running   0          4m5s    10.244.2.6   kind-worker2   <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-njkbs   1/1     Running   0          2m42s   10.244.1.6   kind-worker    <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
```
### Scale replicas to 4
```
kubectl scale deploy nginx --replicas=4

 ~/tech-explore/pod-scheduling  kubectl get pods -o wide --show-labels
NAME                     READY   STATUS    RESTARTS   AGE     IP           NODE           NOMINATED NODE   READINESS GATES   LABELS
nginx-7b8dd6c565-6tc6g   1/1     Running   0          4m32s   10.244.3.6   kind-worker3   <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-8z4rc   1/1     Running   0          5m3s    10.244.2.6   kind-worker2   <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-kmmb5   1/1     Running   0          16s     10.244.3.7   kind-worker3   <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-njkbs   1/1     Running   0          3m40s   10.244.1.6   kind-worker    <none>           <none>            app=nginx,pod-template-hash=7b8dd6c565
```

## References
* https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
* https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
* https://www.youtube.com/watch?v=o_gT_LfFn4s