---
tags: k8s-fundamentals, CKA
---
# Scheduling Workloads
This page contains some experiments on K8S workload scheduling.
* NodeAffinity
* PodAffinity
* PodAntiAffinity
* Pod topologySpreadConstraints
## Experiments
### Setup kind cluster
```
// kind.yaml
kind: Cluster
apiVersion: kind.sigs.k8s.io/v1alpha3
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
```
```
kind create cluster --config ./kind.yaml --image kindest/node:v1.18.0
```
### Label a node and Unlabel a node
```
kubectl label node kind-worker node-role.kubernetes.io/worker=
~/tech-explore/pod-scheduling kubectl get nodes
NAME STATUS ROLES AGE VERSION
kind-control-plane Ready master 109m v1.18.0
kind-worker Ready worker 108m v1.18.0
kind-worker2 Ready <none> 108m v1.18.0
kind-worker3 Ready <none> 108m v1.18.0
kubectl label node kind-worker node-role.kubernetes.io/worker=
~/tech-explore/pod-scheduling kubectl get nodes
NAME STATUS ROLES AGE VERSION
kind-control-plane Ready master 109m v1.18.0
kind-worker Ready <none> 108m v1.18.0
kind-worker2 Ready <none> 108m v1.18.0
kind-worker3 Ready <none> 108m v1.18.0
```
### Create a pod on a random node
```
kubectl run nginx --image=nginx --restart=Never -o yaml --dry-run=client > nginx-pod.yaml
~/tech-explore/pod-scheduling kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx 1/1 Running 0 11s 10.244.3.2 kind-worker3 <none> <none>
```
### Create a pod on a particular node with nodeName
```
// For testing purpose, I am going to shutdown the controller manager, sheduler in kind and, the pod still can be created
docker exec -it kind-control-plane bash
root@kind-control-plane:/etc/kubernetes/manifests# crictl ps
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID
a7f4b8a0562e3 db10073a6f829 2 hours ago Running local-path-provisioner 0 65923e0030098
d77d86c094e02 67da37a9a360e 2 hours ago Running coredns 0 c95ed97ccbcaa
06383b940ee74 67da37a9a360e 2 hours ago Running coredns 0 a2b961215cdf7
e00a7f3c4405d 8aff47b2a8d8d 2 hours ago Running kube-proxy 0 84136aa6c7724
3862a3d86abae 2186a1a396deb 2 hours ago Running kindnet-cni 0 8c37f628331c6
0c598f608ea02 303ce5db0e90d 2 hours ago Running etcd 0 80871c79ca7a9
390dac1bf7f26 06898432911e4 2 hours ago Running kube-apiserver 0 ea71ce42bd71b
f2b9bce63b40d 0086f582d6f07 2 hours ago Running kube-controller-manager 0 e79df33546948
4a73613bd6f64 e7ef4082901cb 2 hours ago Running kube-scheduler 0 0fb43feda0941
cd /etc/kubernetes/manifests
root@kind-control-plane:/etc/kubernetes/manifests# mv kube-controller-manager.yaml ../
root@kind-control-plane:/etc/kubernetes/manifests# mv kube-scheduler.yaml ../
root@kind-control-plane:/etc/kubernetes/manifests# crictl ps
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID
a7f4b8a0562e3 db10073a6f829 2 hours ago Running local-path-provisioner 0 65923e0030098
d77d86c094e02 67da37a9a360e 2 hours ago Running coredns 0 c95ed97ccbcaa
06383b940ee74 67da37a9a360e 2 hours ago Running coredns 0 a2b961215cdf7
e00a7f3c4405d 8aff47b2a8d8d 2 hours ago Running kube-proxy 0 84136aa6c7724
3862a3d86abae 2186a1a396deb 2 hours ago Running kindnet-cni 0 8c37f628331c6
0c598f608ea02 303ce5db0e90d 2 hours ago Running etcd 0 80871c79ca7a9
390dac1bf7f26 06898432911e4 2 hours ago Running kube-apiserver 0 ea71ce42bd71b
cat > nginx-pod.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
nodeName: kind-worker
containers:
- image: nginx
name: nginx
resources: {}
dnsPolicy: ClusterFirst
restartPolicy: Never
EOF
kubectl apply -f nginx-pod.yaml
~/tech-explore/pod-scheduling kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx 1/1 Running 0 6s 10.244.1.3 kind-worker <none> <none>
```
### Create a deployment with nodeName
In order to make it work, you might want to enable the kube-controller-manager and kube-scheduler in kind
```
cat > nginx-deploy.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: nginx
name: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: nginx
spec:
nodeName: kind-worker3
containers:
- image: nginx
name: nginx
resources: {}
status: {}
EOF
kubectl apply -f nginx-deploy.yaml
```
### Create a deployment with nodeSelector
```
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: nginx
name: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: nginx
spec:
nodeSelector:
pickme: "" // You have to have one of the node labeled with pickme first
containers:
- image: nginx
name: nginx
resources: {}
status: {}
kubectl apply -f nginx-deploy.yaml
```
### Pod Topology Spread Constraints
* Label nodes: two nodes are labeled with zone=A, one node is labeled with zone=B
```
~/tech-explore/pod-scheduling kubectl get nodes -o wide --show-labels
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME LABELS
kind-control-plane Ready master 5h27m v1.18.0 172.17.0.5 <none> Ubuntu 19.10 4.19.76-linuxkit containerd://1.3.3-14-g449e9269 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-control-plane,kubernetes.io/os=linux,node-role.kubernetes.io/master=
kind-worker Ready <none> 5h26m v1.18.0 172.17.0.4 <none> Ubuntu 19.10 4.19.76-linuxkit containerd://1.3.3-14-g449e9269 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-worker,kubernetes.io/os=linux,zone=A
kind-worker2 Ready <none> 5h26m v1.18.0 172.17.0.2 <none> Ubuntu 19.10 4.19.76-linuxkit containerd://1.3.3-14-g449e9269 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-worker2,kubernetes.io/os=linux,zone=A
kind-worker3 Ready <none> 5h26m v1.18.0 172.17.0.3 <none> Ubuntu 19.10 4.19.76-linuxkit containerd://1.3.3-14-g449e9269 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kind-worker3,kubernetes.io/os=linux,zone=B
```
* Create a deployment
```
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
strategy: {}
template:
metadata:
labels:
app: nginx
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: zone
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app: nginx
containers:
- image: nginx
name: nginx
resources: {}
status: {}
kubectl apply -f <filename.yaml>
```
### Scale replicas to 3
```
kubectl scale deploy nginx --replicas=3
~/tech-explore/pod-scheduling kubectl get pods -o wide --show-labels
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES LABELS
nginx-7b8dd6c565-6tc6g 1/1 Running 0 3m34s 10.244.3.6 kind-worker3 <none> <none> app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-8z4rc 1/1 Running 0 4m5s 10.244.2.6 kind-worker2 <none> <none> app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-njkbs 1/1 Running 0 2m42s 10.244.1.6 kind-worker <none> <none> app=nginx,pod-template-hash=7b8dd6c565
```
### Scale replicas to 4
```
kubectl scale deploy nginx --replicas=4
~/tech-explore/pod-scheduling kubectl get pods -o wide --show-labels
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES LABELS
nginx-7b8dd6c565-6tc6g 1/1 Running 0 4m32s 10.244.3.6 kind-worker3 <none> <none> app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-8z4rc 1/1 Running 0 5m3s 10.244.2.6 kind-worker2 <none> <none> app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-kmmb5 1/1 Running 0 16s 10.244.3.7 kind-worker3 <none> <none> app=nginx,pod-template-hash=7b8dd6c565
nginx-7b8dd6c565-njkbs 1/1 Running 0 3m40s 10.244.1.6 kind-worker <none> <none> app=nginx,pod-template-hash=7b8dd6c565
```
## References
* https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
* https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
* https://www.youtube.com/watch?v=o_gT_LfFn4s
Sign in with Wallet
Connect another wallet