Ceph Non-Resilient Pools - Power platform testing
Container Storage On PowerCSOP-3 ODF 4.12 TestCSOP-62
Verify Replica 1 - Non-resilient pool - Dev Preview

[root@rdr-res3-lon06-bastion-0 ~]# oc get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS
version 4.12.0-0.nightly-ppc64le-2022-11-24-203625 True False 11h Cluster version is 4.12.0-0.nightly-ppc64le-2022-11-24-203625

[root@rdr-res3-lon06-bastion-0 ~]#
[root@rdr-res3-lon06-bastion-0 ~]# oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-storagecluster 5m20s Ready 2022-12-15T07:01:33Z 4.12.0

[root@rdr-res3-lon06-bastion-0 ~]# oc patch storagecluster ocs-storagecluster -n openshift-storage --type json --patch '[{ "op": "replace", "path": "/spec/managedResources/cephNonResilientPools/enable", "value": true }]'
storagecluster.ocs.openshift.io/ocs-storagecluster patched

[root@rdr-res3-lon06-bastion-0 ~]# oc patch cm rook-ceph-operator-config -n openshift-storage -p $'data:\n "CSI_ENABLE_TOPOLOGY": "true"'
configmap/rook-ceph-operator-config patched
[root@rdr-res3-lon06-bastion-0 ~]#
[root@rdr-res3-lon06-bastion-0 ~]# oc patch cm rook-ceph-operator-config -n openshift-storage -p $'data:\n "CSI_TOPOLOGY_DOMAIN_LABELS": "topology.kubernetes.io/zone"'
configmap/rook-ceph-operator-config patched

[root@rdr-res3-lon06-bastion-0 ~]# oc get cephblockpools
NAME PHASE
ocs-storagecluster-cephblockpool Ready
ocs-storagecluster-cephblockpool-worker-0 Ready
ocs-storagecluster-cephblockpool-worker-1 Ready
ocs-storagecluster-cephblockpool-worker-2 Ready
[root@rdr-res3-lon06-bastion-0 ~]#

##Check if the default pool has the “replicated” deviceclass
[root@rdr-res3-lon06-bastion-0 ~]# oc get cephblockpools ocs-storagecluster-cephblockpool -o yaml
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
creationTimestamp: "2022-12-15T07:01:33Z"
finalizers:
- cephblockpool.ceph.rook.io
generation: 2
name: ocs-storagecluster-cephblockpool
namespace: openshift-storage
ownerReferences:
- apiVersion: ocs.openshift.io/v1
blockOwnerDeletion: true
controller: true
kind: StorageCluster
name: ocs-storagecluster
uid: 4b3315df-7e73-48e3-8106-76b41363cd3e
resourceVersion: "334149"
uid: 47ff62e9-ac94-4849-9683-e369dd6c3e1d
spec:
deviceClass: replicated
enableRBDStats: true
erasureCoded:
codingChunks: 0
dataChunks: 0
failureDomain: host
mirroring: {}
quotas: {}
replicated:
replicasPerFailureDomain: 1
size: 3
targetSizeRatio: 0.49
statusCheck:
mirror: {}
status:
observedGeneration: 2
phase: Ready
[root@rdr-res3-lon06-bastion-0 ~]#

##Check if the non-resilient pools have the failure domain(Node/Zone) as the device class

[root@rdr-res3-lon06-bastion-0 ~]# oc get cephblockpool ocs-storagecluster-cephblockpool-worker-0 -o yaml
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
creationTimestamp: "2022-12-15T07:07:46Z"
finalizers:
- cephblockpool.ceph.rook.io
generation: 1
name: ocs-storagecluster-cephblockpool-worker-0
namespace: openshift-storage
ownerReferences:
- apiVersion: ocs.openshift.io/v1
blockOwnerDeletion: true
controller: true
kind: StorageCluster
name: ocs-storagecluster
uid: 4b3315df-7e73-48e3-8106-76b41363cd3e
resourceVersion: "335089"
uid: 330caaa5-4249-4367-ba60-62690fe4f874
spec:
deviceClass: worker-0
enableRBDStats: true
erasureCoded:
codingChunks: 0
dataChunks: 0
failureDomain: host
mirroring: {}
quotas: {}
replicated:
size: 1
statusCheck:
mirror: {}
status:
observedGeneration: 1
phase: Ready
[root@rdr-res3-lon06-bastion-0 ~]#

##storageclass
[root@rdr-res3-lon06-bastion-0 ~]# oc get storageclass
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
localblock kubernetes.io/no-provisioner Delete WaitForFirstConsumer false 28m
ocs-storagecluster-ceph-non-resilient-rbd openshift-storage.rbd.csi.ceph.com Delete WaitForFirstConsumer true 15m
ocs-storagecluster-ceph-rbd openshift-storage.rbd.csi.ceph.com Delete Immediate true 20m
ocs-storagecluster-ceph-rgw openshift-storage.ceph.rook.io/bucket Delete Immediate false 22m
ocs-storagecluster-cephfs openshift-storage.cephfs.csi.ceph.com Delete Immediate true 20m
openshift-storage.noobaa.io openshift-storage.noobaa.io/obc Delete Immediate false 18m
[root@rdr-res3-lon06-bastion-0 ~]#

###Check the new storageclass for non-resilient topology constrained pools

[root@rdr-res3-lon06-bastion-0 ~]# oc get storageclass ocs-storagecluster-ceph-non-resilient-rbd -o yaml
allowVolumeExpansion: true
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
description: 'Ceph Non Resilient Pools : Provides RWO Filesystem volumes, and
RWO and RWX Block volumes'
creationTimestamp: "2022-12-15T07:09:10Z"
name: ocs-storagecluster-ceph-non-resilient-rbd
resourceVersion: "335168"
uid: e5ae2d13-a6db-4f2c-a4f8-812e48235307
parameters:
clusterID: openshift-storage
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/controller-expand-secret-namespace: openshift-storage
csi.storage.k8s.io/fstype: ext4
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
csi.storage.k8s.io/node-stage-secret-namespace: openshift-storage
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: openshift-storage
imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
imageFormat: "2"
pool: ocs-storagecluster-cephblockpool
topologyConstrainedPools: |-
[
{
"poolName": "ocs-storagecluster-cephblockpool-worker-1",
"domainSegments": [
{
"domainLabel": "host",
"value": "worker-1"
}
]
},
{
"poolName": "ocs-storagecluster-cephblockpool-worker-2",
"domainSegments": [
{
"domainLabel": "host",
"value": "worker-2"
}
]
},
{
"poolName": "ocs-storagecluster-cephblockpool-worker-0",
"domainSegments": [
{
"domainLabel": "host",
"value": "worker-0"
}
]
}
]
provisioner: openshift-storage.rbd.csi.ceph.com
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
[root@rdr-res3-lon06-bastion-0 ~]#

##OSD PODS

[root@rdr-res3-lon06-bastion-0 ~]# oc get pods | grep osd
rook-ceph-osd-0-6fd49bf8dd-2kxf5 2/2 Running 0 23m
rook-ceph-osd-1-cb965d6fd-zb7zv 2/2 Running 0 23m
rook-ceph-osd-2-75d89c9dbd-f8649 2/2 Running 0 23m
rook-ceph-osd-3-7f8d9986d8-k7gcx 2/2 Running 0 18m
rook-ceph-osd-4-d9fb99cc5-hsqbh 2/2 Running 0 18m
rook-ceph-osd-5-8674ccc494-mlwjn 2/2 Running 0 18m
rook-ceph-osd-prepare-21340fc0b3eaaa13c7ff7bde3747282d-5mdzh 0/1 Completed 0 23m
rook-ceph-osd-prepare-42c210093a02c8e4b6af91e249a8649d-k8jcd 0/1 Completed 0 23m
rook-ceph-osd-prepare-ad5556d5edf290c46b62a6b32a0f5aba-nqgzn 0/1 Completed 0 23m
rook-ceph-osd-prepare-worker-0-data-0rtcwv-fsddm 0/1 Completed 0 18m
rook-ceph-osd-prepare-worker-1-data-0xwf6n-q6kqm 0/1 Completed 0 18m
rook-ceph-osd-prepare-worker-2-data-0nxg78-vjdxb 0/1 Completed 0 18m
[root@rdr-res3-lon06-bastion-0 ~]#

##Data Pvc's

[root@rdr-res3-lon06-bastion-0 ~]# oc get pvc | grep data
ocs-deviceset-localblock-0-data-06l6cm Bound local-pv-7854945e 500Gi RWO localblock 24m
ocs-deviceset-localblock-0-data-1tx46r Bound local-pv-ceaba4b7 500Gi RWO localblock 24m
ocs-deviceset-localblock-0-data-2764zg Bound local-pv-7353786a 500Gi RWO localblock 24m
worker-0-data-0rtcwv Bound local-pv-b32d86e2 500Gi RWO localblock 18m
worker-1-data-0xwf6n Bound local-pv-f1711088 500Gi RWO localblock 18m
worker-2-data-0nxg78 Bound local-pv-bc3fb692 500Gi RWO localblock 18m
[root@rdr-res3-lon06-bastion-0 ~]#

##Data Pv's

[root@rdr-res3-lon06-bastion-0 ~]# oc get pv | grep data
local-pv-7353786a 500Gi RWO Delete Bound openshift-storage/ocs-deviceset-localblock-0-data-2764zg localblock 31m
local-pv-7854945e 500Gi RWO Delete Bound openshift-storage/ocs-deviceset-localblock-0-data-06l6cm localblock 31m
local-pv-b32d86e2 500Gi RWO Delete Bound openshift-storage/worker-0-data-0rtcwv localblock 31m
local-pv-bc3fb692 500Gi RWO Delete Bound openshift-storage/worker-2-data-0nxg78 localblock 31m
local-pv-ceaba4b7 500Gi RWO Delete Bound openshift-storage/ocs-deviceset-localblock-0-data-1tx46r localblock 31m
local-pv-f1711088 500Gi RWO Delete Bound openshift-storage/worker-1-data-0xwf6n localblock 31m
[root@rdr-res3-lon06-bastion-0 ~]#

##Ceph OSD Tree

[root@rdr-res3-lon06-bastion-0 ~]# oc rsh rook-ceph-tools-67bb47d98b-jhskk
sh-4.4$ ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 2.92978 root default
-5 2.92978 region lon
-4 2.92978 zone lon06
-3 0.97659 host worker-0
0 hdd 0.48830 osd.0 up 1.00000 1.00000
4 worker-0 0.48830 osd.4 up 1.00000 1.00000
-11 0.97659 host worker-1
1 hdd 0.48830 osd.1 up 1.00000 1.00000
5 worker-1 0.48830 osd.5 up 1.00000 1.00000
-9 0.97659 host worker-2
2 hdd 0.48830 osd.2 up 1.00000 1.00000
3 worker-2 0.48830 osd.3 up 1.00000 1.00000
sh-4.4$

##Ceph Pool details

sh-4.4$ ceph osd pool ls detail
pool 1 'ocs-storagecluster-cephblockpool' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 31 flags hashpspool,selfmanaged_snaps stripe_width 0 target_size_ratio 0.49 application rbd
pool 2 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 16 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr_devicehealth
pool 3 'ocs-storagecluster-cephobjectstore.rgw.meta' replicated size 3 min_size 2 crush_rule 2 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 20 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 4 'ocs-storagecluster-cephobjectstore.rgw.buckets.non-ec' replicated size 3 min_size 2 crush_rule 3 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 20 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 5 '.rgw.root' replicated size 3 min_size 2 crush_rule 5 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 21 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 6 'ocs-storagecluster-cephobjectstore.rgw.control' replicated size 3 min_size 2 crush_rule 7 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 20 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 7 'ocs-storagecluster-cephobjectstore.rgw.buckets.index' replicated size 3 min_size 2 crush_rule 6 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 20 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 8 'ocs-storagecluster-cephobjectstore.rgw.otp' replicated size 3 min_size 2 crush_rule 8 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 21 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 9 'ocs-storagecluster-cephobjectstore.rgw.log' replicated size 3 min_size 2 crush_rule 4 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 20 flags hashpspool stripe_width 0 application rook-ceph-rgw
pool 10 'ocs-storagecluster-cephfilesystem-metadata' replicated size 3 min_size 2 crush_rule 9 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 29 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs
pool 11 'ocs-storagecluster-cephfilesystem-data0' replicated size 3 min_size 2 crush_rule 11 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 33 flags hashpspool stripe_width 0 target_size_ratio 0.49 application cephfs
pool 12 'ocs-storagecluster-cephobjectstore.rgw.buckets.data' replicated size 3 min_size 2 crush_rule 10 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 32 flags hashpspool stripe_width 0 target_size_ratio 0.49 application rook-ceph-rgw
pool 13 'ocs-storagecluster-cephblockpool-worker-2' replicated size 1 min_size 1 crush_rule 12 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 45 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd
pool 14 'ocs-storagecluster-cephblockpool-worker-0' replicated size 1 min_size 1 crush_rule 13 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 52 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd
pool 15 'ocs-storagecluster-cephblockpool-worker-1' replicated size 1 min_size 1 crush_rule 14 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 58 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd

sh-4.4$

#######################
##Testing the non-resilient storageclass topology aware provisioning

#Create a pvc

[root@rdr-res3-lon06-bastion-0 ~]# cat <<EOF | oc create -f -
> apiVersion: v1
> kind: PersistentVolumeClaim
> metadata:
> name: non-resilient-rbd-pvc
> spec:
> accessModes:
> - ReadWriteOnce
> resources:
> requests:
> storage: 1Gi
> storageClassName: ocs-storagecluster-ceph-non-resilient-rbd
> EOF
persistentvolumeclaim/non-resilient-rbd-pvc created

#The pvc would be in pending state, waiting for a consumer
[root@rdr-res3-lon06-bastion-0 ~]# oc get pvc | grep non-resilient-rbd-pvc
non-resilient-rbd-pvc Pending ocs-storagecluster-ceph-non-resilient-rbd 22s

#Create a pod to consume the pvc
[root@rdr-res3-lon06-bastion-0 ~]# cat <<EOF | oc create -f -
> apiVersion: v1
> kind: Pod
spec:
> metadata:
> name: task-pv-pod
> spec:
> nodeSelector:
> topology.kubernetes.io/zone: us-east-1a
> volumes:
> - name: task-pv-storage
> persistentVolumeClaim:
> claimName: non-resilient-rbd-pvc
> containers:
> - name: task-pv-container
> image: nginx
> ports:
> - containerPort: 80
name: "http-server"
volumeMounts:
> name: "http-server"
> volumeMounts:
> - mountPath: "/usr/share/nginx/html"
> name: task-pv-storage
> EOF
pod/task-pv-pod created
[root@rdr-res3-lon06-bastion-0 ~]#
[root@rdr-res3-lon06-bastion-0 ~]# cat <<EOF | oc create -f -
> apiVersion: v1
> kind: Pod
spec:
> metadata:
> name: task-pv-pod
> spec:
> nodeSelector:
> kubernetes.io/hostname=worker-0
> volumes:
> - name: task-pv-storage
> persistentVolumeClaim:
> claimName: non-resilient-rbd-pvc
> containers:
> - name: task-pv-container
> image: nginx
> ports:
> - containerPort: 80
name: "http-server"
volumeMounts:
> name: "http-server"
> volumeMounts:
> - mountPath: "/usr/share/nginx/html"
> name: task-pv-storage
> EOF
pod/task-pv-pod created
[root@rdr-res3-lon06-bastion-0 ~]#