# In-transit encryption: ODF External Mode
* Case-1: ODF 4.13, with encryption enabled during creation of storagesystem
-From UI while storagesystem creation by ticking checkbox
-From CLI with encryption in storagecluster spec while creation.
(This is the only officially supported way to use encryption)
* Case-2: ODF 4.13, without enabling encryption during storagesystem creation
* Case-3: An cluster is upgraded from ODF 4.12 to ODF 4.13
* Case-4: In an cluster created with encryption, due to some issues encryption needs to be turned off
## Case-1: ODF 4.13, with encryption enabled during creation of storagesystem
### Enabling Encryption while storagesystem creation
* ##### While creating the storagesystem tick the checkboc for enabling in-transit encryption 
```
network:
connections:
encryption:
enabled: true
```
* ##### Wait till the storagecluster becomes ready & all it's conditions to say Reconcile completed successfully
```
~ $ oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-external-storagecluster 35m Ready true 2023-05-02T06:25:09Z 4.13.0
```
### Apply settings on RHCS cluster
#### ssh into the RHCS cluster
#### Apply the in-transit encryption ceph configurations
```
[root@rhcs-2-node-1 ~]# ceph config set global ms_client_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global ms_cluster_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global ms_service_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global rbd_default_map_options ms_mode=secure
```
---
### Check settings on the RHCS cluster
#### Check encryption settings like ms_client_mode, ms_cluster_mode, ms_service_mode are present & have value secure
`https://docs.ceph.com/en/quincy/rados/configuration/msgr2/#connection-mode-configuration-options`
```
[root@rhcs-2-node-1 ~]# ceph config dump
WHO MASK LEVEL OPTION VALUE RO
global basic container_image registry.redhat.io/rhceph/rhceph-5-rhel8@sha256:3075e8708792ebd527ca14849b6af4a11256a3f881ab09b837d7af0f8b2102ea *
global advanced mon_max_pg_per_osd 1024
global basic ms_client_mode secure *
global basic ms_cluster_mode secure *
global basic ms_service_mode secure *
global advanced rbd_default_map_options ms_mode=secure *
mon advanced auth_allow_insecure_global_id_reclaim false
mon advanced mon_allow_pool_delete true
mon advanced mon_max_pg_per_osd 512
mon advanced public_network 10.1.114.0/23 *
mgr advanced mgr/cephadm/container_init True *
mgr advanced mgr/cephadm/migration_current 5 *
mgr advanced mgr/cephadm/no_five_one_rgw true *
mgr advanced mgr/dashboard/ALERTMANAGER_API_HOST http://rhcs-2-node-1:9093 *
mgr advanced mgr/dashboard/GRAFANA_API_SSL_VERIFY false *
mgr advanced mgr/dashboard/GRAFANA_API_URL https://rhcs-2-node-1:3000 *
mgr advanced mgr/dashboard/PROMETHEUS_API_HOST http://rhcs-2-node-1:9095 *
mgr advanced mgr/dashboard/RGW_API_ACCESS_KEY {"parth": "3P5MOBEZECAETGND4O5E", "multisite1": "3P5MOBEZECAETGND4O5E", "2": "3P5MOBEZECAETGND4O5E", "abc": "3P5MOBEZECAETGND4O5E", "multisit2e": "2BHJY1WBM51QVGLH149I", "abc1": "3P5MOBEZECAETGND4O5E", "realm5": "3P5MOBEZECAETGND4O5E", "realm2": "3P5MOBEZECAETGND4O5E", "realm1": "3P5MOBEZECAETGND4O5E", "rgw1": "3P5MOBEZECAETGND4O5E"} *
mgr advanced mgr/dashboard/RGW_API_SECRET_KEY {"parth": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "multisite1": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "2": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "abc": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "multisit2e": "JT62UYswADlvvpH3MVdK8oDounmda3wN3DKolIDp", "abc1": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "realm5": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "realm2": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "realm1": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2", "rgw1": "6N1ioVCCClEHTHLTXubSAAQL2XZULthZ4lF2Mtk2"} *
mgr advanced mgr/dashboard/ssl_server_port 8443 *
mgr advanced mgr/orchestrator/orchestrator cephadm
osd host:rhcs-2-node-1 basic osd_memory_target 10968240341
osd host:rhcs-2-node-2 basic osd_memory_target 11017632465
osd host:rhcs-2-node-3 basic osd_memory_target 12069899453
osd advanced osd_memory_target_autotune true
mds.cephfs basic mds_join_fs cephfs
client.rgw.multi.zone3 advanced rgw_realm multisit2e *
client.rgw.multi.zone3 advanced rgw_zone zone3 *
client.rgw.multi.zone3.rhcs-2-node-1.uumunx basic rgw_frontends beast port=8000 *
client.rgw.multi.zone3.rhcs-2-node-2.hrswgh basic rgw_frontends beast port=8000 *
client.rgw.objectgw.rhcs-2-node-1.seyimg basic rgw_frontends beast port=8080 *
```
#### Check ceph mon details if both v1 & v2 port are available
```
[root@rhcs-2-node-1 ~]# ceph mon dump
epoch 3
fsid 15b49e52-5bf5-11ed-a2e7-0050568fd833
last_changed 2022-11-04T04:02:06.587546+0000
created 2022-11-04T03:59:46.486027+0000
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:10.1.115.118:3300/0,v1:10.1.115.118:6789/0] mon.rhcs-2-node-1
1: [v2:10.1.115.119:3300/0,v1:10.1.115.119:6789/0] mon.rhcs-2-node-2
2: [v2:10.1.115.120:3300/0,v1:10.1.115.120:6789/0] mon.rhcs-2-node-3
dumped monmap epoch 3
```
#### On odf cluster check if csi is configured to use v2 3300 port
```
~ $ oc get cm rook-ceph-csi-config -oyaml
apiVersion: v1
data:
csi-cluster-config-json: '[{"clusterID":"openshift-storage","monitors":["10.1.115.118:3300","10.1.115.119:3300","10.1.115.120:3300"],"namespace":"openshift-storage"}]'
kind: ConfigMap
metadata:
creationTimestamp: "2023-05-02T06:25:14Z"
name: rook-ceph-csi-config
namespace: openshift-storage
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: false
controller: true
kind: Deployment
name: rook-ceph-operator
uid: adda64a4-a260-4612-bb90-4e8a23402d40
resourceVersion: "74934"
uid: 2baf6aff-37d2-45d2-8529-b7420b954176
```
---
### Test by creating rbd PVC ,Mounting it in a Pod & using the volume
#### Creating & Checking PVC
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: encryption-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep encryption
encryption-rbd-pvc Bound pvc-c271e73b-92a4-4710-9a6b-71a6d799830f 1Gi RWO ocs-storagecluster-ceph-rbd 67s
```
#### Creating & Checking Pod
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: encryption-rbd-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: encryption-rbd-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep encryption
encryption-rbd-pod 1/1 Running 0 2m41s
```
#### rsh into the pod and try using the mounted volume
```
$ oc rsh encryption-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
---
### Test by creating cephfs PVC, Mounting it in a Pod & checking the volume
#### Creating & Checking PVC
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: encryption-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep encryption
encryption-cephfs-pvc Bound pvc-25e473e2-a013-4707-bf09-14d99e7482f5 1Gi RWO ocs-storagecluster-cephfs 19s
encryption-rbd-pvc Bound pvc-c271e73b-92a4-4710-9a6b-71a6d799830f 1Gi RWO ocs-storagecluster-ceph-rbd 67s
```
#### Creating & Checking Pod
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: encryption-cephfs-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: encryption-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep encryption
encryption-cephfs-pod 0/1 ContainerCreating 0 43s
encryption-rbd-pod 1/1 Running 0 2m41s
```
#### rsh into the pod & try using the mounted volume
```
$ oc rsh encryption-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
## Case 2: ODF 4.13, without enabling encryption during storagesystem creation
**Important**
*If encryption is not enabled during storagesystem creation, enabling it afterwards although possible is not a officially supported operation.*
---
### Creating storagesystem without encryption
* While creating storagesystem leave the in-transit encryption tickbox unchecked.
If creating from CLI don't include encryption enabled true in Network Connections Spec.
* Wait for the storagecluster to become ready & all it's conditions to say Reconcile completed successfully
---
### Check if csi is configured to use v1 6789 port
```
~ $ oc get cm rook-ceph-csi-config -oyaml
apiVersion: v1
data:
csi-cluster-config-json: '[{"clusterID":"openshift-storage","monitors":["10.1.115.119:6789","10.1.115.120:6789","10.1.115.118:6789"],"namespace":"openshift-storage"}]'
kind: ConfigMap
metadata:
creationTimestamp: "2023-05-02T15:07:40Z"
name: rook-ceph-csi-config
namespace: openshift-storage
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: false
controller: true
kind: Deployment
name: rook-ceph-operator
uid: a223b149-da3a-43da-b81d-50216e4e0117
resourceVersion: "361936"
uid: 85969ac5-6d9e-42ec-8287-912dbaab938a
```
---
### Test by creating rbd PVC ,Mounting it in a Pod & using the volume
#### Create a rbd pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: normal-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep normal
normal-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 30s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: normal-rbd-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: normal-rbd-pvc
containers:
- name: nginx-container
image: quay.io/mparida/nginx:latest
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep normal
normal-rbd-pod 1/1 Running 0 27s
```
#### rsh into the pod & try using the mounted volume
```
~ $ oc rsh normal-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
---
### Test by creating cephfs PVC ,Mounting it in a Pod & using the volume
#### Create a cephfs pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: normal-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep normal
normal-cephfs-pvc Bound pvc-5564dfa1-58d7-4679-82e6-f5280157bbe5 1Gi RWO ocs-storagecluster-cephfs 8s
normal-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 5m15s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: normal-cephfs-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: normal-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep normal
normal-cephfs-pod 1/1 Running 0 31s
normal-rbd-pod 1/1 Running 0 6m47s
```
#### rsh into the pod & try using the mounted volume
```
~ $ oc rsh normal-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
**Important**
*The below operations to enable encryption in this case are not officially supported and are just for reference only.*
**Very Very Important**
*Enabling in-transit encryption doesnt affect the existing mapped/mounted volumes. Once a volume is mapped/mounted it will keep using the setting it was given during its mapping/mounting, doesn't matter if the encryption setting has changed on the cluster.
The new setting will affect only any new volume you create. For old volumes to use the new settings they have to be remapped/remounted again one by one.*
---
### Enable in-transit encryption
* If created from UI patch the storagecluster to enable in-transit encryption
```
oc patch storagecluster ocs-storagecluster -n openshift-storage --type json --patch '[{ "op": "replace", "path": "/spec/network/connections/encryption", "value": {"enabled": true} }]'
```
If created from CLI edit the storagecluster to add encryption enabled true in storagecluster spec
```
network:
connections:
encryption:
enabled: true
```
* Wait for the storagecluster to get ready
```
~ $ oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-storagecluster 141m Ready 2023-01-19T11:56:18Z 4.12.0
```
#### ssh into the RHCS cluster
#### Apply the in-transit encryption ceph configurations
```
[root@rhcs-2-node-1 ~]# ceph config set global ms_client_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global ms_cluster_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global ms_service_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global rbd_default_map_options ms_mode=secure
```
---
### Check settings on the RHCS cluster
#### Check encryption settings like ms_client_mode, ms_cluster_mode, ms_service_mode are present & have value secure by ceph config dump
`https://docs.ceph.com/en/quincy/rados/configuration/msgr2/#connection-mode-configuration-options`
#### Check ceph mon details if both v1 & v2 port are available
```
[root@rhcs-2-node-1 ~]# ceph mon dump
epoch 3
fsid 15b49e52-5bf5-11ed-a2e7-0050568fd833
last_changed 2022-11-04T04:02:06.587546+0000
created 2022-11-04T03:59:46.486027+0000
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:10.1.115.118:3300/0,v1:10.1.115.118:6789/0] mon.rhcs-2-node-1
1: [v2:10.1.115.119:3300/0,v1:10.1.115.119:6789/0] mon.rhcs-2-node-2
2: [v2:10.1.115.120:3300/0,v1:10.1.115.120:6789/0] mon.rhcs-2-node-3
dumped monmap epoch 3
```
#### On odf cluster check if csi is configured to use v2 3300 port
```
~ $ oc get cm rook-ceph-csi-config -oyaml
apiVersion: v1
data:
csi-cluster-config-json: '[{"clusterID":"openshift-storage","monitors":["10.1.115.118:3300","10.1.115.119:3300","10.1.115.120:3300"],"namespace":"openshift-storage"}]'
kind: ConfigMap
metadata:
creationTimestamp: "2023-05-02T06:25:14Z"
name: rook-ceph-csi-config
namespace: openshift-storage
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: false
controller: true
kind: Deployment
name: rook-ceph-operator
uid: adda64a4-a260-4612-bb90-4e8a23402d40
resourceVersion: "74934"
uid: 2baf6aff-37d2-45d2-8529-b7420b954176
```
---
### Test by creating rbd PVC ,Mounting it in a Pod & using the volume
#### Creating & Checking PVC
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: encryption-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep encryption
encryption-rbd-pvc Bound pvc-c271e73b-92a4-4710-9a6b-71a6d799830f 1Gi RWO ocs-storagecluster-ceph-rbd 67s
```
#### Creating & Checking Pod
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: encryption-rbd-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: encryption-rbd-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep encryption
encryption-rbd-pod 1/1 Running 0 2m41s
```
#### rsh into the pod and try using the mounted volume
```
$ oc rsh encryption-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
---
### Test by creating cephfs PVC, Mounting it in a Pod & checking the volume
#### Creating & Checking PVC
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: encryption-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep encryption
encryption-cephfs-pvc Bound pvc-25e473e2-a013-4707-bf09-14d99e7482f5 1Gi RWO ocs-storagecluster-cephfs 19s
encryption-rbd-pvc Bound pvc-c271e73b-92a4-4710-9a6b-71a6d799830f 1Gi RWO ocs-storagecluster-ceph-rbd 67s
```
#### Creating & Checking Pod
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: encryption-cephfs-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: encryption-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep encryption
encryption-cephfs-pod 0/1 ContainerCreating 0 43s
encryption-rbd-pod 1/1 Running 0 2m41s
```
#### rsh into the pod & try using the mounted volume
```
$ oc rsh encryption-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
## Case-3: An cluster is upgraded from ODF 4.12 to ODF 4.13
### Setting up odf 4.12
#### Install ODF 4.12 by choosing the appropriate subscription channel
```
~ $ oc get csv
NAME DISPLAY VERSION REPLACES PHASE
mcg-operator.v4.12.2-rhodf NooBaa Operator 4.12.2-rhodf mcg-operator.v4.12.1 Succeeded
ocs-operator.v4.12.2-rhodf OpenShift Container Storage 4.12.2-rhodf ocs-operator.v4.12.1 Succeeded
odf-csi-addons-operator.v4.12.2-rhodf CSI Addons 4.12.2-rhodf odf-csi-addons-operator.v4.12.1 Succeeded
odf-operator.v4.12.2-rhodf OpenShift Data Foundation 4.12.2-rhodf odf-operator.v4.12.1 Succeeded
```
#### Create a storagecluster & wait for it to be ready & all it's conditions to say Reconcile completed successfully
```
~ $ oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-external-storagecluster 11m Ready true 2023-05-02T16:11:14Z 4.12.0
```
---
### Create some workload to test after upgrade
* #### Create a rbd pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: 4-12-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep 4-12
4-12-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 30s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: 4-12-rbd-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: 4-12-rbd-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep 4-12
4-12-rbd-pod 1/1 Running 0 27s
```
* #### rsh into the pod & create a file on the volume
```
~ $ oc rsh 4-12-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
* #### Create a cephfs pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: 4-12-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep 4-12
4-12-cephfs-pvc Bound pvc-5564dfa1-58d7-4679-82e6-f5280157bbe5 1Gi RWO ocs-storagecluster-cephfs 8s
4-12-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 5m15s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: 4-12-cephfs-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: 4-12-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep 4-12
4-12-cephfs-pod 1/1 Running 0 31s
4-12-rbd-pod 1/1 Running 0 6m47s
```
* #### rsh into the pod & create a file on the volume
```
~ $ oc rsh 4-12-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
### Upgrade to ODF 4.13
#### Wait for the upgrade to complete & the storagecluster to get ready
```
~ $ oc get csv
NAME DISPLAY VERSION REPLACES PHASE
mcg-operator.v4.13.0-179.stable NooBaa Operator 4.13.0-179.stable mcg-operator.v4.12.2-rhodf Succeeded
ocs-operator.v4.13.0-179.stable OpenShift Container Storage 4.13.0-179.stable ocs-operator.v4.12.2-rhodf Succeeded
odf-csi-addons-operator.v4.13.0-179.stable CSI Addons 4.13.0-179.stable odf-csi-addons-operator.v4.12.2-rhodf Succeeded
odf-operator.v4.13.0-179.stable OpenShift Data Foundation 4.13.0-179.stable odf-operator.v4.12.2-rhodf Succeeded
```
```
~ $ oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-external-storagecluster 15m Ready true 2023-05-03T06:48:57Z 4.13.0
```
---
#### Check if existing workload is usable
```
~ $ oc rsh 4-12-rbd-pod
# cd /usr/share/nginx/html
# ls
a lost+found
# touch b
# ls
a b lost+found
# sync && exit
```
```
~ $ oc rsh 4-12-cephfs-pod
# cd /usr/share/nginx/html
# ls
a
# touch b
# ls
a b
# sync && exit
```
#### On odf cluster check if csi is configured to use v1 6789 port
```
~ $ oc get cm rook-ceph-csi-config -oyaml
apiVersion: v1
data:
csi-cluster-config-json: '[{"clusterID":"openshift-storage","monitors":["10.1.115.118:6789","10.1.115.119:6789","10.1.115.120:6789"],"namespace":"openshift-storage"}]'
kind: ConfigMap
metadata:
creationTimestamp: "2023-05-03T06:48:57Z"
name: rook-ceph-csi-config
namespace: openshift-storage
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: false
controller: true
kind: Deployment
name: rook-ceph-operator
uid: 04dd8823-d45e-44d6-8f3e-31b56e2fe2e9
resourceVersion: "96880"
uid: 0e0ad0db-1f03-4126-922a-f2c09f995833
```
---
### Test by creating rbd PVC ,Mounting it in a Pod & using the volume
#### Create a rbd pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: normal-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep normal
normal-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 30s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: normal-rbd-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: normal-rbd-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep normal
normal-rbd-pod 1/1 Running 0 27s
```
#### rsh into the pod & try using the mounted volume
```
~ $ oc rsh normal-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
---
### Test by creating cephfs PVC ,Mounting it in a Pod & using the volume
#### Create a cephfs pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: normal-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep normal
normal-cephfs-pvc Bound pvc-5564dfa1-58d7-4679-82e6-f5280157bbe5 1Gi RWO ocs-storagecluster-cephfs 8s
normal-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 5m15s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: normal-cephfs-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: normal-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep normal
normal-cephfs-pod 1/1 Running 0 31s
normal-rbd-pod 1/1 Running 0 6m47s
```
#### rsh into the pod & try using the mounted volume
```
~ $ oc rsh normal-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
**Important**
*The below operations to enable encryption in this case are not officially supported and are just for reference only.*
**Very Very Important**
*Enabling in-transit encryption doesnt affect the existing mapped/mounted volumes. Once a volume is mapped/mounted it will keep using the setting it was given during its mapping/mounting, doesn't matter if the encryption setting has changed on the cluster.
The new setting will affect only any new volume you create. For old volumes to use the new settings they have to be remapped/remounted again one by one.*
---
### Enable in-transit encryption
* Edit the storagecluster to add encryption enabled true in storagecluster spec
```
network:
connections:
encryption:
enabled: true
```
* Check if storagecluster is ready
```
~ $ oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-external-storagecluster 41m Ready true 2023-05-03T06:48:57Z 4.13.0
```
#### ssh into the RHCS cluster
#### Apply the in-transit encryption ceph configurations
```
[root@rhcs-2-node-1 ~]# ceph config set global ms_client_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global ms_cluster_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global ms_service_mode secure
[root@rhcs-2-node-1 ~]# ceph config set global rbd_default_map_options ms_mode=secure
```
---
### Check settings on the RHCS cluster
#### Check encryption settings like ms_client_mode, ms_cluster_mode, ms_service_mode are present & have value secure by doping ceph config dump
`https://docs.ceph.com/en/quincy/rados/configuration/msgr2/#connection-mode-configuration-options`
#### Check ceph mon details if both v1 & v2 port are available
```
[root@rhcs-2-node-1 ~]# ceph mon dump
epoch 3
fsid 15b49e52-5bf5-11ed-a2e7-0050568fd833
last_changed 2022-11-04T04:02:06.587546+0000
created 2022-11-04T03:59:46.486027+0000
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:10.1.115.118:3300/0,v1:10.1.115.118:6789/0] mon.rhcs-2-node-1
1: [v2:10.1.115.119:3300/0,v1:10.1.115.119:6789/0] mon.rhcs-2-node-2
2: [v2:10.1.115.120:3300/0,v1:10.1.115.120:6789/0] mon.rhcs-2-node-3
```
#### On odf cluster check if csi is configured to use v2 3300 port
```
~ $ oc get cm rook-ceph-csi-config -oyaml
apiVersion: v1
data:
csi-cluster-config-json: '[{"clusterID":"openshift-storage","monitors":["10.1.115.118:3300","10.1.115.119:3300","10.1.115.120:3300"],"namespace":"openshift-storage"}]'
kind: ConfigMap
metadata:
creationTimestamp: "2023-05-02T06:25:14Z"
name: rook-ceph-csi-config
namespace: openshift-storage
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: false
controller: true
kind: Deployment
name: rook-ceph-operator
uid: adda64a4-a260-4612-bb90-4e8a23402d40
resourceVersion: "74934"
uid: 2baf6aff-37d2-45d2-8529-b7420b954176
```
---
### Test by creating rbd PVC ,Mounting it in a Pod & using the volume
#### Creating & Checking PVC
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: encryption-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep encryption
encryption-rbd-pvc Bound pvc-c271e73b-92a4-4710-9a6b-71a6d799830f 1Gi RWO ocs-storagecluster-ceph-rbd 67s
```
#### Creating & Checking Pod
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: encryption-rbd-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: encryption-rbd-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep encryption
encryption-rbd-pod 1/1 Running 0 2m41s
```
#### rsh into the pod and try using the mounted volume
```
$ oc rsh encryption-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
---
### Test by creating cephfs PVC, Mounting it in a Pod & using the volume
#### Creating & Checking PVC
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: encryption-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-external-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep encryption
encryption-cephfs-pvc Bound pvc-25e473e2-a013-4707-bf09-14d99e7482f5 1Gi RWO ocs-storagecluster-cephfs 19s
encryption-rbd-pvc Bound pvc-c271e73b-92a4-4710-9a6b-71a6d799830f 1Gi RWO ocs-storagecluster-ceph-rbd 67s
```
#### Creating & Checking Pod
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: encryption-cephfs-pod
spec:
nodeSelector:
kubernetes.io/hostname: compute-0
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: encryption-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep encryption
encryption-cephfs-pod 0/1 ContainerCreating 0 43s
encryption-rbd-pod 1/1 Running 0 2m41s
```
#### rsh into the pod & try using the mounted volume
```
$ oc rsh encryption-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
### Verify if the earlied created normal volumes are accesible
```
~ $ oc rsh normal-rbd-pod
# cd /usr/share/nginx/html
# ls
a lost+found
# touch b
# ls
a b lost+found
# sync && exit
```
```
~ $ oc rsh normal-cephfs-pod
# cd /usr/share/nginx/html
# ls
a
# touch b
# ls
a b
# sync && exit
```
---
### Verify if the earlied created 4-12 volumes are accesible
```
~ $ oc rsh 4-12-rbd-pod
# cd /usr/share/nginx/html
# ls
a b lost+found
# touch c
# ls
a b c lost+found
# sync && exit
```
```
~ $ oc rsh 4-12-cephfs-pod
# cd /usr/share/nginx/html
# ls
a b
# touch c
# ls
a b c
# sync && exit
```
---
---
## Case 4: Disabling in-transit encryption
(This case should be tested after case-1 not after other cases)
**Important**
This is not a officially supported action. If someone wants to don't use in-transit encryption anymore, They should delete the storagecluster & recreate another one without encryption.
**Important**
Removing the encryption settings/disabling encryption doesnt affect the existing mapped/mounted volumes. Once a volume is mapped/mounted it will keep using the setting it was given during its mapping/mounting, doesn't matter if the encryption setting has changed on the cluster. The new setting will affect only any new volume you create. For old volumes to use the new settings they have to be remapped/remounted again one by one.
---
### Disabling in-transit encryption
#### Putting the encryption enabled flag to false
```
oc patch storagecluster ocs-external-storagecluster -n openshift-storage --type json --patch '[{ "op": "replace", "path": "/spec/network/connections/encryption", "value": {"enabled": false} }]'
```
#### Check if the storagecluster is ready
```
~ $ oc get storagecluster
NAME AGE PHASE EXTERNAL CREATED AT VERSION
ocs-external-storagecluster 66m Ready true 2023-05-03T06:48:57Z 4.13.0
```
#### ssh into the RHCS cluster
#### Remove all the in-transit encryption related settings
```
[root@rhcs-2-node-1 ~]# ceph config rm global ms_client_mode
[root@rhcs-2-node-1 ~]# ceph config rm global ms_cluster_mode
[root@rhcs-2-node-1 ~]# ceph config rm global ms_service_mode
[root@rhcs-2-node-1 ~]# ceph config rm global rbd_default_map_options
```
#### Check ceph config to ensure settings like ms_client_mode, ms_cluster_mode, ms_service_mode are removed by doing ceph config dump
### rsh into both rbd & cephfs pod to check the volumes
```
$ oc rsh encryption-rbd-pod
# cd /usr/share/nginx/html
# ls
a lost+found
# touch b
# ls
a b lost+found
#
```
```
$ oc rsh encryption-cephfs-pod
# cd /usr/share/nginx/html
# ls
a
# touch b
# ls
a b
#
```
---
### For existing workload to stop using the in-transit encryption the volumes have to be remapped/remounted.
---
### Test by creating rbd PVC ,Mounting it in a Pod & using the volume
#### Create a rbd pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: normal-rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-storagecluster-ceph-rbd
EOF
```
```
~ $ oc get pvc | grep normal
normal-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 30s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: normal-rbd-pod
spec:
nodeSelector:
topology.kubernetes.io/zone: us-east-1a
volumes:
- name: rbd-storage
persistentVolumeClaim:
claimName: normal-rbd-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: rbd-storage
EOF
```
```
~ $ oc get pods | grep normal
normal-rbd-pod 1/1 Running 0 27s
```
#### rsh into the pod & try using the mounted volume
```
~ $ oc rsh normal-rbd-pod
# cd /usr/share/nginx/html
# ls
lost+found
# touch a
# ls
a lost+found
# sync && exit
```
---
### Test by creating cephfs PVC ,Mounting it in a Pod & using the volume
#### Create a cephfs pvc & a pod to use it
```
~ $ cat <<EOF | oc create -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: normal-cephfs-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ocs-storagecluster-cephfs
EOF
```
```
~ $ oc get pvc | grep normal
normal-cephfs-pvc Bound pvc-5564dfa1-58d7-4679-82e6-f5280157bbe5 1Gi RWO ocs-storagecluster-cephfs 8s
normal-rbd-pvc Bound pvc-2001b9a3-0d28-475a-9c4f-56d820c331b5 1Gi RWO ocs-storagecluster-ceph-rbd 5m15s
```
```
~ % cat <<EOF | oc create -f -
apiVersion: v1
kind: Pod
metadata:
name: normal-cephfs-pod
spec:
nodeSelector:
topology.kubernetes.io/zone: us-east-1a
volumes:
- name: cephfs-storage
persistentVolumeClaim:
claimName: normal-cephfs-pvc
containers:
- name: nginx-container
image: nginx
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/usr/share/nginx/html"
name: cephfs-storage
EOF
```
```
~ $ oc get pods | grep normal
normal-cephfs-pod 1/1 Running 0 31s
normal-rbd-pod 1/1 Running 0 6m47s
```
#### rsh into the pod & try using the mounted volume
```
~ $ oc rsh normal-cephfs-pod
# cd /usr/share/nginx/html
# ls
# touch a
# ls
a
# sync && exit
```
---
Sign in with Wallet
Connect another wallet