# OADP 3866

## cluster

oc version
Client Version: 4.12.0-202301042354.p0.g854f807.assembly.stream-854f807
Kustomize Version: v4.5.7
Server Version: 4.14.14
Kubernetes Version: v1.27.10+28ed2d7
whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scrip

## setup

 ansible-playbook mtc-helpers/data-generator/playbooks/playbook_case3.yml --extra-vars 'dir_count=30 files_count=230 files_size=307200 dept_count=1 pvc_size=6Gi dataset_path=/opt/mounts/mnt1/ namespace=perf-datagen-case-0-0-7-cephrbd sc=gp3-csi deployment_name=deploy-perf-datagen-0-0-7-6gi-10-rbd-9 pvc_name=pvc-perf-datagen-0-0-7-6gi-10-rbd-9'
[WARNING]: provided hosts list is empty, only localhost is available. Note that the implicit localhost does not match 'all'
[WARNING]: Found variable using reserved name: namespace

PLAY [localhost] ******************************************************************************************************************************************************************************************************************************

TASK [Gathering Facts] ************************************************************************************************************************************************************************************************************************
ok: [localhost]

TASK [Creating workload namespace] ************************************************************************************************************************************************************************************************************
ok: [localhost]

TASK [pause] **********************************************************************************************************************************************************************************************************************************
Pausing for 5 seconds
(ctrl+C then 'C' = continue early, ctrl+C then 'A' = abort)
ok: [localhost]

TASK [Deploying workload] *********************************************************************************************************************************************************************************************************************
changed: [localhost]

PLAY RECAP ************************************************************************************************************************************************************************************************************************************
localhost                  : ok=4    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   

### oc get all

whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scripts$ oc get all
Warning: apps.openshift.io/v1 DeploymentConfig is deprecated in v4.14+, unavailable in v4.10000+
NAME                                                          READY   STATUS    RESTARTS   AGE
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88   1/1     Running   0          18m
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x   1/1     Running   0          4m42s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt     1/1     Running   0          3m59s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw    1/1     Running   0          3m32s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4    1/1     Running   0          3m17s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk    1/1     Running   0          2m50s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h   1/1     Running   0          2m36s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd   1/1     Running   0          2m16s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl   1/1     Running   0          118s
pod/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m   1/1     Running   0          100s

NAME                                                     READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-0   1/1     1            1           18m
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-1   1/1     1            1           4m42s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-2   1/1     1            1           3m59s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-3   1/1     1            1           3m32s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-4   1/1     1            1           3m17s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-5   1/1     1            1           2m50s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-6   1/1     1            1           2m36s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-7   1/1     1            1           2m16s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-8   1/1     1            1           118s
deployment.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-9   1/1     1            1           100s

NAME                                                                DESIRED   CURRENT   READY   AGE
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5   1         1         1       18m
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698   1         1         1       4m42s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77     1         1         1       3m59s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8    1         1         1       3m32s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7    1         1         1       3m17s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd    1         1         1       2m50s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc   1         1         1       2m36s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b   1         1         1       2m16s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9   1         1         1       118s
replicaset.apps/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d   1         1         1       100s



whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scripts$ oc get pvc
NAME                                  STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
pvc-perf-datagen-0-0-7-6gi-10-rbd-0   Bound    pvc-daf818ed-8737-45a5-993c-c979d2207bd8   6Gi        RWO            gp3-csi        18m
pvc-perf-datagen-0-0-7-6gi-10-rbd-1   Bound    pvc-7f7e9dac-2743-4f5d-8250-bb615f06e73c   6Gi        RWO            gp3-csi        5m1s
pvc-perf-datagen-0-0-7-6gi-10-rbd-2   Bound    pvc-f842028b-c533-4a4f-ab0e-988692246cd7   6Gi        RWO            gp3-csi        4m17s
pvc-perf-datagen-0-0-7-6gi-10-rbd-3   Bound    pvc-6abe9b54-2ed1-4413-b3e0-1f0fe572d6b9   6Gi        RWO            gp3-csi        3m51s
pvc-perf-datagen-0-0-7-6gi-10-rbd-4   Bound    pvc-68cf3ad8-f0f7-4e8d-bb86-d64b4a708723   6Gi        RWO            gp3-csi        3m36s
pvc-perf-datagen-0-0-7-6gi-10-rbd-5   Bound    pvc-6bba1e0c-d340-43f5-a496-873a08679610   6Gi        RWO            gp3-csi        3m9s
pvc-perf-datagen-0-0-7-6gi-10-rbd-6   Bound    pvc-884e4ed5-4667-4d66-b8ff-ff7d44ef0d69   6Gi        RWO            gp3-csi        2m54s
pvc-perf-datagen-0-0-7-6gi-10-rbd-7   Bound    pvc-13014b76-0451-43e2-868b-474dab5aea08   6Gi        RWO            gp3-csi        2m34s
pvc-perf-datagen-0-0-7-6gi-10-rbd-8   Bound    pvc-e57cbace-bc58-49d0-be6e-e5d6e6f97e4f   6Gi        RWO            gp3-csi        2m16s
pvc-perf-datagen-0-0-7-6gi-10-rbd-9   Bound    pvc-1ef234de-7be6-460b-b989-7c8676c4b242   6Gi        RWO            gp3-csi        118s
whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scripts$ oc get pv | grep perf
pvc-13014b76-0451-43e2-868b-474dab5aea08   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-7   gp3-csi                 2m40s
pvc-1ef234de-7be6-460b-b989-7c8676c4b242   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-9   gp3-csi                 2m5s
pvc-68cf3ad8-f0f7-4e8d-bb86-d64b4a708723   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-4   gp3-csi                 3m42s
pvc-6abe9b54-2ed1-4413-b3e0-1f0fe572d6b9   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-3   gp3-csi                 3m57s
pvc-6bba1e0c-d340-43f5-a496-873a08679610   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-5   gp3-csi                 3m15s
pvc-7f7e9dac-2743-4f5d-8250-bb615f06e73c   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-1   gp3-csi                 5m7s
pvc-884e4ed5-4667-4d66-b8ff-ff7d44ef0d69   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-6   gp3-csi                 3m
pvc-daf818ed-8737-45a5-993c-c979d2207bd8   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-0   gp3-csi                 18m
pvc-e57cbace-bc58-49d0-be6e-e5d6e6f97e4f   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-8   gp3-csi                 2m23s
pvc-f842028b-c533-4a4f-ab0e-988692246cd7   6Gi        RWO            Delete           Bound    perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-2   gp3-csi                 4m24s
whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scripts$ 

## dpa

apiVersion: oadp.openshift.io/v1alpha1
kind: DataProtectionApplication
metadata:
  creationTimestamp: "2024-04-10T17:13:59Z"
  generation: 1
  name: dpa-sample
  namespace: openshift-adp
  resourceVersion: "14057854"
  uid: 0b4652b7-7978-42b8-b29f-7591646c03f5
spec:
  backupLocations:
  - name: default
    velero:
      config:
        profile: default
        region: us-west-2
      credential:
        key: cloud
        name: cloud-credentials
      default: true
      objectStorage:
        bucket: cvpbucketuswest2
        prefix: velero
      provider: aws
  configuration:
    nodeAgent:
      enable: true
      uploaderType: kopia
    velero:
      defaultPlugins:
      - openshift
      - aws
      - csi
  snapshotLocations:
  - velero:
      config:
        profile: default
        region: us-west-2
      provider: aws
status:
  conditions:
  - lastTransitionTime: "2024-04-10T17:13:59Z"
    message: Reconcile complete
    reason: Complete
    status: "True"
    type: Reconciled

## backup KOPIA

apiVersion: velero.io/v1
kind: Backup
metadata:
  name: perf-datagen-case-0-0-7-cephrbd-kopia-1
  namespace: openshift-adp
spec:
  includedNamespaces:
  - perf-datagen-case-0-0-7-cephrbd
  storageLocation: default
  ttl: 720h0m0s
  snapshotMoveData: False
  defaultVolumesToFsBackup: True

## backup success

Name:         perf-datagen-case-0-0-7-cephrbd-kopia-1
Namespace:    openshift-adp
Labels:       velero.io/storage-location=default
Annotations:  velero.io/resource-timeout=10m0s
              velero.io/source-cluster-k8s-gitversion=v1.27.10+28ed2d7
              velero.io/source-cluster-k8s-major-version=1
              velero.io/source-cluster-k8s-minor-version=27

Phase:  Completed


Namespaces:
  Included:  perf-datagen-case-0-0-7-cephrbd
  Excluded:  <none>

Resources:
  Included:        *
  Excluded:        <none>
  Cluster-scoped:  auto

Label selector:  <none>

Or label selector:  <none>

Storage Location:  default

Velero-Native Snapshot PVs:  auto
Snapshot Move Data:          false
Data Mover:                  velero

TTL:  720h0m0s

CSISnapshotTimeout:    10m0s
ItemOperationTimeout:  4h0m0s

Hooks:  <none>

Backup Format Version:  1.1.0

Started:    2024-04-10 17:52:32 +0000 UTC
Completed:  2024-04-10 17:57:40 +0000 UTC

Expiration:  2024-05-10 17:52:32 +0000 UTC

Total items to be backed up:  201
Items backed up:              201

Resource List:
  apiextensions.k8s.io/v1/CustomResourceDefinition:
    - clusterserviceversions.operators.coreos.com
  apps/v1/Deployment:
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9
  apps/v1/ReplicaSet:
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d
  authorization.openshift.io/v1/RoleBinding:
    - perf-datagen-case-0-0-7-cephrbd/system:deployers
    - perf-datagen-case-0-0-7-cephrbd/system:image-builders
    - perf-datagen-case-0-0-7-cephrbd/system:image-pullers
  operators.coreos.com/v1alpha1/ClusterServiceVersion:
    - perf-datagen-case-0-0-7-cephrbd/volsync-product.v0.7.4-0.1698026108.p
  rbac.authorization.k8s.io/v1/RoleBinding:
    - perf-datagen-case-0-0-7-cephrbd/system:deployers
    - perf-datagen-case-0-0-7-cephrbd/system:image-builders
    - perf-datagen-case-0-0-7-cephrbd/system:image-pullers
  v1/ConfigMap:
    - perf-datagen-case-0-0-7-cephrbd/kube-root-ca.crt
    - perf-datagen-case-0-0-7-cephrbd/openshift-service-ca.crt
    - perf-datagen-case-0-0-7-cephrbd/settings
  v1/Event:
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb7e59bc9378
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb7ee678d52c
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb7f86da5b27
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb7f8943ef76
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb85d38c9ba8
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb85dc6d5be2
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88.17c4fb85dddae6a0
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5.17c4fb7d69d14f9e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0.17c4fb7d67cf2d9b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc3e23967e8d
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc3ed66e123e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc4038d27e5e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc403b62626a
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc406b1a8306
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc407ab4d516
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x.17c4fc407c5863a3
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698.17c4fc3d33c91b65
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1.17c4fc3d3179d34e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc4831de483d
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc48bc96ad5e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc49b064129b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc49b34422e7
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc49de279d3d
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc49e6710b33
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt.17c4fc49e7d55520
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77.17c4fc4741b83d8b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2.17c4fc47403349ce
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc4e75450922
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc4f0164712f
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc50f5e22207
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc50f8f1bdd0
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc51254640e6
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc512f599298
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw.17c4fc513155912e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8.17c4fc4d8484e170
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3.17c4fc4d82c9c7dc
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc51edae208f
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc527ab7a139
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc53c221a4bd
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc53c49ec159
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc53f550bea4
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc5401507792
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4.17c4fc540328260c
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7.17c4fc50fd585d2c
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4.17c4fc50fb83384e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc5839e9307a
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc58c386f322
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc5af7c3fff3
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc5afaa41c99
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc5b27ffb6b7
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc5b31b0df24
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk.17c4fc5b3362b598
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd.17c4fc574a681b16
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5.17c4fc57475a85f6
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc5baa853491
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc5c3a70b96b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc5ce0195d42
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc5ce3953516
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc6138bdf294
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc6141af5f8b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h.17c4fc6143922ed9
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc.17c4fc5ab665bf1d
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6.17c4fc5ab4683948
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc6048cca1fa
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc60d1e294e8
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc620fc551df
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc621236ec41
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc65cee053e7
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc65d6605814
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd.17c4fc65d7c9aed2
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b.17c4fc5f591b54b2
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7.17c4fc5f551e6e7e
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc6464561089
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc64ee4aa2ec
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc66e34a571d
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc66e5854227
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc671222cecd
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc671b1d08a0
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl.17c4fc671c76d833
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9.17c4fc63731ec76d
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8.17c4fc63712931b2
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc68968c79a1
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc6922e72382
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc6b30b2df6a
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc6b330886c4
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc6b6323256b
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc6b6abf916f
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m.17c4fc6b6c578e15
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d.17c4fc67a5ba6ec8
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9.17c4fc67a3c20720
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-0.17c4fb7d5d045415
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-0.17c4fb7d6b2dc6d0
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-0.17c4fb7d6b483b03
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-0.17c4fb7e349385e5
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-1.17c4fc3d26f5eb21
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-1.17c4fc3d35516bdf
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-1.17c4fc3d357aa9b4
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-1.17c4fc3e00ef9968
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-2.17c4fc4734a75538
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-2.17c4fc47435dc6a4
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-2.17c4fc47436a6130
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-2.17c4fc480b967eef
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-3.17c4fc4d77b7dad2
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-3.17c4fc4d862a499d
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-3.17c4fc4d862edde1
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-3.17c4fc4e4f17e303
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-4.17c4fc50eeed1710
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-4.17c4fc50ff3b11dd
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-4.17c4fc50ff54bf77
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-4.17c4fc51c9033652
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-5.17c4fc573c687a33
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-5.17c4fc574b9a66d7
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-5.17c4fc574baefc45
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-5.17c4fc581343c548
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-6.17c4fc5aa9577fc5
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-6.17c4fc5abc1942d1
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-6.17c4fc5abc30f539
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-6.17c4fc5b8631603c
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-7.17c4fc5f4a1e532a
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-7.17c4fc5f5a7eeb5c
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-7.17c4fc5f5a907f1f
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-7.17c4fc6028874482
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-8.17c4fc63656c9c36
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-8.17c4fc63761a31d7
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-8.17c4fc63762a9757
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-8.17c4fc643db53148
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-9.17c4fc6798cde831
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-9.17c4fc67a81a1b94
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-9.17c4fc67a84d5321
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-9.17c4fc68712fc88c
  v1/Namespace:
    - perf-datagen-case-0-0-7-cephrbd
  v1/PersistentVolume:
    - pvc-13014b76-0451-43e2-868b-474dab5aea08
    - pvc-1ef234de-7be6-460b-b989-7c8676c4b242
    - pvc-68cf3ad8-f0f7-4e8d-bb86-d64b4a708723
    - pvc-6abe9b54-2ed1-4413-b3e0-1f0fe572d6b9
    - pvc-6bba1e0c-d340-43f5-a496-873a08679610
    - pvc-7f7e9dac-2743-4f5d-8250-bb615f06e73c
    - pvc-884e4ed5-4667-4d66-b8ff-ff7d44ef0d69
    - pvc-daf818ed-8737-45a5-993c-c979d2207bd8
    - pvc-e57cbace-bc58-49d0-be6e-e5d6e6f97e4f
    - pvc-f842028b-c533-4a4f-ab0e-988692246cd7
  v1/PersistentVolumeClaim:
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-0
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-1
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-2
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-3
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-4
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-5
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-6
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-7
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-8
    - perf-datagen-case-0-0-7-cephrbd/pvc-perf-datagen-0-0-7-6gi-10-rbd-9
  v1/Pod:
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl
    - perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m
  v1/Secret:
    - perf-datagen-case-0-0-7-cephrbd/builder-dockercfg-27mtm
    - perf-datagen-case-0-0-7-cephrbd/builder-token-fh5l2
    - perf-datagen-case-0-0-7-cephrbd/default-dockercfg-vzdb7
    - perf-datagen-case-0-0-7-cephrbd/default-token-dtctj
    - perf-datagen-case-0-0-7-cephrbd/deployer-dockercfg-st6dx
    - perf-datagen-case-0-0-7-cephrbd/deployer-token-fmpp8
  v1/ServiceAccount:
    - perf-datagen-case-0-0-7-cephrbd/builder
    - perf-datagen-case-0-0-7-cephrbd/default
    - perf-datagen-case-0-0-7-cephrbd/deployer

Velero-Native Snapshots: <none included>

kopia Backups:
  Completed:
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m: vol-0
whayutin@thinkdoe:/tmp/oadp_tmp$ 

### clean up

whayutin@thinkdoe:/tmp/oadp_tmp$ oc project
Using project "perf-datagen-case-0-0-7-cephrbd" on server "https://api.cluster-wdh03202024a.wdh03202024a.mg.dog8code.com:6443".
whayutin@thinkdoe:/tmp/oadp_tmp$ oc delete namespace perf-datagen-case-0-0-7-cephrbd
namespace "perf-datagen-case-0-0-7-cephrbd" deleted
whayutin@thinkdoe:/tmp/oadp_tmp$ 

whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scripts$ oc get pv | grep perf
whayutin@thinkdoe:~/OPENSHIFT/git/QE/mpqe-scale-scripts$ 

## restore

whayutin@thinkdoe:/tmp/oadp_tmp$ cat restore_perf-datagen-case-0-0-7-cephrbd-kopia-1.yaml
apiVersion: velero.io/v1
kind: Restore
metadata:
  name: restore-perf-datagen-case-0-0-7-cephrbd-kopia-1
  namespace: openshift-adp
spec:
  backupName: perf-datagen-case-0-0-7-cephrbd-kopia-1
  restorePVs: true

Name:         restore-perf-datagen-case-0-0-7-cephrbd-kopia-1
Namespace:    openshift-adp
Labels:       <none>
Annotations:  <none>

Phase:                                 InProgress
Estimated total items to be restored:  71
Items restored so far:                 71

Started:    2024-04-10 18:03:56 +0000 UTC
Completed:  <n/a>

Backup:  perf-datagen-case-0-0-7-cephrbd-kopia-1

Namespaces:
  Included:  all namespaces found in the backup
  Excluded:  <none>

Resources:
  Included:        *
  Excluded:        nodes, events, events.events.k8s.io, backups.velero.io, restores.velero.io, resticrepositories.velero.io, csinodes.storage.k8s.io, volumeattachments.storage.k8s.io, backuprepositories.velero.io
  Cluster-scoped:  auto

Namespace mappings:  <none>

Label selector:  <none>

Or label selector:  <none>

Restore PVs:  true

kopia Restores:
  Failed:
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-0-65b864c6c5-mgk88: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-8-665844fbf9-7xbhl: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-9-7c6f7d5c9d-9lp6m: vol-0
  New:
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-1-584dd89698-nfv8x: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-2-cdcc4b77-f8pmt: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-3-cc74c9fd8-69tbw: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-4-9bffb6cb7-9rfd4: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-5-775b87dfd-lzgsk: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-6-7fd478d7fc-db46h: vol-0
    perf-datagen-case-0-0-7-cephrbd/deploy-perf-datagen-0-0-7-6gi-10-rbd-7-685476bc9b-x5gzd: vol-0

Existing Resource Policy:   <none>
ItemOperationTimeout:       4h0m0s

Preserve Service NodePorts:  auto

Resource List:  <restore resource list not found>

## LOGS
* https://termbin.com/bje9
* 3 successful backups prior to perf-datagen; search for "perf-datagen"

### Errors

* node-agent-2r9gb node-agent time="2024-04-10T18:04:37Z" level=error msg="Async fs restore data path failed" controller=PodVolumeRestore error="Failed to run kopia restore: Failed to copy snapshot data to the target: restore error: copy file: error creating file: open /host_pods/71426e7d-4fc1-491e-ad9d-8b572d1f9dfa/volumes/kubernetes.io~csi/pvc-7094b570-6771-488f-a4be-d1140074fecd/mount/python/my-dir-name-5/my-random-file-151.data: no such file or directory" error.file="/remote-source/velero/deps/gomod/pkg/mod/github.com/project-velero/kopia@v0.13.0-velero.1/snapshot/restore/local_fs_output.go:153" error.function="github.com/kopia/kopia/snapshot/restore.(*FilesystemOutput).WriteFile" logSource="/remote-source/velero/app/pkg/controller/pod_volume_restore_controller.go:326" pvr=restore-perf-datagen-case-0-0-7-cephrbd-kopia-1-7glrp

* node-agent-c4gpl node-agent time="2024-04-10T18:04:33Z" level=error msg="Async fs restore data path failed" controller=PodVolumeRestore error="Failed to run kopia restore: Failed to copy snapshot data to the target: restore error: copy file: error creating file: open /host_pods/5a7e627e-816a-43ba-b0c4-a213cf6e33ff/volumes/kubernetes.io~csi/pvc-b7cf5fc2-a6da-43f5-a649-30548b47a73c/mount/python/my-dir-name-3/my-random-file-184.data: no such file or directory" error.file="/remote-source/velero/deps/gomod/pkg/mod/github.com/project-velero/kopia@v0.13.0-velero.1/snapshot/restore/local_fs_output.go:153" error.function="github.com/kopia/kopia/snapshot/restore.(*FilesystemOutput).WriteFile" logSource="/remote-source/velero/app/pkg/controller/pod_volume_restore_controller.go:326" pvr=restore-perf-datagen-case-0-0-7-cephrbd-kopia-1-7zzx6