Merge pull request #16350 from rook/mergify/bp/release-1.18/pr-16320

csi: make csi-operator default deployment (backport #16320)

.github/workflows/canary-integration-test.yml

@@ -842,6 +842,7 @@ jobs:
         run: |
           yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true"
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
@@ -876,6 +877,7 @@ jobs:
       - name: redeploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
 
@@ -939,6 +941,7 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
           cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -992,6 +995,7 @@ jobs:
       - name: deploy rook
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
           cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -1058,6 +1062,7 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -1089,6 +1094,7 @@ jobs:
       - name: redeploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -1153,6 +1159,7 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
@@ -1208,6 +1215,7 @@ jobs:
       - name: deploy rook
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -1278,12 +1286,12 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
           yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
-          kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}'
           yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
           yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2
           kubectl create -f tests/manifests/test-object.yaml
@@ -1369,11 +1377,11 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
-          kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}'
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
 
       - name: wait for prepare pod
@@ -1438,6 +1446,7 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
           yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false
           kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
@@ -1889,6 +1898,8 @@ jobs:
       - name: deploy cluster
         run: |
           tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
+          tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml
+
 
           export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
           yq write -i deploy/examples/cluster-multus-test.yaml "spec.storage.deviceFilter" "${BLOCK}"

Documentation/Helm-Charts/operator-chart.md

@@ -129,7 +129,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.registrar.tag` | Registrar image tag | `"v2.13.0"` |
 | `csi.resizer.repository` | Kubernetes CSI resizer image repository | `"registry.k8s.io/sig-storage/csi-resizer"` |
 | `csi.resizer.tag` | Resizer image tag | `"v1.13.2"` |
-| `csi.rookUseCsiOperator` |  | `false` |
+| `csi.rookUseCsiOperator` |  | `true` |
 | `csi.serviceMonitor.enabled` | Enable ServiceMonitor for Ceph CSI drivers | `false` |
 | `csi.serviceMonitor.interval` | Service monitor scrape interval | `"10s"` |
 | `csi.serviceMonitor.labels` | ServiceMonitor additional labels | `{}` |

build/rbac/get-helm-rbac.sh

@@ -19,6 +19,7 @@ options=(
   --namespace rook-ceph
   --set crds.enabled=false
   --set csi.csiAddons.enabled=true
+  --set csi.rookUseCsiOperator=false
 )
 
 for option in ${ADDITIONAL_HELM_CLI_OPTIONS}; do

deploy/charts/rook-ceph/values.yaml

@@ -88,7 +88,7 @@ ceph-csi-operator:
 # Settings for whether to disable the drivers or other daemons if they are not
 # needed
 csi:
-  rookUseCsiOperator: false
+  rookUseCsiOperator: true
   # -- Enable Ceph CSI RBD driver
   enableRbdDriver: true
   # -- Enable Ceph CSI CephFS driver

deploy/examples/operator-openshift.yaml

@@ -122,6 +122,9 @@ data:
   # Allow using loop devices for osds in test clusters.
   ROOK_CEPH_ALLOW_LOOP_DEVICES: "false"
 
+  # Enable CSI Operator
+  ROOK_USE_CSI_OPERATOR: "true"
+
   # Enable the CSI driver.
   # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml
   ROOK_CSI_ENABLE_CEPHFS: "true"

deploy/examples/operator.yaml

@@ -32,7 +32,7 @@ data:
   ROOK_CEPH_ALLOW_LOOP_DEVICES: "false"
 
   # Enable CSI Operator
-  ROOK_USE_CSI_OPERATOR: "false"
+  ROOK_USE_CSI_OPERATOR: "true"
   # Enable the CSI driver.
   # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml
   ROOK_CSI_ENABLE_CEPHFS: "true"

pkg/operator/ceph/csi/config_test.go

@@ -47,7 +47,8 @@ func TestCreateUpdateClientProfile(t *testing.T) {
 
 	ns := "test"
 	c.Namespace = ns
-	c.SetName("testcluster")
+	c.SetName(c.Namespace)
+	t.Setenv(k8sutil.PodNamespaceEnvVar, ns)
 	c.NamespacedName()
 	c.SetName(c.Namespace)
 	t.Setenv(k8sutil.PodNamespaceEnvVar, ns)

pkg/operator/ceph/csi/operator_driver.go

@@ -136,7 +136,6 @@ func (r *ReconcileCSI) createOrUpdateCephFSDriverResource(cluster cephv1.CephClu
 		Spec: spec,
 	}
 
-	cephFsDriver.Spec.SnapshotPolicy = csiopv1.NoneSnapshotPolicy
 	if CSIParam.VolumeGroupSnapshotCLIFlag != "" {
 		cephFsDriver.Spec.SnapshotPolicy = csiopv1.VolumeGroupSnapshotPolicy
 	}

tests/framework/installer/ceph_helm_installer.go

@@ -67,6 +67,10 @@ func (h *CephInstaller) configureRookOperatorViaHelm(upgrade bool) error {
 		"csiCephFSPluginResource":      nil,
 	}
 
+	if !h.settings.EnableCsiOperator {
+		values["rookUseCsiOperator"] = false
+	}
+
 	// create the operator namespace
 	if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil {
 		return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err)

tests/framework/installer/ceph_manifests.go

@@ -97,6 +97,7 @@ func (m *CephManifestsMaster) GetOperator() string {
 	} else {
 		manifest = m.settings.readManifest("operator.yaml")
 	}
+
 	return m.settings.replaceOperatorSettings(manifest)
 }
 

tests/framework/installer/ceph_settings.go

@@ -52,6 +52,7 @@ type TestCephSettings struct {
 	RookVersion                 string
 	CephVersion                 cephv1.CephVersionSpec
 	KubernetesVersion           string
+	EnableCsiOperator           bool
 }
 
 func (s *TestCephSettings) ApplyEnvVars() {
@@ -94,6 +95,9 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
 	manifest = strings.ReplaceAll(manifest,
 		`# ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize" # default allowed configs`,
 		`ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"`)
+	if !s.EnableCsiOperator {
+		manifest = strings.ReplaceAll(manifest, `ROOK_USE_CSI_OPERATOR: "true"`, `ROOK_USE_CSI_OPERATOR: "false"`)
+	}
 	return manifest
 }
 

tests/integration/ceph_helm_test.go

@@ -71,6 +71,7 @@ func (h *HelmSuite) SetupSuite() {
 		ConnectionsEncrypted: true,
 		RookVersion:          installer.LocalBuildTag,
 		CephVersion:          installer.ReefVersion,
+		EnableCsiOperator:    true,
 	}
 	h.settings.ApplyEnvVars()
 	h.installer, h.k8shelper = StartTestCluster(h.T, h.settings)

tests/integration/ceph_smoke_test.go

@@ -95,6 +95,7 @@ func (s *SmokeSuite) SetupSuite() {
 		ChangeHostName:          true,
 		RookVersion:             installer.LocalBuildTag,
 		CephVersion:             installer.ReturnCephVersion(),
+		EnableCsiOperator:       true,
 	}
 	s.settings.ApplyEnvVars()
 	s.installer, s.k8sh = StartTestCluster(s.T, s.settings)

tests/integration/ceph_upgrade_test.go

@@ -90,6 +90,7 @@ func (s *UpgradeSuite) baseSetup(useHelm bool, initialRookVersion string, initia
 		SkipClusterCleanup:          true,
 		RookVersion:                 initialRookVersion,
 		CephVersion:                 initialCephVersion,
+		EnableCsiOperator:           true,
 	}
 
 	s.installer, s.k8sh = StartTestCluster(s.T, s.settings)
@@ -454,7 +455,7 @@ func (s *UpgradeSuite) upgradeToMaster() {
 	}
 
 	require.NoError(s.T(), s.installer.InstallCSIOperator())
-	require.NoError(s.T(), s.installer.SetOperatorSetting("ROOK_USE_CSI_OPERATOR", "false"))
+	require.NoError(s.T(), s.installer.SetOperatorSetting("ROOK_USE_CSI_OPERATOR", "true"))
 
 	require.NoError(s.T(), s.k8sh.ResourceOperation("apply", s.installer.Manifests.GetCRDs(s.k8sh)))
 

tests/manifests/test-kms-vault-spec-k8s-auth.yaml

@@ -12,3 +12,6 @@ spec:
         VAULT_CACERT: "vault-ca-cert"
         VAULT_AUTH_METHOD: kubernetes
         VAULT_AUTH_KUBERNETES_ROLE: "rook-ceph"
+    keyRotation:
+      enabled: true
+      schedule: "*/1 * * * *"

tests/manifests/test-kms-vault-spec-token-auth.yaml

@@ -12,3 +12,6 @@ spec:
         VAULT_CACERT: "vault-ca-cert"
         VAULT_AUTH_METHOD: token
       tokenSecretName: rook-vault-token
+    keyRotation:
+      enabled: true
+      schedule: "*/1 * * * *"

tests/scripts/collect-logs.sh

@@ -33,7 +33,7 @@ for NAMESPACE in "${NAMESPACES[@]}"; do
   mkdir "${NS_DIR}"
 
   # describe every one of the k8s resources in the namespace which rook commonly uses
-  for KIND in 'pod' 'deployment' 'job' 'daemonset' 'cm'; do
+  for KIND in 'pod' 'deployment' 'job' 'daemonset' 'cm' 'pvc' 'sc'; do
     kubectl -n "$NAMESPACE" get "$KIND" -o wide >"${NS_DIR}"/"$KIND"-list.txt
     for resource in $(kubectl -n "$NAMESPACE" get "$KIND" -o jsonpath='{.items[*].metadata.name}'); do
       kubectl -n "$NAMESPACE" describe "$KIND" "$resource" >"${NS_DIR}"/"$KIND"-describe--"$resource".txt
@@ -56,7 +56,7 @@ for NAMESPACE in "${NAMESPACES[@]}"; do
   for CRD in $(kubectl get crds -o jsonpath='{.items[*].metadata.name}'); do
     for resource in $(kubectl -n "$NAMESPACE" get "$CRD" -o jsonpath='{.items[*].metadata.name}'); do
       crd_main_type="${CRD%%.*}" # e.g., for cephclusters.ceph.rook.io, only use 'cephclusters'
-      kubectl -n "$NAMESPACE" describe "$CRD" "$resource" >"${NS_DIR}"/"$crd_main_type"-describe--"$resource".txt
+      kubectl -n "$NAMESPACE" describe "$CRD" "$resource" >>"${NS_DIR}"/"$crd_main_type"-describe--"$resource".txt
     done
   done
 

tests/scripts/csiaddons.sh

@@ -21,17 +21,17 @@ CSIADDONS_CRD_NAME="csiaddonsnodes.csiaddons.openshift.io"
 CSIADDONS_CONTAINER_NAME="csi-addons"
 
 function setup_csiaddons() {
-    echo "setting up csi-addons"
+  echo "setting up csi-addons"
 
-    echo "deploying controller"
-    kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/crds.yaml
-    kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/rbac.yaml
-    kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/setup-controller.yaml
+  echo "deploying controller"
+  kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/crds.yaml
+  kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/rbac.yaml
+  kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/setup-controller.yaml
 
-    echo "enabling csi-addons"
-    kubectl patch cm rook-ceph-operator-config -n rook-ceph --type merge -p '{"data":{"CSI_ENABLE_CSIADDONS":"true"}}'
+  echo "enabling csi-addons"
+  kubectl patch cm rook-ceph-operator-config -n rook-ceph --type merge -p '{"data":{"CSI_ENABLE_CSIADDONS":"true"}}'
 
-    echo "Successfully created CSI-Addons"
+  echo "Successfully created CSI-Addons"
 }
 
 function verify_crd_created() {
@@ -47,25 +47,24 @@ function verify_crd_created() {
 }
 
 function verify_container_in_pod_by_label() {
-    label=$1
-    pod=$(kubectl get pods -n rook-ceph -l "$label" -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null)
-    if ! [ -n "$pod" ]; then
-        echo "pod with label $label not found!"
-        exit 1
-    fi
+  label=$1
+  pod=$(kubectl get pods -n rook-ceph -l "$label" -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null)
+  if ! [ -n "$pod" ]; then
+    echo "pod with label $label not found!"
+    exit 1
+  fi
 
-    container_status=$(kubectl get pod -n rook-ceph "$pod" -o=jsonpath="{.status.containerStatuses[?(@.name=='$CSIADDONS_CONTAINER_NAME')].ready}" 2>/dev/null)
-    if [ "$container_status" != "true" ]; then
-        echo "csi-addons container not found in $pod pod!"
-        exit 1
-    fi
-    echo "$CSIADDONS_CONTAINER_NAME container is running in $pod"
+  container_status=$(kubectl get pod -n rook-ceph "$pod" -o=jsonpath="{.status.containerStatuses[?(@.name=='$CSIADDONS_CONTAINER_NAME')].ready}" 2>/dev/null)
+  if [ "$container_status" != "true" ]; then
+    echo "csi-addons container not found in $pod pod!"
+    exit 1
+  fi
+  echo "$CSIADDONS_CONTAINER_NAME container is running in $pod"
 }
 
 function verify_container_is_running() {
-    verify_container_in_pod_by_label app=csi-rbdplugin
-    verify_container_in_pod_by_label app=csi-rbdplugin-provisioner
-    verify_container_in_pod_by_label app=csi-cephfsplugin-provisioner
+  verify_container_in_pod_by_label app=rook-ceph.rbd.csi.ceph.com-ctrlplugin
+  verify_container_in_pod_by_label app=rook-ceph.cephfs.csi.ceph.com-ctrlplugin
 }
 
 FUNCTION="$1"

tests/scripts/github-action-helper.sh

@@ -297,6 +297,7 @@ function deploy_cluster() {
   cd "${REPO_DIR}/deploy/examples"
 
   deploy_manifest_with_local_build operator.yaml
+  kubectl create -f csi-operator.yaml
 
   if [ $# == 0 ]; then
     sed -i "s|#deviceFilter:|deviceFilter: $(block_dev_basename)|g" cluster-test.yaml
@@ -479,6 +480,7 @@ function deploy_first_rook_cluster() {
   cd "${REPO_DIR}/deploy/examples"
 
   deploy_manifest_with_local_build operator.yaml
+  deploy_manifest_with_local_build csi-operator.yaml
   yq w -i -d0 cluster-test.yaml spec.dashboard.enabled false
   yq w -i -d0 cluster-test.yaml spec.storage.useAllDevices false
   yq w -i -d0 cluster-test.yaml spec.storage.deviceFilter "${DEVICE_NAME}"1
@@ -679,8 +681,8 @@ function test_csi_rbd_workload {
   kubectl create -f pod.yaml
   timeout 90 sh -c 'until kubectl exec -t pod/csirbd-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test bs=1M count=1; do echo "waiting for test pod to be ready" && sleep 1; done'
   kubectl exec -t pod/csirbd-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test oflag=direct bs=1M count=1
-  kubectl -n rook-ceph logs ds/csi-rbdplugin -c csi-rbdplugin
-  kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=csi-rbdplugin --field-selector=status.phase=Running -o name)"
+  kubectl -n rook-ceph logs ds/rook-ceph.rbd.csi.ceph.com-nodeplugin -c csi-rbdplugin
+  kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=rook-ceph.rbd.csi.ceph.com-nodeplugin --field-selector=status.phase=Running -o name)"
   kubectl exec -t pod/csirbd-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test1 oflag=direct bs=1M count=1
   kubectl exec -t pod/csirbd-demo-pod -- ls -alh /var/lib/www/html/
 }
@@ -692,8 +694,8 @@ function test_csi_cephfs_workload {
   kubectl create -f pod.yaml
   timeout 90 sh -c 'until kubectl exec -t pod/csicephfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test bs=1M count=1; do echo "waiting for test pod to be ready" && sleep 1; done'
   kubectl exec -t pod/csicephfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test oflag=direct bs=1M count=1
-  kubectl -n rook-ceph logs ds/csi-cephfsplugin -c csi-cephfsplugin
-  kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=csi-cephfsplugin --field-selector=status.phase=Running -o name)"
+  kubectl -n rook-ceph logs ds/rook-ceph.cephfs.csi.ceph.com-nodeplugin -c csi-cephfsplugin
+  kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=rook-ceph.cephfs.csi.ceph.com-nodeplugin --field-selector=status.phase=Running -o name)"
   kubectl exec -t pod/csicephfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test1 oflag=direct bs=1M count=1
   kubectl exec -t pod/csicephfs-demo-pod -- ls -alh /var/lib/www/html/
 }
@@ -706,7 +708,7 @@ function test_csi_nfs_workload {
   kubectl create -f pod.yaml
   timeout 90 sh -c 'until kubectl exec -t pod/csinfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test bs=1M count=1; do echo "waiting for test pod to be ready" && sleep 1; done'
   kubectl exec -t pod/csinfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test oflag=direct bs=1M count=1
-  kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=csi-nfsplugin --field-selector=status.phase=Running -o name)"
+  kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=rook-ceph.nfs.csi.ceph.com-nodeplugin --field-selector=status.phase=Running -o name)"
   kubectl exec -t pod/csinfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test1 oflag=direct bs=1M count=1
   kubectl exec -t pod/csinfs-demo-pod -- ls -alh /var/lib/www/html/
 }
@@ -802,6 +804,9 @@ function delete_cluster() {
   kubectl --namespace rook-ceph logs deploy/rook-ceph-operator
   wait_for_cleanup_pod
   kubectl --namespace rook-ceph delete --ignore-not-found=true -f deploy/examples/operator.yaml
+  kubectl --namespace rook-ceph delete clientprofile rook-ceph
+  kubectl --namespace rook-ceph logs deploy/ceph-csi-controller-manager
+  kubectl --namespace rook-ceph delete --ignore-not-found=true -f deploy/examples/csi-operator.yaml
   remove_cluster_prerequisites
 }
 

tests/scripts/validate_cluster.sh

@@ -103,8 +103,9 @@ function test_demo_pool {
 
 function test_csi {
   timeout 360 bash -x <<-'EOF'
-    until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c ^csi-)" -eq 6 ]]; do
+    until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c csi.)" -ge 7 ]]; do
       echo "waiting for csi pods to be ready"
+      kubectl -n rook-ceph get pods | grep csi
       sleep 5
     done
 EOF