diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 45e6fd370..4492c2143 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -842,6 +842,7 @@ jobs: run: | yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true" tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi @@ -876,6 +877,7 @@ jobs: - name: redeploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml @@ -939,6 +941,7 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -992,6 +995,7 @@ jobs: - name: deploy rook run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1058,6 +1062,7 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1089,6 +1094,7 @@ jobs: - name: redeploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1153,6 +1159,7 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml @@ -1208,6 +1215,7 @@ jobs: - name: deploy rook run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -1278,12 +1286,12 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 kubectl create -f tests/manifests/test-object.yaml @@ -1369,11 +1377,11 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - name: wait for prepare pod @@ -1438,6 +1446,7 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml @@ -1889,6 +1898,8 @@ jobs: - name: deploy cluster run: | tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/csi-operator.yaml + export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)" yq write -i deploy/examples/cluster-multus-test.yaml "spec.storage.deviceFilter" "${BLOCK}" diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 143ba521c..7ea5fafdf 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -129,7 +129,7 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.registrar.tag` | Registrar image tag | `"v2.13.0"` | | `csi.resizer.repository` | Kubernetes CSI resizer image repository | `"registry.k8s.io/sig-storage/csi-resizer"` | | `csi.resizer.tag` | Resizer image tag | `"v1.13.2"` | -| `csi.rookUseCsiOperator` | | `false` | +| `csi.rookUseCsiOperator` | | `true` | | `csi.serviceMonitor.enabled` | Enable ServiceMonitor for Ceph CSI drivers | `false` | | `csi.serviceMonitor.interval` | Service monitor scrape interval | `"10s"` | | `csi.serviceMonitor.labels` | ServiceMonitor additional labels | `{}` | diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index 8fbce04d4..45b9b7bb5 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -19,6 +19,7 @@ options=( --namespace rook-ceph --set crds.enabled=false --set csi.csiAddons.enabled=true + --set csi.rookUseCsiOperator=false ) for option in ${ADDITIONAL_HELM_CLI_OPTIONS}; do diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 5d3fcd1aa..4d1d6b793 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -88,7 +88,7 @@ ceph-csi-operator: # Settings for whether to disable the drivers or other daemons if they are not # needed csi: - rookUseCsiOperator: false + rookUseCsiOperator: true # -- Enable Ceph CSI RBD driver enableRbdDriver: true # -- Enable Ceph CSI CephFS driver diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index c12d877af..050850eea 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -122,6 +122,9 @@ data: # Allow using loop devices for osds in test clusters. ROOK_CEPH_ALLOW_LOOP_DEVICES: "false" + # Enable CSI Operator + ROOK_USE_CSI_OPERATOR: "true" + # Enable the CSI driver. # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml ROOK_CSI_ENABLE_CEPHFS: "true" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 52467f318..58846fa7d 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -32,7 +32,7 @@ data: ROOK_CEPH_ALLOW_LOOP_DEVICES: "false" # Enable CSI Operator - ROOK_USE_CSI_OPERATOR: "false" + ROOK_USE_CSI_OPERATOR: "true" # Enable the CSI driver. # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml ROOK_CSI_ENABLE_CEPHFS: "true" diff --git a/pkg/operator/ceph/csi/config_test.go b/pkg/operator/ceph/csi/config_test.go index b29234b80..15fb33b48 100644 --- a/pkg/operator/ceph/csi/config_test.go +++ b/pkg/operator/ceph/csi/config_test.go @@ -47,7 +47,8 @@ func TestCreateUpdateClientProfile(t *testing.T) { ns := "test" c.Namespace = ns - c.SetName("testcluster") + c.SetName(c.Namespace) + t.Setenv(k8sutil.PodNamespaceEnvVar, ns) c.NamespacedName() c.SetName(c.Namespace) t.Setenv(k8sutil.PodNamespaceEnvVar, ns) diff --git a/pkg/operator/ceph/csi/operator_driver.go b/pkg/operator/ceph/csi/operator_driver.go index d176edcae..a4ddc80ef 100644 --- a/pkg/operator/ceph/csi/operator_driver.go +++ b/pkg/operator/ceph/csi/operator_driver.go @@ -136,7 +136,6 @@ func (r *ReconcileCSI) createOrUpdateCephFSDriverResource(cluster cephv1.CephClu Spec: spec, } - cephFsDriver.Spec.SnapshotPolicy = csiopv1.NoneSnapshotPolicy if CSIParam.VolumeGroupSnapshotCLIFlag != "" { cephFsDriver.Spec.SnapshotPolicy = csiopv1.VolumeGroupSnapshotPolicy } diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index b9de394f5..0f49f2a88 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -67,6 +67,10 @@ func (h *CephInstaller) configureRookOperatorViaHelm(upgrade bool) error { "csiCephFSPluginResource": nil, } + if !h.settings.EnableCsiOperator { + values["rookUseCsiOperator"] = false + } + // create the operator namespace if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 63706052e..04b71914a 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -97,6 +97,7 @@ func (m *CephManifestsMaster) GetOperator() string { } else { manifest = m.settings.readManifest("operator.yaml") } + return m.settings.replaceOperatorSettings(manifest) } diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 055c3b6b7..10982e301 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -52,6 +52,7 @@ type TestCephSettings struct { RookVersion string CephVersion cephv1.CephVersionSpec KubernetesVersion string + EnableCsiOperator bool } func (s *TestCephSettings) ApplyEnvVars() { @@ -94,6 +95,9 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { manifest = strings.ReplaceAll(manifest, `# ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize" # default allowed configs`, `ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"`) + if !s.EnableCsiOperator { + manifest = strings.ReplaceAll(manifest, `ROOK_USE_CSI_OPERATOR: "true"`, `ROOK_USE_CSI_OPERATOR: "false"`) + } return manifest } diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index de16e4319..d686c179c 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -71,6 +71,7 @@ func (h *HelmSuite) SetupSuite() { ConnectionsEncrypted: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.ReefVersion, + EnableCsiOperator: true, } h.settings.ApplyEnvVars() h.installer, h.k8shelper = StartTestCluster(h.T, h.settings) diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 153db60e6..a75843e9f 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -95,6 +95,7 @@ func (s *SmokeSuite) SetupSuite() { ChangeHostName: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.ReturnCephVersion(), + EnableCsiOperator: true, } s.settings.ApplyEnvVars() s.installer, s.k8sh = StartTestCluster(s.T, s.settings) diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index ef8401624..2b83f1e8d 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -90,6 +90,7 @@ func (s *UpgradeSuite) baseSetup(useHelm bool, initialRookVersion string, initia SkipClusterCleanup: true, RookVersion: initialRookVersion, CephVersion: initialCephVersion, + EnableCsiOperator: true, } s.installer, s.k8sh = StartTestCluster(s.T, s.settings) @@ -454,7 +455,7 @@ func (s *UpgradeSuite) upgradeToMaster() { } require.NoError(s.T(), s.installer.InstallCSIOperator()) - require.NoError(s.T(), s.installer.SetOperatorSetting("ROOK_USE_CSI_OPERATOR", "false")) + require.NoError(s.T(), s.installer.SetOperatorSetting("ROOK_USE_CSI_OPERATOR", "true")) require.NoError(s.T(), s.k8sh.ResourceOperation("apply", s.installer.Manifests.GetCRDs(s.k8sh))) diff --git a/tests/manifests/test-kms-vault-spec-k8s-auth.yaml b/tests/manifests/test-kms-vault-spec-k8s-auth.yaml index a6065358a..509f104ed 100644 --- a/tests/manifests/test-kms-vault-spec-k8s-auth.yaml +++ b/tests/manifests/test-kms-vault-spec-k8s-auth.yaml @@ -12,3 +12,6 @@ spec: VAULT_CACERT: "vault-ca-cert" VAULT_AUTH_METHOD: kubernetes VAULT_AUTH_KUBERNETES_ROLE: "rook-ceph" + keyRotation: + enabled: true + schedule: "*/1 * * * *" diff --git a/tests/manifests/test-kms-vault-spec-token-auth.yaml b/tests/manifests/test-kms-vault-spec-token-auth.yaml index e8b43451d..2517dc80b 100644 --- a/tests/manifests/test-kms-vault-spec-token-auth.yaml +++ b/tests/manifests/test-kms-vault-spec-token-auth.yaml @@ -12,3 +12,6 @@ spec: VAULT_CACERT: "vault-ca-cert" VAULT_AUTH_METHOD: token tokenSecretName: rook-vault-token + keyRotation: + enabled: true + schedule: "*/1 * * * *" diff --git a/tests/scripts/collect-logs.sh b/tests/scripts/collect-logs.sh index 20a75ea15..42d620730 100755 --- a/tests/scripts/collect-logs.sh +++ b/tests/scripts/collect-logs.sh @@ -33,7 +33,7 @@ for NAMESPACE in "${NAMESPACES[@]}"; do mkdir "${NS_DIR}" # describe every one of the k8s resources in the namespace which rook commonly uses - for KIND in 'pod' 'deployment' 'job' 'daemonset' 'cm'; do + for KIND in 'pod' 'deployment' 'job' 'daemonset' 'cm' 'pvc' 'sc'; do kubectl -n "$NAMESPACE" get "$KIND" -o wide >"${NS_DIR}"/"$KIND"-list.txt for resource in $(kubectl -n "$NAMESPACE" get "$KIND" -o jsonpath='{.items[*].metadata.name}'); do kubectl -n "$NAMESPACE" describe "$KIND" "$resource" >"${NS_DIR}"/"$KIND"-describe--"$resource".txt @@ -56,7 +56,7 @@ for NAMESPACE in "${NAMESPACES[@]}"; do for CRD in $(kubectl get crds -o jsonpath='{.items[*].metadata.name}'); do for resource in $(kubectl -n "$NAMESPACE" get "$CRD" -o jsonpath='{.items[*].metadata.name}'); do crd_main_type="${CRD%%.*}" # e.g., for cephclusters.ceph.rook.io, only use 'cephclusters' - kubectl -n "$NAMESPACE" describe "$CRD" "$resource" >"${NS_DIR}"/"$crd_main_type"-describe--"$resource".txt + kubectl -n "$NAMESPACE" describe "$CRD" "$resource" >>"${NS_DIR}"/"$crd_main_type"-describe--"$resource".txt done done diff --git a/tests/scripts/csiaddons.sh b/tests/scripts/csiaddons.sh index b47705463..81b23346c 100755 --- a/tests/scripts/csiaddons.sh +++ b/tests/scripts/csiaddons.sh @@ -21,17 +21,17 @@ CSIADDONS_CRD_NAME="csiaddonsnodes.csiaddons.openshift.io" CSIADDONS_CONTAINER_NAME="csi-addons" function setup_csiaddons() { - echo "setting up csi-addons" + echo "setting up csi-addons" - echo "deploying controller" - kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/crds.yaml - kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/rbac.yaml - kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/setup-controller.yaml + echo "deploying controller" + kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/crds.yaml + kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/rbac.yaml + kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/$CSIADDONS_VERSION/setup-controller.yaml - echo "enabling csi-addons" - kubectl patch cm rook-ceph-operator-config -n rook-ceph --type merge -p '{"data":{"CSI_ENABLE_CSIADDONS":"true"}}' + echo "enabling csi-addons" + kubectl patch cm rook-ceph-operator-config -n rook-ceph --type merge -p '{"data":{"CSI_ENABLE_CSIADDONS":"true"}}' - echo "Successfully created CSI-Addons" + echo "Successfully created CSI-Addons" } function verify_crd_created() { @@ -47,25 +47,24 @@ function verify_crd_created() { } function verify_container_in_pod_by_label() { - label=$1 - pod=$(kubectl get pods -n rook-ceph -l "$label" -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null) - if ! [ -n "$pod" ]; then - echo "pod with label $label not found!" - exit 1 - fi + label=$1 + pod=$(kubectl get pods -n rook-ceph -l "$label" -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null) + if ! [ -n "$pod" ]; then + echo "pod with label $label not found!" + exit 1 + fi - container_status=$(kubectl get pod -n rook-ceph "$pod" -o=jsonpath="{.status.containerStatuses[?(@.name=='$CSIADDONS_CONTAINER_NAME')].ready}" 2>/dev/null) - if [ "$container_status" != "true" ]; then - echo "csi-addons container not found in $pod pod!" - exit 1 - fi - echo "$CSIADDONS_CONTAINER_NAME container is running in $pod" + container_status=$(kubectl get pod -n rook-ceph "$pod" -o=jsonpath="{.status.containerStatuses[?(@.name=='$CSIADDONS_CONTAINER_NAME')].ready}" 2>/dev/null) + if [ "$container_status" != "true" ]; then + echo "csi-addons container not found in $pod pod!" + exit 1 + fi + echo "$CSIADDONS_CONTAINER_NAME container is running in $pod" } function verify_container_is_running() { - verify_container_in_pod_by_label app=csi-rbdplugin - verify_container_in_pod_by_label app=csi-rbdplugin-provisioner - verify_container_in_pod_by_label app=csi-cephfsplugin-provisioner + verify_container_in_pod_by_label app=rook-ceph.rbd.csi.ceph.com-ctrlplugin + verify_container_in_pod_by_label app=rook-ceph.cephfs.csi.ceph.com-ctrlplugin } FUNCTION="$1" diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 03343dab7..f049e1720 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -297,6 +297,7 @@ function deploy_cluster() { cd "${REPO_DIR}/deploy/examples" deploy_manifest_with_local_build operator.yaml + kubectl create -f csi-operator.yaml if [ $# == 0 ]; then sed -i "s|#deviceFilter:|deviceFilter: $(block_dev_basename)|g" cluster-test.yaml @@ -479,6 +480,7 @@ function deploy_first_rook_cluster() { cd "${REPO_DIR}/deploy/examples" deploy_manifest_with_local_build operator.yaml + deploy_manifest_with_local_build csi-operator.yaml yq w -i -d0 cluster-test.yaml spec.dashboard.enabled false yq w -i -d0 cluster-test.yaml spec.storage.useAllDevices false yq w -i -d0 cluster-test.yaml spec.storage.deviceFilter "${DEVICE_NAME}"1 @@ -679,8 +681,8 @@ function test_csi_rbd_workload { kubectl create -f pod.yaml timeout 90 sh -c 'until kubectl exec -t pod/csirbd-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test bs=1M count=1; do echo "waiting for test pod to be ready" && sleep 1; done' kubectl exec -t pod/csirbd-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test oflag=direct bs=1M count=1 - kubectl -n rook-ceph logs ds/csi-rbdplugin -c csi-rbdplugin - kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=csi-rbdplugin --field-selector=status.phase=Running -o name)" + kubectl -n rook-ceph logs ds/rook-ceph.rbd.csi.ceph.com-nodeplugin -c csi-rbdplugin + kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=rook-ceph.rbd.csi.ceph.com-nodeplugin --field-selector=status.phase=Running -o name)" kubectl exec -t pod/csirbd-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test1 oflag=direct bs=1M count=1 kubectl exec -t pod/csirbd-demo-pod -- ls -alh /var/lib/www/html/ } @@ -692,8 +694,8 @@ function test_csi_cephfs_workload { kubectl create -f pod.yaml timeout 90 sh -c 'until kubectl exec -t pod/csicephfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test bs=1M count=1; do echo "waiting for test pod to be ready" && sleep 1; done' kubectl exec -t pod/csicephfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test oflag=direct bs=1M count=1 - kubectl -n rook-ceph logs ds/csi-cephfsplugin -c csi-cephfsplugin - kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=csi-cephfsplugin --field-selector=status.phase=Running -o name)" + kubectl -n rook-ceph logs ds/rook-ceph.cephfs.csi.ceph.com-nodeplugin -c csi-cephfsplugin + kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=rook-ceph.cephfs.csi.ceph.com-nodeplugin --field-selector=status.phase=Running -o name)" kubectl exec -t pod/csicephfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test1 oflag=direct bs=1M count=1 kubectl exec -t pod/csicephfs-demo-pod -- ls -alh /var/lib/www/html/ } @@ -706,7 +708,7 @@ function test_csi_nfs_workload { kubectl create -f pod.yaml timeout 90 sh -c 'until kubectl exec -t pod/csinfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test bs=1M count=1; do echo "waiting for test pod to be ready" && sleep 1; done' kubectl exec -t pod/csinfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test oflag=direct bs=1M count=1 - kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=csi-nfsplugin --field-selector=status.phase=Running -o name)" + kubectl -n rook-ceph delete "$(kubectl -n rook-ceph get pod --selector=app=rook-ceph.nfs.csi.ceph.com-nodeplugin --field-selector=status.phase=Running -o name)" kubectl exec -t pod/csinfs-demo-pod -- dd if=/dev/random of=/var/lib/www/html/test1 oflag=direct bs=1M count=1 kubectl exec -t pod/csinfs-demo-pod -- ls -alh /var/lib/www/html/ } @@ -802,6 +804,9 @@ function delete_cluster() { kubectl --namespace rook-ceph logs deploy/rook-ceph-operator wait_for_cleanup_pod kubectl --namespace rook-ceph delete --ignore-not-found=true -f deploy/examples/operator.yaml + kubectl --namespace rook-ceph delete clientprofile rook-ceph + kubectl --namespace rook-ceph logs deploy/ceph-csi-controller-manager + kubectl --namespace rook-ceph delete --ignore-not-found=true -f deploy/examples/csi-operator.yaml remove_cluster_prerequisites } diff --git a/tests/scripts/validate_cluster.sh b/tests/scripts/validate_cluster.sh index eeac1344b..eb6c3cd24 100755 --- a/tests/scripts/validate_cluster.sh +++ b/tests/scripts/validate_cluster.sh @@ -103,8 +103,9 @@ function test_demo_pool { function test_csi { timeout 360 bash -x <<-'EOF' - until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c ^csi-)" -eq 6 ]]; do + until [[ "$(kubectl -n rook-ceph get pods --field-selector=status.phase=Running|grep -c csi.)" -ge 7 ]]; do echo "waiting for csi pods to be ready" + kubectl -n rook-ceph get pods | grep csi sleep 5 done EOF