From eb38b091ede9395a29bbeacdd926cb9e8877f8f0 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 5 May 2026 12:15:10 +0530 Subject: [PATCH] csi: wait for the PV deletion Wait for the PV deletion before deleting the ceph resources like pools. Signed-off-by: Madhu Rajanna (cherry picked from commit 799bd66243d0988bad9c8342f9d6b9403bf032a1) --- tests/integration/ceph_base_block_test.go | 47 ++++++++++++++++++++++- tests/integration/ceph_base_file_test.go | 42 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/tests/integration/ceph_base_block_test.go b/tests/integration/ceph_base_block_test.go index 142dda891..842a56f63 100644 --- a/tests/integration/ceph_base_block_test.go +++ b/tests/integration/ceph_base_block_test.go @@ -95,18 +95,34 @@ func blockCSICloneTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s *sui require.NoError(s.T(), err) logger.Infof("delete clone pvc") + // Get the PV name before deleting the PVC + clonePVName, err := k8sh.GetPVCVolumeName(defaultNamespace, clonePVCName) + assert.NoError(s.T(), err) + err = helper.BlockClient.DeletePVC(defaultNamespace, clonePVCName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, clonePVCName)) + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, clonePVName, false, "")) + logger.Infof("clone PV %s deleted successfully", clonePVName) + // delete the parent PVC and app err = k8sh.DeletePod(k8sutil.DefaultNamespace, podName) require.NoError(s.T(), err) logger.Infof("delete parent pvc") + // Get the PV name before deleting the PVC + parentPVName, err := k8sh.GetPVCVolumeName(defaultNamespace, pvcName) + assert.NoError(s.T(), err) + err = helper.BlockClient.DeletePVC(defaultNamespace, pvcName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, pvcName)) + + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, parentPVName, false, "")) + logger.Infof("parent PV %s deleted successfully", parentPVName) } func blockCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s *suite.Suite, storageClassName, namespace string) { @@ -204,10 +220,18 @@ func blockCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s * require.NoError(s.T(), err) logger.Infof("delete restore pvc") + // Get the PV name before deleting the PVC + restorePVName, err := k8sh.GetPVCVolumeName(defaultNamespace, restorePVCName) + assert.NoError(s.T(), err) + err = helper.BlockClient.DeletePVC(defaultNamespace, restorePVCName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, restorePVCName)) + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, restorePVName, false, "")) + logger.Infof("restore PV %s deleted successfully", restorePVName) + // delete the snapshot logger.Infof("delete snapshot") @@ -220,10 +244,18 @@ func blockCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s * require.NoError(s.T(), err) logger.Infof("delete parent pvc") + // Get the PV name before deleting the PVC + parentPVName, err := k8sh.GetPVCVolumeName(defaultNamespace, pvcName) + assert.NoError(s.T(), err) + err = helper.BlockClient.DeletePVC(defaultNamespace, pvcName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, pvcName)) + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, parentPVName, false, "")) + logger.Infof("parent PV %s deleted successfully", parentPVName) + logger.Infof("delete snapshotclass") err = helper.BlockClient.DeleteSnapshotClass(snapshotClassName, snapshotDeletePolicy, namespace) @@ -435,10 +467,23 @@ func createAndWaitForPVC(helper *clients.TestClient, k8sh *utils.K8sHelper, s *s func deleteBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s *suite.Suite, clusterInfo *client.ClusterInfo, poolName, storageClassName, blockName string, requireBlockImagesRemoved bool) { logger.Infof("deleteBlockLite: cleaning up after test") + // Get the PV name before deleting the PVC + pvName, err := k8sh.GetPVCVolumeName(defaultNamespace, blockName) + if err != nil { + logger.Warningf("failed to get PV name for PVC %s: %v", blockName, err) + } + // Delete pvc and storageclass - err := helper.BlockClient.DeletePVC(defaultNamespace, blockName) + err = helper.BlockClient.DeletePVC(defaultNamespace, blockName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, blockName)) + + // Wait for the PV to be deleted as well + if pvName != "" { + assert.True(s.T(), retryPVCheck(k8sh, pvName, false, "")) + logger.Infof("PV %s deleted successfully", pvName) + } + if requireBlockImagesRemoved { assert.NoError(s.T(), retryBlockImageCountCheck(helper, clusterInfo, 0), "Make sure block images were deleted") } diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index 39e5f9098..dd90bf9f2 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -102,18 +102,34 @@ func fileSystemCSICloneTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s require.NoError(s.T(), err) logger.Infof("delete clone pvc") + // Get the PV name before deleting the PVC + clonePVName, err := k8sh.GetPVCVolumeName(defaultNamespace, clonePVCName) + assert.NoError(s.T(), err) + err = helper.FSClient.DeletePVC(defaultNamespace, clonePVCName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, clonePVCName)) + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, clonePVName, false, "")) + logger.Infof("clone PV %s deleted successfully", clonePVName) + // delete the parent PVC and app err = k8sh.DeletePod(k8sutil.DefaultNamespace, podName) require.NoError(s.T(), err) logger.Infof("delete parent pvc") + // Get the PV name before deleting the PVC + parentPVName, err := k8sh.GetPVCVolumeName(defaultNamespace, pvcName) + assert.NoError(s.T(), err) + err = helper.FSClient.DeletePVC(defaultNamespace, pvcName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, pvcName)) + + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, parentPVName, false, "")) + logger.Infof("parent PV %s deleted successfully", parentPVName) } func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper, s *suite.Suite, storageClassName, namespace string, testNFS bool) { @@ -216,10 +232,18 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper require.NoError(s.T(), err) logger.Infof("delete restore pvc") + // Get the PV name before deleting the PVC + restorePVName, err := k8sh.GetPVCVolumeName(defaultNamespace, restorePVCName) + assert.NoError(s.T(), err) + err = helper.FSClient.DeletePVC(defaultNamespace, restorePVCName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, restorePVCName)) + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, restorePVName, false, "")) + logger.Infof("restore PV %s deleted successfully", restorePVName) + // delete the snapshot logger.Infof("delete snapshot") @@ -232,10 +256,18 @@ func fileSystemCSISnapshotTest(helper *clients.TestClient, k8sh *utils.K8sHelper require.NoError(s.T(), err) logger.Infof("delete parent pvc") + // Get the PV name before deleting the PVC + parentPVName, err := k8sh.GetPVCVolumeName(defaultNamespace, pvcName) + assert.NoError(s.T(), err) + err = helper.FSClient.DeletePVC(defaultNamespace, pvcName) assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, pvcName)) + // Wait for the PV to be deleted as well + assert.True(s.T(), retryPVCheck(k8sh, parentPVName, false, "")) + logger.Infof("parent PV %s deleted successfully", parentPVName) + logger.Infof("delete snapshotclass") if !testNFS { err = helper.FSClient.DeleteSnapshotClass(snapshotClassName, snapshotDeletePolicy, namespace) @@ -440,12 +472,22 @@ func cleanupFilesystemConsumer(helper *clients.TestClient, k8sh *utils.K8sHelper k8sh.PrintPodDescribe(namespace, podName) assert.Fail(s.T(), fmt.Sprintf("make sure %s pod is terminated", podName)) } + + // Get the PV name before deleting the PVC + pvName, err := k8sh.GetPVCVolumeName(namespace, podName) + assert.NoError(s.T(), err) + err = helper.FSClient.DeletePVC(namespace, podName) assertNoErrorUnlessNotFound(s, err) isdeleted := k8sh.WaitUntilPVCIsDeleted(namespace, podName) if !isdeleted { assert.Fail(s.T(), fmt.Sprintf("Failed to delete PVC %q", podName)) } + + // Wait for the PV to be deleted + assert.True(s.T(), retryPVCheck(k8sh, pvName, false, "")) + logger.Infof("PV %s deleted successfully", pvName) + isPVListZero := k8sh.WaitUntilZeroPVs() if !isPVListZero { assert.Fail(s.T(), "PV list is not zero")