From ce1fedf8843ceacb09bf61e039e0fdccf8d4aa5f Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 17 Mar 2020 15:28:05 -0600 Subject: [PATCH] ci: only list the images in the needed pool during cleanup The pool cleanup only needs to happen for an individual pool. No need to query the block images in all pools. One of the rgw pools is periodically causing a hang when it is queried, but there is no need to query for it when we are cleaning up the pool tests. Signed-off-by: Travis Nielsen --- tests/framework/clients/block.go | 37 +++++++++++++++-------- tests/framework/clients/pool.go | 12 +++----- tests/integration/ceph_base_block_test.go | 28 +++++++++-------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/tests/framework/clients/block.go b/tests/framework/clients/block.go index ad7a33e12..61fc0877f 100644 --- a/tests/framework/clients/block.go +++ b/tests/framework/clients/block.go @@ -103,11 +103,10 @@ func (b *BlockOperation) DeleteBlock(manifest string) (string, error) { } -// List Function to list all the blocks created/being managed by rook -func (b *BlockOperation) List(namespace string) ([]BlockImage, error) { +// List Function to list all the block images in all pools +func (b *BlockOperation) ListAllImages(namespace string) ([]BlockImage, error) { // first list all the pools so that we can retrieve images from all pools - context := b.k8sClient.MakeContext() - pools, err := client.ListPoolSummaries(context, namespace) + pools, err := client.ListPoolSummaries(b.k8sClient.MakeContext(), namespace) if err != nil { return nil, fmt.Errorf("failed to list pools: %+v", err) } @@ -115,20 +114,32 @@ func (b *BlockOperation) List(namespace string) ([]BlockImage, error) { // for each pool, get further details about all the images in the pool images := []BlockImage{} for _, p := range pools { - cephImages, err := client.ListImages(context, namespace, p.Name) + cephImages, err := b.ListImagesInPool(namespace, p.Name) if err != nil { return nil, fmt.Errorf("failed to get images from pool %s: %+v", p.Name, err) } + images = append(images, cephImages...) + } + return images, nil +} - for _, image := range cephImages { - // add the current image's details to the result set - newImage := BlockImage{ - Name: image.Name, - PoolName: p.Name, - Size: image.Size, - } - images = append(images, newImage) +// List Function to list all the block images in a pool +func (b *BlockOperation) ListImagesInPool(namespace, poolName string) ([]BlockImage, error) { + // for each pool, get further details about all the images in the pool + images := []BlockImage{} + cephImages, err := client.ListImages(b.k8sClient.MakeContext(), namespace, poolName) + if err != nil { + return nil, fmt.Errorf("failed to get images from pool %s: %+v", poolName, err) + } + + for _, image := range cephImages { + // add the current image's details to the result set + newImage := BlockImage{ + Name: image.Name, + PoolName: poolName, + Size: image.Size, } + images = append(images, newImage) } return images, nil diff --git a/tests/framework/clients/pool.go b/tests/framework/clients/pool.go index 9bca1d0d9..5665ae11a 100644 --- a/tests/framework/clients/pool.go +++ b/tests/framework/clients/pool.go @@ -107,14 +107,12 @@ func (p *PoolOperation) CephPoolExists(namespace, name string) (bool, error) { func (p *PoolOperation) DeletePool(blockClient *BlockOperation, namespace, poolName string) error { // Delete all the images in a pool - blockImagesList, _ := blockClient.List(namespace) + blockImagesList, _ := blockClient.ListImagesInPool(namespace, poolName) for _, blockImage := range blockImagesList { - if poolName == blockImage.PoolName { - logger.Infof("force deleting block image %q in pool %q", blockImage, poolName) - err := blockClient.DeleteBlockImage(blockImage, namespace) - if err != nil { - logger.Infof("failed deleting image %q from %q. %v", blockImage, poolName, err) - } + logger.Infof("force deleting block image %q in pool %q", blockImage, poolName) + err := blockClient.DeleteBlockImage(blockImage, namespace) + if err != nil { + logger.Infof("failed deleting image %q from %q. %v", blockImage, poolName, err) } } diff --git a/tests/integration/ceph_base_block_test.go b/tests/integration/ceph_base_block_test.go index 6586163bf..592ad8783 100644 --- a/tests/integration/ceph_base_block_test.go +++ b/tests/integration/ceph_base_block_test.go @@ -17,6 +17,7 @@ limitations under the License. package integration import ( + "fmt" "strconv" "time" @@ -64,16 +65,16 @@ func runBlockCSITest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. logger.Infof("Running on Rook Cluster %s", namespace) logger.Infof("Step 0 : Get Initial List Block") - initBlockImages, _ := helper.BlockClient.List(namespace) + initBlockImages, _ := helper.BlockClient.ListAllImages(namespace) assert.Equal(s.T(), 0, len(initBlockImages), "there should not already be any images in the pool") logger.Infof("step 1: Create block storage") err := helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolName, storageClassName, "Delete", blockName, "ReadWriteOnce") require.NoError(s.T(), err) - require.True(s.T(), retryBlockImageCountCheck(helper, 1, namespace), "Make sure a new block is created") + require.NoError(s.T(), retryBlockImageCountCheck(helper, 1, namespace), "Make sure a new block is created") err = helper.BlockClient.CreateStorageClassAndPVC(true, defaultNamespace, namespace, systemNamespace, poolNameRetained, storageClassNameRetained, "Retain", blockNameRetained, "ReadWriteOnce") require.NoError(s.T(), err) - require.True(s.T(), retryBlockImageCountCheck(helper, 2, namespace), "Make sure another new block is created") + require.NoError(s.T(), retryBlockImageCountCheck(helper, 2, namespace), "Make sure another new block is created") logger.Infof("Block Storage created successfully") require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName), "Make sure PVC is Bound") require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockNameRetained), "Make sure PVC with reclaimPolicy:Retain is Bound") @@ -154,11 +155,11 @@ func deletePVC(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, if retainPolicy == "Delete" { assert.True(s.T(), retryPVCheck(k8sh, pvName, false, "")) logger.Infof("PV: %s deleted successfully", pvName) - assert.True(s.T(), retryBlockImageCountCheck(helper, 1, clusterNamespace), "Make sure a block is deleted") + assert.NoError(s.T(), retryBlockImageCountCheck(helper, 1, clusterNamespace), "Make sure a block is deleted") logger.Infof("Block Storage deleted successfully") } else { assert.True(s.T(), retryPVCheck(k8sh, pvName, true, "Released")) - assert.True(s.T(), retryBlockImageCountCheck(helper, 1, clusterNamespace), "Make sure a block is retained") + assert.NoError(s.T(), retryBlockImageCountCheck(helper, 1, clusterNamespace), "Make sure a block is retained") logger.Infof("Block Storage retained") k8sh.Kubectl("delete", "pv", pvName) } @@ -211,7 +212,7 @@ func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S clusterNamespace, systemNamespace, poolName, storageClassName, blockName, podName string, version cephv1.CephVersionSpec) { // Check initial number of blocks - initialBlocks, err := helper.BlockClient.List(clusterNamespace) + initialBlocks, err := helper.BlockClient.ListAllImages(clusterNamespace) require.NoError(s.T(), err) initBlockCount := len(initialBlocks) assert.Equal(s.T(), 0, initBlockCount, "why is there already a block image in the new pool?") @@ -224,7 +225,7 @@ func setupBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S require.True(s.T(), k8sh.WaitUntilPVCIsBound(defaultNamespace, blockName)) // Make sure new block is created - b, err := helper.BlockClient.List(clusterNamespace) + b, err := helper.BlockClient.ListAllImages(clusterNamespace) assert.NoError(s.T(), err) assert.Equal(s.T(), 1, len(b), "Make sure new block image is created") poolExists, err := helper.PoolClient.CephPoolExists(clusterNamespace, poolName) @@ -239,7 +240,7 @@ func deleteBlockLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite. assertNoErrorUnlessNotFound(s, err) assert.True(s.T(), k8sh.WaitUntilPVCIsDeleted(defaultNamespace, blockName)) if requireBlockImagesRemoved { - assert.True(s.T(), retryBlockImageCountCheck(helper, 0, clusterNamespace), "Make sure block images were deleted") + assert.NoError(s.T(), retryBlockImageCountCheck(helper, 0, clusterNamespace), "Make sure block images were deleted") } err = helper.PoolClient.DeletePool(helper.BlockClient, clusterNamespace, poolName) @@ -283,16 +284,19 @@ func blockTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s s // periodically checking if block image count has changed to expected value // When creating pvc in k8s platform, it may take some time for the block Image to be bounded -func retryBlockImageCountCheck(helper *clients.TestClient, expectedImageCount int, namespace string) bool { +func retryBlockImageCountCheck(helper *clients.TestClient, expectedImageCount int, namespace string) error { for i := 0; i < utils.RetryLoop; i++ { - blockImages, _ := helper.BlockClient.List(namespace) + blockImages, err := helper.BlockClient.ListAllImages(namespace) + if err != nil { + return err + } if expectedImageCount == len(blockImages) { - return true + return nil } logger.Infof("Waiting for block image count to reach %d. current=%d. %+v", expectedImageCount, len(blockImages), blockImages) time.Sleep(time.Second * utils.RetryInterval) } - return false + return fmt.Errorf("timed out waiting for image count to reach %d", expectedImageCount) } func retryPVCheck(k8sh *utils.K8sHelper, name string, exists bool, status string) bool {