From e1a9a9039b50243d64167e6d7c94355e45c8ca00 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 19 May 2020 22:02:37 -0600 Subject: [PATCH] ceph: pool resize test hangs during cleanup In Octopus 15.2.2 there is an rbd change that causes the rbd ls command to hang if there are not sufficient OSDs to meet the replica requirement. The smoke suite just started hanging today since the 15.2.2 image was released with this change. Therefore, the test that resizes the pool to replica 3 must resize back to 1 before cleaning up. Signed-off-by: Travis Nielsen --- tests/framework/clients/pool.go | 2 ++ tests/integration/ceph_smoke_test.go | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/framework/clients/pool.go b/tests/framework/clients/pool.go index 4cb77dab0..9f33418b6 100644 --- a/tests/framework/clients/pool.go +++ b/tests/framework/clients/pool.go @@ -106,8 +106,10 @@ func (p *PoolOperation) CephPoolExists(namespace, name string) (bool, error) { return false, nil } +// DeletePool deletes a pool after deleting all the block images contained by the pool func (p *PoolOperation) DeletePool(blockClient *BlockOperation, namespace, poolName string) error { // Delete all the images in a pool + logger.Infof("listing images in pool %q", poolName) blockImagesList, _ := blockClient.ListImagesInPool(namespace, poolName) for _, blockImage := range blockImagesList { logger.Infof("force deleting block image %q in pool %q", blockImage, poolName) diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 09a19e47e..f7ee02f12 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -214,28 +214,33 @@ func (suite *SmokeSuite) TestPoolResize() { require.Equal(suite.T(), true, poolFound, "pool not found") - err = suite.helper.PoolClient.Update(poolName, suite.namespace, 3) + err = suite.helper.PoolClient.Update(poolName, suite.namespace, 2) require.Nil(suite.T(), err) - poolFound = false + poolResized := false // Wait for pool resize to happen for i := 0; i < 10; i++ { - details, err := suite.helper.PoolClient.GetCephPoolDetails(suite.namespace, poolName) require.Nil(suite.T(), err) if details.Size > 1 { - logger.Infof("pool %s size got updated", poolName) - require.Equal(suite.T(), 3, int(details.Size)) - poolFound = true + logger.Infof("pool %s size was updated", poolName) + require.Equal(suite.T(), 2, int(details.Size)) + poolResized = true + + // resize the pool back to 1 to avoid hangs around not having enough OSDs to satisfy rbd + err = suite.helper.PoolClient.Update(poolName, suite.namespace, 1) + require.Nil(suite.T(), err) + } else if poolResized && details.Size == 1 { + logger.Infof("pool resized back to 1") break } - logger.Infof("pool %s size not updated yet. details: %+v", poolName, details) + logger.Debugf("pool %s size not updated yet. details: %+v", poolName, details) logger.Infof("Waiting for pool %s resize to happen", poolName) time.Sleep(2 * time.Second) } - require.Equal(suite.T(), true, poolFound, fmt.Sprintf("pool %s not found", poolName)) + require.Equal(suite.T(), true, poolResized, fmt.Sprintf("pool %s not found", poolName)) // clean up the pool suite.helper.PoolClient.DeletePool(suite.helper.BlockClient, suite.namespace, poolName)