test: run multiple object stores at once in CI
Multiple CephObjectStores should be able to run at the same time. In CI, test this by: 1. Create the primary object store under test (store A) (this was already done) 2. Verify that store A starts and becomes ready (already done but is moved here) 3. Create a second object store (store B) (new) 4. Verify that store B starts and becomes ready (new) 5. Delete store B, and ensure it is deleted (new) 6., etc. Continue to verify primary store A operations as before (this was already done) This is necessary to ensure that the Rook operator is using `radosgw-admin` and the RGW Admin Ops API properly to support multiple ObjectStores running at once. Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>
This commit is contained in:
1 file changed
+116
-78
@@ -123,7 +123,12 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
}
|
||||
|
||||
logger.Infof("Running on Rook Cluster %s", namespace)
|
||||
createCephObjectStore(s, helper, k8sh, namespace, storeName, 3, tlsEnable)
|
||||
createCephObjectStore(s.T(), helper, k8sh, namespace, storeName, 3, tlsEnable)
|
||||
|
||||
// test that a second object store can be created (and deleted) while the first exists
|
||||
testSecondObjectStore(s.T(), helper, k8sh, namespace, storeName+"-second", tlsEnable)
|
||||
|
||||
// now test operation of the first object store
|
||||
testObjectStoreOperations(s, helper, k8sh, namespace, storeName)
|
||||
}
|
||||
|
||||
@@ -188,27 +193,75 @@ func checkCephObjectUser(
|
||||
}
|
||||
}
|
||||
|
||||
func createCephObjectStore(s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string, replicaSize int, tlsEnable bool) {
|
||||
// create a CephObjectStore and wait for it to report ready status
|
||||
func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string, replicaSize int, tlsEnable bool) {
|
||||
logger.Infof("Create Object Store %q with replica count %d", storeName, replicaSize)
|
||||
rgwServiceName := "rook-ceph-rgw-" + storeName
|
||||
if tlsEnable {
|
||||
generateRgwTlsCertSecret(s, helper, k8sh, namespace, storeName, rgwServiceName)
|
||||
t.Run("generate TLS certs", func(t *testing.T) {
|
||||
generateRgwTlsCertSecret(t, helper, k8sh, namespace, storeName, rgwServiceName)
|
||||
})
|
||||
}
|
||||
t := s.T()
|
||||
t.Run("create CephObjectStore", func(t *testing.T) {
|
||||
err := helper.ObjectClient.Create(namespace, storeName, 3, tlsEnable)
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("wait for RGWs to be running", func(t *testing.T) {
|
||||
// check that ObjectStore is created
|
||||
logger.Infof("Check that RGW pods are Running")
|
||||
for i := 0; i < 24 && k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, 1, "Running") == false; i++ {
|
||||
logger.Infof("(%d) RGW pod check sleeping for 5 seconds ...", i)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, 1, "Running"))
|
||||
assert.True(t, k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, 1, "Running"))
|
||||
logger.Info("RGW pods are running")
|
||||
logger.Infof("Object store %q created successfully", storeName)
|
||||
})
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
// Check object store status
|
||||
t.Run("verify object store status", func(t *testing.T) {
|
||||
retryCount := 30
|
||||
i := 0
|
||||
for i = 0; i < retryCount; i++ {
|
||||
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
if objectStore.Status == nil || objectStore.Status.BucketStatus == nil {
|
||||
logger.Infof("(%d) object status check sleeping for 5 seconds ...%+v", i, objectStore.Status)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
logger.Info("objectstore status is", objectStore.Status)
|
||||
if objectStore.Status.BucketStatus.Health == cephv1.ConditionFailure {
|
||||
logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.BucketStatus)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
assert.Equal(t, cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health)
|
||||
// Info field has the endpoint in it
|
||||
assert.NotEmpty(t, objectStore.Status.Info)
|
||||
assert.NotEmpty(t, objectStore.Status.Info["endpoint"])
|
||||
break
|
||||
}
|
||||
if i == retryCount {
|
||||
t.Fatal("bucket status check failed. status is not connected")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// test creating a second object store while an object store already exists, once the second store
|
||||
// is found to be ready, it will be deleted. "storeName" should be unique.
|
||||
func testSecondObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string, tlsEnable bool) {
|
||||
t.Run("run a second object store", func(t *testing.T) {
|
||||
createCephObjectStore(t, helper, k8sh, namespace, storeName, 1, tlsEnable)
|
||||
|
||||
t.Run("delete the second object store", func(t *testing.T) {
|
||||
deleteObjectStore(t, k8sh, namespace, storeName)
|
||||
assertObjectStoreDeletion(t, k8sh, namespace, storeName)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
|
||||
@@ -228,35 +281,6 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *
|
||||
assert.NotEqual(t, 4, i)
|
||||
})
|
||||
|
||||
// Check object store status
|
||||
t.Run(fmt.Sprintf("verify ceph object store %q status", storeName), func(t *testing.T) {
|
||||
retryCount := 30
|
||||
i := 0
|
||||
for i = 0; i < retryCount; i++ {
|
||||
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
|
||||
assert.Nil(s.T(), err)
|
||||
if objectStore.Status == nil || objectStore.Status.BucketStatus == nil {
|
||||
logger.Infof("(%d) object status check sleeping for 5 seconds ...%+v", i, objectStore.Status)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
logger.Info("objectstore status is", objectStore.Status)
|
||||
if objectStore.Status.BucketStatus.Health == cephv1.ConditionFailure {
|
||||
logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.BucketStatus)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
assert.Equal(s.T(), cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health)
|
||||
// Info field has the endpoint in it
|
||||
assert.NotEmpty(s.T(), objectStore.Status.Info)
|
||||
assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"])
|
||||
break
|
||||
}
|
||||
if i == retryCount {
|
||||
t.Fatal("bucket status check failed. status is not connected")
|
||||
}
|
||||
})
|
||||
|
||||
context := k8sh.MakeContext()
|
||||
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
|
||||
assert.Nil(s.T(), err)
|
||||
@@ -360,11 +384,7 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *
|
||||
})
|
||||
|
||||
t.Run("delete CephObjectStore should be blocked by OBC bucket and CephObjectStoreUser", func(t *testing.T) {
|
||||
err := k8sh.DeleteResourceAndWait(false, "-n", namespace, "CephObjectStore", storeName)
|
||||
assert.NoError(t, err)
|
||||
// wait initially for the controller to detect deletion. Almost always enough, but not
|
||||
// waiting will almost always fail the first check in the loop
|
||||
time.Sleep(2 * time.Second)
|
||||
deleteObjectStore(t, k8sh, namespace, storeName)
|
||||
|
||||
store := &cephv1.CephObjectStore{}
|
||||
i := 0
|
||||
@@ -449,58 +469,76 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *
|
||||
// wait initially since it will almost never detect on the first try without this.
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
store := &cephv1.CephObjectStore{}
|
||||
i := 0
|
||||
retry := 10
|
||||
for i = 0; i < retry; i++ {
|
||||
storeStr, err := k8sh.GetResource("-n", namespace, "CephObjectStore", storeName, "-o", "json")
|
||||
assert.NoError(t, err)
|
||||
logger.Infof("store: \n%s", storeStr)
|
||||
|
||||
err = json.Unmarshal([]byte(storeStr), &store)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cond := cephv1.FindStatusCondition(store.Status.Conditions, cephv1.ConditionDeletionIsBlocked)
|
||||
if cond.Status == v1.ConditionFalse && cond.Reason == cephv1.ObjectHasNoDependentsReason {
|
||||
// Let's give some time to the object to be updated
|
||||
time.Sleep(5 * time.Second)
|
||||
break
|
||||
}
|
||||
logger.Info("waiting 3 more seconds for CephObjectStore to be unblocked by dependents")
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
assert.NotEqual(t, retry, i)
|
||||
|
||||
assert.Equal(t, cephv1.ConditionDeleting, store.Status.Phase) // phase == "Deleting"
|
||||
// verify deletion is NOT blocked b/c object has dependents
|
||||
cond := cephv1.FindStatusCondition(store.Status.Conditions, cephv1.ConditionDeletionIsBlocked)
|
||||
assert.Equal(t, v1.ConditionFalse, cond.Status)
|
||||
assert.Equal(t, cephv1.ObjectHasNoDependentsReason, cond.Reason)
|
||||
|
||||
err := k8sh.WaitUntilResourceIsDeleted("CephObjectStore", namespace, storeName)
|
||||
assert.NoError(t, err)
|
||||
assertObjectStoreDeletion(t, k8sh, namespace, storeName)
|
||||
})
|
||||
|
||||
// TODO : Add case for brownfield/cleanup s3 client}
|
||||
}
|
||||
|
||||
func generateRgwTlsCertSecret(s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName, rgwServiceName string) {
|
||||
func deleteObjectStore(t *testing.T, k8sh *utils.K8sHelper, namespace, storeName string) {
|
||||
err := k8sh.DeleteResourceAndWait(false, "-n", namespace, "CephObjectStore", storeName)
|
||||
assert.NoError(t, err)
|
||||
// wait initially for the controller to detect deletion. Almost always enough, but not
|
||||
// waiting immediately after this will almost always fail the first check in the loop
|
||||
time.Sleep(4 * time.Second)
|
||||
}
|
||||
|
||||
func assertObjectStoreDeletion(t *testing.T, k8sh *utils.K8sHelper, namespace, storeName string) {
|
||||
store := &cephv1.CephObjectStore{}
|
||||
i := 0
|
||||
retry := 10
|
||||
sleepTime := 3 * time.Second
|
||||
for i = 0; i < retry; i++ {
|
||||
storeStr, err := k8sh.GetResource("-n", namespace, "CephObjectStore", storeName, "-o", "json")
|
||||
assert.NoError(t, err)
|
||||
logger.Infof("store: \n%s", storeStr)
|
||||
|
||||
err = json.Unmarshal([]byte(storeStr), &store)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cond := cephv1.FindStatusCondition(store.Status.Conditions, cephv1.ConditionDeletionIsBlocked)
|
||||
if cond == nil {
|
||||
logger.Infof("waiting for CephObjectStore %q to have a deletion condition", storeName)
|
||||
time.Sleep(sleepTime)
|
||||
continue
|
||||
}
|
||||
if cond.Status == v1.ConditionFalse && cond.Reason == cephv1.ObjectHasNoDependentsReason {
|
||||
// no longer blocked by dependents
|
||||
time.Sleep(5 * time.Second) // Let's give some time to the object to be updated
|
||||
break
|
||||
}
|
||||
logger.Infof("waiting 3 more seconds for CephObjectStore %q to be unblocked by dependents", storeName)
|
||||
time.Sleep(sleepTime)
|
||||
}
|
||||
assert.NotEqual(t, retry, i)
|
||||
|
||||
assert.Equal(t, cephv1.ConditionDeleting, store.Status.Phase) // phase == "Deleting"
|
||||
// verify deletion is NOT blocked b/c object has dependents
|
||||
cond := cephv1.FindStatusCondition(store.Status.Conditions, cephv1.ConditionDeletionIsBlocked)
|
||||
assert.Equal(t, v1.ConditionFalse, cond.Status)
|
||||
assert.Equal(t, cephv1.ObjectHasNoDependentsReason, cond.Reason)
|
||||
|
||||
err := k8sh.WaitUntilResourceIsDeleted("CephObjectStore", namespace, storeName)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func generateRgwTlsCertSecret(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName, rgwServiceName string) {
|
||||
ctx := context.TODO()
|
||||
root, err := utils.FindRookRoot()
|
||||
require.NoError(s.T(), err, "failed to get rook root")
|
||||
require.NoError(t, err, "failed to get rook root")
|
||||
tlscertdir, err := ioutil.TempDir(root, "tlscertdir")
|
||||
require.NoError(s.T(), err, "failed to create directory for TLS certs")
|
||||
require.NoError(t, err, "failed to create directory for TLS certs")
|
||||
defer os.RemoveAll(tlscertdir)
|
||||
cmdArgs := utils.CommandArgs{Command: filepath.Join(root, "tests/scripts/generate-tls-config.sh"),
|
||||
CmdArgs: []string{tlscertdir, rgwServiceName, namespace}}
|
||||
cmdOut := utils.ExecuteCommand(cmdArgs)
|
||||
require.NoError(s.T(), cmdOut.Err)
|
||||
require.NoError(t, cmdOut.Err)
|
||||
tlsKeyIn, err := ioutil.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".key"))
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(t, err)
|
||||
tlsCertIn, err := ioutil.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".crt"))
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(t, err)
|
||||
tlsCaCertIn, err := ioutil.ReadFile(filepath.Join(tlscertdir, rgwServiceName+".ca"))
|
||||
require.NoError(s.T(), err)
|
||||
require.NoError(t, err)
|
||||
secretCertOut := fmt.Sprintf("%s%s%s", tlsKeyIn, tlsCertIn, tlsCaCertIn)
|
||||
tlsK8sSecret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -512,5 +550,5 @@ func generateRgwTlsCertSecret(s suite.Suite, helper *clients.TestClient, k8sh *u
|
||||
},
|
||||
}
|
||||
_, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(ctx, tlsK8sSecret, metav1.CreateOptions{})
|
||||
require.Nil(s.T(), err)
|
||||
require.Nil(t, err)
|
||||
}
|
||||
Reference in new issue
Block a user