For the specification see:
<https://github.com/rook/rook/blob/master/design/ceph/object/swift-and-keystone-integration.md>
* extend the API object specs for swift and keystone integration
* adapt rgw to the new go-ceph version
- The parameter lists of the API call have changes, as parameters
ignored by the RGW Admin Ops API are no longer serialized, therefore
the mock has to be adapted.
- There is now validation for the user keys that are passed to the
User get API, therefore things failed when we had empty keys in our
User proxy object.
* expand the reconcile loop for the swift and keystone integration
* fix minor mistakes in design document
* add env var to pass extra args to minikube
Minikube decides CPU cores and memory automatically based on the
available resources on the machine which may be insufficient to
run rook. This commit adds an environment variable to add arbitrary
arguments to the minikube command, so both can be specified if
desired.
* integration tests for swift and keystone
The new integration of swift or s3 and keystone support by rook
does not have any integration tests yet.
This commit introduces integration tests for swift and keystone. The
tests are done against a minimal keystone setup (keystone container
image from Yaook-project (https://yaook.cloud), sqlite as database
backend, cert-manager and trust-manager for test certificate setup).
To prevent hardcoded credentials, passwords are generated
by the tests. The integration tests use the openstack client
(keystone- and swift-functionality) (https://docs.openstack.org/
python-openstackclient/ latest/). This was a concious design decision
to use client tooling as close as possible to the end user instead of
using other go-libraries (such as gophercloud).
* add documentation on swift and keystone
Currently there is no documentation on the use of Swift to access
an object store as well as the use of OpenStack keystone for
authentication.
This commit adds documentation on the use of Swift and OpenStack
keystone, as well as CRD-related documentation and an example setup.
* add integration tests for S3 via keystone
This commit introduces integration tests for s3 and keystone. The
tests are run against the same minimal keystone setup that the tests
for swift and keystone use.
The integration tests use the aws s3 client to use client tooling as
close as possible to the end user instead of using other go-libraries.
Co-authored-by: Jan Klippel <jan.klippel@uhurutec.com>
Co-authored-by: Silvio Ankermann <silvio.ankermann@cloudandheat.com>
Signed-off-by: Sebastian Riese <sebastian.riese@cloudandheat.com>
Signed-off-by: Jan Klippel <jan.klippel@uhurutec.com>
Signed-off-by: Silvio Ankermann <silvio.ankermann@cloudandheat.com>
(cherry picked from commit ee8bcad49d)
147 lines
5.7 KiB
Go
147 lines
5.7 KiB
Go
package integration
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/object"
|
|
rgw "github.com/rook/rook/pkg/operator/ceph/object"
|
|
"github.com/rook/rook/tests/framework/clients"
|
|
"github.com/rook/rook/tests/framework/installer"
|
|
"github.com/rook/rook/tests/framework/utils"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/suite"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
const (
|
|
objectStoreCOSI = "cosi-store"
|
|
cosiUser = "cosi"
|
|
bucketClassName = "cosi-bucketclass"
|
|
deletionPolicy = "Delete"
|
|
cosiBucketName = "cosi-bucket"
|
|
)
|
|
|
|
func testCOSIDriver(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, cephinstaller *installer.CephInstaller, namespace string) {
|
|
if utils.IsPlatformOpenShift() {
|
|
s.T().Skip("ceph cosi driver tests skipped on openshift")
|
|
}
|
|
|
|
t := s.T()
|
|
|
|
t.Run("COSI Controller and CRD installation", func(t *testing.T) {
|
|
_, err := k8sh.Kubectl("create", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-api")
|
|
assert.NoError(t, err, "failed to create COSI CRDs")
|
|
_, err = k8sh.Kubectl("create", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-controller")
|
|
assert.NoError(t, err, "failed to create COSI controller")
|
|
})
|
|
|
|
createCephObjectStore(s.T(), helper, k8sh, cephinstaller, namespace, objectStoreCOSI, 1, false, false)
|
|
|
|
t.Run("Creating CephCOSIDriver CRD", func(t *testing.T) {
|
|
err := helper.COSIClient.CreateCOSI()
|
|
assert.NoError(t, err, "failed to create Ceph COSI Driver CRD")
|
|
|
|
})
|
|
|
|
operatorNamespace := cephinstaller.Manifests.Settings().OperatorNamespace
|
|
t.Run("Check ceph cosi driver running", func(t *testing.T) {
|
|
for i := 24; i < 24 && k8sh.CheckPodCountAndState("ceph-cosi-driver", operatorNamespace, 1, "Running") == false; i++ {
|
|
logger.Infof("ceph-cosi-driver is not running, trying again")
|
|
k8sh.CheckPodCountAndState("ceph-cosi-driver", namespace, 1, "Running")
|
|
}
|
|
assert.True(t, k8sh.CheckPodCountAndState("ceph-cosi-driver", operatorNamespace, 1, "Running"))
|
|
assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=ceph-cosi-driver", operatorNamespace))
|
|
})
|
|
|
|
objectStoreUserSecretName := object.GenerateCephUserSecretName(objectStoreCOSI, cosiUser)
|
|
t.Run("Creating BucketClass", func(t *testing.T) {
|
|
err := helper.COSIClient.CreateBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy)
|
|
assert.NoError(t, err, "failed to create BucketClass")
|
|
})
|
|
|
|
t.Run("Creating BucketClaim", func(t *testing.T) {
|
|
err := helper.COSIClient.CreateBucketClaim(cosiBucketName, bucketClassName)
|
|
assert.NoError(t, err, "failed to create BucketClaim")
|
|
})
|
|
|
|
var cosiBucket string
|
|
t.Run("Check Bucket is ready", func(t *testing.T) {
|
|
_, err := k8sh.GetResource("bucketclaim", "-n", operatorNamespace, cosiBucketName)
|
|
assert.NoError(t, err, "failed to get BucketClaim")
|
|
cosiBucket, err = k8sh.GetResource("bucketclaim", "-n", operatorNamespace, cosiBucketName, "-o", "jsonpath={.status.bucketName}")
|
|
assert.NoError(t, err, "failed to get Bucket name")
|
|
i := 0
|
|
var bucketReady string
|
|
for i = 0; i < 4; i++ {
|
|
bucketReady, err = k8sh.GetResource("bucket", cosiBucket, "-o", "jsonpath={.status.bucketReady}")
|
|
if bucketReady == "true" && err == nil {
|
|
break
|
|
}
|
|
logger.Warningf("bucket %q is not ready, retrying... bucketReady=%q, err=%v", cosiBucket, bucketReady, err)
|
|
time.Sleep(5 * time.Second)
|
|
}
|
|
assert.NotEqual(t, 4, i)
|
|
assert.Equal(t, "true", bucketReady, "Bucket is not ready")
|
|
})
|
|
|
|
t.Run("Check Bucket is created in Backend Ceph", func(t *testing.T) {
|
|
ctx := context.TODO()
|
|
// check if bucket is created in the backend
|
|
context := k8sh.MakeContext()
|
|
clusterInfo := client.AdminTestClusterInfo(namespace)
|
|
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, objectStoreCOSI, metav1.GetOptions{})
|
|
assert.Nil(t, err)
|
|
rgwcontext, err := rgw.NewMultisiteContext(context, clusterInfo, objectStore)
|
|
assert.Nil(t, err)
|
|
var bkt rgw.ObjectBucket
|
|
i := 0
|
|
for i = 0; i < 4; i++ {
|
|
b, code, err := rgw.GetBucket(rgwcontext, cosiBucket)
|
|
if b != nil && err == nil {
|
|
bkt = *b
|
|
break
|
|
}
|
|
logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", cosiBucket, b, code, err)
|
|
logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
|
|
time.Sleep(5 * time.Second)
|
|
}
|
|
assert.NotEqual(t, 4, i)
|
|
assert.Equal(t, cosiBucket, bkt.Name)
|
|
})
|
|
|
|
t.Run("Deleting BucketClaim", func(t *testing.T) {
|
|
err := helper.COSIClient.DeleteBucketClaim(cosiBucketName, bucketClassName)
|
|
assert.NoError(t, err, "failed to delete BucketClaim")
|
|
})
|
|
|
|
t.Run("Deleting BucketClass", func(t *testing.T) {
|
|
err := helper.COSIClient.DeleteBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy)
|
|
assert.NoError(t, err, "failed to delete BucketClass")
|
|
})
|
|
|
|
t.Run("Deleting object user for cosi", func(t *testing.T) {
|
|
err := helper.ObjectUserClient.Delete(namespace, cosiUser)
|
|
assert.NoError(t, err, "failed to delete cosi user")
|
|
})
|
|
|
|
t.Run("delete CephObjectStore", func(t *testing.T) {
|
|
deleteObjectStore(t, k8sh, namespace, objectStoreCOSI)
|
|
assertObjectStoreDeletion(t, k8sh, namespace, objectStoreCOSI)
|
|
})
|
|
|
|
t.Run("delete CephCOSIDriver CRD", func(t *testing.T) {
|
|
err := helper.COSIClient.DeleteCOSI()
|
|
assert.NoError(t, err, "failed to delete Ceph COSI Driver CRD")
|
|
})
|
|
|
|
t.Run("delete COSI Controller", func(t *testing.T) {
|
|
_, err := k8sh.Kubectl("delete", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-api")
|
|
assert.NoError(t, err, "failed to create COSI CRDs")
|
|
_, err = k8sh.Kubectl("delete", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-controller")
|
|
assert.NoError(t, err, "failed to create COSI controller")
|
|
})
|
|
}
|