diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md index 059a059af..0cb7a73fe 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md @@ -43,33 +43,9 @@ The driver is created in the same namespace as Rook operator. ## Admin Operations -### Create a Ceph Object Store User - -Create a CephObjectStoreUser to be used by the COSI driver for provisioning buckets. - -```yaml -apiVersion: ceph.rook.io/v1 -kind: CephObjectStoreUser -metadata: - name: cosi - namespace: rook-ceph -spec: - displayName: "cosi user" - store: my-store - capabilities: - bucket: "*" - user: "*" -``` - -```console -kubectl create -f cosi-user.yaml -``` - -Above step will be automated in future by the Rook operator. - ### Create a BucketClass and BucketAccessClass -The BucketClass and BucketAccessClass are CRDs defined by COSI. The BucketClass defines the storage class for the bucket. The BucketAccessClass defines the access class for the bucket. The BucketClass and BucketAccessClass are defined as below: +The BucketClass and BucketAccessClass are CRDs defined by COSI. The BucketClass defines the storage class for the bucket. The BucketAccessClass defines the access class for the bucket. Rook will automatically create a secret named with `rook-ceph-object-user--cosi` which contains credentials used by the COSI driver. This secret is referred by the BucketClass and BucketAccessClass as defined below: ```yaml kind: BucketClass @@ -97,8 +73,6 @@ parameters: kubectl create -f bucketclass.yaml -f bucketaccessclass.yaml ``` -The `objectStoreUserSecretName` and `objectStoreUserSecretNamespace` are the name and namespace of the CephObjectStoreUser created in the previous step. - ## User Operations ### Create a Bucket diff --git a/deploy/examples/cosi/cosi-user.yaml b/deploy/examples/cosi/cosi-user.yaml deleted file mode 100644 index 9574d7aea..000000000 --- a/deploy/examples/cosi/cosi-user.yaml +++ /dev/null @@ -1,16 +0,0 @@ -################################################################################################################# -# Create an object store user for access to the ceph cosi driver. -# kubectl create -f cosi-user.yaml -################################################################################################################# - -apiVersion: ceph.rook.io/v1 -kind: CephObjectStoreUser -metadata: - name: cosi - namespace: rook-ceph # namespace:cluster -spec: - store: my-store - displayName: "user for cosi" - capabilities: - user: "*" - bucket: "*" diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index c56c69fbc..a5b8333f0 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -24,6 +24,7 @@ import ( "syscall" "time" + "github.com/ceph/go-ceph/rgw/admin" "github.com/coreos/pkg/capnslog" bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned" "github.com/pkg/errors" @@ -81,6 +82,9 @@ var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion // allow this to be overridden for unit tests var cephObjectStoreDependents = CephObjectStoreDependents +// newMultisiteAdminOpsCtxFunc help us mocking the admin ops API client in unit test +var newMultisiteAdminOpsCtxFunc = NewMultisiteAdminOpsContext + // ReconcileCephObjectStore reconciles a cephObjectStore object type ReconcileCephObjectStore struct { client client.Client @@ -239,11 +243,14 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci if err != nil { return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context") } - opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) + opsCtx, err := newMultisiteAdminOpsCtxFunc(objCtx, &cephObjectStore.Spec) if err != nil { return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context") } - + err = r.deleteCOSIUser(opsCtx) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to delete COSI user") + } deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx) if err != nil { return reconcile.Result{}, *cephObjectStore, err @@ -456,9 +463,11 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create object store %q", cephObjectStore.Name) } + } - return reconcile.Result{}, nil + // Create COSI user and secret + return r.reconcileCOSIUser(cephObjectStore) } func (r *ReconcileCephObjectStore) reconcileCephZone(store *cephv1.CephObjectStore, zoneGroupName string, realmName string) (reconcile.Result, error) { @@ -519,3 +528,61 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil } + +func (r *ReconcileCephObjectStore) reconcileCOSIUser(cephObjectStore *cephv1.CephObjectStore) (reconcile.Result, error) { + // Create COSI user and secret + userConfig := generateCOSIUserConfig() + var user admin.User + + // Create COSI user + objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to get object context") + } + + adminOpsCtx, err := newMultisiteAdminOpsCtxFunc(objCtx, &cephObjectStore.Spec) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to get admin ops API context") + } + + user, err = adminOpsCtx.AdminOpsClient.GetUser(r.opManagerContext, *userConfig) + if err != nil { + if errors.Is(err, admin.ErrNoSuchUser) { + logger.Infof("creating COSI user %q", userConfig.ID) + user, err = adminOpsCtx.AdminOpsClient.CreateUser(r.opManagerContext, *userConfig) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to create COSI user %q", userConfig.ID) + } + } else { + return reconcile.Result{}, errors.Wrapf(err, "failed to get COSI user %q", userConfig.ID) + } + } + + // Create COSI user secret + return ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStore, &user, objCtx.Endpoint, cephObjectStore.Namespace, cephObjectStore.Name, cephObjectStore.Spec.Gateway.SSLCertificateRef) +} + +func generateCOSIUserConfig() *admin.User { + userConfig := admin.User{ + ID: cosiUserName, + DisplayName: cosiUserName, + } + + userConfig.UserCaps = cosiUserCaps + + return &userConfig +} + +func (r *ReconcileCephObjectStore) deleteCOSIUser(adminOpsCtx *AdminOpsContext) error { + userConfig := generateCOSIUserConfig() + err := adminOpsCtx.AdminOpsClient.RemoveUser(r.opManagerContext, *userConfig) + if err != nil { + if errors.Is(err, admin.ErrNoSuchUser) { + logger.Debugf("COSI user %q not found", userConfig.ID) + return nil + } else { + return errors.Wrapf(err, "failed to delete COSI user %q", userConfig.ID) + } + } + return nil +} diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index b82d49b45..efdd30353 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -18,7 +18,10 @@ limitations under the License. package object import ( + "bytes" "context" + "io" + "net/http" "os" "reflect" "testing" @@ -26,6 +29,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/ceph/go-ceph/rgw/admin" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -320,6 +324,35 @@ var ( store = "my-store" ) +var mockMultisiteAdminOpsCtxFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (*AdminOpsContext, error) { + mockClient := &MockClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodGet || req.Method == http.MethodPut { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte(userCreateJSON))), + }, nil + } + if req.Method == http.MethodDelete { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte(""))), + }, nil + } + return nil, errors.Errorf("unexpected method %q", req.Method) + }, + } + context := NewContext(objContext.Context, objContext.clusterInfo, store) + adminClient, _ := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient) + + return &AdminOpsContext{ + Context: *context, + AdminOpsUserAccessKey: "EOE7FYCNOBZJ5VFV909G", + AdminOpsUserSecretKey: "qmIqpWm8HxCzmynCrD6U6vKWi4hnDBndOnmxXNsV", + AdminOpsClient: adminClient, + }, nil +} + func TestCephObjectStoreController(t *testing.T) { ctx := context.TODO() // Set DEBUG logging @@ -336,6 +369,13 @@ func TestCephObjectStoreController(t *testing.T) { return nil } + // overwrite adminops context func + oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc + newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc + defer func() { + newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc + }() + setupNewEnvironment := func(additionalObjects ...runtime.Object) *ReconcileCephObjectStore { // reset var we use to check if we have called to commit config changes calledCommitConfigChanges = false @@ -377,9 +417,11 @@ func TestCephObjectStoreController(t *testing.T) { s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{}) s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() + // Create a ReconcileCephObjectStore object with the scheme and fake client. r := &ReconcileCephObjectStore{ client: cl, @@ -688,7 +730,12 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { commitConfigChangesOrig := commitConfigChanges defer func() { commitConfigChanges = commitConfigChangesOrig }() - + // overwrite adminops context func + oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc + newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc + defer func() { + newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc + }() // make sure joining multisite calls to commit config changes calledCommitConfigChanges := false commitConfigChanges = func(c *Context) error { @@ -711,7 +758,7 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectZone{}, &cephv1.CephObjectZoneList{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &cephv1.CephObjectStore{}, &cephv1.CephObjectStoreList{}) - + s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() @@ -916,6 +963,13 @@ func TestCephObjectExternalStoreController(t *testing.T) { rgwAdminOpsUserSecret, } + // overwrite adminops context func + oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc + newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc + defer func() { + newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc + }() + r := getReconciler(objects) t.Run("create an external object store", func(t *testing.T) { diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 5733b6f22..e2e8d22c1 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -53,6 +53,8 @@ const ( SecretKeyName = "secret-key" svcDNSSuffix = "svc" rgwRadosPoolPgNum = "8" + cosiUserName = "cosi" + cosiUserCaps = "buckets=*;users=*" ) var ( diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index d435c2995..7cb2b670b 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -17,13 +17,23 @@ limitations under the License. package object import ( + "context" "encoding/json" + "fmt" "strings" "syscall" "github.com/ceph/go-ceph/rgw/admin" "github.com/pkg/errors" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) const ( @@ -226,3 +236,53 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) { return result, errors.Wrapf(err, "failed to delete s3 user uid=%q", id) } + +func GenerateCephUserSecretName(store, username string) string { + return fmt.Sprintf("rook-ceph-object-user-%s-%s", store, username) +} + +func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) *corev1.Secret { + secretName := GenerateCephUserSecretName(storeName, userConfig.ID) + // Store the keys in a secret + secrets := map[string]string{ + "AccessKey": userConfig.Keys[0].AccessKey, + "SecretKey": userConfig.Keys[0].SecretKey, + "Endpoint": endpoint, + } + if tlsSecretName != "" { + secrets["SSLCertSecretName"] = tlsSecretName + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + Labels: map[string]string{ + "app": AppName, + "user": userConfig.ID, + "rook_cluster": namespace, + "rook_object_store": storeName, + }, + }, + StringData: secrets, + Type: k8sutil.RookType, + } + return secret +} + +func ReconcileCephUserSecret(ctx context.Context, k8sclient client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (reconcile.Result, error) { + // Generate Kubernetes Secret + secret := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName) + + // Set owner ref to the object store user object + err := controllerutil.SetControllerReference(ownerRef, secret, scheme) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name) + } + + // Create Kubernetes Secret + err = opcontroller.CreateOrUpdateObject(ctx, k8sclient, secret) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph object user %q secret", secret.Name) + } + return reconcile.Result{}, nil +} diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index 2834d89c5..1c7624e6c 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -264,7 +263,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci } tlsSecretName := store.Spec.Gateway.SSLCertificateRef - reconcileResponse, err = r.reconcileCephUserSecret(cephObjectStoreUser, tlsSecretName) + reconcileResponse, err = object.ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStoreUser, r.userConfig, r.objContext.Endpoint, cephObjectStoreUser.Namespace, cephObjectStoreUser.Spec.Store, tlsSecretName) if err != nil { r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcileResponse, *cephObjectStoreUser, err @@ -501,62 +500,12 @@ func generateUserConfig(user *cephv1.CephObjectStoreUser) admin.User { return userConfig } -func generateCephUserSecretName(u *cephv1.CephObjectStoreUser) string { - return fmt.Sprintf("rook-ceph-object-user-%s-%s", u.Spec.Store, u.Name) -} - func generateStatusInfo(u *cephv1.CephObjectStoreUser) map[string]string { m := make(map[string]string) - m["secretName"] = generateCephUserSecretName(u) + m["secretName"] = object.GenerateCephUserSecretName(u.Spec.Store, u.Name) return m } -func (r *ReconcileObjectStoreUser) generateCephUserSecret(u *cephv1.CephObjectStoreUser, tlsSecretName string) *corev1.Secret { - // Store the keys in a secret - secrets := map[string]string{ - "AccessKey": r.userConfig.Keys[0].AccessKey, - "SecretKey": r.userConfig.Keys[0].SecretKey, - "Endpoint": r.objContext.Endpoint, - } - if tlsSecretName != "" { - secrets["SSLCertSecretName"] = tlsSecretName - } - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: generateCephUserSecretName(u), - Namespace: u.Namespace, - Labels: map[string]string{ - "app": appName, - "user": u.Name, - "rook_cluster": u.Namespace, - "rook_object_store": u.Spec.Store, - }, - }, - StringData: secrets, - Type: k8sutil.RookType, - } - return secret -} - -func (r *ReconcileObjectStoreUser) reconcileCephUserSecret(cephObjectStoreUser *cephv1.CephObjectStoreUser, tlsSecretName string) (reconcile.Result, error) { - // Generate Kubernetes Secret - secret := r.generateCephUserSecret(cephObjectStoreUser, tlsSecretName) - - // Set owner ref to the object store user object - err := controllerutil.SetControllerReference(cephObjectStoreUser, secret, r.scheme) - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name) - } - - // Create Kubernetes Secret - err = opcontroller.CreateOrUpdateObject(r.opManagerContext, r.client, secret) - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph object user %q secret", secret.Name) - } - - return reconcile.Result{}, nil -} - func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *cephv1.CephObjectStoreUser) error { cephObjectStore, err := r.getObjectStore(cephObjectStoreUser.Spec.Store) if err != nil { diff --git a/tests/integration/ceph_cosi_test.go b/tests/integration/ceph_cosi_test.go index f56e50537..732771aef 100644 --- a/tests/integration/ceph_cosi_test.go +++ b/tests/integration/ceph_cosi_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/object" rgw "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" @@ -55,8 +56,7 @@ func testCOSIDriver(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sH assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=ceph-cosi-driver", operatorNamespace)) }) - createCephObjectUser(s, helper, k8sh, namespace, objectStoreCOSI, cosiUser, true, true) - objectStoreUserSecretName := "rook-ceph-object-user" + "-" + objectStoreCOSI + "-" + cosiUser + objectStoreUserSecretName := object.GenerateCephUserSecretName(objectStoreCOSI, cosiUser) t.Run("Creating BucketClass", func(t *testing.T) { err := helper.COSIClient.CreateBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy) assert.NoError(t, err, "failed to create BucketClass")