Merge pull request #12995 from rook/mergify/bp/release-1.12/pr-12633

object: create cosi user for each object store (backport #12633)

Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md

@@ -43,33 +43,9 @@ The driver is created in the same namespace as Rook operator.
 
 ## Admin Operations
 
-### Create a Ceph Object Store User
-
-Create a CephObjectStoreUser to be used by the COSI driver for provisioning buckets.
-
-```yaml
-apiVersion: ceph.rook.io/v1
-kind: CephObjectStoreUser
-metadata:
-  name: cosi
-  namespace: rook-ceph
-spec:
-  displayName: "cosi user"
-  store: my-store
-  capabilities:
-    bucket: "*"
-    user: "*"
-```
-
-```console
-kubectl create -f cosi-user.yaml
-```
-
-Above step will be automated in future by the Rook operator.
-
 ### Create a BucketClass and BucketAccessClass
 
-The BucketClass and BucketAccessClass are CRDs defined by COSI. The BucketClass defines the storage class for the bucket. The BucketAccessClass defines the access class for the bucket. The BucketClass and BucketAccessClass are defined as below:
+The BucketClass and BucketAccessClass are CRDs defined by COSI. The BucketClass defines the storage class for the bucket. The BucketAccessClass defines the access class for the bucket. Rook will automatically create a secret named with `rook-ceph-object-user-<store-name>-cosi` which contains credentials used by the COSI driver. This secret is referred by the BucketClass and BucketAccessClass as defined below:
 
 ```yaml
 kind: BucketClass
@@ -97,8 +73,6 @@ parameters:
 kubectl create -f bucketclass.yaml -f bucketaccessclass.yaml
 ```
 
-The `objectStoreUserSecretName` and `objectStoreUserSecretNamespace` are the name and namespace of the CephObjectStoreUser created in the previous step.
-
 ## User Operations
 
 ### Create a Bucket

deploy/examples/cosi/cosi-user.yaml

@@ -1,16 +0,0 @@
-#################################################################################################################
-# Create an object store user for access to the ceph cosi driver.
-#  kubectl create -f cosi-user.yaml
-#################################################################################################################
-
-apiVersion: ceph.rook.io/v1
-kind: CephObjectStoreUser
-metadata:
-  name: cosi
-  namespace: rook-ceph # namespace:cluster
-spec:
-  store: my-store
-  displayName: "user for cosi"
-  capabilities:
-    user: "*"
-    bucket: "*"

pkg/operator/ceph/object/controller.go

@@ -24,6 +24,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/coreos/pkg/capnslog"
 	bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned"
 	"github.com/pkg/errors"
@@ -81,6 +82,9 @@ var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion
 // allow this to be overridden for unit tests
 var cephObjectStoreDependents = CephObjectStoreDependents
 
+// newMultisiteAdminOpsCtxFunc help us mocking the admin ops API client in unit test
+var newMultisiteAdminOpsCtxFunc = NewMultisiteAdminOpsContext
+
 // ReconcileCephObjectStore reconciles a cephObjectStore object
 type ReconcileCephObjectStore struct {
 	client           client.Client
@@ -239,11 +243,14 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
 		if err != nil {
 			return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context")
 		}
-		opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec)
+		opsCtx, err := newMultisiteAdminOpsCtxFunc(objCtx, &cephObjectStore.Spec)
 		if err != nil {
 			return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context")
 		}
-
+		err = r.deleteCOSIUser(opsCtx)
+		if err != nil {
+			return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to delete COSI user")
+		}
 		deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx)
 		if err != nil {
 			return reconcile.Result{}, *cephObjectStore, err
@@ -456,9 +463,11 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
 		if err != nil {
 			return reconcile.Result{}, errors.Wrapf(err, "failed to create object store %q", cephObjectStore.Name)
 		}
+
 	}
 
-	return reconcile.Result{}, nil
+	// Create COSI user and secret
+	return r.reconcileCOSIUser(cephObjectStore)
 }
 
 func (r *ReconcileCephObjectStore) reconcileCephZone(store *cephv1.CephObjectStore, zoneGroupName string, realmName string) (reconcile.Result, error) {
@@ -519,3 +528,61 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
 
 	return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil
 }
+
+func (r *ReconcileCephObjectStore) reconcileCOSIUser(cephObjectStore *cephv1.CephObjectStore) (reconcile.Result, error) {
+	// Create COSI user and secret
+	userConfig := generateCOSIUserConfig()
+	var user admin.User
+
+	// Create COSI user
+	objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to get object context")
+	}
+
+	adminOpsCtx, err := newMultisiteAdminOpsCtxFunc(objCtx, &cephObjectStore.Spec)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to get admin ops API context")
+	}
+
+	user, err = adminOpsCtx.AdminOpsClient.GetUser(r.opManagerContext, *userConfig)
+	if err != nil {
+		if errors.Is(err, admin.ErrNoSuchUser) {
+			logger.Infof("creating COSI user %q", userConfig.ID)
+			user, err = adminOpsCtx.AdminOpsClient.CreateUser(r.opManagerContext, *userConfig)
+			if err != nil {
+				return reconcile.Result{}, errors.Wrapf(err, "failed to create COSI user %q", userConfig.ID)
+			}
+		} else {
+			return reconcile.Result{}, errors.Wrapf(err, "failed to get COSI user %q", userConfig.ID)
+		}
+	}
+
+	// Create COSI user secret
+	return ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStore, &user, objCtx.Endpoint, cephObjectStore.Namespace, cephObjectStore.Name, cephObjectStore.Spec.Gateway.SSLCertificateRef)
+}
+
+func generateCOSIUserConfig() *admin.User {
+	userConfig := admin.User{
+		ID:          cosiUserName,
+		DisplayName: cosiUserName,
+	}
+
+	userConfig.UserCaps = cosiUserCaps
+
+	return &userConfig
+}
+
+func (r *ReconcileCephObjectStore) deleteCOSIUser(adminOpsCtx *AdminOpsContext) error {
+	userConfig := generateCOSIUserConfig()
+	err := adminOpsCtx.AdminOpsClient.RemoveUser(r.opManagerContext, *userConfig)
+	if err != nil {
+		if errors.Is(err, admin.ErrNoSuchUser) {
+			logger.Debugf("COSI user %q not found", userConfig.ID)
+			return nil
+		} else {
+			return errors.Wrapf(err, "failed to delete COSI user %q", userConfig.ID)
+		}
+	}
+	return nil
+}

pkg/operator/ceph/object/controller_test.go

@@ -18,7 +18,10 @@ limitations under the License.
 package object
 
 import (
+	"bytes"
 	"context"
+	"io"
+	"net/http"
 	"os"
 	"reflect"
 	"testing"
@@ -26,6 +29,7 @@ import (
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
+	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -320,6 +324,35 @@ var (
 	store     = "my-store"
 )
 
+var mockMultisiteAdminOpsCtxFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (*AdminOpsContext, error) {
+	mockClient := &MockClient{
+		MockDo: func(req *http.Request) (*http.Response, error) {
+			if req.Method == http.MethodGet || req.Method == http.MethodPut {
+				return &http.Response{
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewReader([]byte(userCreateJSON))),
+				}, nil
+			}
+			if req.Method == http.MethodDelete {
+				return &http.Response{
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewReader([]byte(""))),
+				}, nil
+			}
+			return nil, errors.Errorf("unexpected method %q", req.Method)
+		},
+	}
+	context := NewContext(objContext.Context, objContext.clusterInfo, store)
+	adminClient, _ := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient)
+
+	return &AdminOpsContext{
+		Context:               *context,
+		AdminOpsUserAccessKey: "EOE7FYCNOBZJ5VFV909G",
+		AdminOpsUserSecretKey: "qmIqpWm8HxCzmynCrD6U6vKWi4hnDBndOnmxXNsV",
+		AdminOpsClient:        adminClient,
+	}, nil
+}
+
 func TestCephObjectStoreController(t *testing.T) {
 	ctx := context.TODO()
 	// Set DEBUG logging
@@ -336,6 +369,13 @@ func TestCephObjectStoreController(t *testing.T) {
 		return nil
 	}
 
+	// overwrite adminops context func
+	oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc
+	newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc
+	defer func() {
+		newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc
+	}()
+
 	setupNewEnvironment := func(additionalObjects ...runtime.Object) *ReconcileCephObjectStore {
 		// reset var we use to check if we have called to commit config changes
 		calledCommitConfigChanges = false
@@ -377,9 +417,11 @@ func TestCephObjectStoreController(t *testing.T) {
 		s := scheme.Scheme
 		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{})
 		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
+		s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
 
 		// Create a fake client to mock API calls.
 		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
+
 		// Create a ReconcileCephObjectStore object with the scheme and fake client.
 		r := &ReconcileCephObjectStore{
 			client:           cl,
@@ -688,7 +730,12 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
 
 	commitConfigChangesOrig := commitConfigChanges
 	defer func() { commitConfigChanges = commitConfigChangesOrig }()
-
+	// overwrite adminops context func
+	oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc
+	newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc
+	defer func() {
+		newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc
+	}()
 	// make sure joining multisite calls to commit config changes
 	calledCommitConfigChanges := false
 	commitConfigChanges = func(c *Context) error {
@@ -711,7 +758,7 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
 	// Register operator types with the runtime scheme.
 	s := scheme.Scheme
 	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectZone{}, &cephv1.CephObjectZoneList{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &cephv1.CephObjectStore{}, &cephv1.CephObjectStoreList{})
-
+	s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
 	// Create a fake client to mock API calls.
 	cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
 
@@ -916,6 +963,13 @@ func TestCephObjectExternalStoreController(t *testing.T) {
 			rgwAdminOpsUserSecret,
 		}
 
+		// overwrite adminops context func
+		oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc
+		newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc
+		defer func() {
+			newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc
+		}()
+
 		r := getReconciler(objects)
 
 		t.Run("create an external object store", func(t *testing.T) {

pkg/operator/ceph/object/objectstore.go

@@ -53,6 +53,8 @@ const (
 	SecretKeyName         = "secret-key"
 	svcDNSSuffix          = "svc"
 	rgwRadosPoolPgNum     = "8"
+	cosiUserName          = "cosi"
+	cosiUserCaps          = "buckets=*;users=*"
 )
 
 var (

pkg/operator/ceph/object/user.go

@@ -17,13 +17,23 @@ limitations under the License.
 package object
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 	"strings"
 	"syscall"
 
 	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/pkg/errors"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/exec"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 const (
@@ -226,3 +236,53 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) {
 
 	return result, errors.Wrapf(err, "failed to delete s3 user uid=%q", id)
 }
+
+func GenerateCephUserSecretName(store, username string) string {
+	return fmt.Sprintf("rook-ceph-object-user-%s-%s", store, username)
+}
+
+func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) *corev1.Secret {
+	secretName := GenerateCephUserSecretName(storeName, userConfig.ID)
+	// Store the keys in a secret
+	secrets := map[string]string{
+		"AccessKey": userConfig.Keys[0].AccessKey,
+		"SecretKey": userConfig.Keys[0].SecretKey,
+		"Endpoint":  endpoint,
+	}
+	if tlsSecretName != "" {
+		secrets["SSLCertSecretName"] = tlsSecretName
+	}
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: namespace,
+			Labels: map[string]string{
+				"app":               AppName,
+				"user":              userConfig.ID,
+				"rook_cluster":      namespace,
+				"rook_object_store": storeName,
+			},
+		},
+		StringData: secrets,
+		Type:       k8sutil.RookType,
+	}
+	return secret
+}
+
+func ReconcileCephUserSecret(ctx context.Context, k8sclient client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (reconcile.Result, error) {
+	// Generate Kubernetes Secret
+	secret := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName)
+
+	// Set owner ref to the object store user object
+	err := controllerutil.SetControllerReference(ownerRef, secret, scheme)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name)
+	}
+
+	// Create Kubernetes Secret
+	err = opcontroller.CreateOrUpdateObject(ctx, k8sclient, secret)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph object user %q secret", secret.Name)
+	}
+	return reconcile.Result{}, nil
+}

pkg/operator/ceph/object/user/controller.go

@@ -29,7 +29,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -264,7 +263,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
 	}
 
 	tlsSecretName := store.Spec.Gateway.SSLCertificateRef
-	reconcileResponse, err = r.reconcileCephUserSecret(cephObjectStoreUser, tlsSecretName)
+	reconcileResponse, err = object.ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStoreUser, r.userConfig, r.objContext.Endpoint, cephObjectStoreUser.Namespace, cephObjectStoreUser.Spec.Store, tlsSecretName)
 	if err != nil {
 		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcileFailedStatus)
 		return reconcileResponse, *cephObjectStoreUser, err
@@ -501,62 +500,12 @@ func generateUserConfig(user *cephv1.CephObjectStoreUser) admin.User {
 	return userConfig
 }
 
-func generateCephUserSecretName(u *cephv1.CephObjectStoreUser) string {
-	return fmt.Sprintf("rook-ceph-object-user-%s-%s", u.Spec.Store, u.Name)
-}
-
 func generateStatusInfo(u *cephv1.CephObjectStoreUser) map[string]string {
 	m := make(map[string]string)
-	m["secretName"] = generateCephUserSecretName(u)
+	m["secretName"] = object.GenerateCephUserSecretName(u.Spec.Store, u.Name)
 	return m
 }
 
-func (r *ReconcileObjectStoreUser) generateCephUserSecret(u *cephv1.CephObjectStoreUser, tlsSecretName string) *corev1.Secret {
-	// Store the keys in a secret
-	secrets := map[string]string{
-		"AccessKey": r.userConfig.Keys[0].AccessKey,
-		"SecretKey": r.userConfig.Keys[0].SecretKey,
-		"Endpoint":  r.objContext.Endpoint,
-	}
-	if tlsSecretName != "" {
-		secrets["SSLCertSecretName"] = tlsSecretName
-	}
-	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      generateCephUserSecretName(u),
-			Namespace: u.Namespace,
-			Labels: map[string]string{
-				"app":               appName,
-				"user":              u.Name,
-				"rook_cluster":      u.Namespace,
-				"rook_object_store": u.Spec.Store,
-			},
-		},
-		StringData: secrets,
-		Type:       k8sutil.RookType,
-	}
-	return secret
-}
-
-func (r *ReconcileObjectStoreUser) reconcileCephUserSecret(cephObjectStoreUser *cephv1.CephObjectStoreUser, tlsSecretName string) (reconcile.Result, error) {
-	// Generate Kubernetes Secret
-	secret := r.generateCephUserSecret(cephObjectStoreUser, tlsSecretName)
-
-	// Set owner ref to the object store user object
-	err := controllerutil.SetControllerReference(cephObjectStoreUser, secret, r.scheme)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name)
-	}
-
-	// Create Kubernetes Secret
-	err = opcontroller.CreateOrUpdateObject(r.opManagerContext, r.client, secret)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph object user %q secret", secret.Name)
-	}
-
-	return reconcile.Result{}, nil
-}
-
 func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *cephv1.CephObjectStoreUser) error {
 	cephObjectStore, err := r.getObjectStore(cephObjectStoreUser.Spec.Store)
 	if err != nil {

tests/integration/ceph_cosi_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/rook/rook/pkg/operator/ceph/object"
 	rgw "github.com/rook/rook/pkg/operator/ceph/object"
 	"github.com/rook/rook/tests/framework/clients"
 	"github.com/rook/rook/tests/framework/installer"
@@ -55,8 +56,7 @@ func testCOSIDriver(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sH
 		assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=ceph-cosi-driver", operatorNamespace))
 	})
 
-	createCephObjectUser(s, helper, k8sh, namespace, objectStoreCOSI, cosiUser, true, true)
-	objectStoreUserSecretName := "rook-ceph-object-user" + "-" + objectStoreCOSI + "-" + cosiUser
+	objectStoreUserSecretName := object.GenerateCephUserSecretName(objectStoreCOSI, cosiUser)
 	t.Run("Creating BucketClass", func(t *testing.T) {
 		err := helper.COSIClient.CreateBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy)
 		assert.NoError(t, err, "failed to create BucketClass")