From ca43800119f93ee995df64e234f5eab41a089333 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Mon, 28 Jun 2021 14:40:54 +0530 Subject: [PATCH] ceph: add options for cephobjectstore user Adding options for quota, bucket limit, caps for the `cephobjectstoreuser`. Signed-off-by: Jiffin Tony Thottan --- Documentation/ceph-object-store-user-crd.md | 21 +++ .../charts/rook-ceph/templates/resources.yaml | 67 ++++++++ cluster/examples/kubernetes/ceph/crds.yaml | 67 ++++++++ .../examples/kubernetes/ceph/object-user.yaml | 12 ++ go.mod | 2 +- go.sum | 4 +- pkg/apis/ceph.rook.io/v1/types.go | 52 +++++- .../ceph.rook.io/v1/zz_generated.deepcopy.go | 59 ++++++- pkg/operator/ceph/object/objectstore_test.go | 9 +- pkg/operator/ceph/object/user.go | 29 +++- pkg/operator/ceph/object/user/controller.go | 57 ++++++- .../ceph/object/user/controller_test.go | 158 +++++++++++++++++- tests/framework/clients/object_user.go | 4 +- tests/framework/installer/ceph_manifests.go | 12 +- .../installer/ceph_manifests_v1.6.go | 2 +- tests/integration/ceph_base_object_test.go | 29 +++- tests/integration/ceph_upgrade_test.go | 8 +- 17 files changed, 556 insertions(+), 36 deletions(-) diff --git a/Documentation/ceph-object-store-user-crd.md b/Documentation/ceph-object-store-user-crd.md index 34a77d45c..7c47bc1a8 100644 --- a/Documentation/ceph-object-store-user-crd.md +++ b/Documentation/ceph-object-store-user-crd.md @@ -20,6 +20,13 @@ metadata: spec: store: my-store displayName: my-display-name + quotas: + maxBuckets: 100 + maxSize: 10G + maxObjects: 10000 + capabilities: + user: "*" + bucket: "*" ``` ## Object Store User Settings @@ -33,3 +40,17 @@ spec: * `store`: The object store in which the user will be created. This matches the name of the objectstore CRD. * `displayName`: The display name which will be passed to the `radosgw-admin user create` command. +* `quotas`: This represents quota limitation can be set on the user(support added from onwards v1.7.3). + Please refer [here](https://docs.ceph.com/en/latest/radosgw/admin/#quota-management) for details. + * `maxBuckets`: The maximum bucket limit for the user. + * `maxSize`: Maximum size limit of all objects across all the user's buckets. + * `maxObjects`: Maximum number of objects across all the user's buckets. +* `capabilities`: Ceph allows users to be given additional permissions(support added from onwards v1.7.3). + P.S this setting can used only during the creation of the object store user, not afterwards. + See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/#add-remove-admin-capabilities) for more info. + Rook supports adding `read`, `write`, `read, write`, or `*` permissions for the following resources: + * `users` + * `buckets` + * `usage` + * `metadata` + * `zone` diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 1d687d036..b01594519 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -7623,9 +7623,76 @@ spec: spec: description: ObjectStoreUserSpec represent the spec of an Objectstoreuser properties: + capabilities: + description: Additional admin-level capabilities for the Ceph object store user + nullable: true + properties: + bucket: + description: Admin capabilities to read/write Ceph object store buckets. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + metadata: + description: Admin capabilities to read/write Ceph object store metadata. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + usage: + description: Admin capabilities to read/write Ceph object store usage. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + user: + description: Admin capabilities to read/write Ceph object store users. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + zone: + description: Admin capabilities to read/write Ceph object store zones. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + type: object displayName: description: The display name for the ceph users type: string + quotas: + description: ObjectUserQuotaSpec can be used to set quotas for the object store user to limit their usage. See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/?#quota-management) for more + nullable: true + properties: + maxBuckets: + description: Maximum bucket limit for the ceph user + nullable: true + type: integer + maxObjects: + description: Maximum number of objects across all the user's buckets + format: int64 + nullable: true + type: integer + maxSize: + anyOf: + - type: integer + - type: string + description: Maximum size limit of all objects across all the user's buckets See https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity for more info. + nullable: true + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object store: description: The store the user will be created in type: string diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index dac3739ab..02d7b1442 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -7617,9 +7617,76 @@ spec: spec: description: ObjectStoreUserSpec represent the spec of an Objectstoreuser properties: + capabilities: + description: Additional admin-level capabilities for the Ceph object store user + nullable: true + properties: + bucket: + description: Admin capabilities to read/write Ceph object store buckets. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + metadata: + description: Admin capabilities to read/write Ceph object store metadata. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + usage: + description: Admin capabilities to read/write Ceph object store usage. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + user: + description: Admin capabilities to read/write Ceph object store users. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + zone: + description: Admin capabilities to read/write Ceph object store zones. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + type: object displayName: description: The display name for the ceph users type: string + quotas: + description: ObjectUserQuotaSpec can be used to set quotas for the object store user to limit their usage. See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/?#quota-management) for more + nullable: true + properties: + maxBuckets: + description: Maximum bucket limit for the ceph user + nullable: true + type: integer + maxObjects: + description: Maximum number of objects across all the user's buckets + format: int64 + nullable: true + type: integer + maxSize: + anyOf: + - type: integer + - type: string + description: Maximum size limit of all objects across all the user's buckets See https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity for more info. + nullable: true + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object store: description: The store the user will be created in type: string diff --git a/cluster/examples/kubernetes/ceph/object-user.yaml b/cluster/examples/kubernetes/ceph/object-user.yaml index bf2b6b41f..8ae3d2413 100644 --- a/cluster/examples/kubernetes/ceph/object-user.yaml +++ b/cluster/examples/kubernetes/ceph/object-user.yaml @@ -11,3 +11,15 @@ metadata: spec: store: my-store displayName: "my display name" + # Quotas set on the user + # quotas: + # maxBuckets: 100 + # maxSize: 10G + # maxObjects: 10000 + # Additional permissions given to the user + # capabilities: + # user: "*" + # bucket: "*" + # metadata: "*" + # usage: "*" + # zone: "*" diff --git a/go.mod b/go.mod index 9896ab5a6..f9fa78158 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.16 require ( github.com/aws/aws-sdk-go v1.37.19 github.com/banzaicloud/k8s-objectmatcher v1.1.0 - github.com/ceph/go-ceph v0.10.1-0.20210729101705-11f319727ffb + github.com/ceph/go-ceph v0.11.0 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/csi-addons/volume-replication-operator v0.1.1-0.20210525040814-ab575a2879fb github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index bfc2dcd65..bc2d72032 100644 --- a/go.sum +++ b/go.sum @@ -264,8 +264,8 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/centrify/cloud-golang-sdk v0.0.0-20190214225812-119110094d0f h1:gJzxrodnNd/CtPXjO3WYiakyNzHg3rtAi7rO74ejHYU= github.com/centrify/cloud-golang-sdk v0.0.0-20190214225812-119110094d0f/go.mod h1:C0rtzmGXgN78pYR0tGJFhtHgkbAs0lIbHwkB81VxDQE= -github.com/ceph/go-ceph v0.10.1-0.20210729101705-11f319727ffb h1:rkflsGZM6dOf1GcbnPF3J0P72NwKVhqXgleFf3Nuqb4= -github.com/ceph/go-ceph v0.10.1-0.20210729101705-11f319727ffb/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= +github.com/ceph/go-ceph v0.11.0 h1:A1pphV40LL8GQKDPpU4XqCa7gkmozsst7rhCC730/nk= +github.com/ceph/go-ceph v0.11.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 54b9d50ae..c41806f5f 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -21,6 +21,7 @@ import ( rook "github.com/rook/rook/pkg/apis/rook.io" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -1431,12 +1432,59 @@ type CephObjectStoreUserList struct { // ObjectStoreUserSpec represent the spec of an Objectstoreuser type ObjectStoreUserSpec struct { - //The store the user will be created in + // The store the user will be created in // +optional Store string `json:"store,omitempty"` - //The display name for the ceph users + // The display name for the ceph users // +optional DisplayName string `json:"displayName,omitempty"` + // +optional + // +nullable + Capabilities *ObjectUserCapSpec `json:"capabilities,omitempty"` + // +optional + // +nullable + Quotas *ObjectUserQuotaSpec `json:"quotas,omitempty"` +} + +// Additional admin-level capabilities for the Ceph object store user +type ObjectUserCapSpec struct { + // +optional + // +kubebuilder:validation:Enum={"*","read","write","read, write"} + // Admin capabilities to read/write Ceph object store users. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + User string `json:"user,omitempty"` + // +optional + // +kubebuilder:validation:Enum={"*","read","write","read, write"} + // Admin capabilities to read/write Ceph object store buckets. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + Bucket string `json:"bucket,omitempty"` + // +optional + // +kubebuilder:validation:Enum={"*","read","write","read, write"} + // Admin capabilities to read/write Ceph object store metadata. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + MetaData string `json:"metadata,omitempty"` + // +optional + // +kubebuilder:validation:Enum={"*","read","write","read, write"} + // Admin capabilities to read/write Ceph object store usage. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + Usage string `json:"usage,omitempty"` + // +optional + // +kubebuilder:validation:Enum={"*","read","write","read, write"} + // Admin capabilities to read/write Ceph object store zones. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + Zone string `json:"zone,omitempty"` +} + +// ObjectUserQuotaSpec can be used to set quotas for the object store user to limit their usage. See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/?#quota-management) for more +type ObjectUserQuotaSpec struct { + // Maximum bucket limit for the ceph user + // +optional + // +nullable + MaxBuckets *int `json:"maxBuckets,omitempty"` + // Maximum size limit of all objects across all the user's buckets + // See https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity for more info. + // +optional + // +nullable + MaxSize *resource.Quantity `json:"maxSize,omitempty"` + // Maximum number of objects across all the user's buckets + // +optional + // +nullable + MaxObjects *int64 `json:"maxObjects,omitempty"` } // CephObjectRealm represents a Ceph Object Store Gateway Realm diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index d52c33b31..f213a01c5 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -848,7 +848,7 @@ func (in *CephObjectStoreUser) DeepCopyInto(out *CephObjectStoreUser) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) if in.Status != nil { in, out := &in.Status, &out.Status *out = new(ObjectStoreUserStatus) @@ -2336,6 +2336,16 @@ func (in *ObjectStoreStatus) DeepCopy() *ObjectStoreStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStoreUserSpec) DeepCopyInto(out *ObjectStoreUserSpec) { *out = *in + if in.Capabilities != nil { + in, out := &in.Capabilities, &out.Capabilities + *out = new(ObjectUserCapSpec) + **out = **in + } + if in.Quotas != nil { + in, out := &in.Quotas, &out.Quotas + *out = new(ObjectUserQuotaSpec) + (*in).DeepCopyInto(*out) + } return } @@ -2372,6 +2382,53 @@ func (in *ObjectStoreUserStatus) DeepCopy() *ObjectStoreUserStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectUserCapSpec) DeepCopyInto(out *ObjectUserCapSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectUserCapSpec. +func (in *ObjectUserCapSpec) DeepCopy() *ObjectUserCapSpec { + if in == nil { + return nil + } + out := new(ObjectUserCapSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectUserQuotaSpec) DeepCopyInto(out *ObjectUserQuotaSpec) { + *out = *in + if in.MaxBuckets != nil { + in, out := &in.MaxBuckets, &out.MaxBuckets + *out = new(int) + **out = **in + } + if in.MaxSize != nil { + in, out := &in.MaxSize, &out.MaxSize + x := (*in).DeepCopy() + *out = &x + } + if in.MaxObjects != nil { + in, out := &in.MaxObjects, &out.MaxObjects + *out = new(int64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectUserQuotaSpec. +func (in *ObjectUserQuotaSpec) DeepCopy() *ObjectUserQuotaSpec { + if in == nil { + return nil + } + out := new(ObjectUserQuotaSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectZoneGroupSpec) DeepCopyInto(out *ObjectZoneGroupSpec) { *out = *in diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 4c79be443..2bc03cfe0 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -57,7 +57,14 @@ const ( "system": "true", "temp_url_keys": [], "type": "rgw", - "mfa_ids": [] + "mfa_ids": [], + "user_quota": { + "enabled": false, + "check_on_raw": false, + "max_size": -1, + "max_size_kb": 0, + "max_objects": -1 + } }` access_key = "VFKF8SSU9L3L2UR03Z8C" ) diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index a5a7ad4e2..9eda431c8 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -37,13 +37,16 @@ const ( // An ObjectUser defines the details of an object store user. type ObjectUser struct { - UserID string `json:"userId"` - DisplayName *string `json:"displayName"` - Email *string `json:"email"` - AccessKey *string `json:"accessKey"` - SecretKey *string `json:"secretKey"` - SystemUser bool `json:"systemuser"` - AdminOpsUser bool `json:"adminopsuser"` + UserID string `json:"userId"` + DisplayName *string `json:"displayName"` + Email *string `json:"email"` + AccessKey *string `json:"accessKey"` + SecretKey *string `json:"secretKey"` + SystemUser bool `json:"systemuser"` + AdminOpsUser bool `json:"adminopsuser"` + MaxBuckets int `json:"max_buckets"` + UserQuota admin.QuotaSpec `json:"user_quota"` + Caps []admin.UserCapSpec `json:"caps"` } // func decodeUser(data string) (*ObjectUser, int, error) { @@ -56,6 +59,18 @@ func decodeUser(data string) (*ObjectUser, int, error) { rookUser := ObjectUser{UserID: user.ID, DisplayName: &user.DisplayName, Email: &user.Email} + if len(user.Caps) > 0 { + rookUser.Caps = user.Caps + } + + if user.MaxBuckets != nil { + rookUser.MaxBuckets = *user.MaxBuckets + } + + if user.UserQuota.Enabled != nil { + rookUser.UserQuota = user.UserQuota + } + if len(user.Keys) > 0 { rookUser.AccessKey = &user.Keys[0].AccessKey rookUser.SecretKey = &user.Keys[0].SecretKey diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index b28e996e1..f1b9227a6 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -281,13 +281,44 @@ func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectSt } else { return errors.Wrapf(err, "failed to get details from ceph object user %q", u.Name) } + } else if *user.MaxBuckets != *r.userConfig.MaxBuckets { + // TODO handle update for user capabilities + user, err = r.objContext.AdminOpsClient.ModifyUser(context.TODO(), *r.userConfig) + if err != nil { + return errors.Wrapf(err, "failed to create ceph object user %v", &r.userConfig.ID) + } + logCreateOrUpdate = fmt.Sprintf("updated ceph object user %q", u.Name) + } + + var quotaEnabled = false + var maxSize int64 = -1 + var maxObjects int64 = -1 + if u.Spec.Quotas != nil { + if u.Spec.Quotas.MaxObjects != nil { + maxObjects = *u.Spec.Quotas.MaxObjects + quotaEnabled = true + } + if u.Spec.Quotas.MaxSize != nil { + maxSize = u.Spec.Quotas.MaxSize.Value() + quotaEnabled = true + } + } + userQuota := admin.QuotaSpec{ + UID: u.Name, + Enabled: "aEnabled, + MaxSize: &maxSize, + MaxObjects: &maxObjects, + } + err = r.objContext.AdminOpsClient.SetUserQuota(context.TODO(), userQuota) + if err != nil { + return errors.Wrapf(err, "failed to set quotas for user %q", u.Name) } // Set access and secret key r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey r.userConfig.Keys[0].SecretKey = user.Keys[0].SecretKey - logger.Info(logCreateOrUpdate) + return nil } @@ -334,6 +365,30 @@ func generateUserConfig(user *cephv1.CephObjectStoreUser) admin.User { Keys: make([]admin.UserKeySpec, 1), } + defaultMaxBuckets := 1000 + userConfig.MaxBuckets = &defaultMaxBuckets + if user.Spec.Quotas != nil && user.Spec.Quotas.MaxBuckets != nil { + userConfig.MaxBuckets = user.Spec.Quotas.MaxBuckets + } + + if user.Spec.Capabilities != nil { + if user.Spec.Capabilities.User != "" { + userConfig.UserCaps += fmt.Sprintf("users=%s;", user.Spec.Capabilities.User) + } + if user.Spec.Capabilities.Bucket != "" { + userConfig.UserCaps += fmt.Sprintf("buckets=%s;", user.Spec.Capabilities.Bucket) + } + if user.Spec.Capabilities.MetaData != "" { + userConfig.UserCaps += fmt.Sprintf("metadata=%s;", user.Spec.Capabilities.MetaData) + } + if user.Spec.Capabilities.Usage != "" { + userConfig.UserCaps += fmt.Sprintf("usage=%s;", user.Spec.Capabilities.Usage) + } + if user.Spec.Capabilities.Zone != "" { + userConfig.UserCaps += fmt.Sprintf("zone=%s;", user.Spec.Capabilities.Zone) + } + } + return userConfig } diff --git a/pkg/operator/ceph/object/user/controller_test.go b/pkg/operator/ceph/object/user/controller_test.go index b2fec0a04..ad4550e50 100644 --- a/pkg/operator/ceph/object/user/controller_test.go +++ b/pkg/operator/ceph/object/user/controller_test.go @@ -38,6 +38,7 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -88,9 +89,12 @@ const ( ) var ( - name = "my-user" - namespace = "rook-ceph" - store = "my-store" + name = "my-user" + namespace = "rook-ceph" + store = "my-store" + maxbucket = 200 + maxsizestr = "10G" + maxobject int64 = 10000 ) func TestCephObjectStoreUserController(t *testing.T) { @@ -306,7 +310,8 @@ func TestCephObjectStoreUserController(t *testing.T) { newMultisiteAdminOpsCtxFunc = func(objContext *cephobject.Context, spec *cephv1.ObjectStoreSpec) (*cephobject.AdminOpsContext, error) { mockClient := &cephobject.MockClient{ MockDo: func(req *http.Request) (*http.Response, error) { - if req.URL.RawQuery == "display-name=my-user&format=json&uid=my-user" && req.Method == http.MethodGet && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user" { + if (req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=1000&uid=my-user" && (req.Method == http.MethodGet || req.Method == http.MethodPost) && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user") || + (req.URL.RawQuery == "enabled=false&format=json&max-objects=-1&max-size=-1"a="a-type=user&uid=my-user" && req.Method == http.MethodPut && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user") { return &http.Response{ StatusCode: 200, Body: ioutil.NopCloser(bytes.NewReader([]byte(userCreateJSON))), @@ -353,3 +358,148 @@ func TestBuildUpdateStatusInfo(t *testing.T) { assert.NotEmpty(t, statusInfo["secretName"]) assert.Equal(t, "rook-ceph-object-user-my-store-my-user", statusInfo["secretName"]) } + +func TestCreateorUpdateCephUser(t *testing.T) { + // Set DEBUG logging + capnslog.SetGlobalLogLevel(capnslog.DEBUG) + + objectUser := &cephv1.CephObjectStoreUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + Namespace: namespace, + }, + Spec: cephv1.ObjectStoreUserSpec{ + Store: store, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "CephObjectStoreUser", + }, + } + mockClient := &cephobject.MockClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "rook-ceph-rgw-my-store.mycluster.svc/admin/user" { + return nil, fmt.Errorf("unexpected url path %q", req.URL.Path) + } + + if req.Method == http.MethodGet || req.Method == http.MethodPost { + if req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=1000&uid=my-user" || + req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=200&uid=my-user" || + req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=1000&uid=my-user&user-caps=users%3Dread%3Bbuckets%3Dread%3B" || + req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=200&uid=my-user&user-caps=users%3Dread%3Bbuckets%3Dread%3B" { + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader([]byte(userCreateJSON))), + }, nil + } + } + + if req.Method == http.MethodPut { + if req.URL.RawQuery == "enabled=false&format=json&max-objects=-1&max-size=-1"a="a-type=user&uid=my-user" || + req.URL.RawQuery == "enabled=true&format=json&max-objects=10000&max-size=-1"a="a-type=user&uid=my-user" || + req.URL.RawQuery == "enabled=true&format=json&max-objects=-1&max-size=10000000000"a="a-type=user&uid=my-user" || + req.URL.RawQuery == "enabled=true&format=json&max-objects=10000&max-size=10000000000"a="a-type=user&uid=my-user" { + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader([]byte(userCreateJSON))), + }, nil + } + } + + return nil, fmt.Errorf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, req.URL.Path) + }, + } + adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient) + assert.NoError(t, err) + userConfig := generateUserConfig(objectUser) + r := &ReconcileObjectStoreUser{ + objContext: &cephobject.AdminOpsContext{ + AdminOpsClient: adminClient, + }, + userConfig: &userConfig, + } + maxsize, err := resource.ParseQuantity(maxsizestr) + assert.NoError(t, err) + + t.Run("user with empty name", func(t *testing.T) { + err = r.createorUpdateCephUser(objectUser) + assert.Error(t, err) + }) + + t.Run("user without any Quotas or Capabilities", func(t *testing.T) { + objectUser.Name = name + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + + t.Run("setting MaxBuckets for the user", func(t *testing.T) { + objectUser.Spec.Quotas = &cephv1.ObjectUserQuotaSpec{MaxBuckets: &maxbucket} + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + + t.Run("setting Capabilities for the user", func(t *testing.T) { + objectUser.Spec.Quotas = nil + objectUser.Spec.Capabilities = &cephv1.ObjectUserCapSpec{ + User: "read", + Bucket: "read", + } + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + + // Testing UserQuotaSpec : MaxObjects and MaxSize + t.Run("setting MaxObjects for the user", func(t *testing.T) { + objectUser.Spec.Capabilities = nil + objectUser.Spec.Quotas = &cephv1.ObjectUserQuotaSpec{MaxObjects: &maxobject} + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + t.Run("setting MaxSize for the user", func(t *testing.T) { + objectUser.Spec.Quotas = &cephv1.ObjectUserQuotaSpec{MaxSize: &maxsize} + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + t.Run("resetting MaxSize and MaxObjects for the user", func(t *testing.T) { + objectUser.Spec.Quotas = nil + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + t.Run("setting both MaxSize and MaxObjects for the user", func(t *testing.T) { + objectUser.Spec.Quotas = &cephv1.ObjectUserQuotaSpec{MaxObjects: &maxobject, MaxSize: &maxsize} + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + t.Run("resetting MaxSize and MaxObjects again for the user", func(t *testing.T) { + objectUser.Spec.Quotas = nil + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) + + t.Run("setting both Quotas and Capabilities for the user", func(t *testing.T) { + objectUser.Spec.Capabilities = &cephv1.ObjectUserCapSpec{ + User: "read", + Bucket: "read", + } + objectUser.Spec.Quotas = &cephv1.ObjectUserQuotaSpec{MaxBuckets: &maxbucket, MaxObjects: &maxobject, MaxSize: &maxsize} + userConfig = generateUserConfig(objectUser) + r.userConfig = &userConfig + err = r.createorUpdateCephUser(objectUser) + assert.NoError(t, err) + }) +} diff --git a/tests/framework/clients/object_user.go b/tests/framework/clients/object_user.go index 6849624ef..5ca3052b9 100644 --- a/tests/framework/clients/object_user.go +++ b/tests/framework/clients/object_user.go @@ -74,10 +74,10 @@ func (o *ObjectUserOperation) UserSecretExists(namespace string, store string, u } // ObjectUserCreate Function to create a object store user in rook -func (o *ObjectUserOperation) Create(namespace string, userid string, displayName string, store string) error { +func (o *ObjectUserOperation) Create(userid, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) error { logger.Infof("creating the object store user via CRD") - if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(userid, displayName, store)); err != nil { + if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStoreUser(userid, displayName, store, usercaps, maxsize, maxbuckets, maxobjects)); err != nil { return err } return nil diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index ca913f9b8..4c0704732 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -42,7 +42,7 @@ type CephManifests interface { GetNFS(name, pool string, daemonCount int) string GetRBDMirror(name string, daemonCount int) string GetObjectStore(name string, replicaCount, port int, tlsEnable bool) string - GetObjectStoreUser(name, displayName, store string) string + GetObjectStoreUser(name, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) string GetBucketStorageClass(storeName, storageClassName, reclaimPolicy, region string) string GetOBC(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string GetClient(name string, caps map[string]string) string @@ -433,7 +433,7 @@ spec: ` } -func (m *CephManifestsMaster) GetObjectStoreUser(name string, displayName string, store string) string { +func (m *CephManifestsMaster) GetObjectStoreUser(name, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: @@ -441,7 +441,13 @@ metadata: namespace: ` + m.settings.Namespace + ` spec: displayName: ` + displayName + ` - store: ` + store + store: ` + store + ` + quotas: + maxBuckets: ` + strconv.Itoa(maxbuckets) + ` + maxObjects: ` + strconv.Itoa(maxobjects) + ` + maxSize: ` + maxsize + ` + capabilities: + user: ` + usercaps } //GetBucketStorageClass returns the manifest to create object bucket diff --git a/tests/framework/installer/ceph_manifests_v1.6.go b/tests/framework/installer/ceph_manifests_v1.6.go index 0bb4da38f..17fe28542 100644 --- a/tests/framework/installer/ceph_manifests_v1.6.go +++ b/tests/framework/installer/ceph_manifests_v1.6.go @@ -363,7 +363,7 @@ spec: ` } -func (m *CephManifestsV1_6) GetObjectStoreUser(name string, displayName string, store string) string { +func (m *CephManifestsV1_6) GetObjectStoreUser(name, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index f56abf07c..f3ec0a5f7 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "testing" "time" @@ -55,6 +56,9 @@ var ( maxObject = "2" newMaxObject = "3" bucketStorageClassName = "rook-smoke-delete-bucket" + maxBucket = 1 + maxSize = "100000" + userCap = "read" ) // Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order @@ -111,11 +115,11 @@ func objectStoreCleanUp(s suite.Suite, helper *clients.TestClient, k8sh *utils.K func createCephObjectUser( s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName, userID string, - checkPhase bool, -) { + checkPhase, checkQuotaAndCaps bool) { s.T().Helper() - - cosuErr := helper.ObjectUserClient.Create(namespace, userID, userdisplayname, storeName) + maxObjectInt, err := strconv.Atoi(maxObject) + assert.Nil(s.T(), err) + cosuErr := helper.ObjectUserClient.Create(userID, userdisplayname, storeName, userCap, maxSize, maxBucket, maxObjectInt) assert.Nil(s.T(), cosuErr) logger.Infof("Waiting 5 seconds for the object user to be created") time.Sleep(5 * time.Second) @@ -125,13 +129,13 @@ func createCephObjectUser( time.Sleep(5 * time.Second) } - checkCephObjectUser(s, helper, k8sh, namespace, storeName, userID, checkPhase) + checkCephObjectUser(s, helper, k8sh, namespace, storeName, userID, checkPhase, checkQuotaAndCaps) } func checkCephObjectUser( s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName, userID string, - checkPhase bool, + checkPhase, checkQuotaAndCaps bool, ) { s.T().Helper() @@ -149,6 +153,17 @@ func checkCephObjectUser( assert.NoError(s.T(), err) assert.Equal(s.T(), k8sutil.ReadyStatus, phase) } + if checkQuotaAndCaps { + // following fields in CephObjectStoreUser CRD doesn't exist before Rook v1.7 + maxObjectInt, err := strconv.Atoi(maxObject) + assert.Nil(s.T(), err) + maxSizeInt, err := strconv.Atoi(maxSize) + assert.Nil(s.T(), err) + assert.Equal(s.T(), maxBucket, userInfo.MaxBuckets) + assert.Equal(s.T(), int64(maxObjectInt), *userInfo.UserQuota.MaxObjects) + assert.Equal(s.T(), int64(maxSizeInt), *userInfo.UserQuota.MaxSize) + assert.Equal(s.T(), userCap, userInfo.Caps[0].Perm) + } } func createCephObjectStore(s suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string, replicaSize int, tlsEnable bool) { @@ -179,7 +194,7 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh * clusterInfo := client.AdminClusterInfo(namespace) t := s.T() t.Run(fmt.Sprintf("create CephObjectStoreUser %q", storeName), func(t *testing.T) { - createCephObjectUser(s, helper, k8sh, namespace, storeName, userid, true) + createCephObjectUser(s, helper, k8sh, namespace, storeName, userid, true, true) i := 0 for i = 0; i < 4; i++ { if helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) { diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index e7f2a0c88..84f2ad25f 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -135,7 +135,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { logger.Infof("Initializing object user before the upgrade") objectUserID := "upgraded-user" - createCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, false) + createCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, false, false) logger.Info("Initializing object bucket claim before the upgrade") bucketStorageClassName := "rook-smoke-delete-bucket" @@ -194,7 +194,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { rbdFilesToRead = append(rbdFilesToRead, newFile) cephfsFilesToRead = append(cephfsFilesToRead, newFile) - checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true) + checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true, false) // should be Bound after upgrade to Rook master // do not need retry b/c the OBC controller runs parallel to Rook-Ceph orchestration @@ -213,7 +213,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead) logger.Infof("Verified upgrade from nautilus to octopus") - checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true) + checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true, false) // // Upgrade from octopus to pacific @@ -226,7 +226,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead) logger.Infof("Verified upgrade from octopus to pacific") - checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true) + checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true, false) } func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {