From cf61966205ebe3f6a17f99acf3ff28af2100eff3 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Wed, 17 Jun 2020 16:54:49 +0530 Subject: [PATCH 1/4] ceph: remove rgwerrno from setuserquota Signed-off-by: Jiffin Tony Thottan --- pkg/operator/ceph/object/bucket/provisioner.go | 6 +++--- pkg/operator/ceph/object/user.go | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index 835ab602b..69aa8766e 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -94,8 +94,8 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje return nil, err } - _, errCode, err := cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets) - if errCode > 0 { + _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets) + if err != nil { p.deleteOBCResourceLogError(p.bucketName) return nil, err } @@ -127,7 +127,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu } // need to quota into -1 for restricting creation of new buckets in rgw - _, _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1) + _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1) if err != nil { p.deleteOBCResourceLogError("") return nil, err diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index dd0586c20..c0cfd534e 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -183,23 +183,23 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) { return result, errors.Wrap(err, "failed to delete s3 user") } -func SetQuotaUserBucketMax(c *Context, id string, max int) (string, int, error) { +func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) { logger.Infof("Setting user %q max buckets to %d", id, max) args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)} - result, errCode, err := setUserQuota(c, id, args) - if errCode != RGWErrorNone { + result, err := setUserQuota(c, id, args) + if err != nil { err = errors.Wrap(err, "failed setting bucket max") } - return result, errCode, err + return result, err } -func setUserQuota(c *Context, id string, args []string) (string, int, error) { +func setUserQuota(c *Context, id string, args []string) (string, error) { args = append([]string{"quota", "set", "--uid", id}, args...) result, err := runAdminCommand(c, args...) if err != nil { err = errors.Wrap(err, "failed to set max buckets for user") } - return result, RGWErrorNone, err + return result, err } func LinkUser(c *Context, id, bucket string) (string, int, error) { From 5a6cafcdfcff76cb537ae2c4da4942695ecad61c Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Thu, 9 Jul 2020 23:09:16 +0530 Subject: [PATCH 2/4] ceph: add missing explaination for apis in user.go Signed-off-by: Jiffin Tony Thottan --- pkg/operator/ceph/object/user.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index c0cfd534e..f12c61122 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -183,6 +183,7 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) { return result, errors.Wrap(err, "failed to delete s3 user") } +// SetQuotaUserBucketMax will set maximum bucket quota for a user func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) { logger.Infof("Setting user %q max buckets to %d", id, max) args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)} @@ -202,6 +203,7 @@ func setUserQuota(c *Context, id string, args []string) (string, error) { return result, err } +// LinkUser will link a user to a bucket func LinkUser(c *Context, id, bucket string) (string, int, error) { logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket) args := []string{"bucket", "link", "--uid", id, "--bucket", bucket} @@ -215,6 +217,7 @@ func LinkUser(c *Context, id, bucket string) (string, int, error) { return result, RGWErrorNone, nil } +// UnlinkUser will unlink the user from a bucket func UnlinkUser(c *Context, id, bucket string) (string, int, error) { logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket) args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket} From 96a53389b19b4c25f4829e9b86a8bb476ca770a8 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Thu, 28 May 2020 11:12:25 +0530 Subject: [PATCH 3/4] ceph: adding new quota apis Adding new apis such as EnableUserQuota() for enabling quota, SetQuotaUserObjectMax() and SetQuotaUserMaxSize() for setting user specific quotas for objects and size respectively. Please note the size is quota is determined in GBs. Signed-off-by: Jiffin Tony Thottan --- pkg/operator/ceph/object/user.go | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index f12c61122..defb66762 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -230,3 +230,37 @@ func UnlinkUser(c *Context, id, bucket string) (string, int, error) { } return result, RGWErrorNone, nil } + +// EnableUserQuota will allows to enable quota defined for a user +func EnableUserQuota(c *Context, id string) (string, error) { + logger.Debug("Enabling user quota for %q", id) + args := append([]string{"quota", "enable", "--quota-scope", "user", "--uid", id}) + result, err := runAdminCommand(c, args...) + if err != nil { + err = errors.Wrap(err, "failed to enable quota for the user") + } + return result, err + +} + +// SetQuotaUserObject allows to set maximum limit on objects for a user +func SetQuotaUserObjectMax(c *Context, id string, maxobjects string) (string, error) { + logger.Debugf("Setting user %q max objects to %s", id, maxobjects) + args := []string{"--quota-scope", "user", "--max-objects", maxobjects} + result, err := setUserQuota(c, id, args) + if err != nil { + err = errors.Wrap(err, "failed setting object max") + } + return result, err +} + +// SetQuotaUserMaxSize allows to set maximum size for a user +func SetQuotaUserMaxSize(c *Context, id string, maxsize string) (string, error) { + logger.Debugf("Setting user %q max size to %s", id, maxsize) + args := []string{"--quota-scope", "user", "--max-size", maxsize} + result, err := setUserQuota(c, id, args) + if err != nil { + err = errors.Wrap(err, "failed setting max size") + } + return result, err +} From f4240cb31d8922fcc463bab58470ce4bb3902de9 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Tue, 7 Jul 2020 11:02:45 +0530 Subject: [PATCH 4/4] ceph: add quota support for obc Closes: #5274 Signed-off-by: Jiffin Tony Thottan --- Documentation/ceph-object-bucket-claim.md | 7 ++- PendingReleaseNotes.md | 1 + .../ceph/object-bucket-claim-delete.yaml | 4 ++ .../ceph/object-bucket-claim-retain.yaml | 4 ++ .../ceph/object/bucket/provisioner.go | 45 +++++++++++++++++++ pkg/operator/ceph/object/bucket/util.go | 9 ++++ tests/framework/clients/bucket.go | 8 ++-- tests/framework/installer/ceph_manifests.go | 8 ++-- .../installer/ceph_manifests_v1.2.go | 6 ++- tests/integration/ceph_base_object_test.go | 37 ++++++++++----- tests/integration/ceph_smoke_test.go | 1 + 11 files changed, 107 insertions(+), 23 deletions(-) diff --git a/Documentation/ceph-object-bucket-claim.md b/Documentation/ceph-object-bucket-claim.md index b973a024a..6f437f260 100644 --- a/Documentation/ceph-object-bucket-claim.md +++ b/Documentation/ceph-object-bucket-claim.md @@ -28,7 +28,8 @@ spec: generateBucketName: photo-booth [4] storageClassName: rook-ceph-bucket [4] additionalConfig: [5] - ANY_KEY: VALUE ... + maxObjects: "1000" + maxSize: "2G" ``` 1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap. 1. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret. @@ -39,7 +40,9 @@ an entire object store. If both `bucketName` and `generateBucketName` are supplied then `BucketName` has precedence and `GenerateBucketName` is ignored. If both `bucketName` and `generateBucketName` are blank or omitted then the storage class is expected to contain the name of an _existing_ bucket. It's an error if all three bucket related names are blank or omitted. 1. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy. -1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Examples can include config values such as tenant, user and policy settings, etc. +1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported: + - `maxObjects`: The maximum number of objects in the bucket + - `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K. ### OBC Custom Resource after Bucket Provisioning ```yaml diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 7907ca065..dbe986a67 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -23,6 +23,7 @@ - OBC changes: - Updated lib bucket provisioner version to support multithread and required change can be found in [operator.yaml](cluster/examples/kubernetes/ceph/operator.yaml#L449) - Can be extended to add support for other providers + - Added support for [quota](Documentation/ceph-object-bucket-claim.md#obc-custom-resource), have options for object count and total size. - CephObjectStore CRD changes: - Health displayed in the Status field - Supports connecting to external Ceph Rados Gateways, refer to the [external object section](Documentation/ceph-object.html#connect-to-external-object-store) diff --git a/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml b/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml index 5f8a3c4f4..601f84fe0 100644 --- a/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml +++ b/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml @@ -11,3 +11,7 @@ spec: #bucketName: generateBucketName: ceph-bkt storageClassName: rook-ceph-delete-bucket + additionalConfig: + # To set for quota for OBC + #maxObjects: "1000" + #maxSize: "2G" diff --git a/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml b/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml index da42270d6..e5aaa426c 100644 --- a/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml +++ b/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml @@ -11,3 +11,7 @@ spec: #bucketName: generateBucketName: ceph-bkt storageClassName: rook-ceph-retain-bucket + additionalConfig: + # To set for quota for OBC + #maxObjects: "1000" + #maxSize: "2G" diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index 69aa8766e..d16aa1027 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -101,6 +101,13 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje } logger.Infof("set user %q bucket max to %d", p.cephUserName, maxBuckets) + // setting quota limit if it is enabled + err = p.setAdditionalSettings(options) + if err != nil { + p.deleteOBCResource(p.bucketName) + return nil, err + } + return p.composeObjectBucket(), nil } @@ -181,6 +188,14 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu p.deleteOBCResourceLogError("") return nil, err } + + // setting quota limit if it is enabled + err = p.setAdditionalSettings(options) + if err != nil { + p.deleteOBCResource("") + return nil, err + } + // returned ob with connection info return p.composeObjectBucket(), nil } @@ -494,3 +509,33 @@ func (p *Provisioner) deleteOBCResourceLogError(name string) { logger.Warningf("error deleting OBC resource. %v", err) } } + +// Check for additional options mentioned in OBC and set them accordingly +func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error { + maxObjects := MaxObjectQuota(options) + maxSize := MaxSizeQuota(options) + if maxObjects == "" && maxSize == "" { + return nil + } + + var err error + if maxObjects != "" { + if _, err = cephObject.SetQuotaUserObjectMax(p.objectContext, p.cephUserName, maxObjects); err != nil { + logger.Errorf("Setting MaxObject failed %v", err) + return err + } + } + if maxSize != "" { + _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize) + if _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize); err != nil { + logger.Errorf("Setting MaxSize failed %v", err) + return err + } + } + if _, err = cephObject.EnableUserQuota(p.objectContext, p.cephUserName); err != nil { + logger.Errorf("Enabling user quota for obc failed %v", err) + return err + } + + return nil +} diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index 536c269e6..30cffce0b 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -23,6 +23,7 @@ import ( "github.com/coreos/pkg/capnslog" bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner" + apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -134,3 +135,11 @@ func randomString(n int) string { } return string(b) } + +func MaxObjectQuota(options *apibkt.BucketOptions) string { + return options.ObjectBucketClaim.Spec.AdditionalConfig["maxObjects"] +} + +func MaxSizeQuota(options *apibkt.BucketOptions) string { + return options.ObjectBucketClaim.Spec.AdditionalConfig["maxSize"] +} diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go index 71f4581f4..9a988eb73 100644 --- a/tests/framework/clients/bucket.go +++ b/tests/framework/clients/bucket.go @@ -44,12 +44,12 @@ func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName s return err } -func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, createBucket bool) error { - return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket)) +func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { + return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) } -func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, createBucket bool) error { - return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket)) +func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { + return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) } // CheckOBC, returns true if the obc, secret and configmap are all in the "check" state, diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index f1fc55bb6..47d83dc06 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -49,7 +49,7 @@ type CephManifests interface { GetObjectStore(namespace, name string, replicaCount, port int) string GetObjectStoreUser(namespace, name, displayName, store string) string GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region string) string - GetObc(obcName, storageClassName, bucketName string, createBucket bool) string + GetObc(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string GetClient(namespace, name string, caps map[string]string) string } @@ -2294,7 +2294,7 @@ parameters: } //GetObc returns the manifest to create object bucket claim -func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string { +func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -2305,7 +2305,9 @@ metadata: name: ` + claimName + ` spec: ` + bucketParameter + `: ` + objectBucketName + ` - storageClassName: ` + storageClassName + storageClassName: ` + storageClassName + ` + additionalConfig: + maxObjects: "` + maxObject + `"` } func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps map[string]string) string { diff --git a/tests/framework/installer/ceph_manifests_v1.2.go b/tests/framework/installer/ceph_manifests_v1.2.go index 267a16b4a..be0a53617 100644 --- a/tests/framework/installer/ceph_manifests_v1.2.go +++ b/tests/framework/installer/ceph_manifests_v1.2.go @@ -1903,7 +1903,7 @@ parameters: } //GetObc returns the manifest to create object bucket claim -func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string { +func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1914,7 +1914,9 @@ metadata: name: ` + claimName + ` spec: ` + bucketParameter + `: ` + objectBucketName + ` - storageClassName: ` + storageClassName + storageClassName: ` + storageClassName + ` + additionalConfig: + maxObjects: "` + maxObject + `"` } func (m *CephManifestsV1_2) GetClient(namespace string, claimName string, caps map[string]string) string { diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 6fe84e74e..782b48aa1 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -36,11 +36,14 @@ var ( userid = "rook-user" userdisplayname = "A rook RGW user" bucketname = "smokebkt" - objBody = "Test Rook Object Data" - objectKey = "rookObj1" + ObjBody = "Test Rook Object Data" + ObjectKey1 = "rookObj1" + ObjectKey2 = "rookObj2" + ObjectKey3 = "rookObj3" contentType = "plain/text" obcName = "smoke-delete-bucket" region = "us-east-1" + maxObject = "2" ) // Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order @@ -119,7 +122,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite bucketStorageClassName := "rook-smoke-delete-bucket" cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region) require.Nil(s.T(), cobErr) - cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, true) + cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true) require.Nil(s.T(), cobcErr) for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "created"); i++ { @@ -154,22 +157,32 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) logger.Infof("Step 5 : Put Object on bucket") - _, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType) + _, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType) require.Nil(s.T(), poErr) logger.Infof("Step 6 : Get Object from bucket") - read, err := s3client.GetObjectInBucket(bucketname, objectKey) + read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1) require.Nil(s.T(), err) - require.Equal(s.T(), objBody, read) + require.Equal(s.T(), ObjBody, read) logger.Infof("Object Created and Retrieved on bucket successfully") - logger.Infof("Step 7 : Delete object on bucket") - _, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey) - require.Nil(s.T(), delobjErr) - logger.Infof("Object deleted on bucket successfully") + logger.Infof("Step 7 : Testing Quota for the OBC") + logger.Infof("Adding one more object to the bucket") + _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType) + require.Nil(s.T(), poErr) + logger.Infof("Testing the max object limit") + _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType) + require.Error(s.T(), poErr) - logger.Infof("Step 8 : Delete Object Bucket Claim") - dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, true) + logger.Infof("Step 8 : Delete objects on bucket") + _, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1) + require.Nil(s.T(), delobjErr) + _, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2) + require.Nil(s.T(), delobjErr) + logger.Infof("Objects deleted on bucket successfully") + + logger.Infof("Step 9 : Delete Object Bucket Claim") + dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true) require.Nil(s.T(), dobcErr) logger.Infof("Checking to see if the obc, secret and cm have all been deleted") for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ { diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 600232f67..85ddfbfbf 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -56,6 +56,7 @@ import ( // - Create/delete buckets // - Create/delete users // - PUT/GET objects +// - Quota limit wrt no of objects // ************************************************ func TestCephSmokeSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) {