diff --git a/Documentation/ceph-object-bucket-claim.md b/Documentation/ceph-object-bucket-claim.md index b973a024a..6f437f260 100644 --- a/Documentation/ceph-object-bucket-claim.md +++ b/Documentation/ceph-object-bucket-claim.md @@ -28,7 +28,8 @@ spec: generateBucketName: photo-booth [4] storageClassName: rook-ceph-bucket [4] additionalConfig: [5] - ANY_KEY: VALUE ... + maxObjects: "1000" + maxSize: "2G" ``` 1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap. 1. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret. @@ -39,7 +40,9 @@ an entire object store. If both `bucketName` and `generateBucketName` are supplied then `BucketName` has precedence and `GenerateBucketName` is ignored. If both `bucketName` and `generateBucketName` are blank or omitted then the storage class is expected to contain the name of an _existing_ bucket. It's an error if all three bucket related names are blank or omitted. 1. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy. -1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Examples can include config values such as tenant, user and policy settings, etc. +1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported: + - `maxObjects`: The maximum number of objects in the bucket + - `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K. ### OBC Custom Resource after Bucket Provisioning ```yaml diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 7907ca065..dbe986a67 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -23,6 +23,7 @@ - OBC changes: - Updated lib bucket provisioner version to support multithread and required change can be found in [operator.yaml](cluster/examples/kubernetes/ceph/operator.yaml#L449) - Can be extended to add support for other providers + - Added support for [quota](Documentation/ceph-object-bucket-claim.md#obc-custom-resource), have options for object count and total size. - CephObjectStore CRD changes: - Health displayed in the Status field - Supports connecting to external Ceph Rados Gateways, refer to the [external object section](Documentation/ceph-object.html#connect-to-external-object-store) diff --git a/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml b/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml index 5f8a3c4f4..601f84fe0 100644 --- a/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml +++ b/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml @@ -11,3 +11,7 @@ spec: #bucketName: generateBucketName: ceph-bkt storageClassName: rook-ceph-delete-bucket + additionalConfig: + # To set for quota for OBC + #maxObjects: "1000" + #maxSize: "2G" diff --git a/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml b/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml index da42270d6..e5aaa426c 100644 --- a/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml +++ b/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml @@ -11,3 +11,7 @@ spec: #bucketName: generateBucketName: ceph-bkt storageClassName: rook-ceph-retain-bucket + additionalConfig: + # To set for quota for OBC + #maxObjects: "1000" + #maxSize: "2G" diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index 835ab602b..d16aa1027 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -94,13 +94,20 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje return nil, err } - _, errCode, err := cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets) - if errCode > 0 { + _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets) + if err != nil { p.deleteOBCResourceLogError(p.bucketName) return nil, err } logger.Infof("set user %q bucket max to %d", p.cephUserName, maxBuckets) + // setting quota limit if it is enabled + err = p.setAdditionalSettings(options) + if err != nil { + p.deleteOBCResource(p.bucketName) + return nil, err + } + return p.composeObjectBucket(), nil } @@ -127,7 +134,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu } // need to quota into -1 for restricting creation of new buckets in rgw - _, _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1) + _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1) if err != nil { p.deleteOBCResourceLogError("") return nil, err @@ -181,6 +188,14 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu p.deleteOBCResourceLogError("") return nil, err } + + // setting quota limit if it is enabled + err = p.setAdditionalSettings(options) + if err != nil { + p.deleteOBCResource("") + return nil, err + } + // returned ob with connection info return p.composeObjectBucket(), nil } @@ -494,3 +509,33 @@ func (p *Provisioner) deleteOBCResourceLogError(name string) { logger.Warningf("error deleting OBC resource. %v", err) } } + +// Check for additional options mentioned in OBC and set them accordingly +func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error { + maxObjects := MaxObjectQuota(options) + maxSize := MaxSizeQuota(options) + if maxObjects == "" && maxSize == "" { + return nil + } + + var err error + if maxObjects != "" { + if _, err = cephObject.SetQuotaUserObjectMax(p.objectContext, p.cephUserName, maxObjects); err != nil { + logger.Errorf("Setting MaxObject failed %v", err) + return err + } + } + if maxSize != "" { + _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize) + if _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize); err != nil { + logger.Errorf("Setting MaxSize failed %v", err) + return err + } + } + if _, err = cephObject.EnableUserQuota(p.objectContext, p.cephUserName); err != nil { + logger.Errorf("Enabling user quota for obc failed %v", err) + return err + } + + return nil +} diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index d16cb2835..1e16485ce 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -22,6 +22,7 @@ import ( "github.com/coreos/pkg/capnslog" bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner" + apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -134,3 +135,11 @@ func randomString(n int) string { } return string(b) } + +func MaxObjectQuota(options *apibkt.BucketOptions) string { + return options.ObjectBucketClaim.Spec.AdditionalConfig["maxObjects"] +} + +func MaxSizeQuota(options *apibkt.BucketOptions) string { + return options.ObjectBucketClaim.Spec.AdditionalConfig["maxSize"] +} diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go index dd0586c20..defb66762 100644 --- a/pkg/operator/ceph/object/user.go +++ b/pkg/operator/ceph/object/user.go @@ -183,25 +183,27 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) { return result, errors.Wrap(err, "failed to delete s3 user") } -func SetQuotaUserBucketMax(c *Context, id string, max int) (string, int, error) { +// SetQuotaUserBucketMax will set maximum bucket quota for a user +func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) { logger.Infof("Setting user %q max buckets to %d", id, max) args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)} - result, errCode, err := setUserQuota(c, id, args) - if errCode != RGWErrorNone { + result, err := setUserQuota(c, id, args) + if err != nil { err = errors.Wrap(err, "failed setting bucket max") } - return result, errCode, err + return result, err } -func setUserQuota(c *Context, id string, args []string) (string, int, error) { +func setUserQuota(c *Context, id string, args []string) (string, error) { args = append([]string{"quota", "set", "--uid", id}, args...) result, err := runAdminCommand(c, args...) if err != nil { err = errors.Wrap(err, "failed to set max buckets for user") } - return result, RGWErrorNone, err + return result, err } +// LinkUser will link a user to a bucket func LinkUser(c *Context, id, bucket string) (string, int, error) { logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket) args := []string{"bucket", "link", "--uid", id, "--bucket", bucket} @@ -215,6 +217,7 @@ func LinkUser(c *Context, id, bucket string) (string, int, error) { return result, RGWErrorNone, nil } +// UnlinkUser will unlink the user from a bucket func UnlinkUser(c *Context, id, bucket string) (string, int, error) { logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket) args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket} @@ -227,3 +230,37 @@ func UnlinkUser(c *Context, id, bucket string) (string, int, error) { } return result, RGWErrorNone, nil } + +// EnableUserQuota will allows to enable quota defined for a user +func EnableUserQuota(c *Context, id string) (string, error) { + logger.Debug("Enabling user quota for %q", id) + args := append([]string{"quota", "enable", "--quota-scope", "user", "--uid", id}) + result, err := runAdminCommand(c, args...) + if err != nil { + err = errors.Wrap(err, "failed to enable quota for the user") + } + return result, err + +} + +// SetQuotaUserObject allows to set maximum limit on objects for a user +func SetQuotaUserObjectMax(c *Context, id string, maxobjects string) (string, error) { + logger.Debugf("Setting user %q max objects to %s", id, maxobjects) + args := []string{"--quota-scope", "user", "--max-objects", maxobjects} + result, err := setUserQuota(c, id, args) + if err != nil { + err = errors.Wrap(err, "failed setting object max") + } + return result, err +} + +// SetQuotaUserMaxSize allows to set maximum size for a user +func SetQuotaUserMaxSize(c *Context, id string, maxsize string) (string, error) { + logger.Debugf("Setting user %q max size to %s", id, maxsize) + args := []string{"--quota-scope", "user", "--max-size", maxsize} + result, err := setUserQuota(c, id, args) + if err != nil { + err = errors.Wrap(err, "failed setting max size") + } + return result, err +} diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go index 71f4581f4..9a988eb73 100644 --- a/tests/framework/clients/bucket.go +++ b/tests/framework/clients/bucket.go @@ -44,12 +44,12 @@ func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName s return err } -func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, createBucket bool) error { - return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket)) +func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { + return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) } -func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, createBucket bool) error { - return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket)) +func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error { + return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket)) } // CheckOBC, returns true if the obc, secret and configmap are all in the "check" state, diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 780d1fe0a..6f4cfd743 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -49,7 +49,7 @@ type CephManifests interface { GetObjectStore(namespace, name string, replicaCount, port int) string GetObjectStoreUser(namespace, name, displayName, store string) string GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region string) string - GetObc(obcName, storageClassName, bucketName string, createBucket bool) string + GetObc(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string GetClient(namespace, name string, caps map[string]string) string } @@ -2295,7 +2295,7 @@ parameters: } //GetObc returns the manifest to create object bucket claim -func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string { +func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -2306,7 +2306,9 @@ metadata: name: ` + claimName + ` spec: ` + bucketParameter + `: ` + objectBucketName + ` - storageClassName: ` + storageClassName + storageClassName: ` + storageClassName + ` + additionalConfig: + maxObjects: "` + maxObject + `"` } func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps map[string]string) string { diff --git a/tests/framework/installer/ceph_manifests_v1.2.go b/tests/framework/installer/ceph_manifests_v1.2.go index e8e74b0d3..ee9c78dc0 100644 --- a/tests/framework/installer/ceph_manifests_v1.2.go +++ b/tests/framework/installer/ceph_manifests_v1.2.go @@ -1904,7 +1904,7 @@ parameters: } //GetObc returns the manifest to create object bucket claim -func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string { +func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1915,7 +1915,9 @@ metadata: name: ` + claimName + ` spec: ` + bucketParameter + `: ` + objectBucketName + ` - storageClassName: ` + storageClassName + storageClassName: ` + storageClassName + ` + additionalConfig: + maxObjects: "` + maxObject + `"` } func (m *CephManifestsV1_2) GetClient(namespace string, claimName string, caps map[string]string) string { diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 6fe84e74e..782b48aa1 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -36,11 +36,14 @@ var ( userid = "rook-user" userdisplayname = "A rook RGW user" bucketname = "smokebkt" - objBody = "Test Rook Object Data" - objectKey = "rookObj1" + ObjBody = "Test Rook Object Data" + ObjectKey1 = "rookObj1" + ObjectKey2 = "rookObj2" + ObjectKey3 = "rookObj3" contentType = "plain/text" obcName = "smoke-delete-bucket" region = "us-east-1" + maxObject = "2" ) // Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order @@ -119,7 +122,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite bucketStorageClassName := "rook-smoke-delete-bucket" cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region) require.Nil(s.T(), cobErr) - cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, true) + cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true) require.Nil(s.T(), cobcErr) for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "created"); i++ { @@ -154,22 +157,32 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey) logger.Infof("Step 5 : Put Object on bucket") - _, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType) + _, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType) require.Nil(s.T(), poErr) logger.Infof("Step 6 : Get Object from bucket") - read, err := s3client.GetObjectInBucket(bucketname, objectKey) + read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1) require.Nil(s.T(), err) - require.Equal(s.T(), objBody, read) + require.Equal(s.T(), ObjBody, read) logger.Infof("Object Created and Retrieved on bucket successfully") - logger.Infof("Step 7 : Delete object on bucket") - _, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey) - require.Nil(s.T(), delobjErr) - logger.Infof("Object deleted on bucket successfully") + logger.Infof("Step 7 : Testing Quota for the OBC") + logger.Infof("Adding one more object to the bucket") + _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType) + require.Nil(s.T(), poErr) + logger.Infof("Testing the max object limit") + _, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType) + require.Error(s.T(), poErr) - logger.Infof("Step 8 : Delete Object Bucket Claim") - dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, true) + logger.Infof("Step 8 : Delete objects on bucket") + _, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1) + require.Nil(s.T(), delobjErr) + _, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2) + require.Nil(s.T(), delobjErr) + logger.Infof("Objects deleted on bucket successfully") + + logger.Infof("Step 9 : Delete Object Bucket Claim") + dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true) require.Nil(s.T(), dobcErr) logger.Infof("Checking to see if the obc, secret and cm have all been deleted") for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ { diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 600232f67..85ddfbfbf 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -56,6 +56,7 @@ import ( // - Create/delete buckets // - Create/delete users // - PUT/GET objects +// - Quota limit wrt no of objects // ************************************************ func TestCephSmokeSuite(t *testing.T) { if installer.SkipTestSuite(installer.CephTestSuite) {