Merge pull request #5674 from thotz/quotaobc
ceph: adding support for quota in object bucket claims
This commit is contained in:
12 files changed
+153
-32
No files matched your search
@@ -28,7 +28,8 @@ spec:
|
||||
generateBucketName: photo-booth [4]
|
||||
storageClassName: rook-ceph-bucket [4]
|
||||
additionalConfig: [5]
|
||||
ANY_KEY: VALUE ...
|
||||
maxObjects: "1000"
|
||||
maxSize: "2G"
|
||||
```
|
||||
1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap.
|
||||
1. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret.
|
||||
@@ -39,7 +40,9 @@ an entire object store.
|
||||
If both `bucketName` and `generateBucketName` are supplied then `BucketName` has precedence and `GenerateBucketName` is ignored.
|
||||
If both `bucketName` and `generateBucketName` are blank or omitted then the storage class is expected to contain the name of an _existing_ bucket. It's an error if all three bucket related names are blank or omitted.
|
||||
1. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy.
|
||||
1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Examples can include config values such as tenant, user and policy settings, etc.
|
||||
1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported:
|
||||
- `maxObjects`: The maximum number of objects in the bucket
|
||||
- `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K.
|
||||
|
||||
### OBC Custom Resource after Bucket Provisioning
|
||||
```yaml
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
- OBC changes:
|
||||
- Updated lib bucket provisioner version to support multithread and required change can be found in [operator.yaml](cluster/examples/kubernetes/ceph/operator.yaml#L449)
|
||||
- Can be extended to add support for other providers
|
||||
- Added support for [quota](Documentation/ceph-object-bucket-claim.md#obc-custom-resource), have options for object count and total size.
|
||||
- CephObjectStore CRD changes:
|
||||
- Health displayed in the Status field
|
||||
- Supports connecting to external Ceph Rados Gateways, refer to the [external object section](Documentation/ceph-object.html#connect-to-external-object-store)
|
||||
|
||||
@@ -11,3 +11,7 @@ spec:
|
||||
#bucketName:
|
||||
generateBucketName: ceph-bkt
|
||||
storageClassName: rook-ceph-delete-bucket
|
||||
additionalConfig:
|
||||
# To set for quota for OBC
|
||||
#maxObjects: "1000"
|
||||
#maxSize: "2G"
|
||||
@@ -11,3 +11,7 @@ spec:
|
||||
#bucketName:
|
||||
generateBucketName: ceph-bkt
|
||||
storageClassName: rook-ceph-retain-bucket
|
||||
additionalConfig:
|
||||
# To set for quota for OBC
|
||||
#maxObjects: "1000"
|
||||
#maxSize: "2G"
|
||||
@@ -94,13 +94,20 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, errCode, err := cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets)
|
||||
if errCode > 0 {
|
||||
_, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets)
|
||||
if err != nil {
|
||||
p.deleteOBCResourceLogError(p.bucketName)
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof("set user %q bucket max to %d", p.cephUserName, maxBuckets)
|
||||
|
||||
// setting quota limit if it is enabled
|
||||
err = p.setAdditionalSettings(options)
|
||||
if err != nil {
|
||||
p.deleteOBCResource(p.bucketName)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.composeObjectBucket(), nil
|
||||
}
|
||||
|
||||
@@ -127,7 +134,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
|
||||
}
|
||||
|
||||
// need to quota into -1 for restricting creation of new buckets in rgw
|
||||
_, _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1)
|
||||
_, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1)
|
||||
if err != nil {
|
||||
p.deleteOBCResourceLogError("")
|
||||
return nil, err
|
||||
@@ -181,6 +188,14 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
|
||||
p.deleteOBCResourceLogError("")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// setting quota limit if it is enabled
|
||||
err = p.setAdditionalSettings(options)
|
||||
if err != nil {
|
||||
p.deleteOBCResource("")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// returned ob with connection info
|
||||
return p.composeObjectBucket(), nil
|
||||
}
|
||||
@@ -494,3 +509,33 @@ func (p *Provisioner) deleteOBCResourceLogError(name string) {
|
||||
logger.Warningf("error deleting OBC resource. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Check for additional options mentioned in OBC and set them accordingly
|
||||
func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error {
|
||||
maxObjects := MaxObjectQuota(options)
|
||||
maxSize := MaxSizeQuota(options)
|
||||
if maxObjects == "" && maxSize == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
if maxObjects != "" {
|
||||
if _, err = cephObject.SetQuotaUserObjectMax(p.objectContext, p.cephUserName, maxObjects); err != nil {
|
||||
logger.Errorf("Setting MaxObject failed %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if maxSize != "" {
|
||||
_, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize)
|
||||
if _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize); err != nil {
|
||||
logger.Errorf("Setting MaxSize failed %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = cephObject.EnableUserQuota(p.objectContext, p.cephUserName); err != nil {
|
||||
logger.Errorf("Enabling user quota for obc failed %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
|
||||
"github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner"
|
||||
apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api"
|
||||
"github.com/pkg/errors"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
storagev1 "k8s.io/api/storage/v1"
|
||||
@@ -134,3 +135,11 @@ func randomString(n int) string {
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func MaxObjectQuota(options *apibkt.BucketOptions) string {
|
||||
return options.ObjectBucketClaim.Spec.AdditionalConfig["maxObjects"]
|
||||
}
|
||||
|
||||
func MaxSizeQuota(options *apibkt.BucketOptions) string {
|
||||
return options.ObjectBucketClaim.Spec.AdditionalConfig["maxSize"]
|
||||
}
|
||||
@@ -183,25 +183,27 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) {
|
||||
return result, errors.Wrap(err, "failed to delete s3 user")
|
||||
}
|
||||
|
||||
func SetQuotaUserBucketMax(c *Context, id string, max int) (string, int, error) {
|
||||
// SetQuotaUserBucketMax will set maximum bucket quota for a user
|
||||
func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) {
|
||||
logger.Infof("Setting user %q max buckets to %d", id, max)
|
||||
args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)}
|
||||
result, errCode, err := setUserQuota(c, id, args)
|
||||
if errCode != RGWErrorNone {
|
||||
result, err := setUserQuota(c, id, args)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed setting bucket max")
|
||||
}
|
||||
return result, errCode, err
|
||||
return result, err
|
||||
}
|
||||
|
||||
func setUserQuota(c *Context, id string, args []string) (string, int, error) {
|
||||
func setUserQuota(c *Context, id string, args []string) (string, error) {
|
||||
args = append([]string{"quota", "set", "--uid", id}, args...)
|
||||
result, err := runAdminCommand(c, args...)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed to set max buckets for user")
|
||||
}
|
||||
return result, RGWErrorNone, err
|
||||
return result, err
|
||||
}
|
||||
|
||||
// LinkUser will link a user to a bucket
|
||||
func LinkUser(c *Context, id, bucket string) (string, int, error) {
|
||||
logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket)
|
||||
args := []string{"bucket", "link", "--uid", id, "--bucket", bucket}
|
||||
@@ -215,6 +217,7 @@ func LinkUser(c *Context, id, bucket string) (string, int, error) {
|
||||
return result, RGWErrorNone, nil
|
||||
}
|
||||
|
||||
// UnlinkUser will unlink the user from a bucket
|
||||
func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
|
||||
logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket)
|
||||
args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket}
|
||||
@@ -227,3 +230,37 @@ func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
|
||||
}
|
||||
return result, RGWErrorNone, nil
|
||||
}
|
||||
|
||||
// EnableUserQuota will allows to enable quota defined for a user
|
||||
func EnableUserQuota(c *Context, id string) (string, error) {
|
||||
logger.Debug("Enabling user quota for %q", id)
|
||||
args := append([]string{"quota", "enable", "--quota-scope", "user", "--uid", id})
|
||||
result, err := runAdminCommand(c, args...)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed to enable quota for the user")
|
||||
}
|
||||
return result, err
|
||||
|
||||
}
|
||||
|
||||
// SetQuotaUserObject allows to set maximum limit on objects for a user
|
||||
func SetQuotaUserObjectMax(c *Context, id string, maxobjects string) (string, error) {
|
||||
logger.Debugf("Setting user %q max objects to %s", id, maxobjects)
|
||||
args := []string{"--quota-scope", "user", "--max-objects", maxobjects}
|
||||
result, err := setUserQuota(c, id, args)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed setting object max")
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
// SetQuotaUserMaxSize allows to set maximum size for a user
|
||||
func SetQuotaUserMaxSize(c *Context, id string, maxsize string) (string, error) {
|
||||
logger.Debugf("Setting user %q max size to %s", id, maxsize)
|
||||
args := []string{"--quota-scope", "user", "--max-size", maxsize}
|
||||
result, err := setUserQuota(c, id, args)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed setting max size")
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
@@ -44,12 +44,12 @@ func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName s
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, createBucket bool) error {
|
||||
return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket))
|
||||
func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
|
||||
return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
|
||||
}
|
||||
|
||||
func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, createBucket bool) error {
|
||||
return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket))
|
||||
func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
|
||||
return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
|
||||
}
|
||||
|
||||
// CheckOBC, returns true if the obc, secret and configmap are all in the "check" state,
|
||||
|
||||
@@ -49,7 +49,7 @@ type CephManifests interface {
|
||||
GetObjectStore(namespace, name string, replicaCount, port int) string
|
||||
GetObjectStoreUser(namespace, name, displayName, store string) string
|
||||
GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region string) string
|
||||
GetObc(obcName, storageClassName, bucketName string, createBucket bool) string
|
||||
GetObc(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string
|
||||
GetClient(namespace, name string, caps map[string]string) string
|
||||
}
|
||||
|
||||
@@ -2295,7 +2295,7 @@ parameters:
|
||||
}
|
||||
|
||||
//GetObc returns the manifest to create object bucket claim
|
||||
func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string {
|
||||
func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string {
|
||||
bucketParameter := "generateBucketName"
|
||||
if varBucketName {
|
||||
bucketParameter = "bucketName"
|
||||
@@ -2306,7 +2306,9 @@ metadata:
|
||||
name: ` + claimName + `
|
||||
spec:
|
||||
` + bucketParameter + `: ` + objectBucketName + `
|
||||
storageClassName: ` + storageClassName
|
||||
storageClassName: ` + storageClassName + `
|
||||
additionalConfig:
|
||||
maxObjects: "` + maxObject + `"`
|
||||
}
|
||||
|
||||
func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps map[string]string) string {
|
||||
|
||||
@@ -1904,7 +1904,7 @@ parameters:
|
||||
}
|
||||
|
||||
//GetObc returns the manifest to create object bucket claim
|
||||
func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string {
|
||||
func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string {
|
||||
bucketParameter := "generateBucketName"
|
||||
if varBucketName {
|
||||
bucketParameter = "bucketName"
|
||||
@@ -1915,7 +1915,9 @@ metadata:
|
||||
name: ` + claimName + `
|
||||
spec:
|
||||
` + bucketParameter + `: ` + objectBucketName + `
|
||||
storageClassName: ` + storageClassName
|
||||
storageClassName: ` + storageClassName + `
|
||||
additionalConfig:
|
||||
maxObjects: "` + maxObject + `"`
|
||||
}
|
||||
|
||||
func (m *CephManifestsV1_2) GetClient(namespace string, claimName string, caps map[string]string) string {
|
||||
|
||||
@@ -36,11 +36,14 @@ var (
|
||||
userid = "rook-user"
|
||||
userdisplayname = "A rook RGW user"
|
||||
bucketname = "smokebkt"
|
||||
objBody = "Test Rook Object Data"
|
||||
objectKey = "rookObj1"
|
||||
ObjBody = "Test Rook Object Data"
|
||||
ObjectKey1 = "rookObj1"
|
||||
ObjectKey2 = "rookObj2"
|
||||
ObjectKey3 = "rookObj3"
|
||||
contentType = "plain/text"
|
||||
obcName = "smoke-delete-bucket"
|
||||
region = "us-east-1"
|
||||
maxObject = "2"
|
||||
)
|
||||
|
||||
// Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order
|
||||
@@ -119,7 +122,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
bucketStorageClassName := "rook-smoke-delete-bucket"
|
||||
cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region)
|
||||
require.Nil(s.T(), cobErr)
|
||||
cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, true)
|
||||
cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
|
||||
require.Nil(s.T(), cobcErr)
|
||||
|
||||
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "created"); i++ {
|
||||
@@ -154,22 +157,32 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
|
||||
|
||||
logger.Infof("Step 5 : Put Object on bucket")
|
||||
_, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType)
|
||||
_, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType)
|
||||
require.Nil(s.T(), poErr)
|
||||
|
||||
logger.Infof("Step 6 : Get Object from bucket")
|
||||
read, err := s3client.GetObjectInBucket(bucketname, objectKey)
|
||||
read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1)
|
||||
require.Nil(s.T(), err)
|
||||
require.Equal(s.T(), objBody, read)
|
||||
require.Equal(s.T(), ObjBody, read)
|
||||
logger.Infof("Object Created and Retrieved on bucket successfully")
|
||||
|
||||
logger.Infof("Step 7 : Delete object on bucket")
|
||||
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey)
|
||||
require.Nil(s.T(), delobjErr)
|
||||
logger.Infof("Object deleted on bucket successfully")
|
||||
logger.Infof("Step 7 : Testing Quota for the OBC")
|
||||
logger.Infof("Adding one more object to the bucket")
|
||||
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType)
|
||||
require.Nil(s.T(), poErr)
|
||||
logger.Infof("Testing the max object limit")
|
||||
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType)
|
||||
require.Error(s.T(), poErr)
|
||||
|
||||
logger.Infof("Step 8 : Delete Object Bucket Claim")
|
||||
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, true)
|
||||
logger.Infof("Step 8 : Delete objects on bucket")
|
||||
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1)
|
||||
require.Nil(s.T(), delobjErr)
|
||||
_, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2)
|
||||
require.Nil(s.T(), delobjErr)
|
||||
logger.Infof("Objects deleted on bucket successfully")
|
||||
|
||||
logger.Infof("Step 9 : Delete Object Bucket Claim")
|
||||
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
|
||||
require.Nil(s.T(), dobcErr)
|
||||
logger.Infof("Checking to see if the obc, secret and cm have all been deleted")
|
||||
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
|
||||
|
||||
@@ -56,6 +56,7 @@ import (
|
||||
// - Create/delete buckets
|
||||
// - Create/delete users
|
||||
// - PUT/GET objects
|
||||
// - Quota limit wrt no of objects
|
||||
// ************************************************
|
||||
func TestCephSmokeSuite(t *testing.T) {
|
||||
if installer.SkipTestSuite(installer.CephTestSuite) {
|
||||
|
||||
Reference in new issue
Block a user