Merge pull request #5674 from thotz/quotaobc

ceph: adding support for quota in object bucket claims
This commit is contained in:
Travis Nielsen authored and GitHub committed 2020-07-30 22:51:58 -06:00
commit e0ea87a353
12 files changed
+153 -32

No files matched your search

+5 -2
View File
@@ -28,7 +28,8 @@ spec:
generateBucketName: photo-booth [4]
storageClassName: rook-ceph-bucket [4]
additionalConfig: [5]
ANY_KEY: VALUE ...
maxObjects: "1000"
maxSize: "2G"
```
1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap.
1. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret.
@@ -39,7 +40,9 @@ an entire object store.
If both `bucketName` and `generateBucketName` are supplied then `BucketName` has precedence and `GenerateBucketName` is ignored.
If both `bucketName` and `generateBucketName` are blank or omitted then the storage class is expected to contain the name of an _existing_ bucket. It's an error if all three bucket related names are blank or omitted.
1. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy.
1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Examples can include config values such as tenant, user and policy settings, etc.
1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported:
- `maxObjects`: The maximum number of objects in the bucket
- `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K.
### OBC Custom Resource after Bucket Provisioning
```yaml
+1
View File
@@ -23,6 +23,7 @@
- OBC changes:
- Updated lib bucket provisioner version to support multithread and required change can be found in [operator.yaml](cluster/examples/kubernetes/ceph/operator.yaml#L449)
- Can be extended to add support for other providers
- Added support for [quota](Documentation/ceph-object-bucket-claim.md#obc-custom-resource), have options for object count and total size.
- CephObjectStore CRD changes:
- Health displayed in the Status field
- Supports connecting to external Ceph Rados Gateways, refer to the [external object section](Documentation/ceph-object.html#connect-to-external-object-store)
@@ -11,3 +11,7 @@ spec:
#bucketName:
generateBucketName: ceph-bkt
storageClassName: rook-ceph-delete-bucket
additionalConfig:
# To set for quota for OBC
#maxObjects: "1000"
#maxSize: "2G"
@@ -11,3 +11,7 @@ spec:
#bucketName:
generateBucketName: ceph-bkt
storageClassName: rook-ceph-retain-bucket
additionalConfig:
# To set for quota for OBC
#maxObjects: "1000"
#maxSize: "2G"
+48 -3
View File
@@ -94,13 +94,20 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
return nil, err
}
_, errCode, err := cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets)
if errCode > 0 {
_, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, maxBuckets)
if err != nil {
p.deleteOBCResourceLogError(p.bucketName)
return nil, err
}
logger.Infof("set user %q bucket max to %d", p.cephUserName, maxBuckets)
// setting quota limit if it is enabled
err = p.setAdditionalSettings(options)
if err != nil {
p.deleteOBCResource(p.bucketName)
return nil, err
}
return p.composeObjectBucket(), nil
}
@@ -127,7 +134,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
}
// need to quota into -1 for restricting creation of new buckets in rgw
_, _, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1)
_, err = cephObject.SetQuotaUserBucketMax(p.objectContext, p.cephUserName, -1)
if err != nil {
p.deleteOBCResourceLogError("")
return nil, err
@@ -181,6 +188,14 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
p.deleteOBCResourceLogError("")
return nil, err
}
// setting quota limit if it is enabled
err = p.setAdditionalSettings(options)
if err != nil {
p.deleteOBCResource("")
return nil, err
}
// returned ob with connection info
return p.composeObjectBucket(), nil
}
@@ -494,3 +509,33 @@ func (p *Provisioner) deleteOBCResourceLogError(name string) {
logger.Warningf("error deleting OBC resource. %v", err)
}
}
// Check for additional options mentioned in OBC and set them accordingly
func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error {
maxObjects := MaxObjectQuota(options)
maxSize := MaxSizeQuota(options)
if maxObjects == "" && maxSize == "" {
return nil
}
var err error
if maxObjects != "" {
if _, err = cephObject.SetQuotaUserObjectMax(p.objectContext, p.cephUserName, maxObjects); err != nil {
logger.Errorf("Setting MaxObject failed %v", err)
return err
}
}
if maxSize != "" {
_, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize)
if _, err = cephObject.SetQuotaUserMaxSize(p.objectContext, p.cephUserName, maxSize); err != nil {
logger.Errorf("Setting MaxSize failed %v", err)
return err
}
}
if _, err = cephObject.EnableUserQuota(p.objectContext, p.cephUserName); err != nil {
logger.Errorf("Enabling user quota for obc failed %v", err)
return err
}
return nil
}
+9
View File
@@ -22,6 +22,7 @@ import (
"github.com/coreos/pkg/capnslog"
bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
"github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner"
apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
@@ -134,3 +135,11 @@ func randomString(n int) string {
}
return string(b)
}
func MaxObjectQuota(options *apibkt.BucketOptions) string {
return options.ObjectBucketClaim.Spec.AdditionalConfig["maxObjects"]
}
func MaxSizeQuota(options *apibkt.BucketOptions) string {
return options.ObjectBucketClaim.Spec.AdditionalConfig["maxSize"]
}
+43 -6
View File
@@ -183,25 +183,27 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) {
return result, errors.Wrap(err, "failed to delete s3 user")
}
func SetQuotaUserBucketMax(c *Context, id string, max int) (string, int, error) {
// SetQuotaUserBucketMax will set maximum bucket quota for a user
func SetQuotaUserBucketMax(c *Context, id string, max int) (string, error) {
logger.Infof("Setting user %q max buckets to %d", id, max)
args := []string{"--quota-scope", "user", "--max-buckets", strconv.Itoa(max)}
result, errCode, err := setUserQuota(c, id, args)
if errCode != RGWErrorNone {
result, err := setUserQuota(c, id, args)
if err != nil {
err = errors.Wrap(err, "failed setting bucket max")
}
return result, errCode, err
return result, err
}
func setUserQuota(c *Context, id string, args []string) (string, int, error) {
func setUserQuota(c *Context, id string, args []string) (string, error) {
args = append([]string{"quota", "set", "--uid", id}, args...)
result, err := runAdminCommand(c, args...)
if err != nil {
err = errors.Wrap(err, "failed to set max buckets for user")
}
return result, RGWErrorNone, err
return result, err
}
// LinkUser will link a user to a bucket
func LinkUser(c *Context, id, bucket string) (string, int, error) {
logger.Infof("Linking (user: %s) (bucket: %s)", id, bucket)
args := []string{"bucket", "link", "--uid", id, "--bucket", bucket}
@@ -215,6 +217,7 @@ func LinkUser(c *Context, id, bucket string) (string, int, error) {
return result, RGWErrorNone, nil
}
// UnlinkUser will unlink the user from a bucket
func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
logger.Infof("Unlinking (user: %s) (bucket: %s)", id, bucket)
args := []string{"bucket", "unlink", "--uid", id, "--bucket", bucket}
@@ -227,3 +230,37 @@ func UnlinkUser(c *Context, id, bucket string) (string, int, error) {
}
return result, RGWErrorNone, nil
}
// EnableUserQuota will allows to enable quota defined for a user
func EnableUserQuota(c *Context, id string) (string, error) {
logger.Debug("Enabling user quota for %q", id)
args := append([]string{"quota", "enable", "--quota-scope", "user", "--uid", id})
result, err := runAdminCommand(c, args...)
if err != nil {
err = errors.Wrap(err, "failed to enable quota for the user")
}
return result, err
}
// SetQuotaUserObject allows to set maximum limit on objects for a user
func SetQuotaUserObjectMax(c *Context, id string, maxobjects string) (string, error) {
logger.Debugf("Setting user %q max objects to %s", id, maxobjects)
args := []string{"--quota-scope", "user", "--max-objects", maxobjects}
result, err := setUserQuota(c, id, args)
if err != nil {
err = errors.Wrap(err, "failed setting object max")
}
return result, err
}
// SetQuotaUserMaxSize allows to set maximum size for a user
func SetQuotaUserMaxSize(c *Context, id string, maxsize string) (string, error) {
logger.Debugf("Setting user %q max size to %s", id, maxsize)
args := []string{"--quota-scope", "user", "--max-size", maxsize}
result, err := setUserQuota(c, id, args)
if err != nil {
err = errors.Wrap(err, "failed setting max size")
}
return result, err
}
+4 -4
View File
@@ -44,12 +44,12 @@ func (b *BucketOperation) DeleteBucketStorageClass(namespace string, storeName s
return err
}
func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, createBucket bool) error {
return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket))
func (b *BucketOperation) CreateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
return b.k8sh.ResourceOperation("create", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
}
func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, createBucket bool) error {
return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, createBucket))
func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
return b.k8sh.ResourceOperation("delete", b.manifests.GetObc(obcName, storageClassName, bucketName, maxObject, createBucket))
}
// CheckOBC, returns true if the obc, secret and configmap are all in the "check" state,
+5 -3
View File
@@ -49,7 +49,7 @@ type CephManifests interface {
GetObjectStore(namespace, name string, replicaCount, port int) string
GetObjectStoreUser(namespace, name, displayName, store string) string
GetBucketStorageClass(namespace, storeName, storageClassName, reclaimPolicy, region string) string
GetObc(obcName, storageClassName, bucketName string, createBucket bool) string
GetObc(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string
GetClient(namespace, name string, caps map[string]string) string
}
@@ -2295,7 +2295,7 @@ parameters:
}
//GetObc returns the manifest to create object bucket claim
func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string {
func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string {
bucketParameter := "generateBucketName"
if varBucketName {
bucketParameter = "bucketName"
@@ -2306,7 +2306,9 @@ metadata:
name: ` + claimName + `
spec:
` + bucketParameter + `: ` + objectBucketName + `
storageClassName: ` + storageClassName
storageClassName: ` + storageClassName + `
additionalConfig:
maxObjects: "` + maxObject + `"`
}
func (m *CephManifestsMaster) GetClient(namespace string, claimName string, caps map[string]string) string {
@@ -1904,7 +1904,7 @@ parameters:
}
//GetObc returns the manifest to create object bucket claim
func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string {
func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string {
bucketParameter := "generateBucketName"
if varBucketName {
bucketParameter = "bucketName"
@@ -1915,7 +1915,9 @@ metadata:
name: ` + claimName + `
spec:
` + bucketParameter + `: ` + objectBucketName + `
storageClassName: ` + storageClassName
storageClassName: ` + storageClassName + `
additionalConfig:
maxObjects: "` + maxObject + `"`
}
func (m *CephManifestsV1_2) GetClient(namespace string, claimName string, caps map[string]string) string {
+25 -12
View File
@@ -36,11 +36,14 @@ var (
userid = "rook-user"
userdisplayname = "A rook RGW user"
bucketname = "smokebkt"
objBody = "Test Rook Object Data"
objectKey = "rookObj1"
ObjBody = "Test Rook Object Data"
ObjectKey1 = "rookObj1"
ObjectKey2 = "rookObj2"
ObjectKey3 = "rookObj3"
contentType = "plain/text"
obcName = "smoke-delete-bucket"
region = "us-east-1"
maxObject = "2"
)
// Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order
@@ -119,7 +122,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
bucketStorageClassName := "rook-smoke-delete-bucket"
cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, bucketStorageClassName, "Delete", region)
require.Nil(s.T(), cobErr)
cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, true)
cobcErr := helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
require.Nil(s.T(), cobcErr)
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "created"); i++ {
@@ -154,22 +157,32 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
logger.Infof("Step 5 : Put Object on bucket")
_, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType)
_, poErr := s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey1, contentType)
require.Nil(s.T(), poErr)
logger.Infof("Step 6 : Get Object from bucket")
read, err := s3client.GetObjectInBucket(bucketname, objectKey)
read, err := s3client.GetObjectInBucket(bucketname, ObjectKey1)
require.Nil(s.T(), err)
require.Equal(s.T(), objBody, read)
require.Equal(s.T(), ObjBody, read)
logger.Infof("Object Created and Retrieved on bucket successfully")
logger.Infof("Step 7 : Delete object on bucket")
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey)
require.Nil(s.T(), delobjErr)
logger.Infof("Object deleted on bucket successfully")
logger.Infof("Step 7 : Testing Quota for the OBC")
logger.Infof("Adding one more object to the bucket")
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey2, contentType)
require.Nil(s.T(), poErr)
logger.Infof("Testing the max object limit")
_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType)
require.Error(s.T(), poErr)
logger.Infof("Step 8 : Delete Object Bucket Claim")
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, true)
logger.Infof("Step 8 : Delete objects on bucket")
_, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1)
require.Nil(s.T(), delobjErr)
_, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2)
require.Nil(s.T(), delobjErr)
logger.Infof("Objects deleted on bucket successfully")
logger.Infof("Step 9 : Delete Object Bucket Claim")
dobcErr := helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketname, maxObject, true)
require.Nil(s.T(), dobcErr)
logger.Infof("Checking to see if the obc, secret and cm have all been deleted")
for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
+1
View File
@@ -56,6 +56,7 @@ import (
// - Create/delete buckets
// - Create/delete users
// - PUT/GET objects
// - Quota limit wrt no of objects
// ************************************************
func TestCephSmokeSuite(t *testing.T) {
if installer.SkipTestSuite(installer.CephTestSuite) {