Merge pull request #7264 from fritchie/pool-quotas

ceph: add ability to set pool quotas

Documentation/ceph-pool-crd.md

@@ -192,6 +192,11 @@ stretched) then you will have 2 replicas per datacenter where each replica ends
     * `disabled`: whether to enable or disable pool mirroring status
     * `interval`: time interval to refresh the mirroring status (default 60s)
 
+* `quotas`: Set byte and object quotas. See the [ceph documentation](https://docs.ceph.com/en/latest/rados/operations/pools/#set-pool-quotas) for more info.
+  * `maxBytes`: quota in bytes (default: 0)
+  * `maxObjects`: quota in objects (default: 0)
+    > **NOTE**: A value of 0 disables the quota.
+
 ### Add specific pool properties
 
 With `poolProperties` you can set any pool property:

cluster/charts/rook-ceph/templates/resources.yaml

@@ -1216,6 +1216,16 @@ spec:
                             type: string
                           startTime:
                             type: string
+                quotas:
+                  type: object
+                  nullable: true
+                  properties:
+                    maxBytes:
+                      type: integer
+                      minimum: 0
+                    maxObjects:
+                      type: integer
+                      minimum: 0
                 statusCheck:
                   type: object
                   x-kubernetes-preserve-unknown-fields: true

cluster/examples/kubernetes/ceph/crds.yaml

@@ -1263,6 +1263,16 @@ spec:
                             type: string
                           startTime:
                             type: string
+                quotas:
+                  type: object
+                  nullable: true
+                  properties:
+                    maxBytes:
+                      type: integer
+                      minimum: 0
+                    maxObjects:
+                      type: integer
+                      minimum: 0
                 statusCheck:
                   type: object
                   x-kubernetes-preserve-unknown-fields: true

cluster/examples/kubernetes/ceph/pool.yaml

@@ -54,6 +54,11 @@ spec:
     mirror:
       disabled: false
       interval: 60s
+  # quota in bytes and/or objects, default value is 0 (unlimited)
+  # see https://docs.ceph.com/en/latest/rados/operations/pools/#set-pool-quotas
+  # quotas:
+    # maxBytes: 10737418240 # 10GiB = 10 * 1024 * 1024 * 1024
+    # maxObjects: 1000000000 # 1 billion objects
   # A key/value list of annotations
   annotations:
   #  key: value

pkg/apis/ceph.rook.io/v1/types.go

@@ -393,6 +393,9 @@ type PoolSpec struct {
 
 	// The mirroring statusCheck
 	StatusCheck MirrorHealthCheckSpec `json:"statusCheck"`
+
+	// The quota settings
+	Quotas QuotaSpec `json:"quotas"`
 }
 
 type MirrorHealthCheckSpec struct {
@@ -477,6 +480,15 @@ type SnapshotScheduleSpec struct {
 	StartTime string `json:"startTime,omitempty"`
 }
 
+// QuotaSpec represents the spec for quotas in a pool
+type QuotaSpec struct {
+	// MaxBytes represents the quota in bytes
+	MaxBytes *uint64 `json:"maxBytes,omitempty"`
+
+	// MaxObjects represents the quota in objects
+	MaxObjects *uint64 `json:"maxObjects,omitempty"`
+}
+
 // ErasureCodeSpec represents the spec for erasure code in a pool
 type ErasureCodedSpec struct {
 	// Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type)

pkg/daemon/ceph/client/pool.go

@@ -283,6 +283,23 @@ func setCommonPoolProperties(context *clusterd.Context, clusterInfo *ClusterInfo
 		}
 	}
 
+	// set max_bytes quota
+	if pool.Quotas.MaxBytes != nil {
+		// set max_bytes quota, 0 value disables quota
+		err := setPoolQuota(context, clusterInfo, poolName, "max_bytes", strconv.FormatUint(*pool.Quotas.MaxBytes, 10))
+		if err != nil {
+			return errors.Wrapf(err, "failed to set max_bytes quota for pool %q", poolName)
+		}
+	}
+	// set max_objects quota
+	if pool.Quotas.MaxObjects != nil {
+		// set max_objects quota, 0 value disables quota
+		err := setPoolQuota(context, clusterInfo, poolName, "max_objects", strconv.FormatUint(*pool.Quotas.MaxObjects, 10))
+		if err != nil {
+			return errors.Wrapf(err, "failed to set max_objects quota for pool %q", poolName)
+		}
+	}
+
 	return nil
 }
 
@@ -488,6 +505,17 @@ func SetPoolProperty(context *clusterd.Context, clusterInfo *ClusterInfo, name,
 	return nil
 }
 
+// setPoolQuota sets quotas on a given pool
+func setPoolQuota(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, quotaType, quotaVal string) error {
+	args := []string{"osd", "pool", "set-quota", poolName, quotaType, quotaVal}
+	logger.Infof("setting quota %q=%q on pool %q", quotaType, quotaVal, poolName)
+	_, err := NewCephCommand(context, clusterInfo, args).Run()
+	if err != nil {
+		return errors.Wrapf(err, "failed to set %q quota on pool %q", quotaType, poolName)
+	}
+	return nil
+}
+
 // SetPoolReplicatedSizeProperty sets the replica size of a pool
 func SetPoolReplicatedSizeProperty(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, size string) error {
 	propName := "size"

tests/framework/installer/ceph_manifests.go

@@ -1269,6 +1269,16 @@ spec:
                             type: string
                           startTime:
                             type: string
+                quotas:
+                  type: object
+                  nullable: true
+                  properties:
+                    maxBytes:
+                      type: integer
+                      minimum: 0
+                    maxObjects:
+                      type: integer
+                      minimum: 0
                 statusCheck:
                   type: object
                   x-kubernetes-preserve-unknown-fields: true
@@ -3769,6 +3779,9 @@ spec:
   mirroring:
     enabled: true
     mode: image
+  quotas:
+    maxBytes: 10737418240
+    maxObjects: 1000000
   statusCheck:
     mirror:
       disabled: false