diff --git a/Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md b/Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md index 8f8941c4f..4016ee495 100644 --- a/Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md +++ b/Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md @@ -32,6 +32,10 @@ spec: distributed: 1 # distributed=<0, 1> (disabled=0) # export: # export=<0-256> (disabled=-1) # random: # random=[0.0, 1.0](disabled=0.0) + # Quota size of the subvolume group. + #quota: 10G + # data pool name for the subvolume group layout instead of the default data pool. + #dataPoolName: myfs-replicated ``` ## Settings @@ -48,7 +52,11 @@ If any setting is unspecified, a suitable default will be used automatically. * `filesystemName`: The metadata name of the CephFilesystem CR where the subvolume group will be created. -* `pinning`: To distribute load across MDS ranks in predictable and stable ways. Reference: https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups. +* `quota`: Quota size of the Ceph Filesystem subvolume group. + +* `dataPoolName`: The data pool name for the subvolume group layout instead of the default data pool. + +* `pinning`: To distribute load across MDS ranks in predictable and stable ways. See the Ceph doc for [Pinning subvolume groups](https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups). * `distributed`: Range: <0, 1>, for disabling it set to 0 * `export`: Range: <0-256>, for disabling it set to -1 * `random`: Range: [0.0, 1.0], for disabling it set to 0.0 diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index bf533a263..64197c517 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -1576,6 +1576,30 @@ reference + + +quota
+ +k8s.io/apimachinery/pkg/api/resource.Quantity + + + +(Optional) +

Quota size of the Ceph Filesystem subvolume group.

+ + + + +dataPoolName
+ +string + + + +(Optional) +

The data pool name for the Ceph Filesystem subvolume group layout, if the default CephFS pool is not desired.

+ +

CephFilesystemSubVolumeGroupSpecPinning diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 38eea18c8..f61b5f9ad 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -7983,6 +7983,9 @@ spec: spec: description: Spec represents the specification of a Ceph Filesystem SubVolumeGroup properties: + dataPoolName: + description: The data pool name for the Ceph Filesystem subvolume group layout, if the default CephFS pool is not desired. + type: string filesystemName: description: |- FilesystemName is the name of Ceph Filesystem SubVolumeGroup volume name. Typically it's the name of @@ -8024,6 +8027,13 @@ spec: x-kubernetes-validations: - message: only one pinning type should be set rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random)) + quota: + anyOf: + - type: integer + - type: string + description: Quota size of the Ceph Filesystem subvolume group. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true required: - filesystemName type: object diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 963903fa5..511facee7 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -7977,6 +7977,9 @@ spec: spec: description: Spec represents the specification of a Ceph Filesystem SubVolumeGroup properties: + dataPoolName: + description: The data pool name for the Ceph Filesystem subvolume group layout, if the default CephFS pool is not desired. + type: string filesystemName: description: |- FilesystemName is the name of Ceph Filesystem SubVolumeGroup volume name. Typically it's the name of @@ -8018,6 +8021,13 @@ spec: x-kubernetes-validations: - message: only one pinning type should be set rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random)) + quota: + anyOf: + - type: integer + - type: string + description: Quota size of the Ceph Filesystem subvolume group. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true required: - filesystemName type: object diff --git a/deploy/examples/subvolumegroup.yaml b/deploy/examples/subvolumegroup.yaml index 629cbe4b8..50345142a 100644 --- a/deploy/examples/subvolumegroup.yaml +++ b/deploy/examples/subvolumegroup.yaml @@ -17,3 +17,7 @@ spec: distributed: 1 # distributed=<0, 1> (disabled=0) # export: # export=<0-256> (disabled=-1) # random: # random=[0.0, 1.0](disabled=0.0) + # Quota size of the subvolume group. + #quota: 10G + # data pool name for the subvolume group layout instead of the default data pool. + #dataPoolName: myfs-replicated diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 134763977..9babbeeca 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -3022,6 +3022,12 @@ type CephFilesystemSubVolumeGroupSpec struct { // only one out of (export, distributed, random) can be set at a time // +optional Pinning CephFilesystemSubVolumeGroupSpecPinning `json:"pinning,omitempty"` + // Quota size of the Ceph Filesystem subvolume group. + // +optional + Quota *resource.Quantity `json:"quota,omitempty"` + // The data pool name for the Ceph Filesystem subvolume group layout, if the default CephFS pool is not desired. + // +optional + DataPoolName string `json:"dataPoolName"` } // CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup diff --git a/pkg/daemon/ceph/client/subvolumegroup.go b/pkg/daemon/ceph/client/subvolumegroup.go index 671eb384a..bfe991a37 100644 --- a/pkg/daemon/ceph/client/subvolumegroup.go +++ b/pkg/daemon/ceph/client/subvolumegroup.go @@ -17,9 +17,11 @@ limitations under the License. package client import ( + "encoding/json" "fmt" "strconv" "strings" + "syscall" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -30,21 +32,89 @@ import ( // CreateCephFSSubVolumeGroup create a CephFS subvolume group. // volName is the name of the Ceph FS volume, the same as the CephFilesystem CR name. -func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { +func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string, svgSpec *cephv1.CephFilesystemSubVolumeGroupSpec) error { logger.Infof("creating cephfs %q subvolume group %q", volName, groupName) - // [--pool_layout ] [--uid ] [--gid ] [--mode ] + // [] [--pool_layout ] [--uid ] [--gid ] [--mode ] args := []string{"fs", "subvolumegroup", "create", volName, groupName} + if svgSpec != nil { + if svgSpec.Quota != nil { + // convert the size to bytes as ceph expect the size in bytes + args = append(args, fmt.Sprintf("--size=%d", svgSpec.Quota.Value())) + } + if svgSpec.DataPoolName != "" { + args = append(args, fmt.Sprintf("--pool_layout=%s", svgSpec.DataPoolName)) + } + } + + svgInfo, err := getCephFSSubVolumeGroupInfo(context, clusterInfo, volName, groupName) + if err != nil { + // return error other than not found. + if code, ok := exec.ExitStatus(err); ok && code != int(syscall.ENOENT) { + return errors.Wrapf(err, "failed to create subvolume group %q in filesystem %q", groupName, volName) + } + } + + // if the subvolumegroup exists, resize the subvolumegroup + if err == nil && svgSpec != nil && svgSpec.Quota != nil && svgSpec.Quota.CmpInt64(svgInfo.BytesQuota) != 0 { + err = resizeCephFSSubVolumeGroup(context, clusterInfo, volName, groupName, svgSpec) + if err != nil { + return errors.Wrapf(err, "failed to create subvolume group %q in filesystem %q", groupName, volName) + } + } + cmd := NewCephCommand(context, clusterInfo, args) cmd.JsonOutput = false output, err := cmd.Run() if err != nil { - return errors.Wrapf(err, "failed to create subvolume group %q. %s", volName, output) + return errors.Wrapf(err, "failed to create subvolume group %q in filesystem %q. %s", groupName, volName, output) } - logger.Infof("successfully created cephfs %q subvolume group %q", volName, groupName) + logger.Infof("successfully created subvolume group %q in filesystem %q", groupName, volName) return nil } +// resizeCephFSSubVolumeGroup resize a CephFS subvolume group. +// volName is the name of the Ceph FS volume, the same as the CephFilesystem CR name. +func resizeCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string, svgSpec *cephv1.CephFilesystemSubVolumeGroupSpec) error { + logger.Infof("resizing cephfs %q subvolume group %q", volName, groupName) + // [--no-shrink] + args := []string{"fs", "subvolumegroup", "resize", volName, groupName, "--no-shrink", fmt.Sprintf("%d", svgSpec.Quota.Value())} + cmd := NewCephCommand(context, clusterInfo, args) + cmd.JsonOutput = false + output, err := cmd.Run() + if err != nil { + return errors.Wrapf(err, "failed to resize subvolume group %q in filesystem %q. %s", groupName, volName, output) + } + + logger.Infof("successfully resized subvolume group %q in filesystem %q to %s", groupName, volName, svgSpec.Quota) + return nil +} + +type subvolumeGroupInfo struct { + BytesQuota int64 `json:"bytes_quota"` + BytesUsed int64 `json:"bytes_used"` + DataPool string `json:"data_pool"` +} + +// getCephFSSubVolumeGroupInfo get subvolumegroup info of the group name. +// volName is the name of the Ceph FS volume, the same as the CephFilesystem CR name. +func getCephFSSubVolumeGroupInfo(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) (*subvolumeGroupInfo, error) { + args := []string{"fs", "subvolumegroup", "info", volName, groupName} + cmd := NewCephCommand(context, clusterInfo, args) + cmd.JsonOutput = true + output, err := cmd.Run() + if err != nil { + return nil, errors.Wrapf(err, "failed to get subvolume group %q in filesystem %q. %s", groupName, volName, output) + } + + svgInfo := subvolumeGroupInfo{} + err = json.Unmarshal(output, &svgInfo) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal into subvolumeGroupInfo") + } + return &svgInfo, nil +} + // DeleteCephFSSubVolumeGroup delete a CephFS subvolume group. func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { logger.Infof("deleting cephfs %q subvolume group %q", volName, groupName) diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 56cf9a046..898875c6b 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -74,7 +74,7 @@ func createFilesystem( } } - err := cephclient.CreateCephFSSubVolumeGroup(context, clusterInfo, fs.Name, defaultCSISubvolumeGroup) + err := cephclient.CreateCephFSSubVolumeGroup(context, clusterInfo, fs.Name, defaultCSISubvolumeGroup, nil) if err != nil { return errors.Wrapf(err, "failed to create subvolume group %q", defaultCSISubvolumeGroup) } diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index 637bc4563..ffe8aaf5d 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -233,6 +233,8 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData return `{"standbys":[], "filesystems":[]}`, nil } else if reflect.DeepEqual(args[0:5], []string{"fs", "subvolumegroup", "create", fsName, defaultCSISubvolumeGroup}) { return "", nil + } else if reflect.DeepEqual(args[0:5], []string{"fs", "subvolumegroup", "info", fsName, defaultCSISubvolumeGroup}) { + return "", nil } else if contains(args, "osd") && contains(args, "lspools") { return "[]", nil } else if contains(args, "mds") && contains(args, "fail") { diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index c77c16baf..a291a8e04 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -318,7 +318,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesyst func (r *ReconcileCephFilesystemSubVolumeGroup) createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup) error { logger.Infof("creating ceph filesystem subvolume group %s in namespace %s", cephFilesystemSubVolumeGroup.Name, cephFilesystemSubVolumeGroup.Namespace) - err := cephclient.CreateCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, getSubvolumeGroupName(cephFilesystemSubVolumeGroup)) + err := cephclient.CreateCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, getSubvolumeGroupName(cephFilesystemSubVolumeGroup), &cephFilesystemSubVolumeGroup.Spec) if err != nil { return errors.Wrapf(err, "failed to create ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 471d79e76..bb8fb0175 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -652,7 +652,9 @@ metadata: name: ` + groupName + ` namespace: ` + m.settings.Namespace + ` spec: - filesystemName: ` + fsName + filesystemName: ` + fsName + ` + quota: 10G + dataPoolName: ` + fsName + "-data0" } func (m *CephManifestsMaster) GetCOSIDriver() string {