diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index c8b64e1ba..e7592e645 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -1546,6 +1546,22 @@ list of Ceph Filesystem volumes with ceph fs volume ls. To learn mo abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes

+ + +pinning
+ + +CephFilesystemSubVolumeGroupSpecPinning + + + + +(Optional) +

Pinning configuration of CephFilesystemSubVolumeGroup, +reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups +only one out of (export, distributed, random) can be set at a time

+ + @@ -3628,6 +3644,73 @@ list of Ceph Filesystem volumes with ceph fs volume ls. To learn mo abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes

+ + +pinning
+ + +CephFilesystemSubVolumeGroupSpecPinning + + + + +(Optional) +

Pinning configuration of CephFilesystemSubVolumeGroup, +reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups +only one out of (export, distributed, random) can be set at a time

+ + + + +

CephFilesystemSubVolumeGroupSpecPinning +

+

+(Appears on:CephFilesystemSubVolumeGroupSpec) +

+
+

CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup

+
+ + + + + + + + + + + + + + + + + + + +
FieldDescription
+export
+ +int + +
+(Optional) +
+distributed
+ +int + +
+(Optional) +
+random,
+ +float64 + +
+(Optional) +

CephFilesystemSubVolumeGroupStatus diff --git a/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml b/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml index 2ea90fc28..adbf45dda 100644 --- a/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml +++ b/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml @@ -11,6 +11,14 @@ spec: name: csi # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created filesystemName: {{ $filesystem.name }} + # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + # only one out of (export, distributed, random) can be set at a time + # by default pinning is set with value: distributed=1 + # for disabling default values set (distributed=0) + pinning: + distributed: 1 # distributed=<0, 1> (disabled=0) + # export: # export=<0-256> (disabled=-1) + # random: # random=[0.0, 1.0](disabled=0.0) --- apiVersion: ceph.rook.io/v1 kind: CephFilesystem diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index b1ec77255..09d3b9da1 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -7990,6 +7990,28 @@ spec: name: description: The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR. type: string + pinning: + description: Pinning configuration of CephFilesystemSubVolumeGroup, reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups only one out of (export, distributed, random) can be set at a time + properties: + distributed: + maximum: 1 + minimum: 0 + nullable: true + type: integer + export: + maximum: 256 + minimum: -1 + nullable: true + type: integer + random: + maximum: 1 + minimum: 0 + nullable: true + type: number + type: object + x-kubernetes-validations: + - message: only one pinning type should be set + rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random)) required: - filesystemName type: object diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index ee86e05f5..439eeed9a 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -7984,6 +7984,28 @@ spec: name: description: The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR. type: string + pinning: + description: Pinning configuration of CephFilesystemSubVolumeGroup, reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups only one out of (export, distributed, random) can be set at a time + properties: + distributed: + maximum: 1 + minimum: 0 + nullable: true + type: integer + export: + maximum: 256 + minimum: -1 + nullable: true + type: integer + random: + maximum: 1 + minimum: 0 + nullable: true + type: number + type: object + x-kubernetes-validations: + - message: only one pinning type should be set + rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random)) required: - filesystemName type: object diff --git a/deploy/examples/filesystem-ec.yaml b/deploy/examples/filesystem-ec.yaml index a4abf5a46..baab5dc7b 100644 --- a/deploy/examples/filesystem-ec.yaml +++ b/deploy/examples/filesystem-ec.yaml @@ -103,3 +103,11 @@ spec: name: csi # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created filesystemName: myfs + # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + # only one out of (export, distributed, random) can be set at a time + # by default pinning is set with value: distributed=1 + # for disabling default values set (distributed=0) + pinning: + distributed: 1 # distributed=<0, 1> (disabled=0) + # export: # export=<0-256> (disabled=-1) + # random: # random=[0.0, 1.0](disabled=0.0) diff --git a/deploy/examples/filesystem-test.yaml b/deploy/examples/filesystem-test.yaml index 52c00392e..f9d6cc98c 100644 --- a/deploy/examples/filesystem-test.yaml +++ b/deploy/examples/filesystem-test.yaml @@ -35,3 +35,11 @@ spec: name: csi # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created filesystemName: myfs + # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + # only one out of (export, distributed, random) can be set at a time + # by default pinning is set with value: distributed=1 + # for disabling default values set (distributed=0) + pinning: + distributed: 1 # distributed=<0, 1> (disabled=0) + # export: # export=<0-256> (disabled=-1) + # random: # random=[0.0, 1.0](disabled=0.0) diff --git a/deploy/examples/filesystem.yaml b/deploy/examples/filesystem.yaml index f824deebc..385beb36d 100644 --- a/deploy/examples/filesystem.yaml +++ b/deploy/examples/filesystem.yaml @@ -148,3 +148,11 @@ spec: name: csi # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created filesystemName: myfs + # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + # only one out of (export, distributed, random) can be set at a time + # by default pinning is set with value: distributed=1 + # for disabling default values set (distributed=0) + pinning: + distributed: 1 # distributed=<0, 1> (disabled=0) + # export: # export=<0-256> (disabled=-1) + # random: # random=[0.0, 1.0](disabled=0.0) diff --git a/deploy/examples/subvolumegroup.yaml b/deploy/examples/subvolumegroup.yaml index 77157123a..629cbe4b8 100644 --- a/deploy/examples/subvolumegroup.yaml +++ b/deploy/examples/subvolumegroup.yaml @@ -9,3 +9,11 @@ spec: # name: group-a # filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created filesystemName: myfs + # reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + # only one out of (export, distributed, random) can be set at a time + # by default pinning is set with value: distributed=1 + # for disabling default values set (distributed=0) + pinning: + distributed: 1 # distributed=<0, 1> (disabled=0) + # export: # export=<0-256> (disabled=-1) + # random: # random=[0.0, 1.0](disabled=0.0) diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 235f18932..0a4575bbf 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2930,6 +2930,31 @@ type CephFilesystemSubVolumeGroupSpec struct { // list of Ceph Filesystem volumes with `ceph fs volume ls`. To learn more about Ceph Filesystem // abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes FilesystemName string `json:"filesystemName"` + // Pinning configuration of CephFilesystemSubVolumeGroup, + // reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups + // only one out of (export, distributed, random) can be set at a time + // +optional + Pinning CephFilesystemSubVolumeGroupSpecPinning `json:"pinning,omitempty"` +} + +// CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup +// +kubebuilder:validation:XValidation:message="only one pinning type should be set",rule="(has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random))" +type CephFilesystemSubVolumeGroupSpecPinning struct { + // +kubebuilder:validation:Minimum=-1 + // +kubebuilder:validation:Maximum=256 + // +optional + // +nullable + Export *int `json:"export,omitempty"` + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + // +optional + // +nullable + Distributed *int `json:"distributed,omitempty"` + // +kubebuilder:validation:Minimum=0.0 + // +kubebuilder:validation:Maximum=1.0 + // +optional + // +nullable + Random *float64 `json:"random,,omitempty"` } // CephFilesystemSubVolumeGroupStatus represents the Status of Ceph Filesystem SubVolumeGroup diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 6d7341efe..57bd0bceb 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1122,7 +1122,7 @@ func (in *CephFilesystemSubVolumeGroup) DeepCopyInto(out *CephFilesystemSubVolum *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) if in.Status != nil { in, out := &in.Status, &out.Status *out = new(CephFilesystemSubVolumeGroupStatus) @@ -1185,6 +1185,7 @@ func (in *CephFilesystemSubVolumeGroupList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CephFilesystemSubVolumeGroupSpec) DeepCopyInto(out *CephFilesystemSubVolumeGroupSpec) { *out = *in + in.Pinning.DeepCopyInto(&out.Pinning) return } @@ -1198,6 +1199,37 @@ func (in *CephFilesystemSubVolumeGroupSpec) DeepCopy() *CephFilesystemSubVolumeG return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephFilesystemSubVolumeGroupSpecPinning) DeepCopyInto(out *CephFilesystemSubVolumeGroupSpecPinning) { + *out = *in + if in.Export != nil { + in, out := &in.Export, &out.Export + *out = new(int) + **out = **in + } + if in.Distributed != nil { + in, out := &in.Distributed, &out.Distributed + *out = new(int) + **out = **in + } + if in.Random != nil { + in, out := &in.Random, &out.Random + *out = new(float64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephFilesystemSubVolumeGroupSpecPinning. +func (in *CephFilesystemSubVolumeGroupSpecPinning) DeepCopy() *CephFilesystemSubVolumeGroupSpecPinning { + if in == nil { + return nil + } + out := new(CephFilesystemSubVolumeGroupSpecPinning) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CephFilesystemSubVolumeGroupStatus) DeepCopyInto(out *CephFilesystemSubVolumeGroupStatus) { *out = *in diff --git a/pkg/daemon/ceph/client/subvolumegroup.go b/pkg/daemon/ceph/client/subvolumegroup.go index 5e659a6b7..933fa4075 100644 --- a/pkg/daemon/ceph/client/subvolumegroup.go +++ b/pkg/daemon/ceph/client/subvolumegroup.go @@ -17,8 +17,13 @@ limitations under the License. package client import ( + "fmt" + "strconv" + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + "k8s.io/apimachinery/pkg/types" ) // CreateCephFSSubVolumeGroup create a CephFS subvolume group. @@ -41,7 +46,6 @@ func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterI // DeleteCephFSSubVolumeGroup delete a CephFS subvolume group. func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { logger.Infof("deleting cephfs %q subvolume group %q", volName, groupName) - // --force? args := []string{"fs", "subvolumegroup", "rm", volName, groupName} cmd := NewCephCommand(context, clusterInfo, args) cmd.JsonOutput = false @@ -55,3 +59,73 @@ func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterI logger.Infof("successfully deleted cephfs %q subvolume group %q", volName, groupName) return nil } + +// PinCephFSSubVolumeGroup pin the cephfs subvolume group +func PinCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName string, cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup, cephFilesystemSubVolumeGroupName string) error { + // namespace is the namespace of the svg CR, name is the svg name spec otherwise svg CR name + namespaceName := types.NamespacedName{Namespace: cephFilesystemSubVolumeGroup.Namespace, Name: cephFilesystemSubVolumeGroupName} + logger.Infof("validating pinning configuration of cephfs subvolume group %v of filesystem %q", namespaceName, volName) + err := validatePinningValues(cephFilesystemSubVolumeGroup.Spec.Pinning) + if err != nil { + return errors.Wrapf(err, "failed to pin subvolume group %q", cephFilesystemSubVolumeGroupName) + } + + logger.Infof("pinning cephfs subvolume group %v of filesystem %q", namespaceName, volName) + args := []string{"fs", "subvolumegroup", "pin", volName, cephFilesystemSubVolumeGroupName} + if cephFilesystemSubVolumeGroup.Spec.Pinning.Distributed != nil { + setting := strconv.Itoa(*cephFilesystemSubVolumeGroup.Spec.Pinning.Distributed) + args = append(args, "distributed", setting) + } else if cephFilesystemSubVolumeGroup.Spec.Pinning.Export != nil { + setting := strconv.Itoa(*cephFilesystemSubVolumeGroup.Spec.Pinning.Export) + args = append(args, "export", setting) + } else if cephFilesystemSubVolumeGroup.Spec.Pinning.Random != nil { + setting := strconv.FormatFloat(*cephFilesystemSubVolumeGroup.Spec.Pinning.Random, 'f', -1, 64) + args = append(args, "random", setting) + } else { + // set by default value + args = append(args, "distributed", "1") + } + logger.Infof("subvolume group pinning args %v", args) + + cmd := NewCephCommand(context, clusterInfo, args) + cmd.JsonOutput = false + output, err := cmd.Run() + if err != nil { + return errors.Wrapf(err, "failed to pin subvolume group %q. %s", cephFilesystemSubVolumeGroupName, output) + } + + logger.Infof("successfully pinned cephfs subvolume group %v", namespaceName) + return nil +} + +func validatePinningValues(pinning cephv1.CephFilesystemSubVolumeGroupSpecPinning) error { + numNils := 0 + var err error + if pinning.Export != nil { + numNils++ + if *pinning.Export > 256 { + err = errors.Errorf("validate pinning type failed, Export: value too big %d", *pinning.Export) + } else if *pinning.Export < -1 { + err = errors.Errorf("validate pinning type failed, Export: negative value %d not allowed except -1", *pinning.Export) + } + } + if pinning.Distributed != nil { + numNils++ + if !(*pinning.Distributed == 1) && !(*pinning.Distributed == 0) { + err = errors.Errorf("validate pinning type failed, Distributed: unknown value %d", *pinning.Distributed) + } + } + if pinning.Random != nil { + numNils++ + if (*pinning.Random < 0) || (*pinning.Random > 1.0) { + err = errors.Errorf("validate pinning type failed, Random: value %.2f is not between 0.0 and 1.1 (inclusive)", *pinning.Random) + } + } + if numNils > 1 { + return fmt.Errorf("only one can be set") + } + if numNils == 0 { + return nil // pinning disabled + } + return err +} diff --git a/pkg/daemon/ceph/client/subvolumegroup_test.go b/pkg/daemon/ceph/client/subvolumegroup_test.go new file mode 100644 index 000000000..077309e41 --- /dev/null +++ b/pkg/daemon/ceph/client/subvolumegroup_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/stretchr/testify/assert" +) + +func TestValidatePinningValues(t *testing.T) { + // set Distributed correctly + var testDistributedData cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testDistributedValue = 1 + testDistributedData.Distributed = &testDistributedValue + err := validatePinningValues(testDistributedData) + assert.NoError(t, err) + + // set Distributed wrongly + var testDistributedData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testDistributedValue1 = 5 + testDistributedData1.Distributed = &testDistributedValue1 + err = validatePinningValues(testDistributedData1) + assert.Error(t, err) + + // set Random correctly + var testRandomData cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testRandomdValue = 1.0 + testRandomData.Random = &testRandomdValue + err = validatePinningValues(testRandomData) + assert.NoError(t, err) + + // set Random wrongly + var testRandomData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testRandomdValue1 = 5.0 + testRandomData1.Random = &testRandomdValue1 + err = validatePinningValues(testRandomData1) + assert.Error(t, err) + + // set export correctly + var testExportData cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testExportValue = 1 + testExportData.Distributed = &testExportValue + err = validatePinningValues(testExportData) + assert.NoError(t, err) + + // set export wrongly + var testExportData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testExportValue1 = 500 + testExportData1.Distributed = &testExportValue1 + err = validatePinningValues(testExportData1) + assert.Error(t, err) + + // more than one set at a time, error + var testData cephv1.CephFilesystemSubVolumeGroupSpecPinning + var testValue = 1 + testData.Distributed = &testValue + testData.Export = &testValue + err = validatePinningValues(testData) + assert.Error(t, err) + + // nothing is set, noerror + var testData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning + err = validatePinningValues(testData1) + assert.NoError(t, err) +} diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index 7417dc522..dc9a18066 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -255,6 +255,12 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config") } + + err = cephclient.PinCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup, getSubvolumeGroupName(cephFilesystemSubVolumeGroup)) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to pin filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) + } + r.updateStatus(observedGeneration, request.NamespacedName, cephv1.ConditionReady) // Return and do not requeue logger.Debugf("done reconciling cephFilesystemSubVolumeGroup %q", namespacedName) diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go index 052d4bc50..2486444f2 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go @@ -195,6 +195,8 @@ func TestCephClientController(t *testing.T) { MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "create" { return "", nil + } else if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "pin" { + return "", nil } return "", errors.Errorf("unknown command. %v", args)