Merge pull request #13369 from rook/mergify/bp/release-1.13/pr-13283

subvolumegroup: add pinning spec in subvolumegroup CRD (backport #13283)
This commit is contained in:
Travis Nielsen authored and GitHub committed 2023-12-11 12:46:23 -07:00
commit 65c9ec87f8
14 files changed
+389 -2

No files matched your search

+83
View File
@@ -1546,6 +1546,22 @@ list of Ceph Filesystem volumes with <code>ceph fs volume ls</code>. To learn mo
abstractions see <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes</a></p>
</td>
</tr>
<tr>
<td>
<code>pinning</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">
CephFilesystemSubVolumeGroupSpecPinning
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>Pinning configuration of CephFilesystemSubVolumeGroup,
reference <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups</a>
only one out of (export, distributed, random) can be set at a time</p>
</td>
</tr>
</table>
</td>
</tr>
@@ -3628,6 +3644,73 @@ list of Ceph Filesystem volumes with <code>ceph fs volume ls</code>. To learn mo
abstractions see <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes</a></p>
</td>
</tr>
<tr>
<td>
<code>pinning</code><br/>
<em>
<a href="#ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">
CephFilesystemSubVolumeGroupSpecPinning
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>Pinning configuration of CephFilesystemSubVolumeGroup,
reference <a href="https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups">https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups</a>
only one out of (export, distributed, random) can be set at a time</p>
</td>
</tr>
</tbody>
</table>
<h3 id="ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">CephFilesystemSubVolumeGroupSpecPinning
</h3>
<p>
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpec">CephFilesystemSubVolumeGroupSpec</a>)
</p>
<div>
<p>CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup</p>
</div>
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>export</code><br/>
<em>
int
</em>
</td>
<td>
<em>(Optional)</em>
</td>
</tr>
<tr>
<td>
<code>distributed</code><br/>
<em>
int
</em>
</td>
<td>
<em>(Optional)</em>
</td>
</tr>
<tr>
<td>
<code>random,</code><br/>
<em>
float64
</em>
</td>
<td>
<em>(Optional)</em>
</td>
</tr>
</tbody>
</table>
<h3 id="ceph.rook.io/v1.CephFilesystemSubVolumeGroupStatus">CephFilesystemSubVolumeGroupStatus
@@ -11,6 +11,14 @@ spec:
name: csi
# filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
filesystemName: {{ $filesystem.name }}
# reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
# only one out of (export, distributed, random) can be set at a time
# by default pinning is set with value: distributed=1
# for disabling default values set (distributed=0)
pinning:
distributed: 1 # distributed=<0, 1> (disabled=0)
# export: # export=<0-256> (disabled=-1)
# random: # random=[0.0, 1.0](disabled=0.0)
---
apiVersion: ceph.rook.io/v1
kind: CephFilesystem
@@ -7990,6 +7990,28 @@ spec:
name:
description: The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
type: string
pinning:
description: Pinning configuration of CephFilesystemSubVolumeGroup, reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups only one out of (export, distributed, random) can be set at a time
properties:
distributed:
maximum: 1
minimum: 0
nullable: true
type: integer
export:
maximum: 256
minimum: -1
nullable: true
type: integer
random:
maximum: 1
minimum: 0
nullable: true
type: number
type: object
x-kubernetes-validations:
- message: only one pinning type should be set
rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random))
required:
- filesystemName
type: object
+22
View File
@@ -7984,6 +7984,28 @@ spec:
name:
description: The name of the subvolume group. If not set, the default is the name of the subvolumeGroup CR.
type: string
pinning:
description: Pinning configuration of CephFilesystemSubVolumeGroup, reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups only one out of (export, distributed, random) can be set at a time
properties:
distributed:
maximum: 1
minimum: 0
nullable: true
type: integer
export:
maximum: 256
minimum: -1
nullable: true
type: integer
random:
maximum: 1
minimum: 0
nullable: true
type: number
type: object
x-kubernetes-validations:
- message: only one pinning type should be set
rule: (has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random))
required:
- filesystemName
type: object
+8
View File
@@ -103,3 +103,11 @@ spec:
name: csi
# filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
filesystemName: myfs
# reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
# only one out of (export, distributed, random) can be set at a time
# by default pinning is set with value: distributed=1
# for disabling default values set (distributed=0)
pinning:
distributed: 1 # distributed=<0, 1> (disabled=0)
# export: # export=<0-256> (disabled=-1)
# random: # random=[0.0, 1.0](disabled=0.0)
+8
View File
@@ -35,3 +35,11 @@ spec:
name: csi
# filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
filesystemName: myfs
# reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
# only one out of (export, distributed, random) can be set at a time
# by default pinning is set with value: distributed=1
# for disabling default values set (distributed=0)
pinning:
distributed: 1 # distributed=<0, 1> (disabled=0)
# export: # export=<0-256> (disabled=-1)
# random: # random=[0.0, 1.0](disabled=0.0)
+8
View File
@@ -148,3 +148,11 @@ spec:
name: csi
# filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
filesystemName: myfs
# reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
# only one out of (export, distributed, random) can be set at a time
# by default pinning is set with value: distributed=1
# for disabling default values set (distributed=0)
pinning:
distributed: 1 # distributed=<0, 1> (disabled=0)
# export: # export=<0-256> (disabled=-1)
# random: # random=[0.0, 1.0](disabled=0.0)
+8
View File
@@ -9,3 +9,11 @@ spec:
# name: group-a
# filesystemName is the metadata name of the CephFilesystem CR where the subvolume group will be created
filesystemName: myfs
# reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
# only one out of (export, distributed, random) can be set at a time
# by default pinning is set with value: distributed=1
# for disabling default values set (distributed=0)
pinning:
distributed: 1 # distributed=<0, 1> (disabled=0)
# export: # export=<0-256> (disabled=-1)
# random: # random=[0.0, 1.0](disabled=0.0)
+25
View File
@@ -2930,6 +2930,31 @@ type CephFilesystemSubVolumeGroupSpec struct {
// list of Ceph Filesystem volumes with `ceph fs volume ls`. To learn more about Ceph Filesystem
// abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes
FilesystemName string `json:"filesystemName"`
// Pinning configuration of CephFilesystemSubVolumeGroup,
// reference https://docs.ceph.com/en/latest/cephfs/fs-volumes/#pinning-subvolumes-and-subvolume-groups
// only one out of (export, distributed, random) can be set at a time
// +optional
Pinning CephFilesystemSubVolumeGroupSpecPinning `json:"pinning,omitempty"`
}
// CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup
// +kubebuilder:validation:XValidation:message="only one pinning type should be set",rule="(has(self.export) && !has(self.distributed) && !has(self.random)) || (!has(self.export) && has(self.distributed) && !has(self.random)) || (!has(self.export) && !has(self.distributed) && has(self.random)) || (!has(self.export) && !has(self.distributed) && !has(self.random))"
type CephFilesystemSubVolumeGroupSpecPinning struct {
// +kubebuilder:validation:Minimum=-1
// +kubebuilder:validation:Maximum=256
// +optional
// +nullable
Export *int `json:"export,omitempty"`
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=1
// +optional
// +nullable
Distributed *int `json:"distributed,omitempty"`
// +kubebuilder:validation:Minimum=0.0
// +kubebuilder:validation:Maximum=1.0
// +optional
// +nullable
Random *float64 `json:"random,,omitempty"`
}
// CephFilesystemSubVolumeGroupStatus represents the Status of Ceph Filesystem SubVolumeGroup
@@ -1122,7 +1122,7 @@ func (in *CephFilesystemSubVolumeGroup) DeepCopyInto(out *CephFilesystemSubVolum
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Spec.DeepCopyInto(&out.Spec)
if in.Status != nil {
in, out := &in.Status, &out.Status
*out = new(CephFilesystemSubVolumeGroupStatus)
@@ -1185,6 +1185,7 @@ func (in *CephFilesystemSubVolumeGroupList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CephFilesystemSubVolumeGroupSpec) DeepCopyInto(out *CephFilesystemSubVolumeGroupSpec) {
*out = *in
in.Pinning.DeepCopyInto(&out.Pinning)
return
}
@@ -1198,6 +1199,37 @@ func (in *CephFilesystemSubVolumeGroupSpec) DeepCopy() *CephFilesystemSubVolumeG
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CephFilesystemSubVolumeGroupSpecPinning) DeepCopyInto(out *CephFilesystemSubVolumeGroupSpecPinning) {
*out = *in
if in.Export != nil {
in, out := &in.Export, &out.Export
*out = new(int)
**out = **in
}
if in.Distributed != nil {
in, out := &in.Distributed, &out.Distributed
*out = new(int)
**out = **in
}
if in.Random != nil {
in, out := &in.Random, &out.Random
*out = new(float64)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephFilesystemSubVolumeGroupSpecPinning.
func (in *CephFilesystemSubVolumeGroupSpecPinning) DeepCopy() *CephFilesystemSubVolumeGroupSpecPinning {
if in == nil {
return nil
}
out := new(CephFilesystemSubVolumeGroupSpecPinning)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CephFilesystemSubVolumeGroupStatus) DeepCopyInto(out *CephFilesystemSubVolumeGroupStatus) {
*out = *in
+75 -1
View File
@@ -17,8 +17,13 @@ limitations under the License.
package client
import (
"fmt"
"strconv"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/apimachinery/pkg/types"
)
// CreateCephFSSubVolumeGroup create a CephFS subvolume group.
@@ -41,7 +46,6 @@ func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterI
// DeleteCephFSSubVolumeGroup delete a CephFS subvolume group.
func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error {
logger.Infof("deleting cephfs %q subvolume group %q", volName, groupName)
// --force?
args := []string{"fs", "subvolumegroup", "rm", volName, groupName}
cmd := NewCephCommand(context, clusterInfo, args)
cmd.JsonOutput = false
@@ -55,3 +59,73 @@ func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterI
logger.Infof("successfully deleted cephfs %q subvolume group %q", volName, groupName)
return nil
}
// PinCephFSSubVolumeGroup pin the cephfs subvolume group
func PinCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName string, cephFilesystemSubVolumeGroup *cephv1.CephFilesystemSubVolumeGroup, cephFilesystemSubVolumeGroupName string) error {
// namespace is the namespace of the svg CR, name is the svg name spec otherwise svg CR name
namespaceName := types.NamespacedName{Namespace: cephFilesystemSubVolumeGroup.Namespace, Name: cephFilesystemSubVolumeGroupName}
logger.Infof("validating pinning configuration of cephfs subvolume group %v of filesystem %q", namespaceName, volName)
err := validatePinningValues(cephFilesystemSubVolumeGroup.Spec.Pinning)
if err != nil {
return errors.Wrapf(err, "failed to pin subvolume group %q", cephFilesystemSubVolumeGroupName)
}
logger.Infof("pinning cephfs subvolume group %v of filesystem %q", namespaceName, volName)
args := []string{"fs", "subvolumegroup", "pin", volName, cephFilesystemSubVolumeGroupName}
if cephFilesystemSubVolumeGroup.Spec.Pinning.Distributed != nil {
setting := strconv.Itoa(*cephFilesystemSubVolumeGroup.Spec.Pinning.Distributed)
args = append(args, "distributed", setting)
} else if cephFilesystemSubVolumeGroup.Spec.Pinning.Export != nil {
setting := strconv.Itoa(*cephFilesystemSubVolumeGroup.Spec.Pinning.Export)
args = append(args, "export", setting)
} else if cephFilesystemSubVolumeGroup.Spec.Pinning.Random != nil {
setting := strconv.FormatFloat(*cephFilesystemSubVolumeGroup.Spec.Pinning.Random, 'f', -1, 64)
args = append(args, "random", setting)
} else {
// set by default value
args = append(args, "distributed", "1")
}
logger.Infof("subvolume group pinning args %v", args)
cmd := NewCephCommand(context, clusterInfo, args)
cmd.JsonOutput = false
output, err := cmd.Run()
if err != nil {
return errors.Wrapf(err, "failed to pin subvolume group %q. %s", cephFilesystemSubVolumeGroupName, output)
}
logger.Infof("successfully pinned cephfs subvolume group %v", namespaceName)
return nil
}
func validatePinningValues(pinning cephv1.CephFilesystemSubVolumeGroupSpecPinning) error {
numNils := 0
var err error
if pinning.Export != nil {
numNils++
if *pinning.Export > 256 {
err = errors.Errorf("validate pinning type failed, Export: value too big %d", *pinning.Export)
} else if *pinning.Export < -1 {
err = errors.Errorf("validate pinning type failed, Export: negative value %d not allowed except -1", *pinning.Export)
}
}
if pinning.Distributed != nil {
numNils++
if !(*pinning.Distributed == 1) && !(*pinning.Distributed == 0) {
err = errors.Errorf("validate pinning type failed, Distributed: unknown value %d", *pinning.Distributed)
}
}
if pinning.Random != nil {
numNils++
if (*pinning.Random < 0) || (*pinning.Random > 1.0) {
err = errors.Errorf("validate pinning type failed, Random: value %.2f is not between 0.0 and 1.1 (inclusive)", *pinning.Random)
}
}
if numNils > 1 {
return fmt.Errorf("only one can be set")
}
if numNils == 0 {
return nil // pinning disabled
}
return err
}
@@ -0,0 +1,81 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/stretchr/testify/assert"
)
func TestValidatePinningValues(t *testing.T) {
// set Distributed correctly
var testDistributedData cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testDistributedValue = 1
testDistributedData.Distributed = &testDistributedValue
err := validatePinningValues(testDistributedData)
assert.NoError(t, err)
// set Distributed wrongly
var testDistributedData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testDistributedValue1 = 5
testDistributedData1.Distributed = &testDistributedValue1
err = validatePinningValues(testDistributedData1)
assert.Error(t, err)
// set Random correctly
var testRandomData cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testRandomdValue = 1.0
testRandomData.Random = &testRandomdValue
err = validatePinningValues(testRandomData)
assert.NoError(t, err)
// set Random wrongly
var testRandomData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testRandomdValue1 = 5.0
testRandomData1.Random = &testRandomdValue1
err = validatePinningValues(testRandomData1)
assert.Error(t, err)
// set export correctly
var testExportData cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testExportValue = 1
testExportData.Distributed = &testExportValue
err = validatePinningValues(testExportData)
assert.NoError(t, err)
// set export wrongly
var testExportData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testExportValue1 = 500
testExportData1.Distributed = &testExportValue1
err = validatePinningValues(testExportData1)
assert.Error(t, err)
// more than one set at a time, error
var testData cephv1.CephFilesystemSubVolumeGroupSpecPinning
var testValue = 1
testData.Distributed = &testValue
testData.Export = &testValue
err = validatePinningValues(testData)
assert.Error(t, err)
// nothing is set, noerror
var testData1 cephv1.CephFilesystemSubVolumeGroupSpecPinning
err = validatePinningValues(testData1)
assert.NoError(t, err)
}
@@ -255,6 +255,12 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to save cluster config")
}
err = cephclient.PinCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup, getSubvolumeGroupName(cephFilesystemSubVolumeGroup))
if err != nil {
return reconcile.Result{}, errors.Wrapf(err, "failed to pin filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
}
r.updateStatus(observedGeneration, request.NamespacedName, cephv1.ConditionReady)
// Return and do not requeue
logger.Debugf("done reconciling cephFilesystemSubVolumeGroup %q", namespacedName)
@@ -195,6 +195,8 @@ func TestCephClientController(t *testing.T) {
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "create" {
return "", nil
} else if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "pin" {
return "", nil
}
return "", errors.Errorf("unknown command. %v", args)