subvolumegroup: handle external mode

If the cluster is external, users might still want to take advantage of
the ceph-csi configmap configuration for ceph filesystem subvolume
group. This configuration is handled by the controller whenever a
cephFilesystemSubVolumeGroup CR is created or deleted.
If the present cluster CephCluster is external, the controller
(cephFilesystemSubVolumeGroup) won't handle creation nor deletion of sub volume groups.
It just assumes that the given name is correct and the controller goes ahead with the
ceph-csi configuration.

Closes: #9551
Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2022-02-03 16:28:07 +01:00
1 parent d24eab2229
commit 49f75c8ec3
3 files changed
+110 -42

No files matched your search

+7 -6
View File
@@ -24,33 +24,34 @@ import (
// CreateCephFSSubVolumeGroup create a CephFS subvolume group.
// volName is the name of the Ceph FS volume, the same as the CephFilesystem CR name.
func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error {
logger.Infof("creating cephfs sub volume group %q", volName)
logger.Infof("creating cephfs subvolume group %q", volName)
// [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]
args := []string{"fs", "subvolumegroup", "create", volName, groupName}
cmd := NewCephCommand(context, clusterInfo, args)
cmd.JsonOutput = false
output, err := cmd.Run()
if err != nil {
return errors.Wrapf(err, "failed to create sub volume group %q. %s", volName, output)
return errors.Wrapf(err, "failed to create subvolume group %q. %s", volName, output)
}
logger.Infof("successfully created cephfs sub volume group %q", volName)
logger.Infof("successfully created cephfs subvolume group %q", volName)
return nil
}
// DeleteCephFSSubVolumeGroup delete a CephFS subvolume group.
func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error {
logger.Infof("deleting cephfs sub volume group %q", volName)
logger.Infof("deleting cephfs subvolume group %q", volName)
// --force?
args := []string{"fs", "subvolumegroup", "rm", volName, groupName}
cmd := NewCephCommand(context, clusterInfo, args)
cmd.JsonOutput = false
_, err := cmd.Run()
output, err := cmd.Run()
if err != nil {
logger.Debugf("failed to delete subvolume group %q. %s. %v", volName, output, err)
// Intentionally don't wrap the error so the caller can inspect the return code
return err
}
logger.Infof("successfully deleted cephfs sub volume group %q", volName)
logger.Infof("successfully deleted cephfs subvolume group %q", volName)
return nil
}
@@ -144,7 +144,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
}
// Make sure a CephCluster is present otherwise do nothing
_, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
if !isReadyToReconcile {
// This handles the case where the Ceph Cluster is gone and we want to delete that CR
// We skip the deleteSubVolumeGroup() function since everything is gone already
@@ -175,13 +175,18 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
// DELETE: the CR was deleted
if !cephFilesystemSubVolumeGroup.GetDeletionTimestamp().IsZero() {
logger.Debugf("deleting subvolume group %q", cephFilesystemSubVolumeGroup.Name)
err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
// On external cluster, we don't delete the subvolume group, it has to be deleted manually
if cephCluster.Spec.External.Enable {
logger.Warning("external subvolume group deletion is not supported, delete it manually")
} else {
err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
}
return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
}
err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), r.clusterInfo, nil)
@@ -200,33 +205,43 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
}
// Build the NamespacedName to fetch the Filesystem and make sure it exists, if not we cannot
// create the subvolumegroup
cephFilesystem := &cephv1.CephFilesystem{}
cephFilesystemNamespacedName := types.NamespacedName{Name: cephFilesystemSubVolumeGroup.Spec.FilesystemName, Namespace: request.Namespace}
err = r.client.Get(r.opManagerContext, cephFilesystemNamespacedName, cephFilesystem)
if err != nil {
if kerrors.IsNotFound(err) {
return reconcile.Result{}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolumegroup %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name)
// create the subvolume group
// On external mode the filesystem is created externally, so we don't need to check for CRD and
// assume it's there
if !cephCluster.Spec.External.Enable {
cephFilesystem := &cephv1.CephFilesystem{}
cephFilesystemNamespacedName := types.NamespacedName{Name: cephFilesystemSubVolumeGroup.Spec.FilesystemName, Namespace: request.Namespace}
err = r.client.Get(r.opManagerContext, cephFilesystemNamespacedName, cephFilesystem)
if err != nil {
if kerrors.IsNotFound(err) {
return reconcile.Result{}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolume group %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name)
}
// Error reading the object - requeue the request.
return reconcile.Result{}, errors.Wrap(err, "failed to get cephFilesystemSubVolumeGroup")
}
// Error reading the object - requeue the request.
return reconcile.Result{}, errors.Wrap(err, "failed to get cephFilesystemSubVolumeGroup")
}
// If the CephFilesystem is not ready to accept commands, we should wait for it to be ready
if cephFilesystem.Status.Phase != cephv1.ConditionReady {
// We know the CR is present so it should a matter of second for it to become ready
return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolumegroup %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name)
// If the CephFilesystem is not ready to accept commands, we should wait for it to be ready
if cephFilesystem.Status.Phase != cephv1.ConditionReady {
// We know the CR is present so it should a matter of second for it to become ready
return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolume group %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name)
}
}
// Create or Update ceph filesystem subvolume group
err = r.createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
// On external mode the subvolume group is created externally, so we don't need to try to create
// it and assume it's there already
if cephCluster.Spec.External.Enable {
logger.Debug("external subvolume group creation is not supported, create it manually, the controller will assume it's there")
} else {
err = r.createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup)
if err != nil {
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
logger.Info(opcontroller.OperatorNotInitializedMessage)
return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
}
r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure)
return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
}
r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure)
return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name)
}
// Update CSI config map
@@ -244,7 +259,11 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
}
// Success! Let's update the status
r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady)
if cephCluster.Spec.External.Enable {
r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionConnected)
} else {
r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady)
}
// Return and do not requeue
logger.Debug("done reconciling")
@@ -268,12 +287,12 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) deleteSubVolumeGroup(cephFilesys
logger.Infof("deleting ceph filesystem subvolume group object %q", cephFilesystemSubVolumeGroup.Name)
if err := cephclient.DeleteCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name); err != nil {
code, ok := exec.ExitStatus(err)
// If the subvolumegroup does not exit, we should not return an error
// If the subvolume group does not exit, we should not return an error
if ok && code == int(syscall.ENOENT) {
logger.Debugf("ceph filesystem subvolume group %q do not exist", cephFilesystemSubVolumeGroup.Name)
return nil
}
// If the subvolumegroup has subvolumes the command will fail with:
// If the subvolume group has subvolumes the command will fail with:
// Error ENOTEMPTY: error in rmdir /volumes/csi
if ok && (code == int(syscall.ENOTEMPTY)) {
return errors.Wrapf(err, "failed to delete ceph filesystem subvolume group %q, remove the subvolumes first", cephFilesystemSubVolumeGroup.Name)
@@ -294,7 +313,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien
logger.Debug("CephFilesystemSubVolumeGroup resource not found. Ignoring since object must be deleted.")
return
}
logger.Warningf("failed to retrieve ceph ceph filesystem subvolume group %q to update status to %q. %v", name, status, err)
logger.Warningf("failed to retrieve ceph filesystem subvolume group %q to update status to %q. %v", name, status, err)
return
}
if cephFilesystemSubVolumeGroup.Status == nil {
@@ -304,7 +323,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien
cephFilesystemSubVolumeGroup.Status.Phase = status
cephFilesystemSubVolumeGroup.Status.Info = map[string]string{"clusterID": buildClusterID(cephFilesystemSubVolumeGroup)}
if err := reporting.UpdateStatus(client, cephFilesystemSubVolumeGroup); err != nil {
logger.Errorf("failed to set ceph ceph filesystem subvolume group %q status to %q. %v", name, status, err)
logger.Errorf("failed to set ceph filesystem subvolume group %q status to %q. %v", name, status, err)
return
}
logger.Debugf("ceph ceph filesystem subvolume group %q status updated to %q", name, status)
@@ -21,6 +21,8 @@ import (
"os"
"testing"
"github.com/pkg/errors"
"github.com/coreos/pkg/capnslog"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
@@ -45,7 +47,6 @@ func TestCephClientController(t *testing.T) {
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
logger.Info("RUN 1")
var (
name = "group-a"
namespace = "rook-ceph"
@@ -192,11 +193,11 @@ func TestCephClientController(t *testing.T) {
executor = &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[0] == "auth" && args[1] == "fs" && args[2] == "subvolumegroup" && args[3] == "create" {
if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "create" {
return "", nil
}
return "", nil
return "", errors.Errorf("unknown command. %v", args)
},
}
c.Executor = executor
@@ -228,6 +229,53 @@ func TestCephClientController(t *testing.T) {
assert.Equal(t, cephv1.ConditionReady, cephFilesystemSubVolumeGroup.Status.Phase)
assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"])
// test that csi configmap is created
cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{})
assert.NoError(t, err)
assert.NotEmpty(t, cm.Data[csi.ConfigKey])
assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID")
assert.Contains(t, cm.Data[csi.ConfigKey], "group-a")
err = c.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, csi.ConfigName, metav1.DeleteOptions{})
assert.NoError(t, err)
})
t.Run("success - external mode csi config is updated", func(t *testing.T) {
cephCluster.Spec.External.Enable = true
objects := []runtime.Object{
cephFilesystemSubVolumeGroup,
cephCluster,
}
// Create a fake client to mock API calls.
cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
c.Client = cl
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{})
// Create a ReconcileCephFilesystemSubVolumeGroup object with the scheme and fake client.
r = &ReconcileCephFilesystemSubVolumeGroup{
client: cl,
scheme: s,
context: c,
opManagerContext: ctx,
}
// Enable CSI
csi.EnableRBD = true
os.Setenv("POD_NAMESPACE", namespace)
// Create CSI config map
ownerRef := &metav1.OwnerReference{}
ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, "")
err := csi.CreateCsiConfigMap(namespace, c.Clientset, ownerInfo)
assert.NoError(t, err)
res, err := r.Reconcile(ctx, req)
assert.NoError(t, err)
assert.False(t, res.Requeue)
err = r.client.Get(ctx, req.NamespacedName, cephFilesystemSubVolumeGroup)
assert.NoError(t, err)
assert.Equal(t, cephv1.ConditionConnected, cephFilesystemSubVolumeGroup.Status.Phase)
assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"])
// test that csi configmap is created
cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{})
assert.NoError(t, err)