From 49f75c8ec3252dc9af4ed07461bd0006e8d151c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 3 Feb 2022 15:33:05 +0100 Subject: [PATCH] subvolumegroup: handle external mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the cluster is external, users might still want to take advantage of the ceph-csi configmap configuration for ceph filesystem subvolume group. This configuration is handled by the controller whenever a cephFilesystemSubVolumeGroup CR is created or deleted. If the present cluster CephCluster is external, the controller (cephFilesystemSubVolumeGroup) won't handle creation nor deletion of sub volume groups. It just assumes that the given name is correct and the controller goes ahead with the ceph-csi configuration. Closes: #9551 Signed-off-by: Sébastien Han --- pkg/daemon/ceph/client/subvolumegroup.go | 13 +-- .../ceph/file/subvolumegroup/controller.go | 85 ++++++++++++------- .../file/subvolumegroup/controller_test.go | 54 +++++++++++- 3 files changed, 110 insertions(+), 42 deletions(-) diff --git a/pkg/daemon/ceph/client/subvolumegroup.go b/pkg/daemon/ceph/client/subvolumegroup.go index a73a4863e..f679b94ed 100644 --- a/pkg/daemon/ceph/client/subvolumegroup.go +++ b/pkg/daemon/ceph/client/subvolumegroup.go @@ -24,33 +24,34 @@ import ( // CreateCephFSSubVolumeGroup create a CephFS subvolume group. // volName is the name of the Ceph FS volume, the same as the CephFilesystem CR name. func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { - logger.Infof("creating cephfs sub volume group %q", volName) + logger.Infof("creating cephfs subvolume group %q", volName) // [--pool_layout ] [--uid ] [--gid ] [--mode ] args := []string{"fs", "subvolumegroup", "create", volName, groupName} cmd := NewCephCommand(context, clusterInfo, args) cmd.JsonOutput = false output, err := cmd.Run() if err != nil { - return errors.Wrapf(err, "failed to create sub volume group %q. %s", volName, output) + return errors.Wrapf(err, "failed to create subvolume group %q. %s", volName, output) } - logger.Infof("successfully created cephfs sub volume group %q", volName) + logger.Infof("successfully created cephfs subvolume group %q", volName) return nil } // DeleteCephFSSubVolumeGroup delete a CephFS subvolume group. func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { - logger.Infof("deleting cephfs sub volume group %q", volName) + logger.Infof("deleting cephfs subvolume group %q", volName) // --force? args := []string{"fs", "subvolumegroup", "rm", volName, groupName} cmd := NewCephCommand(context, clusterInfo, args) cmd.JsonOutput = false - _, err := cmd.Run() + output, err := cmd.Run() if err != nil { + logger.Debugf("failed to delete subvolume group %q. %s. %v", volName, output, err) // Intentionally don't wrap the error so the caller can inspect the return code return err } - logger.Infof("successfully deleted cephfs sub volume group %q", volName) + logger.Infof("successfully deleted cephfs subvolume group %q", volName) return nil } diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index 9c9af8fa6..f5f538ef2 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -144,7 +144,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteSubVolumeGroup() function since everything is gone already @@ -175,13 +175,18 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ // DELETE: the CR was deleted if !cephFilesystemSubVolumeGroup.GetDeletionTimestamp().IsZero() { logger.Debugf("deleting subvolume group %q", cephFilesystemSubVolumeGroup.Name) - err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + // On external cluster, we don't delete the subvolume group, it has to be deleted manually + if cephCluster.Spec.External.Enable { + logger.Warning("external subvolume group deletion is not supported, delete it manually") + } else { + err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } - return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), r.clusterInfo, nil) @@ -200,33 +205,43 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } // Build the NamespacedName to fetch the Filesystem and make sure it exists, if not we cannot - // create the subvolumegroup - cephFilesystem := &cephv1.CephFilesystem{} - cephFilesystemNamespacedName := types.NamespacedName{Name: cephFilesystemSubVolumeGroup.Spec.FilesystemName, Namespace: request.Namespace} - err = r.client.Get(r.opManagerContext, cephFilesystemNamespacedName, cephFilesystem) - if err != nil { - if kerrors.IsNotFound(err) { - return reconcile.Result{}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolumegroup %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + // create the subvolume group + // On external mode the filesystem is created externally, so we don't need to check for CRD and + // assume it's there + if !cephCluster.Spec.External.Enable { + cephFilesystem := &cephv1.CephFilesystem{} + cephFilesystemNamespacedName := types.NamespacedName{Name: cephFilesystemSubVolumeGroup.Spec.FilesystemName, Namespace: request.Namespace} + err = r.client.Get(r.opManagerContext, cephFilesystemNamespacedName, cephFilesystem) + if err != nil { + if kerrors.IsNotFound(err) { + return reconcile.Result{}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolume group %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + } + // Error reading the object - requeue the request. + return reconcile.Result{}, errors.Wrap(err, "failed to get cephFilesystemSubVolumeGroup") } - // Error reading the object - requeue the request. - return reconcile.Result{}, errors.Wrap(err, "failed to get cephFilesystemSubVolumeGroup") - } - // If the CephFilesystem is not ready to accept commands, we should wait for it to be ready - if cephFilesystem.Status.Phase != cephv1.ConditionReady { - // We know the CR is present so it should a matter of second for it to become ready - return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolumegroup %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + // If the CephFilesystem is not ready to accept commands, we should wait for it to be ready + if cephFilesystem.Status.Phase != cephv1.ConditionReady { + // We know the CR is present so it should a matter of second for it to become ready + return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolume group %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + } } // Create or Update ceph filesystem subvolume group - err = r.createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + // On external mode the subvolume group is created externally, so we don't need to try to create + // it and assume it's there already + if cephCluster.Spec.External.Enable { + logger.Debug("external subvolume group creation is not supported, create it manually, the controller will assume it's there") + } else { + err = r.createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) + return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } - r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) - return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } // Update CSI config map @@ -244,7 +259,11 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } // Success! Let's update the status - r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady) + if cephCluster.Spec.External.Enable { + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionConnected) + } else { + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady) + } // Return and do not requeue logger.Debug("done reconciling") @@ -268,12 +287,12 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) deleteSubVolumeGroup(cephFilesys logger.Infof("deleting ceph filesystem subvolume group object %q", cephFilesystemSubVolumeGroup.Name) if err := cephclient.DeleteCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name); err != nil { code, ok := exec.ExitStatus(err) - // If the subvolumegroup does not exit, we should not return an error + // If the subvolume group does not exit, we should not return an error if ok && code == int(syscall.ENOENT) { logger.Debugf("ceph filesystem subvolume group %q do not exist", cephFilesystemSubVolumeGroup.Name) return nil } - // If the subvolumegroup has subvolumes the command will fail with: + // If the subvolume group has subvolumes the command will fail with: // Error ENOTEMPTY: error in rmdir /volumes/csi if ok && (code == int(syscall.ENOTEMPTY)) { return errors.Wrapf(err, "failed to delete ceph filesystem subvolume group %q, remove the subvolumes first", cephFilesystemSubVolumeGroup.Name) @@ -294,7 +313,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien logger.Debug("CephFilesystemSubVolumeGroup resource not found. Ignoring since object must be deleted.") return } - logger.Warningf("failed to retrieve ceph ceph filesystem subvolume group %q to update status to %q. %v", name, status, err) + logger.Warningf("failed to retrieve ceph filesystem subvolume group %q to update status to %q. %v", name, status, err) return } if cephFilesystemSubVolumeGroup.Status == nil { @@ -304,7 +323,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien cephFilesystemSubVolumeGroup.Status.Phase = status cephFilesystemSubVolumeGroup.Status.Info = map[string]string{"clusterID": buildClusterID(cephFilesystemSubVolumeGroup)} if err := reporting.UpdateStatus(client, cephFilesystemSubVolumeGroup); err != nil { - logger.Errorf("failed to set ceph ceph filesystem subvolume group %q status to %q. %v", name, status, err) + logger.Errorf("failed to set ceph filesystem subvolume group %q status to %q. %v", name, status, err) return } logger.Debugf("ceph ceph filesystem subvolume group %q status updated to %q", name, status) diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go index 1d0e5a555..fca3bff14 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go @@ -21,6 +21,8 @@ import ( "os" "testing" + "github.com/pkg/errors" + "github.com/coreos/pkg/capnslog" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" @@ -45,7 +47,6 @@ func TestCephClientController(t *testing.T) { capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - logger.Info("RUN 1") var ( name = "group-a" namespace = "rook-ceph" @@ -192,11 +193,11 @@ func TestCephClientController(t *testing.T) { executor = &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "auth" && args[1] == "fs" && args[2] == "subvolumegroup" && args[3] == "create" { + if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "create" { return "", nil } - return "", nil + return "", errors.Errorf("unknown command. %v", args) }, } c.Executor = executor @@ -228,6 +229,53 @@ func TestCephClientController(t *testing.T) { assert.Equal(t, cephv1.ConditionReady, cephFilesystemSubVolumeGroup.Status.Phase) assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"]) + // test that csi configmap is created + cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotEmpty(t, cm.Data[csi.ConfigKey]) + assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") + assert.Contains(t, cm.Data[csi.ConfigKey], "group-a") + err = c.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, csi.ConfigName, metav1.DeleteOptions{}) + assert.NoError(t, err) + }) + + t.Run("success - external mode csi config is updated", func(t *testing.T) { + cephCluster.Spec.External.Enable = true + objects := []runtime.Object{ + cephFilesystemSubVolumeGroup, + cephCluster, + } + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() + c.Client = cl + + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + // Create a ReconcileCephFilesystemSubVolumeGroup object with the scheme and fake client. + r = &ReconcileCephFilesystemSubVolumeGroup{ + client: cl, + scheme: s, + context: c, + opManagerContext: ctx, + } + + // Enable CSI + csi.EnableRBD = true + os.Setenv("POD_NAMESPACE", namespace) + // Create CSI config map + ownerRef := &metav1.OwnerReference{} + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, "") + err := csi.CreateCsiConfigMap(namespace, c.Clientset, ownerInfo) + assert.NoError(t, err) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + + err = r.client.Get(ctx, req.NamespacedName, cephFilesystemSubVolumeGroup) + assert.NoError(t, err) + assert.Equal(t, cephv1.ConditionConnected, cephFilesystemSubVolumeGroup.Status.Phase) + assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"]) + // test that csi configmap is created cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) assert.NoError(t, err)