diff --git a/pkg/daemon/ceph/client/subvolumegroup.go b/pkg/daemon/ceph/client/subvolumegroup.go index a73a4863e..f679b94ed 100644 --- a/pkg/daemon/ceph/client/subvolumegroup.go +++ b/pkg/daemon/ceph/client/subvolumegroup.go @@ -24,33 +24,34 @@ import ( // CreateCephFSSubVolumeGroup create a CephFS subvolume group. // volName is the name of the Ceph FS volume, the same as the CephFilesystem CR name. func CreateCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { - logger.Infof("creating cephfs sub volume group %q", volName) + logger.Infof("creating cephfs subvolume group %q", volName) // [--pool_layout ] [--uid ] [--gid ] [--mode ] args := []string{"fs", "subvolumegroup", "create", volName, groupName} cmd := NewCephCommand(context, clusterInfo, args) cmd.JsonOutput = false output, err := cmd.Run() if err != nil { - return errors.Wrapf(err, "failed to create sub volume group %q. %s", volName, output) + return errors.Wrapf(err, "failed to create subvolume group %q. %s", volName, output) } - logger.Infof("successfully created cephfs sub volume group %q", volName) + logger.Infof("successfully created cephfs subvolume group %q", volName) return nil } // DeleteCephFSSubVolumeGroup delete a CephFS subvolume group. func DeleteCephFSSubVolumeGroup(context *clusterd.Context, clusterInfo *ClusterInfo, volName, groupName string) error { - logger.Infof("deleting cephfs sub volume group %q", volName) + logger.Infof("deleting cephfs subvolume group %q", volName) // --force? args := []string{"fs", "subvolumegroup", "rm", volName, groupName} cmd := NewCephCommand(context, clusterInfo, args) cmd.JsonOutput = false - _, err := cmd.Run() + output, err := cmd.Run() if err != nil { + logger.Debugf("failed to delete subvolume group %q. %s. %v", volName, output, err) // Intentionally don't wrap the error so the caller can inspect the return code return err } - logger.Infof("successfully deleted cephfs sub volume group %q", volName) + logger.Infof("successfully deleted cephfs subvolume group %q", volName) return nil } diff --git a/pkg/operator/ceph/file/subvolumegroup/controller.go b/pkg/operator/ceph/file/subvolumegroup/controller.go index 9c9af8fa6..f5f538ef2 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller.go @@ -144,7 +144,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteSubVolumeGroup() function since everything is gone already @@ -175,13 +175,18 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ // DELETE: the CR was deleted if !cephFilesystemSubVolumeGroup.GetDeletionTimestamp().IsZero() { logger.Debugf("deleting subvolume group %q", cephFilesystemSubVolumeGroup.Name) - err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + // On external cluster, we don't delete the subvolume group, it has to be deleted manually + if cephCluster.Spec.External.Enable { + logger.Warning("external subvolume group deletion is not supported, delete it manually") + } else { + err := r.deleteSubVolumeGroup(cephFilesystemSubVolumeGroup) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } - return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephFilesystemSubVolumeGroup), r.clusterInfo, nil) @@ -200,33 +205,43 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } // Build the NamespacedName to fetch the Filesystem and make sure it exists, if not we cannot - // create the subvolumegroup - cephFilesystem := &cephv1.CephFilesystem{} - cephFilesystemNamespacedName := types.NamespacedName{Name: cephFilesystemSubVolumeGroup.Spec.FilesystemName, Namespace: request.Namespace} - err = r.client.Get(r.opManagerContext, cephFilesystemNamespacedName, cephFilesystem) - if err != nil { - if kerrors.IsNotFound(err) { - return reconcile.Result{}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolumegroup %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + // create the subvolume group + // On external mode the filesystem is created externally, so we don't need to check for CRD and + // assume it's there + if !cephCluster.Spec.External.Enable { + cephFilesystem := &cephv1.CephFilesystem{} + cephFilesystemNamespacedName := types.NamespacedName{Name: cephFilesystemSubVolumeGroup.Spec.FilesystemName, Namespace: request.Namespace} + err = r.client.Get(r.opManagerContext, cephFilesystemNamespacedName, cephFilesystem) + if err != nil { + if kerrors.IsNotFound(err) { + return reconcile.Result{}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolume group %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + } + // Error reading the object - requeue the request. + return reconcile.Result{}, errors.Wrap(err, "failed to get cephFilesystemSubVolumeGroup") } - // Error reading the object - requeue the request. - return reconcile.Result{}, errors.Wrap(err, "failed to get cephFilesystemSubVolumeGroup") - } - // If the CephFilesystem is not ready to accept commands, we should wait for it to be ready - if cephFilesystem.Status.Phase != cephv1.ConditionReady { - // We know the CR is present so it should a matter of second for it to become ready - return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolumegroup %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + // If the CephFilesystem is not ready to accept commands, we should wait for it to be ready + if cephFilesystem.Status.Phase != cephv1.ConditionReady { + // We know the CR is present so it should a matter of second for it to become ready + return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, errors.Wrapf(err, "failed to fetch ceph filesystem %q, cannot create subvolume group %q", cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name) + } } // Create or Update ceph filesystem subvolume group - err = r.createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + // On external mode the subvolume group is created externally, so we don't need to try to create + // it and assume it's there already + if cephCluster.Spec.External.Enable { + logger.Debug("external subvolume group creation is not supported, create it manually, the controller will assume it's there") + } else { + err = r.createOrUpdateSubVolumeGroup(cephFilesystemSubVolumeGroup) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) + return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } - r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) - return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph filesystem subvolume group %q", cephFilesystemSubVolumeGroup.Name) } // Update CSI config map @@ -244,7 +259,11 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ } // Success! Let's update the status - r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady) + if cephCluster.Spec.External.Enable { + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionConnected) + } else { + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady) + } // Return and do not requeue logger.Debug("done reconciling") @@ -268,12 +287,12 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) deleteSubVolumeGroup(cephFilesys logger.Infof("deleting ceph filesystem subvolume group object %q", cephFilesystemSubVolumeGroup.Name) if err := cephclient.DeleteCephFSSubVolumeGroup(r.context, r.clusterInfo, cephFilesystemSubVolumeGroup.Spec.FilesystemName, cephFilesystemSubVolumeGroup.Name); err != nil { code, ok := exec.ExitStatus(err) - // If the subvolumegroup does not exit, we should not return an error + // If the subvolume group does not exit, we should not return an error if ok && code == int(syscall.ENOENT) { logger.Debugf("ceph filesystem subvolume group %q do not exist", cephFilesystemSubVolumeGroup.Name) return nil } - // If the subvolumegroup has subvolumes the command will fail with: + // If the subvolume group has subvolumes the command will fail with: // Error ENOTEMPTY: error in rmdir /volumes/csi if ok && (code == int(syscall.ENOTEMPTY)) { return errors.Wrapf(err, "failed to delete ceph filesystem subvolume group %q, remove the subvolumes first", cephFilesystemSubVolumeGroup.Name) @@ -294,7 +313,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien logger.Debug("CephFilesystemSubVolumeGroup resource not found. Ignoring since object must be deleted.") return } - logger.Warningf("failed to retrieve ceph ceph filesystem subvolume group %q to update status to %q. %v", name, status, err) + logger.Warningf("failed to retrieve ceph filesystem subvolume group %q to update status to %q. %v", name, status, err) return } if cephFilesystemSubVolumeGroup.Status == nil { @@ -304,7 +323,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateStatus(client client.Clien cephFilesystemSubVolumeGroup.Status.Phase = status cephFilesystemSubVolumeGroup.Status.Info = map[string]string{"clusterID": buildClusterID(cephFilesystemSubVolumeGroup)} if err := reporting.UpdateStatus(client, cephFilesystemSubVolumeGroup); err != nil { - logger.Errorf("failed to set ceph ceph filesystem subvolume group %q status to %q. %v", name, status, err) + logger.Errorf("failed to set ceph filesystem subvolume group %q status to %q. %v", name, status, err) return } logger.Debugf("ceph ceph filesystem subvolume group %q status updated to %q", name, status) diff --git a/pkg/operator/ceph/file/subvolumegroup/controller_test.go b/pkg/operator/ceph/file/subvolumegroup/controller_test.go index 1d0e5a555..fca3bff14 100644 --- a/pkg/operator/ceph/file/subvolumegroup/controller_test.go +++ b/pkg/operator/ceph/file/subvolumegroup/controller_test.go @@ -21,6 +21,8 @@ import ( "os" "testing" + "github.com/pkg/errors" + "github.com/coreos/pkg/capnslog" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" @@ -45,7 +47,6 @@ func TestCephClientController(t *testing.T) { capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - logger.Info("RUN 1") var ( name = "group-a" namespace = "rook-ceph" @@ -192,11 +193,11 @@ func TestCephClientController(t *testing.T) { executor = &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "auth" && args[1] == "fs" && args[2] == "subvolumegroup" && args[3] == "create" { + if args[0] == "fs" && args[1] == "subvolumegroup" && args[2] == "create" { return "", nil } - return "", nil + return "", errors.Errorf("unknown command. %v", args) }, } c.Executor = executor @@ -228,6 +229,53 @@ func TestCephClientController(t *testing.T) { assert.Equal(t, cephv1.ConditionReady, cephFilesystemSubVolumeGroup.Status.Phase) assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"]) + // test that csi configmap is created + cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotEmpty(t, cm.Data[csi.ConfigKey]) + assert.Contains(t, cm.Data[csi.ConfigKey], "clusterID") + assert.Contains(t, cm.Data[csi.ConfigKey], "group-a") + err = c.Clientset.CoreV1().ConfigMaps(namespace).Delete(ctx, csi.ConfigName, metav1.DeleteOptions{}) + assert.NoError(t, err) + }) + + t.Run("success - external mode csi config is updated", func(t *testing.T) { + cephCluster.Spec.External.Enable = true + objects := []runtime.Object{ + cephFilesystemSubVolumeGroup, + cephCluster, + } + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() + c.Client = cl + + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + // Create a ReconcileCephFilesystemSubVolumeGroup object with the scheme and fake client. + r = &ReconcileCephFilesystemSubVolumeGroup{ + client: cl, + scheme: s, + context: c, + opManagerContext: ctx, + } + + // Enable CSI + csi.EnableRBD = true + os.Setenv("POD_NAMESPACE", namespace) + // Create CSI config map + ownerRef := &metav1.OwnerReference{} + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, "") + err := csi.CreateCsiConfigMap(namespace, c.Clientset, ownerInfo) + assert.NoError(t, err) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + + err = r.client.Get(ctx, req.NamespacedName, cephFilesystemSubVolumeGroup) + assert.NoError(t, err) + assert.Equal(t, cephv1.ConditionConnected, cephFilesystemSubVolumeGroup.Status.Phase) + assert.NotEmpty(t, cephFilesystemSubVolumeGroup.Status.Info["clusterID"]) + // test that csi configmap is created cm, err := c.Clientset.CoreV1().ConfigMaps(namespace).Get(ctx, csi.ConfigName, metav1.GetOptions{}) assert.NoError(t, err)