diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index 434585b5e..284b20f62 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -49,6 +49,9 @@ const ( // RgwType defines the rgw DaemonType RgwType = "rgw" + // NfsType defines the nfs DaemonType + NfsType = "nfs" + // RbdMirrorType defines the rbd-mirror DaemonType RbdMirrorType = "rbd-mirror" diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 0c0cc2025..127ac0326 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -34,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -572,3 +573,68 @@ func TestDaemonEnvVars(t *testing.T) { got = ApplyNetworkEnv(clusterSpec) assert.Equal(t, want, got) } + +func TestGetDaemonsToSkipReconcile(t *testing.T) { + tests := []struct { + name string + labels map[string]string + expectedSkip bool + expectedDaemonID string + }{ + { + name: "has skip-reconcile label", + labels: map[string]string{ + k8sutil.AppAttr: "rook-ceph-nfs", + cephv1.SkipReconcileLabelKey: "true", + config.NfsType: "a", + }, + expectedSkip: true, + expectedDaemonID: "a", + }, + { + name: "no skip-reconcile label", + labels: map[string]string{ + k8sutil.AppAttr: "rook-ceph-nfs", + config.NfsType: "b", + }, + expectedSkip: false, + expectedDaemonID: "b", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + namespace := "rook-ceph" + daemonName := "nfs" + appLabel := "rook-ceph-nfs" + + clientset := test.New(t, 1) + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%s", appLabel, daemonName, tt.expectedDaemonID), + Namespace: namespace, + Labels: tt.labels, + }, + } + + _, err := clientset.AppsV1().Deployments(namespace).Create(context.TODO(), dep, metav1.CreateOptions{}) + assert.NoError(t, err) + + clusterdCtx := &clusterd.Context{ + Clientset: clientset, + } + + result, err := GetDaemonsToSkipReconcile(context.TODO(), clusterdCtx, namespace, daemonName, appLabel) + assert.NoError(t, err) + + if tt.expectedSkip { + assert.Len(t, result, 1) + assert.True(t, result.Has(tt.expectedDaemonID), "expected daemon ID %q to be skipped", tt.expectedDaemonID) + } else { + assert.Len(t, result, 0) + assert.False(t, result.Has(tt.expectedDaemonID), "expected daemon ID %q NOT to be skipped", tt.expectedDaemonID) + } + }) + } +} diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index eea695862..725c93c67 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -27,6 +27,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" v1 "k8s.io/api/core/v1" @@ -54,9 +55,19 @@ type daemonConfig struct { // Create the ganesha server func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error { + nfsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(r.clusterInfo.Context, r.context, n.Namespace, config.NfsType, AppName) + if err != nil { + return errors.Wrap(err, "failed to check for NFS daemons to skip reconcile") + } + for i := 0; i < n.Spec.Server.Active; i++ { id := k8sutil.IndexToName(i) + if nfsToSkipReconcile.Has(id) { + logger.Warningf("skipping reconcile of nfs daemon %q with label %q", id, cephv1.SkipReconcileLabelKey) + continue + } + configName, configHash, err := r.createConfigMap(n, id) if err != nil { return errors.Wrap(err, "failed to create config") diff --git a/pkg/operator/ceph/nfs/nfs_test.go b/pkg/operator/ceph/nfs/nfs_test.go index be90a8ab9..1a05e6d0a 100644 --- a/pkg/operator/ceph/nfs/nfs_test.go +++ b/pkg/operator/ceph/nfs/nfs_test.go @@ -26,11 +26,16 @@ import ( "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/config" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -207,3 +212,118 @@ func TestReconcileCephNFS_upCephNFS(t *testing.T) { } assert.ElementsMatch(t, []string{"rook-ceph-nfs-my-nfs-a", "rook-ceph-nfs-my-nfs-b"}, names) } + +func TestUpCephNFS_SkipsReconcile(t *testing.T) { + ns := "skip-nfs-test" + s := scheme.Scheme + daemonID := "a" + + clientset := k8sfake.NewSimpleClientset() + client := fake.NewClientBuilder().WithScheme(s).Build() + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-nfs-my-nfs-a", + Namespace: ns, + Labels: map[string]string{ + k8sutil.AppAttr: AppName, + cephv1.SkipReconcileLabelKey: "true", + config.NfsType: daemonID, + }, + }, + } + _, err := clientset.AppsV1().Deployments(ns).Create(context.TODO(), dep, metav1.CreateOptions{}) + assert.NoError(t, err) + + executor := &exectest.MockExecutor{} + + r := &ReconcileCephNFS{ + scheme: s, + context: &clusterd.Context{ + Executor: executor, + Client: client, + Clientset: clientset, + }, + clusterInfo: &cephclient.ClusterInfo{ + FSID: "fsid-test", + CephVersion: cephver.Squid, + Context: context.TODO(), + Namespace: ns, + }, + cephClusterSpec: &cephv1.ClusterSpec{ + CephVersion: cephv1.CephVersionSpec{ + Image: "quay.io/ceph/ceph:v19", + }, + }, + } + + nfs := &cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs", + Namespace: ns, + }, + Spec: cephv1.NFSGaneshaSpec{ + Server: cephv1.GaneshaServerSpec{ + Active: 1, + }, + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "myfs-pool", + Namespace: "nfs-ns", + }, + }, + } + + err = r.upCephNFS(nfs) + assert.NoError(t, err) +} + +func TestUpCephNFS_SkipReconcileFails(t *testing.T) { + ns := "skip-nfs-test-fail" + s := scheme.Scheme + + // Create a clientset that always returns error for List + clientset := &k8sfake.Clientset{} + clientset.PrependReactor("list", "deployments", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("simulated list failure") + }) + + r := &ReconcileCephNFS{ + scheme: s, + context: &clusterd.Context{ + Executor: &exectest.MockExecutor{}, + Client: fake.NewClientBuilder().WithScheme(s).Build(), + Clientset: clientset, + }, + clusterInfo: &cephclient.ClusterInfo{ + FSID: "fsid-fail", + CephVersion: cephver.Squid, + Context: context.TODO(), + Namespace: ns, + }, + cephClusterSpec: &cephv1.ClusterSpec{ + CephVersion: cephv1.CephVersionSpec{ + Image: "quay.io/ceph/ceph:v19", + }, + }, + } + + nfs := &cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs-fail", + Namespace: ns, + }, + Spec: cephv1.NFSGaneshaSpec{ + Server: cephv1.GaneshaServerSpec{ + Active: 1, + }, + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "some-pool", + Namespace: "nfs-ns", + }, + }, + } + + err := r.upCephNFS(nfs) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to check for NFS daemons to skip reconcile") +}