nfs: skip NFS daemon reconciliation when labeled with skip-reconcile

This change adds support for skipping reconciliation of CephNFS daemons
that are labeled with `ceph.rook.io/skip-reconcile=true`.
Similar to MDS, MGR, and RGW components, this allows cluster operators
to prevent Rook from modifying specific NFS daemon deployments.

Signed-off-by: Patryk Rostkowski <patrostkowski@gmail.com>
(cherry picked from commit 612e54e1ae)

pkg/operator/ceph/config/config.go

@@ -49,6 +49,9 @@ const (
 	// RgwType defines the rgw DaemonType
 	RgwType = "rgw"
 
+	// NfsType defines the nfs DaemonType
+	NfsType = "nfs"
+
 	// RbdMirrorType defines the rbd-mirror DaemonType
 	RbdMirrorType = "rbd-mirror"
 

pkg/operator/ceph/controller/spec_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -572,3 +573,68 @@ func TestDaemonEnvVars(t *testing.T) {
 	got = ApplyNetworkEnv(clusterSpec)
 	assert.Equal(t, want, got)
 }
+
+func TestGetDaemonsToSkipReconcile(t *testing.T) {
+	tests := []struct {
+		name             string
+		labels           map[string]string
+		expectedSkip     bool
+		expectedDaemonID string
+	}{
+		{
+			name: "has skip-reconcile label",
+			labels: map[string]string{
+				k8sutil.AppAttr:              "rook-ceph-nfs",
+				cephv1.SkipReconcileLabelKey: "true",
+				config.NfsType:               "a",
+			},
+			expectedSkip:     true,
+			expectedDaemonID: "a",
+		},
+		{
+			name: "no skip-reconcile label",
+			labels: map[string]string{
+				k8sutil.AppAttr: "rook-ceph-nfs",
+				config.NfsType:  "b",
+			},
+			expectedSkip:     false,
+			expectedDaemonID: "b",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			namespace := "rook-ceph"
+			daemonName := "nfs"
+			appLabel := "rook-ceph-nfs"
+
+			clientset := test.New(t, 1)
+
+			dep := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-%s-%s", appLabel, daemonName, tt.expectedDaemonID),
+					Namespace: namespace,
+					Labels:    tt.labels,
+				},
+			}
+
+			_, err := clientset.AppsV1().Deployments(namespace).Create(context.TODO(), dep, metav1.CreateOptions{})
+			assert.NoError(t, err)
+
+			clusterdCtx := &clusterd.Context{
+				Clientset: clientset,
+			}
+
+			result, err := GetDaemonsToSkipReconcile(context.TODO(), clusterdCtx, namespace, daemonName, appLabel)
+			assert.NoError(t, err)
+
+			if tt.expectedSkip {
+				assert.Len(t, result, 1)
+				assert.True(t, result.Has(tt.expectedDaemonID), "expected daemon ID %q to be skipped", tt.expectedDaemonID)
+			} else {
+				assert.Len(t, result, 0)
+				assert.False(t, result.Has(tt.expectedDaemonID), "expected daemon ID %q NOT to be skipped", tt.expectedDaemonID)
+			}
+		})
+	}
+}

pkg/operator/ceph/nfs/nfs.go

@@ -27,6 +27,7 @@ import (
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/config"
+	"github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/exec"
 	v1 "k8s.io/api/core/v1"
@@ -54,9 +55,19 @@ type daemonConfig struct {
 
 // Create the ganesha server
 func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error {
+	nfsToSkipReconcile, err := controller.GetDaemonsToSkipReconcile(r.clusterInfo.Context, r.context, n.Namespace, config.NfsType, AppName)
+	if err != nil {
+		return errors.Wrap(err, "failed to check for NFS daemons to skip reconcile")
+	}
+
 	for i := 0; i < n.Spec.Server.Active; i++ {
 		id := k8sutil.IndexToName(i)
 
+		if nfsToSkipReconcile.Has(id) {
+			logger.Warningf("skipping reconcile of nfs daemon %q with label %q", id, cephv1.SkipReconcileLabelKey)
+			continue
+		}
+
 		configName, configHash, err := r.createConfigMap(n, id)
 		if err != nil {
 			return errors.Wrap(err, "failed to create config")

pkg/operator/ceph/nfs/nfs_test.go

@@ -26,11 +26,16 @@ import (
 	"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
 	"github.com/rook/rook/pkg/clusterd"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/rook/rook/pkg/operator/ceph/config"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
+	"github.com/rook/rook/pkg/operator/k8sutil"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	k8sfake "k8s.io/client-go/kubernetes/fake"
+	k8stesting "k8s.io/client-go/testing"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
@@ -207,3 +212,118 @@ func TestReconcileCephNFS_upCephNFS(t *testing.T) {
 	}
 	assert.ElementsMatch(t, []string{"rook-ceph-nfs-my-nfs-a", "rook-ceph-nfs-my-nfs-b"}, names)
 }
+
+func TestUpCephNFS_SkipsReconcile(t *testing.T) {
+	ns := "skip-nfs-test"
+	s := scheme.Scheme
+	daemonID := "a"
+
+	clientset := k8sfake.NewSimpleClientset()
+	client := fake.NewClientBuilder().WithScheme(s).Build()
+
+	dep := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "rook-ceph-nfs-my-nfs-a",
+			Namespace: ns,
+			Labels: map[string]string{
+				k8sutil.AppAttr:              AppName,
+				cephv1.SkipReconcileLabelKey: "true",
+				config.NfsType:               daemonID,
+			},
+		},
+	}
+	_, err := clientset.AppsV1().Deployments(ns).Create(context.TODO(), dep, metav1.CreateOptions{})
+	assert.NoError(t, err)
+
+	executor := &exectest.MockExecutor{}
+
+	r := &ReconcileCephNFS{
+		scheme: s,
+		context: &clusterd.Context{
+			Executor:  executor,
+			Client:    client,
+			Clientset: clientset,
+		},
+		clusterInfo: &cephclient.ClusterInfo{
+			FSID:        "fsid-test",
+			CephVersion: cephver.Squid,
+			Context:     context.TODO(),
+			Namespace:   ns,
+		},
+		cephClusterSpec: &cephv1.ClusterSpec{
+			CephVersion: cephv1.CephVersionSpec{
+				Image: "quay.io/ceph/ceph:v19",
+			},
+		},
+	}
+
+	nfs := &cephv1.CephNFS{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-nfs",
+			Namespace: ns,
+		},
+		Spec: cephv1.NFSGaneshaSpec{
+			Server: cephv1.GaneshaServerSpec{
+				Active: 1,
+			},
+			RADOS: cephv1.GaneshaRADOSSpec{
+				Pool:      "myfs-pool",
+				Namespace: "nfs-ns",
+			},
+		},
+	}
+
+	err = r.upCephNFS(nfs)
+	assert.NoError(t, err)
+}
+
+func TestUpCephNFS_SkipReconcileFails(t *testing.T) {
+	ns := "skip-nfs-test-fail"
+	s := scheme.Scheme
+
+	// Create a clientset that always returns error for List
+	clientset := &k8sfake.Clientset{}
+	clientset.PrependReactor("list", "deployments", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+		return true, nil, errors.New("simulated list failure")
+	})
+
+	r := &ReconcileCephNFS{
+		scheme: s,
+		context: &clusterd.Context{
+			Executor:  &exectest.MockExecutor{},
+			Client:    fake.NewClientBuilder().WithScheme(s).Build(),
+			Clientset: clientset,
+		},
+		clusterInfo: &cephclient.ClusterInfo{
+			FSID:        "fsid-fail",
+			CephVersion: cephver.Squid,
+			Context:     context.TODO(),
+			Namespace:   ns,
+		},
+		cephClusterSpec: &cephv1.ClusterSpec{
+			CephVersion: cephv1.CephVersionSpec{
+				Image: "quay.io/ceph/ceph:v19",
+			},
+		},
+	}
+
+	nfs := &cephv1.CephNFS{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-nfs-fail",
+			Namespace: ns,
+		},
+		Spec: cephv1.NFSGaneshaSpec{
+			Server: cephv1.GaneshaServerSpec{
+				Active: 1,
+			},
+			RADOS: cephv1.GaneshaRADOSSpec{
+				Pool:      "some-pool",
+				Namespace: "nfs-ns",
+			},
+		},
+	}
+
+	err := r.upCephNFS(nfs)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to check for NFS daemons to skip reconcile")
+}