Merge pull request #8607 from leseb/refact

ceph: refactor operator initialization sequence and add more controllers (csi, lib-bucket-prov, discover daemon, flex)

cmd/rook/ceph/operator.go

@@ -17,12 +17,12 @@ limitations under the License.
 package ceph
 
 import (
+	"os"
+
 	"github.com/pkg/errors"
 	"github.com/rook/rook/cmd/rook/rook"
 	"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
 	operator "github.com/rook/rook/pkg/operator/ceph"
-	cluster "github.com/rook/rook/pkg/operator/ceph/cluster"
-
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/flags"
@@ -41,7 +41,7 @@ https://github.com/rook/rook`,
 }
 
 func init() {
-	operatorCmd.Flags().BoolVar(&cluster.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers")
+	operatorCmd.Flags().BoolVar(&operator.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers")
 
 	flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
 	flags.SetLoggingFlags(operatorCmd.Flags())
@@ -49,9 +49,7 @@ func init() {
 }
 
 func startOperator(cmd *cobra.Command, args []string) error {
-
 	rook.SetLogLevel()
-
 	rook.LogStartupInfo(operatorCmd.Flags())
 
 	logger.Info("starting Rook-Ceph operator")
@@ -62,6 +60,11 @@ func startOperator(cmd *cobra.Command, args []string) error {
 		rook.TerminateFatal(err)
 	}
 
+	// Fail if operator namespace is not provided
+	if os.Getenv(k8sutil.PodNamespaceEnvVar) == "" {
+		rook.TerminateFatal(errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %q", k8sutil.PodNamespaceEnvVar))
+	}
+
 	rook.CheckOperatorResources(context.Clientset)
 	rookImage := rook.GetOperatorImage(context.Clientset, containerName)
 	rookBaseImageCephVersion, err := rook.GetOperatorBaseImageCephVersion(context)

cmd/rook/ceph/osd.go

@@ -17,6 +17,7 @@ limitations under the License.
 package ceph
 
 import (
+	ctx "context"
 	"encoding/json"
 	"os"
 	"strings"
@@ -226,6 +227,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
 	ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID)
 	ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(&ownerRef, clusterInfo.Namespace)
 	clusterInfo.OwnerInfo = ownerInfo
+	clusterInfo.Context = ctx.Background()
 	kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerInfo)
 	agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat,
 		cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, cfg.pvcBacked)

cmd/rook/rook/rook.go

@@ -33,7 +33,6 @@ import (
 	"github.com/rook/rook/pkg/version"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-	"github.com/tevino/abool"
 	v1 "k8s.io/api/core/v1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -170,8 +169,6 @@ func NewContext() *clusterd.Context {
 	context.NetworkClient, err = netclient.NewForConfig(context.KubeConfig)
 	TerminateOnError(err, "failed to create network clientset")
 
-	context.RequestCancelOrchestration = abool.New()
-
 	return context
 }
 
@@ -228,23 +225,21 @@ func TerminateOnError(err error, msg string) {
 // TerminateFatal terminates the process with an exit code of 1
 // and writes the given reason to stderr and the termination log file.
 func TerminateFatal(reason error) {
-	fmt.Fprintln(os.Stderr, reason)
-
 	file, err := os.OpenFile(terminationLog, os.O_APPEND|os.O_WRONLY, 0600)
 	if err != nil {
-		fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %+v", err))
+		fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %v", err))
 	} else {
 		// #nosec G307 Calling defer to close the file without checking the error return is not a risk for a simple file open and close
 		defer file.Close()
 		if _, err = file.WriteString(reason.Error()); err != nil {
-			fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %+v", err))
+			fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %v", err))
 		}
 		if err := file.Close(); err != nil {
-			logger.Errorf("failed to close file. %v", err)
+			logger.Fatalf("failed to close file. %v", err)
 		}
 	}
 
-	os.Exit(1)
+	logger.Fatalln(reason)
 }
 
 // GetOperatorBaseImageCephVersion returns the Ceph version of the operator image

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/hashicorp/vault/api v1.1.2-0.20210713235431-1fc8af4c041f
 	github.com/hashicorp/vault/sdk v0.2.2-0.20210825150427-9b1f4d486f5d
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.0
-	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875
+	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210907154902-775800a3d0b0
 	github.com/libopenstorage/secrets v0.0.0-20210709082113-dde442ea20ec
 	github.com/openshift/cluster-api v0.0.0-20191129101638-b09907ac6668
 	github.com/openshift/machine-api-operator v0.2.1-0.20190903202259-474e14e4965a
@@ -26,7 +26,6 @@ require (
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
-	github.com/tevino/abool v1.2.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/ini.v1 v1.57.0
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -1067,8 +1067,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875 h1:jX3VXgmNOye8XYKjwcTVXcBYcPv3jj657fwX8DN/HiM=
-github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875/go.mod h1:XpQ9HGG9uF5aJCBP+s6w5kSiyTIVSqCV8+XAE4qms5E=
+github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210907154902-775800a3d0b0 h1:h7ghDWni/MDzN1AXXfvuwU88eazM0lvgntaouyJyALs=
+github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210907154902-775800a3d0b0/go.mod h1:XpQ9HGG9uF5aJCBP+s6w5kSiyTIVSqCV8+XAE4qms5E=
 github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
 github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
@@ -1486,8 +1486,6 @@ github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d/go.mod h1:BST
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.83+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.171+incompatible h1:K3fcS92NS8cRntIdu8Uqy2ZSePvX73nNhOkKuPGJLXQ=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.171+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
-github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
-github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=

pkg/clusterd/context.go

@@ -22,7 +22,6 @@ import (
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
 	"github.com/rook/rook/pkg/util/exec"
 	"github.com/rook/rook/pkg/util/sys"
-	"github.com/tevino/abool"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
@@ -32,7 +31,6 @@ import (
 
 // Context for loading or applying the configuration state of a service.
 type Context struct {
-
 	// The kubernetes config used for this context
 	KubeConfig *rest.Config
 
@@ -71,7 +69,4 @@ type Context struct {
 
 	// The local devices detected on the node
 	Devices []*sys.LocalDisk
-
-	// RequestCancelOrchestration manages the orchestration and its possible cancellation
-	RequestCancelOrchestration *abool.AtomicBool
 }

pkg/daemon/ceph/agent/controller.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package agent
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume"
+	opagent "github.com/rook/rook/pkg/operator/ceph/agent"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/ceph/provisioner"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+	flexcontroller "sigs.k8s.io/sig-storage-lib-external-provisioner/v6/controller"
+)
+
+const (
+	controllerName        = "rook-ceph-operator-flex-controller"
+	provisionerName       = "ceph.rook.io/block"
+	provisionerNameLegacy = "rook.io/block"
+)
+
+var (
+	// The supported configurations for the volume provisioner
+	provisionerConfigs = map[string]string{
+		provisionerName:       flexvolume.FlexvolumeVendor,
+		provisionerNameLegacy: flexvolume.FlexvolumeVendorLegacy,
+	}
+)
+
+// ReconcileAgent reconciles the Flex driver
+type ReconcileAgent struct {
+	client           client.Client
+	context          *clusterd.Context
+	opManagerContext context.Context
+	opConfig         opcontroller.OperatorConfig
+}
+
+// Add creates a new CephClient Controller and adds it to the Manager. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+	return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
+	return &ReconcileAgent{
+		client:           mgr.GetClient(),
+		context:          context,
+		opConfig:         opConfig,
+		opManagerContext: opManagerContext,
+	}
+}
+
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+	logger.Infof("%s successfully started", controllerName)
+
+	// Watch for ConfigMap (operator config)
+	err = c.Watch(&source.Kind{
+		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Reconcile reads that state of the operator config map and makes changes based on the state read
+// The Controller will requeue the Request to be processed again if the returned error is non-nil or
+// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
+func (r *ReconcileAgent) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
+	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
+	reconcileResponse, err := r.reconcile(request)
+	if err != nil {
+		logger.Errorf("failed to reconcile %v", err)
+	}
+
+	return reconcileResponse, err
+}
+
+func (r *ReconcileAgent) reconcile(request reconcile.Request) (reconcile.Result, error) {
+	if opcontroller.FlexDriverEnabled(r.context) {
+		rookAgent := opagent.New(r.context.Clientset)
+		if err := rookAgent.Start(r.opConfig.OperatorNamespace, r.opConfig.Image, r.opConfig.ServiceAccount); err != nil {
+			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to start agent daemonset")
+		}
+
+		serverVersion, err := r.context.Clientset.Discovery().ServerVersion()
+		if err != nil {
+			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version")
+		}
+		for name, vendor := range provisionerConfigs {
+			volumeProvisioner := provisioner.New(r.context, vendor)
+			pc := flexcontroller.NewProvisionController(
+				r.context.Clientset,
+				name,
+				volumeProvisioner,
+				serverVersion.GitVersion,
+			)
+			// TODO: register each go routine!!!
+			go pc.Run(r.opManagerContext)
+			logger.Infof("rook-provisioner %q started using %q flex vendor dir", name, vendor)
+		}
+	}
+
+	logger.Info("done reconciling")
+	return reconcile.Result{}, nil
+}

pkg/daemon/ceph/agent/flexvolume/controller.go

@@ -398,7 +398,7 @@ func (c *Controller) GetClientAccessInfo(args []string, clientAccessInfo *Client
 	ctx := context.TODO()
 	// args: 0 ClusterNamespace, 1 PodNamespace, 2 MountUser, 3 MountSecret
 	clusterNamespace := args[0]
-	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, clusterNamespace)
+	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, ctx, clusterNamespace)
 	if err != nil {
 		return errors.Wrapf(err, "failed to load cluster information from clusters namespace %s", clusterNamespace)
 	}

pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	ctx "context"
+
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
@@ -229,7 +231,7 @@ func (vm *VolumeManager) isAttached(image, pool, clusterNamespace string) (strin
 }
 
 func getClusterInfo(context *clusterd.Context, clusterNamespace string) (string, string, error) {
-	clusterInfo, _, _, err := mon.LoadClusterInfo(context, clusterNamespace)
+	clusterInfo, _, _, err := mon.LoadClusterInfo(context, ctx.TODO(), clusterNamespace)
 	if err != nil {
 		return "", "", errors.Wrapf(err, "failed to load cluster information from cluster %s", clusterNamespace)
 	}

pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go

@@ -162,7 +162,7 @@ func TestAttach(t *testing.T) {
 		},
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	_, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo)
+	_, _, _, err = mon.CreateOrLoadClusterInfo(context, ctx, clusterNamespace, ownerInfo)
 	assert.NoError(t, err)
 	devicePath, err := vm.Attach("image1", "testpool", "admin", "never-gonna-give-you-up", clusterNamespace)
 	assert.Equal(t, "/dev/rbd3", devicePath)
@@ -244,7 +244,7 @@ func TestDetach(t *testing.T) {
 		},
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	_, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo)
+	_, _, _, err = mon.CreateOrLoadClusterInfo(context, ctx, clusterNamespace, ownerInfo)
 	assert.NoError(t, err)
 	err = vm.Detach("image1", "testpool", "admin", "", clusterNamespace, false)
 	assert.Nil(t, err)
@@ -300,7 +300,7 @@ func TestDetachCustomKeyring(t *testing.T) {
 		},
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	_, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo)
+	_, _, _, err = mon.CreateOrLoadClusterInfo(context, ctx, clusterNamespace, ownerInfo)
 	assert.NoError(t, err)
 	err = vm.Detach("image1", "testpool", "user1", "", clusterNamespace, false)
 	assert.Nil(t, err)

pkg/daemon/ceph/agent/predicate.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package agent
+
+import (
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// predicateController is the predicate function to trigger reconcile on operator configuration cm change
+func predicateController() predicate.Funcs {
+	return predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			if cm, ok := e.Object.(*v1.ConfigMap); ok {
+				return cm.Name == opcontroller.OperatorSettingConfigMapName
+			}
+
+			return false
+		},
+
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			if old, ok := e.ObjectOld.(*v1.ConfigMap); ok {
+				if new, ok := e.ObjectNew.(*v1.ConfigMap); ok {
+					if old.Name == opcontroller.OperatorSettingConfigMapName && new.Name == opcontroller.OperatorSettingConfigMapName {
+						return old.Data["ROOK_ENABLE_FLEX_DRIVER"] != new.Data["ROOK_ENABLE_FLEX_DRIVER"]
+					}
+				}
+			}
+
+			return false
+		},
+
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			return false
+		},
+
+		GenericFunc: func(e event.GenericEvent) bool {
+			return false
+		},
+	}
+}

pkg/daemon/ceph/client/command.go

@@ -127,6 +127,9 @@ func NewRBDCommand(context *clusterd.Context, clusterInfo *ClusterInfo, args []s
 
 func (c *CephToolCommand) run() ([]byte, error) {
 	command, args := FinalizeCephCommandArgs(c.tool, c.clusterInfo, c.args, c.context.ConfigDir)
+	if c.clusterInfo.Context.Err() != nil {
+		return nil, c.clusterInfo.Context.Err()
+	}
 	if c.JsonOutput {
 		args = append(args, "--format", "json")
 	} else {

pkg/daemon/ceph/client/command_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package client
 
 import (
+	"context"
 	"strconv"
 	"testing"
 	"time"
@@ -140,4 +141,18 @@ func TestNewRBDCommand(t *testing.T) {
 		assert.EqualError(t, err, "no pods found with selector \"rook-ceph-mgr\"")
 	})
 
+	t.Run("context canceled nothing to run", func(t *testing.T) {
+		clusterInfo := AdminClusterInfo("rook")
+		ctx, cancel := context.WithCancel(context.TODO())
+		clusterInfo.Context = ctx
+		cancel()
+		executor := &exectest.MockExecutor{}
+		context := &clusterd.Context{Executor: executor, RemoteExecutor: exec.RemotePodCommandExecutor{ClientSet: test.New(t, 3)}}
+		cmd := NewRBDCommand(context, clusterInfo, args)
+		_, err := cmd.Run()
+		assert.Error(t, err)
+		// This is not the best but it shows we go through the right codepath
+		assert.EqualError(t, err, "context canceled")
+	})
+
 }

pkg/daemon/ceph/client/config.go

@@ -18,7 +18,6 @@ limitations under the License.
 package client
 
 import (
-	"context"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -139,8 +138,7 @@ func generateConfigFile(context *clusterd.Context, clusterInfo *ClusterInfo, pat
 }
 
 func mergeDefaultConfigWithRookConfigOverride(clusterdContext *clusterd.Context, clusterInfo *ClusterInfo, configFile *ini.File) error {
-	ctx := context.TODO()
-	cm, err := clusterdContext.Clientset.CoreV1().ConfigMaps(clusterInfo.Namespace).Get(ctx, k8sutil.ConfigOverrideName, metav1.GetOptions{})
+	cm, err := clusterdContext.Clientset.CoreV1().ConfigMaps(clusterInfo.Namespace).Get(clusterInfo.Context, k8sutil.ConfigOverrideName, metav1.GetOptions{})
 	if err != nil {
 		if !kerrors.IsNotFound(err) {
 			return errors.Wrapf(err, "failed to read configmap %q", k8sutil.ConfigOverrideName)

pkg/daemon/ceph/client/config_test.go

@@ -108,6 +108,7 @@ func TestGenerateConfigFile(t *testing.T) {
 		},
 		CephVersion: cephver.Nautilus,
 		CephCred:    CephCred{Username: "admin", Secret: "mysecret"},
+		Context:     ctx,
 	}
 
 	isInitialized := clusterInfo.IsInitialized(true)

pkg/daemon/ceph/client/info.go

@@ -17,6 +17,7 @@ limitations under the License.
 package client
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"testing"
@@ -46,6 +47,14 @@ type ClusterInfo struct {
 	name              string
 	OsdUpgradeTimeout time.Duration
 	NetworkSpec       cephv1.NetworkSpec
+	// A context to cancel the context it is used to determine whether the reconcile loop should
+	// exist (if the context has been cancelled). This cannot be in main clusterd context since this
+	// is a pointer passed through the entire life cycle or the operator. If the context is
+	// cancelled it will immedialy be re-created, thus existing reconciles loops will not be
+	// cancelled.
+	// Whereas if passed through clusterInfo, we don't have that problem since clusterInfo is
+	// re-hydrated when a context is cancelled.
+	Context context.Context
 }
 
 // MonInfo is a collection of information about a Ceph mon.
@@ -90,6 +99,7 @@ func AdminClusterInfo(namespace string) *ClusterInfo {
 		},
 		name:      "testing",
 		OwnerInfo: ownerInfo,
+		Context:   context.TODO(),
 	}
 }
 
@@ -120,6 +130,10 @@ func (c *ClusterInfo) IsInitialized(logError bool) bool {
 		if logError {
 			logger.Error("ceph secret is empty")
 		}
+	} else if c.Context == nil {
+		if logError {
+			logger.Error("nil context")
+		}
 	} else {
 		isInitialized = true
 	}

pkg/daemon/ceph/client/test/info.go

@@ -17,6 +17,7 @@ limitations under the License.
 package test
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -53,6 +54,7 @@ func CreateTestClusterInfo(monCount int) *client.ClusterInfo {
 		},
 		Monitors:  map[string]*client.MonInfo{},
 		OwnerInfo: ownerInfo,
+		Context:   context.TODO(),
 	}
 	mons := []string{"a", "b", "c", "d", "e"}
 	for i := 0; i < monCount; i++ {

pkg/daemon/ceph/client/upgrade_test.go

@@ -300,7 +300,7 @@ func TestGetRetryConfig(t *testing.T) {
 }
 
 func TestOSDUpdateShouldCheckOkToStop(t *testing.T) {
-	clusterInfo := &ClusterInfo{}
+	clusterInfo := AdminClusterInfo("mycluster")
 	lsOutput := ""
 	treeOutput := ""
 	context := &clusterd.Context{

pkg/daemon/ceph/osd/daemon.go

@@ -320,7 +320,7 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd
 			// If we detect a partition we have to make sure that ceph-volume will be able to consume it
 			// ceph-volume version 14.2.8 has the right code to support partitions
 			if !agent.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) {
-				logger.Infof("skipping device %q because it is a partition and ceph version is too old, you need at least ceph %q", device.Name, cephVolumeRawModeMinCephVersion.String())
+				logger.Infof("skipping device %q because it is a partition and ceph version is too old %q, you need at least ceph %q", device.Name, agent.clusterInfo.CephVersion.String(), cephVolumeRawModeMinCephVersion.String())
 				continue
 			}
 

pkg/daemon/ceph/osd/device_test.go

@@ -40,7 +40,7 @@ func TestOSDBootstrap(t *testing.T) {
 
 	context := &clusterd.Context{Executor: executor, ConfigDir: configDir}
 	defer os.RemoveAll(context.ConfigDir)
-	err := createOSDBootstrapKeyring(context, &client.ClusterInfo{Namespace: "name"}, configDir)
+	err := createOSDBootstrapKeyring(context, client.AdminClusterInfo("mycluster"), configDir)
 	assert.Nil(t, err)
 
 	targetPath := path.Join(configDir, bootstrapOsdKeyring)

pkg/daemon/ceph/osd/kms/k8s.go

@@ -17,7 +17,6 @@ limitations under the License.
 package kms
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/pkg/errors"
@@ -42,14 +41,13 @@ const (
 
 // storeSecretInKubernetes stores the dmcrypt key in a Kubernetes Secret
 func (c *Config) storeSecretInKubernetes(pvcName, key string) error {
-	ctx := context.TODO()
 	s, err := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo)
 	if err != nil {
 		return err
 	}
 
 	// Create the Kubernetes Secret
-	_, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, s, metav1.CreateOptions{})
+	_, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, s, metav1.CreateOptions{})
 	if err != nil && !kerrors.IsAlreadyExists(err) {
 		return errors.Wrapf(err, "failed to save ceph osd encryption key as a secret for pvc %q", pvcName)
 	}

pkg/daemon/ceph/osd/remove.go

@@ -17,7 +17,6 @@ limitations under the License.
 package osd
 
 import (
-	"context"
 	"fmt"
 	"strconv"
 
@@ -68,7 +67,6 @@ func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osds
 }
 
 func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int, preservePVC bool) {
-	ctx := context.TODO()
 	// Get the host where the OSD is found
 	hostName, err := client.GetCrushHostName(clusterdContext, clusterInfo, osdID)
 	if err != nil {
@@ -84,7 +82,7 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf
 
 	// Remove the OSD deployment
 	deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID)
-	deployment, err := clusterdContext.Clientset.AppsV1().Deployments(clusterInfo.Namespace).Get(ctx, deploymentName, metav1.GetOptions{})
+	deployment, err := clusterdContext.Clientset.AppsV1().Deployments(clusterInfo.Namespace).Get(clusterInfo.Context, deploymentName, metav1.GetOptions{})
 	if err != nil {
 		logger.Errorf("failed to fetch the deployment %q. %v", deploymentName, err)
 	} else {
@@ -97,7 +95,7 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf
 		}
 		if pvcName, ok := deployment.GetLabels()[osd.OSDOverPVCLabelKey]; ok {
 			labelSelector := fmt.Sprintf("%s=%s", osd.OSDOverPVCLabelKey, pvcName)
-			prepareJobList, err := clusterdContext.Clientset.BatchV1().Jobs(clusterInfo.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
+			prepareJobList, err := clusterdContext.Clientset.BatchV1().Jobs(clusterInfo.Namespace).List(clusterInfo.Context, metav1.ListOptions{LabelSelector: labelSelector})
 			if err != nil && !kerrors.IsNotFound(err) {
 				logger.Errorf("failed to list osd prepare jobs with pvc %q. %v ", pvcName, err)
 			}
@@ -114,20 +112,20 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf
 			if preservePVC {
 				// Detach the OSD PVC from Rook. We will continue OSD deletion even if failed to remove PVC label
 				logger.Infof("detach the OSD PVC %q from Rook", pvcName)
-				if pvc, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Get(ctx, pvcName, metav1.GetOptions{}); err != nil {
+				if pvc, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Get(clusterInfo.Context, pvcName, metav1.GetOptions{}); err != nil {
 					logger.Errorf("failed to get pvc for OSD %q. %v", pvcName, err)
 				} else {
 					labels := pvc.GetLabels()
 					delete(labels, osd.CephDeviceSetPVCIDLabelKey)
 					pvc.SetLabels(labels)
-					if _, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil {
+					if _, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Update(clusterInfo.Context, pvc, metav1.UpdateOptions{}); err != nil {
 						logger.Errorf("failed to remove label %q from pvc for OSD %q. %v", osd.CephDeviceSetPVCIDLabelKey, pvcName, err)
 					}
 				}
 			} else {
 				// Remove the OSD PVC
 				logger.Infof("removing the OSD PVC %q", pvcName)
-				if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil {
+				if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(clusterInfo.Context, pvcName, metav1.DeleteOptions{}); err != nil {
 					if err != nil {
 						// Continue deleting the OSD PVC even if PVC deletion fails
 						logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err)

pkg/daemon/ceph/osd/volume_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package osd
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -382,11 +383,12 @@ func TestConfigureCVDevices(t *testing.T) {
 			return "", errors.Errorf("unknown command %s %s", command, args)
 		}
 
-		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
 		clusterInfo := &cephclient.ClusterInfo{
 			CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8},
 			FSID:        clusterFSID,
+			Context:     context.TODO(),
 		}
+		context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir}
 		agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{DeviceClass: "myds"}}
 		devices := createPVCAvailableDevices()
 		deviceOSDs, err := agent.configureCVDevices(context, devices)

pkg/operator/ceph/client/controller.go

@@ -64,24 +64,26 @@ var controllerTypeMeta = metav1.TypeMeta{
 
 // ReconcileCephClient reconciles a CephClient object
 type ReconcileCephClient struct {
-	client      client.Client
-	scheme      *runtime.Scheme
-	context     *clusterd.Context
-	clusterInfo *cephclient.ClusterInfo
+	client           client.Client
+	scheme           *runtime.Scheme
+	context          *clusterd.Context
+	clusterInfo      *cephclient.ClusterInfo
+	opManagerContext context.Context
 }
 
 // Add creates a new CephClient Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, context *clusterd.Context) error {
-	return add(mgr, newReconciler(mgr, context))
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+	return add(mgr, newReconciler(mgr, context, opManagerContext))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler {
 	return &ReconcileCephClient{
-		client:  mgr.GetClient(),
-		scheme:  mgr.GetScheme(),
-		context: context,
+		client:           mgr.GetClient(),
+		scheme:           mgr.GetScheme(),
+		context:          context,
+		opManagerContext: opManagerContext,
 	}
 }
 
@@ -128,7 +130,7 @@ func (r *ReconcileCephClient) Reconcile(context context.Context, request reconci
 func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Result, error) {
 	// Fetch the CephClient instance
 	cephClient := &cephv1.CephClient{}
-	err := r.client.Get(context.TODO(), request.NamespacedName, cephClient)
+	err := r.client.Get(r.opManagerContext, request.NamespacedName, cephClient)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("cephClient resource not found. Ignoring since object must be deleted.")
@@ -146,7 +148,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
 
 	// The CR was just created, initializing status fields
 	if cephClient.Status == nil {
-		updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing)
+		r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing)
 	}
 
 	// Make sure a CephCluster is present otherwise do nothing
@@ -172,10 +174,11 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
 	}
 
 	// Populate clusterInfo during each reconcile
-	r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace)
+	r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
 	}
+	r.clusterInfo.Context = r.opManagerContext
 
 	// DELETE: the CR was deleted
 	if !cephClient.GetDeletionTimestamp().IsZero() {
@@ -208,12 +211,12 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
 			logger.Info(opcontroller.OperatorNotInitializedMessage)
 			return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
 		}
-		updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure)
+		r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure)
 		return reconcile.Result{}, errors.Wrapf(err, "failed to create or update client %q", cephClient.Name)
 	}
 
 	// Success! Let's update the status
-	updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady)
+	r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady)
 
 	// Return and do not requeue
 	logger.Debug("done reconciling")
@@ -222,7 +225,6 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
 
 // Create the client
 func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient) error {
-	ctx := context.TODO()
 	logger.Infof("creating client %s in namespace %s", cephClient.Name, cephClient.Namespace)
 
 	// Generate the CephX details
@@ -261,11 +263,11 @@ func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient
 	}
 
 	// Create or Update Kubernetes Secret
-	_, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
+	_, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Get(r.clusterInfo.Context, secret.Name, metav1.GetOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debugf("creating secret for %q", secret.Name)
-			if _, err := r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+			if _, err := r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Create(r.clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil {
 				return errors.Wrapf(err, "failed to create secret for %q", secret.Name)
 			}
 			logger.Infof("created client %q", cephClient.Name)
@@ -274,7 +276,7 @@ func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient
 		return errors.Wrapf(err, "failed to get secret for %q", secret.Name)
 	}
 	logger.Debugf("updating secret for %s", secret.Name)
-	_, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
+	_, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Update(r.clusterInfo.Context, secret, metav1.UpdateOptions{})
 	if err != nil {
 		return errors.Wrapf(err, "failed to update secret for %q", secret.Name)
 	}
@@ -332,9 +334,9 @@ func generateClientName(name string) string {
 }
 
 // updateStatus updates an object with a given status
-func updateStatus(client client.Client, name types.NamespacedName, status cephv1.ConditionType) {
+func (r *ReconcileCephClient) updateStatus(client client.Client, name types.NamespacedName, status cephv1.ConditionType) {
 	cephClient := &cephv1.CephClient{}
-	if err := client.Get(context.TODO(), name, cephClient); err != nil {
+	if err := client.Get(r.opManagerContext, name, cephClient); err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("CephClient resource not found. Ignoring since object must be deleted.")
 			return

pkg/operator/ceph/client/controller_test.go

@@ -32,7 +32,6 @@ import (
 	testop "github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -155,10 +154,9 @@ func TestCephClientController(t *testing.T) {
 		},
 	}
 	c := &clusterd.Context{
-		Executor:                   executor,
-		Clientset:                  testop.New(t, 1),
-		RookClientset:              rookclient.NewSimpleClientset(),
-		RequestCancelOrchestration: abool.New(),
+		Executor:      executor,
+		Clientset:     testop.New(t, 1),
+		RookClientset: rookclient.NewSimpleClientset(),
 	}
 
 	// Register operator types with the runtime scheme.
@@ -170,9 +168,10 @@ func TestCephClientController(t *testing.T) {
 
 	// Create a ReconcileCephClient object with the scheme and fake client.
 	r := &ReconcileCephClient{
-		client:  cl,
-		scheme:  s,
-		context: c,
+		client:           cl,
+		scheme:           s,
+		context:          c,
+		opManagerContext: ctx,
 	}
 
 	// Mock request to simulate Reconcile() being called on an event for a
@@ -217,9 +216,10 @@ func TestCephClientController(t *testing.T) {
 	cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
 	// Create a ReconcileCephClient object with the scheme and fake client.
 	r = &ReconcileCephClient{
-		client:  cl,
-		scheme:  s,
-		context: c,
+		client:           cl,
+		scheme:           s,
+		context:          c,
+		opManagerContext: ctx,
 	}
 	assert.True(t, res.Requeue)
 
@@ -274,15 +274,10 @@ func TestCephClientController(t *testing.T) {
 	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{})
 	// Create a ReconcileCephClient object with the scheme and fake client.
 	r = &ReconcileCephClient{
-		client:  cl,
-		scheme:  s,
-		context: c,
-	}
-
-	r = &ReconcileCephClient{
-		client:  cl,
-		scheme:  s,
-		context: c,
+		client:           cl,
+		scheme:           s,
+		context:          c,
+		opManagerContext: context.TODO(),
 	}
 
 	res, err = r.Reconcile(ctx, req)

pkg/operator/ceph/cluster/cephstatus.go

@@ -84,13 +84,13 @@ func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.Clu
 }
 
 // checkCephStatus periodically checks the health of the cluster
-func (c *cephStatusChecker) checkCephStatus(stopCh chan struct{}) {
+func (c *cephStatusChecker) checkCephStatus(context context.Context) {
 	// check the status immediately before starting the loop
 	c.checkStatus()
 
 	for {
 		select {
-		case <-stopCh:
+		case <-context.Done():
 			logger.Infof("stopping monitoring of ceph status")
 			return
 
@@ -174,7 +174,7 @@ func (c *cephStatusChecker) configureHealthSettings(status cephclient.CephStatus
 // updateStatus updates an object with a given status
 func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus, condition cephv1.ConditionType, reason cephv1.ConditionReason, message string, conditionStatus v1.ConditionStatus) {
 	clusterName := c.clusterInfo.NamespacedName()
-	cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(context.TODO(), clusterName.Name, metav1.GetOptions{})
+	cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(c.clusterInfo.Context, clusterName.Name, metav1.GetOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
@@ -241,10 +241,9 @@ func formatTime(t time.Time) string {
 }
 
 func (c *ClusterController) updateClusterCephVersion(image string, cephVersion cephver.CephVersion) {
-	ctx := context.TODO()
 	logger.Infof("cluster %q: version %q detected for image %q", c.namespacedName.Namespace, cephVersion.String(), image)
 
-	cephCluster, err := c.context.RookClientset.CephV1().CephClusters(c.namespacedName.Namespace).Get(ctx, c.namespacedName.Name, metav1.GetOptions{})
+	cephCluster, err := c.context.RookClientset.CephV1().CephClusters(c.namespacedName.Namespace).Get(c.OpManagerCtx, c.namespacedName.Name, metav1.GetOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
@@ -324,7 +323,7 @@ func (c *cephStatusChecker) getRookPodsOnNode(node string) ([]v1.Pod, error) {
 	listOpts := metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("spec.nodeName=%s", node),
 	}
-	pods, err := c.context.Clientset.CoreV1().Pods(clusterName.Namespace).List(context.TODO(), listOpts)
+	pods, err := c.context.Clientset.CoreV1().Pods(clusterName.Namespace).List(c.clusterInfo.Context, listOpts)
 	if err != nil {
 		return podsOnNode, errors.Wrapf(err, "failed to get pods on node %q", node)
 	}

pkg/operator/ceph/cluster/cleanup.go

@@ -58,9 +58,9 @@ var (
 	sanitizeIterationDefault int32 = 1
 )
 
-func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
+func (c *ClusterController) startClusterCleanUp(context context.Context, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) {
 	logger.Infof("starting clean up for cluster %q", cluster.Name)
-	err := c.waitForCephDaemonCleanUp(stopCleanupCh, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
+	err := c.waitForCephDaemonCleanUp(context, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second)
 	if err != nil {
 		logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err)
 		return
@@ -183,7 +183,7 @@ func getCleanupPlacement(c cephv1.ClusterSpec) cephv1.Placement {
 	return cephv1.Placement{Tolerations: tolerations}
 }
 
-func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, retryInterval time.Duration) error {
+func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cluster *cephv1.CephCluster, retryInterval time.Duration) error {
 	logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace)
 	for {
 		select {
@@ -200,15 +200,14 @@ func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{}
 
 			logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q",
 				cluster.Namespace, retryInterval.String())
-		case <-stopCleanupCh:
-			return errors.New("cancelling the host cleanup job")
+		case <-context.Done():
+			return errors.Errorf("cancelling the host cleanup job. %s", context.Err())
 		}
 	}
 }
 
 // getCephHosts returns a list of host names where ceph daemon pods are running
 func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
-	ctx := context.TODO()
 	cephAppNames := []string{mon.AppName, mgr.AppName, osd.AppName, object.AppName, mds.AppName, rbd.AppName, mirror.AppName}
 	nodeNameList := sets.NewString()
 	hostNameList := []string{}
@@ -217,7 +216,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
 	// get all the node names where ceph daemons are running
 	for _, app := range cephAppNames {
 		appLabelSelector := fmt.Sprintf("app=%s", app)
-		podList, err := c.context.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: appLabelSelector})
+		podList, err := c.context.Clientset.CoreV1().Pods(namespace).List(c.OpManagerCtx, metav1.ListOptions{LabelSelector: appLabelSelector})
 		if err != nil {
 			return hostNameList, errors.Wrapf(err, "could not list the %q pods", app)
 		}
@@ -244,7 +243,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
 }
 
 func (c *ClusterController) getCleanUpDetails(namespace string) (string, string, error) {
-	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, namespace)
+	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.OpManagerCtx, namespace)
 	if err != nil {
 		return "", "", errors.Wrap(err, "failed to get cluster info")
 	}

pkg/operator/ceph/cluster/cleanup_test.go

@@ -49,19 +49,7 @@ func TestCleanupJobSpec(t *testing.T) {
 		Clientset:     clientset,
 		RookClientset: rookfake.NewSimpleClientset(),
 	}
-	operatorConfigCallbacks := []func() error{
-		func() error {
-			logger.Infof("test success callback")
-			return nil
-		},
-	}
-	addCallbacks := []func() error{
-		func() error {
-			logger.Infof("test success callback")
-			return nil
-		},
-	}
-	controller := NewClusterController(context, "", &attachment.MockAttachment{}, operatorConfigCallbacks, addCallbacks)
+	controller := NewClusterController(context, "", &attachment.MockAttachment{})
 	podTemplateSpec := controller.cleanUpJobTemplateSpec(cluster, "monSecret", "28b87851-8dc1-46c8-b1ec-90ec51a47c89")
 	assert.Equal(t, expectedHostPath, podTemplateSpec.Spec.Containers[0].Env[0].Value)
 	assert.Equal(t, expectedNamespace, podTemplateSpec.Spec.Containers[0].Env[1].Value)

pkg/operator/ceph/cluster/cluster.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"os/exec"
 	"path"
-	"strings"
 	"sync"
 	"syscall"
 
@@ -58,17 +57,14 @@ type cluster struct {
 	Spec               *cephv1.ClusterSpec
 	namespacedName     types.NamespacedName
 	mons               *mon.Cluster
-	stopCh             chan struct{}
-	closedStopCh       bool
 	ownerInfo          *k8sutil.OwnerInfo
 	isUpgrade          bool
-	watchersActivated  bool
-	monitoringChannels map[string]*clusterHealth
+	monitoringRoutines map[string]*clusterHealth
 }
 
 type clusterHealth struct {
-	stopChan          chan struct{}
-	monitoringRunning bool
+	internalCtx    context.Context
+	internalCancel context.CancelFunc
 }
 
 func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerInfo *k8sutil.OwnerInfo) *cluster {
@@ -81,8 +77,7 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync
 		Spec:               &c.Spec,
 		context:            context,
 		namespacedName:     types.NamespacedName{Namespace: c.Namespace, Name: c.Name},
-		monitoringChannels: make(map[string]*clusterHealth),
-		stopCh:             make(chan struct{}),
+		monitoringRoutines: make(map[string]*clusterHealth),
 		ownerInfo:          ownerInfo,
 		mons:               mon.New(context, c.Namespace, c.Spec, ownerInfo, csiMutex),
 	}
@@ -104,6 +99,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
 	}
 	clusterInfo.OwnerInfo = c.ownerInfo
 	clusterInfo.SetName(c.namespacedName.Name)
+	clusterInfo.Context = c.ClusterInfo.Context
 	c.ClusterInfo = clusterInfo
 	c.ClusterInfo.NetworkSpec = c.Spec.Network
 
@@ -112,9 +108,8 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
 		return errors.New("the cluster identity was not established")
 	}
 
-	// Check whether we need to cancel the orchestration
-	if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
-		return err
+	if c.ClusterInfo.Context.Err() != nil {
+		return c.ClusterInfo.Context.Err()
 	}
 
 	// Execute actions after the monitors are up and running
@@ -124,15 +119,6 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep
 		return errors.Wrap(err, "failed to execute post actions after all the ceph monitors started")
 	}
 
-	// If this is an upgrade, notify all the child controllers
-	if c.isUpgrade {
-		logger.Info("upgrade in progress, notifying child CRs")
-		err := c.notifyChildControllerOfUpgrade()
-		if err != nil {
-			return errors.Wrap(err, "failed to notify child CRs of upgrade")
-		}
-	}
-
 	// Start Ceph manager
 	controller.UpdateCondition(c.context, c.namespacedName, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring Ceph Mgr(s)")
 	mgrs := mgr.New(c.context, c.ClusterInfo, *c.Spec, rookImage)
@@ -179,7 +165,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
 		}
 	}
 
-	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, cluster.Namespace)
+	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace)
 	if err != nil {
 		logger.Infof("clusterInfo not yet found, must be a new cluster")
 	} else {
@@ -200,7 +186,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
 		// Test if the cluster has already been configured if the mgr deployment has been created.
 		// If the mgr does not exist, the mons have never been verified to be in quorum.
 		opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, mgr.AppName)}
-		mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(context.TODO(), opts)
+		mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(c.OpManagerCtx, opts)
 		if err == nil && len(mgrDeployments.Items) > 0 && cluster.ClusterInfo != nil {
 			c.configureCephMonitoring(cluster, clusterInfo)
 		}
@@ -245,6 +231,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
 
 	controller.UpdateCondition(c.context, c.namespacedName, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring the Ceph cluster")
 
+	cluster.ClusterInfo.Context = c.OpManagerCtx
 	// Run the orchestration
 	err = cluster.reconcileCephDaemons(c.rookImage, *cephVersion)
 	if err != nil {
@@ -256,104 +243,15 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
 	return nil
 }
 
-func (c *cluster) notifyChildControllerOfUpgrade() error {
-	ctx := context.TODO()
-	version := strings.Replace(c.ClusterInfo.CephVersion.String(), " ", "-", -1)
-
-	// List all child controllers
-	cephFilesystems, err := c.context.RookClientset.CephV1().CephFilesystems(c.Namespace).List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return errors.Wrap(err, "failed to list ceph filesystem CRs")
-	}
-	for _, cephFilesystem := range cephFilesystems.Items {
-		if cephFilesystem.Labels == nil {
-			cephFilesystem.Labels = map[string]string{}
-		}
-		cephFilesystem.Labels["ceph_version"] = version
-		localCephFilesystem := cephFilesystem
-		_, err := c.context.RookClientset.CephV1().CephFilesystems(c.Namespace).Update(ctx, &localCephFilesystem, metav1.UpdateOptions{})
-		if err != nil {
-			return errors.Wrapf(err, "failed to update ceph filesystem CR %q with new label", cephFilesystem.Name)
-		}
-	}
-
-	cephObjectStores, err := c.context.RookClientset.CephV1().CephObjectStores(c.Namespace).List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return errors.Wrap(err, "failed to list ceph object store CRs")
-	}
-	for _, cephObjectStore := range cephObjectStores.Items {
-		if cephObjectStore.Labels == nil {
-			cephObjectStore.Labels = map[string]string{}
-		}
-		cephObjectStore.Labels["ceph_version"] = version
-		localCephObjectStore := cephObjectStore
-		_, err := c.context.RookClientset.CephV1().CephObjectStores(c.Namespace).Update(ctx, &localCephObjectStore, metav1.UpdateOptions{})
-		if err != nil {
-			return errors.Wrapf(err, "failed to update ceph object store CR %q with new label", cephObjectStore.Name)
-		}
-	}
-
-	cephNFSes, err := c.context.RookClientset.CephV1().CephNFSes(c.Namespace).List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return errors.Wrap(err, "failed to list ceph nfs CRs")
-	}
-	for _, cephNFS := range cephNFSes.Items {
-		if cephNFS.Labels == nil {
-			cephNFS.Labels = map[string]string{}
-		}
-		cephNFS.Labels["ceph_version"] = version
-		localCephNFS := cephNFS
-		_, err := c.context.RookClientset.CephV1().CephNFSes(c.Namespace).Update(ctx, &localCephNFS, metav1.UpdateOptions{})
-		if err != nil {
-			return errors.Wrapf(err, "failed to update ceph nfs CR %q with new label", cephNFS.Name)
-		}
-	}
-
-	cephRBDMirrors, err := c.context.RookClientset.CephV1().CephRBDMirrors(c.Namespace).List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return errors.Wrap(err, "failed to list ceph rbd-mirror CRs")
-	}
-	for _, cephRBDMirror := range cephRBDMirrors.Items {
-		if cephRBDMirror.Labels == nil {
-			cephRBDMirror.Labels = map[string]string{}
-		}
-		cephRBDMirror.Labels["ceph_version"] = version
-		localCephRBDMirror := cephRBDMirror
-		_, err := c.context.RookClientset.CephV1().CephRBDMirrors(c.Namespace).Update(ctx, &localCephRBDMirror, metav1.UpdateOptions{})
-		if err != nil {
-			return errors.Wrapf(err, "failed to update ceph rbd-mirror CR %q with new label", cephRBDMirror.Name)
-		}
-	}
-
-	cephFilesystemMirrors, err := c.context.RookClientset.CephV1().CephFilesystemMirrors(c.Namespace).List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return errors.Wrap(err, "failed to list cephfs mirror CRs")
-	}
-	for _, cephFilesystemMirror := range cephFilesystemMirrors.Items {
-		if cephFilesystemMirror.Labels == nil {
-			cephFilesystemMirror.Labels = map[string]string{}
-		}
-		cephFilesystemMirror.Labels["ceph_version"] = version
-		localCephFilesystemMirror := cephFilesystemMirror
-		_, err := c.context.RookClientset.CephV1().CephFilesystemMirrors(c.Namespace).Update(ctx, &localCephFilesystemMirror, metav1.UpdateOptions{})
-		if err != nil {
-			return errors.Wrapf(err, "failed to update ceph nfs CR %q with new label", cephFilesystemMirror.Name)
-		}
-	}
-
-	return nil
-}
-
 // Validate the cluster Specs
 func preClusterStartValidation(cluster *cluster) error {
-	ctx := context.TODO()
 	if cluster.Spec.Mon.Count == 0 {
 		logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount)
 		cluster.Spec.Mon.Count = mon.DefaultMonCount
 	}
 	if !cluster.Spec.Mon.AllowMultiplePerNode {
 		// Check that there are enough nodes to have a chance of starting the requested number of mons
-		nodes, err := cluster.context.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+		nodes, err := cluster.context.Clientset.CoreV1().Nodes().List(cluster.ClusterInfo.Context, metav1.ListOptions{})
 		if err == nil && len(nodes.Items) < cluster.Spec.Mon.Count {
 			return errors.Errorf("cannot start %d mons on %d node(s) when allowMultiplePerNode is false", cluster.Spec.Mon.Count, len(nodes.Items))
 		}
@@ -381,7 +279,7 @@ func preClusterStartValidation(cluster *cluster) error {
 			}
 
 			// Get network attachment definition
-			_, err := cluster.context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(ctx, nad, metav1.GetOptions{})
+			_, err := cluster.context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(cluster.ClusterInfo.Context, nad, metav1.GetOptions{})
 			if err != nil {
 				if kerrors.IsNotFound(err) {
 					return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector)

pkg/operator/ceph/cluster/cluster_external.go

@@ -49,8 +49,9 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
 	// loop until we find the secret necessary to connect to the external cluster
 	// then populate clusterInfo
 
-	cluster.ClusterInfo = mon.PopulateExternalClusterInfo(c.context, c.namespacedName.Namespace, cluster.ownerInfo)
+	cluster.ClusterInfo = mon.PopulateExternalClusterInfo(c.context, c.OpManagerCtx, c.namespacedName.Namespace, cluster.ownerInfo)
 	cluster.ClusterInfo.SetName(c.namespacedName.Name)
+	cluster.ClusterInfo.Context = c.OpManagerCtx
 
 	if !client.IsKeyringBase64Encoded(cluster.ClusterInfo.CephCred.Secret) {
 		return errors.Errorf("invalid user health checker key for user %q", cluster.ClusterInfo.CephCred.Username)

pkg/operator/ceph/cluster/cluster_test.go

@@ -22,6 +22,7 @@ import (
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
 	testop "github.com/rook/rook/pkg/operator/test"
 )
 
@@ -34,27 +35,27 @@ func TestPreClusterStartValidation(t *testing.T) {
 		args    args
 		wantErr bool
 	}{
-		{"no settings", args{&cluster{Spec: &cephv1.ClusterSpec{}, context: &clusterd.Context{Clientset: testop.New(t, 3)}}}, false},
-		{"even mons", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 2}}}}, false},
-		{"missing stretch zones", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
+		{"no settings", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), Spec: &cephv1.ClusterSpec{}, context: &clusterd.Context{Clientset: testop.New(t, 3)}}}, false},
+		{"even mons", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 2}}}}, false},
+		{"missing stretch zones", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
 			{Name: "a"},
 		}}}}}}, true},
-		{"missing arbiter", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
+		{"missing arbiter", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
 			{Name: "a"},
 			{Name: "b"},
 			{Name: "c"},
 		}}}}}}, true},
-		{"missing zone name", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
+		{"missing zone name", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
 			{Arbiter: true},
 			{Name: "b"},
 			{Name: "c"},
 		}}}}}}, true},
-		{"valid stretch cluster", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 3, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
+		{"valid stretch cluster", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 3, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
 			{Name: "a", Arbiter: true},
 			{Name: "b"},
 			{Name: "c"},
 		}}}}}}, false},
-		{"not enough stretch nodes", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 5, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
+		{"not enough stretch nodes", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 5, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
 			{Name: "a", Arbiter: true},
 			{Name: "b"},
 			{Name: "c"},

pkg/operator/ceph/cluster/controller.go

@@ -21,9 +21,7 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
-	"time"
 
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
@@ -41,7 +39,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
+	apituntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -52,8 +50,7 @@ import (
 )
 
 const (
-	controllerName           = "ceph-cluster-controller"
-	detectCephVersionTimeout = 15 * time.Minute
+	controllerName = "ceph-cluster-controller"
 )
 
 const (
@@ -84,35 +81,35 @@ var ControllerTypeMeta = metav1.TypeMeta{
 
 // ClusterController controls an instance of a Rook cluster
 type ClusterController struct {
-	context                 *clusterd.Context
-	volumeAttachment        attachment.Attachment
-	rookImage               string
-	clusterMap              map[string]*cluster
-	operatorConfigCallbacks []func() error
-	addClusterCallbacks     []func() error
-	csiConfigMutex          *sync.Mutex
-	osdChecker              *osd.OSDHealthMonitor
-	client                  client.Client
-	namespacedName          types.NamespacedName
-	recorder                *k8sutil.EventReporter
+	context          *clusterd.Context
+	volumeAttachment attachment.Attachment
+	rookImage        string
+	clusterMap       map[string]*cluster
+	csiConfigMutex   *sync.Mutex
+	osdChecker       *osd.OSDHealthMonitor
+	client           client.Client
+	namespacedName   types.NamespacedName
+	recorder         *k8sutil.EventReporter
+	OpManagerCtx     context.Context
 }
 
 // ReconcileCephCluster reconciles a CephFilesystem object
 type ReconcileCephCluster struct {
 	client            client.Client
-	scheme            *runtime.Scheme
+	scheme            *apituntime.Scheme
 	context           *clusterd.Context
 	clusterController *ClusterController
+	opManagerContext  context.Context
 }
 
 // Add creates a new CephCluster Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController) error {
-	return add(mgr, newReconciler(mgr, ctx, clusterController), ctx)
+func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) error {
+	return add(mgr, newReconciler(mgr, ctx, clusterController, opManagerContext), ctx)
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) reconcile.Reconciler {
 	// add "rook-" prefix to the controller name to make sure it is clear to all reading the events
 	// that they are coming from Rook. The controller name already has context that it is for Ceph
 	// and from the cluster controller.
@@ -123,6 +120,7 @@ func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController
 		scheme:            mgr.GetScheme(),
 		context:           ctx,
 		clusterController: clusterController,
+		opManagerContext:  opManagerContext,
 	}
 }
 
@@ -142,7 +140,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context)
 			},
 		},
 		&handler.EnqueueRequestForObject{},
-		watchControllerPredicate(context))
+		watchControllerPredicate())
 	if err != nil {
 		return err
 	}
@@ -219,10 +217,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context)
 func (r *ReconcileCephCluster) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
 	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
 	reconcileResponse, cephCluster, err := r.reconcile(request)
-	if err != nil && strings.Contains(err.Error(), opcontroller.CancellingOrchestrationMessage) {
-		logger.Infof("Cluster update requested. %s", opcontroller.CancellingOrchestrationMessage)
-		return opcontroller.ImmediateRetryResultNoBackoff, nil
-	}
 
 	return reporting.ReportReconcileResult(logger, r.clusterController.recorder,
 		cephCluster, reconcileResponse, err)
@@ -240,7 +234,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
 
 	// Fetch the cephCluster instance
 	cephCluster := &cephv1.CephCluster{}
-	err := r.client.Get(context.TODO(), request.NamespacedName, cephCluster)
+	err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("cephCluster resource not found. Ignoring since object must be deleted.")
@@ -293,26 +287,20 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
 	doCleanup := true
 
 	// Start cluster clean up only if cleanupPolicy is applied to the ceph cluster
-	stopCleanupCh := make(chan struct{})
+	internalCtx := context.Context(r.opManagerContext)
 	if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.Spec.External.Enable {
 		monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(cephCluster.Namespace)
 		if err != nil {
-			logger.Warningf("failed to get mon secret. Skip cluster cleanup and remove finalizer. %v", err)
+			logger.Warningf("failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err)
 			doCleanup = false
 		}
 
 		if doCleanup {
 			cephHosts, err := r.clusterController.getCephHosts(cephCluster.Namespace)
 			if err != nil {
-				close(stopCleanupCh)
 				return reconcile.Result{}, cephCluster, errors.Wrapf(err, "failed to find valid ceph hosts in the cluster %q", cephCluster.Namespace)
 			}
-			// Go will garbage collect the stopCleanupCh if it is left open once the cluster cleanup
-			// goroutine is no longer running (i.e., referencing the channel)
-			go r.clusterController.startClusterCleanUp(stopCleanupCh, cephCluster, cephHosts, monSecret, clusterFSID)
-		} else {
-			// stop channel not needed if the cleanup goroutine isn't started
-			close(stopCleanupCh)
+			go r.clusterController.startClusterCleanUp(internalCtx, cephCluster, cephHosts, monSecret, clusterFSID)
 		}
 	}
 
@@ -322,7 +310,6 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
 		if err != nil {
 			// If the cluster cannot be deleted, requeue the request for deletion to see if the conditions
 			// will eventually be satisfied such as the volumes being removed
-			close(stopCleanupCh)
 			return response, cephCluster, errors.Wrapf(err, "failed to clean up CephCluster %q", nsName.String())
 		}
 	}
@@ -338,15 +325,13 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
 }
 
 // NewClusterController create controller for watching cluster custom resources created
-func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, operatorConfigCallbacks []func() error, addClusterCallbacks []func() error) *ClusterController {
+func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment) *ClusterController {
 	return &ClusterController{
-		context:                 context,
-		volumeAttachment:        volumeAttachment,
-		rookImage:               rookImage,
-		clusterMap:              make(map[string]*cluster),
-		operatorConfigCallbacks: operatorConfigCallbacks,
-		addClusterCallbacks:     addClusterCallbacks,
-		csiConfigMutex:          &sync.Mutex{},
+		context:          context,
+		volumeAttachment: volumeAttachment,
+		rookImage:        rookImage,
+		clusterMap:       make(map[string]*cluster),
+		csiConfigMutex:   &sync.Mutex{},
 	}
 }
 
@@ -370,20 +355,9 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster,
 	// Set the spec
 	cluster.Spec = &clusterObj.Spec
 
-	// Note that this lock is held through the callback process, as this creates CSI resources, but we must lock in
-	// this scope as the clusterMap is authoritative on cluster count and thus involved in the check for CSI resource
-	// deletion. If we ever add additional callback functions, we should tighten this lock.
-	c.csiConfigMutex.Lock()
 	c.clusterMap[cluster.Namespace] = cluster
 	logger.Infof("reconciling ceph cluster in namespace %q", cluster.Namespace)
 
-	for _, callback := range c.addClusterCallbacks {
-		if err := callback(); err != nil {
-			logger.Errorf("%v", err)
-		}
-	}
-	c.csiConfigMutex.Unlock()
-
 	// Start the main ceph cluster orchestration
 	return c.initializeCluster(cluster)
 }
@@ -400,17 +374,13 @@ func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (r
 	logger.Infof("cleaning up CephCluster %q", nsName)
 
 	if cluster, ok := c.clusterMap[cluster.Namespace]; ok {
-		// if not already stopped, stop clientcontroller and bucketController
-		if !cluster.closedStopCh {
-			close(cluster.stopCh)
-			cluster.closedStopCh = true
-		}
-
+		// We used to stop the bucket controller here but when we get a DELETE event for the CephCluster
+		// we will reload the CRD manager anyway so the bucket controller go routine will be stopped
+		// since the op manager context is cancelled.
 		// close the goroutines watching the health of the cluster (mons, osds, ceph status)
 		for _, daemon := range monitorDaemonList {
-			if monitoring, ok := cluster.monitoringChannels[daemon]; ok && monitoring.monitoringRunning {
-				close(cluster.monitoringChannels[daemon].stopChan)
-				cluster.monitoringChannels[daemon].monitoringRunning = false
+			if monitoring, ok := cluster.monitoringRoutines[daemon]; ok && monitoring.internalCtx.Err() == nil { // if the context hasn't been cancelled
+				cluster.monitoringRoutines[daemon].internalCancel()
 			}
 		}
 	}
@@ -503,8 +473,7 @@ func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephClust
 }
 
 func (c *ClusterController) checkPVPresentInCluster(drivers []string, clusterID string) (bool, error) {
-	ctx := context.TODO()
-	pv, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
+	pv, err := c.context.Clientset.CoreV1().PersistentVolumes().List(c.OpManagerCtx, metav1.ListOptions{})
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to list PV")
 	}

pkg/operator/ceph/cluster/controller_test.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	testop "github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,7 +41,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
-func TestReconcile_DeleteCephCluster(t *testing.T) {
+func TestReconcileDeleteCephCluster(t *testing.T) {
 	ctx := context.TODO()
 	cephNs := "rook-ceph"
 	clusterName := "my-cluster"
@@ -79,8 +78,7 @@ func TestReconcile_DeleteCephCluster(t *testing.T) {
 		clusterdCtx := &clusterd.Context{
 			Clientset: k8sfake.NewSimpleClientset(),
 			// reconcile looks for fake dependencies in the dynamic clientset
-			DynamicClientset:           dynamicfake.NewSimpleDynamicClient(scheme, fakePool),
-			RequestCancelOrchestration: abool.New(),
+			DynamicClientset: dynamicfake.NewSimpleDynamicClient(scheme, fakePool),
 		}
 
 		// set up ClusterController
@@ -90,21 +88,9 @@ func TestReconcile_DeleteCephCluster(t *testing.T) {
 				return &rookalpha.VolumeList{Items: []rookalpha.Volume{}}, nil
 			},
 		}
-		operatorConfigCallbacks := []func() error{
-			func() error {
-				t.Log("test op config callback")
-				return nil
-			},
-		}
-		addCallbacks := []func() error{
-			func() error {
-				t.Log("test success callback")
-				return nil
-			},
-		}
 
 		// create the cluster controller and tell it that the cluster has been deleted
-		controller := NewClusterController(clusterdCtx, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
+		controller := NewClusterController(clusterdCtx, "", volumeAttachmentController)
 		fakeRecorder := record.NewFakeRecorder(5)
 		controller.recorder = k8sutil.NewEventReporter(fakeRecorder)
 
@@ -118,6 +104,7 @@ func TestReconcile_DeleteCephCluster(t *testing.T) {
 			scheme:            scheme,
 			context:           clusterdCtx,
 			clusterController: controller,
+			opManagerContext:  context.TODO(),
 		}
 
 		req := reconcile.Request{NamespacedName: nsName}
@@ -201,20 +188,8 @@ func Test_checkIfVolumesExist(t *testing.T) {
 
 			},
 		}
-		operatorConfigCallbacks := []func() error{
-			func() error {
-				logger.Infof("test success callback")
-				return nil
-			},
-		}
-		addCallbacks := []func() error{
-			func() error {
-				logger.Infof("test success callback")
-				return nil
-			},
-		}
 		// create the cluster controller and tell it that the cluster has been deleted
-		controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
+		controller := NewClusterController(context, "", volumeAttachmentController)
 		clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
 
 		// The test returns a volume on the first call
@@ -244,23 +219,8 @@ func Test_checkIfVolumesExist(t *testing.T) {
 
 			},
 		}
-		operatorConfigCallbacks := []func() error{
-			func() error {
-				logger.Infof("test success callback")
-				return nil
-			},
-		}
-		addCallbacks := []func() error{
-			func() error {
-				logger.Infof("test success callback")
-				os.Setenv("ROOK_ENABLE_FLEX_DRIVER", "true")
-				os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace)
-				defer os.Unsetenv("ROOK_ENABLE_FLEX_DRIVER")
-				return nil
-			},
-		}
 		// create the cluster controller and tell it that the cluster has been deleted
-		controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks)
+		controller := NewClusterController(context, "", volumeAttachmentController)
 		clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
 		assert.NoError(t, controller.checkIfVolumesExist(clusterToDelete))
 

pkg/operator/ceph/cluster/crash/add.go

@@ -17,6 +17,7 @@ limitations under the License.
 package crash
 
 import (
+	"context"
 	"reflect"
 	"strings"
 
@@ -24,6 +25,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -47,16 +49,18 @@ const (
 )
 
 // Add adds a new Controller based on nodedrain.ReconcileNode and registers the relevant watches and handlers
-func Add(mgr manager.Manager, context *clusterd.Context) error {
-	return add(mgr, newReconciler(mgr, context))
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+	return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
 	return &ReconcileNode{
-		client:  mgr.GetClient(),
-		scheme:  mgr.GetScheme(),
-		context: context,
+		client:           mgr.GetClient(),
+		scheme:           mgr.GetScheme(),
+		context:          context,
+		opManagerContext: opManagerContext,
+		opConfig:         opConfig,
 	}
 }
 

pkg/operator/ceph/cluster/crash/crash.go

@@ -17,7 +17,6 @@ limitations under the License.
 package crash
 
 import (
-	"context"
 	"fmt"
 	"path"
 
@@ -123,7 +122,7 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations []
 		return nil
 	}
 
-	return controllerutil.CreateOrUpdate(context.TODO(), r.client, deploy, mutateFunc)
+	return controllerutil.CreateOrUpdate(r.opManagerContext, r.client, deploy, mutateFunc)
 }
 
 // createOrUpdateCephCron is a wrapper around controllerutil.CreateOrUpdate
@@ -164,7 +163,7 @@ func (r *ReconcileNode) createOrUpdateCephCron(cephCluster cephv1.CephCluster, c
 	// minimum k8s version required for v1 cronJob is 'v1.21.0'. Apply v1 if k8s version is at least 'v1.21.0', else apply v1beta1 cronJob.
 	if useCronJobV1 {
 		// delete v1beta1 cronJob if it already exists
-		err := r.client.Delete(context.TODO(), &v1beta1.CronJob{ObjectMeta: objectMeta})
+		err := r.client.Delete(r.opManagerContext, &v1beta1.CronJob{ObjectMeta: objectMeta})
 		if err != nil && !apierrors.IsNotFound(err) {
 			return controllerutil.OperationResultNone, errors.Wrapf(err, "failed to delete CronJob v1Beta1 %q", prunerName)
 		}
@@ -183,7 +182,7 @@ func (r *ReconcileNode) createOrUpdateCephCron(cephCluster cephv1.CephCluster, c
 			return nil
 		}
 
-		return controllerutil.CreateOrUpdate(context.TODO(), r.client, cronJob, mutateFunc)
+		return controllerutil.CreateOrUpdate(r.opManagerContext, r.client, cronJob, mutateFunc)
 	}
 	cronJob := &v1beta1.CronJob{ObjectMeta: objectMeta}
 	err := controllerutil.SetControllerReference(&cephCluster, cronJob, r.scheme)
@@ -200,7 +199,7 @@ func (r *ReconcileNode) createOrUpdateCephCron(cephCluster cephv1.CephCluster, c
 		return nil
 	}
 
-	return controllerutil.CreateOrUpdate(context.TODO(), r.client, cronJob, mutateFunc)
+	return controllerutil.CreateOrUpdate(r.opManagerContext, r.client, cronJob, mutateFunc)
 }
 
 func getCrashDirInitContainer(cephCluster cephv1.CephCluster) corev1.Container {

pkg/operator/ceph/cluster/crash/crash_test.go

@@ -27,7 +27,6 @@ import (
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	v1 "k8s.io/api/batch/v1"
 	"k8s.io/api/batch/v1beta1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -48,9 +47,8 @@ func TestCreateOrUpdateCephCron(t *testing.T) {
 	cephVersion := &cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}
 	ctx := context.TODO()
 	context := &clusterd.Context{
-		Clientset:                  test.New(t, 1),
-		RookClientset:              rookclient.NewSimpleClientset(),
-		RequestCancelOrchestration: abool.New(),
+		Clientset:     test.New(t, 1),
+		RookClientset: rookclient.NewSimpleClientset(),
 	}
 
 	s := scheme.Scheme

pkg/operator/ceph/cluster/crash/keyring.go

@@ -96,7 +96,7 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo
 	// Create Kubernetes Secret
 	err = k.CreateSecret(s)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", crashCollectorSecret, clusterInfo.Namespace)
+		return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
 	}
 
 	logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace)

pkg/operator/ceph/cluster/crash/reconcile.go

@@ -69,9 +69,11 @@ const (
 // ReconcileNode reconciles ReplicaSets
 type ReconcileNode struct {
 	// client can be used to retrieve objects from the APIServer.
-	scheme  *runtime.Scheme
-	client  client.Client
-	context *clusterd.Context
+	scheme           *runtime.Scheme
+	client           client.Client
+	context          *clusterd.Context
+	opManagerContext context.Context
+	opConfig         opcontroller.OperatorConfig
 }
 
 // Reconcile reconciles a node and ensures that it has a crashcollector deployment
@@ -92,7 +94,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
 
 	// get the node object
 	node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: request.Name}}
-	err := r.client.Get(context.TODO(), request.NamespacedName, node)
+	err := r.client.Get(r.opManagerContext, request.NamespacedName, node)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			// if a node is not present, check if there are any crashcollector deployment for that node and delete it.
@@ -130,7 +132,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
 	for namespace, cephPods := range namespaceToPodList {
 		// get dataDirHostPath from the CephCluster
 		cephClusters := &cephv1.CephClusterList{}
-		err := r.client.List(context.TODO(), cephClusters, client.InNamespace(namespace))
+		err := r.client.List(r.opManagerContext, cephClusters, client.InNamespace(namespace))
 		if err != nil {
 			return reconcile.Result{}, errors.Wrapf(err, "could not get cephcluster in namespaces %q", namespace)
 		}
@@ -150,7 +152,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
 			namespaceListOpts := client.InNamespace(request.Namespace)
 
 			// Try to fetch the list of existing deployment and remove them
-			err := r.client.List(context.TODO(), deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName}, namespaceListOpts)
+			err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName}, namespaceListOpts)
 			if err != nil {
 				logger.Errorf("failed to list crash collector deployments, delete it/them manually. %v", err)
 				return reconcile.Result{}, nil
@@ -171,7 +173,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result,
 
 		// checking if secret "rook-ceph-crash-collector-keyring" is present which is required to create crashcollector pods
 		secret := &corev1.Secret{}
-		err = r.client.Get(context.TODO(), types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret)
+		err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret)
 		if err != nil {
 			if kerrors.IsNotFound(err) {
 				logger.Debugf("secret %q not found. retrying in %q. %v", crashCollectorKeyName, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err)
@@ -231,7 +233,7 @@ func (r *ReconcileNode) cephPodList() ([]corev1.Pod, error) {
 
 	for _, app := range cephAppNames {
 		podList := &corev1.PodList{}
-		err := r.client.List(context.TODO(), podList, client.MatchingLabels{k8sutil.AppAttr: app})
+		err := r.client.List(r.opManagerContext, podList, client.MatchingLabels{k8sutil.AppAttr: app})
 		if err != nil {
 			return cephPods, errors.Wrapf(err, "could not list the %q pods", app)
 		}
@@ -245,7 +247,7 @@ func (r *ReconcileNode) cephPodList() ([]corev1.Pod, error) {
 func (r *ReconcileNode) listCrashCollectorAndDelete(nodeName, ns string) error {
 	deploymentList := &appsv1.DeploymentList{}
 	namespaceListOpts := client.InNamespace(ns)
-	err := r.client.List(context.TODO(), deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName, NodeNameLabel: nodeName}, namespaceListOpts)
+	err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName, NodeNameLabel: nodeName}, namespaceListOpts)
 	if err != nil {
 		return errors.Wrap(err, "failed to list crash collector deployments")
 	}
@@ -271,7 +273,7 @@ func (r *ReconcileNode) deleteCrashCollector(deployment appsv1.Deployment) error
 		},
 	}
 
-	err := r.client.Delete(context.TODO(), dep)
+	err := r.client.Delete(r.opManagerContext, dep)
 	if err != nil && !kerrors.IsNotFound(err) {
 		return errors.Wrapf(err, "could not delete crash collector deployment %q", deploymentName)
 	}
@@ -298,7 +300,7 @@ func (r *ReconcileNode) reconcileCrashRetention(namespace string, cephCluster ce
 		// minimum k8s version required for v1 cronJob is 'v1.21.0'. Apply v1 if k8s version is at least 'v1.21.0', else apply v1beta1 cronJob.
 		if useCronJobV1 {
 			// delete v1beta1 cronJob if it already exists
-			err = r.client.Delete(context.TODO(), &v1beta1.CronJob{ObjectMeta: objectMeta})
+			err = r.client.Delete(r.opManagerContext, &v1beta1.CronJob{ObjectMeta: objectMeta})
 			if err != nil && !kerrors.IsNotFound(err) {
 				return errors.Wrapf(err, "failed to delete CronJob v1beta1 %q", prunerName)
 			}
@@ -307,7 +309,7 @@ func (r *ReconcileNode) reconcileCrashRetention(namespace string, cephCluster ce
 			cronJob = &v1beta1.CronJob{ObjectMeta: objectMeta}
 		}
 
-		err := r.client.Delete(context.TODO(), cronJob)
+		err := r.client.Delete(r.opManagerContext, cronJob)
 		if err != nil {
 			if kerrors.IsNotFound(err) {
 				logger.Debug("cronJob resource not found. Ignoring since object must be deleted.")

pkg/operator/ceph/cluster/mgr/config.go

@@ -17,7 +17,6 @@ limitations under the License.
 package mgr
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/rook/rook/pkg/operator/ceph/config"
@@ -57,7 +56,6 @@ func (c *Cluster) dashboardPort() int {
 }
 
 func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
-	ctx := context.TODO()
 	user := fmt.Sprintf("mgr.%s", m.DaemonID)
 	access := []string{"mon", "allow profile mgr", "mds", "allow *", "osd", "allow *"}
 	s := keyring.GetSecretStore(c.context, c.clusterInfo, c.clusterInfo.OwnerInfo)
@@ -68,7 +66,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
 	}
 
 	// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
-	err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(ctx, m.ResourceName, metav1.DeleteOptions{})
+	err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, m.ResourceName, metav1.DeleteOptions{})
 	if err != nil {
 		if errors.IsNotFound(err) {
 			logger.Debugf("legacy mgr key %q is already removed", m.ResourceName)

pkg/operator/ceph/cluster/mgr/dashboard.go

@@ -56,7 +56,6 @@ var (
 )
 
 func (c *Cluster) configureDashboardService(activeDaemon string) error {
-	ctx := context.TODO()
 	dashboardService, err := c.makeDashboardService(AppName, activeDaemon)
 	if err != nil {
 		return err
@@ -68,7 +67,7 @@ func (c *Cluster) configureDashboardService(activeDaemon string) error {
 		}
 	} else {
 		// delete the dashboard service if it exists
-		err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(ctx, dashboardService.Name, metav1.DeleteOptions{})
+		err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, dashboardService.Name, metav1.DeleteOptions{})
 		if err != nil && !kerrors.IsNotFound(err) {
 			return errors.Wrap(err, "failed to delete dashboard service")
 		}
@@ -253,8 +252,7 @@ func (c *Cluster) setLoginCredentials(password string) error {
 }
 
 func (c *Cluster) getOrGenerateDashboardPassword() (string, error) {
-	ctx := context.TODO()
-	secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(ctx, dashboardPasswordName, metav1.GetOptions{})
+	secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, dashboardPasswordName, metav1.GetOptions{})
 	if err == nil {
 		logger.Info("the dashboard secret was already generated")
 		return decodeSecret(secret)
@@ -286,7 +284,7 @@ func (c *Cluster) getOrGenerateDashboardPassword() (string, error) {
 		return "", errors.Wrapf(err, "failed to set owner reference to dashboard secret %q", secret.Name)
 	}
 
-	_, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, secret, metav1.CreateOptions{})
+	_, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, secret, metav1.CreateOptions{})
 	if err != nil {
 		return "", errors.Wrap(err, "failed to save dashboard secret")
 	}

pkg/operator/ceph/cluster/mgr/dashboard_test.go

@@ -115,6 +115,7 @@ func TestStartSecureDashboard(t *testing.T) {
 		Namespace:   "myns",
 		CephVersion: cephver.Nautilus,
 		OwnerInfo:   ownerInfo,
+		Context:     ctx,
 	}
 	c := &Cluster{clusterInfo: clusterInfo, context: &clusterd.Context{Clientset: clientset, Executor: executor},
 		spec: cephv1.ClusterSpec{

pkg/operator/ceph/cluster/mgr/mgr.go

@@ -18,7 +18,6 @@ limitations under the License.
 package mgr
 
 import (
-	"context"
 	"fmt"
 	"path"
 	"strconv"
@@ -112,7 +111,6 @@ func (c *Cluster) getDaemonIDs() []string {
 
 // Start begins the process of running a cluster of Ceph mgrs.
 func (c *Cluster) Start() error {
-	ctx := context.TODO()
 	// Validate pod's memory if specified
 	err := controller.CheckPodMemory(cephv1.ResourcesKeyMgr, cephv1.GetMgrResources(c.spec.Resources), cephMgrPodMinimumMemory)
 	if err != nil {
@@ -124,11 +122,9 @@ func (c *Cluster) Start() error {
 	var deploymentsToWaitFor []*v1.Deployment
 
 	for _, daemonID := range daemonIDs {
-		// Check whether we need to cancel the orchestration
-		if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
-			return err
+		if c.clusterInfo.Context.Err() != nil {
+			return c.clusterInfo.Context.Err()
 		}
-
 		resourceName := fmt.Sprintf("%s-%s", AppName, daemonID)
 		mgrConfig := &mgrConfig{
 			DaemonID:     daemonID,
@@ -155,7 +151,7 @@ func (c *Cluster) Start() error {
 			return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name)
 		}
 
-		newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{})
+		newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, d, metav1.CreateOptions{})
 		if err != nil {
 			if !kerrors.IsAlreadyExists(err) {
 				return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName)
@@ -233,7 +229,7 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
 	// In case the mgr count was reduced, delete the extra mgrs
 	for i := maxMgrCount - 1; i >= len(daemonIDs); i-- {
 		mgrName := fmt.Sprintf("%s-%s", AppName, k8sutil.IndexToName(i))
-		err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), mgrName, metav1.DeleteOptions{})
+		err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrName, metav1.DeleteOptions{})
 		if err == nil {
 			logger.Infof("removed extra mgr %q", mgrName)
 		} else if !kerrors.IsNotFound(err) {
@@ -246,7 +242,7 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) {
 // in the sidecar
 func (c *Cluster) ReconcileActiveMgrServices(daemonNameToUpdate string) error {
 	// If the services are already set to this daemon, no need to attempt to update
-	svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), AppName, metav1.GetOptions{})
+	svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, AppName, metav1.GetOptions{})
 	if err != nil {
 		logger.Errorf("failed to check current mgr service, proceeding to update. %v", err)
 	} else {

pkg/operator/ceph/cluster/mgr/mgr_test.go

@@ -37,7 +37,6 @@ import (
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/tevino/abool"
 	apps "k8s.io/api/apps/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
@@ -74,13 +73,13 @@ func TestStartMgr(t *testing.T) {
 
 	defer os.RemoveAll(configDir)
 	ctx := &clusterd.Context{
-		Executor:                   executor,
-		ConfigDir:                  configDir,
-		Clientset:                  clientset,
-		RequestCancelOrchestration: abool.New(),
-		Client:                     cl}
+		Executor:  executor,
+		ConfigDir: configDir,
+		Clientset: clientset,
+		Client:    cl,
+	}
 	ownerInfo := cephclient.NewMinimumOwnerInfo(t)
-	clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo, CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Build: 5}}
+	clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo, CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Build: 5}, Context: context.TODO()}
 	clusterInfo.SetName("test")
 	clusterSpec := cephv1.ClusterSpec{
 		Annotations:        map[rook.KeyType]rook.Annotations{cephv1.KeyMgr: {"my": "annotation"}},
@@ -207,8 +206,7 @@ func TestMgrSidecarReconcile(t *testing.T) {
 		ConfigDir: configDir,
 		Clientset: clientset,
 	}
-	ownerInfo := cephclient.NewMinimumOwnerInfo(t)
-	clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", OwnerInfo: ownerInfo}
+	clusterInfo := cephclient.AdminClusterInfo("mycluster")
 	clusterInfo.SetName("test")
 	c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo}
 
@@ -280,7 +278,7 @@ func TestConfigureModules(t *testing.T) {
 
 	clientset := testop.New(t, 3)
 	context := &clusterd.Context{Executor: executor, Clientset: clientset}
-	clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
+	clusterInfo := cephclient.AdminClusterInfo("mycluster")
 	c := &Cluster{
 		context:     context,
 		clusterInfo: clusterInfo,

pkg/operator/ceph/cluster/mgr/orchestrator_test.go

@@ -16,6 +16,7 @@ limitations under the License.
 package mgr
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -30,7 +31,6 @@ import (
 
 func TestOrchestratorModules(t *testing.T) {
 	executor := &exectest.MockExecutor{}
-	context := &clusterd.Context{Executor: executor}
 	rookModuleEnabled := false
 	rookBackendSet := false
 	backendErrorCount := 0
@@ -60,7 +60,9 @@ func TestOrchestratorModules(t *testing.T) {
 
 	clusterInfo := &cephclient.ClusterInfo{
 		CephVersion: cephver.Nautilus,
+		Context:     context.TODO(),
 	}
+	context := &clusterd.Context{Executor: executor}
 
 	c := &Cluster{clusterInfo: clusterInfo, context: context}
 	c.exitCode = func(err error) (int, bool) {

pkg/operator/ceph/cluster/mon/config.go

@@ -57,6 +57,10 @@ const (
 	externalConnectionRetry = 60 * time.Second
 )
 
+var (
+	ClusterInfoNoClusterNoSecret = errors.New("not expected to create new cluster info and did not find existing secret")
+)
+
 func (c *Cluster) genMonSharedKeyring() string {
 	return fmt.Sprintf(
 		keyringTemplate,
@@ -77,32 +81,32 @@ func dataDirRelativeHostPath(monName string) string {
 }
 
 // LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID
-func LoadClusterInfo(context *clusterd.Context, namespace string) (*cephclient.ClusterInfo, int, *Mapping, error) {
-	return CreateOrLoadClusterInfo(context, namespace, nil)
+func LoadClusterInfo(ctx *clusterd.Context, context context.Context, namespace string) (*cephclient.ClusterInfo, int, *Mapping, error) {
+	return CreateOrLoadClusterInfo(ctx, context, namespace, nil)
 }
 
 // CreateOrLoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID
-func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, int, *Mapping, error) {
-	ctx := context.TODO()
+func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, context context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, int, *Mapping, error) {
 	var clusterInfo *cephclient.ClusterInfo
 	maxMonID := -1
 	monMapping := &Mapping{
 		Schedule: map[string]*MonScheduleInfo{},
 	}
 
-	secrets, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, AppName, metav1.GetOptions{})
+	secrets, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(context, AppName, metav1.GetOptions{})
 	if err != nil {
 		if !kerrors.IsNotFound(err) {
 			return nil, maxMonID, monMapping, errors.Wrap(err, "failed to get mon secrets")
 		}
 		if ownerInfo == nil {
-			return nil, maxMonID, monMapping, errors.New("not expected to create new cluster info and did not find existing secret")
+			return nil, maxMonID, monMapping, ClusterInfoNoClusterNoSecret
 		}
 
 		clusterInfo, err = createNamedClusterInfo(clusterdContext, namespace)
 		if err != nil {
 			return nil, maxMonID, monMapping, errors.Wrap(err, "failed to create mon secrets")
 		}
+		clusterInfo.Context = context
 
 		err = createClusterAccessSecret(clusterdContext.Clientset, namespace, clusterInfo, ownerInfo)
 		if err != nil {
@@ -113,6 +117,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string
 			Namespace:     namespace,
 			FSID:          string(secrets.Data[fsidSecretNameKey]),
 			MonitorSecret: string(secrets.Data[monSecretNameKey]),
+			Context:       context,
 		}
 		if cephUsername, ok := secrets.Data[cephUsernameKey]; ok {
 			clusterInfo.CephCred.Username = string(cephUsername)
@@ -123,7 +128,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string
 
 			secrets.Data[cephUsernameKey] = []byte(cephclient.AdminUsername)
 			secrets.Data[cephUserSecretKey] = adminSecretKey
-			if _, err = clusterdContext.Clientset.CoreV1().Secrets(namespace).Update(ctx, secrets, metav1.UpdateOptions{}); err != nil {
+			if _, err = clusterdContext.Clientset.CoreV1().Secrets(namespace).Update(context, secrets, metav1.UpdateOptions{}); err != nil {
 				return nil, maxMonID, monMapping, errors.Wrap(err, "failed to update mon secrets")
 			}
 		} else {
@@ -148,7 +153,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string
 
 	// If the admin secret is "admin-secret", look for the deprecated secret that has the external creds
 	if clusterInfo.CephCred.Secret == adminSecretNameKey {
-		secret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, OperatorCreds, metav1.GetOptions{})
+		secret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(context, OperatorCreds, metav1.GetOptions{})
 		if err != nil {
 			return clusterInfo, maxMonID, monMapping, err
 		}
@@ -263,7 +268,6 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string
 }
 
 func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, clusterInfo *cephclient.ClusterInfo, ownerInfo *k8sutil.OwnerInfo) error {
-	ctx := context.TODO()
 	logger.Infof("creating mon secrets for a new cluster")
 	var err error
 
@@ -286,7 +290,7 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string,
 	if err != nil {
 		return errors.Wrapf(err, "failed to set owner reference to mon secret %q", secret.Name)
 	}
-	if _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+	if _, err = clientset.CoreV1().Secrets(namespace).Create(clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil {
 		return errors.Wrap(err, "failed to save mon secrets")
 	}
 
@@ -369,9 +373,9 @@ func ExtractKey(contents string) (string, error) {
 }
 
 // PopulateExternalClusterInfo Add validation in the code to fail if the external cluster has no OSDs keep waiting
-func PopulateExternalClusterInfo(context *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) *cephclient.ClusterInfo {
+func PopulateExternalClusterInfo(context *clusterd.Context, ctx context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) *cephclient.ClusterInfo {
 	for {
-		clusterInfo, _, _, err := LoadClusterInfo(context, namespace)
+		clusterInfo, _, _, err := LoadClusterInfo(context, ctx, namespace)
 		if err != nil {
 			logger.Warningf("waiting for the csi connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
 			logger.Debugf("%v", err)

pkg/operator/ceph/cluster/mon/config_test.go

@@ -57,7 +57,7 @@ func TestCreateClusterSecrets(t *testing.T) {
 	}
 	namespace := "ns"
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
-	info, maxID, mapping, err := CreateOrLoadClusterInfo(context, namespace, ownerInfo)
+	info, maxID, mapping, err := CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
 	assert.NoError(t, err)
 	assert.Equal(t, -1, maxID)
 	require.NotNil(t, info)
@@ -80,7 +80,7 @@ func TestCreateClusterSecrets(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Check that the cluster info can now be loaded
-	info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo)
+	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
 	assert.NoError(t, err)
 	assert.Equal(t, "client.admin", info.CephCred.Username)
 	assert.Equal(t, adminSecret, info.CephCred.Secret)
@@ -89,7 +89,7 @@ func TestCreateClusterSecrets(t *testing.T) {
 	secret.Data[adminSecretNameKey] = []byte(adminSecretNameKey)
 	_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
 	assert.NoError(t, err)
-	_, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo)
+	_, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
 	assert.Error(t, err)
 
 	// Load the external cluster with the legacy external creds
@@ -100,7 +100,7 @@ func TestCreateClusterSecrets(t *testing.T) {
 	}
 	_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
 	assert.NoError(t, err)
-	info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo)
+	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
 	assert.NoError(t, err)
 	assert.Equal(t, "testid", info.CephCred.Username)
 	assert.Equal(t, "testkey", info.CephCred.Secret)

pkg/operator/ceph/cluster/mon/health.go

@@ -109,14 +109,14 @@ func NewHealthChecker(monCluster *Cluster) *HealthChecker {
 }
 
 // Check periodically checks the health of the monitors
-func (hc *HealthChecker) Check(stopCh chan struct{}) {
+func (hc *HealthChecker) Check(context context.Context) {
 	for {
 		// Update Mon Timeout with CR details
 		updateMonTimeout(hc.monCluster)
 		// Update Mon Interval with CR details
 		updateMonInterval(hc.monCluster, hc)
 		select {
-		case <-stopCh:
+		case <-context.Done():
 			logger.Infof("stopping monitoring of mons in namespace %q", hc.monCluster.Namespace)
 			return
 
@@ -351,7 +351,6 @@ func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool {
 }
 
 func (c *Cluster) removeOrphanMonResources() {
-	ctx := context.TODO()
 	if c.spec.Mon.VolumeClaimTemplate == nil {
 		logger.Debug("skipping check for orphaned mon pvcs since using the host path")
 		return
@@ -360,7 +359,7 @@ func (c *Cluster) removeOrphanMonResources() {
 	logger.Info("checking for orphaned mon resources")
 
 	opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)}
-	pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(ctx, opts)
+	pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(c.ClusterInfo.Context, opts)
 	if err != nil {
 		logger.Infof("failed to check for orphaned mon pvcs. %v", err)
 		return
@@ -369,7 +368,7 @@ func (c *Cluster) removeOrphanMonResources() {
 	for _, pvc := range pvcs.Items {
 		logger.Debugf("checking if pvc %q is orphaned", pvc.Name)
 
-		_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{})
+		_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, pvc.Name, metav1.GetOptions{})
 		if err == nil {
 			logger.Debugf("skipping pvc removal since the mon daemon %q still requires it", pvc.Name)
 			continue
@@ -383,7 +382,7 @@ func (c *Cluster) removeOrphanMonResources() {
 		var gracePeriod int64 // delete immediately
 		propagation := metav1.DeletePropagationForeground
 		options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
-		err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(ctx, pvc.Name, *options)
+		err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, pvc.Name, *options)
 		if err != nil {
 			logger.Warningf("failed to delete orphaned monitor pvc %q. %v", pvc.Name, err)
 		}
@@ -391,9 +390,8 @@ func (c *Cluster) removeOrphanMonResources() {
 }
 
 func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
-	ctx := context.TODO()
 	// get the existing deployment
-	d, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, resourceName(name), metav1.GetOptions{})
+	d, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, resourceName(name), metav1.GetOptions{})
 	if err != nil {
 		return errors.Wrapf(err, "failed to get mon %q", name)
 	}
@@ -408,7 +406,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
 
 	// update the deployment
 	logger.Infof("scaling the mon %q deployment to replica %d", name, desiredReplicas)
-	_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(ctx, d, metav1.UpdateOptions{})
+	_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(c.ClusterInfo.Context, d, metav1.UpdateOptions{})
 	if err != nil {
 		return errors.Wrapf(err, "failed to update mon %q replicas from %d to %d", name, originalReplicas, desiredReplicas)
 	}
@@ -493,7 +491,6 @@ func (c *Cluster) failoverMon(name string) error {
 
 // make a best effort to remove the mon and all its resources
 func (c *Cluster) removeMon(daemonName string) error {
-	ctx := context.TODO()
 	logger.Infof("ensuring removal of unhealthy monitor %s", daemonName)
 
 	resourceName := resourceName(daemonName)
@@ -502,7 +499,7 @@ func (c *Cluster) removeMon(daemonName string) error {
 	var gracePeriod int64
 	propagation := metav1.DeletePropagationForeground
 	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
-	if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(ctx, resourceName, *options); err != nil {
+	if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Infof("dead mon %s was already gone", resourceName)
 		} else {
@@ -519,7 +516,7 @@ func (c *Cluster) removeMon(daemonName string) error {
 	delete(c.mapping.Schedule, daemonName)
 
 	// Remove the service endpoint
-	if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(ctx, resourceName, *options); err != nil {
+	if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Infof("dead mon service %s was already gone", resourceName)
 		} else {
@@ -528,7 +525,7 @@ func (c *Cluster) removeMon(daemonName string) error {
 	}
 
 	// Remove the PVC backing the mon if it existed
-	if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(ctx, resourceName, metav1.DeleteOptions{}); err != nil {
+	if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, metav1.DeleteOptions{}); err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Infof("mon pvc did not exist %q", resourceName)
 		} else {
@@ -666,7 +663,7 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error {
 
 	// Get all the mon pods
 	label := fmt.Sprintf("app=%s", AppName)
-	pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: label})
+	pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: label})
 	if err != nil {
 		return errors.Wrap(err, "failed to list mon pods")
 	}

pkg/operator/ceph/cluster/mon/health_test.go

@@ -36,7 +36,6 @@ import (
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/tevino/abool"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -61,10 +60,9 @@ func TestCheckHealth(t *testing.T) {
 	configDir, _ := ioutil.TempDir("", "")
 	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
-		Clientset:                  clientset,
-		ConfigDir:                  configDir,
-		Executor:                   executor,
-		RequestCancelOrchestration: abool.New(),
+		Clientset: clientset,
+		ConfigDir: configDir,
+		Executor:  executor,
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
 	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
@@ -94,6 +92,7 @@ func TestCheckHealth(t *testing.T) {
 		return SchedulingResult{Node: node}, nil
 	}
 
+	c.ClusterInfo.Context = ctx
 	err = c.checkHealth()
 	assert.Nil(t, err)
 	logger.Infof("mons after checkHealth: %v", c.ClusterInfo.Monitors)
@@ -165,7 +164,7 @@ func TestEvictMonOnSameNode(t *testing.T) {
 			return "{\"key\":\"mysecurekey\"}", nil
 		},
 	}
-	context := &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Executor: executor, RequestCancelOrchestration: abool.New()}
+	context := &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Executor: executor}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
 	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
 	setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 0}, "myversion")
@@ -195,6 +194,7 @@ func TestEvictMonOnSameNode(t *testing.T) {
 	assert.Equal(t, 2, c.maxMonID)
 
 	// Should evict either mon a or mon c since they are on the same node and failover to mon d
+	c.ClusterInfo.Context = ctx
 	err = c.evictMonIfMultipleOnSameNode()
 	assert.NoError(t, err)
 	_, err = clientset.AppsV1().Deployments(c.Namespace).Get(ctx, "rook-ceph-mon-d", metav1.GetOptions{})
@@ -267,10 +267,9 @@ func TestCheckHealthNotFound(t *testing.T) {
 	configDir, _ := ioutil.TempDir("", "")
 	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
-		Clientset:                  clientset,
-		ConfigDir:                  configDir,
-		Executor:                   executor,
-		RequestCancelOrchestration: abool.New(),
+		Clientset: clientset,
+		ConfigDir: configDir,
+		Executor:  executor,
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
 	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
@@ -330,10 +329,9 @@ func TestAddRemoveMons(t *testing.T) {
 	configDir, _ := ioutil.TempDir("", "")
 	defer os.RemoveAll(configDir)
 	context := &clusterd.Context{
-		Clientset:                  clientset,
-		ConfigDir:                  configDir,
-		Executor:                   executor,
-		RequestCancelOrchestration: abool.New(),
+		Clientset: clientset,
+		ConfigDir: configDir,
+		Executor:  executor,
 	}
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
 	c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})

pkg/operator/ceph/cluster/mon/mon.go

@@ -20,7 +20,6 @@ limitations under the License.
 package mon
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
@@ -181,13 +180,15 @@ func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, o
 
 // Start begins the process of running a cluster of Ceph mons.
 func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookVersion string, cephVersion cephver.CephVersion, spec cephv1.ClusterSpec) (*cephclient.ClusterInfo, error) {
-
 	// Only one goroutine can orchestrate the mons at a time
 	c.acquireOrchestrationLock()
 	defer c.releaseOrchestrationLock()
 
 	clusterInfo.OwnerInfo = c.ownerInfo
 	c.ClusterInfo = clusterInfo
+	if c.ClusterInfo.Context == nil {
+		panic("nil context")
+	}
 	c.rookVersion = rookVersion
 	c.spec = spec
 
@@ -199,7 +200,7 @@ func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookVersion string,
 	// Validate pod's memory if specified
 	err := controller.CheckPodMemory(cephv1.ResourcesKeyMon, cephv1.GetMonResources(c.spec.Resources), cephMonPodMinimumMemory)
 	if err != nil {
-		return nil, errors.Wrap(err, "error checking pod memory")
+		return nil, errors.Wrap(err, "failed to check pod memory")
 	}
 
 	logger.Infof("start running mons")
@@ -246,11 +247,9 @@ func (c *Cluster) startMons(targetCount int) error {
 	if existingCount < len(mons) {
 		// Start the new mons one at a time
 		for i := existingCount; i < targetCount; i++ {
-			// Check whether we need to cancel the orchestration
-			if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
-				return err
+			if c.ClusterInfo.Context.Err() != nil {
+				return c.ClusterInfo.Context.Err()
 			}
-
 			if err := c.ensureMonsRunning(mons, i, targetCount, true); err != nil {
 				return err
 			}
@@ -471,14 +470,16 @@ func (c *Cluster) ensureMonsRunning(mons []*monConfig, i, targetCount int, requi
 func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion) error {
 	var err error
 
+	context := c.ClusterInfo.Context
 	// get the cluster info from secret
-	c.ClusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, c.ownerInfo)
+	c.ClusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, context, c.Namespace, c.ownerInfo)
 	if err != nil {
 		return errors.Wrap(err, "failed to get cluster info")
 	}
 
 	c.ClusterInfo.CephVersion = cephVersion
 	c.ClusterInfo.OwnerInfo = c.ownerInfo
+	c.ClusterInfo.Context = context
 
 	// save cluster monitor config
 	if err = c.saveMonConfig(); err != nil {
@@ -594,7 +595,6 @@ func resourceName(name string) string {
 // scheduleMonitor selects a node for a monitor deployment.
 // see startMon() and design/ceph/ceph-mon-pv.md for additional details.
 func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
-	ctx := context.TODO()
 	// build the canary deployment.
 	d, err := c.makeDeployment(mon, true)
 	if err != nil {
@@ -631,17 +631,17 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
 		// to the real monitor deployment.
 		pvc, err := c.makeDeploymentPVC(mon, true)
 		if err != nil {
-			return nil, errors.Wrapf(err, "sched-mon: failed to make monitor %s pvc", d.Name)
+			return nil, errors.Wrapf(err, "failed to make monitor %s pvc", d.Name)
 		}
 
-		_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(ctx, pvc, metav1.CreateOptions{})
+		_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{})
 		if err == nil {
-			logger.Infof("sched-mon: created canary monitor %s pvc %s", d.Name, pvc.Name)
+			logger.Infof("created canary monitor %s pvc %s", d.Name, pvc.Name)
 		} else {
 			if kerrors.IsAlreadyExists(err) {
-				logger.Debugf("sched-mon: creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
+				logger.Debugf("creating mon %s pvc %s: already exists.", d.Name, pvc.Name)
 			} else {
-				return nil, errors.Wrapf(err, "sched-mon: error creating mon %s pvc %s", d.Name, pvc.Name)
+				return nil, errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name)
 			}
 		}
 
@@ -654,25 +654,28 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) {
 	// already exists it may have been scheduled with a different crd config.
 	createdDeployment := false
 	for i := 0; i < canaryRetries; i++ {
-		_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(ctx, d, metav1.CreateOptions{})
+		if c.ClusterInfo.Context.Err() != nil {
+			return nil, c.ClusterInfo.Context.Err()
+		}
+		_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{})
 		if err == nil {
 			createdDeployment = true
-			logger.Infof("sched-mon: created canary deployment %s", d.Name)
+			logger.Infof("created canary deployment %s", d.Name)
 			break
 		} else if kerrors.IsAlreadyExists(err) {
 			if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, d.Name); err != nil {
-				return nil, errors.Wrapf(err, "sched-mon: error deleting canary deployment %s", d.Name)
+				return nil, errors.Wrapf(err, "failed to delete canary deployment %s", d.Name)
 			}
-			logger.Infof("sched-mon: deleted existing canary deployment %s", d.Name)
+			logger.Infof("deleted existing canary deployment %s", d.Name)
 			time.Sleep(time.Second * canaryRetryDelaySeconds)
 		} else {
-			return nil, errors.Wrapf(err, "sched-mon: error creating canary monitor deployment %s", d.Name)
+			return nil, errors.Wrapf(err, "failed to create canary monitor deployment %s", d.Name)
 		}
 	}
 
 	// failed after retrying
 	if !createdDeployment {
-		return nil, errors.Errorf("sched-mon: failed to create canary deployment %s", d.Name)
+		return nil, errors.Errorf("failed to create canary deployment %s", d.Name)
 	}
 
 	// caller should arrange for the deployment to be removed
@@ -696,17 +699,14 @@ func (c *Cluster) getMonPlacement(zone string) cephv1.Placement {
 }
 
 func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingResult, error) {
-	ctx := context.TODO()
 	// target node decision, and deployment/pvc to cleanup
 	result := SchedulingResult{}
 
 	// wait for the scheduler to make a placement decision
 	for i := 0; i < canaryRetries; i++ {
-		// Check whether we need to cancel the orchestration
-		if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
-			return result, err
+		if c.ClusterInfo.Context.Err() != nil {
+			return result, c.ClusterInfo.Context.Err()
 		}
-
 		if i > 0 {
 			time.Sleep(time.Second * canaryRetryDelaySeconds)
 		}
@@ -715,42 +715,45 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes
 			LabelSelector: labels.Set(d.Spec.Selector.MatchLabels).String(),
 		}
 
-		pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(ctx, listOptions)
+		pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, listOptions)
 		if err != nil {
-			return result, errors.Wrapf(err, "sched-mon: error listing canary pods %s", d.Name)
+			return result, errors.Wrapf(err, "failed to list canary pods %s", d.Name)
 		}
 
 		if len(pods.Items) == 0 {
-			logger.Infof("sched-mon: waiting for canary pod creation %s", d.Name)
+			logger.Infof("waiting for canary pod creation %s", d.Name)
 			continue
 		}
 
 		pod := pods.Items[0]
 		if pod.Spec.NodeName == "" {
-			logger.Debugf("sched-mon: monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
+			logger.Debugf("monitor %s canary pod %s not yet scheduled", d.Name, pod.Name)
 			continue
 		}
 
-		node, err := c.context.Clientset.CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, metav1.GetOptions{})
+		node, err := c.context.Clientset.CoreV1().Nodes().Get(c.ClusterInfo.Context, pod.Spec.NodeName, metav1.GetOptions{})
 		if err != nil {
-			return result, errors.Wrapf(err, "sched-mon: error getting node %s", pod.Spec.NodeName)
+			return result, errors.Wrapf(err, "failed to get node %s", pod.Spec.NodeName)
 		}
 
 		result.Node = node
-		logger.Infof("sched-mon: canary monitor deployment %s scheduled to %s", d.Name, node.Name)
+		logger.Infof("canary monitor deployment %s scheduled to %s", d.Name, node.Name)
 		return result, nil
 	}
 
-	return result, errors.New("sched-mon: canary pod scheduling failed retries")
+	return result, errors.New("failed to schedule canary pod(s)")
 }
 
 func (c *Cluster) initMonIPs(mons []*monConfig) error {
 	for _, m := range mons {
+		if c.ClusterInfo.Context.Err() != nil {
+			return c.ClusterInfo.Context.Err()
+		}
 		if c.spec.Network.IsHost() {
 			logger.Infof("setting mon endpoints for hostnetwork mode")
 			node, ok := c.mapping.Schedule[m.DaemonName]
 			if !ok || node == nil {
-				return errors.Errorf("node for mon %q doesn't exist in assignment map", m.DaemonName)
+				return errors.Errorf("failed to found node for mon %q in assignment map", m.DaemonName)
 			}
 			m.PublicIP = node.Address
 		} else {
@@ -769,7 +772,6 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error {
 // Delete mon canary deployments (and associated PVCs) using deployment labels
 // to select this kind of temporary deployments
 func (c *Cluster) removeCanaryDeployments() {
-	ctx := context.TODO()
 	canaryDeployments, err := k8sutil.GetDeployments(c.context.Clientset, c.Namespace, "app=rook-ceph-mon,mon_canary=true")
 	if err != nil {
 		logger.Warningf("failed to get the list of monitor canary deployments. %v", err)
@@ -782,7 +784,7 @@ func (c *Cluster) removeCanaryDeployments() {
 		var gracePeriod int64
 		propagation := metav1.DeletePropagationForeground
 		options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
-		if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(ctx, canary.Name, *options); err != nil {
+		if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, canary.Name, *options); err != nil {
 			logger.Warningf("failed to delete canary monitor deployment %q. %v", canary.Name, err)
 		}
 	}
@@ -805,14 +807,12 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 	// enforced using a node selector, or (2) configuration permits k8s to handle
 	// scheduling for the monitor.
 	for _, mon := range mons {
-		// Check whether we need to cancel the orchestration
-		if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
-			return err
+		if c.ClusterInfo.Context.Err() != nil {
+			return c.ClusterInfo.Context.Err()
 		}
-
 		// scheduling for this monitor has already been completed
 		if _, ok := c.mapping.Schedule[mon.DaemonName]; ok {
-			logger.Debugf("assignmon: mon %s already scheduled", mon.DaemonName)
+			logger.Debugf("mon %s already scheduled", mon.DaemonName)
 			continue
 		}
 
@@ -822,7 +822,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 		// before a decision is stored in the node mapping.
 		deployment, err := scheduleMonitor(c, mon)
 		if err != nil {
-			return errors.Wrap(err, "assignmon: error scheduling monitor")
+			return errors.Wrap(err, "failed to schedule monitor")
 		}
 
 		// start waiting for the deployment
@@ -841,7 +841,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 
 			nodeChoice := result.Node
 			if nodeChoice == nil {
-				logger.Errorf("assignmon: could not schedule monitor %q", mon.DaemonName)
+				logger.Errorf("failed to schedule monitor %q", mon.DaemonName)
 				failedMonSchedule = true
 				return
 			}
@@ -851,15 +851,15 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 			// directly to a node selector on the monitor pod.
 			var schedule *MonScheduleInfo
 			if c.spec.Network.IsHost() || c.monVolumeClaimTemplate(mon) == nil {
-				logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
+				logger.Infof("mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name)
 				schedule, err = getNodeInfoFromNode(*nodeChoice)
 				if err != nil {
-					logger.Errorf("assignmon: couldn't get node info for node %q. %v", nodeChoice.Name, err)
+					logger.Errorf("failed to get node info for node %q. %v", nodeChoice.Name, err)
 					failedMonSchedule = true
 					return
 				}
 			} else {
-				logger.Infof("assignmon: mon %q placement using native scheduler", mon.DaemonName)
+				logger.Infof("mon %q placement using native scheduler", mon.DaemonName)
 			}
 
 			if c.spec.IsStretchCluster() {
@@ -882,7 +882,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error {
 		return errors.New("failed to schedule mons")
 	}
 
-	logger.Debug("assignmons: mons have been scheduled")
+	logger.Debug("mons have been scheduled")
 	return nil
 }
 
@@ -906,7 +906,6 @@ func (c *Cluster) monVolumeClaimTemplate(mon *monConfig) *v1.PersistentVolumeCla
 }
 
 func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) error {
-	ctx := context.TODO()
 	if len(mons) == 0 {
 		return errors.New("cannot start 0 mons")
 	}
@@ -916,7 +915,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e
 	// 1) New clusters where we are starting one deployment at a time. We only need to check for quorum once when we add a new mon.
 	// 2) Clusters being restored where no mon deployments are running. We need to start all the deployments before checking quorum.
 	onlyCheckQuorumOnce := false
-	deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)})
+	deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Infof("0 of %d expected mon deployments exist. creating new deployment(s).", len(mons))
@@ -1029,7 +1028,6 @@ func (c *Cluster) saveMonConfig() error {
 }
 
 func (c *Cluster) persistExpectedMonDaemons() error {
-	ctx := context.TODO()
 	configMap := &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      EndpointConfigMapName,
@@ -1068,13 +1066,13 @@ func (c *Cluster) persistExpectedMonDaemons() error {
 		csi.ConfigKey: csiConfigValue,
 	}
 
-	if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil {
+	if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, configMap, metav1.CreateOptions{}); err != nil {
 		if !kerrors.IsAlreadyExists(err) {
 			return errors.Wrap(err, "failed to create mon endpoint config map")
 		}
 
 		logger.Debugf("updating config map %s that already exists", configMap.Name)
-		if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(ctx, configMap, metav1.UpdateOptions{}); err != nil {
+		if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil {
 			return errors.Wrap(err, "failed to update mon endpoint config map")
 		}
 	}
@@ -1083,7 +1081,7 @@ func (c *Cluster) persistExpectedMonDaemons() error {
 }
 
 func (c *Cluster) getStoredMaxMonID() (string, error) {
-	configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
+	configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{})
 	if err != nil && !kerrors.IsNotFound(err) {
 		return "", errors.Wrap(err, "could not load maxMonId")
 	}
@@ -1106,7 +1104,7 @@ func (c *Cluster) commitMaxMonID(monName string) error {
 		return errors.Wrapf(err, "invalid mon name %q", monName)
 	}
 
-	configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
+	configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to find existing mon endpoint config map")
 	}
@@ -1125,7 +1123,7 @@ func (c *Cluster) commitMaxMonID(monName string) error {
 	logger.Infof("updating maxMonID from %d to %d after committing mon %q", existingMax, committedMonID, monName)
 	configmap.Data[MaxMonIDKey] = strconv.Itoa(committedMonID)
 
-	if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(context.TODO(), configmap, metav1.UpdateOptions{}); err != nil {
+	if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil {
 		return errors.Wrap(err, "failed to update mon endpoint config map for the maxMonID")
 	}
 	return nil
@@ -1141,7 +1139,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
 			return errors.Wrapf(err, "failed to make mon %q pvc", d.Name)
 		}
 
-		existingPvc, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Get(context.TODO(), m.ResourceName, metav1.GetOptions{})
+		existingPvc, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
 		if err != nil {
 			return errors.Wrapf(err, "failed to fetch pvc for mon %q", m.ResourceName)
 		}
@@ -1190,7 +1188,6 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error {
 //           - if PVC      -> remove node selector, if present
 //
 func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
-	ctx := context.TODO()
 	// check if the monitor deployment already exists. if the deployment does
 	// exist, also determine if it using pvc storage.
 	pvcExists := false
@@ -1207,7 +1204,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
 		return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name)
 	}
 
-	existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, d.Name, metav1.GetOptions{})
+	existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, d.Name, metav1.GetOptions{})
 	if err == nil {
 		deploymentExists = true
 		pvcExists = controller.DaemonVolumesContainsPVC(existingDeployment.Spec.Template.Spec.Volumes)
@@ -1262,7 +1259,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
 		if err != nil {
 			return errors.Wrapf(err, "failed to make mon %s pvc", d.Name)
 		}
-		_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(ctx, pvc, metav1.CreateOptions{})
+		_, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{})
 		if err != nil {
 			if kerrors.IsAlreadyExists(err) {
 				logger.Debugf("cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name)
@@ -1286,7 +1283,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error {
 		map[string]string{k8sutil.AppAttr: AppName}, nodeSelector)
 
 	logger.Debugf("Starting mon: %+v", d.Name)
-	_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(ctx, d, metav1.CreateOptions{})
+	_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{})
 	if err != nil {
 		return errors.Wrapf(err, "failed to create mon deployment %s", d.Name)
 	}

pkg/operator/ceph/cluster/mon/mon_test.go

@@ -29,6 +29,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
@@ -40,7 +41,6 @@ import (
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -94,10 +94,9 @@ func newTestStartClusterWithQuorumResponse(t *testing.T, namespace string, monRe
 		},
 	}
 	return &clusterd.Context{
-		Clientset:                  clientset,
-		Executor:                   executor,
-		ConfigDir:                  configDir,
-		RequestCancelOrchestration: abool.New(),
+		Clientset: clientset,
+		Executor:  executor,
+		ConfigDir: configDir,
 	}, nil
 }
 
@@ -140,7 +139,6 @@ func TestResourceName(t *testing.T) {
 }
 
 func TestStartMonDeployment(t *testing.T) {
-	ctx := context.TODO()
 	namespace := "ns"
 	context, err := newTestStartCluster(t, namespace)
 	assert.NoError(t, err)
@@ -151,7 +149,7 @@ func TestStartMonDeployment(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{Name: EndpointConfigMapName},
 		Data:       map[string]string{"maxMonId": "1"},
 	}
-	_, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(ctx, cm, metav1.CreateOptions{})
+	_, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, cm, metav1.CreateOptions{})
 	assert.NoError(t, err)
 
 	// Start mon a on a specific node since there is no volumeClaimTemplate
@@ -159,7 +157,7 @@ func TestStartMonDeployment(t *testing.T) {
 	schedule := &MonScheduleInfo{Hostname: "host-a", Zone: "zonea"}
 	err = c.startMon(m, schedule)
 	assert.NoError(t, err)
-	deployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, m.ResourceName, metav1.GetOptions{})
+	deployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
 	assert.NoError(t, err)
 	assert.Equal(t, schedule.Hostname, deployment.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"])
 
@@ -169,14 +167,15 @@ func TestStartMonDeployment(t *testing.T) {
 	c.spec.Mon.VolumeClaimTemplate = &v1.PersistentVolumeClaim{}
 	err = c.startMon(m, schedule)
 	assert.NoError(t, err)
-	deployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, m.ResourceName, metav1.GetOptions{})
+	deployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{})
 	assert.NoError(t, err)
 	// no node selector when there is a volumeClaimTemplate and the mon is assigned to a zone
 	assert.Equal(t, 0, len(deployment.Spec.Template.Spec.NodeSelector))
 }
 
 func TestStartMonPods(t *testing.T) {
-	ctx := context.TODO()
+	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
+	os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
 	namespace := "ns"
 	context, err := newTestStartCluster(t, namespace)
 	assert.NoError(t, err)
@@ -187,17 +186,16 @@ func TestStartMonPods(t *testing.T) {
 	_, err = c.Start(c.ClusterInfo, c.rookVersion, cephver.Nautilus, c.spec)
 	assert.NoError(t, err)
 
-	validateStart(ctx, t, c)
+	validateStart(t, c)
 
-	// starting again should be a no-op, but still results in an error
+	// starting again should be a no-op
 	_, err = c.Start(c.ClusterInfo, c.rookVersion, cephver.Nautilus, c.spec)
 	assert.NoError(t, err)
 
-	validateStart(ctx, t, c)
+	validateStart(t, c)
 }
 
 func TestOperatorRestart(t *testing.T) {
-	ctx := context.TODO()
 	namespace := "ns"
 	context, err := newTestStartCluster(t, namespace)
 	assert.NoError(t, err)
@@ -209,7 +207,7 @@ func TestOperatorRestart(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, info.IsInitialized(true))
 
-	validateStart(ctx, t, c)
+	validateStart(t, c)
 
 	c = newCluster(context, namespace, true, v1.ResourceRequirements{})
 	c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
@@ -219,12 +217,11 @@ func TestOperatorRestart(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, info.IsInitialized(true))
 
-	validateStart(ctx, t, c)
+	validateStart(t, c)
 }
 
 // safety check that if hostNetwork is used no changes occur on an operator restart
 func TestOperatorRestartHostNetwork(t *testing.T) {
-	ctx := context.TODO()
 	namespace := "ns"
 	context, err := newTestStartCluster(t, namespace)
 	assert.NoError(t, err)
@@ -238,7 +235,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, info.IsInitialized(true))
 
-	validateStart(ctx, t, c)
+	validateStart(t, c)
 
 	// cluster with host networking
 	c = newCluster(context, namespace, false, v1.ResourceRequirements{})
@@ -250,16 +247,16 @@ func TestOperatorRestartHostNetwork(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, info.IsInitialized(true), info)
 
-	validateStart(ctx, t, c)
+	validateStart(t, c)
 }
 
-func validateStart(ctx context.Context, t *testing.T, c *Cluster) {
-	s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(ctx, AppName, metav1.GetOptions{})
+func validateStart(t *testing.T, c *Cluster) {
+	s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{})
 	assert.NoError(t, err) // there shouldn't be an error due the secret existing
 	assert.Equal(t, 4, len(s.Data))
 
 	// there is only one pod created. the other two won't be created since the first one doesn't start
-	_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, "rook-ceph-mon-a", metav1.GetOptions{})
+	_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, "rook-ceph-mon-a", metav1.GetOptions{})
 	assert.NoError(t, err)
 }
 
@@ -343,6 +340,7 @@ func TestMaxMonID(t *testing.T) {
 	defer os.RemoveAll(configDir)
 	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
 	c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{})
+	c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
 
 	// when the configmap is not found, the maxMonID is -1
 	maxMonID, err := c.getStoredMaxMonID()
@@ -437,7 +435,7 @@ func TestWaitForQuorum(t *testing.T) {
 	assert.NoError(t, err)
 	requireAllInQuorum := false
 	expectedMons := []string{"a"}
-	clusterInfo := &cephclient.ClusterInfo{Namespace: namespace}
+	clusterInfo := cephclient.AdminClusterInfo("mycluster")
 	err = waitForQuorumWithMons(context, clusterInfo, expectedMons, 0, requireAllInQuorum)
 	assert.NoError(t, err)
 }

pkg/operator/ceph/cluster/mon/service.go

@@ -17,7 +17,6 @@ limitations under the License.
 package mon
 
 import (
-	"context"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -29,7 +28,6 @@ import (
 )
 
 func (c *Cluster) createService(mon *monConfig) (string, error) {
-	ctx := context.TODO()
 	svcDef := &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      mon.ResourceName,
@@ -62,7 +60,7 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
 	// For example, in disaster recovery the service might have been deleted accidentally, but we have the
 	// expected endpoint from the mon configmap.
 	if mon.PublicIP != "" {
-		_, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(ctx, svcDef.Name, metav1.GetOptions{})
+		_, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(c.ClusterInfo.Context, svcDef.Name, metav1.GetOptions{})
 		if err != nil && kerrors.IsNotFound(err) {
 			logger.Infof("ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP)
 			svcDef.Spec.ClusterIP = mon.PublicIP

pkg/operator/ceph/cluster/mon/service_test.go

@@ -23,6 +23,7 @@ import (
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
@@ -33,6 +34,7 @@ func TestCreateService(t *testing.T) {
 	ctx := context.TODO()
 	clientset := test.New(t, 1)
 	c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{})
+	c.ClusterInfo = client.AdminClusterInfo("rook-ceph")
 	m := &monConfig{ResourceName: "rook-ceph-mon-b", DaemonName: "b"}
 	clusterIP, err := c.createService(m)
 	assert.NoError(t, err)

pkg/operator/ceph/cluster/monitoring.go

@@ -18,11 +18,12 @@ limitations under the License.
 package cluster
 
 import (
+	"context"
+
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
-	"github.com/rook/rook/pkg/operator/ceph/object/bucket"
 )
 
 var (
@@ -30,37 +31,27 @@ var (
 )
 
 func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInfo *cephclient.ClusterInfo) {
-	var isDisabled bool
-
+	var isEnabled bool
 	for _, daemon := range monitorDaemonList {
 		// Is the monitoring enabled for that daemon?
-		isDisabled = isMonitoringDisabled(daemon, cluster.Spec)
-		if health, ok := cluster.monitoringChannels[daemon]; ok {
-			if health.monitoringRunning {
-				// If the goroutine was running but the CR was updated to stop the monitoring we need to close the channel
-				if isDisabled {
-					// close the channel so the goroutine can stop
-					close(cluster.monitoringChannels[daemon].stopChan)
-					// Set monitoring to false since it's not running anymore
-					cluster.monitoringChannels[daemon].monitoringRunning = false
-				} else {
-					logger.Debugf("ceph %s health go routine is already running for cluster %q", daemon, cluster.Namespace)
-				}
-			} else {
-				// if not already running and not disabled, we run it
-				if !isDisabled {
-					// Run the go routine
-					c.startMonitoringCheck(cluster, clusterInfo, daemon)
-
-					// Set the flag to indicate monitoring is running
-					cluster.monitoringChannels[daemon].monitoringRunning = true
+		isEnabled = isMonitoringEnabled(daemon, cluster.Spec)
+		if health, ok := cluster.monitoringRoutines[daemon]; ok {
+			// If the context Err() is nil this means it hasn't been cancelled yet
+			if health.internalCtx.Err() == nil {
+				logger.Debugf("monitoring routine for %q is already running", daemon)
+				if !isEnabled {
+					cluster.monitoringRoutines[daemon].internalCancel()
 				}
 			}
 		} else {
-			if !isDisabled {
-				cluster.monitoringChannels[daemon] = &clusterHealth{
-					stopChan:          make(chan struct{}),
-					monitoringRunning: true, // Set the flag to indicate monitoring is running
+			if isEnabled {
+				// Instantiate the monitoring goroutine context from the parent context
+				// They can individually be cancelled and will be cancelled when the parent context is cancelled
+				internalCtx, internalCancel := context.WithCancel(c.OpManagerCtx)
+
+				cluster.monitoringRoutines[daemon] = &clusterHealth{
+					internalCtx:    internalCtx,
+					internalCancel: internalCancel,
 				}
 
 				// Run the go routine
@@ -68,41 +59,18 @@ func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInf
 			}
 		}
 	}
-
-	// Start watchers
-	if cluster.watchersActivated {
-		logger.Debugf("cluster is already being watched by bucket and client provisioner for cluster %q", cluster.Namespace)
-		return
-	}
-
-	// Start the object bucket provisioner
-	bucketProvisioner := bucket.NewProvisioner(c.context, clusterInfo)
-	// If cluster is external, pass down the user to the bucket controller
-
-	// note: the error return below is ignored and is expected to be removed from the
-	//   bucket library's `NewProvisioner` function
-	bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner)
-	go func() {
-		err := bucketController.Run(cluster.stopCh)
-		if err != nil {
-			logger.Errorf("failed to run bucket controller. %v", err)
-		}
-	}()
-
-	// enable the cluster watcher once
-	cluster.watchersActivated = true
 }
 
-func isMonitoringDisabled(daemon string, clusterSpec *cephv1.ClusterSpec) bool {
+func isMonitoringEnabled(daemon string, clusterSpec *cephv1.ClusterSpec) bool {
 	switch daemon {
 	case "mon":
-		return clusterSpec.HealthCheck.DaemonHealth.Monitor.Disabled
+		return !clusterSpec.HealthCheck.DaemonHealth.Monitor.Disabled
 
 	case "osd":
-		return clusterSpec.HealthCheck.DaemonHealth.ObjectStorageDaemon.Disabled
+		return !clusterSpec.HealthCheck.DaemonHealth.ObjectStorageDaemon.Disabled
 
 	case "status":
-		return clusterSpec.HealthCheck.DaemonHealth.Status.Disabled
+		return !clusterSpec.HealthCheck.DaemonHealth.Status.Disabled
 	}
 
 	return false
@@ -113,18 +81,18 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *
 	case "mon":
 		healthChecker := mon.NewHealthChecker(cluster.mons)
 		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
-		go healthChecker.Check(cluster.monitoringChannels[daemon].stopChan)
+		go healthChecker.Check(cluster.monitoringRoutines[daemon].internalCtx)
 
 	case "osd":
 		if !cluster.Spec.External.Enable {
 			c.osdChecker = osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
 			logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
-			go c.osdChecker.Start(cluster.monitoringChannels[daemon].stopChan)
+			go c.osdChecker.Start(cluster.monitoringRoutines[daemon].internalCtx)
 		}
 
 	case "status":
 		cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec)
 		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
-		go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan)
+		go cephChecker.checkCephStatus(cluster.monitoringRoutines[daemon].internalCtx)
 	}
 }

pkg/operator/ceph/cluster/monitoring_test.go

@@ -23,7 +23,7 @@ import (
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 )
 
-func TestIsMonitoringDisabled(t *testing.T) {
+func TestIsMonitoringEnabled(t *testing.T) {
 	type args struct {
 		daemon      string
 		clusterSpec *cephv1.ClusterSpec
@@ -33,12 +33,12 @@ func TestIsMonitoringDisabled(t *testing.T) {
 		args args
 		want bool
 	}{
-		{"isDisabled", args{"mon", &cephv1.ClusterSpec{}}, false},
-		{"isEnabled", args{"mon", &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Monitor: cephv1.HealthCheckSpec{Disabled: true}}}}}, true},
+		{"isEnabled", args{"mon", &cephv1.ClusterSpec{}}, true},
+		{"isDisabled", args{"mon", &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Monitor: cephv1.HealthCheckSpec{Disabled: true}}}}}, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := isMonitoringDisabled(tt.args.daemon, tt.args.clusterSpec); got != tt.want {
+			if got := isMonitoringEnabled(tt.args.daemon, tt.args.clusterSpec); got != tt.want {
 				t.Errorf("isMonitoringEnabled() = %v, want %v", got, tt.want)
 			}
 		})

pkg/operator/ceph/cluster/operator_watchers.go

@@ -1,70 +0,0 @@
-/*
-Copyright 2020 The Rook Authors. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package cluster to manage a Ceph cluster.
-package cluster
-
-import (
-	"os"
-	"reflect"
-
-	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
-	"github.com/rook/rook/pkg/operator/k8sutil"
-	v1 "k8s.io/api/core/v1"
-)
-
-// StartOperatorSettingsWatch starts the operator settings watcher
-// TODO: use controller runtime, so we can use a Context to cancel the watch instead of using another channel
-// The cache package mentions that it'd be nice to use a Context for cancellation too
-func (c *ClusterController) StartOperatorSettingsWatch(stopCh chan struct{}) {
-	operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
-	// watch for "rook-ceph-operator-config" ConfigMap
-	k8sutil.StartOperatorSettingsWatch(c.context, operatorNamespace, opcontroller.OperatorSettingConfigMapName,
-		c.operatorConfigChange,
-		func(oldObj, newObj interface{}) {
-			if reflect.DeepEqual(oldObj, newObj) {
-				return
-			}
-			c.operatorConfigChange(newObj)
-		}, nil, stopCh)
-}
-
-// StopWatch stop watchers
-func (c *ClusterController) StopWatch() {
-	for _, cluster := range c.clusterMap {
-		// check channel is open before closing
-		if !cluster.closedStopCh {
-			close(cluster.stopCh)
-			cluster.closedStopCh = true
-		}
-	}
-	c.clusterMap = make(map[string]*cluster)
-}
-
-func (c *ClusterController) operatorConfigChange(obj interface{}) {
-	cm, ok := obj.(*v1.ConfigMap)
-	if !ok {
-		logger.Warningf("Expected ConfigMap but handler received %T. %#v", obj, obj)
-		return
-	}
-
-	logger.Infof("ConfigMap %q changes detected. Updating configurations", cm.Name)
-	for _, callback := range c.operatorConfigCallbacks {
-		if err := callback(); err != nil {
-			logger.Errorf("%v", err)
-		}
-	}
-}

pkg/operator/ceph/cluster/osd/create.go

@@ -153,11 +153,9 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
 
 	awaitingStatusConfigMaps := sets.NewString()
 	for _, volume := range c.deviceSets {
-		// Check whether we need to cancel the orchestration
-		if err := opcontroller.CheckForCancelledOrchestration(c.context); err != nil {
-			return awaitingStatusConfigMaps, err
+		if c.clusterInfo.Context.Err() != nil {
+			return awaitingStatusConfigMaps, c.clusterInfo.Context.Err()
 		}
-
 		dataSource, dataOK := volume.PVCSources[bluestorePVCData]
 
 		// The data PVC template is required.
@@ -310,11 +308,9 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov
 
 	awaitingStatusConfigMaps := sets.NewString()
 	for _, node := range c.ValidStorage.Nodes {
-		// Check whether we need to cancel the orchestration
-		if err := opcontroller.CheckForCancelledOrchestration(c.context); err != nil {
-			return awaitingStatusConfigMaps, err
+		if c.clusterInfo.Context.Err() != nil {
+			return awaitingStatusConfigMaps, c.clusterInfo.Context.Err()
 		}
-
 		// fully resolve the storage config and resources for this node
 		// don't care about osd device class resources since it will be overwritten later for prepareosd resources
 		n := c.resolveNode(node.Name, "")

pkg/operator/ceph/cluster/osd/create_test.go

@@ -28,7 +28,6 @@ import (
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	apiresource "k8s.io/apimachinery/pkg/api/resource"
@@ -307,7 +306,6 @@ func Test_startProvisioningOverPVCs(t *testing.T) {
 
 	clientset := test.NewComplexClientset(t) // fake clientset with generate name functionality
 
-	requestCancelOrchestration := *abool.New()
 	// clusterd.Context is created in doSetup()
 
 	clusterInfo := &cephclient.ClusterInfo{
@@ -316,6 +314,7 @@ func Test_startProvisioningOverPVCs(t *testing.T) {
 	}
 	clusterInfo.SetName("mycluster")
 	clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
+	clusterInfo.Context = context.TODO()
 
 	spec := cephv1.ClusterSpec{}
 	fakeK8sVersion := "v1.13.0"
@@ -329,8 +328,7 @@ func Test_startProvisioningOverPVCs(t *testing.T) {
 		test.SetFakeKubernetesVersion(clientset, fakeK8sVersion) // PVCs require k8s version v1.13+
 		errs = newProvisionErrors()
 		ctx := &clusterd.Context{
-			Clientset:                  clientset,
-			RequestCancelOrchestration: &requestCancelOrchestration,
+			Clientset: clientset,
 		}
 		c = New(ctx, clusterInfo, spec, "rook/rook:master")
 		config = c.newProvisionConfig()
@@ -425,18 +423,6 @@ func Test_startProvisioningOverPVCs(t *testing.T) {
 		fakeK8sVersion = "v1.13.0"
 	})
 
-	t.Run("request cancel orchestration", func(t *testing.T) {
-		// spec = <working spec from prior test>
-		clientset = test.NewComplexClientset(t) // reset to empty fake k8s environment
-		requestCancelOrchestration.Set()
-		doSetup()
-		awaitingStatusConfigMaps, err = c.startProvisioningOverPVCs(config, errs)
-		assert.Error(t, err)
-		assert.Zero(t, errs.len())
-		assert.Zero(t, awaitingStatusConfigMaps.Len())
-		requestCancelOrchestration.UnSet()
-	})
-
 	t.Run("error if no volume claim template", func(t *testing.T) {
 		spec = cephv1.ClusterSpec{
 			Storage: cephv1.StorageScopeSpec{
@@ -468,7 +454,6 @@ func Test_startProvisioningOverNodes(t *testing.T) {
 	dataDirHostPath := "/var/lib/mycluster"
 
 	clientset := test.New(t, 3) // fake clientset with 3 nodes
-	requestCancelOrchestration := *abool.New()
 	// clusterd.Context is created in doSetup()
 
 	// names of status configmaps for nodes
@@ -483,6 +468,7 @@ func Test_startProvisioningOverNodes(t *testing.T) {
 	}
 	clusterInfo.SetName("mycluster")
 	clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
+	clusterInfo.Context = context.TODO()
 
 	var useAllDevices bool
 	spec := cephv1.ClusterSpec{}
@@ -496,8 +482,7 @@ func Test_startProvisioningOverNodes(t *testing.T) {
 	doSetup := func() {
 		errs = newProvisionErrors()
 		ctx := &clusterd.Context{
-			Clientset:                  clientset,
-			RequestCancelOrchestration: &requestCancelOrchestration,
+			Clientset: clientset,
 		}
 		c = New(ctx, clusterInfo, spec, "rook/rook:master")
 		config = c.newProvisionConfig()
@@ -599,16 +584,6 @@ func Test_startProvisioningOverNodes(t *testing.T) {
 		)
 	})
 
-	t.Run("request cancel orchestration", func(t *testing.T) {
-		requestCancelOrchestration.Set()
-		doSetup()
-		prepareJobsRun, err = c.startProvisioningOverNodes(config, errs)
-		assert.Error(t, err)
-		assert.Zero(t, errs.len())
-		assert.Zero(t, prepareJobsRun.Len())
-		requestCancelOrchestration.UnSet()
-	})
-
 	t.Run("use no nodes", func(t *testing.T) {
 		spec = cephv1.ClusterSpec{
 			Storage: cephv1.StorageScopeSpec{

pkg/operator/ceph/cluster/osd/deviceSet.go

@@ -189,7 +189,6 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet cephv1.StorageClassDe
 }
 
 func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolumeClaim, deviceSetName string, pvcTemplate v1.PersistentVolumeClaim, setIndex int) (*v1.PersistentVolumeClaim, error) {
-	ctx := context.TODO()
 	// old labels and PVC ID for backward compatibility
 	pvcID := legacyDeviceSetPVCID(deviceSetName, setIndex)
 
@@ -215,7 +214,7 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum
 	}
 
 	// No PVC found, creating a new one
-	deployedPVC, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.clusterInfo.Namespace).Create(ctx, pvc, metav1.CreateOptions{})
+	deployedPVC, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, pvc, metav1.CreateOptions{})
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to create PVC %q for device set %q", pvc.Name, deviceSetName)
 	}

pkg/operator/ceph/cluster/osd/health.go

@@ -69,7 +69,7 @@ func NewOSDHealthMonitor(context *clusterd.Context, clusterInfo *client.ClusterI
 }
 
 // Start runs monitoring logic for osds status at set intervals
-func (m *OSDHealthMonitor) Start(stopCh chan struct{}) {
+func (m *OSDHealthMonitor) Start(context context.Context) {
 
 	for {
 		select {
@@ -77,8 +77,8 @@ func (m *OSDHealthMonitor) Start(stopCh chan struct{}) {
 			logger.Debug("checking osd processes status.")
 			m.checkOSDHealth()
 
-		case <-stopCh:
-			logger.Infof("Stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace)
+		case <-context.Done():
+			logger.Infof("stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace)
 			return
 		}
 	}
@@ -192,7 +192,7 @@ func (m *OSDHealthMonitor) updateCephStatus(devices []string) {
 	for _, device := range devices {
 		cephClusterStorage.DeviceClasses = append(cephClusterStorage.DeviceClasses, cephv1.DeviceClasses{Name: device})
 	}
-	err := m.context.Client.Get(context.TODO(), m.clusterInfo.NamespacedName(), cephCluster)
+	err := m.context.Client.Get(m.clusterInfo.Context, m.clusterInfo.NamespacedName(), cephCluster)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")

pkg/operator/ceph/cluster/osd/health_test.go

@@ -101,11 +101,11 @@ func TestOSDHealthCheck(t *testing.T) {
 }
 
 func TestMonitorStart(t *testing.T) {
-	stopCh := make(chan struct{})
+	context, cancel := context.WithCancel(context.TODO())
 	osdMon := NewOSDHealthMonitor(&clusterd.Context{}, client.AdminClusterInfo("ns"), true, cephv1.CephClusterHealthCheckSpec{})
 	logger.Infof("starting osd monitor")
-	go osdMon.Start(stopCh)
-	close(stopCh)
+	go osdMon.Start(context)
+	cancel()
 }
 
 func TestNewOSDHealthMonitor(t *testing.T) {

pkg/operator/ceph/cluster/osd/integration_test.go

@@ -18,6 +18,7 @@ package osd
 
 import (
 	"context"
+	contextStandard "context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -37,7 +38,6 @@ import (
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -96,7 +96,8 @@ func TestOSDIntegration(t *testing.T) {
 
 // This is the actual test. If it hangs, we should consider that an error.
 func testOSDIntegration(t *testing.T) {
-	ctx := context.TODO()
+	ctx := contextStandard.TODO()
+	contextCancel, cancel := contextStandard.WithCancel(ctx)
 	namespace := "osd-integration"
 	clusterName := "my-cluster"
 
@@ -191,13 +192,13 @@ func testOSDIntegration(t *testing.T) {
 	clusterInfo.CephVersion = cephver.Pacific
 	clusterInfo.SetName("mycluster")
 	clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
+	clusterInfo.Context = ctx
 	executor := osdIntegrationTestExecutor(t, clientset, namespace)
 
 	context := &clusterd.Context{
-		Clientset:                  clientset,
-		ConfigDir:                  "/var/lib/rook",
-		Executor:                   executor,
-		RequestCancelOrchestration: abool.New(),
+		Clientset: clientset,
+		ConfigDir: "/var/lib/rook",
+		Executor:  executor,
 	}
 	spec := cephv1.ClusterSpec{
 		CephVersion: cephv1.CephVersionSpec{
@@ -225,9 +226,10 @@ func testOSDIntegration(t *testing.T) {
 
 	var startErr error
 	var done bool
-	runReconcile := func() {
+	runReconcile := func(ctx contextStandard.Context) {
 		// reset environment
 		c = New(context, clusterInfo, spec, "myversion")
+		clusterInfo.Context = ctx
 		statusMapWatcher.Reset()
 
 		// reset counters
@@ -249,7 +251,7 @@ func testOSDIntegration(t *testing.T) {
 
 	// NOTE: these tests all use the same environment
 	t.Run("initial creation", func(t *testing.T) {
-		go runReconcile()
+		go runReconcile(contextCancel)
 
 		cms := waitForNumConfigMaps(clientset, namespace, 12) // 3 nodes + 9 new PVCs
 		for _, cm := range cms {
@@ -265,7 +267,7 @@ func testOSDIntegration(t *testing.T) {
 	})
 
 	t.Run("reconcile again with no changes", func(t *testing.T) {
-		go runReconcile()
+		go runReconcile(contextCancel)
 
 		cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
@@ -286,7 +288,7 @@ func testOSDIntegration(t *testing.T) {
 		spec.Storage.StorageClassDeviceSets[1].Count = 6     // 3 more (1 more per node)
 		osdsPerNode = 3                                      // vda, vdb, vdc
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 
 		cms := waitForNumConfigMaps(clientset, namespace, 8) // 3 nodes + 5 new PVCs
 		for _, cm := range cms {
@@ -306,7 +308,7 @@ func testOSDIntegration(t *testing.T) {
 		spec.Storage.StorageClassDeviceSets[0].Count = 10     // 2 more portable
 		osdsPerNode = 4                                       // vd[a-d]
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms := waitForNumConfigMaps(clientset, namespace, 5) // 3 nodes + 2 new PVCs
 		i := 1
 		for _, cm := range cms {
@@ -323,7 +325,9 @@ func testOSDIntegration(t *testing.T) {
 				time.Sleep(10 * time.Millisecond)
 				// after the second status map is made ready, cancel the orchestration. wait a short
 				// while to make sure the watcher picks up the updated change
-				c.context.RequestCancelOrchestration.Set()
+				cancel()
+				// refresh the context so the rest of the test can continue
+				contextCancel = contextStandard.TODO()
 				break
 			}
 			i++
@@ -337,7 +341,7 @@ func testOSDIntegration(t *testing.T) {
 		numUpdates := len(deploymentsUpdated)
 		t.Logf("deployments updated: %d", numUpdates)
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms = waitForNumConfigMaps(clientset, namespace, 5) // 3 nodes + 2 new PVCs
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -354,7 +358,7 @@ func testOSDIntegration(t *testing.T) {
 		spec.Storage.Selection.DeviceFilter = "/dev/vd[abcde]" // 3 more (1 more per node)
 		osdsPerNode = 5                                        // vd[a-e]
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -372,7 +376,7 @@ func testOSDIntegration(t *testing.T) {
 		assert.Len(t, deploymentsUpdated, 28)
 
 		// should get back to healthy after
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms = waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -387,7 +391,7 @@ func testOSDIntegration(t *testing.T) {
 
 	t.Run("failures during deployment updates", func(t *testing.T) {
 		failUpdatingDeployments = []string{"osd-15", "osd-22"}
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -401,7 +405,7 @@ func testOSDIntegration(t *testing.T) {
 		assert.Len(t, deploymentsUpdated, 31) // should attempt to update all deployments
 
 		failUpdatingDeployments = []string{}
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms = waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -419,7 +423,7 @@ func testOSDIntegration(t *testing.T) {
 		osdsPerNode = 6                                         // vd[a-f]
 
 		failCreatingDeployments = []string{"osd-31", "osd-33"}
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -433,7 +437,7 @@ func testOSDIntegration(t *testing.T) {
 		assert.Len(t, deploymentsUpdated, 31)
 
 		failCreatingDeployments = []string{}
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms = waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -455,7 +459,7 @@ func testOSDIntegration(t *testing.T) {
 		}
 		spec.Storage.StorageClassDeviceSets = append(spec.Storage.StorageClassDeviceSets, newSCDS)
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -473,7 +477,7 @@ func testOSDIntegration(t *testing.T) {
 			newDummyPVC("metadata", namespace, "10Gi", "uncle-rogers-secret-stuff"),
 		}
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms = waitForNumConfigMaps(clientset, namespace, 6) // 3 nodes + 3 new PVCs
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()
@@ -497,7 +501,7 @@ func testOSDIntegration(t *testing.T) {
 		_, err := clientset.CoreV1().ConfigMaps(namespace).Create(ctx, danglingCM, metav1.CreateOptions{})
 		assert.NoError(t, err)
 
-		go runReconcile()
+		go runReconcile(contextCancel)
 		cms := waitForNumConfigMaps(clientset, namespace, 4) // 3 nodes + dangling
 		for _, cm := range cms {
 			cpy := cm.DeepCopy()

pkg/operator/ceph/cluster/osd/osd.go

@@ -240,10 +240,9 @@ func (c *Cluster) Start() error {
 }
 
 func (c *Cluster) getExistingOSDDeploymentsOnPVCs() (sets.String, error) {
-	ctx := context.TODO()
 	listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s", k8sutil.AppAttr, AppName, OSDOverPVCLabelKey)}
 
-	deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(ctx, listOpts)
+	deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to query existing OSD deployments")
 	}
@@ -406,12 +405,11 @@ func (c *Cluster) getOSDPropsForPVC(pvcName, osdDeviceClass string) (osdProperti
 // First look for the node selector that was previously used for the OSD, or if a new OSD
 // check for the assignment of the OSD prepare job.
 func (c *Cluster) getPVCHostName(pvcName string) (string, error) {
-	ctx := context.TODO()
 	listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", OSDOverPVCLabelKey, pvcName)}
 
 	// Check for the existence of the OSD deployment where the node selector was applied
 	// in a previous reconcile.
-	deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(ctx, listOpts)
+	deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to get deployment for osd with pvc %q", pvcName)
 	}
@@ -426,7 +424,7 @@ func (c *Cluster) getPVCHostName(pvcName string) (string, error) {
 
 	// Since the deployment wasn't found it must be a new deployment so look at the node
 	// assignment of the OSD prepare pod
-	pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(ctx, listOpts)
+	pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to get pod for osd with pvc %q", pvcName)
 	}

pkg/operator/ceph/cluster/osd/osd_test.go

@@ -35,7 +35,6 @@ import (
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/tevino/abool"
 	apps "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -69,6 +68,7 @@ func TestStart(t *testing.T) {
 	clusterInfo := &cephclient.ClusterInfo{
 		Namespace:   "ns",
 		CephVersion: cephver.Nautilus,
+		Context:     context.TODO(),
 	}
 	context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
 	spec := cephv1.ClusterSpec{}
@@ -151,6 +151,7 @@ func TestAddRemoveNode(t *testing.T) {
 	clusterInfo := &cephclient.ClusterInfo{
 		Namespace:   namespace,
 		CephVersion: cephver.Nautilus,
+		Context:     ctx,
 	}
 	clusterInfo.SetName("rook-ceph-test")
 	clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
@@ -162,11 +163,10 @@ func TestAddRemoveNode(t *testing.T) {
 	}
 
 	context := &clusterd.Context{
-		Clientset:                  clientset,
-		ConfigDir:                  "/var/lib/rook",
-		Executor:                   executor,
-		RequestCancelOrchestration: abool.New(),
-		RookClientset:              fakeclient.NewSimpleClientset(),
+		Clientset:     clientset,
+		ConfigDir:     "/var/lib/rook",
+		Executor:      executor,
+		RookClientset: fakeclient.NewSimpleClientset(),
 	}
 	spec := cephv1.ClusterSpec{
 		DataDirHostPath: context.ConfigDir,
@@ -324,10 +324,11 @@ func TestAddNodeFailure(t *testing.T) {
 	clusterInfo := &cephclient.ClusterInfo{
 		Namespace:   "ns-add-remove",
 		CephVersion: cephver.Nautilus,
+		Context:     context.TODO(),
 	}
 	clusterInfo.SetName("testcluster")
 	clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
-	context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()}
+	context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
 	spec := cephv1.ClusterSpec{
 		DataDirHostPath: context.ConfigDir,
 		Storage: cephv1.StorageScopeSpec{

pkg/operator/ceph/cluster/osd/status.go

@@ -18,14 +18,12 @@ limitations under the License.
 package osd
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/operator/ceph/config"
-	"github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -196,13 +194,12 @@ func (c *Cluster) updateAndCreateOSDsLoop(
 	errs *provisionErrors, // add errors here
 ) (shouldRestart bool, err error) {
 	cmClient := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace)
-	ctx := context.TODO()
 	selector := statusConfigMapSelector()
 
 	listOptions := metav1.ListOptions{
 		LabelSelector: selector,
 	}
-	configMapList, err := cmClient.List(ctx, listOptions)
+	configMapList, err := cmClient.List(c.clusterInfo.Context, listOptions)
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to list OSD provisioning status ConfigMaps")
 	}
@@ -218,7 +215,7 @@ func (c *Cluster) updateAndCreateOSDsLoop(
 		Watch:           true,
 		ResourceVersion: configMapList.ResourceVersion,
 	}
-	watcher, err := cmClient.Watch(ctx, watchOptions)
+	watcher, err := cmClient.Watch(c.clusterInfo.Context, watchOptions)
 	defer watcher.Stop()
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to start watching OSD provisioning status ConfigMaps")
@@ -269,9 +266,8 @@ func (c *Cluster) updateAndCreateOSDsLoop(
 			updateConfig.updateExistingOSDs(errs)
 
 		case <-minuteTicker.C:
-			// Check whether we need to cancel the orchestration
-			if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
-				return false, err
+			if c.clusterInfo.Context.Err() != nil {
+				return false, c.clusterInfo.Context.Err()
 			}
 			// Log progress
 			c, cExp := createConfig.progress()
@@ -350,19 +346,18 @@ func (c *Cluster) deleteStatusConfigMap(nodeOrPVCName string) {
 }
 
 func (c *Cluster) deleteAllStatusConfigMaps() {
-	ctx := context.TODO()
 	listOpts := metav1.ListOptions{
 		LabelSelector: statusConfigMapSelector(),
 	}
 	cmClientset := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace)
-	cms, err := cmClientset.List(ctx, listOpts)
+	cms, err := cmClientset.List(c.clusterInfo.Context, listOpts)
 	if err != nil {
 		logger.Warningf("failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err)
 		return
 	}
 	for _, cm := range cms.Items {
 		logger.Debugf("cleaning up dangling OSD prepare status configmap %q", cm.Name)
-		err := cmClientset.Delete(ctx, cm.Name, metav1.DeleteOptions{})
+		err := cmClientset.Delete(c.clusterInfo.Context, cm.Name, metav1.DeleteOptions{})
 		if err != nil {
 			logger.Warningf("failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err)
 		}

pkg/operator/ceph/cluster/osd/update.go

@@ -17,7 +17,6 @@ limitations under the License.
 package osd
 
 import (
-	"context"
 	"fmt"
 	"strconv"
 	"strings"
@@ -73,8 +72,6 @@ func (c *updateConfig) doneUpdating() bool {
 }
 
 func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
-	ctx := context.TODO()
-
 	if c.doneUpdating() {
 		return // no more OSDs to update
 	}
@@ -117,7 +114,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
 		}
 
 		depName := deploymentName(osdID)
-		dep, err := c.cluster.context.Clientset.AppsV1().Deployments(c.cluster.clusterInfo.Namespace).Get(ctx, depName, metav1.GetOptions{})
+		dep, err := c.cluster.context.Clientset.AppsV1().Deployments(c.cluster.clusterInfo.Namespace).Get(c.cluster.clusterInfo.Context, depName, metav1.GetOptions{})
 		if err != nil {
 			errs.addError("failed to update OSD %d. failed to find existing deployment %q. %v", osdID, depName, err)
 			continue
@@ -194,7 +191,6 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
 // getOSDUpdateInfo returns an update queue of OSDs which need updated and an existence list of OSD
 // Deployments which already exist.
 func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existenceList, error) {
-	ctx := context.TODO()
 	namespace := c.clusterInfo.Namespace
 
 	selector := fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)
@@ -202,7 +198,7 @@ func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existe
 		// list only rook-ceph-osd Deployments
 		LabelSelector: selector,
 	}
-	deps, err := c.context.Clientset.AppsV1().Deployments(namespace).List(ctx, listOpts)
+	deps, err := c.context.Clientset.AppsV1().Deployments(namespace).List(c.clusterInfo.Context, listOpts)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to query existing OSD deployments to see if they need updated")
 	}
@@ -338,12 +334,11 @@ func (e *existenceList) Exists(osdID int) bool {
 
 // return a function that will list only OSD deployments with the IDs given
 func (c *Cluster) getFuncToListDeploymentsWithIDs(osdIDs []string) func() (*appsv1.DeploymentList, error) {
-	ctx := context.TODO()
 	selector := fmt.Sprintf("ceph-osd-id in (%s)", strings.Join(osdIDs, ", "))
 	listOpts := metav1.ListOptions{
 		LabelSelector: selector, // e.g. 'ceph-osd-id in (1, 3, 5, 7, 9)'
 	}
 	return func() (*appsv1.DeploymentList, error) {
-		return c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(ctx, listOpts)
+		return c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts)
 	}
 }

pkg/operator/ceph/cluster/osd/update_test.go

@@ -101,6 +101,7 @@ func Test_updateExistingOSDs(t *testing.T) {
 		clusterInfo := &cephclient.ClusterInfo{
 			Namespace:   namespace,
 			CephVersion: cephver.Pacific,
+			Context:     context.TODO(),
 		}
 		clusterInfo.SetName("mycluster")
 		clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)

pkg/operator/ceph/cluster/predicate.go

@@ -105,7 +105,7 @@ func isHotPlugCM(obj runtime.Object) bool {
 	return false
 }
 
-func watchControllerPredicate(rookContext *clusterd.Context) predicate.Funcs {
+func watchControllerPredicate() predicate.Funcs {
 	return predicate.Funcs{
 		CreateFunc: func(e event.CreateEvent) bool {
 			logger.Debug("create event from a CR")
@@ -131,18 +131,25 @@ func watchControllerPredicate(rookContext *clusterd.Context) predicate.Funcs {
 				}
 				diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer)
 				if diff != "" {
-					// Set the cancellation flag to stop any ongoing orchestration
-					rookContext.RequestCancelOrchestration.Set()
-
 					logger.Infof("CR has changed for %q. diff=%s", objNew.Name, diff)
-					return true
+
+					if objNew.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
+						logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace())
+						return false
+					}
+
+					// Stop any ongoing orchestration
+					controller.ReloadManager()
+
+					return false
 
 				} else if !objOld.GetDeletionTimestamp().Equal(objNew.GetDeletionTimestamp()) {
-					// Set the cancellation flag to stop any ongoing orchestration
-					rookContext.RequestCancelOrchestration.Set()
-
 					logger.Infof("CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name)
-					return true
+
+					// Stop any ongoing orchestration
+					controller.ReloadManager()
+
+					return false
 
 				} else if objOld.GetGeneration() != objNew.GetGeneration() {
 					logger.Debugf("skipping resource %q update with unchanged spec", objNew.Name)

pkg/operator/ceph/cluster/rbd/config.go

@@ -17,7 +17,6 @@ limitations under the License.
 package rbd
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/pkg/errors"
@@ -51,7 +50,6 @@ type daemonConfig struct {
 }
 
 func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo, daemonConfig *daemonConfig) (string, error) {
-	ctx := context.TODO()
 	user := fullDaemonName(daemonConfig.DaemonID)
 	access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"}
 	s := keyring.GetSecretStore(r.context, clusterInfo, daemonConfig.ownerInfo)
@@ -62,7 +60,7 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo
 	}
 
 	// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
-	err = r.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Delete(ctx, daemonConfig.ResourceName, metav1.DeleteOptions{})
+	err = r.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Delete(r.opManagerContext, daemonConfig.ResourceName, metav1.DeleteOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debugf("legacy rbd-mirror key %q is already removed", daemonConfig.ResourceName)
@@ -80,14 +78,13 @@ func fullDaemonName(daemonID string) string {
 }
 
 func (r *ReconcileCephRBDMirror) reconcileAddBoostrapPeer(cephRBDMirror *cephv1.CephRBDMirror, namespacedName types.NamespacedName) (reconcile.Result, error) {
-	ctx := context.TODO()
 	// List all the peers secret, we can have more than one peer we might want to configure
 	// For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring
 
 	logger.Warning("(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.")
 	for _, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames {
 		logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret)
-		s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(ctx, peerSecret, metav1.GetOptions{})
+		s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{})
 		// We don't care about IsNotFound here, we still need to fail
 		if err != nil {
 			return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret)

pkg/operator/ceph/cluster/rbd/controller.go

@@ -28,7 +28,6 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
-	opconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/ceph/reporting"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -67,14 +66,18 @@ var controllerTypeMeta = metav1.TypeMeta{
 	APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version),
 }
 
+var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion
+
 // ReconcileCephRBDMirror reconciles a cephRBDMirror object
 type ReconcileCephRBDMirror struct {
-	context         *clusterd.Context
-	clusterInfo     *cephclient.ClusterInfo
-	client          client.Client
-	scheme          *runtime.Scheme
-	cephClusterSpec *cephv1.ClusterSpec
-	peers           map[string]*peerSpec
+	context          *clusterd.Context
+	clusterInfo      *cephclient.ClusterInfo
+	client           client.Client
+	scheme           *runtime.Scheme
+	cephClusterSpec  *cephv1.ClusterSpec
+	peers            map[string]*peerSpec
+	opManagerContext context.Context
+	opConfig         opcontroller.OperatorConfig
 }
 
 // peerSpec represents peer details
@@ -86,17 +89,19 @@ type peerSpec struct {
 
 // Add creates a new cephRBDMirror Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, context *clusterd.Context) error {
-	return add(mgr, newReconciler(mgr, context))
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+	return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
 	return &ReconcileCephRBDMirror{
-		client:  mgr.GetClient(),
-		scheme:  mgr.GetScheme(),
-		context: context,
-		peers:   make(map[string]*peerSpec),
+		client:           mgr.GetClient(),
+		scheme:           mgr.GetScheme(),
+		context:          context,
+		peers:            make(map[string]*peerSpec),
+		opConfig:         opConfig,
+		opManagerContext: opManagerContext,
 	}
 }
 
@@ -125,25 +130,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 		}
 	}
 
-	// Build Handler function to return the list of ceph rbd-mirror
-	// This is used by the watchers below
-	handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephRBDMirrorList{}, mgr.GetScheme())
-	if err != nil {
-		return err
-	}
-
-	// Watch for CephCluster Spec changes that we want to propagate to us
-	err = c.Watch(&source.Kind{Type: &cephv1.CephCluster{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       opcontroller.ClusterResource.Kind,
-			APIVersion: opcontroller.ClusterResource.APIVersion,
-		},
-	},
-	}, handler.EnqueueRequestsFromMapFunc(handlerFunc), opcontroller.WatchCephClusterPredicate())
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -155,7 +141,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco
 	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
 	reconcileResponse, err := r.reconcile(request)
 	if err != nil {
-		updateStatus(r.client, request.NamespacedName, k8sutil.FailedStatus)
+		r.updateStatus(r.client, request.NamespacedName, k8sutil.FailedStatus)
 		logger.Errorf("failed to reconcile %v", err)
 	}
 
@@ -165,7 +151,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco
 func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile.Result, error) {
 	// Fetch the cephRBDMirror instance
 	cephRBDMirror := &cephv1.CephRBDMirror{}
-	err := r.client.Get(context.TODO(), request.NamespacedName, cephRBDMirror)
+	err := r.client.Get(r.opManagerContext, request.NamespacedName, cephRBDMirror)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("cephRBDMirror resource not found. Ignoring since object must be deleted.")
@@ -177,7 +163,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
 
 	// The CR was just created, initializing status fields
 	if cephRBDMirror.Status == nil {
-		updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus)
+		r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus)
 	}
 
 	// validate the pool settings
@@ -195,22 +181,38 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
 
 	// Populate clusterInfo
 	// Always populate it during each reconcile
-	r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace)
+	r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
 	if err != nil {
 		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to populate cluster info")
 	}
 
-	// Populate CephVersion
-	daemon := string(opconfig.MonType)
-	currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon)
+	// Detect desired CephCluster version
+	runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion(
+		r.opConfig.Image,
+		cephRBDMirror.Namespace,
+		controllerName,
+		k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme),
+		r.context,
+		r.cephClusterSpec,
+		r.clusterInfo,
+	)
 	if err != nil {
 		if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
 			logger.Info(opcontroller.OperatorNotInitializedMessage)
-			return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil
+			return reconcile.Result{}, nil
 		}
-		return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon)
+		return reconcile.Result{}, errors.Wrap(err, "failed to detect running and desired ceph version")
 	}
-	r.clusterInfo.CephVersion = currentCephVersion
+
+	// If the version of the Ceph monitor differs from the CephCluster CR image version we assume
+	// the cluster is being upgraded. So the controller will just wait for the upgrade to finish and
+	// then versions should match. Obviously using the cmd reporter job adds up to the deployment time
+	if !reflect.DeepEqual(runningCephVersion, desiredCephVersion) {
+		// Upgrade is in progress, let's wait for the mons to be done
+		return opcontroller.WaitForRequeueIfCephClusterIsUpgrading,
+			opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion)
+	}
+	r.clusterInfo.CephVersion = *runningCephVersion
 
 	// Add bootstrap peer if any
 	logger.Debug("reconciling ceph rbd mirror peers addition")
@@ -227,7 +229,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
 	}
 
 	// Set Ready status, we are done reconciling
-	updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus)
+	r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus)
 
 	// Return and do not requeue
 	logger.Debug("done reconciling ceph rbd mirror")
@@ -254,9 +256,9 @@ func (r *ReconcileCephRBDMirror) reconcileCreateCephRBDMirror(cephRBDMirror *cep
 }
 
 // updateStatus updates an object with a given status
-func updateStatus(client client.Client, name types.NamespacedName, status string) {
+func (r *ReconcileCephRBDMirror) updateStatus(client client.Client, name types.NamespacedName, status string) {
 	rbdMirror := &cephv1.CephRBDMirror{}
-	err := client.Get(context.TODO(), name, rbdMirror)
+	err := client.Get(r.opManagerContext, name, rbdMirror)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.")

pkg/operator/ceph/cluster/rbd/controller_test.go

@@ -27,6 +27,8 @@ import (
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
 	"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
 	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -52,18 +54,18 @@ const (
 func TestCephRBDMirrorController(t *testing.T) {
 	ctx := context.TODO()
 	var (
-		name      = "my-mirror"
-		namespace = "rook-ceph"
+		name           = "my-mirror"
+		namespace      = "rook-ceph"
+		peerSecretName = "peer-secret"
 	)
 	// Set DEBUG logging
 	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
 	os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
 
-	//
-	// TEST 1 SETUP
-	//
-	// FAILURE because no CephCluster
-	//
+	currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
+		return &version.Pacific, &version.Pacific, nil
+	}
+
 	// An rbd-mirror resource with metadata and spec.
 	rbdMirror := &cephv1.CephRBDMirror{
 		ObjectMeta: metav1.ObjectMeta{
@@ -81,6 +83,30 @@ func TestCephRBDMirrorController(t *testing.T) {
 		rbdMirror,
 	}
 
+	// Mock request to simulate Reconcile() being called on an event for a
+	// watched resource .
+	req := reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	cephCluster := &cephv1.CephCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      namespace,
+			Namespace: namespace,
+		},
+		Status: cephv1.ClusterStatus{
+			Phase: "",
+			CephStatus: &cephv1.CephStatus{
+				Health: "",
+			},
+		},
+	}
+
+	s := scheme.Scheme
+
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 			if args[0] == "status" {
@@ -108,123 +134,101 @@ func TestCephRBDMirrorController(t *testing.T) {
 		Clientset:     clientset,
 	}
 
-	// Register operator types with the runtime scheme.
-	s := scheme.Scheme
-	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{})
-	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
+	t.Run("error - no ceph cluster", func(t *testing.T) {
+		// Register operator types with the runtime scheme.
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{})
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
 
-	// Create a fake client to mock API calls.
-	cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
 
-	// Create a ReconcileCephRBDMirror object with the scheme and fake client.
-	r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c}
+		// Create a ReconcileCephRBDMirror object with the scheme and fake client.
+		r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx}
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.True(t, res.Requeue)
+	})
 
-	// Mock request to simulate Reconcile() being called on an event for a
-	// watched resource .
-	req := reconcile.Request{
-		NamespacedName: types.NamespacedName{
-			Name:      name,
-			Namespace: namespace,
-		},
-	}
-	logger.Info("STARTING PHASE 1")
-	res, err := r.Reconcile(ctx, req)
-	assert.NoError(t, err)
-	assert.True(t, res.Requeue)
-	logger.Info("PHASE 1 DONE")
+	t.Run("error - ceph cluster not ready", func(t *testing.T) {
+		object = append(object, cephCluster)
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+		// Create a ReconcileCephRBDMirror object with the scheme and fake client.
+		r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx}
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.True(t, res.Requeue)
+	})
 
-	//
-	// TEST 2:
-	//
-	// FAILURE we have a cluster but it's not ready
-	//
-	cephCluster := &cephv1.CephCluster{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      namespace,
-			Namespace: namespace,
-		},
-		Status: cephv1.ClusterStatus{
-			Phase: "",
-			CephStatus: &cephv1.CephStatus{
-				Health: "",
+	t.Run("error - wrong peers configuration", func(t *testing.T) {
+		// Mock clusterInfo
+		secrets := map[string][]byte{
+			"fsid":         []byte(name),
+			"mon-secret":   []byte("monsecret"),
+			"admin-secret": []byte("adminsecret"),
+		}
+		secret := &v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "rook-ceph-mon",
+				Namespace: namespace,
 			},
-		},
-	}
-	object = append(object, cephCluster)
-	// Create a fake client to mock API calls.
-	cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
-	// Create a ReconcileCephRBDMirror object with the scheme and fake client.
-	r = &ReconcileCephRBDMirror{client: cl, scheme: s, context: c}
-	logger.Info("STARTING PHASE 2")
-	res, err = r.Reconcile(ctx, req)
-	assert.NoError(t, err)
-	assert.True(t, res.Requeue)
-	logger.Info("PHASE 2 DONE")
+			Data: secrets,
+			Type: k8sutil.RookType,
+		}
+		_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+		assert.NoError(t, err)
 
-	//
-	// TEST 3:
-	//
-	// SUCCESS! The CephCluster is ready
-	//
+		// Add ready status to the CephCluster
+		cephCluster.Status.Phase = k8sutil.ReadyStatus
+		cephCluster.Status.CephStatus.Health = "HEALTH_OK"
 
-	// Mock clusterInfo
-	secrets := map[string][]byte{
-		"fsid":         []byte(name),
-		"mon-secret":   []byte("monsecret"),
-		"admin-secret": []byte("adminsecret"),
-	}
-	secret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "rook-ceph-mon",
-			Namespace: namespace,
-		},
-		Data: secrets,
-		Type: k8sutil.RookType,
-	}
-	_, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
-	assert.NoError(t, err)
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
 
-	// Add ready status to the CephCluster
-	cephCluster.Status.Phase = k8sutil.ReadyStatus
-	cephCluster.Status.CephStatus.Health = "HEALTH_OK"
+		// Create a ReconcileCephRBDMirror object with the scheme and fake client.
+		r := &ReconcileCephRBDMirror{
+			client:           cl,
+			scheme:           s,
+			context:          c,
+			peers:            make(map[string]*peerSpec),
+			opManagerContext: ctx,
+		}
 
-	// Create a fake client to mock API calls.
-	cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+		rbdMirror.Spec.Peers.SecretNames = []string{peerSecretName}
+		err = r.client.Update(context.TODO(), rbdMirror)
+		assert.NoError(t, err)
+		res, err := r.Reconcile(ctx, req)
+		assert.Error(t, err)
+		assert.True(t, res.Requeue)
 
-	// Create a ReconcileCephRBDMirror object with the scheme and fake client.
-	r = &ReconcileCephRBDMirror{
-		client:  cl,
-		scheme:  s,
-		context: c,
-		peers:   make(map[string]*peerSpec),
-	}
+	})
 
-	logger.Info("STARTING PHASE 4")
-
-	peerSecretName := "peer-secret"
-	rbdMirror.Spec.Peers.SecretNames = []string{peerSecretName}
-	err = r.client.Update(context.TODO(), rbdMirror)
-	assert.NoError(t, err)
-	res, err = r.Reconcile(ctx, req)
-	assert.Error(t, err)
-	assert.True(t, res.Requeue)
-
-	logger.Info("STARTING PHASE 5")
-	bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token
-	peerSecret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      peerSecretName,
-			Namespace: namespace,
-		},
-		Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")},
-		Type: k8sutil.RookType,
-	}
-	_, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{})
-	assert.NoError(t, err)
-	res, err = r.Reconcile(ctx, req)
-	assert.NoError(t, err)
-	assert.False(t, res.Requeue)
-	err = r.client.Get(context.TODO(), req.NamespacedName, rbdMirror)
-	assert.NoError(t, err)
-	assert.Equal(t, "Ready", rbdMirror.Status.Phase, rbdMirror)
+	t.Run("success - peers are configured", func(t *testing.T) {
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+		r := &ReconcileCephRBDMirror{
+			client:           cl,
+			scheme:           s,
+			context:          c,
+			peers:            make(map[string]*peerSpec),
+			opManagerContext: ctx,
+		}
+		bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token
+		peerSecret := &v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      peerSecretName,
+				Namespace: namespace,
+			},
+			Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")},
+			Type: k8sutil.RookType,
+		}
+		_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{})
+		assert.NoError(t, err)
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+		err = r.client.Get(context.TODO(), req.NamespacedName, rbdMirror)
+		assert.NoError(t, err)
+		assert.Equal(t, "Ready", rbdMirror.Status.Phase, rbdMirror)
+	})
 }

pkg/operator/ceph/cluster/rbd/mirror.go

@@ -18,7 +18,6 @@ limitations under the License.
 package rbd
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
@@ -44,7 +43,6 @@ var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait
 
 // Start begins the process of running rbd mirroring daemons.
 func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) error {
-	ctx := context.TODO()
 	// Validate pod's memory if specified
 	err := controller.CheckPodMemory(cephv1.ResourcesKeyRBDMirror, cephRBDMirror.Spec.Resources, cephRbdMirrorPodMinimumMemory)
 	if err != nil {
@@ -86,7 +84,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
 		return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name)
 	}
 
-	if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(ctx, d, metav1.CreateOptions{}); err != nil {
+	if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(r.opManagerContext, d, metav1.CreateOptions{}); err != nil {
 		if !kerrors.IsAlreadyExists(err) {
 			return errors.Wrapf(err, "failed to create %q deployment", resourceName)
 		}
@@ -95,11 +93,11 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro
 		if err := updateDeploymentAndWait(r.context, r.clusterInfo, d, config.RbdMirrorType, daemonConf.DaemonID, r.cephClusterSpec.SkipUpgradeChecks, false); err != nil {
 			// fail could be an issue updating label selector (immutable), so try del and recreate
 			logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err)
-			err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(ctx, cephRBDMirror.Name, metav1.DeleteOptions{})
+			err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(r.opManagerContext, cephRBDMirror.Name, metav1.DeleteOptions{})
 			if err != nil {
 				return errors.Wrapf(err, "failed to delete rbd-mirror %q during del-and-recreate update attempt", resourceName)
 			}
-			if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(ctx, d, metav1.CreateOptions{}); err != nil {
+			if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(r.opManagerContext, d, metav1.CreateOptions{}); err != nil {
 				return errors.Wrapf(err, "failed to recreate rbd-mirror deployment %q during del-and-recreate update attempt", resourceName)
 			}
 		}

pkg/operator/ceph/cluster/register_controllers.go

@@ -1,111 +0,0 @@
-/*
-Copyright 2019 The Rook Authors. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cluster
-
-import (
-	"github.com/pkg/errors"
-	"github.com/rook/rook/pkg/clusterd"
-	"github.com/rook/rook/pkg/operator/ceph/client"
-	"github.com/rook/rook/pkg/operator/ceph/cluster/crash"
-	"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
-	"github.com/rook/rook/pkg/operator/ceph/disruption/clusterdisruption"
-	"github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig"
-	"github.com/rook/rook/pkg/operator/ceph/disruption/machinedisruption"
-	"github.com/rook/rook/pkg/operator/ceph/disruption/machinelabel"
-	"github.com/rook/rook/pkg/operator/ceph/file"
-	"github.com/rook/rook/pkg/operator/ceph/file/mirror"
-	"github.com/rook/rook/pkg/operator/ceph/nfs"
-	"github.com/rook/rook/pkg/operator/ceph/object"
-	"github.com/rook/rook/pkg/operator/ceph/object/realm"
-	objectuser "github.com/rook/rook/pkg/operator/ceph/object/user"
-	"github.com/rook/rook/pkg/operator/ceph/object/zone"
-	"github.com/rook/rook/pkg/operator/ceph/object/zonegroup"
-	"github.com/rook/rook/pkg/operator/ceph/pool"
-
-	"sigs.k8s.io/controller-runtime/pkg/manager"
-)
-
-var (
-	// EnableMachineDisruptionBudget checks whether machine disruption budget is enabled
-	EnableMachineDisruptionBudget bool
-)
-
-// AddToManagerFuncsMaintenance is a list of functions to add all Controllers to the Manager (entrypoint for controller)
-var AddToManagerFuncsMaintenance = []func(manager.Manager, *controllerconfig.Context) error{
-	clusterdisruption.Add,
-}
-
-// MachineDisruptionBudgetAddToManagerFuncs is a list of fencing related functions to add all Controllers to the Manager (entrypoint for controller)
-var MachineDisruptionBudgetAddToManagerFuncs = []func(manager.Manager, *controllerconfig.Context) error{
-	machinelabel.Add,
-	machinedisruption.Add,
-}
-
-// AddToManagerFuncs is a list of functions to add all Controllers to the Manager (entrypoint for controller)
-var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context) error{
-	crash.Add,
-	pool.Add,
-	objectuser.Add,
-	realm.Add,
-	zonegroup.Add,
-	zone.Add,
-	object.Add,
-	file.Add,
-	nfs.Add,
-	rbd.Add,
-	client.Add,
-	mirror.Add,
-}
-
-// AddToManager adds all the registered controllers to the passed manager.
-// each controller package will have an Add method listed in AddToManagerFuncs
-// which will setup all the necessary watch
-func AddToManager(m manager.Manager, c *controllerconfig.Context, clusterController *ClusterController) error {
-	if c == nil {
-		return errors.New("nil context passed")
-	}
-
-	// Run CephCluster CR
-	if err := Add(m, c.ClusterdContext, clusterController); err != nil {
-		return err
-	}
-
-	// Add Ceph child CR controllers
-	for _, f := range AddToManagerFuncs {
-		if err := f(m, c.ClusterdContext); err != nil {
-			return err
-		}
-	}
-
-	// Add maintenance controllers
-	for _, f := range AddToManagerFuncsMaintenance {
-		if err := f(m, c); err != nil {
-			return err
-		}
-	}
-
-	// If machine disruption budget is enabled let's add the controllers
-	if EnableMachineDisruptionBudget {
-		for _, f := range MachineDisruptionBudgetAddToManagerFuncs {
-			if err := f(m, c); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}

pkg/operator/ceph/cluster/version.go

@@ -18,19 +18,23 @@ limitations under the License.
 package cluster
 
 import (
-	"time"
-
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	daemonclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
-	"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
 )
 
 func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cephver.CephVersion, bool, error) {
-	version, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, detectCephVersionTimeout)
+	version, err := controller.DetectCephVersion(
+		c.rookImage,
+		cluster.Namespace,
+		detectVersionName,
+		cluster.ownerInfo,
+		c.context.Clientset,
+		cluster.Spec,
+	)
 	if err != nil {
 		return nil, false, err
 	}
@@ -113,44 +117,6 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion,
 	return false, nil
 }
 
-// detectCephVersion loads the ceph version from the image and checks that it meets the version requirements to
-// run in the cluster
-func (c *cluster) detectCephVersion(rookImage, cephImage string, timeout time.Duration) (*cephver.CephVersion, error) {
-	logger.Infof("detecting the ceph image version for image %s...", cephImage)
-	versionReporter, err := cmdreporter.New(
-		c.context.Clientset, c.ownerInfo,
-		detectVersionName, detectVersionName, c.Namespace,
-		[]string{"ceph"}, []string{"--version"},
-		rookImage, cephImage)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to set up ceph version job")
-	}
-
-	job := versionReporter.Job()
-	job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter"
-
-	// Apply the same placement for the ceph version detection as the mon daemons except for PodAntiAffinity
-	cephv1.GetMonPlacement(c.Spec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)
-	job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil
-
-	stdout, stderr, retcode, err := versionReporter.Run(timeout)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to complete ceph version job")
-	}
-	if retcode != 0 {
-		return nil, errors.Errorf(`ceph version job returned failure with retcode %d.
-  stdout: %s
-  stderr: %s`, retcode, stdout, stderr)
-	}
-
-	version, err := cephver.ExtractCephVersion(stdout)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to extract ceph version")
-	}
-	logger.Infof("detected ceph image version: %q", version)
-	return version, nil
-}
-
 func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
 	if !c.Spec.External.Enable {
 		if !version.IsAtLeast(cephver.Minimum) {
@@ -175,8 +141,9 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
 	// state we are in and if we should upgrade or not
 
 	// Try to load clusterInfo so we can compare the running version with the one from the spec image
-	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.Namespace)
+	clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.ClusterInfo.Context, c.Namespace)
 	if err == nil {
+		clusterInfo.Context = c.ClusterInfo.Context
 		// Write connection info (ceph config file and keyring) for ceph commands
 		err = mon.WriteConnectionConfig(c.context, clusterInfo)
 		if err != nil {

pkg/operator/ceph/cluster/version_test.go

@@ -18,11 +18,13 @@ limitations under the License.
 package cluster
 
 import (
+	"context"
 	"encoding/json"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/client"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	testop "github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
@@ -149,6 +151,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) {
 func TestMinVersion(t *testing.T) {
 	c := testSpec(t)
 	c.Spec.CephVersion.AllowUnsupported = true
+	c.ClusterInfo = &client.ClusterInfo{Context: context.TODO()}
 
 	// All versions less than 14.2.5 are invalid
 	v := &cephver.CephVersion{Major: 13, Minor: 2, Extra: 3}
@@ -167,6 +170,7 @@ func TestMinVersion(t *testing.T) {
 
 func TestSupportedVersion(t *testing.T) {
 	c := testSpec(t)
+	c.ClusterInfo = &client.ClusterInfo{Context: context.TODO()}
 
 	// Supported versions are valid
 	v := &cephver.CephVersion{Major: 14, Minor: 2, Extra: 12}

pkg/operator/ceph/config/keyring/store.go

@@ -20,8 +20,6 @@ limitations under the License.
 package keyring
 
 import (
-	"context"
-
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
@@ -102,9 +100,8 @@ func (k *SecretStore) CreateOrUpdate(resourceName string, keyring string) error
 
 // Delete deletes the keyring secret for the resource.
 func (k *SecretStore) Delete(resourceName string) error {
-	ctx := context.TODO()
 	secretName := keyringSecretName(resourceName)
-	err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Delete(ctx, secretName, metav1.DeleteOptions{})
+	err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Delete(k.clusterInfo.Context, secretName, metav1.DeleteOptions{})
 	if err != nil && !kerrors.IsNotFound(err) {
 		logger.Warningf("failed to delete keyring secret for %q. user may need to delete the resource manually. %v", secretName, err)
 	}
@@ -114,13 +111,12 @@ func (k *SecretStore) Delete(resourceName string) error {
 
 // CreateSecret creates or update a kubernetes secret
 func (k *SecretStore) CreateSecret(secret *v1.Secret) error {
-	ctx := context.TODO()
 	secretName := secret.ObjectMeta.Name
-	_, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Get(ctx, secretName, metav1.GetOptions{})
+	_, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Get(k.clusterInfo.Context, secretName, metav1.GetOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debugf("creating secret for %s", secretName)
-			if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+			if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Create(k.clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil {
 				return errors.Wrapf(err, "failed to create secret for %s", secretName)
 			}
 			return nil
@@ -129,7 +125,7 @@ func (k *SecretStore) CreateSecret(secret *v1.Secret) error {
 	}
 
 	logger.Debugf("updating secret for %s", secretName)
-	if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
+	if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Update(k.clusterInfo.Context, secret, metav1.UpdateOptions{}); err != nil {
 		return errors.Wrapf(err, "failed to update secret for %s", secretName)
 	}
 	return nil

pkg/operator/ceph/config/keyring/store_test.go

@@ -50,7 +50,7 @@ func TestGenerateKey(t *testing.T) {
 	}
 	ns := "rook-ceph"
 	ownerInfo := k8sutil.OwnerInfo{}
-	s := GetSecretStore(ctx, &cephclient.ClusterInfo{Namespace: ns}, &ownerInfo)
+	s := GetSecretStore(ctx, cephclient.AdminClusterInfo(ns), &ownerInfo)
 
 	generateKey = "generatedsecretkey"
 	failGenerateKey = false

pkg/operator/ceph/config/monstore_test.go

@@ -50,7 +50,7 @@ func TestMonStore_Set(t *testing.T) {
 			return "", nil
 		}
 
-	monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"})
+	monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster"))
 
 	// setting with spaces converts to underscores
 	e := monStore.Set("global", "debug ms", "10")
@@ -95,7 +95,7 @@ func TestMonStore_Delete(t *testing.T) {
 			return "", nil
 		}
 
-	monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"})
+	monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster"))
 
 	// ceph config rm called as expected
 	e := monStore.Delete("global", "debug ms")
@@ -134,7 +134,7 @@ func TestMonStore_GetDaemon(t *testing.T) {
 			return execReturn, nil
 		}
 
-	monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"})
+	monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster"))
 
 	// ceph config get called as expected
 	options, e := monStore.GetDaemon("client.rgw.test.a")
@@ -177,7 +177,7 @@ func TestMonStore_DeleteDaemon(t *testing.T) {
 			return execReturn, nil
 		}
 
-	monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"})
+	monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster"))
 
 	// ceph config rm rgw_enable_usage_log called as expected
 	e := monStore.DeleteDaemon("client.rgw.test.a")
@@ -208,7 +208,7 @@ func TestMonStore_SetAll(t *testing.T) {
 			return "", nil
 		}
 
-	monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"})
+	monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster"))
 
 	cfgOverrides := []Option{
 		configOverride("global", "debug ms", "10"), // setting w/ spaces converts to underscores

pkg/operator/ceph/config/store.go

@@ -19,7 +19,6 @@ limitations under the License.
 package config
 
 import (
-	"context"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -62,9 +61,8 @@ func GetStore(context *clusterd.Context, namespace string, ownerInfo *k8sutil.Ow
 
 // CreateOrUpdate creates or updates the stored Ceph config based on the cluster info.
 func (s *Store) CreateOrUpdate(clusterInfo *cephclient.ClusterInfo) error {
-	ctx := context.TODO()
 	// these are used for all ceph daemons on the commandline and must *always* be stored
-	if err := s.createOrUpdateMonHostSecrets(ctx, clusterInfo); err != nil {
+	if err := s.createOrUpdateMonHostSecrets(clusterInfo); err != nil {
 		return errors.Wrap(err, "failed to store mon host configs")
 	}
 
@@ -72,7 +70,7 @@ func (s *Store) CreateOrUpdate(clusterInfo *cephclient.ClusterInfo) error {
 }
 
 // update "mon_host" and "mon_initial_members" in the stored config
-func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *cephclient.ClusterInfo) error {
+func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephclient.ClusterInfo) error {
 
 	// extract a list of just the monitor names, which will populate the "mon initial members"
 	// and "mon hosts" global config field
@@ -97,11 +95,11 @@ func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *c
 		return errors.Wrapf(err, "failed to set owner reference to moh host secret %q", secret.Name)
 	}
 
-	_, err = clientset.CoreV1().Secrets(s.namespace).Get(ctx, StoreName, metav1.GetOptions{})
+	_, err = clientset.CoreV1().Secrets(s.namespace).Get(clusterInfo.Context, StoreName, metav1.GetOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			logger.Debugf("creating config secret %q", secret.Name)
-			if _, err := clientset.CoreV1().Secrets(s.namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+			if _, err := clientset.CoreV1().Secrets(s.namespace).Create(clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil {
 				return errors.Wrapf(err, "failed to create config secret %+v", secret)
 			}
 		} else {
@@ -110,7 +108,7 @@ func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *c
 	}
 
 	logger.Debugf("updating config secret %q", secret.Name)
-	if _, err := clientset.CoreV1().Secrets(s.namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
+	if _, err := clientset.CoreV1().Secrets(s.namespace).Update(clusterInfo.Context, secret, metav1.UpdateOptions{}); err != nil {
 		return errors.Wrapf(err, "failed to update config secret %+v", secret)
 	}
 

pkg/operator/ceph/controller.go

@@ -0,0 +1,172 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package operator
+
+import (
+	"context"
+	"strings"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/coreos/pkg/capnslog"
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/discover"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	v1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	controllerName = "rook-ceph-operator-config-controller"
+)
+
+// ReconcileConfig reconciles a Ceph Operator config
+type ReconcileConfig struct {
+	client           client.Client
+	context          *clusterd.Context
+	config           opcontroller.OperatorConfig
+	opManagerContext context.Context
+}
+
+// Add creates a new Operator configuration Controller and adds it to the Manager. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+	return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, config opcontroller.OperatorConfig) reconcile.Reconciler {
+	return &ReconcileConfig{
+		client:           mgr.GetClient(),
+		context:          context,
+		config:           config,
+		opManagerContext: opManagerContext,
+	}
+}
+
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+	logger.Infof("%s successfully started", controllerName)
+
+	// Watch for ConfigMap (operator config)
+	err = c.Watch(&source.Kind{
+		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(mgr.GetClient()))
+	if err != nil {
+		return err
+	}
+
+	// Watch for Secret (admission controller secret)
+	err = c.Watch(&source.Kind{
+		Type: &v1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(mgr.GetClient()))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Reconcile reads that state of the cluster for a CephClient object and makes changes based on the state read
+// and what is in the CephClient.Spec
+// The Controller will requeue the Request to be processed again if the returned error is non-nil or
+// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
+func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
+	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
+	reconcileResponse, err := r.reconcile(request)
+	if err != nil {
+		logger.Errorf("failed to reconcile %v", err)
+	}
+
+	return reconcileResponse, err
+}
+
+func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result, error) {
+	// Fetch the operator's configmap
+	opConfig := &v1.ConfigMap{}
+	logger.Debugf("reconciling %s", request.NamespacedName)
+	err := r.client.Get(r.opManagerContext, request.NamespacedName, opConfig)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			logger.Debug("operator's configmap resource not found. will use default value or env var.")
+		} else {
+			// Error reading the object - requeue the request.
+			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap")
+		}
+	} else {
+		// Populate the operator's config
+		r.config.Parameters = opConfig.Data
+	}
+
+	// Reconcile Ceph CLI timeout, since the clusterd context is passed to by pointer to all CRD
+	// controllers they will receive the update
+	opcontroller.SetCephCommandsTimeout(r.config.Parameters)
+
+	// Reconcile Operator's logging level
+	err = reconcileOperatorLogLevel(opConfig.Data)
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, err
+	}
+
+	// Reconcile discovery daemon
+	err = r.reconcileDiscoveryDaemon()
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, err
+	}
+
+	// Reconcile webhook secret
+	// This is done in the predicate function
+
+	logger.Infof("%s done reconciling", controllerName)
+	return reconcile.Result{}, nil
+}
+
+func reconcileOperatorLogLevel(data map[string]string) error {
+	rookLogLevel := k8sutil.GetValue(data, "ROOK_LOG_LEVEL", "INFO")
+	logLevel, err := capnslog.ParseLevel(strings.ToUpper(rookLogLevel))
+	if err != nil {
+		return errors.Wrapf(err, "failed to load ROOK_LOG_LEVEL %q.", rookLogLevel)
+	}
+
+	capnslog.SetGlobalLogLevel(logLevel)
+	return nil
+}
+
+func (r *ReconcileConfig) reconcileDiscoveryDaemon() error {
+	rookDiscover := discover.New(r.context.Clientset)
+	if opcontroller.DiscoveryDaemonEnabled(r.config.Parameters) {
+		if err := rookDiscover.Start(r.opManagerContext, r.config.OperatorNamespace, r.config.Image, r.config.ServiceAccount, true); err != nil {
+			return errors.Wrap(err, "failed to start device discovery daemonset")
+		}
+	} else {
+		if err := rookDiscover.Stop(r.opManagerContext, r.config.OperatorNamespace); err != nil {
+			return errors.Wrap(err, "failed to stop device discovery daemonset")
+		}
+	}
+
+	return nil
+}

pkg/operator/ceph/controller/controller_utils.go

@@ -18,7 +18,6 @@ package controller
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -35,6 +34,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+// OperatorConfig represents the configuration of the operator
+type OperatorConfig struct {
+	OperatorNamespace string
+	Image             string
+	ServiceAccount    string
+	NamespaceToWatch  string
+	Parameters        map[string]string
+}
+
 const (
 	// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
 	OperatorSettingConfigMapName string = "rook-ceph-operator-config"
@@ -45,9 +53,6 @@ const (
 
 	// OperatorNotInitializedMessage is the message we print when the Operator is not ready to reconcile, typically the ceph.conf has not been generated yet
 	OperatorNotInitializedMessage = "skipping reconcile since operator is still initializing"
-
-	// CancellingOrchestrationMessage is the message to indicate a reconcile was cancelled
-	CancellingOrchestrationMessage = "CANCELLING CURRENT ORCHESTRATION"
 )
 
 var (
@@ -61,6 +66,9 @@ var (
 	// WaitForRequeueIfCephClusterNotReady waits for the CephCluster to be ready
 	WaitForRequeueIfCephClusterNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
 
+	// WaitForRequeueIfCephClusterIsUpgrading waits until the upgrade is complete
+	WaitForRequeueIfCephClusterIsUpgrading = reconcile.Result{Requeue: true, RequeueAfter: time.Minute}
+
 	// WaitForRequeueIfFinalizerBlocked waits for resources to be cleaned up before the finalizer can be removed
 	WaitForRequeueIfFinalizerBlocked = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
 
@@ -71,21 +79,23 @@ var (
 	OperatorCephBaseImageVersion string
 )
 
-func FlexDriverEnabled(context *clusterd.Context) bool {
-	// Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the flex driver
-	value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false")
+func FlexDriverEnabled(ctx *clusterd.Context) bool {
+	// Ignore the error. In the remote chance that the configmap fails to be read, we will default
+	// to disabling the flex driver
+	// It's ok to use a TODO context here since FlexDriverEnabled is called by a manager which
+	// already has a context
+	context := context.TODO()
+	value, _ := k8sutil.GetOperatorSetting(context, ctx.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false")
 	return value == "true"
 }
 
-func DiscoveryDaemonEnabled(context *clusterd.Context) bool {
-	// Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the discovery daemon
-	value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_DISCOVERY_DAEMON", "false")
-	return value == "true"
+func DiscoveryDaemonEnabled(data map[string]string) bool {
+	return k8sutil.GetValue(data, "ROOK_ENABLE_DISCOVERY_DAEMON", "false") == "true"
 }
 
 // SetCephCommandsTimeout sets the timeout value of Ceph commands which are executed from Rook
-func SetCephCommandsTimeout(context *clusterd.Context) {
-	strTimeoutSeconds, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15")
+func SetCephCommandsTimeout(data map[string]string) {
+	strTimeoutSeconds := k8sutil.GetValue(data, "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15")
 	timeoutSeconds, err := strconv.Atoi(strTimeoutSeconds)
 	if err != nil || timeoutSeconds < 1 {
 		logger.Warningf("ROOK_CEPH_COMMANDS_TIMEOUT is %q but it should be >= 1, set the default value 15", strTimeoutSeconds)
@@ -94,18 +104,6 @@ func SetCephCommandsTimeout(context *clusterd.Context) {
 	exec.CephCommandsTimeout = time.Duration(timeoutSeconds) * time.Second
 }
 
-// CheckForCancelledOrchestration checks whether a cancellation has been requested
-func CheckForCancelledOrchestration(context *clusterd.Context) error {
-	defer context.RequestCancelOrchestration.UnSet()
-
-	// Check whether we need to cancel the orchestration
-	if context.RequestCancelOrchestration.IsSet() {
-		return errors.New(CancellingOrchestrationMessage)
-	}
-
-	return nil
-}
-
 // canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely.
 func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool {
 	// Get a list of all the keys causing the HEALTH_ERR status.

pkg/operator/ceph/controller/controller_utils_test.go

@@ -17,17 +17,12 @@ limitations under the License.
 package controller
 
 import (
-	"context"
 	"testing"
 	"time"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/util/exec"
 	"github.com/stretchr/testify/assert"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
 )
 
 func CreateTestClusterFromStatusDetails(details map[string]cephv1.CephHealthMessage) cephv1.CephCluster {
@@ -79,28 +74,14 @@ func TestCanIgnoreHealthErrStatusInReconcile(t *testing.T) {
 }
 
 func TestSetCephCommandsTimeout(t *testing.T) {
-	clientset := fake.NewSimpleClientset()
-	ctx := context.TODO()
-	cm := &v1.ConfigMap{}
-	cm.Name = "rook-ceph-operator-config"
-	_, err := clientset.CoreV1().ConfigMaps("").Create(ctx, cm, metav1.CreateOptions{})
-	assert.NoError(t, err)
-	context := &clusterd.Context{Clientset: clientset}
-
-	SetCephCommandsTimeout(context)
+	SetCephCommandsTimeout(map[string]string{})
 	assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout)
 
 	exec.CephCommandsTimeout = 0
-	cm.Data = map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "0"}
-	_, err = clientset.CoreV1().ConfigMaps("").Update(ctx, cm, metav1.UpdateOptions{})
-	assert.NoError(t, err)
-	SetCephCommandsTimeout(context)
+	SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "0"})
 	assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout)
 
 	exec.CephCommandsTimeout = 0
-	cm.Data = map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"}
-	_, err = clientset.CoreV1().ConfigMaps("").Update(ctx, cm, metav1.UpdateOptions{})
-	assert.NoError(t, err)
-	SetCephCommandsTimeout(context)
+	SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"})
 	assert.Equal(t, 1*time.Second, exec.CephCommandsTimeout)
 }

pkg/operator/ceph/controller/finalizer.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -74,6 +75,10 @@ func AddFinalizerIfNotPresent(client client.Client, obj client.Object) error {
 
 // RemoveFinalizer removes a finalizer from an object
 func RemoveFinalizer(client client.Client, obj client.Object) error {
+	err := client.Get(context.TODO(), types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, obj)
+	if err != nil {
+		return errors.Wrap(err, "failed to get the latest version of the object")
+	}
 	objectFinalizer := buildFinalizerName(obj.GetObjectKind().GroupVersionKind().Kind)
 	accessor, err := meta.Accessor(obj)
 	if err != nil {

pkg/operator/ceph/controller/predicate.go

@@ -18,7 +18,9 @@ package controller
 
 import (
 	"encoding/json"
+	"os"
 	"strings"
+	"syscall"
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/google/go-cmp/cmp"
@@ -321,58 +323,6 @@ func WatchControllerPredicate() predicate.Funcs {
 	}
 }
 
-// WatchCephClusterPredicate is a predicate used by child controllers such as Filesystem or Object
-// It watch for CR changes on the CephCluster object and reconciles if this needs to be propagated
-// For instance the logCollector option from the CephCluster spec affects the configuration of rgw pods
-// So if it changes we must update the deployment
-func WatchCephClusterPredicate() predicate.Funcs {
-	return predicate.Funcs{
-		CreateFunc: func(e event.CreateEvent) bool {
-			logger.Debug("create event from a CR")
-			return true
-		},
-		DeleteFunc: func(e event.DeleteEvent) bool {
-			logger.Debug("delete event from a CR")
-			return true
-		},
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			// resource.Quantity has non-exportable fields, so we use its comparator method
-			resourceQtyComparer := cmp.Comparer(func(x, y resource.Quantity) bool { return x.Cmp(y) == 0 })
-
-			switch objOld := e.ObjectOld.(type) {
-			case *cephv1.CephCluster:
-				objNew := e.ObjectNew.(*cephv1.CephCluster)
-				logger.Debug("update event on CephCluster CR")
-				// If the labels "do_not_reconcile" is set on the object, let's not reconcile that request
-				isDoNotReconcile := IsDoNotReconcile(objNew.GetLabels())
-				if isDoNotReconcile {
-					logger.Debugf("object %q matched on update but %q label is set, doing nothing", DoNotReconcileLabelName, objNew.Name)
-					return false
-				}
-				diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer)
-				if diff != "" {
-					// The image change (upgrade) is being taking care by watchControllerPredicate() in the cluster package
-					if objOld.Spec.CephVersion.Image != objNew.Spec.CephVersion.Image {
-						return false
-					}
-					// If the log collector setting changes let's reconcile the child controllers
-					if !cmp.Equal(objOld.Spec.LogCollector, objNew.Spec.LogCollector) {
-						logger.Debug("log collector option changed, reconciling")
-						return true
-					}
-				} else if objOld.GetGeneration() != objNew.GetGeneration() {
-					logger.Debugf("skipping resource %q update with unchanged spec", objNew.Name)
-				}
-			}
-
-			return false
-		},
-		GenericFunc: func(e event.GenericEvent) bool {
-			return false
-		},
-	}
-}
-
 func objectToBeDeleted(oldObj, newObj client.Object) bool {
 	return !oldObj.GetDeletionTimestamp().Equal(newObj.GetDeletionTimestamp())
 }
@@ -675,3 +625,8 @@ func IsDoNotReconcile(labels map[string]string) bool {
 
 	return false
 }
+
+func ReloadManager() {
+	p, _ := os.FindProcess(os.Getpid())
+	_ = p.Signal(syscall.SIGHUP)
+}

pkg/operator/ceph/controller/spec.go

@@ -18,7 +18,6 @@ limitations under the License.
 package controller
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"path"
@@ -697,7 +696,7 @@ func ConfigureExternalMetricsEndpoint(ctx *clusterd.Context, monitoringSpec ceph
 	}
 
 	// Get the endpoint to see if anything needs to be updated
-	currentEndpoints, err := ctx.Clientset.CoreV1().Endpoints(clusterInfo.Namespace).Get(context.TODO(), endpoint.Name, metav1.GetOptions{})
+	currentEndpoints, err := ctx.Clientset.CoreV1().Endpoints(clusterInfo.Namespace).Get(clusterInfo.Context, endpoint.Name, metav1.GetOptions{})
 	if err != nil && !kerrors.IsNotFound(err) {
 		return errors.Wrap(err, "failed to fetch endpoints")
 	}

pkg/operator/ceph/controller/spec_test.go

@@ -249,16 +249,13 @@ func TestExtractMgrIP(t *testing.T) {
 }
 
 func TestConfigureExternalMetricsEndpoint(t *testing.T) {
+	clusterInfo := cephclient.AdminClusterInfo("rook-ceph")
 	t.Run("spec and current active mgr endpoint identical with no existing endpoint object", func(t *testing.T) {
 		monitoringSpec := cephv1.MonitoringSpec{
 			Enabled:              true,
 			RulesNamespace:       "rook-ceph",
 			ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}},
 		}
-		clusterInfo := &cephclient.ClusterInfo{
-			FSID:      "id",
-			Namespace: "rook-ceph",
-		}
 		executor := &exectest.MockExecutor{
 			MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 				logger.Infof("Command: %s %v", command, args)
@@ -289,10 +286,6 @@ func TestConfigureExternalMetricsEndpoint(t *testing.T) {
 			RulesNamespace:       "rook-ceph",
 			ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}},
 		}
-		clusterInfo := &cephclient.ClusterInfo{
-			FSID:      "id",
-			Namespace: "rook-ceph",
-		}
 		executor := &exectest.MockExecutor{
 			MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 				logger.Infof("Command: %s %v", command, args)
@@ -322,10 +315,6 @@ func TestConfigureExternalMetricsEndpoint(t *testing.T) {
 			RulesNamespace:       "rook-ceph",
 			ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}},
 		}
-		clusterInfo := &cephclient.ClusterInfo{
-			FSID:      "id",
-			Namespace: "rook-ceph",
-		}
 		executor := &exectest.MockExecutor{
 			MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 				logger.Infof("Command: %s %v", command, args)
@@ -360,10 +349,6 @@ func TestConfigureExternalMetricsEndpoint(t *testing.T) {
 			RulesNamespace:       "rook-ceph",
 			ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}},
 		}
-		clusterInfo := &cephclient.ClusterInfo{
-			FSID:      "id",
-			Namespace: "rook-ceph",
-		}
 		executor := &exectest.MockExecutor{
 			MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
 				logger.Infof("Command: %s %v", command, args)

pkg/operator/ceph/controller/version.go

@@ -17,13 +17,24 @@ limitations under the License.
 package controller
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	"github.com/rook/rook/pkg/operator/ceph/config"
+	"github.com/rook/rook/pkg/operator/ceph/version"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
+	"k8s.io/client-go/kubernetes"
 )
 
+const detectCephVersionTimeout = 15 * time.Minute
+
 // ValidateCephVersionsBetweenLocalAndExternalClusters makes sure an external cluster can be connected
 // by checking the external ceph versions available and comparing it with the local image provided
 func ValidateCephVersionsBetweenLocalAndExternalClusters(context *clusterd.Context, clusterInfo *client.ClusterInfo) (cephver.CephVersion, error) {
@@ -47,3 +58,79 @@ func GetImageVersion(cephCluster cephv1.CephCluster) (*cephver.CephVersion, erro
 
 	return nil, errors.New("attempt to determine ceph version for the current cluster image timed out")
 }
+
+// DetectCephVersion loads the ceph version from the image and checks that it meets the version requirements to
+// run in the cluster
+func DetectCephVersion(rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*version.CephVersion, error) {
+	cephImage := cephClusterSpec.CephVersion.Image
+	logger.Infof("detecting the ceph image version for image %s...", cephImage)
+	versionReporter, err := cmdreporter.New(
+		clientset,
+		ownerInfo,
+		jobName,
+		jobName,
+		namespace,
+		[]string{"ceph"},
+		[]string{"--version"},
+		rookImage,
+		cephImage,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to set up ceph version job")
+	}
+
+	job := versionReporter.Job()
+	job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter"
+
+	// Apply the same placement for the ceph version detection as the mon daemons except for PodAntiAffinity
+	cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)
+	job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil
+
+	stdout, stderr, retcode, err := versionReporter.Run(detectCephVersionTimeout)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to complete ceph version job")
+	}
+	if retcode != 0 {
+		return nil, errors.Errorf(`ceph version job returned failure with retcode %d. `+
+			`stdout: %s. `+
+			`stderr: %s`,
+			retcode,
+			stdout,
+			stderr,
+		)
+	}
+
+	version, err := cephver.ExtractCephVersion(stdout)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to extract ceph version")
+	}
+	logger.Infof("detected ceph image version: %q", version)
+	return version, nil
+}
+
+func CurrentAndDesiredCephVersion(rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
+	// Detect desired CephCluster version
+	desiredCephVersion, err := DetectCephVersion(rookImage, namespace, fmt.Sprintf("%s-detect-version", jobName), ownerInfo, context.Clientset, cephClusterSpec)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to detect ceph image version")
+	}
+
+	// Check the ceph version of the running monitors
+	runningMonDaemonVersion, err := client.LeastUptodateDaemonVersion(context, clusterInfo, config.MonType)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to retrieve current ceph %q version", config.MonType)
+	}
+
+	return desiredCephVersion, &runningMonDaemonVersion, nil
+}
+
+func ErrorCephUpgradingRequeue(runningCephVersion, desiredCephVersion *version.CephVersion) error {
+	return errors.Errorf(`waiting for ceph monitors upgrade to finish. `+
+		`current version: %s. `+
+		`expected version: %s. `+
+		`will reconcile again in %s`,
+		runningCephVersion.String(),
+		desiredCephVersion.String(),
+		WaitForRequeueIfCephClusterIsUpgrading.RequeueAfter.String(),
+	)
+}

pkg/operator/ceph/controller_test.go

@@ -0,0 +1,202 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package operator
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/pkg/capnslog"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+	"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/test"
+	"github.com/rook/rook/pkg/util/exec"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+func TestOperatorController(t *testing.T) {
+	ctx := context.TODO()
+	// Set DEBUG logging
+	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
+	os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
+	name := controller.OperatorSettingConfigMapName
+	namespace := "rook-ceph"
+	req := reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	t.Run("success - normal run", func(t *testing.T) {
+		fakeClientSet := test.New(t, 1)
+		test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0")
+		c := &clusterd.Context{
+			Clientset:     fakeClientSet,
+			RookClientset: rookclient.NewSimpleClientset(),
+		}
+
+		// Register operator types with the runtime scheme.
+		s := scheme.Scheme
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{})
+
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).Build()
+
+		// Create a ReconcileCSI object with the scheme and fake client.
+		r := &ReconcileConfig{
+			client:  cl,
+			context: c,
+			config: controller.OperatorConfig{
+				OperatorNamespace: namespace,
+				Image:             "rook",
+				ServiceAccount:    "foo",
+			},
+		}
+
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+	})
+
+	t.Run("success - env set for command timeout", func(t *testing.T) {
+		fakeClientSet := test.New(t, 1)
+		test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0")
+		c := &clusterd.Context{
+			Clientset:     fakeClientSet,
+			RookClientset: rookclient.NewSimpleClientset(),
+		}
+
+		// Register operator types with the runtime scheme.
+		s := scheme.Scheme
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{})
+
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).Build()
+
+		// Create a ReconcileCSI object with the scheme and fake client.
+		r := &ReconcileConfig{
+			client:  cl,
+			context: c,
+			config: controller.OperatorConfig{
+				OperatorNamespace: namespace,
+				Image:             "rook",
+				ServiceAccount:    "foo",
+			},
+		}
+		os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "10")
+		defer os.Unsetenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS")
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+		assert.Equal(t, time.Second*10, exec.CephCommandsTimeout)
+	})
+
+	t.Run("success - cm set for command timeout", func(t *testing.T) {
+		fakeClientSet := test.New(t, 1)
+		test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0")
+		c := &clusterd.Context{
+			Clientset:     fakeClientSet,
+			RookClientset: rookclient.NewSimpleClientset(),
+		}
+
+		// Register operator types with the runtime scheme.
+		// s := scheme.Scheme
+		s := clientgoscheme.Scheme
+		// s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{})
+
+		opConfigCM := &v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      controller.OperatorSettingConfigMapName,
+				Namespace: namespace,
+			},
+			Data: map[string]string{
+				"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "20",
+			},
+		}
+
+		object := []runtime.Object{
+			opConfigCM,
+		}
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+
+		// Create a ReconcileCSI object with the scheme and fake client.
+		r := &ReconcileConfig{
+			client:  cl,
+			context: c,
+			config: controller.OperatorConfig{
+				OperatorNamespace: namespace,
+				Image:             "rook",
+				ServiceAccount:    "foo",
+			},
+		}
+
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+		assert.Equal(t, time.Second*20, exec.CephCommandsTimeout)
+	})
+
+	t.Run("success - env set for discovery daemon", func(t *testing.T) {
+		fakeClientSet := test.New(t, 1)
+		test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0")
+		c := &clusterd.Context{
+			Clientset:     fakeClientSet,
+			RookClientset: rookclient.NewSimpleClientset(),
+		}
+
+		// Register operator types with the runtime scheme.
+		s := scheme.Scheme
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{})
+
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).Build()
+
+		// Create a ReconcileCSI object with the scheme and fake client.
+		r := &ReconcileConfig{
+			client:  cl,
+			context: c,
+			config: controller.OperatorConfig{
+				OperatorNamespace: namespace,
+				Image:             "rook",
+				ServiceAccount:    "foo",
+			},
+		}
+		os.Setenv("ROOK_ENABLE_DISCOVERY_DAEMON", "true")
+		defer os.Unsetenv("ROOK_ENABLE_DISCOVERY_DAEMON")
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+		ds, err := c.Clientset.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{})
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(ds.Items), ds)
+		assert.Equal(t, "rook-discover", ds.Items[0].Name, ds)
+	})
+}

pkg/operator/ceph/cr_manager.go

@@ -21,8 +21,28 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/daemon/ceph/agent"
+	"github.com/rook/rook/pkg/operator/ceph/client"
 	"github.com/rook/rook/pkg/operator/ceph/cluster"
+	"github.com/rook/rook/pkg/operator/ceph/cluster/crash"
+	"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/ceph/csi"
+	"github.com/rook/rook/pkg/operator/ceph/disruption/clusterdisruption"
 	"github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig"
+	"github.com/rook/rook/pkg/operator/ceph/disruption/machinedisruption"
+	"github.com/rook/rook/pkg/operator/ceph/disruption/machinelabel"
+	"github.com/rook/rook/pkg/operator/ceph/file"
+	"github.com/rook/rook/pkg/operator/ceph/file/mirror"
+	"github.com/rook/rook/pkg/operator/ceph/nfs"
+	"github.com/rook/rook/pkg/operator/ceph/object"
+	"github.com/rook/rook/pkg/operator/ceph/object/bucket"
+	"github.com/rook/rook/pkg/operator/ceph/object/realm"
+	objectuser "github.com/rook/rook/pkg/operator/ceph/object/user"
+	"github.com/rook/rook/pkg/operator/ceph/object/zone"
+	"github.com/rook/rook/pkg/operator/ceph/object/zonegroup"
+	"github.com/rook/rook/pkg/operator/ceph/pool"
 	"k8s.io/apimachinery/pkg/runtime"
 
 	mapiv1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1"
@@ -52,7 +72,86 @@ var (
 	webhookResources = []webhook.Validator{&cephv1.CephCluster{}, &cephv1.CephBlockPool{}, &cephv1.CephObjectStore{}}
 )
 
-func (o *Operator) startManager(context context.Context, namespaceToWatch string, mgrErrorCh chan error) {
+var (
+	// EnableMachineDisruptionBudget checks whether machine disruption budget is enabled
+	EnableMachineDisruptionBudget bool
+)
+
+// AddToManagerFuncsMaintenance is a list of functions to add all Controllers to the Manager (entrypoint for controller)
+var AddToManagerFuncsMaintenance = []func(manager.Manager, *controllerconfig.Context) error{
+	clusterdisruption.Add,
+}
+
+// MachineDisruptionBudgetAddToManagerFuncs is a list of fencing related functions to add all Controllers to the Manager (entrypoint for controller)
+var MachineDisruptionBudgetAddToManagerFuncs = []func(manager.Manager, *controllerconfig.Context) error{
+	machinelabel.Add,
+	machinedisruption.Add,
+}
+
+// AddToManagerFuncs is a list of functions to add all Controllers to the Manager (entrypoint for controller)
+var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Context, opcontroller.OperatorConfig) error{
+	crash.Add,
+	pool.Add,
+	objectuser.Add,
+	realm.Add,
+	zonegroup.Add,
+	zone.Add,
+	object.Add,
+	file.Add,
+	nfs.Add,
+	rbd.Add,
+	client.Add,
+	mirror.Add,
+	Add,
+	csi.Add,
+	agent.Add,
+	bucket.Add,
+}
+
+// AddToManagerOpFunc is a list of functions to add all Controllers to the Manager (entrypoint for
+// controller)
+// var AddToManagerOpFunc = []func(manager.Manager, *clusterd.Context, opcontroller.OperatorConfig) error{}
+
+// AddToManager adds all the registered controllers to the passed manager.
+// each controller package will have an Add method listed in AddToManagerFuncs
+// which will setup all the necessary watch
+func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context) error {
+	if c == nil {
+		return errors.New("nil context passed")
+	}
+
+	// Run CephCluster CR
+	if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext); err != nil {
+		return err
+	}
+
+	// Add Ceph child CR controllers
+	for _, f := range AddToManagerFuncs {
+		if err := f(m, c.ClusterdContext, opManagerContext, *o.config); err != nil {
+			return err
+		}
+	}
+
+	// Add maintenance controllers
+	for _, f := range AddToManagerFuncsMaintenance {
+		if err := f(m, c); err != nil {
+			return err
+		}
+	}
+
+	// If machine disruption budget is enabled let's add the controllers
+	if EnableMachineDisruptionBudget {
+		for _, f := range MachineDisruptionBudgetAddToManagerFuncs {
+			if err := f(m, c); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan error) {
 	logger.Info("setting up schemes")
 	// Setup Scheme for all resources
 	scheme := runtime.NewScheme()
@@ -67,7 +166,7 @@ func (o *Operator) startManager(context context.Context, namespaceToWatch string
 	// Set up a manager
 	mgrOpts := manager.Options{
 		LeaderElection: false,
-		Namespace:      namespaceToWatch,
+		Namespace:      o.config.NamespaceToWatch,
 		Scheme:         scheme,
 		CertDir:        certDir,
 	}
@@ -79,23 +178,7 @@ func (o *Operator) startManager(context context.Context, namespaceToWatch string
 		return
 	}
 
-	// options to pass to the controllers
-	controllerOpts := &controllerconfig.Context{
-		RookImage:         o.config.Image,
-		ClusterdContext:   o.context,
-		OperatorNamespace: o.config.OperatorNamespace,
-		ReconcileCanaries: &controllerconfig.LockingBool{},
-	}
-
-	// Add the registered controllers to the manager (entrypoint for controllers)
-	err = cluster.AddToManager(mgr, controllerOpts, o.clusterController)
-	if err != nil {
-		mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager")
-		return
-	}
-
-	// Add Webhooks
-	// TODO: this needs a callback with a watcher for this secret so we can quickly enable the webhook if it changes
+	// Add webhook if needed
 	isPresent, err := isSecretPresent(context, o.context)
 	if err != nil {
 		mgrErrorCh <- errors.Wrap(err, "failed to retrieve admission webhook secret")
@@ -117,9 +200,25 @@ func (o *Operator) startManager(context context.Context, namespaceToWatch string
 		}
 	}
 
+	// options to pass to the controllers
+	controllerOpts := &controllerconfig.Context{
+		ClusterdContext:   o.context,
+		ReconcileCanaries: &controllerconfig.LockingBool{},
+		OpManagerContext:  context,
+	}
+
+	// Add the registered controllers to the manager (entrypoint for controllers)
+	err = o.addToManager(mgr, controllerOpts, context)
+	if err != nil {
+		mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager")
+		return
+	}
+
 	logger.Info("starting the controller-runtime manager")
 	if err := mgr.Start(context); err != nil {
 		mgrErrorCh <- errors.Wrap(err, "failed to run the controller-runtime manager")
 		return
 	}
+
+	logger.Info("successfully started the controller-runtime manager")
 }

pkg/operator/ceph/csi/cluster_config.go

@@ -87,9 +87,9 @@ func monEndpoints(mons map[string]*cephclient.MonInfo) []string {
 	return endpoints
 }
 
-// UpdateCsiClusterConfig returns a json-formatted string containing
+// updateCsiClusterConfig returns a json-formatted string containing
 // the cluster-to-mon mapping required to configure ceph csi.
-func UpdateCsiClusterConfig(
+func updateCsiClusterConfig(
 	curr, clusterKey string, mons map[string]*cephclient.MonInfo) (string, error) {
 
 	var (
@@ -158,8 +158,6 @@ func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface, ownerI
 func SaveClusterConfig(
 	clientset kubernetes.Interface, clusterNamespace string,
 	clusterInfo *cephclient.ClusterInfo, l sync.Locker) error {
-	ctx := context.TODO()
-
 	if !CSIEnabled() {
 		return nil
 	}
@@ -173,7 +171,7 @@ func SaveClusterConfig(
 	logger.Debugf("Using %+v for CSI ConfigMap Namespace", csiNamespace)
 
 	// fetch current ConfigMap contents
-	configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(ctx,
+	configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(clusterInfo.Context,
 		ConfigName, metav1.GetOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to fetch current csi config map")
@@ -184,7 +182,7 @@ func SaveClusterConfig(
 	if currData == "" {
 		currData = "[]"
 	}
-	newData, err := UpdateCsiClusterConfig(
+	newData, err := updateCsiClusterConfig(
 		currData, clusterNamespace, clusterInfo.Monitors)
 	if err != nil {
 		return errors.Wrap(err, "failed to update csi config map data")
@@ -192,7 +190,7 @@ func SaveClusterConfig(
 	configMap.Data[ConfigKey] = newData
 
 	// update ConfigMap with new contents
-	if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(ctx, configMap, metav1.UpdateOptions{}); err != nil {
+	if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(clusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil {
 		return errors.Wrapf(err, "failed to update csi config map")
 	}
 

pkg/operator/ceph/csi/cluster_config_test.go

@@ -28,7 +28,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
 	mons := map[string]*cephclient.MonInfo{
 		"foo": {Name: "foo", Endpoint: "1.2.3.4:5000"},
 	}
-	s, err := UpdateCsiClusterConfig("[]", "alpha", mons)
+	s, err := updateCsiClusterConfig("[]", "alpha", mons)
 	assert.NoError(t, err)
 	assert.Equal(t, s,
 		`[{"clusterID":"alpha","monitors":["1.2.3.4:5000"]}]`)
@@ -36,7 +36,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
 	// add a 2nd mon to the current cluster
 	mons["bar"] = &cephclient.MonInfo{
 		Name: "bar", Endpoint: "10.11.12.13:5000"}
-	s, err = UpdateCsiClusterConfig(s, "alpha", mons)
+	s, err = updateCsiClusterConfig(s, "alpha", mons)
 	assert.NoError(t, err)
 	cc, err := parseCsiClusterConfig(s)
 	assert.NoError(t, err)
@@ -52,7 +52,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
 		"flam": {Name: "flam", Endpoint: "20.1.1.2:5000"},
 		"blam": {Name: "blam", Endpoint: "20.1.1.3:5000"},
 	}
-	s, err = UpdateCsiClusterConfig(s, "beta", mons2)
+	s, err = updateCsiClusterConfig(s, "beta", mons2)
 	assert.NoError(t, err)
 	cc, err = parseCsiClusterConfig(s)
 	assert.NoError(t, err)
@@ -69,7 +69,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
 
 	// remove a mon from the 2nd cluster
 	delete(mons2, "blam")
-	s, err = UpdateCsiClusterConfig(s, "beta", mons2)
+	s, err = updateCsiClusterConfig(s, "beta", mons2)
 	assert.NoError(t, err)
 	cc, err = parseCsiClusterConfig(s)
 	assert.NoError(t, err)
@@ -84,6 +84,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) {
 	assert.Equal(t, len(cc[1].Monitors), 2)
 
 	// does it return error on garbage input?
-	_, err = UpdateCsiClusterConfig("qqq", "beta", mons2)
+	_, err = updateCsiClusterConfig("qqq", "beta", mons2)
 	assert.Error(t, err)
 }

pkg/operator/ceph/csi/controller.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"context"
+	"os"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/ceph/csi/peermap"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	v1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	controllerName = "rook-ceph-operator-csi-controller"
+)
+
+// ReconcileCSI reconciles a ceph-csi driver
+type ReconcileCSI struct {
+	client           client.Client
+	context          *clusterd.Context
+	opManagerContext context.Context
+	opConfig         opcontroller.OperatorConfig
+}
+
+// Add creates a new Ceph CSI Controller and adds it to the Manager. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+	return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
+	return &ReconcileCSI{
+		client:           mgr.GetClient(),
+		context:          context,
+		opConfig:         opConfig,
+		opManagerContext: opManagerContext,
+	}
+}
+
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+	logger.Infof("%s successfully started", controllerName)
+
+	// Watch for ConfigMap (operator config)
+	err = c.Watch(&source.Kind{
+		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController())
+	if err != nil {
+		return err
+	}
+
+	// Watch for CephCluster
+	err = c.Watch(&source.Kind{
+		Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Reconcile reads that state of the operator config map and makes changes based on the state read
+// The Controller will requeue the Request to be processed again if the returned error is non-nil or
+// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
+func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
+	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
+	reconcileResponse, err := r.reconcile(request)
+	if err != nil {
+		logger.Errorf("failed to reconcile %v", err)
+	}
+
+	return reconcileResponse, err
+}
+
+func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) {
+	// See if there is a CephCluster
+	cephClusters := &cephv1.CephClusterList{}
+	err := r.client.List(r.opManagerContext, cephClusters, &client.ListOptions{})
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			logger.Debug("no ceph cluster found not deploying ceph csi driver")
+			return reconcile.Result{}, nil
+		}
+		// Error reading the object - requeue the request.
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters")
+	}
+
+	// // Do not nothing if no ceph cluster is present
+	if len(cephClusters.Items) == 0 {
+		logger.Debug("no ceph cluster found not deploying ceph csi driver")
+		return reconcile.Result{}, nil
+	} else {
+		for _, cluster := range cephClusters.Items {
+			if !cluster.DeletionTimestamp.IsZero() {
+				logger.Debug("ceph cluster is being deleting, no need to reconcile the csi driver")
+				return reconcile.Result{}, nil
+			}
+
+			if !cluster.Spec.External.Enable && cluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
+				logger.Debug("ceph cluster has cleanup policy, the cluster will soon go away, no need to reconcile the csi driver")
+				return reconcile.Result{}, nil
+			}
+		}
+	}
+
+	// Fetch the operator's configmap
+	opConfig := &v1.ConfigMap{}
+	err = r.client.Get(r.opManagerContext, request.NamespacedName, opConfig)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			logger.Debug("operator's configmap resource not found. will use default value or env var.")
+			r.opConfig.Parameters = make(map[string]string)
+		} else {
+			// Error reading the object - requeue the request.
+			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap")
+		}
+	} else {
+		// Populate the operator's config
+		r.opConfig.Parameters = opConfig.Data
+	}
+
+	serverVersion, err := r.context.Clientset.Discovery().ServerVersion()
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version")
+	}
+
+	if serverVersion.Major < KubeMinMajor || serverVersion.Major == KubeMinMajor && serverVersion.Minor < ProvDeploymentSuppVersion {
+		logger.Infof("CSI drivers only supported in K8s 1.14 or newer. version=%s", serverVersion.String())
+		// disable csi control variables to disable other csi functions
+		EnableRBD = false
+		EnableCephFS = false
+		return opcontroller.ImmediateRetryResult, nil
+	}
+
+	ownerRef, err := k8sutil.GetDeploymentOwnerReference(r.context.Clientset, os.Getenv(k8sutil.PodNameEnvVar), r.opConfig.OperatorNamespace)
+	if err != nil {
+		logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err)
+	}
+	if ownerRef != nil {
+		blockOwnerDeletion := false
+		ownerRef.BlockOwnerDeletion = &blockOwnerDeletion
+	}
+
+	ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, r.opConfig.OperatorNamespace)
+	// create an empty config map. config map will be filled with data
+	// later when clusters have mons
+	err = CreateCsiConfigMap(r.opConfig.OperatorNamespace, r.context.Clientset, ownerInfo)
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map")
+	}
+
+	err = peermap.CreateOrUpdateConfig(r.context, &peermap.PeerIDMappings{})
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
+	}
+
+	err = r.validateAndConfigureDrivers(serverVersion, ownerInfo)
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed configure ceph csi")
+	}
+
+	return reconcile.Result{}, nil
+}

pkg/operator/ceph/csi/controller_test.go

@@ -0,0 +1,158 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/coreos/pkg/capnslog"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+	"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/operator/ceph/controller"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/operator/test"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+func TestCephCSIController(t *testing.T) {
+	ctx := context.TODO()
+	var (
+		name      = "rook-ceph"
+		namespace = "rook-ceph"
+	)
+	// Set DEBUG logging
+	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
+	os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
+	os.Setenv(k8sutil.PodNameEnvVar, "rook-ceph-operator")
+	os.Setenv(k8sutil.PodNamespaceEnvVar, namespace)
+
+	os.Setenv("ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "true")
+	CSIParam = Param{
+		CSIPluginImage:   "image",
+		RegistrarImage:   "image",
+		ProvisionerImage: "image",
+		AttacherImage:    "image",
+		SnapshotterImage: "image",
+	}
+	req := reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+
+	t.Run("failure because no CephCluster", func(t *testing.T) {
+		fakeClientSet := test.New(t, 1)
+		test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0")
+		c := &clusterd.Context{
+			Clientset:     fakeClientSet,
+			RookClientset: rookclient.NewSimpleClientset(),
+		}
+
+		_, err := c.Clientset.CoreV1().Pods(namespace).Create(ctx, test.FakeOperatorPod(namespace), metav1.CreateOptions{})
+		assert.NoError(t, err)
+		_, err = c.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), test.FakeReplicaSet(namespace), metav1.CreateOptions{})
+		assert.NoError(t, err)
+		// Register operator types with the runtime scheme.
+		s := scheme.Scheme
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}, &cephv1.CephClusterList{})
+
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).Build()
+		c.Client = cl
+
+		// Create a ReconcileCSI object with the scheme and fake client.
+		r := &ReconcileCSI{
+			client:  cl,
+			context: c,
+			opConfig: controller.OperatorConfig{
+				OperatorNamespace: namespace,
+				Image:             "rook",
+				ServiceAccount:    "foo",
+			},
+		}
+
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+	})
+
+	t.Run("success ceph csi deployment", func(t *testing.T) {
+		fakeClientSet := test.New(t, 1)
+		test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0")
+		c := &clusterd.Context{
+			Clientset:     fakeClientSet,
+			RookClientset: rookclient.NewSimpleClientset(),
+		}
+		_, err := c.Clientset.CoreV1().Pods(namespace).Create(ctx, test.FakeOperatorPod(namespace), metav1.CreateOptions{})
+		assert.NoError(t, err)
+		_, err = c.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), test.FakeReplicaSet(namespace), metav1.CreateOptions{})
+		assert.NoError(t, err)
+		cephCluster := &cephv1.CephCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      namespace,
+				Namespace: namespace,
+			},
+			Status: cephv1.ClusterStatus{
+				Phase: "",
+				CephVersion: &cephv1.ClusterVersion{
+					Version: "14.2.9-0",
+				},
+				CephStatus: &cephv1.CephStatus{
+					Health: "",
+				},
+			},
+		}
+		s := scheme.Scheme
+		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
+
+		object := []runtime.Object{
+			cephCluster,
+		}
+		// Create a fake client to mock API calls.
+		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+		c.Client = cl
+		// // Create a ReconcileCSI object with the scheme and fake client.
+		r := &ReconcileCSI{
+			client:  cl,
+			context: c,
+			opConfig: controller.OperatorConfig{
+				OperatorNamespace: namespace,
+				Image:             "rook",
+				ServiceAccount:    "foo",
+			},
+		}
+
+		res, err := r.Reconcile(ctx, req)
+		assert.NoError(t, err)
+		assert.False(t, res.Requeue)
+
+		ds, err := c.Clientset.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{})
+		assert.NoError(t, err)
+		assert.Equal(t, 2, len(ds.Items), ds)
+	})
+}

pkg/operator/ceph/csi/csi.go

@@ -20,136 +20,83 @@ import (
 	"strconv"
 
 	"github.com/pkg/errors"
-	"github.com/rook/rook/pkg/clusterd"
-	controllerutil "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"k8s.io/apimachinery/pkg/version"
-	"k8s.io/client-go/kubernetes"
 )
 
-func ValidateAndConfigureDrivers(context *clusterd.Context, namespace, rookImage, securityAccount string, serverVersion *version.Info, ownerInfo *k8sutil.OwnerInfo) {
-	csiLock.Lock()
-	defer csiLock.Unlock()
+func (r *ReconcileCSI) validateAndConfigureDrivers(serverVersion *version.Info, ownerInfo *k8sutil.OwnerInfo) error {
 	var (
 		v   *CephCSIVersion
 		err error
 	)
 
-	if err = setParams(context.Clientset); err != nil {
-		logger.Errorf("failed to configure CSI parameters. %v", err)
-		return
+	if err = r.setParams(); err != nil {
+		return errors.Wrapf(err, "failed to configure CSI parameters")
 	}
 
 	if err = validateCSIParam(); err != nil {
-		logger.Errorf("failed to validate CSI parameters. %v", err)
-		return
+		return errors.Wrapf(err, "failed to validate CSI parameters")
 	}
 
 	if !AllowUnsupported && CSIEnabled() {
-		if v, err = validateCSIVersion(context.Clientset, namespace, rookImage, securityAccount, ownerInfo); err != nil {
-			logger.Errorf("invalid csi version. %+v", err)
-			return
+		if v, err = r.validateCSIVersion(ownerInfo); err != nil {
+			return errors.Wrapf(err, "invalid csi version")
 		}
 	} else {
-		logger.Info("Skipping csi version check, since unsupported versions are allowed or csi is disabled")
+		logger.Info("skipping csi version check, since unsupported versions are allowed or csi is disabled")
 	}
 
 	if CSIEnabled() {
 		maxRetries := 3
 		for i := 0; i < maxRetries; i++ {
-			if err = startDrivers(context.Clientset, context.RookClientset, namespace, serverVersion, ownerInfo, v); err != nil {
+			if err = r.startDrivers(serverVersion, ownerInfo, v); err != nil {
 				logger.Errorf("failed to start Ceph csi drivers, will retry starting csi drivers %d more times. %v", maxRetries-i-1, err)
 			} else {
 				break
 			}
 		}
+		return errors.Wrap(err, "failed to start ceph csi drivers")
 	}
 
-	stopDrivers(context.Clientset, namespace, serverVersion)
+	// Check whether RBD or CephFS needs to be disabled
+	r.stopDrivers(serverVersion)
+
+	return nil
 }
 
-func setParams(clientset kubernetes.Interface) error {
+func (r *ReconcileCSI) setParams() error {
 	var err error
 
-	csiEnableRBD, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ENABLE_RBD", "true")
-	if err != nil {
-		return errors.Wrap(err, "unable to determine if CSI driver for RBD is enabled")
-	}
-	if EnableRBD, err = strconv.ParseBool(csiEnableRBD); err != nil {
+	if EnableRBD, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_RBD", "true")); err != nil {
 		return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_RBD'")
 	}
 
-	csiEnableCephFS, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ENABLE_CEPHFS", "true")
-	if err != nil {
-		return errors.Wrap(err, "unable to determine if CSI driver for CephFS is enabled")
-	}
-	if EnableCephFS, err = strconv.ParseBool(csiEnableCephFS); err != nil {
+	if EnableCephFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil {
 		return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_CEPHFS'")
 	}
 
-	csiAllowUnsupported, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "false")
-	if err != nil {
-		return errors.Wrap(err, "unable to determine if unsupported version is allowed")
-	}
-	if AllowUnsupported, err = strconv.ParseBool(csiAllowUnsupported); err != nil {
+	if AllowUnsupported, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "false")); err != nil {
 		return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ALLOW_UNSUPPORTED_VERSION'")
 	}
 
-	csiEnableCSIGRPCMetrics, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ENABLE_GRPC_METRICS", "false")
-	if err != nil {
-		return errors.Wrap(err, "unable to determine if CSI GRPC metrics is enabled")
-	}
-	if EnableCSIGRPCMetrics, err = strconv.ParseBool(csiEnableCSIGRPCMetrics); err != nil {
+	if EnableCSIGRPCMetrics, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_GRPC_METRICS", "false")); err != nil {
 		return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_GRPC_METRICS'")
 	}
 
-	csiEnableCSIHostNetwork, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_HOST_NETWORK", "true")
-	if err != nil {
-		return errors.Wrap(err, "failed to determine if CSI Host Network is enabled")
-	}
-	if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(csiEnableCSIHostNetwork); err != nil {
+	if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil {
 		return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'")
 	}
 
-	CSIParam.CSIPluginImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI plugin image")
-	}
-	CSIParam.RegistrarImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI registrar image")
-	}
-	CSIParam.ProvisionerImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI provisioner image")
-	}
-	CSIParam.AttacherImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI attacher image")
-	}
-	CSIParam.SnapshotterImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI snapshotter image")
-	}
-	CSIParam.KubeletDirPath, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI kubelet directory path")
-	}
-	CSIParam.VolumeReplicationImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage)
-	if err != nil {
-		return errors.Wrap(err, "unable to configure Volume replication controller image")
-	}
-
-	csiCephFSPodLabels, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_CEPHFS_POD_LABELS", "")
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI CephFS pod labels")
-	}
+	CSIParam.CSIPluginImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage)
+	CSIParam.RegistrarImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage)
+	CSIParam.ProvisionerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage)
+	CSIParam.AttacherImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage)
+	CSIParam.SnapshotterImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage)
+	CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath)
+	CSIParam.VolumeReplicationImage = k8sutil.GetValue(r.opConfig.Parameters, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage)
+	csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "")
 	CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels)
-
-	csiRBDPodLabels, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_RBD_POD_LABELS", "")
-	if err != nil {
-		return errors.Wrap(err, "unable to configure CSI RBD pod labels")
-	}
+	csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "")
 	CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels)
 
 	return nil

pkg/operator/ceph/csi/peermap/config.go

@@ -168,7 +168,7 @@ func getClusterPoolIDMap(clusterContext *clusterd.Context, clusterInfo *cephclie
 	logger.Debugf("pool details of local cluster %+v", localPoolDetails)
 
 	for _, peerSecret := range pool.Spec.Mirroring.Peers.SecretNames {
-		s, err := clusterContext.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(context.TODO(), peerSecret, metav1.GetOptions{})
+		s, err := clusterContext.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(clusterInfo.Context, peerSecret, metav1.GetOptions{})
 		if err != nil {
 			return mappings, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret)
 		}

pkg/operator/ceph/csi/peermap/config_test.go

@@ -275,7 +275,7 @@ var mockExecutor = &exectest.MockExecutor{
 }
 
 func TestSinglePeerMappings(t *testing.T) {
-	clusterInfo := &cephclient.ClusterInfo{Namespace: ns}
+	clusterInfo := cephclient.AdminClusterInfo(ns)
 	fakeContext := &clusterd.Context{
 		Executor:  mockExecutor,
 		Clientset: test.New(t, 3),
@@ -299,7 +299,7 @@ func TestSinglePeerMappings(t *testing.T) {
 }
 
 func TestMultiPeerMappings(t *testing.T) {
-	clusterInfo := &cephclient.ClusterInfo{Namespace: ns}
+	clusterInfo := cephclient.AdminClusterInfo(ns)
 	fakeContext := &clusterd.Context{
 		Executor:  mockExecutor,
 		Clientset: test.New(t, 3),
@@ -358,41 +358,8 @@ func TestDecodePeerToken(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func fakeOperatorPod() *corev1.Pod {
-	p := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test",
-			Namespace: ns,
-			OwnerReferences: []metav1.OwnerReference{
-				{
-					Kind: "ReplicaSet",
-					Name: "testReplicaSet",
-				},
-			},
-		},
-		Spec: corev1.PodSpec{},
-	}
-	return p
-}
-
-func fakeReplicaSet() *appsv1.ReplicaSet {
-	r := &appsv1.ReplicaSet{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "testReplicaSet",
-			Namespace: ns,
-			OwnerReferences: []metav1.OwnerReference{
-				{
-					Kind: "Deployment",
-				},
-			},
-		},
-	}
-
-	return r
-}
-
 func TestCreateOrUpdateConfig(t *testing.T) {
-	os.Setenv("POD_NAME", "test")
+	os.Setenv("POD_NAME", "rook-ceph-operator")
 	defer os.Setenv("POD_NAME", "")
 	os.Setenv("POD_NAMESPACE", ns)
 	defer os.Setenv("POD_NAMESPACE", "")
@@ -414,11 +381,11 @@ func TestCreateOrUpdateConfig(t *testing.T) {
 	}
 
 	// Create fake pod
-	_, err = fakeContext.Clientset.CoreV1().Pods(ns).Create(context.TODO(), fakeOperatorPod(), metav1.CreateOptions{})
+	_, err = fakeContext.Clientset.CoreV1().Pods(ns).Create(context.TODO(), test.FakeOperatorPod(ns), metav1.CreateOptions{})
 	assert.NoError(t, err)
 
 	// Create fake replicaset
-	_, err = fakeContext.Clientset.AppsV1().ReplicaSets(ns).Create(context.TODO(), fakeReplicaSet(), metav1.CreateOptions{})
+	_, err = fakeContext.Clientset.AppsV1().ReplicaSets(ns).Create(context.TODO(), test.FakeReplicaSet(ns), metav1.CreateOptions{})
 	assert.NoError(t, err)
 
 	// Create empty ID mapping configMap

pkg/operator/ceph/csi/predicate.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"regexp"
+
+	"github.com/google/go-cmp/cmp"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// predicateController is the predicate function to trigger reconcile on operator configuration cm change
+func predicateController() predicate.Funcs {
+	return predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			// if the operator configuration file is created we want to reconcile
+			if cm, ok := e.Object.(*v1.ConfigMap); ok {
+				// We don't want to use cm.Generation here, it case the operator was stopped and the
+				// ConfigMap was created
+				return cm.Name == opcontroller.OperatorSettingConfigMapName
+			}
+
+			// If a Ceph Cluster is created we want to reconcile the csi driver
+			if cephCluster, ok := e.Object.(*cephv1.CephCluster); ok {
+				// This allows us to avoid a double reconcile of the CSI controller if this is not
+				// the first generation of the CephCluster. So only return true if this is the very
+				// first instance of the CephCluster
+				// Corner case is when the cluster is created but the operator is down AND the cm
+				// does not exist... However, these days the operator config map is part of the
+				// operator.yaml so it's probably acceptable?
+				return cephCluster.Generation == 1
+			}
+
+			return false
+		},
+
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			resourceQtyComparer := cmp.Comparer(func(x, y resource.Quantity) bool { return x.Cmp(y) == 0 })
+			if old, ok := e.ObjectOld.(*v1.ConfigMap); ok {
+				if new, ok := e.ObjectNew.(*v1.ConfigMap); ok {
+					if old.Name == opcontroller.OperatorSettingConfigMapName && new.Name == opcontroller.OperatorSettingConfigMapName {
+						diff := cmp.Diff(old.Data, new.Data, resourceQtyComparer)
+						logger.Debugf("operator configmap diff:\n %s", diff)
+						return findCSIChange(diff)
+					}
+				}
+			}
+
+			return false
+		},
+
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			return false
+		},
+
+		GenericFunc: func(e event.GenericEvent) bool {
+			return false
+		},
+	}
+}
+
+func findCSIChange(str string) bool {
+	var re = regexp.MustCompile(`(?m)^(\+|-).*(\"ROOK_CSI_|\"CSI_).*,$`)
+	found := re.FindAllString(str, -1)
+	if len(found) > 0 {
+		for _, match := range found {
+			// logger.Debugf("ceph csi config changed with: %q", match)
+			logger.Infof("ceph csi config changed with: %q", match)
+		}
+		return true
+	}
+
+	return false
+}

pkg/operator/ceph/csi/predicate_test.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_findCSIChange(t *testing.T) {
+	t.Run("no match", func(t *testing.T) {
+		var str = `  map[string]string{
+        "CSI_FORCE_CEPHFS_KERNEL_CLIENT":     "true",
+        "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false",
+        "ROOK_CSI_ENABLE_GRPC_METRICS":       "true",
+        "ROOK_CSI_ENABLE_RBD":                "true",
+        "ROOK_ENABLE_DISCOVERY_DAEMON":       "false",
+-       "ROOK_LOG_LEVEL":                     "INFO",
++       "ROOK_LOG_LEVEL":                     "DEBUG",
+        "ROOK_OBC_WATCH_OPERATOR_NAMESPACE":  "true",
+  }`
+		b := findCSIChange(str)
+		assert.False(t, b)
+	})
+
+	t.Run("match on addition", func(t *testing.T) {
+		var str = `  map[string]string{
+        "CSI_FORCE_CEPHFS_KERNEL_CLIENT":     "true",
+        "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false",
++       "ROOK_CSI_ENABLE_CEPHFS":             "false",
+        "ROOK_CSI_ENABLE_GRPC_METRICS":       "true",
+        "ROOK_CSI_ENABLE_RBD":                "true",
+        "ROOK_ENABLE_DISCOVERY_DAEMON":       "false",
+-       "ROOK_LOG_LEVEL":                     "INFO",
++       "ROOK_LOG_LEVEL":                     "DEBUG",
+        "ROOK_OBC_WATCH_OPERATOR_NAMESPACE":  "true",
+  }`
+		b := findCSIChange(str)
+		assert.True(t, b)
+	})
+
+	t.Run("match on deletion", func(t *testing.T) {
+		var str = `  map[string]string{
+        "CSI_FORCE_CEPHFS_KERNEL_CLIENT":     "true",
+        "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false",
+-       "ROOK_CSI_ENABLE_CEPHFS":             "true",
+        "ROOK_CSI_ENABLE_GRPC_METRICS":       "true",
+        "ROOK_CSI_ENABLE_RBD":                "true",
+        "ROOK_ENABLE_DISCOVERY_DAEMON":       "false",
+-       "ROOK_LOG_LEVEL":                     "INFO",
++       "ROOK_LOG_LEVEL":                     "DEBUG",
+        "ROOK_OBC_WATCH_OPERATOR_NAMESPACE":  "true",
+  }`
+		b := findCSIChange(str)
+		assert.True(t, b)
+	})
+
+	t.Run("match on addition and deletion", func(t *testing.T) {
+		var str = `  map[string]string{
+        "CSI_FORCE_CEPHFS_KERNEL_CLIENT":     "true",
+        "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false",
+-       "ROOK_CSI_ENABLE_CEPHFS":             "true",
++       "ROOK_CSI_ENABLE_CEPHFS":             "false",
+        "ROOK_CSI_ENABLE_GRPC_METRICS":       "true",
+        "ROOK_CSI_ENABLE_RBD":                "true",
+        "ROOK_ENABLE_DISCOVERY_DAEMON":       "false",
+-       "ROOK_LOG_LEVEL":                     "INFO",
++       "ROOK_LOG_LEVEL":                     "DEBUG",
+        "ROOK_OBC_WATCH_OPERATOR_NAMESPACE":  "true",
+  }`
+		b := findCSIChange(str)
+		assert.True(t, b)
+	})
+
+	t.Run("match with CSI_ name", func(t *testing.T) {
+		var str = `  map[string]string{
+        "CSI_FORCE_CEPHFS_KERNEL_CLIENT":     "true",
+        "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false",
+-       "CSI_PROVISIONER_TOLERATIONS":             "true",
++       "CSI_PROVISIONER_TOLERATIONS":             "false",
+        "ROOK_CSI_ENABLE_GRPC_METRICS":       "true",
+        "ROOK_CSI_ENABLE_RBD":                "true",
+        "ROOK_ENABLE_DISCOVERY_DAEMON":       "false",
+-       "ROOK_LOG_LEVEL":                     "INFO",
++       "ROOK_LOG_LEVEL":                     "DEBUG",
+        "ROOK_OBC_WATCH_OPERATOR_NAMESPACE":  "true",
+  }`
+		b := findCSIChange(str)
+		assert.True(t, b)
+	})
+
+	t.Run("match with CSI_ and ROOK_CSI_ name", func(t *testing.T) {
+		var str = `  map[string]string{
+        "CSI_FORCE_CEPHFS_KERNEL_CLIENT":     "true",
+        "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false",
+-       "CSI_PROVISIONER_TOLERATIONS":             "true",
++       "CSI_PROVISIONER_TOLERATIONS":             "false",
+-       "ROOK_CSI_ENABLE_CEPHFS":             "true",
++       "ROOK_CSI_ENABLE_CEPHFS":             "false",
+        "ROOK_CSI_ENABLE_GRPC_METRICS":       "true",
+        "ROOK_CSI_ENABLE_RBD":                "true",
+        "ROOK_ENABLE_DISCOVERY_DAEMON":       "false",
+-       "ROOK_LOG_LEVEL":                     "INFO",
++       "ROOK_LOG_LEVEL":                     "DEBUG",
+        "ROOK_OBC_WATCH_OPERATOR_NAMESPACE":  "true",
+  }`
+		b := findCSIChange(str)
+		assert.True(t, b)
+	})
+}

pkg/operator/ceph/csi/secrets.go

@@ -159,7 +159,7 @@ func createOrUpdateCSISecret(clusterInfo *client.ClusterInfo, csiRBDProvisionerS
 		// Create Kubernetes Secret
 		err = k.CreateSecret(s)
 		if err != nil {
-			return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", secret, clusterInfo.Namespace)
+			return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
 		}
 
 	}

pkg/operator/ceph/csi/spec.go

@@ -22,11 +22,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
-	rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
-	controllerutil "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
 
@@ -36,7 +33,6 @@ import (
 	k8scsi "k8s.io/api/storage/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/version"
-	"k8s.io/client-go/kubernetes"
 )
 
 type Param struct {
@@ -92,7 +88,6 @@ var (
 	ConfigName = "rook-ceph-csi-config"
 	ConfigKey  = "csi-cluster-config-json"
 
-	csiLock      sync.Mutex
 	csiDriverobj csiDriver
 )
 
@@ -208,8 +203,7 @@ func validateCSIParam() error {
 	return nil
 }
 
-func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Interface, namespace string, ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error {
-	ctx := context.TODO()
+func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error {
 	var (
 		err                                                   error
 		rbdPlugin, cephfsPlugin                               *apps.DaemonSet
@@ -219,12 +213,12 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 
 	tp := templateParam{
 		Param:     CSIParam,
-		Namespace: namespace,
+		Namespace: r.opConfig.OperatorNamespace,
 	}
 	// if the user didn't specify a custom DriverNamePrefix use
 	// the namespace (and a dot).
 	if tp.DriverNamePrefix == "" {
-		tp.DriverNamePrefix = fmt.Sprintf("%s.", namespace)
+		tp.DriverNamePrefix = fmt.Sprintf("%s.", r.opConfig.OperatorNamespace)
 	}
 
 	CephFSDriverName = tp.DriverNamePrefix + "cephfs.csi.ceph.com"
@@ -239,13 +233,13 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		// than maximum supported version for betav1 object.(unavailable in v1.22+)
 		// Ignore if not found.
 		if EnableRBD && ver.Minor <= kubeMaxVerForBeta1csiDriver {
-			err = beta1CsiDriver{}.deleteCSIDriverInfo(ctx, clientset, RBDDriverName)
+			err = beta1CsiDriver{}.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, RBDDriverName)
 			if err != nil {
 				logger.Errorf("failed to delete %q Driver Info. %v", RBDDriverName, err)
 			}
 		}
 		if EnableCephFS && ver.Minor <= kubeMaxVerForBeta1csiDriver {
-			err = beta1CsiDriver{}.deleteCSIDriverInfo(ctx, clientset, CephFSDriverName)
+			err = beta1CsiDriver{}.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, CephFSDriverName)
 			if err != nil {
 				logger.Errorf("failed to delete %q Driver Info. %v", CephFSDriverName, err)
 			}
@@ -255,46 +249,37 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 	tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics)
 
 	// If not set or set to anything but "false", the kernel client will be enabled
-	kClient, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_FORCE_CEPHFS_KERNEL_CLIENT setting")
-	}
-	if strings.EqualFold(kClient, "false") {
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") {
 		tp.ForceCephFSKernelClient = "false"
 	} else {
 		tp.ForceCephFSKernelClient = "true"
 	}
+
 	// parse GRPC and Liveness ports
-	tp.CephFSGRPCMetricsPort, err = getPortFromConfig(clientset, "CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort)
+	tp.CephFSGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort)
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI CephFS GRPC metrics port.")
 	}
-	tp.CephFSLivenessMetricsPort, err = getPortFromConfig(clientset, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort)
+	tp.CephFSLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort)
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.")
 	}
 
-	tp.RBDGRPCMetricsPort, err = getPortFromConfig(clientset, "CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort)
+	tp.RBDGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort)
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI RBD GRPC metrics port.")
 	}
-	tp.RBDLivenessMetricsPort, err = getPortFromConfig(clientset, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort)
+	tp.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort)
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI RBD liveness metrics port.")
 	}
 
 	// default value `system-node-critical` is the highest available priority
-	tp.PluginPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PLUGIN_PRIORITY_CLASSNAME", "")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_PLUGIN_PRIORITY_CLASSNAME setting")
-	}
+	tp.PluginPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_PRIORITY_CLASSNAME", "")
 
 	// default value `system-cluster-critical` is applied for some
 	// critical pods in cluster but less priority than plugin pods
-	tp.ProvisionerPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_PROVISIONER_PRIORITY_CLASSNAME setting")
-	}
+	tp.ProvisionerPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "")
 
 	// OMAP generator will be enabled by default
 	// If AllowUnsupported is set to false and if CSI version is less than
@@ -303,11 +288,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		tp.EnableOMAPGenerator = false
 	}
 
-	enableOMAPGenerator, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_OMAP_GENERATOR", "false")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_ENABLE_OMAP_GENERATOR setting")
-	}
-	if strings.EqualFold(enableOMAPGenerator, "true") {
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_OMAP_GENERATOR", "false"), "true") {
 		tp.EnableOMAPGenerator = true
 	}
 
@@ -316,45 +297,27 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		tp.EnableRBDSnapshotter = true
 		tp.EnableCephFSSnapshotter = true
 	}
-	enableRBDSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_RBD_SNAPSHOTTER", "true")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_ENABLE_RBD_SNAPSHOTTER setting")
-	}
-	if strings.EqualFold(enableRBDSnapshotter, "false") {
+
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") {
 		tp.EnableRBDSnapshotter = false
 	}
-	enableCephFSSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_ENABLE_CEPHFS_SNAPSHOTTER setting")
-	}
-	if strings.EqualFold(enableCephFSSnapshotter, "false") {
+
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") {
 		tp.EnableCephFSSnapshotter = false
 	}
 
 	tp.EnableVolumeReplicationSideCar = false
-	enableVolumeReplicationSideCar, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_VOLUME_REPLICATION", "false")
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_ENABLE_VOLUME_REPLICATION setting")
-	}
-	if strings.EqualFold(enableVolumeReplicationSideCar, "true") {
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_REPLICATION", "false"), "true") {
 		tp.EnableVolumeReplicationSideCar = true
 	}
 
-	updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate)
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting")
-	}
-	if strings.EqualFold(updateStrategy, onDelete) {
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) {
 		tp.CephFSPluginUpdateStrategy = onDelete
 	} else {
 		tp.CephFSPluginUpdateStrategy = rollingUpdate
 	}
 
-	updateStrategy, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate)
-	if err != nil {
-		return errors.Wrap(err, "failed to load CSI_RBD_PLUGIN_UPDATE_STRATEGY setting")
-	}
-	if strings.EqualFold(updateStrategy, onDelete) {
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) {
 		tp.RBDPluginUpdateStrategy = onDelete
 	} else {
 		tp.RBDPluginUpdateStrategy = rollingUpdate
@@ -362,13 +325,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 
 	logger.Infof("Kubernetes version is %s.%s", ver.Major, ver.Minor)
 
-	tp.ResizerImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage)
-	if err != nil {
-		return errors.Wrap(err, "failed to load ROOK_CSI_RESIZER_IMAGE setting")
-	}
-	if tp.ResizerImage == "" {
-		tp.ResizerImage = DefaultResizerImage
-	}
+	tp.ResizerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage)
 
 	if ver.Major == KubeMinMajor && ver.Minor < kubeMinVerForFilesystemRestore {
 		logger.Warning("CSI Filesystem volume expansion requires Kubernetes version >=1.15.0")
@@ -377,11 +334,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		logger.Warning("CSI Block volume expansion requires Kubernetes version >=1.16.0")
 	}
 
-	logLevel, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_LOG_LEVEL", "")
-	if err != nil {
-		// logging a warning and intentionally continuing with the default log level
-		logger.Warningf("failed to load CSI_LOG_LEVEL. Defaulting to %d. %v", defaultLogLevel, err)
-	}
+	logLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LOG_LEVEL", "")
 	tp.LogLevel = defaultLogLevel
 	if logLevel != "" {
 		l, err := strconv.ParseUint(logLevel, 10, 8)
@@ -393,7 +346,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 	}
 
 	tp.ProvisionerReplicas = 2
-	nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{})
 	if err == nil {
 		if len(nodes.Items) == 1 {
 			tp.ProvisionerReplicas = 1
@@ -417,7 +370,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		if err != nil {
 			return errors.Wrap(err, "failed to load rbd plugin service template")
 		}
-		rbdService.Namespace = namespace
+		rbdService.Namespace = r.opConfig.OperatorNamespace
 	}
 	if EnableCephFS {
 		cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath, tp)
@@ -434,50 +387,50 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		if err != nil {
 			return errors.Wrap(err, "failed to load cephfs plugin service template")
 		}
-		cephfsService.Namespace = namespace
+		cephfsService.Namespace = r.opConfig.OperatorNamespace
 	}
 
 	// get common provisioner tolerations and node affinity
-	provisionerTolerations := getToleration(clientset, provisionerTolerationsEnv, []corev1.Toleration{})
-	provisionerNodeAffinity := getNodeAffinity(clientset, provisionerNodeAffinityEnv, &corev1.NodeAffinity{})
+	provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{})
+	provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{})
 	// get common plugin tolerations and node affinity
-	pluginTolerations := getToleration(clientset, pluginTolerationsEnv, []corev1.Toleration{})
-	pluginNodeAffinity := getNodeAffinity(clientset, pluginNodeAffinityEnv, &corev1.NodeAffinity{})
+	pluginTolerations := getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []corev1.Toleration{})
+	pluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &corev1.NodeAffinity{})
 
 	if rbdPlugin != nil {
 		// get RBD plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified
-		rbdPluginTolerations := getToleration(clientset, rbdPluginTolerationsEnv, pluginTolerations)
-		rbdPluginNodeAffinity := getNodeAffinity(clientset, rbdPluginNodeAffinityEnv, pluginNodeAffinity)
+		rbdPluginTolerations := getToleration(r.opConfig.Parameters, rbdPluginTolerationsEnv, pluginTolerations)
+		rbdPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdPluginNodeAffinityEnv, pluginNodeAffinity)
 		// apply RBD plugin tolerations and node affinity
 		applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations)
 		// apply resource request and limit to rbdplugin containers
-		applyResourcesToContainers(clientset, rbdPluginResource, &rbdPlugin.Spec.Template.Spec)
+		applyResourcesToContainers(r.opConfig.Parameters, rbdPluginResource, &rbdPlugin.Spec.Template.Spec)
 		err = ownerInfo.SetControllerReference(rbdPlugin)
 		if err != nil {
 			return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name)
 		}
-		multusApplied, err := applyCephClusterNetworkConfig(ctx, &rbdPlugin.Spec.Template.ObjectMeta, rookclientset)
+		multusApplied, err := r.applyCephClusterNetworkConfig(r.opManagerContext, &rbdPlugin.Spec.Template.ObjectMeta)
 		if err != nil {
-			return errors.Wrapf(err, "failed to apply network config to rbd plugin daemonset: %+v", rbdPlugin)
+			return errors.Wrapf(err, "failed to apply network config to rbd plugin daemonset %q", rbdPlugin.Name)
 		}
 		if multusApplied {
 			rbdPlugin.Spec.Template.Spec.HostNetwork = false
 		}
-		err = k8sutil.CreateDaemonSet(csiRBDPlugin, namespace, clientset, rbdPlugin)
+		err = k8sutil.CreateDaemonSet(csiRBDPlugin, r.opConfig.OperatorNamespace, r.context.Clientset, rbdPlugin)
 		if err != nil {
-			return errors.Wrapf(err, "failed to start rbdplugin daemonset: %+v", rbdPlugin)
+			return errors.Wrapf(err, "failed to start rbdplugin daemonset %q", rbdPlugin.Name)
 		}
 		k8sutil.AddRookVersionLabelToDaemonSet(rbdPlugin)
 	}
 
 	if rbdProvisionerDeployment != nil {
 		// get RBD provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified
-		rbdProvisionerTolerations := getToleration(clientset, rbdProvisionerTolerationsEnv, provisionerTolerations)
-		rbdProvisionerNodeAffinity := getNodeAffinity(clientset, rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity)
+		rbdProvisionerTolerations := getToleration(r.opConfig.Parameters, rbdProvisionerTolerationsEnv, provisionerTolerations)
+		rbdProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity)
 		// apply RBD provisioner tolerations and node affinity
 		applyToPodSpec(&rbdProvisionerDeployment.Spec.Template.Spec, rbdProvisionerNodeAffinity, rbdProvisionerTolerations)
 		// apply resource request and limit to rbd provisioner containers
-		applyResourcesToContainers(clientset, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec)
+		applyResourcesToContainers(r.opConfig.Parameters, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec)
 		err = ownerInfo.SetControllerReference(rbdProvisionerDeployment)
 		if err != nil {
 			return errors.Wrapf(err, "failed to set owner reference to rbd provisioner deployment %q", rbdProvisionerDeployment.Name)
@@ -488,65 +441,65 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 			Type: apps.RecreateDeploymentStrategyType,
 		}
 
-		_, err = applyCephClusterNetworkConfig(ctx, &rbdProvisionerDeployment.Spec.Template.ObjectMeta, rookclientset)
+		_, err = r.applyCephClusterNetworkConfig(r.opManagerContext, &rbdProvisionerDeployment.Spec.Template.ObjectMeta)
 		if err != nil {
-			return errors.Wrapf(err, "failed to apply network config to rbd plugin provisioner deployment: %+v", rbdProvisionerDeployment)
+			return errors.Wrapf(err, "failed to apply network config to rbd plugin provisioner deployment %q", rbdProvisionerDeployment.Name)
 		}
-		_, err = k8sutil.CreateOrUpdateDeployment(clientset, rbdProvisionerDeployment)
+		_, err = k8sutil.CreateOrUpdateDeployment(r.context.Clientset, rbdProvisionerDeployment)
 		if err != nil {
-			return errors.Wrapf(err, "failed to start rbd provisioner deployment: %+v", rbdProvisionerDeployment)
+			return errors.Wrapf(err, "failed to start rbd provisioner deployment %q", rbdProvisionerDeployment.Name)
 		}
 		k8sutil.AddRookVersionLabelToDeployment(rbdProvisionerDeployment)
 		logger.Info("successfully started CSI Ceph RBD driver")
 	}
 
 	if rbdService != nil {
-		rbdService.Namespace = namespace
+		rbdService.Namespace = r.opConfig.OperatorNamespace
 		err = ownerInfo.SetControllerReference(rbdService)
 		if err != nil {
 			return errors.Wrapf(err, "failed to set owner reference to rbd service %q", rbdService)
 		}
-		_, err = k8sutil.CreateOrUpdateService(clientset, namespace, rbdService)
+		_, err = k8sutil.CreateOrUpdateService(r.context.Clientset, r.opConfig.OperatorNamespace, rbdService)
 		if err != nil {
-			return errors.Wrapf(err, "failed to create rbd service: %+v", rbdService)
+			return errors.Wrapf(err, "failed to create rbd service %q", rbdService.Name)
 		}
 	}
 
 	if cephfsPlugin != nil {
 		// get CephFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified
-		cephFSPluginTolerations := getToleration(clientset, cephFSPluginTolerationsEnv, pluginTolerations)
-		cephFSPluginNodeAffinity := getNodeAffinity(clientset, cephFSPluginNodeAffinityEnv, pluginNodeAffinity)
+		cephFSPluginTolerations := getToleration(r.opConfig.Parameters, cephFSPluginTolerationsEnv, pluginTolerations)
+		cephFSPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSPluginNodeAffinityEnv, pluginNodeAffinity)
 		// apply CephFS plugin tolerations and node affinity
 		applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations)
 		// apply resource request and limit to cephfs plugin containers
-		applyResourcesToContainers(clientset, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec)
+		applyResourcesToContainers(r.opConfig.Parameters, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec)
 		err = ownerInfo.SetControllerReference(cephfsPlugin)
 		if err != nil {
 			return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name)
 		}
-		multusApplied, err := applyCephClusterNetworkConfig(ctx, &cephfsPlugin.Spec.Template.ObjectMeta, rookclientset)
+		multusApplied, err := r.applyCephClusterNetworkConfig(r.opManagerContext, &cephfsPlugin.Spec.Template.ObjectMeta)
 		if err != nil {
-			return errors.Wrapf(err, "failed to apply network config to cephfs plugin daemonset: %+v", cephfsPlugin)
+			return errors.Wrapf(err, "failed to apply network config to cephfs plugin daemonset %q", cephfsPlugin.Name)
 		}
 		if multusApplied {
 			cephfsPlugin.Spec.Template.Spec.HostNetwork = false
 		}
-		err = k8sutil.CreateDaemonSet(csiCephFSPlugin, namespace, clientset, cephfsPlugin)
+		err = k8sutil.CreateDaemonSet(csiCephFSPlugin, r.opConfig.OperatorNamespace, r.context.Clientset, cephfsPlugin)
 		if err != nil {
-			return errors.Wrapf(err, "failed to start cephfs plugin daemonset: %+v", cephfsPlugin)
+			return errors.Wrapf(err, "failed to start cephfs plugin daemonset %q", cephfsPlugin.Name)
 		}
 		k8sutil.AddRookVersionLabelToDaemonSet(cephfsPlugin)
 	}
 
 	if cephfsProvisionerDeployment != nil {
 		// get CephFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified
-		cephFSProvisionerTolerations := getToleration(clientset, cephFSProvisionerTolerationsEnv, provisionerTolerations)
-		cephFSProvisionerNodeAffinity := getNodeAffinity(clientset, cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity)
+		cephFSProvisionerTolerations := getToleration(r.opConfig.Parameters, cephFSProvisionerTolerationsEnv, provisionerTolerations)
+		cephFSProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity)
 		// apply CephFS provisioner tolerations and node affinity
 		applyToPodSpec(&cephfsProvisionerDeployment.Spec.Template.Spec, cephFSProvisionerNodeAffinity, cephFSProvisionerTolerations)
 		// get resource details for cephfs provisioner
 		// apply resource request and limit to cephfs provisioner containers
-		applyResourcesToContainers(clientset, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec)
+		applyResourcesToContainers(r.opConfig.Parameters, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec)
 		err = ownerInfo.SetControllerReference(cephfsProvisionerDeployment)
 		if err != nil {
 			return errors.Wrapf(err, "failed to set owner reference to cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name)
@@ -557,13 +510,13 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 			Type: apps.RecreateDeploymentStrategyType,
 		}
 
-		_, err = applyCephClusterNetworkConfig(ctx, &cephfsProvisionerDeployment.Spec.Template.ObjectMeta, rookclientset)
+		_, err = r.applyCephClusterNetworkConfig(r.opManagerContext, &cephfsProvisionerDeployment.Spec.Template.ObjectMeta)
 		if err != nil {
-			return errors.Wrapf(err, "failed to apply network config to cephfs plugin provisioner deployment: %+v", cephfsProvisionerDeployment)
+			return errors.Wrapf(err, "failed to apply network config to cephfs plugin provisioner deployment %q", cephfsProvisionerDeployment.Name)
 		}
-		_, err = k8sutil.CreateOrUpdateDeployment(clientset, cephfsProvisionerDeployment)
+		_, err = k8sutil.CreateOrUpdateDeployment(r.context.Clientset, cephfsProvisionerDeployment)
 		if err != nil {
-			return errors.Wrapf(err, "failed to start cephfs provisioner deployment: %+v", cephfsProvisionerDeployment)
+			return errors.Wrapf(err, "failed to start cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name)
 		}
 		k8sutil.AddRookVersionLabelToDeployment(cephfsProvisionerDeployment)
 		logger.Info("successfully started CSI CephFS driver")
@@ -573,31 +526,20 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		if err != nil {
 			return errors.Wrapf(err, "failed to set owner reference to cephfs service %q", cephfsService)
 		}
-		_, err = k8sutil.CreateOrUpdateService(clientset, namespace, cephfsService)
+		_, err = k8sutil.CreateOrUpdateService(r.context.Clientset, r.opConfig.OperatorNamespace, cephfsService)
 		if err != nil {
-			return errors.Wrapf(err, "failed to create cephfs service: %+v", cephfsService)
+			return errors.Wrapf(err, "failed to create cephfs service %q", cephfsService.Name)
 		}
 	}
 
 	if EnableRBD {
-		fsGroupPolicyForRBD, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy))
-		if err != nil {
-			// logging a warning and intentionally continuing with the default log level
-			logger.Warningf("failed to parse CSI_RBD_FSGROUPPOLICY. Defaulting to %q. %v", k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy, err)
-		}
-		err = csiDriverobj.createCSIDriverInfo(ctx, clientset, RBDDriverName, fsGroupPolicyForRBD)
+		err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, RBDDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)))
 		if err != nil {
 			return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName)
 		}
 	}
 	if EnableCephFS {
-		fsGroupPolicyForCephFS, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy))
-		if err != nil {
-			// logging a warning and intentionally continuing with the default
-			// log level
-			logger.Warningf("failed to parse CSI_CEPHFS_FSGROUPPOLICY. Defaulting to %q. %v", k8scsi.NoneFSGroupPolicy, err)
-		}
-		err = csiDriverobj.createCSIDriverInfo(ctx, clientset, CephFSDriverName, fsGroupPolicyForCephFS)
+		err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, CephFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)))
 		if err != nil {
 			return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName)
 		}
@@ -606,10 +548,10 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 	return nil
 }
 
-func stopDrivers(clientset kubernetes.Interface, namespace string, ver *version.Info) {
+func (r *ReconcileCSI) stopDrivers(ver *version.Info) {
 	if !EnableRBD {
 		logger.Info("CSI Ceph RBD driver disabled")
-		succeeded := deleteCSIDriverResources(clientset, ver, namespace, csiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName)
+		succeeded := r.deleteCSIDriverResources(ver, csiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName)
 		if succeeded {
 			logger.Info("successfully removed CSI Ceph RBD driver")
 		} else {
@@ -619,7 +561,7 @@ func stopDrivers(clientset kubernetes.Interface, namespace string, ver *version.
 
 	if !EnableCephFS {
 		logger.Info("CSI CephFS driver disabled")
-		succeeded := deleteCSIDriverResources(clientset, ver, namespace, csiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName)
+		succeeded := r.deleteCSIDriverResources(ver, csiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName)
 		if succeeded {
 			logger.Info("successfully removed CSI CephFS driver")
 		} else {
@@ -628,33 +570,31 @@ func stopDrivers(clientset kubernetes.Interface, namespace string, ver *version.
 	}
 }
 
-func deleteCSIDriverResources(
-	clientset kubernetes.Interface, ver *version.Info, namespace, daemonset, deployment, service, driverName string) bool {
-	ctx := context.TODO()
+func (r *ReconcileCSI) deleteCSIDriverResources(ver *version.Info, daemonset, deployment, service, driverName string) bool {
 	succeeded := true
 	csiDriverobj = beta1CsiDriver{}
 	if ver.Major > KubeMinMajor || ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForV1csiDriver {
 		csiDriverobj = v1CsiDriver{}
 	}
-	err := k8sutil.DeleteDaemonset(clientset, namespace, daemonset)
+	err := k8sutil.DeleteDaemonset(r.context.Clientset, r.opConfig.OperatorNamespace, daemonset)
 	if err != nil {
 		logger.Errorf("failed to delete the %q. %v", daemonset, err)
 		succeeded = false
 	}
 
-	err = k8sutil.DeleteDeployment(clientset, namespace, deployment)
+	err = k8sutil.DeleteDeployment(r.context.Clientset, r.opConfig.OperatorNamespace, deployment)
 	if err != nil {
 		logger.Errorf("failed to delete the %q. %v", deployment, err)
 		succeeded = false
 	}
 
-	err = k8sutil.DeleteService(clientset, namespace, service)
+	err = k8sutil.DeleteService(r.context.Clientset, r.opConfig.OperatorNamespace, service)
 	if err != nil {
 		logger.Errorf("failed to delete the %q. %v", service, err)
 		succeeded = false
 	}
 
-	err = csiDriverobj.deleteCSIDriverInfo(ctx, clientset, driverName)
+	err = csiDriverobj.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, driverName)
 	if err != nil {
 		logger.Errorf("failed to delete %q Driver Info. %v", driverName, err)
 		succeeded = false
@@ -662,9 +602,9 @@ func deleteCSIDriverResources(
 	return succeeded
 }
 
-func applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.ObjectMeta, rookclientset rookclient.Interface) (bool, error) {
+func (r *ReconcileCSI) applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.ObjectMeta) (bool, error) {
 	var isMultusApplied bool
-	cephClusters, err := rookclientset.CephV1().CephClusters(objectMeta.Namespace).List(ctx, metav1.ListOptions{})
+	cephClusters, err := r.context.RookClientset.CephV1().CephClusters(objectMeta.Namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return false, errors.Wrap(err, "failed to find CephClusters")
 	}
@@ -682,27 +622,32 @@ func applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.Objec
 }
 
 // ValidateCSIVersion checks if the configured ceph-csi image is supported
-func validateCSIVersion(clientset kubernetes.Interface, namespace, rookImage, serviceAccountName string, ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) {
+func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) {
 	timeout := 15 * time.Minute
 
 	logger.Infof("detecting the ceph csi image version for image %q", CSIParam.CSIPluginImage)
 
 	versionReporter, err := cmdreporter.New(
-		clientset,
+		r.context.Clientset,
 		ownerInfo,
-		detectCSIVersionName, detectCSIVersionName, namespace,
-		[]string{"cephcsi"}, []string{"--version"},
-		rookImage, CSIParam.CSIPluginImage)
+		detectCSIVersionName,
+		detectCSIVersionName,
+		r.opConfig.OperatorNamespace,
+		[]string{"cephcsi"},
+		[]string{"--version"},
+		r.opConfig.Image,
+		CSIParam.CSIPluginImage,
+	)
 
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to set up ceph CSI version job")
 	}
 
 	job := versionReporter.Job()
-	job.Spec.Template.Spec.ServiceAccountName = serviceAccountName
+	job.Spec.Template.Spec.ServiceAccountName = r.opConfig.ServiceAccount
 
 	// Apply csi provisioner toleration for csi version check job
-	job.Spec.Template.Spec.Tolerations = getToleration(clientset, provisionerTolerationsEnv, []corev1.Toleration{})
+	job.Spec.Template.Spec.Tolerations = getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{})
 	stdout, _, retcode, err := versionReporter.Run(timeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to complete ceph CSI version job")

pkg/operator/ceph/csi/spec_test.go

@@ -1,49 +0,0 @@
-/*
-Copyright 2019 The Rook Authors. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package csi
-
-import (
-	"testing"
-
-	rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake"
-	"github.com/rook/rook/pkg/clusterd"
-	"github.com/rook/rook/pkg/operator/test"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestStartCSI(t *testing.T) {
-	CSIParam = Param{
-		CSIPluginImage:   "image",
-		RegistrarImage:   "image",
-		ProvisionerImage: "image",
-		AttacherImage:    "image",
-		SnapshotterImage: "image",
-	}
-	clientset := test.New(t, 3)
-	context := &clusterd.Context{
-		Clientset:     clientset,
-		RookClientset: rookfake.NewSimpleClientset(),
-	}
-	serverVersion, err := clientset.Discovery().ServerVersion()
-	if err != nil {
-		assert.Nil(t, err)
-	}
-	AllowUnsupported = true
-	err = startDrivers(context.Clientset, context.RookClientset, "ns", serverVersion, nil, nil)
-	assert.Nil(t, err)
-}