diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index 3407a1a7f..aff489d59 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -17,12 +17,12 @@ limitations under the License. package ceph import ( + "os" + "github.com/pkg/errors" "github.com/rook/rook/cmd/rook/rook" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" operator "github.com/rook/rook/pkg/operator/ceph" - cluster "github.com/rook/rook/pkg/operator/ceph/cluster" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" @@ -41,7 +41,7 @@ https://github.com/rook/rook`, } func init() { - operatorCmd.Flags().BoolVar(&cluster.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers") + operatorCmd.Flags().BoolVar(&operator.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers") flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix) flags.SetLoggingFlags(operatorCmd.Flags()) @@ -49,9 +49,7 @@ func init() { } func startOperator(cmd *cobra.Command, args []string) error { - rook.SetLogLevel() - rook.LogStartupInfo(operatorCmd.Flags()) logger.Info("starting Rook-Ceph operator") @@ -62,6 +60,11 @@ func startOperator(cmd *cobra.Command, args []string) error { rook.TerminateFatal(err) } + // Fail if operator namespace is not provided + if os.Getenv(k8sutil.PodNamespaceEnvVar) == "" { + rook.TerminateFatal(errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %q", k8sutil.PodNamespaceEnvVar)) + } + rook.CheckOperatorResources(context.Clientset) rookImage := rook.GetOperatorImage(context.Clientset, containerName) rookBaseImageCephVersion, err := rook.GetOperatorBaseImageCephVersion(context) diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 3e20f3a72..f0cbbbc2b 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -17,6 +17,7 @@ limitations under the License. package ceph import ( + ctx "context" "encoding/json" "os" "strings" @@ -226,6 +227,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(&ownerRef, clusterInfo.Namespace) clusterInfo.OwnerInfo = ownerInfo + clusterInfo.Context = ctx.Background() kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerInfo) agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat, cfg.storeConfig, &clusterInfo, cfg.nodeName, kv, cfg.pvcBacked) diff --git a/cmd/rook/rook/rook.go b/cmd/rook/rook/rook.go index 8b700f3ae..e2235213f 100644 --- a/cmd/rook/rook/rook.go +++ b/cmd/rook/rook/rook.go @@ -33,7 +33,6 @@ import ( "github.com/rook/rook/pkg/version" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/tevino/abool" v1 "k8s.io/api/core/v1" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/util/uuid" @@ -170,8 +169,6 @@ func NewContext() *clusterd.Context { context.NetworkClient, err = netclient.NewForConfig(context.KubeConfig) TerminateOnError(err, "failed to create network clientset") - context.RequestCancelOrchestration = abool.New() - return context } @@ -228,23 +225,21 @@ func TerminateOnError(err error, msg string) { // TerminateFatal terminates the process with an exit code of 1 // and writes the given reason to stderr and the termination log file. func TerminateFatal(reason error) { - fmt.Fprintln(os.Stderr, reason) - file, err := os.OpenFile(terminationLog, os.O_APPEND|os.O_WRONLY, 0600) if err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %+v", err)) + fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %v", err)) } else { // #nosec G307 Calling defer to close the file without checking the error return is not a risk for a simple file open and close defer file.Close() if _, err = file.WriteString(reason.Error()); err != nil { - fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %+v", err)) + fmt.Fprintln(os.Stderr, fmt.Errorf("failed to write message to termination log: %v", err)) } if err := file.Close(); err != nil { - logger.Errorf("failed to close file. %v", err) + logger.Fatalf("failed to close file. %v", err) } } - os.Exit(1) + logger.Fatalln(reason) } // GetOperatorBaseImageCephVersion returns the Ceph version of the operator image diff --git a/go.mod b/go.mod index 50fbbbdce..e8b833a50 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/vault/api v1.1.2-0.20210713235431-1fc8af4c041f github.com/hashicorp/vault/sdk v0.2.2-0.20210825150427-9b1f4d486f5d github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.0 - github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875 + github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210907154902-775800a3d0b0 github.com/libopenstorage/secrets v0.0.0-20210709082113-dde442ea20ec github.com/openshift/cluster-api v0.0.0-20191129101638-b09907ac6668 github.com/openshift/machine-api-operator v0.2.1-0.20190903202259-474e14e4965a @@ -26,7 +26,6 @@ require ( github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 - github.com/tevino/abool v1.2.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/ini.v1 v1.57.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 8a7cf3938..32d87abf8 100644 --- a/go.sum +++ b/go.sum @@ -1067,8 +1067,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875 h1:jX3VXgmNOye8XYKjwcTVXcBYcPv3jj657fwX8DN/HiM= -github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875/go.mod h1:XpQ9HGG9uF5aJCBP+s6w5kSiyTIVSqCV8+XAE4qms5E= +github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210907154902-775800a3d0b0 h1:h7ghDWni/MDzN1AXXfvuwU88eazM0lvgntaouyJyALs= +github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210907154902-775800a3d0b0/go.mod h1:XpQ9HGG9uF5aJCBP+s6w5kSiyTIVSqCV8+XAE4qms5E= github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8= github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= @@ -1486,8 +1486,6 @@ github.com/svanharmelen/jsonapi v0.0.0-20180618144545-0c0828c3f16d/go.mod h1:BST github.com/tencentcloud/tencentcloud-sdk-go v3.0.83+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentcloud/tencentcloud-sdk-go v3.0.171+incompatible h1:K3fcS92NS8cRntIdu8Uqy2ZSePvX73nNhOkKuPGJLXQ= github.com/tencentcloud/tencentcloud-sdk-go v3.0.171+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= -github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA= -github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4= diff --git a/pkg/clusterd/context.go b/pkg/clusterd/context.go index eccd4704f..f8c318478 100644 --- a/pkg/clusterd/context.go +++ b/pkg/clusterd/context.go @@ -22,7 +22,6 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/sys" - "github.com/tevino/abool" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -32,7 +31,6 @@ import ( // Context for loading or applying the configuration state of a service. type Context struct { - // The kubernetes config used for this context KubeConfig *rest.Config @@ -71,7 +69,4 @@ type Context struct { // The local devices detected on the node Devices []*sys.LocalDisk - - // RequestCancelOrchestration manages the orchestration and its possible cancellation - RequestCancelOrchestration *abool.AtomicBool } diff --git a/pkg/daemon/ceph/agent/controller.go b/pkg/daemon/ceph/agent/controller.go new file mode 100644 index 000000000..43791b083 --- /dev/null +++ b/pkg/daemon/ceph/agent/controller.go @@ -0,0 +1,135 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package agent + +import ( + "context" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume" + opagent "github.com/rook/rook/pkg/operator/ceph/agent" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/provisioner" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + flexcontroller "sigs.k8s.io/sig-storage-lib-external-provisioner/v6/controller" +) + +const ( + controllerName = "rook-ceph-operator-flex-controller" + provisionerName = "ceph.rook.io/block" + provisionerNameLegacy = "rook.io/block" +) + +var ( + // The supported configurations for the volume provisioner + provisionerConfigs = map[string]string{ + provisionerName: flexvolume.FlexvolumeVendor, + provisionerNameLegacy: flexvolume.FlexvolumeVendorLegacy, + } +) + +// ReconcileAgent reconciles the Flex driver +type ReconcileAgent struct { + client client.Client + context *clusterd.Context + opManagerContext context.Context + opConfig opcontroller.OperatorConfig +} + +// Add creates a new CephClient Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { + return &ReconcileAgent{ + client: mgr.GetClient(), + context: context, + opConfig: opConfig, + opManagerContext: opManagerContext, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + logger.Infof("%s successfully started", controllerName) + + // Watch for ConfigMap (operator config) + err = c.Watch(&source.Kind{ + Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController()) + if err != nil { + return err + } + + return nil +} + +// Reconcile reads that state of the operator config map and makes changes based on the state read +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileAgent) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface + reconcileResponse, err := r.reconcile(request) + if err != nil { + logger.Errorf("failed to reconcile %v", err) + } + + return reconcileResponse, err +} + +func (r *ReconcileAgent) reconcile(request reconcile.Request) (reconcile.Result, error) { + if opcontroller.FlexDriverEnabled(r.context) { + rookAgent := opagent.New(r.context.Clientset) + if err := rookAgent.Start(r.opConfig.OperatorNamespace, r.opConfig.Image, r.opConfig.ServiceAccount); err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to start agent daemonset") + } + + serverVersion, err := r.context.Clientset.Discovery().ServerVersion() + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version") + } + for name, vendor := range provisionerConfigs { + volumeProvisioner := provisioner.New(r.context, vendor) + pc := flexcontroller.NewProvisionController( + r.context.Clientset, + name, + volumeProvisioner, + serverVersion.GitVersion, + ) + // TODO: register each go routine!!! + go pc.Run(r.opManagerContext) + logger.Infof("rook-provisioner %q started using %q flex vendor dir", name, vendor) + } + } + + logger.Info("done reconciling") + return reconcile.Result{}, nil +} diff --git a/pkg/daemon/ceph/agent/flexvolume/controller.go b/pkg/daemon/ceph/agent/flexvolume/controller.go index a6eab2692..e997ad999 100644 --- a/pkg/daemon/ceph/agent/flexvolume/controller.go +++ b/pkg/daemon/ceph/agent/flexvolume/controller.go @@ -398,7 +398,7 @@ func (c *Controller) GetClientAccessInfo(args []string, clientAccessInfo *Client ctx := context.TODO() // args: 0 ClusterNamespace, 1 PodNamespace, 2 MountUser, 3 MountSecret clusterNamespace := args[0] - clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, clusterNamespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, ctx, clusterNamespace) if err != nil { return errors.Wrapf(err, "failed to load cluster information from clusters namespace %s", clusterNamespace) } diff --git a/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager.go b/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager.go index 56147e32e..f937d4c8e 100644 --- a/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager.go +++ b/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager.go @@ -23,6 +23,8 @@ import ( "strings" "time" + ctx "context" + "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" @@ -229,7 +231,7 @@ func (vm *VolumeManager) isAttached(image, pool, clusterNamespace string) (strin } func getClusterInfo(context *clusterd.Context, clusterNamespace string) (string, string, error) { - clusterInfo, _, _, err := mon.LoadClusterInfo(context, clusterNamespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(context, ctx.TODO(), clusterNamespace) if err != nil { return "", "", errors.Wrapf(err, "failed to load cluster information from cluster %s", clusterNamespace) } diff --git a/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go b/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go index 6d1ae9f1e..34f297c8f 100644 --- a/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go +++ b/pkg/daemon/ceph/agent/flexvolume/manager/ceph/manager_test.go @@ -162,7 +162,7 @@ func TestAttach(t *testing.T) { }, } ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo) + _, _, _, err = mon.CreateOrLoadClusterInfo(context, ctx, clusterNamespace, ownerInfo) assert.NoError(t, err) devicePath, err := vm.Attach("image1", "testpool", "admin", "never-gonna-give-you-up", clusterNamespace) assert.Equal(t, "/dev/rbd3", devicePath) @@ -244,7 +244,7 @@ func TestDetach(t *testing.T) { }, } ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo) + _, _, _, err = mon.CreateOrLoadClusterInfo(context, ctx, clusterNamespace, ownerInfo) assert.NoError(t, err) err = vm.Detach("image1", "testpool", "admin", "", clusterNamespace, false) assert.Nil(t, err) @@ -300,7 +300,7 @@ func TestDetachCustomKeyring(t *testing.T) { }, } ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - _, _, _, err = mon.CreateOrLoadClusterInfo(context, clusterNamespace, ownerInfo) + _, _, _, err = mon.CreateOrLoadClusterInfo(context, ctx, clusterNamespace, ownerInfo) assert.NoError(t, err) err = vm.Detach("image1", "testpool", "user1", "", clusterNamespace, false) assert.Nil(t, err) diff --git a/pkg/daemon/ceph/agent/predicate.go b/pkg/daemon/ceph/agent/predicate.go new file mode 100644 index 000000000..0cb002c71 --- /dev/null +++ b/pkg/daemon/ceph/agent/predicate.go @@ -0,0 +1,57 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package agent + +import ( + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// predicateController is the predicate function to trigger reconcile on operator configuration cm change +func predicateController() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + if cm, ok := e.Object.(*v1.ConfigMap); ok { + return cm.Name == opcontroller.OperatorSettingConfigMapName + } + + return false + }, + + UpdateFunc: func(e event.UpdateEvent) bool { + if old, ok := e.ObjectOld.(*v1.ConfigMap); ok { + if new, ok := e.ObjectNew.(*v1.ConfigMap); ok { + if old.Name == opcontroller.OperatorSettingConfigMapName && new.Name == opcontroller.OperatorSettingConfigMapName { + return old.Data["ROOK_ENABLE_FLEX_DRIVER"] != new.Data["ROOK_ENABLE_FLEX_DRIVER"] + } + } + } + + return false + }, + + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + } +} diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go index fcfdea3ea..a9cf1272e 100644 --- a/pkg/daemon/ceph/client/command.go +++ b/pkg/daemon/ceph/client/command.go @@ -127,6 +127,9 @@ func NewRBDCommand(context *clusterd.Context, clusterInfo *ClusterInfo, args []s func (c *CephToolCommand) run() ([]byte, error) { command, args := FinalizeCephCommandArgs(c.tool, c.clusterInfo, c.args, c.context.ConfigDir) + if c.clusterInfo.Context.Err() != nil { + return nil, c.clusterInfo.Context.Err() + } if c.JsonOutput { args = append(args, "--format", "json") } else { diff --git a/pkg/daemon/ceph/client/command_test.go b/pkg/daemon/ceph/client/command_test.go index aed9bb3ca..574047b9f 100644 --- a/pkg/daemon/ceph/client/command_test.go +++ b/pkg/daemon/ceph/client/command_test.go @@ -17,6 +17,7 @@ limitations under the License. package client import ( + "context" "strconv" "testing" "time" @@ -140,4 +141,18 @@ func TestNewRBDCommand(t *testing.T) { assert.EqualError(t, err, "no pods found with selector \"rook-ceph-mgr\"") }) + t.Run("context canceled nothing to run", func(t *testing.T) { + clusterInfo := AdminClusterInfo("rook") + ctx, cancel := context.WithCancel(context.TODO()) + clusterInfo.Context = ctx + cancel() + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor, RemoteExecutor: exec.RemotePodCommandExecutor{ClientSet: test.New(t, 3)}} + cmd := NewRBDCommand(context, clusterInfo, args) + _, err := cmd.Run() + assert.Error(t, err) + // This is not the best but it shows we go through the right codepath + assert.EqualError(t, err, "context canceled") + }) + } diff --git a/pkg/daemon/ceph/client/config.go b/pkg/daemon/ceph/client/config.go index aedb65d5a..2bf2aa932 100644 --- a/pkg/daemon/ceph/client/config.go +++ b/pkg/daemon/ceph/client/config.go @@ -18,7 +18,6 @@ limitations under the License. package client import ( - "context" "fmt" "io/ioutil" "net" @@ -139,8 +138,7 @@ func generateConfigFile(context *clusterd.Context, clusterInfo *ClusterInfo, pat } func mergeDefaultConfigWithRookConfigOverride(clusterdContext *clusterd.Context, clusterInfo *ClusterInfo, configFile *ini.File) error { - ctx := context.TODO() - cm, err := clusterdContext.Clientset.CoreV1().ConfigMaps(clusterInfo.Namespace).Get(ctx, k8sutil.ConfigOverrideName, metav1.GetOptions{}) + cm, err := clusterdContext.Clientset.CoreV1().ConfigMaps(clusterInfo.Namespace).Get(clusterInfo.Context, k8sutil.ConfigOverrideName, metav1.GetOptions{}) if err != nil { if !kerrors.IsNotFound(err) { return errors.Wrapf(err, "failed to read configmap %q", k8sutil.ConfigOverrideName) diff --git a/pkg/daemon/ceph/client/config_test.go b/pkg/daemon/ceph/client/config_test.go index ed6ed4dd5..8a60614a8 100644 --- a/pkg/daemon/ceph/client/config_test.go +++ b/pkg/daemon/ceph/client/config_test.go @@ -108,6 +108,7 @@ func TestGenerateConfigFile(t *testing.T) { }, CephVersion: cephver.Nautilus, CephCred: CephCred{Username: "admin", Secret: "mysecret"}, + Context: ctx, } isInitialized := clusterInfo.IsInitialized(true) diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index 6c5ab5f52..217e99d2f 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -17,6 +17,7 @@ limitations under the License. package client import ( + "context" "fmt" "net" "testing" @@ -46,6 +47,14 @@ type ClusterInfo struct { name string OsdUpgradeTimeout time.Duration NetworkSpec cephv1.NetworkSpec + // A context to cancel the context it is used to determine whether the reconcile loop should + // exist (if the context has been cancelled). This cannot be in main clusterd context since this + // is a pointer passed through the entire life cycle or the operator. If the context is + // cancelled it will immedialy be re-created, thus existing reconciles loops will not be + // cancelled. + // Whereas if passed through clusterInfo, we don't have that problem since clusterInfo is + // re-hydrated when a context is cancelled. + Context context.Context } // MonInfo is a collection of information about a Ceph mon. @@ -90,6 +99,7 @@ func AdminClusterInfo(namespace string) *ClusterInfo { }, name: "testing", OwnerInfo: ownerInfo, + Context: context.TODO(), } } @@ -120,6 +130,10 @@ func (c *ClusterInfo) IsInitialized(logError bool) bool { if logError { logger.Error("ceph secret is empty") } + } else if c.Context == nil { + if logError { + logger.Error("nil context") + } } else { isInitialized = true } diff --git a/pkg/daemon/ceph/client/test/info.go b/pkg/daemon/ceph/client/test/info.go index 6efd21917..052fe696b 100644 --- a/pkg/daemon/ceph/client/test/info.go +++ b/pkg/daemon/ceph/client/test/info.go @@ -17,6 +17,7 @@ limitations under the License. package test import ( + "context" "fmt" "io/ioutil" "os" @@ -53,6 +54,7 @@ func CreateTestClusterInfo(monCount int) *client.ClusterInfo { }, Monitors: map[string]*client.MonInfo{}, OwnerInfo: ownerInfo, + Context: context.TODO(), } mons := []string{"a", "b", "c", "d", "e"} for i := 0; i < monCount; i++ { diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index 2035d84bb..92b9514cc 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -300,7 +300,7 @@ func TestGetRetryConfig(t *testing.T) { } func TestOSDUpdateShouldCheckOkToStop(t *testing.T) { - clusterInfo := &ClusterInfo{} + clusterInfo := AdminClusterInfo("mycluster") lsOutput := "" treeOutput := "" context := &clusterd.Context{ diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index b158fc7d7..b1edce0fb 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -320,7 +320,7 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd // If we detect a partition we have to make sure that ceph-volume will be able to consume it // ceph-volume version 14.2.8 has the right code to support partitions if !agent.clusterInfo.CephVersion.IsAtLeast(cephVolumeRawModeMinCephVersion) { - logger.Infof("skipping device %q because it is a partition and ceph version is too old, you need at least ceph %q", device.Name, cephVolumeRawModeMinCephVersion.String()) + logger.Infof("skipping device %q because it is a partition and ceph version is too old %q, you need at least ceph %q", device.Name, agent.clusterInfo.CephVersion.String(), cephVolumeRawModeMinCephVersion.String()) continue } diff --git a/pkg/daemon/ceph/osd/device_test.go b/pkg/daemon/ceph/osd/device_test.go index 3b9fc8f04..899e54116 100644 --- a/pkg/daemon/ceph/osd/device_test.go +++ b/pkg/daemon/ceph/osd/device_test.go @@ -40,7 +40,7 @@ func TestOSDBootstrap(t *testing.T) { context := &clusterd.Context{Executor: executor, ConfigDir: configDir} defer os.RemoveAll(context.ConfigDir) - err := createOSDBootstrapKeyring(context, &client.ClusterInfo{Namespace: "name"}, configDir) + err := createOSDBootstrapKeyring(context, client.AdminClusterInfo("mycluster"), configDir) assert.Nil(t, err) targetPath := path.Join(configDir, bootstrapOsdKeyring) diff --git a/pkg/daemon/ceph/osd/kms/k8s.go b/pkg/daemon/ceph/osd/kms/k8s.go index d1e693314..bd4860aef 100644 --- a/pkg/daemon/ceph/osd/kms/k8s.go +++ b/pkg/daemon/ceph/osd/kms/k8s.go @@ -17,7 +17,6 @@ limitations under the License. package kms import ( - "context" "fmt" "github.com/pkg/errors" @@ -42,14 +41,13 @@ const ( // storeSecretInKubernetes stores the dmcrypt key in a Kubernetes Secret func (c *Config) storeSecretInKubernetes(pvcName, key string) error { - ctx := context.TODO() s, err := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo) if err != nil { return err } // Create the Kubernetes Secret - _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, s, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, s, metav1.CreateOptions{}) if err != nil && !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to save ceph osd encryption key as a secret for pvc %q", pvcName) } diff --git a/pkg/daemon/ceph/osd/remove.go b/pkg/daemon/ceph/osd/remove.go index a4f032666..dcb3c05a4 100644 --- a/pkg/daemon/ceph/osd/remove.go +++ b/pkg/daemon/ceph/osd/remove.go @@ -17,7 +17,6 @@ limitations under the License. package osd import ( - "context" "fmt" "strconv" @@ -68,7 +67,6 @@ func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osds } func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInfo, osdID int, preservePVC bool) { - ctx := context.TODO() // Get the host where the OSD is found hostName, err := client.GetCrushHostName(clusterdContext, clusterInfo, osdID) if err != nil { @@ -84,7 +82,7 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf // Remove the OSD deployment deploymentName := fmt.Sprintf("rook-ceph-osd-%d", osdID) - deployment, err := clusterdContext.Clientset.AppsV1().Deployments(clusterInfo.Namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + deployment, err := clusterdContext.Clientset.AppsV1().Deployments(clusterInfo.Namespace).Get(clusterInfo.Context, deploymentName, metav1.GetOptions{}) if err != nil { logger.Errorf("failed to fetch the deployment %q. %v", deploymentName, err) } else { @@ -97,7 +95,7 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf } if pvcName, ok := deployment.GetLabels()[osd.OSDOverPVCLabelKey]; ok { labelSelector := fmt.Sprintf("%s=%s", osd.OSDOverPVCLabelKey, pvcName) - prepareJobList, err := clusterdContext.Clientset.BatchV1().Jobs(clusterInfo.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + prepareJobList, err := clusterdContext.Clientset.BatchV1().Jobs(clusterInfo.Namespace).List(clusterInfo.Context, metav1.ListOptions{LabelSelector: labelSelector}) if err != nil && !kerrors.IsNotFound(err) { logger.Errorf("failed to list osd prepare jobs with pvc %q. %v ", pvcName, err) } @@ -114,20 +112,20 @@ func removeOSD(clusterdContext *clusterd.Context, clusterInfo *client.ClusterInf if preservePVC { // Detach the OSD PVC from Rook. We will continue OSD deletion even if failed to remove PVC label logger.Infof("detach the OSD PVC %q from Rook", pvcName) - if pvc, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Get(ctx, pvcName, metav1.GetOptions{}); err != nil { + if pvc, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Get(clusterInfo.Context, pvcName, metav1.GetOptions{}); err != nil { logger.Errorf("failed to get pvc for OSD %q. %v", pvcName, err) } else { labels := pvc.GetLabels() delete(labels, osd.CephDeviceSetPVCIDLabelKey) pvc.SetLabels(labels) - if _, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil { + if _, err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Update(clusterInfo.Context, pvc, metav1.UpdateOptions{}); err != nil { logger.Errorf("failed to remove label %q from pvc for OSD %q. %v", osd.CephDeviceSetPVCIDLabelKey, pvcName, err) } } } else { // Remove the OSD PVC logger.Infof("removing the OSD PVC %q", pvcName) - if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { + if err := clusterdContext.Clientset.CoreV1().PersistentVolumeClaims(clusterInfo.Namespace).Delete(clusterInfo.Context, pvcName, metav1.DeleteOptions{}); err != nil { if err != nil { // Continue deleting the OSD PVC even if PVC deletion fails logger.Errorf("failed to delete pvc for OSD %q. %v", pvcName, err) diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index bc5be24f4..82d6531f4 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -17,6 +17,7 @@ limitations under the License. package osd import ( + "context" "fmt" "io/ioutil" "os" @@ -382,11 +383,12 @@ func TestConfigureCVDevices(t *testing.T) { return "", errors.Errorf("unknown command %s %s", command, args) } - context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir} clusterInfo := &cephclient.ClusterInfo{ CephVersion: cephver.CephVersion{Major: 14, Minor: 2, Extra: 8}, FSID: clusterFSID, + Context: context.TODO(), } + context := &clusterd.Context{Executor: executor, ConfigDir: cephConfigDir} agent := &OsdAgent{clusterInfo: clusterInfo, nodeName: nodeName, pvcBacked: true, storeConfig: config.StoreConfig{DeviceClass: "myds"}} devices := createPVCAvailableDevices() deviceOSDs, err := agent.configureCVDevices(context, devices) diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index 725bed314..131d574dd 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -64,24 +64,26 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileCephClient reconciles a CephClient object type ReconcileCephClient struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context } // Add creates a new CephClient Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { return &ReconcileCephClient{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, } } @@ -128,7 +130,7 @@ func (r *ReconcileCephClient) Reconcile(context context.Context, request reconci func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephClient instance cephClient := &cephv1.CephClient{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephClient) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephClient) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephClient resource not found. Ignoring since object must be deleted.") @@ -146,7 +148,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re // The CR was just created, initializing status fields if cephClient.Status == nil { - updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing) } // Make sure a CephCluster is present otherwise do nothing @@ -172,10 +174,11 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re } // Populate clusterInfo during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } + r.clusterInfo.Context = r.opManagerContext // DELETE: the CR was deleted if !cephClient.GetDeletionTimestamp().IsZero() { @@ -208,12 +211,12 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re logger.Info(opcontroller.OperatorNotInitializedMessage) return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil } - updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) return reconcile.Result{}, errors.Wrapf(err, "failed to create or update client %q", cephClient.Name) } // Success! Let's update the status - updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady) // Return and do not requeue logger.Debug("done reconciling") @@ -222,7 +225,6 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re // Create the client func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient) error { - ctx := context.TODO() logger.Infof("creating client %s in namespace %s", cephClient.Name, cephClient.Namespace) // Generate the CephX details @@ -261,11 +263,11 @@ func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient } // Create or Update Kubernetes Secret - _, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) + _, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Get(r.clusterInfo.Context, secret.Name, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("creating secret for %q", secret.Name) - if _, err := r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Create(r.clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to create secret for %q", secret.Name) } logger.Infof("created client %q", cephClient.Name) @@ -274,7 +276,7 @@ func (r *ReconcileCephClient) createOrUpdateClient(cephClient *cephv1.CephClient return errors.Wrapf(err, "failed to get secret for %q", secret.Name) } logger.Debugf("updating secret for %s", secret.Name) - _, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Update(ctx, secret, metav1.UpdateOptions{}) + _, err = r.context.Clientset.CoreV1().Secrets(cephClient.Namespace).Update(r.clusterInfo.Context, secret, metav1.UpdateOptions{}) if err != nil { return errors.Wrapf(err, "failed to update secret for %q", secret.Name) } @@ -332,9 +334,9 @@ func generateClientName(name string) string { } // updateStatus updates an object with a given status -func updateStatus(client client.Client, name types.NamespacedName, status cephv1.ConditionType) { +func (r *ReconcileCephClient) updateStatus(client client.Client, name types.NamespacedName, status cephv1.ConditionType) { cephClient := &cephv1.CephClient{} - if err := client.Get(context.TODO(), name, cephClient); err != nil { + if err := client.Get(r.opManagerContext, name, cephClient); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephClient resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/client/controller_test.go b/pkg/operator/ceph/client/controller_test.go index f90bc913e..215dd6542 100644 --- a/pkg/operator/ceph/client/controller_test.go +++ b/pkg/operator/ceph/client/controller_test.go @@ -32,7 +32,6 @@ import ( testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -155,10 +154,9 @@ func TestCephClientController(t *testing.T) { }, } c := &clusterd.Context{ - Executor: executor, - Clientset: testop.New(t, 1), - RookClientset: rookclient.NewSimpleClientset(), - RequestCancelOrchestration: abool.New(), + Executor: executor, + Clientset: testop.New(t, 1), + RookClientset: rookclient.NewSimpleClientset(), } // Register operator types with the runtime scheme. @@ -170,9 +168,10 @@ func TestCephClientController(t *testing.T) { // Create a ReconcileCephClient object with the scheme and fake client. r := &ReconcileCephClient{ - client: cl, - scheme: s, - context: c, + client: cl, + scheme: s, + context: c, + opManagerContext: ctx, } // Mock request to simulate Reconcile() being called on an event for a @@ -217,9 +216,10 @@ func TestCephClientController(t *testing.T) { cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() // Create a ReconcileCephClient object with the scheme and fake client. r = &ReconcileCephClient{ - client: cl, - scheme: s, - context: c, + client: cl, + scheme: s, + context: c, + opManagerContext: ctx, } assert.True(t, res.Requeue) @@ -274,15 +274,10 @@ func TestCephClientController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephClient object with the scheme and fake client. r = &ReconcileCephClient{ - client: cl, - scheme: s, - context: c, - } - - r = &ReconcileCephClient{ - client: cl, - scheme: s, - context: c, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), } res, err = r.Reconcile(ctx, req) diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index acec9a1e0..899cd19a0 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -84,13 +84,13 @@ func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.Clu } // checkCephStatus periodically checks the health of the cluster -func (c *cephStatusChecker) checkCephStatus(stopCh chan struct{}) { +func (c *cephStatusChecker) checkCephStatus(context context.Context) { // check the status immediately before starting the loop c.checkStatus() for { select { - case <-stopCh: + case <-context.Done(): logger.Infof("stopping monitoring of ceph status") return @@ -174,7 +174,7 @@ func (c *cephStatusChecker) configureHealthSettings(status cephclient.CephStatus // updateStatus updates an object with a given status func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus, condition cephv1.ConditionType, reason cephv1.ConditionReason, message string, conditionStatus v1.ConditionStatus) { clusterName := c.clusterInfo.NamespacedName() - cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(context.TODO(), clusterName.Name, metav1.GetOptions{}) + cephCluster, err := c.context.RookClientset.CephV1().CephClusters(clusterName.Namespace).Get(c.clusterInfo.Context, clusterName.Name, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") @@ -241,10 +241,9 @@ func formatTime(t time.Time) string { } func (c *ClusterController) updateClusterCephVersion(image string, cephVersion cephver.CephVersion) { - ctx := context.TODO() logger.Infof("cluster %q: version %q detected for image %q", c.namespacedName.Namespace, cephVersion.String(), image) - cephCluster, err := c.context.RookClientset.CephV1().CephClusters(c.namespacedName.Namespace).Get(ctx, c.namespacedName.Name, metav1.GetOptions{}) + cephCluster, err := c.context.RookClientset.CephV1().CephClusters(c.namespacedName.Namespace).Get(c.OpManagerCtx, c.namespacedName.Name, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") @@ -324,7 +323,7 @@ func (c *cephStatusChecker) getRookPodsOnNode(node string) ([]v1.Pod, error) { listOpts := metav1.ListOptions{ FieldSelector: fmt.Sprintf("spec.nodeName=%s", node), } - pods, err := c.context.Clientset.CoreV1().Pods(clusterName.Namespace).List(context.TODO(), listOpts) + pods, err := c.context.Clientset.CoreV1().Pods(clusterName.Namespace).List(c.clusterInfo.Context, listOpts) if err != nil { return podsOnNode, errors.Wrapf(err, "failed to get pods on node %q", node) } diff --git a/pkg/operator/ceph/cluster/cleanup.go b/pkg/operator/ceph/cluster/cleanup.go index c0ed852e2..c86ccdee6 100644 --- a/pkg/operator/ceph/cluster/cleanup.go +++ b/pkg/operator/ceph/cluster/cleanup.go @@ -58,9 +58,9 @@ var ( sanitizeIterationDefault int32 = 1 ) -func (c *ClusterController) startClusterCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) { +func (c *ClusterController) startClusterCleanUp(context context.Context, cluster *cephv1.CephCluster, cephHosts []string, monSecret, clusterFSID string) { logger.Infof("starting clean up for cluster %q", cluster.Name) - err := c.waitForCephDaemonCleanUp(stopCleanupCh, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second) + err := c.waitForCephDaemonCleanUp(context, cluster, time.Duration(clusterCleanUpPolicyRetryInterval)*time.Second) if err != nil { logger.Errorf("failed to wait till ceph daemons are destroyed. %v", err) return @@ -183,7 +183,7 @@ func getCleanupPlacement(c cephv1.ClusterSpec) cephv1.Placement { return cephv1.Placement{Tolerations: tolerations} } -func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{}, cluster *cephv1.CephCluster, retryInterval time.Duration) error { +func (c *ClusterController) waitForCephDaemonCleanUp(context context.Context, cluster *cephv1.CephCluster, retryInterval time.Duration) error { logger.Infof("waiting for all the ceph daemons to be cleaned up in the cluster %q", cluster.Namespace) for { select { @@ -200,15 +200,14 @@ func (c *ClusterController) waitForCephDaemonCleanUp(stopCleanupCh chan struct{} logger.Debugf("waiting for ceph daemons in cluster %q to be cleaned up. Retrying in %q", cluster.Namespace, retryInterval.String()) - case <-stopCleanupCh: - return errors.New("cancelling the host cleanup job") + case <-context.Done(): + return errors.Errorf("cancelling the host cleanup job. %s", context.Err()) } } } // getCephHosts returns a list of host names where ceph daemon pods are running func (c *ClusterController) getCephHosts(namespace string) ([]string, error) { - ctx := context.TODO() cephAppNames := []string{mon.AppName, mgr.AppName, osd.AppName, object.AppName, mds.AppName, rbd.AppName, mirror.AppName} nodeNameList := sets.NewString() hostNameList := []string{} @@ -217,7 +216,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) { // get all the node names where ceph daemons are running for _, app := range cephAppNames { appLabelSelector := fmt.Sprintf("app=%s", app) - podList, err := c.context.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: appLabelSelector}) + podList, err := c.context.Clientset.CoreV1().Pods(namespace).List(c.OpManagerCtx, metav1.ListOptions{LabelSelector: appLabelSelector}) if err != nil { return hostNameList, errors.Wrapf(err, "could not list the %q pods", app) } @@ -244,7 +243,7 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) { } func (c *ClusterController) getCleanUpDetails(namespace string) (string, string, error) { - clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, namespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.OpManagerCtx, namespace) if err != nil { return "", "", errors.Wrap(err, "failed to get cluster info") } diff --git a/pkg/operator/ceph/cluster/cleanup_test.go b/pkg/operator/ceph/cluster/cleanup_test.go index 4f3876ae0..5ff10b483 100644 --- a/pkg/operator/ceph/cluster/cleanup_test.go +++ b/pkg/operator/ceph/cluster/cleanup_test.go @@ -49,19 +49,7 @@ func TestCleanupJobSpec(t *testing.T) { Clientset: clientset, RookClientset: rookfake.NewSimpleClientset(), } - operatorConfigCallbacks := []func() error{ - func() error { - logger.Infof("test success callback") - return nil - }, - } - addCallbacks := []func() error{ - func() error { - logger.Infof("test success callback") - return nil - }, - } - controller := NewClusterController(context, "", &attachment.MockAttachment{}, operatorConfigCallbacks, addCallbacks) + controller := NewClusterController(context, "", &attachment.MockAttachment{}) podTemplateSpec := controller.cleanUpJobTemplateSpec(cluster, "monSecret", "28b87851-8dc1-46c8-b1ec-90ec51a47c89") assert.Equal(t, expectedHostPath, podTemplateSpec.Spec.Containers[0].Env[0].Value) assert.Equal(t, expectedNamespace, podTemplateSpec.Spec.Containers[0].Env[1].Value) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index eb1182d3c..de46e96ad 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -22,7 +22,6 @@ import ( "fmt" "os/exec" "path" - "strings" "sync" "syscall" @@ -58,17 +57,14 @@ type cluster struct { Spec *cephv1.ClusterSpec namespacedName types.NamespacedName mons *mon.Cluster - stopCh chan struct{} - closedStopCh bool ownerInfo *k8sutil.OwnerInfo isUpgrade bool - watchersActivated bool - monitoringChannels map[string]*clusterHealth + monitoringRoutines map[string]*clusterHealth } type clusterHealth struct { - stopChan chan struct{} - monitoringRunning bool + internalCtx context.Context + internalCancel context.CancelFunc } func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerInfo *k8sutil.OwnerInfo) *cluster { @@ -81,8 +77,7 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync Spec: &c.Spec, context: context, namespacedName: types.NamespacedName{Namespace: c.Namespace, Name: c.Name}, - monitoringChannels: make(map[string]*clusterHealth), - stopCh: make(chan struct{}), + monitoringRoutines: make(map[string]*clusterHealth), ownerInfo: ownerInfo, mons: mon.New(context, c.Namespace, c.Spec, ownerInfo, csiMutex), } @@ -104,6 +99,7 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep } clusterInfo.OwnerInfo = c.ownerInfo clusterInfo.SetName(c.namespacedName.Name) + clusterInfo.Context = c.ClusterInfo.Context c.ClusterInfo = clusterInfo c.ClusterInfo.NetworkSpec = c.Spec.Network @@ -112,9 +108,8 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep return errors.New("the cluster identity was not established") } - // Check whether we need to cancel the orchestration - if err := controller.CheckForCancelledOrchestration(c.context); err != nil { - return err + if c.ClusterInfo.Context.Err() != nil { + return c.ClusterInfo.Context.Err() } // Execute actions after the monitors are up and running @@ -124,15 +119,6 @@ func (c *cluster) reconcileCephDaemons(rookImage string, cephVersion cephver.Cep return errors.Wrap(err, "failed to execute post actions after all the ceph monitors started") } - // If this is an upgrade, notify all the child controllers - if c.isUpgrade { - logger.Info("upgrade in progress, notifying child CRs") - err := c.notifyChildControllerOfUpgrade() - if err != nil { - return errors.Wrap(err, "failed to notify child CRs of upgrade") - } - } - // Start Ceph manager controller.UpdateCondition(c.context, c.namespacedName, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring Ceph Mgr(s)") mgrs := mgr.New(c.context, c.ClusterInfo, *c.Spec, rookImage) @@ -179,7 +165,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { } } - clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, cluster.Namespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace) if err != nil { logger.Infof("clusterInfo not yet found, must be a new cluster") } else { @@ -200,7 +186,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { // Test if the cluster has already been configured if the mgr deployment has been created. // If the mgr does not exist, the mons have never been verified to be in quorum. opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, mgr.AppName)} - mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(context.TODO(), opts) + mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(c.OpManagerCtx, opts) if err == nil && len(mgrDeployments.Items) > 0 && cluster.ClusterInfo != nil { c.configureCephMonitoring(cluster, clusterInfo) } @@ -245,6 +231,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error { controller.UpdateCondition(c.context, c.namespacedName, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring the Ceph cluster") + cluster.ClusterInfo.Context = c.OpManagerCtx // Run the orchestration err = cluster.reconcileCephDaemons(c.rookImage, *cephVersion) if err != nil { @@ -256,104 +243,15 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error { return nil } -func (c *cluster) notifyChildControllerOfUpgrade() error { - ctx := context.TODO() - version := strings.Replace(c.ClusterInfo.CephVersion.String(), " ", "-", -1) - - // List all child controllers - cephFilesystems, err := c.context.RookClientset.CephV1().CephFilesystems(c.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list ceph filesystem CRs") - } - for _, cephFilesystem := range cephFilesystems.Items { - if cephFilesystem.Labels == nil { - cephFilesystem.Labels = map[string]string{} - } - cephFilesystem.Labels["ceph_version"] = version - localCephFilesystem := cephFilesystem - _, err := c.context.RookClientset.CephV1().CephFilesystems(c.Namespace).Update(ctx, &localCephFilesystem, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to update ceph filesystem CR %q with new label", cephFilesystem.Name) - } - } - - cephObjectStores, err := c.context.RookClientset.CephV1().CephObjectStores(c.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list ceph object store CRs") - } - for _, cephObjectStore := range cephObjectStores.Items { - if cephObjectStore.Labels == nil { - cephObjectStore.Labels = map[string]string{} - } - cephObjectStore.Labels["ceph_version"] = version - localCephObjectStore := cephObjectStore - _, err := c.context.RookClientset.CephV1().CephObjectStores(c.Namespace).Update(ctx, &localCephObjectStore, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to update ceph object store CR %q with new label", cephObjectStore.Name) - } - } - - cephNFSes, err := c.context.RookClientset.CephV1().CephNFSes(c.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list ceph nfs CRs") - } - for _, cephNFS := range cephNFSes.Items { - if cephNFS.Labels == nil { - cephNFS.Labels = map[string]string{} - } - cephNFS.Labels["ceph_version"] = version - localCephNFS := cephNFS - _, err := c.context.RookClientset.CephV1().CephNFSes(c.Namespace).Update(ctx, &localCephNFS, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to update ceph nfs CR %q with new label", cephNFS.Name) - } - } - - cephRBDMirrors, err := c.context.RookClientset.CephV1().CephRBDMirrors(c.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list ceph rbd-mirror CRs") - } - for _, cephRBDMirror := range cephRBDMirrors.Items { - if cephRBDMirror.Labels == nil { - cephRBDMirror.Labels = map[string]string{} - } - cephRBDMirror.Labels["ceph_version"] = version - localCephRBDMirror := cephRBDMirror - _, err := c.context.RookClientset.CephV1().CephRBDMirrors(c.Namespace).Update(ctx, &localCephRBDMirror, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to update ceph rbd-mirror CR %q with new label", cephRBDMirror.Name) - } - } - - cephFilesystemMirrors, err := c.context.RookClientset.CephV1().CephFilesystemMirrors(c.Namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list cephfs mirror CRs") - } - for _, cephFilesystemMirror := range cephFilesystemMirrors.Items { - if cephFilesystemMirror.Labels == nil { - cephFilesystemMirror.Labels = map[string]string{} - } - cephFilesystemMirror.Labels["ceph_version"] = version - localCephFilesystemMirror := cephFilesystemMirror - _, err := c.context.RookClientset.CephV1().CephFilesystemMirrors(c.Namespace).Update(ctx, &localCephFilesystemMirror, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to update ceph nfs CR %q with new label", cephFilesystemMirror.Name) - } - } - - return nil -} - // Validate the cluster Specs func preClusterStartValidation(cluster *cluster) error { - ctx := context.TODO() if cluster.Spec.Mon.Count == 0 { logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) cluster.Spec.Mon.Count = mon.DefaultMonCount } if !cluster.Spec.Mon.AllowMultiplePerNode { // Check that there are enough nodes to have a chance of starting the requested number of mons - nodes, err := cluster.context.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + nodes, err := cluster.context.Clientset.CoreV1().Nodes().List(cluster.ClusterInfo.Context, metav1.ListOptions{}) if err == nil && len(nodes.Items) < cluster.Spec.Mon.Count { return errors.Errorf("cannot start %d mons on %d node(s) when allowMultiplePerNode is false", cluster.Spec.Mon.Count, len(nodes.Items)) } @@ -381,7 +279,7 @@ func preClusterStartValidation(cluster *cluster) error { } // Get network attachment definition - _, err := cluster.context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(ctx, nad, metav1.GetOptions{}) + _, err := cluster.context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(cluster.ClusterInfo.Context, nad, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector) diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index fbdc7fa02..e7f4246b0 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -49,8 +49,9 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // loop until we find the secret necessary to connect to the external cluster // then populate clusterInfo - cluster.ClusterInfo = mon.PopulateExternalClusterInfo(c.context, c.namespacedName.Namespace, cluster.ownerInfo) + cluster.ClusterInfo = mon.PopulateExternalClusterInfo(c.context, c.OpManagerCtx, c.namespacedName.Namespace, cluster.ownerInfo) cluster.ClusterInfo.SetName(c.namespacedName.Name) + cluster.ClusterInfo.Context = c.OpManagerCtx if !client.IsKeyringBase64Encoded(cluster.ClusterInfo.CephCred.Secret) { return errors.Errorf("invalid user health checker key for user %q", cluster.ClusterInfo.CephCred.Username) diff --git a/pkg/operator/ceph/cluster/cluster_test.go b/pkg/operator/ceph/cluster/cluster_test.go index d121c79f6..41be7dd51 100644 --- a/pkg/operator/ceph/cluster/cluster_test.go +++ b/pkg/operator/ceph/cluster/cluster_test.go @@ -22,6 +22,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" testop "github.com/rook/rook/pkg/operator/test" ) @@ -34,27 +35,27 @@ func TestPreClusterStartValidation(t *testing.T) { args args wantErr bool }{ - {"no settings", args{&cluster{Spec: &cephv1.ClusterSpec{}, context: &clusterd.Context{Clientset: testop.New(t, 3)}}}, false}, - {"even mons", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 2}}}}, false}, - {"missing stretch zones", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ + {"no settings", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), Spec: &cephv1.ClusterSpec{}, context: &clusterd.Context{Clientset: testop.New(t, 3)}}}, false}, + {"even mons", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 2}}}}, false}, + {"missing stretch zones", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ {Name: "a"}, }}}}}}, true}, - {"missing arbiter", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ + {"missing arbiter", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ {Name: "a"}, {Name: "b"}, {Name: "c"}, }}}}}}, true}, - {"missing zone name", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ + {"missing zone name", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ {Arbiter: true}, {Name: "b"}, {Name: "c"}, }}}}}}, true}, - {"valid stretch cluster", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 3, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ + {"valid stretch cluster", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 3, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ {Name: "a", Arbiter: true}, {Name: "b"}, {Name: "c"}, }}}}}}, false}, - {"not enough stretch nodes", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 5, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ + {"not enough stretch nodes", args{&cluster{ClusterInfo: client.AdminClusterInfo("rook-ceph"), context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 5, StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ {Name: "a", Arbiter: true}, {Name: "b"}, {Name: "c"}, diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 8de257bcc..d0fad5b03 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -21,9 +21,7 @@ import ( "context" "fmt" "os" - "strings" "sync" - "time" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -41,7 +39,7 @@ import ( corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" + apituntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -52,8 +50,7 @@ import ( ) const ( - controllerName = "ceph-cluster-controller" - detectCephVersionTimeout = 15 * time.Minute + controllerName = "ceph-cluster-controller" ) const ( @@ -84,35 +81,35 @@ var ControllerTypeMeta = metav1.TypeMeta{ // ClusterController controls an instance of a Rook cluster type ClusterController struct { - context *clusterd.Context - volumeAttachment attachment.Attachment - rookImage string - clusterMap map[string]*cluster - operatorConfigCallbacks []func() error - addClusterCallbacks []func() error - csiConfigMutex *sync.Mutex - osdChecker *osd.OSDHealthMonitor - client client.Client - namespacedName types.NamespacedName - recorder *k8sutil.EventReporter + context *clusterd.Context + volumeAttachment attachment.Attachment + rookImage string + clusterMap map[string]*cluster + csiConfigMutex *sync.Mutex + osdChecker *osd.OSDHealthMonitor + client client.Client + namespacedName types.NamespacedName + recorder *k8sutil.EventReporter + OpManagerCtx context.Context } // ReconcileCephCluster reconciles a CephFilesystem object type ReconcileCephCluster struct { client client.Client - scheme *runtime.Scheme + scheme *apituntime.Scheme context *clusterd.Context clusterController *ClusterController + opManagerContext context.Context } // Add creates a new CephCluster Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController) error { - return add(mgr, newReconciler(mgr, ctx, clusterController), ctx) +func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) error { + return add(mgr, newReconciler(mgr, ctx, clusterController, opManagerContext), ctx) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) reconcile.Reconciler { // add "rook-" prefix to the controller name to make sure it is clear to all reading the events // that they are coming from Rook. The controller name already has context that it is for Ceph // and from the cluster controller. @@ -123,6 +120,7 @@ func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController scheme: mgr.GetScheme(), context: ctx, clusterController: clusterController, + opManagerContext: opManagerContext, } } @@ -142,7 +140,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) }, }, &handler.EnqueueRequestForObject{}, - watchControllerPredicate(context)) + watchControllerPredicate()) if err != nil { return err } @@ -219,10 +217,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) func (r *ReconcileCephCluster) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, cephCluster, err := r.reconcile(request) - if err != nil && strings.Contains(err.Error(), opcontroller.CancellingOrchestrationMessage) { - logger.Infof("Cluster update requested. %s", opcontroller.CancellingOrchestrationMessage) - return opcontroller.ImmediateRetryResultNoBackoff, nil - } return reporting.ReportReconcileResult(logger, r.clusterController.recorder, cephCluster, reconcileResponse, err) @@ -240,7 +234,7 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R // Fetch the cephCluster instance cephCluster := &cephv1.CephCluster{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephCluster) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephCluster resource not found. Ignoring since object must be deleted.") @@ -293,26 +287,20 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) doCleanup := true // Start cluster clean up only if cleanupPolicy is applied to the ceph cluster - stopCleanupCh := make(chan struct{}) + internalCtx := context.Context(r.opManagerContext) if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.Spec.External.Enable { monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(cephCluster.Namespace) if err != nil { - logger.Warningf("failed to get mon secret. Skip cluster cleanup and remove finalizer. %v", err) + logger.Warningf("failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err) doCleanup = false } if doCleanup { cephHosts, err := r.clusterController.getCephHosts(cephCluster.Namespace) if err != nil { - close(stopCleanupCh) return reconcile.Result{}, cephCluster, errors.Wrapf(err, "failed to find valid ceph hosts in the cluster %q", cephCluster.Namespace) } - // Go will garbage collect the stopCleanupCh if it is left open once the cluster cleanup - // goroutine is no longer running (i.e., referencing the channel) - go r.clusterController.startClusterCleanUp(stopCleanupCh, cephCluster, cephHosts, monSecret, clusterFSID) - } else { - // stop channel not needed if the cleanup goroutine isn't started - close(stopCleanupCh) + go r.clusterController.startClusterCleanUp(internalCtx, cephCluster, cephHosts, monSecret, clusterFSID) } } @@ -322,7 +310,6 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) if err != nil { // If the cluster cannot be deleted, requeue the request for deletion to see if the conditions // will eventually be satisfied such as the volumes being removed - close(stopCleanupCh) return response, cephCluster, errors.Wrapf(err, "failed to clean up CephCluster %q", nsName.String()) } } @@ -338,15 +325,13 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) } // NewClusterController create controller for watching cluster custom resources created -func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment, operatorConfigCallbacks []func() error, addClusterCallbacks []func() error) *ClusterController { +func NewClusterController(context *clusterd.Context, rookImage string, volumeAttachment attachment.Attachment) *ClusterController { return &ClusterController{ - context: context, - volumeAttachment: volumeAttachment, - rookImage: rookImage, - clusterMap: make(map[string]*cluster), - operatorConfigCallbacks: operatorConfigCallbacks, - addClusterCallbacks: addClusterCallbacks, - csiConfigMutex: &sync.Mutex{}, + context: context, + volumeAttachment: volumeAttachment, + rookImage: rookImage, + clusterMap: make(map[string]*cluster), + csiConfigMutex: &sync.Mutex{}, } } @@ -370,20 +355,9 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster, // Set the spec cluster.Spec = &clusterObj.Spec - // Note that this lock is held through the callback process, as this creates CSI resources, but we must lock in - // this scope as the clusterMap is authoritative on cluster count and thus involved in the check for CSI resource - // deletion. If we ever add additional callback functions, we should tighten this lock. - c.csiConfigMutex.Lock() c.clusterMap[cluster.Namespace] = cluster logger.Infof("reconciling ceph cluster in namespace %q", cluster.Namespace) - for _, callback := range c.addClusterCallbacks { - if err := callback(); err != nil { - logger.Errorf("%v", err) - } - } - c.csiConfigMutex.Unlock() - // Start the main ceph cluster orchestration return c.initializeCluster(cluster) } @@ -400,17 +374,13 @@ func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (r logger.Infof("cleaning up CephCluster %q", nsName) if cluster, ok := c.clusterMap[cluster.Namespace]; ok { - // if not already stopped, stop clientcontroller and bucketController - if !cluster.closedStopCh { - close(cluster.stopCh) - cluster.closedStopCh = true - } - + // We used to stop the bucket controller here but when we get a DELETE event for the CephCluster + // we will reload the CRD manager anyway so the bucket controller go routine will be stopped + // since the op manager context is cancelled. // close the goroutines watching the health of the cluster (mons, osds, ceph status) for _, daemon := range monitorDaemonList { - if monitoring, ok := cluster.monitoringChannels[daemon]; ok && monitoring.monitoringRunning { - close(cluster.monitoringChannels[daemon].stopChan) - cluster.monitoringChannels[daemon].monitoringRunning = false + if monitoring, ok := cluster.monitoringRoutines[daemon]; ok && monitoring.internalCtx.Err() == nil { // if the context hasn't been cancelled + cluster.monitoringRoutines[daemon].internalCancel() } } } @@ -503,8 +473,7 @@ func (c *ClusterController) csiVolumesAllowForDeletion(cluster *cephv1.CephClust } func (c *ClusterController) checkPVPresentInCluster(drivers []string, clusterID string) (bool, error) { - ctx := context.TODO() - pv, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + pv, err := c.context.Clientset.CoreV1().PersistentVolumes().List(c.OpManagerCtx, metav1.ListOptions{}) if err != nil { return false, errors.Wrapf(err, "failed to list PV") } diff --git a/pkg/operator/ceph/cluster/controller_test.go b/pkg/operator/ceph/cluster/controller_test.go index 9cbe1b049..16a733ced 100644 --- a/pkg/operator/ceph/cluster/controller_test.go +++ b/pkg/operator/ceph/cluster/controller_test.go @@ -29,7 +29,6 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -func TestReconcile_DeleteCephCluster(t *testing.T) { +func TestReconcileDeleteCephCluster(t *testing.T) { ctx := context.TODO() cephNs := "rook-ceph" clusterName := "my-cluster" @@ -79,8 +78,7 @@ func TestReconcile_DeleteCephCluster(t *testing.T) { clusterdCtx := &clusterd.Context{ Clientset: k8sfake.NewSimpleClientset(), // reconcile looks for fake dependencies in the dynamic clientset - DynamicClientset: dynamicfake.NewSimpleDynamicClient(scheme, fakePool), - RequestCancelOrchestration: abool.New(), + DynamicClientset: dynamicfake.NewSimpleDynamicClient(scheme, fakePool), } // set up ClusterController @@ -90,21 +88,9 @@ func TestReconcile_DeleteCephCluster(t *testing.T) { return &rookalpha.VolumeList{Items: []rookalpha.Volume{}}, nil }, } - operatorConfigCallbacks := []func() error{ - func() error { - t.Log("test op config callback") - return nil - }, - } - addCallbacks := []func() error{ - func() error { - t.Log("test success callback") - return nil - }, - } // create the cluster controller and tell it that the cluster has been deleted - controller := NewClusterController(clusterdCtx, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks) + controller := NewClusterController(clusterdCtx, "", volumeAttachmentController) fakeRecorder := record.NewFakeRecorder(5) controller.recorder = k8sutil.NewEventReporter(fakeRecorder) @@ -118,6 +104,7 @@ func TestReconcile_DeleteCephCluster(t *testing.T) { scheme: scheme, context: clusterdCtx, clusterController: controller, + opManagerContext: context.TODO(), } req := reconcile.Request{NamespacedName: nsName} @@ -201,20 +188,8 @@ func Test_checkIfVolumesExist(t *testing.T) { }, } - operatorConfigCallbacks := []func() error{ - func() error { - logger.Infof("test success callback") - return nil - }, - } - addCallbacks := []func() error{ - func() error { - logger.Infof("test success callback") - return nil - }, - } // create the cluster controller and tell it that the cluster has been deleted - controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks) + controller := NewClusterController(context, "", volumeAttachmentController) clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}} // The test returns a volume on the first call @@ -244,23 +219,8 @@ func Test_checkIfVolumesExist(t *testing.T) { }, } - operatorConfigCallbacks := []func() error{ - func() error { - logger.Infof("test success callback") - return nil - }, - } - addCallbacks := []func() error{ - func() error { - logger.Infof("test success callback") - os.Setenv("ROOK_ENABLE_FLEX_DRIVER", "true") - os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace) - defer os.Unsetenv("ROOK_ENABLE_FLEX_DRIVER") - return nil - }, - } // create the cluster controller and tell it that the cluster has been deleted - controller := NewClusterController(context, "", volumeAttachmentController, operatorConfigCallbacks, addCallbacks) + controller := NewClusterController(context, "", volumeAttachmentController) clusterToDelete := &cephv1.CephCluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}} assert.NoError(t, controller.checkIfVolumesExist(clusterToDelete)) diff --git a/pkg/operator/ceph/cluster/crash/add.go b/pkg/operator/ceph/cluster/crash/add.go index 40a755c65..de315edc6 100644 --- a/pkg/operator/ceph/cluster/crash/add.go +++ b/pkg/operator/ceph/cluster/crash/add.go @@ -17,6 +17,7 @@ limitations under the License. package crash import ( + "context" "reflect" "strings" @@ -24,6 +25,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -47,16 +49,18 @@ const ( ) // Add adds a new Controller based on nodedrain.ReconcileNode and registers the relevant watches and handlers -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileNode{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, + opConfig: opConfig, } } diff --git a/pkg/operator/ceph/cluster/crash/crash.go b/pkg/operator/ceph/cluster/crash/crash.go index 8ff255ac4..0fef8b65b 100644 --- a/pkg/operator/ceph/cluster/crash/crash.go +++ b/pkg/operator/ceph/cluster/crash/crash.go @@ -17,7 +17,6 @@ limitations under the License. package crash import ( - "context" "fmt" "path" @@ -123,7 +122,7 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations [] return nil } - return controllerutil.CreateOrUpdate(context.TODO(), r.client, deploy, mutateFunc) + return controllerutil.CreateOrUpdate(r.opManagerContext, r.client, deploy, mutateFunc) } // createOrUpdateCephCron is a wrapper around controllerutil.CreateOrUpdate @@ -164,7 +163,7 @@ func (r *ReconcileNode) createOrUpdateCephCron(cephCluster cephv1.CephCluster, c // minimum k8s version required for v1 cronJob is 'v1.21.0'. Apply v1 if k8s version is at least 'v1.21.0', else apply v1beta1 cronJob. if useCronJobV1 { // delete v1beta1 cronJob if it already exists - err := r.client.Delete(context.TODO(), &v1beta1.CronJob{ObjectMeta: objectMeta}) + err := r.client.Delete(r.opManagerContext, &v1beta1.CronJob{ObjectMeta: objectMeta}) if err != nil && !apierrors.IsNotFound(err) { return controllerutil.OperationResultNone, errors.Wrapf(err, "failed to delete CronJob v1Beta1 %q", prunerName) } @@ -183,7 +182,7 @@ func (r *ReconcileNode) createOrUpdateCephCron(cephCluster cephv1.CephCluster, c return nil } - return controllerutil.CreateOrUpdate(context.TODO(), r.client, cronJob, mutateFunc) + return controllerutil.CreateOrUpdate(r.opManagerContext, r.client, cronJob, mutateFunc) } cronJob := &v1beta1.CronJob{ObjectMeta: objectMeta} err := controllerutil.SetControllerReference(&cephCluster, cronJob, r.scheme) @@ -200,7 +199,7 @@ func (r *ReconcileNode) createOrUpdateCephCron(cephCluster cephv1.CephCluster, c return nil } - return controllerutil.CreateOrUpdate(context.TODO(), r.client, cronJob, mutateFunc) + return controllerutil.CreateOrUpdate(r.opManagerContext, r.client, cronJob, mutateFunc) } func getCrashDirInitContainer(cephCluster cephv1.CephCluster) corev1.Container { diff --git a/pkg/operator/ceph/cluster/crash/crash_test.go b/pkg/operator/ceph/cluster/crash/crash_test.go index fa4d9786e..3ead1f591 100644 --- a/pkg/operator/ceph/cluster/crash/crash_test.go +++ b/pkg/operator/ceph/cluster/crash/crash_test.go @@ -27,7 +27,6 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" v1 "k8s.io/api/batch/v1" "k8s.io/api/batch/v1beta1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -48,9 +47,8 @@ func TestCreateOrUpdateCephCron(t *testing.T) { cephVersion := &cephver.CephVersion{Major: 16, Minor: 2, Extra: 0} ctx := context.TODO() context := &clusterd.Context{ - Clientset: test.New(t, 1), - RookClientset: rookclient.NewSimpleClientset(), - RequestCancelOrchestration: abool.New(), + Clientset: test.New(t, 1), + RookClientset: rookclient.NewSimpleClientset(), } s := scheme.Scheme diff --git a/pkg/operator/ceph/cluster/crash/keyring.go b/pkg/operator/ceph/cluster/crash/keyring.go index a6400f8b0..157fe4c7c 100644 --- a/pkg/operator/ceph/cluster/crash/keyring.go +++ b/pkg/operator/ceph/cluster/crash/keyring.go @@ -96,7 +96,7 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo // Create Kubernetes Secret err = k.CreateSecret(s) if err != nil { - return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", crashCollectorSecret, clusterInfo.Namespace) + return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace) } logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace) diff --git a/pkg/operator/ceph/cluster/crash/reconcile.go b/pkg/operator/ceph/cluster/crash/reconcile.go index f18bdfb7d..c524af63a 100644 --- a/pkg/operator/ceph/cluster/crash/reconcile.go +++ b/pkg/operator/ceph/cluster/crash/reconcile.go @@ -69,9 +69,11 @@ const ( // ReconcileNode reconciles ReplicaSets type ReconcileNode struct { // client can be used to retrieve objects from the APIServer. - scheme *runtime.Scheme - client client.Client - context *clusterd.Context + scheme *runtime.Scheme + client client.Client + context *clusterd.Context + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Reconcile reconciles a node and ensures that it has a crashcollector deployment @@ -92,7 +94,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, // get the node object node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: request.Name}} - err := r.client.Get(context.TODO(), request.NamespacedName, node) + err := r.client.Get(r.opManagerContext, request.NamespacedName, node) if err != nil { if kerrors.IsNotFound(err) { // if a node is not present, check if there are any crashcollector deployment for that node and delete it. @@ -130,7 +132,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, for namespace, cephPods := range namespaceToPodList { // get dataDirHostPath from the CephCluster cephClusters := &cephv1.CephClusterList{} - err := r.client.List(context.TODO(), cephClusters, client.InNamespace(namespace)) + err := r.client.List(r.opManagerContext, cephClusters, client.InNamespace(namespace)) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "could not get cephcluster in namespaces %q", namespace) } @@ -150,7 +152,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, namespaceListOpts := client.InNamespace(request.Namespace) // Try to fetch the list of existing deployment and remove them - err := r.client.List(context.TODO(), deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName}, namespaceListOpts) + err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName}, namespaceListOpts) if err != nil { logger.Errorf("failed to list crash collector deployments, delete it/them manually. %v", err) return reconcile.Result{}, nil @@ -171,7 +173,7 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, // checking if secret "rook-ceph-crash-collector-keyring" is present which is required to create crashcollector pods secret := &corev1.Secret{} - err = r.client.Get(context.TODO(), types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret) + err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: crashCollectorKeyName, Namespace: namespace}, secret) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("secret %q not found. retrying in %q. %v", crashCollectorKeyName, waitForRequeueIfSecretNotCreated.RequeueAfter.String(), err) @@ -231,7 +233,7 @@ func (r *ReconcileNode) cephPodList() ([]corev1.Pod, error) { for _, app := range cephAppNames { podList := &corev1.PodList{} - err := r.client.List(context.TODO(), podList, client.MatchingLabels{k8sutil.AppAttr: app}) + err := r.client.List(r.opManagerContext, podList, client.MatchingLabels{k8sutil.AppAttr: app}) if err != nil { return cephPods, errors.Wrapf(err, "could not list the %q pods", app) } @@ -245,7 +247,7 @@ func (r *ReconcileNode) cephPodList() ([]corev1.Pod, error) { func (r *ReconcileNode) listCrashCollectorAndDelete(nodeName, ns string) error { deploymentList := &appsv1.DeploymentList{} namespaceListOpts := client.InNamespace(ns) - err := r.client.List(context.TODO(), deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName, NodeNameLabel: nodeName}, namespaceListOpts) + err := r.client.List(r.opManagerContext, deploymentList, client.MatchingLabels{k8sutil.AppAttr: AppName, NodeNameLabel: nodeName}, namespaceListOpts) if err != nil { return errors.Wrap(err, "failed to list crash collector deployments") } @@ -271,7 +273,7 @@ func (r *ReconcileNode) deleteCrashCollector(deployment appsv1.Deployment) error }, } - err := r.client.Delete(context.TODO(), dep) + err := r.client.Delete(r.opManagerContext, dep) if err != nil && !kerrors.IsNotFound(err) { return errors.Wrapf(err, "could not delete crash collector deployment %q", deploymentName) } @@ -298,7 +300,7 @@ func (r *ReconcileNode) reconcileCrashRetention(namespace string, cephCluster ce // minimum k8s version required for v1 cronJob is 'v1.21.0'. Apply v1 if k8s version is at least 'v1.21.0', else apply v1beta1 cronJob. if useCronJobV1 { // delete v1beta1 cronJob if it already exists - err = r.client.Delete(context.TODO(), &v1beta1.CronJob{ObjectMeta: objectMeta}) + err = r.client.Delete(r.opManagerContext, &v1beta1.CronJob{ObjectMeta: objectMeta}) if err != nil && !kerrors.IsNotFound(err) { return errors.Wrapf(err, "failed to delete CronJob v1beta1 %q", prunerName) } @@ -307,7 +309,7 @@ func (r *ReconcileNode) reconcileCrashRetention(namespace string, cephCluster ce cronJob = &v1beta1.CronJob{ObjectMeta: objectMeta} } - err := r.client.Delete(context.TODO(), cronJob) + err := r.client.Delete(r.opManagerContext, cronJob) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cronJob resource not found. Ignoring since object must be deleted.") diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index ae69cf77c..15cac9f4f 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -17,7 +17,6 @@ limitations under the License. package mgr import ( - "context" "fmt" "github.com/rook/rook/pkg/operator/ceph/config" @@ -57,7 +56,6 @@ func (c *Cluster) dashboardPort() int { } func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) { - ctx := context.TODO() user := fmt.Sprintf("mgr.%s", m.DaemonID) access := []string{"mon", "allow profile mgr", "mds", "allow *", "osd", "allow *"} s := keyring.GetSecretStore(c.context, c.clusterInfo, c.clusterInfo.OwnerInfo) @@ -68,7 +66,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) { } // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x - err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(ctx, m.ResourceName, metav1.DeleteOptions{}) + err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, m.ResourceName, metav1.DeleteOptions{}) if err != nil { if errors.IsNotFound(err) { logger.Debugf("legacy mgr key %q is already removed", m.ResourceName) diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 42047133b..f1190f849 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -56,7 +56,6 @@ var ( ) func (c *Cluster) configureDashboardService(activeDaemon string) error { - ctx := context.TODO() dashboardService, err := c.makeDashboardService(AppName, activeDaemon) if err != nil { return err @@ -68,7 +67,7 @@ func (c *Cluster) configureDashboardService(activeDaemon string) error { } } else { // delete the dashboard service if it exists - err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(ctx, dashboardService.Name, metav1.DeleteOptions{}) + err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, dashboardService.Name, metav1.DeleteOptions{}) if err != nil && !kerrors.IsNotFound(err) { return errors.Wrap(err, "failed to delete dashboard service") } @@ -253,8 +252,7 @@ func (c *Cluster) setLoginCredentials(password string) error { } func (c *Cluster) getOrGenerateDashboardPassword() (string, error) { - ctx := context.TODO() - secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(ctx, dashboardPasswordName, metav1.GetOptions{}) + secret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, dashboardPasswordName, metav1.GetOptions{}) if err == nil { logger.Info("the dashboard secret was already generated") return decodeSecret(secret) @@ -286,7 +284,7 @@ func (c *Cluster) getOrGenerateDashboardPassword() (string, error) { return "", errors.Wrapf(err, "failed to set owner reference to dashboard secret %q", secret.Name) } - _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, secret, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to save dashboard secret") } diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go index 9cc2b7c13..c820d3946 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard_test.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go @@ -115,6 +115,7 @@ func TestStartSecureDashboard(t *testing.T) { Namespace: "myns", CephVersion: cephver.Nautilus, OwnerInfo: ownerInfo, + Context: ctx, } c := &Cluster{clusterInfo: clusterInfo, context: &clusterd.Context{Clientset: clientset, Executor: executor}, spec: cephv1.ClusterSpec{ diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 11dba2a51..061e4f349 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -18,7 +18,6 @@ limitations under the License. package mgr import ( - "context" "fmt" "path" "strconv" @@ -112,7 +111,6 @@ func (c *Cluster) getDaemonIDs() []string { // Start begins the process of running a cluster of Ceph mgrs. func (c *Cluster) Start() error { - ctx := context.TODO() // Validate pod's memory if specified err := controller.CheckPodMemory(cephv1.ResourcesKeyMgr, cephv1.GetMgrResources(c.spec.Resources), cephMgrPodMinimumMemory) if err != nil { @@ -124,11 +122,9 @@ func (c *Cluster) Start() error { var deploymentsToWaitFor []*v1.Deployment for _, daemonID := range daemonIDs { - // Check whether we need to cancel the orchestration - if err := controller.CheckForCancelledOrchestration(c.context); err != nil { - return err + if c.clusterInfo.Context.Err() != nil { + return c.clusterInfo.Context.Err() } - resourceName := fmt.Sprintf("%s-%s", AppName, daemonID) mgrConfig := &mgrConfig{ DaemonID: daemonID, @@ -155,7 +151,7 @@ func (c *Cluster) Start() error { return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name) } - newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{}) + newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, d, metav1.CreateOptions{}) if err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName) @@ -233,7 +229,7 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) { // In case the mgr count was reduced, delete the extra mgrs for i := maxMgrCount - 1; i >= len(daemonIDs); i-- { mgrName := fmt.Sprintf("%s-%s", AppName, k8sutil.IndexToName(i)) - err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), mgrName, metav1.DeleteOptions{}) + err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrName, metav1.DeleteOptions{}) if err == nil { logger.Infof("removed extra mgr %q", mgrName) } else if !kerrors.IsNotFound(err) { @@ -246,7 +242,7 @@ func (c *Cluster) removeExtraMgrs(daemonIDs []string) { // in the sidecar func (c *Cluster) ReconcileActiveMgrServices(daemonNameToUpdate string) error { // If the services are already set to this daemon, no need to attempt to update - svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), AppName, metav1.GetOptions{}) + svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, AppName, metav1.GetOptions{}) if err != nil { logger.Errorf("failed to check current mgr service, proceeding to update. %v", err) } else { diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index afc99d3fd..718e19cf8 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -37,7 +37,6 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tevino/abool" apps "k8s.io/api/apps/v1" policyv1 "k8s.io/api/policy/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" @@ -74,13 +73,13 @@ func TestStartMgr(t *testing.T) { defer os.RemoveAll(configDir) ctx := &clusterd.Context{ - Executor: executor, - ConfigDir: configDir, - Clientset: clientset, - RequestCancelOrchestration: abool.New(), - Client: cl} + Executor: executor, + ConfigDir: configDir, + Clientset: clientset, + Client: cl, + } ownerInfo := cephclient.NewMinimumOwnerInfo(t) - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo, CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Build: 5}} + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid", OwnerInfo: ownerInfo, CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Build: 5}, Context: context.TODO()} clusterInfo.SetName("test") clusterSpec := cephv1.ClusterSpec{ Annotations: map[rook.KeyType]rook.Annotations{cephv1.KeyMgr: {"my": "annotation"}}, @@ -207,8 +206,7 @@ func TestMgrSidecarReconcile(t *testing.T) { ConfigDir: configDir, Clientset: clientset, } - ownerInfo := cephclient.NewMinimumOwnerInfo(t) - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", OwnerInfo: ownerInfo} + clusterInfo := cephclient.AdminClusterInfo("mycluster") clusterInfo.SetName("test") c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo} @@ -280,7 +278,7 @@ func TestConfigureModules(t *testing.T) { clientset := testop.New(t, 3) context := &clusterd.Context{Executor: executor, Clientset: clientset} - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo := cephclient.AdminClusterInfo("mycluster") c := &Cluster{ context: context, clusterInfo: clusterInfo, diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go index 66cf8ab56..243a53e4d 100644 --- a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go +++ b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go @@ -16,6 +16,7 @@ limitations under the License. package mgr import ( + "context" "testing" "time" @@ -30,7 +31,6 @@ import ( func TestOrchestratorModules(t *testing.T) { executor := &exectest.MockExecutor{} - context := &clusterd.Context{Executor: executor} rookModuleEnabled := false rookBackendSet := false backendErrorCount := 0 @@ -60,7 +60,9 @@ func TestOrchestratorModules(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ CephVersion: cephver.Nautilus, + Context: context.TODO(), } + context := &clusterd.Context{Executor: executor} c := &Cluster{clusterInfo: clusterInfo, context: context} c.exitCode = func(err error) (int, bool) { diff --git a/pkg/operator/ceph/cluster/mon/config.go b/pkg/operator/ceph/cluster/mon/config.go index 66be39ceb..9cf20cce6 100644 --- a/pkg/operator/ceph/cluster/mon/config.go +++ b/pkg/operator/ceph/cluster/mon/config.go @@ -57,6 +57,10 @@ const ( externalConnectionRetry = 60 * time.Second ) +var ( + ClusterInfoNoClusterNoSecret = errors.New("not expected to create new cluster info and did not find existing secret") +) + func (c *Cluster) genMonSharedKeyring() string { return fmt.Sprintf( keyringTemplate, @@ -77,32 +81,32 @@ func dataDirRelativeHostPath(monName string) string { } // LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID -func LoadClusterInfo(context *clusterd.Context, namespace string) (*cephclient.ClusterInfo, int, *Mapping, error) { - return CreateOrLoadClusterInfo(context, namespace, nil) +func LoadClusterInfo(ctx *clusterd.Context, context context.Context, namespace string) (*cephclient.ClusterInfo, int, *Mapping, error) { + return CreateOrLoadClusterInfo(ctx, context, namespace, nil) } // CreateOrLoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID -func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, int, *Mapping, error) { - ctx := context.TODO() +func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, context context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, int, *Mapping, error) { var clusterInfo *cephclient.ClusterInfo maxMonID := -1 monMapping := &Mapping{ Schedule: map[string]*MonScheduleInfo{}, } - secrets, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, AppName, metav1.GetOptions{}) + secrets, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(context, AppName, metav1.GetOptions{}) if err != nil { if !kerrors.IsNotFound(err) { return nil, maxMonID, monMapping, errors.Wrap(err, "failed to get mon secrets") } if ownerInfo == nil { - return nil, maxMonID, monMapping, errors.New("not expected to create new cluster info and did not find existing secret") + return nil, maxMonID, monMapping, ClusterInfoNoClusterNoSecret } clusterInfo, err = createNamedClusterInfo(clusterdContext, namespace) if err != nil { return nil, maxMonID, monMapping, errors.Wrap(err, "failed to create mon secrets") } + clusterInfo.Context = context err = createClusterAccessSecret(clusterdContext.Clientset, namespace, clusterInfo, ownerInfo) if err != nil { @@ -113,6 +117,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string Namespace: namespace, FSID: string(secrets.Data[fsidSecretNameKey]), MonitorSecret: string(secrets.Data[monSecretNameKey]), + Context: context, } if cephUsername, ok := secrets.Data[cephUsernameKey]; ok { clusterInfo.CephCred.Username = string(cephUsername) @@ -123,7 +128,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string secrets.Data[cephUsernameKey] = []byte(cephclient.AdminUsername) secrets.Data[cephUserSecretKey] = adminSecretKey - if _, err = clusterdContext.Clientset.CoreV1().Secrets(namespace).Update(ctx, secrets, metav1.UpdateOptions{}); err != nil { + if _, err = clusterdContext.Clientset.CoreV1().Secrets(namespace).Update(context, secrets, metav1.UpdateOptions{}); err != nil { return nil, maxMonID, monMapping, errors.Wrap(err, "failed to update mon secrets") } } else { @@ -148,7 +153,7 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, namespace string // If the admin secret is "admin-secret", look for the deprecated secret that has the external creds if clusterInfo.CephCred.Secret == adminSecretNameKey { - secret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, OperatorCreds, metav1.GetOptions{}) + secret, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(context, OperatorCreds, metav1.GetOptions{}) if err != nil { return clusterInfo, maxMonID, monMapping, err } @@ -263,7 +268,6 @@ func loadMonConfig(clientset kubernetes.Interface, namespace string) (map[string } func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, clusterInfo *cephclient.ClusterInfo, ownerInfo *k8sutil.OwnerInfo) error { - ctx := context.TODO() logger.Infof("creating mon secrets for a new cluster") var err error @@ -286,7 +290,7 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string, if err != nil { return errors.Wrapf(err, "failed to set owner reference to mon secret %q", secret.Name) } - if _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err = clientset.CoreV1().Secrets(namespace).Create(clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil { return errors.Wrap(err, "failed to save mon secrets") } @@ -369,9 +373,9 @@ func ExtractKey(contents string) (string, error) { } // PopulateExternalClusterInfo Add validation in the code to fail if the external cluster has no OSDs keep waiting -func PopulateExternalClusterInfo(context *clusterd.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) *cephclient.ClusterInfo { +func PopulateExternalClusterInfo(context *clusterd.Context, ctx context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) *cephclient.ClusterInfo { for { - clusterInfo, _, _, err := LoadClusterInfo(context, namespace) + clusterInfo, _, _, err := LoadClusterInfo(context, ctx, namespace) if err != nil { logger.Warningf("waiting for the csi connection info of the external cluster. retrying in %s.", externalConnectionRetry.String()) logger.Debugf("%v", err) diff --git a/pkg/operator/ceph/cluster/mon/config_test.go b/pkg/operator/ceph/cluster/mon/config_test.go index b31bdc859..03d04b069 100644 --- a/pkg/operator/ceph/cluster/mon/config_test.go +++ b/pkg/operator/ceph/cluster/mon/config_test.go @@ -57,7 +57,7 @@ func TestCreateClusterSecrets(t *testing.T) { } namespace := "ns" ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() - info, maxID, mapping, err := CreateOrLoadClusterInfo(context, namespace, ownerInfo) + info, maxID, mapping, err := CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo) assert.NoError(t, err) assert.Equal(t, -1, maxID) require.NotNil(t, info) @@ -80,7 +80,7 @@ func TestCreateClusterSecrets(t *testing.T) { assert.NoError(t, err) // Check that the cluster info can now be loaded - info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo) + info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo) assert.NoError(t, err) assert.Equal(t, "client.admin", info.CephCred.Username) assert.Equal(t, adminSecret, info.CephCred.Secret) @@ -89,7 +89,7 @@ func TestCreateClusterSecrets(t *testing.T) { secret.Data[adminSecretNameKey] = []byte(adminSecretNameKey) _, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}) assert.NoError(t, err) - _, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo) + _, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo) assert.Error(t, err) // Load the external cluster with the legacy external creds @@ -100,7 +100,7 @@ func TestCreateClusterSecrets(t *testing.T) { } _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) assert.NoError(t, err) - info, _, _, err = CreateOrLoadClusterInfo(context, namespace, ownerInfo) + info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo) assert.NoError(t, err) assert.Equal(t, "testid", info.CephCred.Username) assert.Equal(t, "testkey", info.CephCred.Secret) diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index eaf45094b..4cbb3510e 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -109,14 +109,14 @@ func NewHealthChecker(monCluster *Cluster) *HealthChecker { } // Check periodically checks the health of the monitors -func (hc *HealthChecker) Check(stopCh chan struct{}) { +func (hc *HealthChecker) Check(context context.Context) { for { // Update Mon Timeout with CR details updateMonTimeout(hc.monCluster) // Update Mon Interval with CR details updateMonInterval(hc.monCluster, hc) select { - case <-stopCh: + case <-context.Done(): logger.Infof("stopping monitoring of mons in namespace %q", hc.monCluster.Namespace) return @@ -351,7 +351,6 @@ func (c *Cluster) failMon(monCount, desiredMonCount int, name string) bool { } func (c *Cluster) removeOrphanMonResources() { - ctx := context.TODO() if c.spec.Mon.VolumeClaimTemplate == nil { logger.Debug("skipping check for orphaned mon pvcs since using the host path") return @@ -360,7 +359,7 @@ func (c *Cluster) removeOrphanMonResources() { logger.Info("checking for orphaned mon resources") opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName)} - pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(ctx, opts) + pvcs, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).List(c.ClusterInfo.Context, opts) if err != nil { logger.Infof("failed to check for orphaned mon pvcs. %v", err) return @@ -369,7 +368,7 @@ func (c *Cluster) removeOrphanMonResources() { for _, pvc := range pvcs.Items { logger.Debugf("checking if pvc %q is orphaned", pvc.Name) - _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{}) + _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, pvc.Name, metav1.GetOptions{}) if err == nil { logger.Debugf("skipping pvc removal since the mon daemon %q still requires it", pvc.Name) continue @@ -383,7 +382,7 @@ func (c *Cluster) removeOrphanMonResources() { var gracePeriod int64 // delete immediately propagation := metav1.DeletePropagationForeground options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} - err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(ctx, pvc.Name, *options) + err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, pvc.Name, *options) if err != nil { logger.Warningf("failed to delete orphaned monitor pvc %q. %v", pvc.Name, err) } @@ -391,9 +390,8 @@ func (c *Cluster) removeOrphanMonResources() { } func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error { - ctx := context.TODO() // get the existing deployment - d, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, resourceName(name), metav1.GetOptions{}) + d, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, resourceName(name), metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "failed to get mon %q", name) } @@ -408,7 +406,7 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error { // update the deployment logger.Infof("scaling the mon %q deployment to replica %d", name, desiredReplicas) - _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(ctx, d, metav1.UpdateOptions{}) + _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Update(c.ClusterInfo.Context, d, metav1.UpdateOptions{}) if err != nil { return errors.Wrapf(err, "failed to update mon %q replicas from %d to %d", name, originalReplicas, desiredReplicas) } @@ -493,7 +491,6 @@ func (c *Cluster) failoverMon(name string) error { // make a best effort to remove the mon and all its resources func (c *Cluster) removeMon(daemonName string) error { - ctx := context.TODO() logger.Infof("ensuring removal of unhealthy monitor %s", daemonName) resourceName := resourceName(daemonName) @@ -502,7 +499,7 @@ func (c *Cluster) removeMon(daemonName string) error { var gracePeriod int64 propagation := metav1.DeletePropagationForeground options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} - if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(ctx, resourceName, *options); err != nil { + if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil { if kerrors.IsNotFound(err) { logger.Infof("dead mon %s was already gone", resourceName) } else { @@ -519,7 +516,7 @@ func (c *Cluster) removeMon(daemonName string) error { delete(c.mapping.Schedule, daemonName) // Remove the service endpoint - if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(ctx, resourceName, *options); err != nil { + if err := c.context.Clientset.CoreV1().Services(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, *options); err != nil { if kerrors.IsNotFound(err) { logger.Infof("dead mon service %s was already gone", resourceName) } else { @@ -528,7 +525,7 @@ func (c *Cluster) removeMon(daemonName string) error { } // Remove the PVC backing the mon if it existed - if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(ctx, resourceName, metav1.DeleteOptions{}); err != nil { + if err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Delete(c.ClusterInfo.Context, resourceName, metav1.DeleteOptions{}); err != nil { if kerrors.IsNotFound(err) { logger.Infof("mon pvc did not exist %q", resourceName) } else { @@ -666,7 +663,7 @@ func (c *Cluster) evictMonIfMultipleOnSameNode() error { // Get all the mon pods label := fmt.Sprintf("app=%s", AppName) - pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: label}) + pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: label}) if err != nil { return errors.Wrap(err, "failed to list mon pods") } diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index ac67b4cc9..bbec6a522 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -36,7 +36,6 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tevino/abool" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,10 +60,9 @@ func TestCheckHealth(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - RequestCancelOrchestration: abool.New(), + Clientset: clientset, + ConfigDir: configDir, + Executor: executor, } ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) @@ -94,6 +92,7 @@ func TestCheckHealth(t *testing.T) { return SchedulingResult{Node: node}, nil } + c.ClusterInfo.Context = ctx err = c.checkHealth() assert.Nil(t, err) logger.Infof("mons after checkHealth: %v", c.ClusterInfo.Monitors) @@ -165,7 +164,7 @@ func TestEvictMonOnSameNode(t *testing.T) { return "{\"key\":\"mysecurekey\"}", nil }, } - context := &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Executor: executor, RequestCancelOrchestration: abool.New()} + context := &clusterd.Context{Clientset: clientset, ConfigDir: configDir, Executor: executor} ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) setCommonMonProperties(c, 1, cephv1.MonSpec{Count: 0}, "myversion") @@ -195,6 +194,7 @@ func TestEvictMonOnSameNode(t *testing.T) { assert.Equal(t, 2, c.maxMonID) // Should evict either mon a or mon c since they are on the same node and failover to mon d + c.ClusterInfo.Context = ctx err = c.evictMonIfMultipleOnSameNode() assert.NoError(t, err) _, err = clientset.AppsV1().Deployments(c.Namespace).Get(ctx, "rook-ceph-mon-d", metav1.GetOptions{}) @@ -267,10 +267,9 @@ func TestCheckHealthNotFound(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - RequestCancelOrchestration: abool.New(), + Clientset: clientset, + ConfigDir: configDir, + Executor: executor, } ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) @@ -330,10 +329,9 @@ func TestAddRemoveMons(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: configDir, - Executor: executor, - RequestCancelOrchestration: abool.New(), + Clientset: clientset, + ConfigDir: configDir, + Executor: executor, } ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New(context, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index c6c6566b7..52ff53ae9 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -20,7 +20,6 @@ limitations under the License. package mon import ( - "context" "encoding/json" "fmt" "reflect" @@ -181,13 +180,15 @@ func New(context *clusterd.Context, namespace string, spec cephv1.ClusterSpec, o // Start begins the process of running a cluster of Ceph mons. func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookVersion string, cephVersion cephver.CephVersion, spec cephv1.ClusterSpec) (*cephclient.ClusterInfo, error) { - // Only one goroutine can orchestrate the mons at a time c.acquireOrchestrationLock() defer c.releaseOrchestrationLock() clusterInfo.OwnerInfo = c.ownerInfo c.ClusterInfo = clusterInfo + if c.ClusterInfo.Context == nil { + panic("nil context") + } c.rookVersion = rookVersion c.spec = spec @@ -199,7 +200,7 @@ func (c *Cluster) Start(clusterInfo *cephclient.ClusterInfo, rookVersion string, // Validate pod's memory if specified err := controller.CheckPodMemory(cephv1.ResourcesKeyMon, cephv1.GetMonResources(c.spec.Resources), cephMonPodMinimumMemory) if err != nil { - return nil, errors.Wrap(err, "error checking pod memory") + return nil, errors.Wrap(err, "failed to check pod memory") } logger.Infof("start running mons") @@ -246,11 +247,9 @@ func (c *Cluster) startMons(targetCount int) error { if existingCount < len(mons) { // Start the new mons one at a time for i := existingCount; i < targetCount; i++ { - // Check whether we need to cancel the orchestration - if err := controller.CheckForCancelledOrchestration(c.context); err != nil { - return err + if c.ClusterInfo.Context.Err() != nil { + return c.ClusterInfo.Context.Err() } - if err := c.ensureMonsRunning(mons, i, targetCount, true); err != nil { return err } @@ -471,14 +470,16 @@ func (c *Cluster) ensureMonsRunning(mons []*monConfig, i, targetCount int, requi func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion) error { var err error + context := c.ClusterInfo.Context // get the cluster info from secret - c.ClusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, c.Namespace, c.ownerInfo) + c.ClusterInfo, c.maxMonID, c.mapping, err = CreateOrLoadClusterInfo(c.context, context, c.Namespace, c.ownerInfo) if err != nil { return errors.Wrap(err, "failed to get cluster info") } c.ClusterInfo.CephVersion = cephVersion c.ClusterInfo.OwnerInfo = c.ownerInfo + c.ClusterInfo.Context = context // save cluster monitor config if err = c.saveMonConfig(); err != nil { @@ -594,7 +595,6 @@ func resourceName(name string) string { // scheduleMonitor selects a node for a monitor deployment. // see startMon() and design/ceph/ceph-mon-pv.md for additional details. func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { - ctx := context.TODO() // build the canary deployment. d, err := c.makeDeployment(mon, true) if err != nil { @@ -631,17 +631,17 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { // to the real monitor deployment. pvc, err := c.makeDeploymentPVC(mon, true) if err != nil { - return nil, errors.Wrapf(err, "sched-mon: failed to make monitor %s pvc", d.Name) + return nil, errors.Wrapf(err, "failed to make monitor %s pvc", d.Name) } - _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(ctx, pvc, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{}) if err == nil { - logger.Infof("sched-mon: created canary monitor %s pvc %s", d.Name, pvc.Name) + logger.Infof("created canary monitor %s pvc %s", d.Name, pvc.Name) } else { if kerrors.IsAlreadyExists(err) { - logger.Debugf("sched-mon: creating mon %s pvc %s: already exists.", d.Name, pvc.Name) + logger.Debugf("creating mon %s pvc %s: already exists.", d.Name, pvc.Name) } else { - return nil, errors.Wrapf(err, "sched-mon: error creating mon %s pvc %s", d.Name, pvc.Name) + return nil, errors.Wrapf(err, "failed to create mon %s pvc %s", d.Name, pvc.Name) } } @@ -654,25 +654,28 @@ func scheduleMonitor(c *Cluster, mon *monConfig) (*apps.Deployment, error) { // already exists it may have been scheduled with a different crd config. createdDeployment := false for i := 0; i < canaryRetries; i++ { - _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(ctx, d, metav1.CreateOptions{}) + if c.ClusterInfo.Context.Err() != nil { + return nil, c.ClusterInfo.Context.Err() + } + _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{}) if err == nil { createdDeployment = true - logger.Infof("sched-mon: created canary deployment %s", d.Name) + logger.Infof("created canary deployment %s", d.Name) break } else if kerrors.IsAlreadyExists(err) { if err := k8sutil.DeleteDeployment(c.context.Clientset, c.Namespace, d.Name); err != nil { - return nil, errors.Wrapf(err, "sched-mon: error deleting canary deployment %s", d.Name) + return nil, errors.Wrapf(err, "failed to delete canary deployment %s", d.Name) } - logger.Infof("sched-mon: deleted existing canary deployment %s", d.Name) + logger.Infof("deleted existing canary deployment %s", d.Name) time.Sleep(time.Second * canaryRetryDelaySeconds) } else { - return nil, errors.Wrapf(err, "sched-mon: error creating canary monitor deployment %s", d.Name) + return nil, errors.Wrapf(err, "failed to create canary monitor deployment %s", d.Name) } } // failed after retrying if !createdDeployment { - return nil, errors.Errorf("sched-mon: failed to create canary deployment %s", d.Name) + return nil, errors.Errorf("failed to create canary deployment %s", d.Name) } // caller should arrange for the deployment to be removed @@ -696,17 +699,14 @@ func (c *Cluster) getMonPlacement(zone string) cephv1.Placement { } func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingResult, error) { - ctx := context.TODO() // target node decision, and deployment/pvc to cleanup result := SchedulingResult{} // wait for the scheduler to make a placement decision for i := 0; i < canaryRetries; i++ { - // Check whether we need to cancel the orchestration - if err := controller.CheckForCancelledOrchestration(c.context); err != nil { - return result, err + if c.ClusterInfo.Context.Err() != nil { + return result, c.ClusterInfo.Context.Err() } - if i > 0 { time.Sleep(time.Second * canaryRetryDelaySeconds) } @@ -715,42 +715,45 @@ func realWaitForMonitorScheduling(c *Cluster, d *apps.Deployment) (SchedulingRes LabelSelector: labels.Set(d.Spec.Selector.MatchLabels).String(), } - pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(ctx, listOptions) + pods, err := c.context.Clientset.CoreV1().Pods(c.Namespace).List(c.ClusterInfo.Context, listOptions) if err != nil { - return result, errors.Wrapf(err, "sched-mon: error listing canary pods %s", d.Name) + return result, errors.Wrapf(err, "failed to list canary pods %s", d.Name) } if len(pods.Items) == 0 { - logger.Infof("sched-mon: waiting for canary pod creation %s", d.Name) + logger.Infof("waiting for canary pod creation %s", d.Name) continue } pod := pods.Items[0] if pod.Spec.NodeName == "" { - logger.Debugf("sched-mon: monitor %s canary pod %s not yet scheduled", d.Name, pod.Name) + logger.Debugf("monitor %s canary pod %s not yet scheduled", d.Name, pod.Name) continue } - node, err := c.context.Clientset.CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, metav1.GetOptions{}) + node, err := c.context.Clientset.CoreV1().Nodes().Get(c.ClusterInfo.Context, pod.Spec.NodeName, metav1.GetOptions{}) if err != nil { - return result, errors.Wrapf(err, "sched-mon: error getting node %s", pod.Spec.NodeName) + return result, errors.Wrapf(err, "failed to get node %s", pod.Spec.NodeName) } result.Node = node - logger.Infof("sched-mon: canary monitor deployment %s scheduled to %s", d.Name, node.Name) + logger.Infof("canary monitor deployment %s scheduled to %s", d.Name, node.Name) return result, nil } - return result, errors.New("sched-mon: canary pod scheduling failed retries") + return result, errors.New("failed to schedule canary pod(s)") } func (c *Cluster) initMonIPs(mons []*monConfig) error { for _, m := range mons { + if c.ClusterInfo.Context.Err() != nil { + return c.ClusterInfo.Context.Err() + } if c.spec.Network.IsHost() { logger.Infof("setting mon endpoints for hostnetwork mode") node, ok := c.mapping.Schedule[m.DaemonName] if !ok || node == nil { - return errors.Errorf("node for mon %q doesn't exist in assignment map", m.DaemonName) + return errors.Errorf("failed to found node for mon %q in assignment map", m.DaemonName) } m.PublicIP = node.Address } else { @@ -769,7 +772,6 @@ func (c *Cluster) initMonIPs(mons []*monConfig) error { // Delete mon canary deployments (and associated PVCs) using deployment labels // to select this kind of temporary deployments func (c *Cluster) removeCanaryDeployments() { - ctx := context.TODO() canaryDeployments, err := k8sutil.GetDeployments(c.context.Clientset, c.Namespace, "app=rook-ceph-mon,mon_canary=true") if err != nil { logger.Warningf("failed to get the list of monitor canary deployments. %v", err) @@ -782,7 +784,7 @@ func (c *Cluster) removeCanaryDeployments() { var gracePeriod int64 propagation := metav1.DeletePropagationForeground options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} - if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(ctx, canary.Name, *options); err != nil { + if err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Delete(c.ClusterInfo.Context, canary.Name, *options); err != nil { logger.Warningf("failed to delete canary monitor deployment %q. %v", canary.Name, err) } } @@ -805,14 +807,12 @@ func (c *Cluster) assignMons(mons []*monConfig) error { // enforced using a node selector, or (2) configuration permits k8s to handle // scheduling for the monitor. for _, mon := range mons { - // Check whether we need to cancel the orchestration - if err := controller.CheckForCancelledOrchestration(c.context); err != nil { - return err + if c.ClusterInfo.Context.Err() != nil { + return c.ClusterInfo.Context.Err() } - // scheduling for this monitor has already been completed if _, ok := c.mapping.Schedule[mon.DaemonName]; ok { - logger.Debugf("assignmon: mon %s already scheduled", mon.DaemonName) + logger.Debugf("mon %s already scheduled", mon.DaemonName) continue } @@ -822,7 +822,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { // before a decision is stored in the node mapping. deployment, err := scheduleMonitor(c, mon) if err != nil { - return errors.Wrap(err, "assignmon: error scheduling monitor") + return errors.Wrap(err, "failed to schedule monitor") } // start waiting for the deployment @@ -841,7 +841,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { nodeChoice := result.Node if nodeChoice == nil { - logger.Errorf("assignmon: could not schedule monitor %q", mon.DaemonName) + logger.Errorf("failed to schedule monitor %q", mon.DaemonName) failedMonSchedule = true return } @@ -851,15 +851,15 @@ func (c *Cluster) assignMons(mons []*monConfig) error { // directly to a node selector on the monitor pod. var schedule *MonScheduleInfo if c.spec.Network.IsHost() || c.monVolumeClaimTemplate(mon) == nil { - logger.Infof("assignmon: mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name) + logger.Infof("mon %s assigned to node %s", mon.DaemonName, nodeChoice.Name) schedule, err = getNodeInfoFromNode(*nodeChoice) if err != nil { - logger.Errorf("assignmon: couldn't get node info for node %q. %v", nodeChoice.Name, err) + logger.Errorf("failed to get node info for node %q. %v", nodeChoice.Name, err) failedMonSchedule = true return } } else { - logger.Infof("assignmon: mon %q placement using native scheduler", mon.DaemonName) + logger.Infof("mon %q placement using native scheduler", mon.DaemonName) } if c.spec.IsStretchCluster() { @@ -882,7 +882,7 @@ func (c *Cluster) assignMons(mons []*monConfig) error { return errors.New("failed to schedule mons") } - logger.Debug("assignmons: mons have been scheduled") + logger.Debug("mons have been scheduled") return nil } @@ -906,7 +906,6 @@ func (c *Cluster) monVolumeClaimTemplate(mon *monConfig) *v1.PersistentVolumeCla } func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) error { - ctx := context.TODO() if len(mons) == 0 { return errors.New("cannot start 0 mons") } @@ -916,7 +915,7 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e // 1) New clusters where we are starting one deployment at a time. We only need to check for quorum once when we add a new mon. // 2) Clusters being restored where no mon deployments are running. We need to start all the deployments before checking quorum. onlyCheckQuorumOnce := false - deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}) + deployments, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).List(c.ClusterInfo.Context, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}) if err != nil { if kerrors.IsNotFound(err) { logger.Infof("0 of %d expected mon deployments exist. creating new deployment(s).", len(mons)) @@ -1029,7 +1028,6 @@ func (c *Cluster) saveMonConfig() error { } func (c *Cluster) persistExpectedMonDaemons() error { - ctx := context.TODO() configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: EndpointConfigMapName, @@ -1068,13 +1066,13 @@ func (c *Cluster) persistExpectedMonDaemons() error { csi.ConfigKey: csiConfigValue, } - if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil { + if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, configMap, metav1.CreateOptions{}); err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrap(err, "failed to create mon endpoint config map") } logger.Debugf("updating config map %s that already exists", configMap.Name) - if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(ctx, configMap, metav1.UpdateOptions{}); err != nil { + if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil { return errors.Wrap(err, "failed to update mon endpoint config map") } } @@ -1083,7 +1081,7 @@ func (c *Cluster) persistExpectedMonDaemons() error { } func (c *Cluster) getStoredMaxMonID() (string, error) { - configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{}) + configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{}) if err != nil && !kerrors.IsNotFound(err) { return "", errors.Wrap(err, "could not load maxMonId") } @@ -1106,7 +1104,7 @@ func (c *Cluster) commitMaxMonID(monName string) error { return errors.Wrapf(err, "invalid mon name %q", monName) } - configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{}) + configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to find existing mon endpoint config map") } @@ -1125,7 +1123,7 @@ func (c *Cluster) commitMaxMonID(monName string) error { logger.Infof("updating maxMonID from %d to %d after committing mon %q", existingMax, committedMonID, monName) configmap.Data[MaxMonIDKey] = strconv.Itoa(committedMonID) - if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(context.TODO(), configmap, metav1.UpdateOptions{}); err != nil { + if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil { return errors.Wrap(err, "failed to update mon endpoint config map for the maxMonID") } return nil @@ -1141,7 +1139,7 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error { return errors.Wrapf(err, "failed to make mon %q pvc", d.Name) } - existingPvc, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Get(context.TODO(), m.ResourceName, metav1.GetOptions{}) + existingPvc, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "failed to fetch pvc for mon %q", m.ResourceName) } @@ -1190,7 +1188,6 @@ func (c *Cluster) updateMon(m *monConfig, d *apps.Deployment) error { // - if PVC -> remove node selector, if present // func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { - ctx := context.TODO() // check if the monitor deployment already exists. if the deployment does // exist, also determine if it using pvc storage. pvcExists := false @@ -1207,7 +1204,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name) } - existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, d.Name, metav1.GetOptions{}) + existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, d.Name, metav1.GetOptions{}) if err == nil { deploymentExists = true pvcExists = controller.DaemonVolumesContainsPVC(existingDeployment.Spec.Template.Spec.Volumes) @@ -1262,7 +1259,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { if err != nil { return errors.Wrapf(err, "failed to make mon %s pvc", d.Name) } - _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(ctx, pvc, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().PersistentVolumeClaims(c.Namespace).Create(c.ClusterInfo.Context, pvc, metav1.CreateOptions{}) if err != nil { if kerrors.IsAlreadyExists(err) { logger.Debugf("cannot create mon %s pvc %s: already exists.", d.Name, pvc.Name) @@ -1286,7 +1283,7 @@ func (c *Cluster) startMon(m *monConfig, schedule *MonScheduleInfo) error { map[string]string{k8sutil.AppAttr: AppName}, nodeSelector) logger.Debugf("Starting mon: %+v", d.Name) - _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(ctx, d, metav1.CreateOptions{}) + _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(c.ClusterInfo.Context, d, metav1.CreateOptions{}) if err != nil { return errors.Wrapf(err, "failed to create mon deployment %s", d.Name) } diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 08bd62a6c..a84ac61d7 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -29,6 +29,7 @@ import ( "testing" "time" + "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -40,7 +41,6 @@ import ( "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -94,10 +94,9 @@ func newTestStartClusterWithQuorumResponse(t *testing.T, namespace string, monRe }, } return &clusterd.Context{ - Clientset: clientset, - Executor: executor, - ConfigDir: configDir, - RequestCancelOrchestration: abool.New(), + Clientset: clientset, + Executor: executor, + ConfigDir: configDir, }, nil } @@ -140,7 +139,6 @@ func TestResourceName(t *testing.T) { } func TestStartMonDeployment(t *testing.T) { - ctx := context.TODO() namespace := "ns" context, err := newTestStartCluster(t, namespace) assert.NoError(t, err) @@ -151,7 +149,7 @@ func TestStartMonDeployment(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: EndpointConfigMapName}, Data: map[string]string{"maxMonId": "1"}, } - _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(ctx, cm, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(c.ClusterInfo.Context, cm, metav1.CreateOptions{}) assert.NoError(t, err) // Start mon a on a specific node since there is no volumeClaimTemplate @@ -159,7 +157,7 @@ func TestStartMonDeployment(t *testing.T) { schedule := &MonScheduleInfo{Hostname: "host-a", Zone: "zonea"} err = c.startMon(m, schedule) assert.NoError(t, err) - deployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, m.ResourceName, metav1.GetOptions{}) + deployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{}) assert.NoError(t, err) assert.Equal(t, schedule.Hostname, deployment.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"]) @@ -169,14 +167,15 @@ func TestStartMonDeployment(t *testing.T) { c.spec.Mon.VolumeClaimTemplate = &v1.PersistentVolumeClaim{} err = c.startMon(m, schedule) assert.NoError(t, err) - deployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, m.ResourceName, metav1.GetOptions{}) + deployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, m.ResourceName, metav1.GetOptions{}) assert.NoError(t, err) // no node selector when there is a volumeClaimTemplate and the mon is assigned to a zone assert.Equal(t, 0, len(deployment.Spec.Template.Spec.NodeSelector)) } func TestStartMonPods(t *testing.T) { - ctx := context.TODO() + capnslog.SetGlobalLogLevel(capnslog.DEBUG) + os.Setenv("ROOK_LOG_LEVEL", "DEBUG") namespace := "ns" context, err := newTestStartCluster(t, namespace) assert.NoError(t, err) @@ -187,17 +186,16 @@ func TestStartMonPods(t *testing.T) { _, err = c.Start(c.ClusterInfo, c.rookVersion, cephver.Nautilus, c.spec) assert.NoError(t, err) - validateStart(ctx, t, c) + validateStart(t, c) - // starting again should be a no-op, but still results in an error + // starting again should be a no-op _, err = c.Start(c.ClusterInfo, c.rookVersion, cephver.Nautilus, c.spec) assert.NoError(t, err) - validateStart(ctx, t, c) + validateStart(t, c) } func TestOperatorRestart(t *testing.T) { - ctx := context.TODO() namespace := "ns" context, err := newTestStartCluster(t, namespace) assert.NoError(t, err) @@ -209,7 +207,7 @@ func TestOperatorRestart(t *testing.T) { assert.NoError(t, err) assert.True(t, info.IsInitialized(true)) - validateStart(ctx, t, c) + validateStart(t, c) c = newCluster(context, namespace, true, v1.ResourceRequirements{}) c.ClusterInfo = clienttest.CreateTestClusterInfo(1) @@ -219,12 +217,11 @@ func TestOperatorRestart(t *testing.T) { assert.NoError(t, err) assert.True(t, info.IsInitialized(true)) - validateStart(ctx, t, c) + validateStart(t, c) } // safety check that if hostNetwork is used no changes occur on an operator restart func TestOperatorRestartHostNetwork(t *testing.T) { - ctx := context.TODO() namespace := "ns" context, err := newTestStartCluster(t, namespace) assert.NoError(t, err) @@ -238,7 +235,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) { assert.NoError(t, err) assert.True(t, info.IsInitialized(true)) - validateStart(ctx, t, c) + validateStart(t, c) // cluster with host networking c = newCluster(context, namespace, false, v1.ResourceRequirements{}) @@ -250,16 +247,16 @@ func TestOperatorRestartHostNetwork(t *testing.T) { assert.NoError(t, err) assert.True(t, info.IsInitialized(true), info) - validateStart(ctx, t, c) + validateStart(t, c) } -func validateStart(ctx context.Context, t *testing.T, c *Cluster) { - s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(ctx, AppName, metav1.GetOptions{}) +func validateStart(t *testing.T, c *Cluster) { + s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(c.ClusterInfo.Context, AppName, metav1.GetOptions{}) assert.NoError(t, err) // there shouldn't be an error due the secret existing assert.Equal(t, 4, len(s.Data)) // there is only one pod created. the other two won't be created since the first one doesn't start - _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(ctx, "rook-ceph-mon-a", metav1.GetOptions{}) + _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(c.ClusterInfo.Context, "rook-ceph-mon-a", metav1.GetOptions{}) assert.NoError(t, err) } @@ -343,6 +340,7 @@ func TestMaxMonID(t *testing.T) { defer os.RemoveAll(configDir) ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", cephv1.ClusterSpec{}, ownerInfo, &sync.Mutex{}) + c.ClusterInfo = clienttest.CreateTestClusterInfo(1) // when the configmap is not found, the maxMonID is -1 maxMonID, err := c.getStoredMaxMonID() @@ -437,7 +435,7 @@ func TestWaitForQuorum(t *testing.T) { assert.NoError(t, err) requireAllInQuorum := false expectedMons := []string{"a"} - clusterInfo := &cephclient.ClusterInfo{Namespace: namespace} + clusterInfo := cephclient.AdminClusterInfo("mycluster") err = waitForQuorumWithMons(context, clusterInfo, expectedMons, 0, requireAllInQuorum) assert.NoError(t, err) } diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index acfde8ff0..ea6f3e8a2 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -17,7 +17,6 @@ limitations under the License. package mon import ( - "context" "strconv" "github.com/pkg/errors" @@ -29,7 +28,6 @@ import ( ) func (c *Cluster) createService(mon *monConfig) (string, error) { - ctx := context.TODO() svcDef := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: mon.ResourceName, @@ -62,7 +60,7 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { // For example, in disaster recovery the service might have been deleted accidentally, but we have the // expected endpoint from the mon configmap. if mon.PublicIP != "" { - _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(ctx, svcDef.Name, metav1.GetOptions{}) + _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get(c.ClusterInfo.Context, svcDef.Name, metav1.GetOptions{}) if err != nil && kerrors.IsNotFound(err) { logger.Infof("ensuring the clusterIP for mon %q is %q", mon.DaemonName, mon.PublicIP) svcDef.Spec.ClusterIP = mon.PublicIP diff --git a/pkg/operator/ceph/cluster/mon/service_test.go b/pkg/operator/ceph/cluster/mon/service_test.go index 95f525a8d..4b4ebf861 100644 --- a/pkg/operator/ceph/cluster/mon/service_test.go +++ b/pkg/operator/ceph/cluster/mon/service_test.go @@ -23,6 +23,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" @@ -33,6 +34,7 @@ func TestCreateService(t *testing.T) { ctx := context.TODO() clientset := test.New(t, 1) c := New(&clusterd.Context{Clientset: clientset}, "ns", cephv1.ClusterSpec{}, &k8sutil.OwnerInfo{}, &sync.Mutex{}) + c.ClusterInfo = client.AdminClusterInfo("rook-ceph") m := &monConfig{ResourceName: "rook-ceph-mon-b", DaemonName: "b"} clusterIP, err := c.createService(m) assert.NoError(t, err) diff --git a/pkg/operator/ceph/cluster/monitoring.go b/pkg/operator/ceph/cluster/monitoring.go index 9748b5566..803ba47fa 100644 --- a/pkg/operator/ceph/cluster/monitoring.go +++ b/pkg/operator/ceph/cluster/monitoring.go @@ -18,11 +18,12 @@ limitations under the License. package cluster import ( + "context" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" - "github.com/rook/rook/pkg/operator/ceph/object/bucket" ) var ( @@ -30,37 +31,27 @@ var ( ) func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInfo *cephclient.ClusterInfo) { - var isDisabled bool - + var isEnabled bool for _, daemon := range monitorDaemonList { // Is the monitoring enabled for that daemon? - isDisabled = isMonitoringDisabled(daemon, cluster.Spec) - if health, ok := cluster.monitoringChannels[daemon]; ok { - if health.monitoringRunning { - // If the goroutine was running but the CR was updated to stop the monitoring we need to close the channel - if isDisabled { - // close the channel so the goroutine can stop - close(cluster.monitoringChannels[daemon].stopChan) - // Set monitoring to false since it's not running anymore - cluster.monitoringChannels[daemon].monitoringRunning = false - } else { - logger.Debugf("ceph %s health go routine is already running for cluster %q", daemon, cluster.Namespace) - } - } else { - // if not already running and not disabled, we run it - if !isDisabled { - // Run the go routine - c.startMonitoringCheck(cluster, clusterInfo, daemon) - - // Set the flag to indicate monitoring is running - cluster.monitoringChannels[daemon].monitoringRunning = true + isEnabled = isMonitoringEnabled(daemon, cluster.Spec) + if health, ok := cluster.monitoringRoutines[daemon]; ok { + // If the context Err() is nil this means it hasn't been cancelled yet + if health.internalCtx.Err() == nil { + logger.Debugf("monitoring routine for %q is already running", daemon) + if !isEnabled { + cluster.monitoringRoutines[daemon].internalCancel() } } } else { - if !isDisabled { - cluster.monitoringChannels[daemon] = &clusterHealth{ - stopChan: make(chan struct{}), - monitoringRunning: true, // Set the flag to indicate monitoring is running + if isEnabled { + // Instantiate the monitoring goroutine context from the parent context + // They can individually be cancelled and will be cancelled when the parent context is cancelled + internalCtx, internalCancel := context.WithCancel(c.OpManagerCtx) + + cluster.monitoringRoutines[daemon] = &clusterHealth{ + internalCtx: internalCtx, + internalCancel: internalCancel, } // Run the go routine @@ -68,41 +59,18 @@ func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInf } } } - - // Start watchers - if cluster.watchersActivated { - logger.Debugf("cluster is already being watched by bucket and client provisioner for cluster %q", cluster.Namespace) - return - } - - // Start the object bucket provisioner - bucketProvisioner := bucket.NewProvisioner(c.context, clusterInfo) - // If cluster is external, pass down the user to the bucket controller - - // note: the error return below is ignored and is expected to be removed from the - // bucket library's `NewProvisioner` function - bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner) - go func() { - err := bucketController.Run(cluster.stopCh) - if err != nil { - logger.Errorf("failed to run bucket controller. %v", err) - } - }() - - // enable the cluster watcher once - cluster.watchersActivated = true } -func isMonitoringDisabled(daemon string, clusterSpec *cephv1.ClusterSpec) bool { +func isMonitoringEnabled(daemon string, clusterSpec *cephv1.ClusterSpec) bool { switch daemon { case "mon": - return clusterSpec.HealthCheck.DaemonHealth.Monitor.Disabled + return !clusterSpec.HealthCheck.DaemonHealth.Monitor.Disabled case "osd": - return clusterSpec.HealthCheck.DaemonHealth.ObjectStorageDaemon.Disabled + return !clusterSpec.HealthCheck.DaemonHealth.ObjectStorageDaemon.Disabled case "status": - return clusterSpec.HealthCheck.DaemonHealth.Status.Disabled + return !clusterSpec.HealthCheck.DaemonHealth.Status.Disabled } return false @@ -113,18 +81,18 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo * case "mon": healthChecker := mon.NewHealthChecker(cluster.mons) logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) - go healthChecker.Check(cluster.monitoringChannels[daemon].stopChan) + go healthChecker.Check(cluster.monitoringRoutines[daemon].internalCtx) case "osd": if !cluster.Spec.External.Enable { c.osdChecker = osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck) logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) - go c.osdChecker.Start(cluster.monitoringChannels[daemon].stopChan) + go c.osdChecker.Start(cluster.monitoringRoutines[daemon].internalCtx) } case "status": cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec) logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) - go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan) + go cephChecker.checkCephStatus(cluster.monitoringRoutines[daemon].internalCtx) } } diff --git a/pkg/operator/ceph/cluster/monitoring_test.go b/pkg/operator/ceph/cluster/monitoring_test.go index a0eed4ff5..d46bf9cf9 100644 --- a/pkg/operator/ceph/cluster/monitoring_test.go +++ b/pkg/operator/ceph/cluster/monitoring_test.go @@ -23,7 +23,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" ) -func TestIsMonitoringDisabled(t *testing.T) { +func TestIsMonitoringEnabled(t *testing.T) { type args struct { daemon string clusterSpec *cephv1.ClusterSpec @@ -33,12 +33,12 @@ func TestIsMonitoringDisabled(t *testing.T) { args args want bool }{ - {"isDisabled", args{"mon", &cephv1.ClusterSpec{}}, false}, - {"isEnabled", args{"mon", &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Monitor: cephv1.HealthCheckSpec{Disabled: true}}}}}, true}, + {"isEnabled", args{"mon", &cephv1.ClusterSpec{}}, true}, + {"isDisabled", args{"mon", &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Monitor: cephv1.HealthCheckSpec{Disabled: true}}}}}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := isMonitoringDisabled(tt.args.daemon, tt.args.clusterSpec); got != tt.want { + if got := isMonitoringEnabled(tt.args.daemon, tt.args.clusterSpec); got != tt.want { t.Errorf("isMonitoringEnabled() = %v, want %v", got, tt.want) } }) diff --git a/pkg/operator/ceph/cluster/operator_watchers.go b/pkg/operator/ceph/cluster/operator_watchers.go deleted file mode 100644 index f76765d45..000000000 --- a/pkg/operator/ceph/cluster/operator_watchers.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2020 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package cluster to manage a Ceph cluster. -package cluster - -import ( - "os" - "reflect" - - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" - "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/core/v1" -) - -// StartOperatorSettingsWatch starts the operator settings watcher -// TODO: use controller runtime, so we can use a Context to cancel the watch instead of using another channel -// The cache package mentions that it'd be nice to use a Context for cancellation too -func (c *ClusterController) StartOperatorSettingsWatch(stopCh chan struct{}) { - operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) - // watch for "rook-ceph-operator-config" ConfigMap - k8sutil.StartOperatorSettingsWatch(c.context, operatorNamespace, opcontroller.OperatorSettingConfigMapName, - c.operatorConfigChange, - func(oldObj, newObj interface{}) { - if reflect.DeepEqual(oldObj, newObj) { - return - } - c.operatorConfigChange(newObj) - }, nil, stopCh) -} - -// StopWatch stop watchers -func (c *ClusterController) StopWatch() { - for _, cluster := range c.clusterMap { - // check channel is open before closing - if !cluster.closedStopCh { - close(cluster.stopCh) - cluster.closedStopCh = true - } - } - c.clusterMap = make(map[string]*cluster) -} - -func (c *ClusterController) operatorConfigChange(obj interface{}) { - cm, ok := obj.(*v1.ConfigMap) - if !ok { - logger.Warningf("Expected ConfigMap but handler received %T. %#v", obj, obj) - return - } - - logger.Infof("ConfigMap %q changes detected. Updating configurations", cm.Name) - for _, callback := range c.operatorConfigCallbacks { - if err := callback(); err != nil { - logger.Errorf("%v", err) - } - } -} diff --git a/pkg/operator/ceph/cluster/osd/create.go b/pkg/operator/ceph/cluster/osd/create.go index eaba45506..6eb799f99 100644 --- a/pkg/operator/ceph/cluster/osd/create.go +++ b/pkg/operator/ceph/cluster/osd/create.go @@ -153,11 +153,9 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi awaitingStatusConfigMaps := sets.NewString() for _, volume := range c.deviceSets { - // Check whether we need to cancel the orchestration - if err := opcontroller.CheckForCancelledOrchestration(c.context); err != nil { - return awaitingStatusConfigMaps, err + if c.clusterInfo.Context.Err() != nil { + return awaitingStatusConfigMaps, c.clusterInfo.Context.Err() } - dataSource, dataOK := volume.PVCSources[bluestorePVCData] // The data PVC template is required. @@ -310,11 +308,9 @@ func (c *Cluster) startProvisioningOverNodes(config *provisionConfig, errs *prov awaitingStatusConfigMaps := sets.NewString() for _, node := range c.ValidStorage.Nodes { - // Check whether we need to cancel the orchestration - if err := opcontroller.CheckForCancelledOrchestration(c.context); err != nil { - return awaitingStatusConfigMaps, err + if c.clusterInfo.Context.Err() != nil { + return awaitingStatusConfigMaps, c.clusterInfo.Context.Err() } - // fully resolve the storage config and resources for this node // don't care about osd device class resources since it will be overwritten later for prepareosd resources n := c.resolveNode(node.Name, "") diff --git a/pkg/operator/ceph/cluster/osd/create_test.go b/pkg/operator/ceph/cluster/osd/create_test.go index 5272da8ea..ca6235967 100644 --- a/pkg/operator/ceph/cluster/osd/create_test.go +++ b/pkg/operator/ceph/cluster/osd/create_test.go @@ -28,7 +28,6 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" apiresource "k8s.io/apimachinery/pkg/api/resource" @@ -307,7 +306,6 @@ func Test_startProvisioningOverPVCs(t *testing.T) { clientset := test.NewComplexClientset(t) // fake clientset with generate name functionality - requestCancelOrchestration := *abool.New() // clusterd.Context is created in doSetup() clusterInfo := &cephclient.ClusterInfo{ @@ -316,6 +314,7 @@ func Test_startProvisioningOverPVCs(t *testing.T) { } clusterInfo.SetName("mycluster") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + clusterInfo.Context = context.TODO() spec := cephv1.ClusterSpec{} fakeK8sVersion := "v1.13.0" @@ -329,8 +328,7 @@ func Test_startProvisioningOverPVCs(t *testing.T) { test.SetFakeKubernetesVersion(clientset, fakeK8sVersion) // PVCs require k8s version v1.13+ errs = newProvisionErrors() ctx := &clusterd.Context{ - Clientset: clientset, - RequestCancelOrchestration: &requestCancelOrchestration, + Clientset: clientset, } c = New(ctx, clusterInfo, spec, "rook/rook:master") config = c.newProvisionConfig() @@ -425,18 +423,6 @@ func Test_startProvisioningOverPVCs(t *testing.T) { fakeK8sVersion = "v1.13.0" }) - t.Run("request cancel orchestration", func(t *testing.T) { - // spec = - clientset = test.NewComplexClientset(t) // reset to empty fake k8s environment - requestCancelOrchestration.Set() - doSetup() - awaitingStatusConfigMaps, err = c.startProvisioningOverPVCs(config, errs) - assert.Error(t, err) - assert.Zero(t, errs.len()) - assert.Zero(t, awaitingStatusConfigMaps.Len()) - requestCancelOrchestration.UnSet() - }) - t.Run("error if no volume claim template", func(t *testing.T) { spec = cephv1.ClusterSpec{ Storage: cephv1.StorageScopeSpec{ @@ -468,7 +454,6 @@ func Test_startProvisioningOverNodes(t *testing.T) { dataDirHostPath := "/var/lib/mycluster" clientset := test.New(t, 3) // fake clientset with 3 nodes - requestCancelOrchestration := *abool.New() // clusterd.Context is created in doSetup() // names of status configmaps for nodes @@ -483,6 +468,7 @@ func Test_startProvisioningOverNodes(t *testing.T) { } clusterInfo.SetName("mycluster") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + clusterInfo.Context = context.TODO() var useAllDevices bool spec := cephv1.ClusterSpec{} @@ -496,8 +482,7 @@ func Test_startProvisioningOverNodes(t *testing.T) { doSetup := func() { errs = newProvisionErrors() ctx := &clusterd.Context{ - Clientset: clientset, - RequestCancelOrchestration: &requestCancelOrchestration, + Clientset: clientset, } c = New(ctx, clusterInfo, spec, "rook/rook:master") config = c.newProvisionConfig() @@ -599,16 +584,6 @@ func Test_startProvisioningOverNodes(t *testing.T) { ) }) - t.Run("request cancel orchestration", func(t *testing.T) { - requestCancelOrchestration.Set() - doSetup() - prepareJobsRun, err = c.startProvisioningOverNodes(config, errs) - assert.Error(t, err) - assert.Zero(t, errs.len()) - assert.Zero(t, prepareJobsRun.Len()) - requestCancelOrchestration.UnSet() - }) - t.Run("use no nodes", func(t *testing.T) { spec = cephv1.ClusterSpec{ Storage: cephv1.StorageScopeSpec{ diff --git a/pkg/operator/ceph/cluster/osd/deviceSet.go b/pkg/operator/ceph/cluster/osd/deviceSet.go index 3081ecf1a..1e79712fe 100644 --- a/pkg/operator/ceph/cluster/osd/deviceSet.go +++ b/pkg/operator/ceph/cluster/osd/deviceSet.go @@ -189,7 +189,6 @@ func (c *Cluster) createDeviceSetPVCsForIndex(newDeviceSet cephv1.StorageClassDe } func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolumeClaim, deviceSetName string, pvcTemplate v1.PersistentVolumeClaim, setIndex int) (*v1.PersistentVolumeClaim, error) { - ctx := context.TODO() // old labels and PVC ID for backward compatibility pvcID := legacyDeviceSetPVCID(deviceSetName, setIndex) @@ -215,7 +214,7 @@ func (c *Cluster) createDeviceSetPVC(existingPVCs map[string]*v1.PersistentVolum } // No PVC found, creating a new one - deployedPVC, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.clusterInfo.Namespace).Create(ctx, pvc, metav1.CreateOptions{}) + deployedPVC, err := c.context.Clientset.CoreV1().PersistentVolumeClaims(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, pvc, metav1.CreateOptions{}) if err != nil { return nil, errors.Wrapf(err, "failed to create PVC %q for device set %q", pvc.Name, deviceSetName) } diff --git a/pkg/operator/ceph/cluster/osd/health.go b/pkg/operator/ceph/cluster/osd/health.go index f1cf93df0..6bff42185 100644 --- a/pkg/operator/ceph/cluster/osd/health.go +++ b/pkg/operator/ceph/cluster/osd/health.go @@ -69,7 +69,7 @@ func NewOSDHealthMonitor(context *clusterd.Context, clusterInfo *client.ClusterI } // Start runs monitoring logic for osds status at set intervals -func (m *OSDHealthMonitor) Start(stopCh chan struct{}) { +func (m *OSDHealthMonitor) Start(context context.Context) { for { select { @@ -77,8 +77,8 @@ func (m *OSDHealthMonitor) Start(stopCh chan struct{}) { logger.Debug("checking osd processes status.") m.checkOSDHealth() - case <-stopCh: - logger.Infof("Stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace) + case <-context.Done(): + logger.Infof("stopping monitoring of OSDs in namespace %q", m.clusterInfo.Namespace) return } } @@ -192,7 +192,7 @@ func (m *OSDHealthMonitor) updateCephStatus(devices []string) { for _, device := range devices { cephClusterStorage.DeviceClasses = append(cephClusterStorage.DeviceClasses, cephv1.DeviceClasses{Name: device}) } - err := m.context.Client.Get(context.TODO(), m.clusterInfo.NamespacedName(), cephCluster) + err := m.context.Client.Get(m.clusterInfo.Context, m.clusterInfo.NamespacedName(), cephCluster) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") diff --git a/pkg/operator/ceph/cluster/osd/health_test.go b/pkg/operator/ceph/cluster/osd/health_test.go index cb3f184ec..ab68ce69d 100644 --- a/pkg/operator/ceph/cluster/osd/health_test.go +++ b/pkg/operator/ceph/cluster/osd/health_test.go @@ -101,11 +101,11 @@ func TestOSDHealthCheck(t *testing.T) { } func TestMonitorStart(t *testing.T) { - stopCh := make(chan struct{}) + context, cancel := context.WithCancel(context.TODO()) osdMon := NewOSDHealthMonitor(&clusterd.Context{}, client.AdminClusterInfo("ns"), true, cephv1.CephClusterHealthCheckSpec{}) logger.Infof("starting osd monitor") - go osdMon.Start(stopCh) - close(stopCh) + go osdMon.Start(context) + cancel() } func TestNewOSDHealthMonitor(t *testing.T) { diff --git a/pkg/operator/ceph/cluster/osd/integration_test.go b/pkg/operator/ceph/cluster/osd/integration_test.go index bf0fd34ea..345522366 100644 --- a/pkg/operator/ceph/cluster/osd/integration_test.go +++ b/pkg/operator/ceph/cluster/osd/integration_test.go @@ -18,6 +18,7 @@ package osd import ( "context" + contextStandard "context" "encoding/json" "fmt" "os" @@ -37,7 +38,6 @@ import ( "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -96,7 +96,8 @@ func TestOSDIntegration(t *testing.T) { // This is the actual test. If it hangs, we should consider that an error. func testOSDIntegration(t *testing.T) { - ctx := context.TODO() + ctx := contextStandard.TODO() + contextCancel, cancel := contextStandard.WithCancel(ctx) namespace := "osd-integration" clusterName := "my-cluster" @@ -191,13 +192,13 @@ func testOSDIntegration(t *testing.T) { clusterInfo.CephVersion = cephver.Pacific clusterInfo.SetName("mycluster") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) + clusterInfo.Context = ctx executor := osdIntegrationTestExecutor(t, clientset, namespace) context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: "/var/lib/rook", - Executor: executor, - RequestCancelOrchestration: abool.New(), + Clientset: clientset, + ConfigDir: "/var/lib/rook", + Executor: executor, } spec := cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{ @@ -225,9 +226,10 @@ func testOSDIntegration(t *testing.T) { var startErr error var done bool - runReconcile := func() { + runReconcile := func(ctx contextStandard.Context) { // reset environment c = New(context, clusterInfo, spec, "myversion") + clusterInfo.Context = ctx statusMapWatcher.Reset() // reset counters @@ -249,7 +251,7 @@ func testOSDIntegration(t *testing.T) { // NOTE: these tests all use the same environment t.Run("initial creation", func(t *testing.T) { - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 12) // 3 nodes + 9 new PVCs for _, cm := range cms { @@ -265,7 +267,7 @@ func testOSDIntegration(t *testing.T) { }) t.Run("reconcile again with no changes", func(t *testing.T) { - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { @@ -286,7 +288,7 @@ func testOSDIntegration(t *testing.T) { spec.Storage.StorageClassDeviceSets[1].Count = 6 // 3 more (1 more per node) osdsPerNode = 3 // vda, vdb, vdc - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 8) // 3 nodes + 5 new PVCs for _, cm := range cms { @@ -306,7 +308,7 @@ func testOSDIntegration(t *testing.T) { spec.Storage.StorageClassDeviceSets[0].Count = 10 // 2 more portable osdsPerNode = 4 // vd[a-d] - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 5) // 3 nodes + 2 new PVCs i := 1 for _, cm := range cms { @@ -323,7 +325,9 @@ func testOSDIntegration(t *testing.T) { time.Sleep(10 * time.Millisecond) // after the second status map is made ready, cancel the orchestration. wait a short // while to make sure the watcher picks up the updated change - c.context.RequestCancelOrchestration.Set() + cancel() + // refresh the context so the rest of the test can continue + contextCancel = contextStandard.TODO() break } i++ @@ -337,7 +341,7 @@ func testOSDIntegration(t *testing.T) { numUpdates := len(deploymentsUpdated) t.Logf("deployments updated: %d", numUpdates) - go runReconcile() + go runReconcile(contextCancel) cms = waitForNumConfigMaps(clientset, namespace, 5) // 3 nodes + 2 new PVCs for _, cm := range cms { cpy := cm.DeepCopy() @@ -354,7 +358,7 @@ func testOSDIntegration(t *testing.T) { spec.Storage.Selection.DeviceFilter = "/dev/vd[abcde]" // 3 more (1 more per node) osdsPerNode = 5 // vd[a-e] - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -372,7 +376,7 @@ func testOSDIntegration(t *testing.T) { assert.Len(t, deploymentsUpdated, 28) // should get back to healthy after - go runReconcile() + go runReconcile(contextCancel) cms = waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -387,7 +391,7 @@ func testOSDIntegration(t *testing.T) { t.Run("failures during deployment updates", func(t *testing.T) { failUpdatingDeployments = []string{"osd-15", "osd-22"} - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -401,7 +405,7 @@ func testOSDIntegration(t *testing.T) { assert.Len(t, deploymentsUpdated, 31) // should attempt to update all deployments failUpdatingDeployments = []string{} - go runReconcile() + go runReconcile(contextCancel) cms = waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -419,7 +423,7 @@ func testOSDIntegration(t *testing.T) { osdsPerNode = 6 // vd[a-f] failCreatingDeployments = []string{"osd-31", "osd-33"} - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -433,7 +437,7 @@ func testOSDIntegration(t *testing.T) { assert.Len(t, deploymentsUpdated, 31) failCreatingDeployments = []string{} - go runReconcile() + go runReconcile(contextCancel) cms = waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -455,7 +459,7 @@ func testOSDIntegration(t *testing.T) { } spec.Storage.StorageClassDeviceSets = append(spec.Storage.StorageClassDeviceSets, newSCDS) - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 3) // 3 nodes for _, cm := range cms { cpy := cm.DeepCopy() @@ -473,7 +477,7 @@ func testOSDIntegration(t *testing.T) { newDummyPVC("metadata", namespace, "10Gi", "uncle-rogers-secret-stuff"), } - go runReconcile() + go runReconcile(contextCancel) cms = waitForNumConfigMaps(clientset, namespace, 6) // 3 nodes + 3 new PVCs for _, cm := range cms { cpy := cm.DeepCopy() @@ -497,7 +501,7 @@ func testOSDIntegration(t *testing.T) { _, err := clientset.CoreV1().ConfigMaps(namespace).Create(ctx, danglingCM, metav1.CreateOptions{}) assert.NoError(t, err) - go runReconcile() + go runReconcile(contextCancel) cms := waitForNumConfigMaps(clientset, namespace, 4) // 3 nodes + dangling for _, cm := range cms { cpy := cm.DeepCopy() diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 50cc160e3..deea8c0bb 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -240,10 +240,9 @@ func (c *Cluster) Start() error { } func (c *Cluster) getExistingOSDDeploymentsOnPVCs() (sets.String, error) { - ctx := context.TODO() listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s", k8sutil.AppAttr, AppName, OSDOverPVCLabelKey)} - deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(ctx, listOpts) + deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) if err != nil { return nil, errors.Wrap(err, "failed to query existing OSD deployments") } @@ -406,12 +405,11 @@ func (c *Cluster) getOSDPropsForPVC(pvcName, osdDeviceClass string) (osdProperti // First look for the node selector that was previously used for the OSD, or if a new OSD // check for the assignment of the OSD prepare job. func (c *Cluster) getPVCHostName(pvcName string) (string, error) { - ctx := context.TODO() listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", OSDOverPVCLabelKey, pvcName)} // Check for the existence of the OSD deployment where the node selector was applied // in a previous reconcile. - deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(ctx, listOpts) + deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) if err != nil { return "", errors.Wrapf(err, "failed to get deployment for osd with pvc %q", pvcName) } @@ -426,7 +424,7 @@ func (c *Cluster) getPVCHostName(pvcName string) (string, error) { // Since the deployment wasn't found it must be a new deployment so look at the node // assignment of the OSD prepare pod - pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(ctx, listOpts) + pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) if err != nil { return "", errors.Wrapf(err, "failed to get pod for osd with pvc %q", pvcName) } diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index 9967ae4fd..83acd33c6 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -35,7 +35,6 @@ import ( "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -69,6 +68,7 @@ func TestStart(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ Namespace: "ns", CephVersion: cephver.Nautilus, + Context: context.TODO(), } context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{} @@ -151,6 +151,7 @@ func TestAddRemoveNode(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ Namespace: namespace, CephVersion: cephver.Nautilus, + Context: ctx, } clusterInfo.SetName("rook-ceph-test") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) @@ -162,11 +163,10 @@ func TestAddRemoveNode(t *testing.T) { } context := &clusterd.Context{ - Clientset: clientset, - ConfigDir: "/var/lib/rook", - Executor: executor, - RequestCancelOrchestration: abool.New(), - RookClientset: fakeclient.NewSimpleClientset(), + Clientset: clientset, + ConfigDir: "/var/lib/rook", + Executor: executor, + RookClientset: fakeclient.NewSimpleClientset(), } spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -324,10 +324,11 @@ func TestAddNodeFailure(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ Namespace: "ns-add-remove", CephVersion: cephver.Nautilus, + Context: context.TODO(), } clusterInfo.SetName("testcluster") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) - context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()} + context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, Storage: cephv1.StorageScopeSpec{ diff --git a/pkg/operator/ceph/cluster/osd/status.go b/pkg/operator/ceph/cluster/osd/status.go index e1fc4ea88..e314c9f82 100644 --- a/pkg/operator/ceph/cluster/osd/status.go +++ b/pkg/operator/ceph/cluster/osd/status.go @@ -18,14 +18,12 @@ limitations under the License. package osd import ( - "context" "encoding/json" "fmt" "time" "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/ceph/config" - "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -196,13 +194,12 @@ func (c *Cluster) updateAndCreateOSDsLoop( errs *provisionErrors, // add errors here ) (shouldRestart bool, err error) { cmClient := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace) - ctx := context.TODO() selector := statusConfigMapSelector() listOptions := metav1.ListOptions{ LabelSelector: selector, } - configMapList, err := cmClient.List(ctx, listOptions) + configMapList, err := cmClient.List(c.clusterInfo.Context, listOptions) if err != nil { return false, errors.Wrapf(err, "failed to list OSD provisioning status ConfigMaps") } @@ -218,7 +215,7 @@ func (c *Cluster) updateAndCreateOSDsLoop( Watch: true, ResourceVersion: configMapList.ResourceVersion, } - watcher, err := cmClient.Watch(ctx, watchOptions) + watcher, err := cmClient.Watch(c.clusterInfo.Context, watchOptions) defer watcher.Stop() if err != nil { return false, errors.Wrapf(err, "failed to start watching OSD provisioning status ConfigMaps") @@ -269,9 +266,8 @@ func (c *Cluster) updateAndCreateOSDsLoop( updateConfig.updateExistingOSDs(errs) case <-minuteTicker.C: - // Check whether we need to cancel the orchestration - if err := controller.CheckForCancelledOrchestration(c.context); err != nil { - return false, err + if c.clusterInfo.Context.Err() != nil { + return false, c.clusterInfo.Context.Err() } // Log progress c, cExp := createConfig.progress() @@ -350,19 +346,18 @@ func (c *Cluster) deleteStatusConfigMap(nodeOrPVCName string) { } func (c *Cluster) deleteAllStatusConfigMaps() { - ctx := context.TODO() listOpts := metav1.ListOptions{ LabelSelector: statusConfigMapSelector(), } cmClientset := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace) - cms, err := cmClientset.List(ctx, listOpts) + cms, err := cmClientset.List(c.clusterInfo.Context, listOpts) if err != nil { logger.Warningf("failed to clean up any dangling OSD prepare status configmaps. failed to list OSD prepare status configmaps. %v", err) return } for _, cm := range cms.Items { logger.Debugf("cleaning up dangling OSD prepare status configmap %q", cm.Name) - err := cmClientset.Delete(ctx, cm.Name, metav1.DeleteOptions{}) + err := cmClientset.Delete(c.clusterInfo.Context, cm.Name, metav1.DeleteOptions{}) if err != nil { logger.Warningf("failed to clean up dangling OSD prepare status configmap %q. %v", cm.Name, err) } diff --git a/pkg/operator/ceph/cluster/osd/update.go b/pkg/operator/ceph/cluster/osd/update.go index e5dcfdbdf..00c63b8c2 100644 --- a/pkg/operator/ceph/cluster/osd/update.go +++ b/pkg/operator/ceph/cluster/osd/update.go @@ -17,7 +17,6 @@ limitations under the License. package osd import ( - "context" "fmt" "strconv" "strings" @@ -73,8 +72,6 @@ func (c *updateConfig) doneUpdating() bool { } func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { - ctx := context.TODO() - if c.doneUpdating() { return // no more OSDs to update } @@ -117,7 +114,7 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { } depName := deploymentName(osdID) - dep, err := c.cluster.context.Clientset.AppsV1().Deployments(c.cluster.clusterInfo.Namespace).Get(ctx, depName, metav1.GetOptions{}) + dep, err := c.cluster.context.Clientset.AppsV1().Deployments(c.cluster.clusterInfo.Namespace).Get(c.cluster.clusterInfo.Context, depName, metav1.GetOptions{}) if err != nil { errs.addError("failed to update OSD %d. failed to find existing deployment %q. %v", osdID, depName, err) continue @@ -194,7 +191,6 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { // getOSDUpdateInfo returns an update queue of OSDs which need updated and an existence list of OSD // Deployments which already exist. func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existenceList, error) { - ctx := context.TODO() namespace := c.clusterInfo.Namespace selector := fmt.Sprintf("%s=%s", k8sutil.AppAttr, AppName) @@ -202,7 +198,7 @@ func (c *Cluster) getOSDUpdateInfo(errs *provisionErrors) (*updateQueue, *existe // list only rook-ceph-osd Deployments LabelSelector: selector, } - deps, err := c.context.Clientset.AppsV1().Deployments(namespace).List(ctx, listOpts) + deps, err := c.context.Clientset.AppsV1().Deployments(namespace).List(c.clusterInfo.Context, listOpts) if err != nil { return nil, nil, errors.Wrap(err, "failed to query existing OSD deployments to see if they need updated") } @@ -338,12 +334,11 @@ func (e *existenceList) Exists(osdID int) bool { // return a function that will list only OSD deployments with the IDs given func (c *Cluster) getFuncToListDeploymentsWithIDs(osdIDs []string) func() (*appsv1.DeploymentList, error) { - ctx := context.TODO() selector := fmt.Sprintf("ceph-osd-id in (%s)", strings.Join(osdIDs, ", ")) listOpts := metav1.ListOptions{ LabelSelector: selector, // e.g. 'ceph-osd-id in (1, 3, 5, 7, 9)' } return func() (*appsv1.DeploymentList, error) { - return c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(ctx, listOpts) + return c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, listOpts) } } diff --git a/pkg/operator/ceph/cluster/osd/update_test.go b/pkg/operator/ceph/cluster/osd/update_test.go index f077e15cf..9f25dc63b 100644 --- a/pkg/operator/ceph/cluster/osd/update_test.go +++ b/pkg/operator/ceph/cluster/osd/update_test.go @@ -101,6 +101,7 @@ func Test_updateExistingOSDs(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{ Namespace: namespace, CephVersion: cephver.Pacific, + Context: context.TODO(), } clusterInfo.SetName("mycluster") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) diff --git a/pkg/operator/ceph/cluster/predicate.go b/pkg/operator/ceph/cluster/predicate.go index 5a764135b..88010f020 100644 --- a/pkg/operator/ceph/cluster/predicate.go +++ b/pkg/operator/ceph/cluster/predicate.go @@ -105,7 +105,7 @@ func isHotPlugCM(obj runtime.Object) bool { return false } -func watchControllerPredicate(rookContext *clusterd.Context) predicate.Funcs { +func watchControllerPredicate() predicate.Funcs { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { logger.Debug("create event from a CR") @@ -131,18 +131,25 @@ func watchControllerPredicate(rookContext *clusterd.Context) predicate.Funcs { } diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer) if diff != "" { - // Set the cancellation flag to stop any ongoing orchestration - rookContext.RequestCancelOrchestration.Set() - logger.Infof("CR has changed for %q. diff=%s", objNew.Name, diff) - return true + + if objNew.Spec.CleanupPolicy.HasDataDirCleanPolicy() { + logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set. not reloading the manager", objNew.GetName(), objNew.GetNamespace()) + return false + } + + // Stop any ongoing orchestration + controller.ReloadManager() + + return false } else if !objOld.GetDeletionTimestamp().Equal(objNew.GetDeletionTimestamp()) { - // Set the cancellation flag to stop any ongoing orchestration - rookContext.RequestCancelOrchestration.Set() - logger.Infof("CR %q is going be deleted, cancelling any ongoing orchestration", objNew.Name) - return true + + // Stop any ongoing orchestration + controller.ReloadManager() + + return false } else if objOld.GetGeneration() != objNew.GetGeneration() { logger.Debugf("skipping resource %q update with unchanged spec", objNew.Name) diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index 03a8296ae..8aa302464 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -17,7 +17,6 @@ limitations under the License. package rbd import ( - "context" "fmt" "github.com/pkg/errors" @@ -51,7 +50,6 @@ type daemonConfig struct { } func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo, daemonConfig *daemonConfig) (string, error) { - ctx := context.TODO() user := fullDaemonName(daemonConfig.DaemonID) access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"} s := keyring.GetSecretStore(r.context, clusterInfo, daemonConfig.ownerInfo) @@ -62,7 +60,7 @@ func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo } // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x - err = r.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Delete(ctx, daemonConfig.ResourceName, metav1.DeleteOptions{}) + err = r.context.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Delete(r.opManagerContext, daemonConfig.ResourceName, metav1.DeleteOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("legacy rbd-mirror key %q is already removed", daemonConfig.ResourceName) @@ -80,14 +78,13 @@ func fullDaemonName(daemonID string) string { } func (r *ReconcileCephRBDMirror) reconcileAddBoostrapPeer(cephRBDMirror *cephv1.CephRBDMirror, namespacedName types.NamespacedName) (reconcile.Result, error) { - ctx := context.TODO() // List all the peers secret, we can have more than one peer we might want to configure // For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring logger.Warning("(DEPRECATED) use of peer secret names in CephRBDMirror is deprecated. Please use CephBlockPool CR to configure peer secret names and import peers.") for _, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames { logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret) - s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(ctx, peerSecret, metav1.GetOptions{}) + s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{}) // We don't care about IsNotFound here, we still need to fail if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret) diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 964e9e770..f57974e33 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -28,7 +28,6 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - opconfig "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" @@ -67,14 +66,18 @@ var controllerTypeMeta = metav1.TypeMeta{ APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } +var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion + // ReconcileCephRBDMirror reconciles a cephRBDMirror object type ReconcileCephRBDMirror struct { - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - client client.Client - scheme *runtime.Scheme - cephClusterSpec *cephv1.ClusterSpec - peers map[string]*peerSpec + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + cephClusterSpec *cephv1.ClusterSpec + peers map[string]*peerSpec + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // peerSpec represents peer details @@ -86,17 +89,19 @@ type peerSpec struct { // Add creates a new cephRBDMirror Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileCephRBDMirror{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, - peers: make(map[string]*peerSpec), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + peers: make(map[string]*peerSpec), + opConfig: opConfig, + opManagerContext: opManagerContext, } } @@ -125,25 +130,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } } - // Build Handler function to return the list of ceph rbd-mirror - // This is used by the watchers below - handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephRBDMirrorList{}, mgr.GetScheme()) - if err != nil { - return err - } - - // Watch for CephCluster Spec changes that we want to propagate to us - err = c.Watch(&source.Kind{Type: &cephv1.CephCluster{ - TypeMeta: metav1.TypeMeta{ - Kind: opcontroller.ClusterResource.Kind, - APIVersion: opcontroller.ClusterResource.APIVersion, - }, - }, - }, handler.EnqueueRequestsFromMapFunc(handlerFunc), opcontroller.WatchCephClusterPredicate()) - if err != nil { - return err - } - return nil } @@ -155,7 +141,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.FailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.FailedStatus) logger.Errorf("failed to reconcile %v", err) } @@ -165,7 +151,7 @@ func (r *ReconcileCephRBDMirror) Reconcile(context context.Context, request reco func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the cephRBDMirror instance cephRBDMirror := &cephv1.CephRBDMirror{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephRBDMirror) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephRBDMirror) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephRBDMirror resource not found. Ignoring since object must be deleted.") @@ -177,7 +163,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile // The CR was just created, initializing status fields if cephRBDMirror.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) } // validate the pool settings @@ -195,22 +181,38 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile // Populate clusterInfo // Always populate it during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to populate cluster info") } - // Populate CephVersion - daemon := string(opconfig.MonType) - currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon) + // Detect desired CephCluster version + runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion( + r.opConfig.Image, + cephRBDMirror.Namespace, + controllerName, + k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme), + r.context, + r.cephClusterSpec, + r.clusterInfo, + ) if err != nil { if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + return reconcile.Result{}, nil } - return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon) + return reconcile.Result{}, errors.Wrap(err, "failed to detect running and desired ceph version") } - r.clusterInfo.CephVersion = currentCephVersion + + // If the version of the Ceph monitor differs from the CephCluster CR image version we assume + // the cluster is being upgraded. So the controller will just wait for the upgrade to finish and + // then versions should match. Obviously using the cmd reporter job adds up to the deployment time + if !reflect.DeepEqual(runningCephVersion, desiredCephVersion) { + // Upgrade is in progress, let's wait for the mons to be done + return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, + opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion) + } + r.clusterInfo.CephVersion = *runningCephVersion // Add bootstrap peer if any logger.Debug("reconciling ceph rbd mirror peers addition") @@ -227,7 +229,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue logger.Debug("done reconciling ceph rbd mirror") @@ -254,9 +256,9 @@ func (r *ReconcileCephRBDMirror) reconcileCreateCephRBDMirror(cephRBDMirror *cep } // updateStatus updates an object with a given status -func updateStatus(client client.Client, name types.NamespacedName, status string) { +func (r *ReconcileCephRBDMirror) updateStatus(client client.Client, name types.NamespacedName, status string) { rbdMirror := &cephv1.CephRBDMirror{} - err := client.Get(context.TODO(), name, rbdMirror) + err := client.Get(r.opManagerContext, name, rbdMirror) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephRBDMirror resource not found. Ignoring since object must be deleted.") diff --git a/pkg/operator/ceph/cluster/rbd/controller_test.go b/pkg/operator/ceph/cluster/rbd/controller_test.go index 1675da907..289835d3f 100644 --- a/pkg/operator/ceph/cluster/rbd/controller_test.go +++ b/pkg/operator/ceph/cluster/rbd/controller_test.go @@ -27,6 +27,8 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -52,18 +54,18 @@ const ( func TestCephRBDMirrorController(t *testing.T) { ctx := context.TODO() var ( - name = "my-mirror" - namespace = "rook-ceph" + name = "my-mirror" + namespace = "rook-ceph" + peerSecretName = "peer-secret" ) // Set DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - // - // TEST 1 SETUP - // - // FAILURE because no CephCluster - // + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Pacific, &version.Pacific, nil + } + // An rbd-mirror resource with metadata and spec. rbdMirror := &cephv1.CephRBDMirror{ ObjectMeta: metav1.ObjectMeta{ @@ -81,6 +83,30 @@ func TestCephRBDMirrorController(t *testing.T) { rbdMirror, } + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Status: cephv1.ClusterStatus{ + Phase: "", + CephStatus: &cephv1.CephStatus{ + Health: "", + }, + }, + } + + s := scheme.Scheme + executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { if args[0] == "status" { @@ -108,123 +134,101 @@ func TestCephRBDMirrorController(t *testing.T) { Clientset: clientset, } - // Register operator types with the runtime scheme. - s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{}) - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + t.Run("error - no ceph cluster", func(t *testing.T) { + // Register operator types with the runtime scheme. + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) - // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileCephRBDMirror object with the scheme and fake client. - r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c} + // Create a ReconcileCephRBDMirror object with the scheme and fake client. + r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx} + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Mock request to simulate Reconcile() being called on an event for a - // watched resource . - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: name, - Namespace: namespace, - }, - } - logger.Info("STARTING PHASE 1") - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 1 DONE") + t.Run("error - ceph cluster not ready", func(t *testing.T) { + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileCephRBDMirror object with the scheme and fake client. + r := &ReconcileCephRBDMirror{client: cl, scheme: s, context: c, opManagerContext: ctx} + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // - cephCluster := &cephv1.CephCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Namespace: namespace, - }, - Status: cephv1.ClusterStatus{ - Phase: "", - CephStatus: &cephv1.CephStatus{ - Health: "", + t.Run("error - wrong peers configuration", func(t *testing.T) { + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, }, - }, - } - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileCephRBDMirror object with the scheme and fake client. - r = &ReconcileCephRBDMirror{client: cl, scheme: s, context: c} - logger.Info("STARTING PHASE 2") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 2 DONE") + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) - // - // TEST 3: - // - // SUCCESS! The CephCluster is ready - // + // Add ready status to the CephCluster + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Add ready status to the CephCluster - cephCluster.Status.Phase = k8sutil.ReadyStatus - cephCluster.Status.CephStatus.Health = "HEALTH_OK" + // Create a ReconcileCephRBDMirror object with the scheme and fake client. + r := &ReconcileCephRBDMirror{ + client: cl, + scheme: s, + context: c, + peers: make(map[string]*peerSpec), + opManagerContext: ctx, + } - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + rbdMirror.Spec.Peers.SecretNames = []string{peerSecretName} + err = r.client.Update(context.TODO(), rbdMirror) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.True(t, res.Requeue) - // Create a ReconcileCephRBDMirror object with the scheme and fake client. - r = &ReconcileCephRBDMirror{ - client: cl, - scheme: s, - context: c, - peers: make(map[string]*peerSpec), - } + }) - logger.Info("STARTING PHASE 4") - - peerSecretName := "peer-secret" - rbdMirror.Spec.Peers.SecretNames = []string{peerSecretName} - err = r.client.Update(context.TODO(), rbdMirror) - assert.NoError(t, err) - res, err = r.Reconcile(ctx, req) - assert.Error(t, err) - assert.True(t, res.Requeue) - - logger.Info("STARTING PHASE 5") - bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token - peerSecret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: peerSecretName, - Namespace: namespace, - }, - Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")}, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{}) - assert.NoError(t, err) - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, rbdMirror) - assert.NoError(t, err) - assert.Equal(t, "Ready", rbdMirror.Status.Phase, rbdMirror) + t.Run("success - peers are configured", func(t *testing.T) { + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + r := &ReconcileCephRBDMirror{ + client: cl, + scheme: s, + context: c, + peers: make(map[string]*peerSpec), + opManagerContext: ctx, + } + bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token + peerSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: peerSecretName, + Namespace: namespace, + }, + Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")}, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, rbdMirror) + assert.NoError(t, err) + assert.Equal(t, "Ready", rbdMirror.Status.Phase, rbdMirror) + }) } diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index af3e01268..cb2fc7c3f 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -18,7 +18,6 @@ limitations under the License. package rbd import ( - "context" "fmt" "github.com/banzaicloud/k8s-objectmatcher/patch" @@ -44,7 +43,6 @@ var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start begins the process of running rbd mirroring daemons. func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) error { - ctx := context.TODO() // Validate pod's memory if specified err := controller.CheckPodMemory(cephv1.ResourcesKeyRBDMirror, cephRBDMirror.Spec.Resources, cephRbdMirrorPodMinimumMemory) if err != nil { @@ -86,7 +84,7 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name) } - if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(ctx, d, metav1.CreateOptions{}); err != nil { + if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(r.opManagerContext, d, metav1.CreateOptions{}); err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create %q deployment", resourceName) } @@ -95,11 +93,11 @@ func (r *ReconcileCephRBDMirror) start(cephRBDMirror *cephv1.CephRBDMirror) erro if err := updateDeploymentAndWait(r.context, r.clusterInfo, d, config.RbdMirrorType, daemonConf.DaemonID, r.cephClusterSpec.SkipUpgradeChecks, false); err != nil { // fail could be an issue updating label selector (immutable), so try del and recreate logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %q. Attempting del-and-recreate. %v", resourceName, err) - err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(ctx, cephRBDMirror.Name, metav1.DeleteOptions{}) + err = r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Delete(r.opManagerContext, cephRBDMirror.Name, metav1.DeleteOptions{}) if err != nil { return errors.Wrapf(err, "failed to delete rbd-mirror %q during del-and-recreate update attempt", resourceName) } - if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(ctx, d, metav1.CreateOptions{}); err != nil { + if _, err := r.context.Clientset.AppsV1().Deployments(cephRBDMirror.Namespace).Create(r.opManagerContext, d, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to recreate rbd-mirror deployment %q during del-and-recreate update attempt", resourceName) } } diff --git a/pkg/operator/ceph/cluster/register_controllers.go b/pkg/operator/ceph/cluster/register_controllers.go deleted file mode 100644 index 93e1d4da6..000000000 --- a/pkg/operator/ceph/cluster/register_controllers.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2019 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cluster - -import ( - "github.com/pkg/errors" - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/ceph/client" - "github.com/rook/rook/pkg/operator/ceph/cluster/crash" - "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" - "github.com/rook/rook/pkg/operator/ceph/disruption/clusterdisruption" - "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" - "github.com/rook/rook/pkg/operator/ceph/disruption/machinedisruption" - "github.com/rook/rook/pkg/operator/ceph/disruption/machinelabel" - "github.com/rook/rook/pkg/operator/ceph/file" - "github.com/rook/rook/pkg/operator/ceph/file/mirror" - "github.com/rook/rook/pkg/operator/ceph/nfs" - "github.com/rook/rook/pkg/operator/ceph/object" - "github.com/rook/rook/pkg/operator/ceph/object/realm" - objectuser "github.com/rook/rook/pkg/operator/ceph/object/user" - "github.com/rook/rook/pkg/operator/ceph/object/zone" - "github.com/rook/rook/pkg/operator/ceph/object/zonegroup" - "github.com/rook/rook/pkg/operator/ceph/pool" - - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -var ( - // EnableMachineDisruptionBudget checks whether machine disruption budget is enabled - EnableMachineDisruptionBudget bool -) - -// AddToManagerFuncsMaintenance is a list of functions to add all Controllers to the Manager (entrypoint for controller) -var AddToManagerFuncsMaintenance = []func(manager.Manager, *controllerconfig.Context) error{ - clusterdisruption.Add, -} - -// MachineDisruptionBudgetAddToManagerFuncs is a list of fencing related functions to add all Controllers to the Manager (entrypoint for controller) -var MachineDisruptionBudgetAddToManagerFuncs = []func(manager.Manager, *controllerconfig.Context) error{ - machinelabel.Add, - machinedisruption.Add, -} - -// AddToManagerFuncs is a list of functions to add all Controllers to the Manager (entrypoint for controller) -var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context) error{ - crash.Add, - pool.Add, - objectuser.Add, - realm.Add, - zonegroup.Add, - zone.Add, - object.Add, - file.Add, - nfs.Add, - rbd.Add, - client.Add, - mirror.Add, -} - -// AddToManager adds all the registered controllers to the passed manager. -// each controller package will have an Add method listed in AddToManagerFuncs -// which will setup all the necessary watch -func AddToManager(m manager.Manager, c *controllerconfig.Context, clusterController *ClusterController) error { - if c == nil { - return errors.New("nil context passed") - } - - // Run CephCluster CR - if err := Add(m, c.ClusterdContext, clusterController); err != nil { - return err - } - - // Add Ceph child CR controllers - for _, f := range AddToManagerFuncs { - if err := f(m, c.ClusterdContext); err != nil { - return err - } - } - - // Add maintenance controllers - for _, f := range AddToManagerFuncsMaintenance { - if err := f(m, c); err != nil { - return err - } - } - - // If machine disruption budget is enabled let's add the controllers - if EnableMachineDisruptionBudget { - for _, f := range MachineDisruptionBudgetAddToManagerFuncs { - if err := f(m, c); err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/operator/ceph/cluster/version.go b/pkg/operator/ceph/cluster/version.go index 8e436d888..fde92d49d 100644 --- a/pkg/operator/ceph/cluster/version.go +++ b/pkg/operator/ceph/cluster/version.go @@ -18,19 +18,23 @@ limitations under the License. package cluster import ( - "time" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" daemonclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/controller" cephver "github.com/rook/rook/pkg/operator/ceph/version" - "github.com/rook/rook/pkg/operator/k8sutil/cmdreporter" ) func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cephver.CephVersion, bool, error) { - version, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, detectCephVersionTimeout) + version, err := controller.DetectCephVersion( + c.rookImage, + cluster.Namespace, + detectVersionName, + cluster.ownerInfo, + c.context.Clientset, + cluster.Spec, + ) if err != nil { return nil, false, err } @@ -113,44 +117,6 @@ func diffImageSpecAndClusterRunningVersion(imageSpecVersion cephver.CephVersion, return false, nil } -// detectCephVersion loads the ceph version from the image and checks that it meets the version requirements to -// run in the cluster -func (c *cluster) detectCephVersion(rookImage, cephImage string, timeout time.Duration) (*cephver.CephVersion, error) { - logger.Infof("detecting the ceph image version for image %s...", cephImage) - versionReporter, err := cmdreporter.New( - c.context.Clientset, c.ownerInfo, - detectVersionName, detectVersionName, c.Namespace, - []string{"ceph"}, []string{"--version"}, - rookImage, cephImage) - if err != nil { - return nil, errors.Wrap(err, "failed to set up ceph version job") - } - - job := versionReporter.Job() - job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter" - - // Apply the same placement for the ceph version detection as the mon daemons except for PodAntiAffinity - cephv1.GetMonPlacement(c.Spec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) - job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil - - stdout, stderr, retcode, err := versionReporter.Run(timeout) - if err != nil { - return nil, errors.Wrap(err, "failed to complete ceph version job") - } - if retcode != 0 { - return nil, errors.Errorf(`ceph version job returned failure with retcode %d. - stdout: %s - stderr: %s`, retcode, stdout, stderr) - } - - version, err := cephver.ExtractCephVersion(stdout) - if err != nil { - return nil, errors.Wrap(err, "failed to extract ceph version") - } - logger.Infof("detected ceph image version: %q", version) - return version, nil -} - func (c *cluster) validateCephVersion(version *cephver.CephVersion) error { if !c.Spec.External.Enable { if !version.IsAtLeast(cephver.Minimum) { @@ -175,8 +141,9 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error { // state we are in and if we should upgrade or not // Try to load clusterInfo so we can compare the running version with the one from the spec image - clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.Namespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(c.context, c.ClusterInfo.Context, c.Namespace) if err == nil { + clusterInfo.Context = c.ClusterInfo.Context // Write connection info (ceph config file and keyring) for ceph commands err = mon.WriteConnectionConfig(c.context, clusterInfo) if err != nil { diff --git a/pkg/operator/ceph/cluster/version_test.go b/pkg/operator/ceph/cluster/version_test.go index 3e56461ef..81cc960b6 100755 --- a/pkg/operator/ceph/cluster/version_test.go +++ b/pkg/operator/ceph/cluster/version_test.go @@ -18,11 +18,13 @@ limitations under the License. package cluster import ( + "context" "encoding/json" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" @@ -149,6 +151,7 @@ func TestDiffImageSpecAndClusterRunningVersion(t *testing.T) { func TestMinVersion(t *testing.T) { c := testSpec(t) c.Spec.CephVersion.AllowUnsupported = true + c.ClusterInfo = &client.ClusterInfo{Context: context.TODO()} // All versions less than 14.2.5 are invalid v := &cephver.CephVersion{Major: 13, Minor: 2, Extra: 3} @@ -167,6 +170,7 @@ func TestMinVersion(t *testing.T) { func TestSupportedVersion(t *testing.T) { c := testSpec(t) + c.ClusterInfo = &client.ClusterInfo{Context: context.TODO()} // Supported versions are valid v := &cephver.CephVersion{Major: 14, Minor: 2, Extra: 12} diff --git a/pkg/operator/ceph/config/keyring/store.go b/pkg/operator/ceph/config/keyring/store.go index 799bca9bb..a8d39ee09 100644 --- a/pkg/operator/ceph/config/keyring/store.go +++ b/pkg/operator/ceph/config/keyring/store.go @@ -20,8 +20,6 @@ limitations under the License. package keyring import ( - "context" - "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" @@ -102,9 +100,8 @@ func (k *SecretStore) CreateOrUpdate(resourceName string, keyring string) error // Delete deletes the keyring secret for the resource. func (k *SecretStore) Delete(resourceName string) error { - ctx := context.TODO() secretName := keyringSecretName(resourceName) - err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Delete(ctx, secretName, metav1.DeleteOptions{}) + err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Delete(k.clusterInfo.Context, secretName, metav1.DeleteOptions{}) if err != nil && !kerrors.IsNotFound(err) { logger.Warningf("failed to delete keyring secret for %q. user may need to delete the resource manually. %v", secretName, err) } @@ -114,13 +111,12 @@ func (k *SecretStore) Delete(resourceName string) error { // CreateSecret creates or update a kubernetes secret func (k *SecretStore) CreateSecret(secret *v1.Secret) error { - ctx := context.TODO() secretName := secret.ObjectMeta.Name - _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Get(ctx, secretName, metav1.GetOptions{}) + _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Get(k.clusterInfo.Context, secretName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("creating secret for %s", secretName) - if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Create(k.clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to create secret for %s", secretName) } return nil @@ -129,7 +125,7 @@ func (k *SecretStore) CreateSecret(secret *v1.Secret) error { } logger.Debugf("updating secret for %s", secretName) - if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + if _, err := k.context.Clientset.CoreV1().Secrets(k.clusterInfo.Namespace).Update(k.clusterInfo.Context, secret, metav1.UpdateOptions{}); err != nil { return errors.Wrapf(err, "failed to update secret for %s", secretName) } return nil diff --git a/pkg/operator/ceph/config/keyring/store_test.go b/pkg/operator/ceph/config/keyring/store_test.go index 603a1e069..a9fe17de4 100644 --- a/pkg/operator/ceph/config/keyring/store_test.go +++ b/pkg/operator/ceph/config/keyring/store_test.go @@ -50,7 +50,7 @@ func TestGenerateKey(t *testing.T) { } ns := "rook-ceph" ownerInfo := k8sutil.OwnerInfo{} - s := GetSecretStore(ctx, &cephclient.ClusterInfo{Namespace: ns}, &ownerInfo) + s := GetSecretStore(ctx, cephclient.AdminClusterInfo(ns), &ownerInfo) generateKey = "generatedsecretkey" failGenerateKey = false diff --git a/pkg/operator/ceph/config/monstore_test.go b/pkg/operator/ceph/config/monstore_test.go index 1d0978a26..bfbdc8309 100644 --- a/pkg/operator/ceph/config/monstore_test.go +++ b/pkg/operator/ceph/config/monstore_test.go @@ -50,7 +50,7 @@ func TestMonStore_Set(t *testing.T) { return "", nil } - monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"}) + monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster")) // setting with spaces converts to underscores e := monStore.Set("global", "debug ms", "10") @@ -95,7 +95,7 @@ func TestMonStore_Delete(t *testing.T) { return "", nil } - monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"}) + monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster")) // ceph config rm called as expected e := monStore.Delete("global", "debug ms") @@ -134,7 +134,7 @@ func TestMonStore_GetDaemon(t *testing.T) { return execReturn, nil } - monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"}) + monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster")) // ceph config get called as expected options, e := monStore.GetDaemon("client.rgw.test.a") @@ -177,7 +177,7 @@ func TestMonStore_DeleteDaemon(t *testing.T) { return execReturn, nil } - monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"}) + monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster")) // ceph config rm rgw_enable_usage_log called as expected e := monStore.DeleteDaemon("client.rgw.test.a") @@ -208,7 +208,7 @@ func TestMonStore_SetAll(t *testing.T) { return "", nil } - monStore := GetMonStore(ctx, &client.ClusterInfo{Namespace: "ns"}) + monStore := GetMonStore(ctx, client.AdminClusterInfo("mycluster")) cfgOverrides := []Option{ configOverride("global", "debug ms", "10"), // setting w/ spaces converts to underscores diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index 5ccfa142c..d7da7e30a 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -19,7 +19,6 @@ limitations under the License. package config import ( - "context" "strings" "github.com/pkg/errors" @@ -62,9 +61,8 @@ func GetStore(context *clusterd.Context, namespace string, ownerInfo *k8sutil.Ow // CreateOrUpdate creates or updates the stored Ceph config based on the cluster info. func (s *Store) CreateOrUpdate(clusterInfo *cephclient.ClusterInfo) error { - ctx := context.TODO() // these are used for all ceph daemons on the commandline and must *always* be stored - if err := s.createOrUpdateMonHostSecrets(ctx, clusterInfo); err != nil { + if err := s.createOrUpdateMonHostSecrets(clusterInfo); err != nil { return errors.Wrap(err, "failed to store mon host configs") } @@ -72,7 +70,7 @@ func (s *Store) CreateOrUpdate(clusterInfo *cephclient.ClusterInfo) error { } // update "mon_host" and "mon_initial_members" in the stored config -func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *cephclient.ClusterInfo) error { +func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephclient.ClusterInfo) error { // extract a list of just the monitor names, which will populate the "mon initial members" // and "mon hosts" global config field @@ -97,11 +95,11 @@ func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *c return errors.Wrapf(err, "failed to set owner reference to moh host secret %q", secret.Name) } - _, err = clientset.CoreV1().Secrets(s.namespace).Get(ctx, StoreName, metav1.GetOptions{}) + _, err = clientset.CoreV1().Secrets(s.namespace).Get(clusterInfo.Context, StoreName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("creating config secret %q", secret.Name) - if _, err := clientset.CoreV1().Secrets(s.namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := clientset.CoreV1().Secrets(s.namespace).Create(clusterInfo.Context, secret, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to create config secret %+v", secret) } } else { @@ -110,7 +108,7 @@ func (s *Store) createOrUpdateMonHostSecrets(ctx context.Context, clusterInfo *c } logger.Debugf("updating config secret %q", secret.Name) - if _, err := clientset.CoreV1().Secrets(s.namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + if _, err := clientset.CoreV1().Secrets(s.namespace).Update(clusterInfo.Context, secret, metav1.UpdateOptions{}); err != nil { return errors.Wrapf(err, "failed to update config secret %+v", secret) } diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go new file mode 100644 index 000000000..5be6eb9dd --- /dev/null +++ b/pkg/operator/ceph/controller.go @@ -0,0 +1,172 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "context" + "strings" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/coreos/pkg/capnslog" + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/discover" + "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + controllerName = "rook-ceph-operator-config-controller" +) + +// ReconcileConfig reconciles a Ceph Operator config +type ReconcileConfig struct { + client client.Client + context *clusterd.Context + config opcontroller.OperatorConfig + opManagerContext context.Context +} + +// Add creates a new Operator configuration Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, config opcontroller.OperatorConfig) reconcile.Reconciler { + return &ReconcileConfig{ + client: mgr.GetClient(), + context: context, + config: config, + opManagerContext: opManagerContext, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + logger.Infof("%s successfully started", controllerName) + + // Watch for ConfigMap (operator config) + err = c.Watch(&source.Kind{ + Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(mgr.GetClient())) + if err != nil { + return err + } + + // Watch for Secret (admission controller secret) + err = c.Watch(&source.Kind{ + Type: &v1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(mgr.GetClient())) + if err != nil { + return err + } + + return nil +} + +// Reconcile reads that state of the cluster for a CephClient object and makes changes based on the state read +// and what is in the CephClient.Spec +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface + reconcileResponse, err := r.reconcile(request) + if err != nil { + logger.Errorf("failed to reconcile %v", err) + } + + return reconcileResponse, err +} + +func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result, error) { + // Fetch the operator's configmap + opConfig := &v1.ConfigMap{} + logger.Debugf("reconciling %s", request.NamespacedName) + err := r.client.Get(r.opManagerContext, request.NamespacedName, opConfig) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("operator's configmap resource not found. will use default value or env var.") + } else { + // Error reading the object - requeue the request. + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") + } + } else { + // Populate the operator's config + r.config.Parameters = opConfig.Data + } + + // Reconcile Ceph CLI timeout, since the clusterd context is passed to by pointer to all CRD + // controllers they will receive the update + opcontroller.SetCephCommandsTimeout(r.config.Parameters) + + // Reconcile Operator's logging level + err = reconcileOperatorLogLevel(opConfig.Data) + if err != nil { + return opcontroller.ImmediateRetryResult, err + } + + // Reconcile discovery daemon + err = r.reconcileDiscoveryDaemon() + if err != nil { + return opcontroller.ImmediateRetryResult, err + } + + // Reconcile webhook secret + // This is done in the predicate function + + logger.Infof("%s done reconciling", controllerName) + return reconcile.Result{}, nil +} + +func reconcileOperatorLogLevel(data map[string]string) error { + rookLogLevel := k8sutil.GetValue(data, "ROOK_LOG_LEVEL", "INFO") + logLevel, err := capnslog.ParseLevel(strings.ToUpper(rookLogLevel)) + if err != nil { + return errors.Wrapf(err, "failed to load ROOK_LOG_LEVEL %q.", rookLogLevel) + } + + capnslog.SetGlobalLogLevel(logLevel) + return nil +} + +func (r *ReconcileConfig) reconcileDiscoveryDaemon() error { + rookDiscover := discover.New(r.context.Clientset) + if opcontroller.DiscoveryDaemonEnabled(r.config.Parameters) { + if err := rookDiscover.Start(r.opManagerContext, r.config.OperatorNamespace, r.config.Image, r.config.ServiceAccount, true); err != nil { + return errors.Wrap(err, "failed to start device discovery daemonset") + } + } else { + if err := rookDiscover.Stop(r.opManagerContext, r.config.OperatorNamespace); err != nil { + return errors.Wrap(err, "failed to stop device discovery daemonset") + } + } + + return nil +} diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index 0357a8715..ad0d4d3e2 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -18,7 +18,6 @@ package controller import ( "context" - "errors" "fmt" "reflect" "strconv" @@ -35,6 +34,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +// OperatorConfig represents the configuration of the operator +type OperatorConfig struct { + OperatorNamespace string + Image string + ServiceAccount string + NamespaceToWatch string + Parameters map[string]string +} + const ( // OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator OperatorSettingConfigMapName string = "rook-ceph-operator-config" @@ -45,9 +53,6 @@ const ( // OperatorNotInitializedMessage is the message we print when the Operator is not ready to reconcile, typically the ceph.conf has not been generated yet OperatorNotInitializedMessage = "skipping reconcile since operator is still initializing" - - // CancellingOrchestrationMessage is the message to indicate a reconcile was cancelled - CancellingOrchestrationMessage = "CANCELLING CURRENT ORCHESTRATION" ) var ( @@ -61,6 +66,9 @@ var ( // WaitForRequeueIfCephClusterNotReady waits for the CephCluster to be ready WaitForRequeueIfCephClusterNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} + // WaitForRequeueIfCephClusterIsUpgrading waits until the upgrade is complete + WaitForRequeueIfCephClusterIsUpgrading = reconcile.Result{Requeue: true, RequeueAfter: time.Minute} + // WaitForRequeueIfFinalizerBlocked waits for resources to be cleaned up before the finalizer can be removed WaitForRequeueIfFinalizerBlocked = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} @@ -71,21 +79,23 @@ var ( OperatorCephBaseImageVersion string ) -func FlexDriverEnabled(context *clusterd.Context) bool { - // Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the flex driver - value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false") +func FlexDriverEnabled(ctx *clusterd.Context) bool { + // Ignore the error. In the remote chance that the configmap fails to be read, we will default + // to disabling the flex driver + // It's ok to use a TODO context here since FlexDriverEnabled is called by a manager which + // already has a context + context := context.TODO() + value, _ := k8sutil.GetOperatorSetting(context, ctx.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_FLEX_DRIVER", "false") return value == "true" } -func DiscoveryDaemonEnabled(context *clusterd.Context) bool { - // Ignore the error. In the remote chance that the configmap fails to be read, we will default to disabling the discovery daemon - value, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_ENABLE_DISCOVERY_DAEMON", "false") - return value == "true" +func DiscoveryDaemonEnabled(data map[string]string) bool { + return k8sutil.GetValue(data, "ROOK_ENABLE_DISCOVERY_DAEMON", "false") == "true" } // SetCephCommandsTimeout sets the timeout value of Ceph commands which are executed from Rook -func SetCephCommandsTimeout(context *clusterd.Context) { - strTimeoutSeconds, _ := k8sutil.GetOperatorSetting(context.Clientset, OperatorSettingConfigMapName, "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15") +func SetCephCommandsTimeout(data map[string]string) { + strTimeoutSeconds := k8sutil.GetValue(data, "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15") timeoutSeconds, err := strconv.Atoi(strTimeoutSeconds) if err != nil || timeoutSeconds < 1 { logger.Warningf("ROOK_CEPH_COMMANDS_TIMEOUT is %q but it should be >= 1, set the default value 15", strTimeoutSeconds) @@ -94,18 +104,6 @@ func SetCephCommandsTimeout(context *clusterd.Context) { exec.CephCommandsTimeout = time.Duration(timeoutSeconds) * time.Second } -// CheckForCancelledOrchestration checks whether a cancellation has been requested -func CheckForCancelledOrchestration(context *clusterd.Context) error { - defer context.RequestCancelOrchestration.UnSet() - - // Check whether we need to cancel the orchestration - if context.RequestCancelOrchestration.IsSet() { - return errors.New(CancellingOrchestrationMessage) - } - - return nil -} - // canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely. func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool { // Get a list of all the keys causing the HEALTH_ERR status. diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index e123494ee..52dac34a3 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -17,17 +17,12 @@ limitations under the License. package controller import ( - "context" "testing" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/util/exec" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" ) func CreateTestClusterFromStatusDetails(details map[string]cephv1.CephHealthMessage) cephv1.CephCluster { @@ -79,28 +74,14 @@ func TestCanIgnoreHealthErrStatusInReconcile(t *testing.T) { } func TestSetCephCommandsTimeout(t *testing.T) { - clientset := fake.NewSimpleClientset() - ctx := context.TODO() - cm := &v1.ConfigMap{} - cm.Name = "rook-ceph-operator-config" - _, err := clientset.CoreV1().ConfigMaps("").Create(ctx, cm, metav1.CreateOptions{}) - assert.NoError(t, err) - context := &clusterd.Context{Clientset: clientset} - - SetCephCommandsTimeout(context) + SetCephCommandsTimeout(map[string]string{}) assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout) exec.CephCommandsTimeout = 0 - cm.Data = map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "0"} - _, err = clientset.CoreV1().ConfigMaps("").Update(ctx, cm, metav1.UpdateOptions{}) - assert.NoError(t, err) - SetCephCommandsTimeout(context) + SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "0"}) assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout) exec.CephCommandsTimeout = 0 - cm.Data = map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"} - _, err = clientset.CoreV1().ConfigMaps("").Update(ctx, cm, metav1.UpdateOptions{}) - assert.NoError(t, err) - SetCephCommandsTimeout(context) + SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"}) assert.Equal(t, 1*time.Second, exec.CephCommandsTimeout) } diff --git a/pkg/operator/ceph/controller/finalizer.go b/pkg/operator/ceph/controller/finalizer.go index 69bf79224..3e3756220 100644 --- a/pkg/operator/ceph/controller/finalizer.go +++ b/pkg/operator/ceph/controller/finalizer.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -74,6 +75,10 @@ func AddFinalizerIfNotPresent(client client.Client, obj client.Object) error { // RemoveFinalizer removes a finalizer from an object func RemoveFinalizer(client client.Client, obj client.Object) error { + err := client.Get(context.TODO(), types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, obj) + if err != nil { + return errors.Wrap(err, "failed to get the latest version of the object") + } objectFinalizer := buildFinalizerName(obj.GetObjectKind().GroupVersionKind().Kind) accessor, err := meta.Accessor(obj) if err != nil { diff --git a/pkg/operator/ceph/controller/predicate.go b/pkg/operator/ceph/controller/predicate.go index 2fe85c5df..6fb9c1102 100644 --- a/pkg/operator/ceph/controller/predicate.go +++ b/pkg/operator/ceph/controller/predicate.go @@ -18,7 +18,9 @@ package controller import ( "encoding/json" + "os" "strings" + "syscall" "github.com/banzaicloud/k8s-objectmatcher/patch" "github.com/google/go-cmp/cmp" @@ -321,58 +323,6 @@ func WatchControllerPredicate() predicate.Funcs { } } -// WatchCephClusterPredicate is a predicate used by child controllers such as Filesystem or Object -// It watch for CR changes on the CephCluster object and reconciles if this needs to be propagated -// For instance the logCollector option from the CephCluster spec affects the configuration of rgw pods -// So if it changes we must update the deployment -func WatchCephClusterPredicate() predicate.Funcs { - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - logger.Debug("create event from a CR") - return true - }, - DeleteFunc: func(e event.DeleteEvent) bool { - logger.Debug("delete event from a CR") - return true - }, - UpdateFunc: func(e event.UpdateEvent) bool { - // resource.Quantity has non-exportable fields, so we use its comparator method - resourceQtyComparer := cmp.Comparer(func(x, y resource.Quantity) bool { return x.Cmp(y) == 0 }) - - switch objOld := e.ObjectOld.(type) { - case *cephv1.CephCluster: - objNew := e.ObjectNew.(*cephv1.CephCluster) - logger.Debug("update event on CephCluster CR") - // If the labels "do_not_reconcile" is set on the object, let's not reconcile that request - isDoNotReconcile := IsDoNotReconcile(objNew.GetLabels()) - if isDoNotReconcile { - logger.Debugf("object %q matched on update but %q label is set, doing nothing", DoNotReconcileLabelName, objNew.Name) - return false - } - diff := cmp.Diff(objOld.Spec, objNew.Spec, resourceQtyComparer) - if diff != "" { - // The image change (upgrade) is being taking care by watchControllerPredicate() in the cluster package - if objOld.Spec.CephVersion.Image != objNew.Spec.CephVersion.Image { - return false - } - // If the log collector setting changes let's reconcile the child controllers - if !cmp.Equal(objOld.Spec.LogCollector, objNew.Spec.LogCollector) { - logger.Debug("log collector option changed, reconciling") - return true - } - } else if objOld.GetGeneration() != objNew.GetGeneration() { - logger.Debugf("skipping resource %q update with unchanged spec", objNew.Name) - } - } - - return false - }, - GenericFunc: func(e event.GenericEvent) bool { - return false - }, - } -} - func objectToBeDeleted(oldObj, newObj client.Object) bool { return !oldObj.GetDeletionTimestamp().Equal(newObj.GetDeletionTimestamp()) } @@ -675,3 +625,8 @@ func IsDoNotReconcile(labels map[string]string) bool { return false } + +func ReloadManager() { + p, _ := os.FindProcess(os.Getpid()) + _ = p.Signal(syscall.SIGHUP) +} diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index e5baa2f1c..8e58b752f 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -18,7 +18,6 @@ limitations under the License. package controller import ( - "context" "fmt" "os" "path" @@ -697,7 +696,7 @@ func ConfigureExternalMetricsEndpoint(ctx *clusterd.Context, monitoringSpec ceph } // Get the endpoint to see if anything needs to be updated - currentEndpoints, err := ctx.Clientset.CoreV1().Endpoints(clusterInfo.Namespace).Get(context.TODO(), endpoint.Name, metav1.GetOptions{}) + currentEndpoints, err := ctx.Clientset.CoreV1().Endpoints(clusterInfo.Namespace).Get(clusterInfo.Context, endpoint.Name, metav1.GetOptions{}) if err != nil && !kerrors.IsNotFound(err) { return errors.Wrap(err, "failed to fetch endpoints") } diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 7836b02ec..e9661e994 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -249,16 +249,13 @@ func TestExtractMgrIP(t *testing.T) { } func TestConfigureExternalMetricsEndpoint(t *testing.T) { + clusterInfo := cephclient.AdminClusterInfo("rook-ceph") t.Run("spec and current active mgr endpoint identical with no existing endpoint object", func(t *testing.T) { monitoringSpec := cephv1.MonitoringSpec{ Enabled: true, RulesNamespace: "rook-ceph", ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}}, } - clusterInfo := &cephclient.ClusterInfo{ - FSID: "id", - Namespace: "rook-ceph", - } executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) @@ -289,10 +286,6 @@ func TestConfigureExternalMetricsEndpoint(t *testing.T) { RulesNamespace: "rook-ceph", ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}}, } - clusterInfo := &cephclient.ClusterInfo{ - FSID: "id", - Namespace: "rook-ceph", - } executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) @@ -322,10 +315,6 @@ func TestConfigureExternalMetricsEndpoint(t *testing.T) { RulesNamespace: "rook-ceph", ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}}, } - clusterInfo := &cephclient.ClusterInfo{ - FSID: "id", - Namespace: "rook-ceph", - } executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) @@ -360,10 +349,6 @@ func TestConfigureExternalMetricsEndpoint(t *testing.T) { RulesNamespace: "rook-ceph", ExternalMgrEndpoints: []v1.EndpointAddress{{IP: "192.168.0.1"}}, } - clusterInfo := &cephclient.ClusterInfo{ - FSID: "id", - Namespace: "rook-ceph", - } executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) diff --git a/pkg/operator/ceph/controller/version.go b/pkg/operator/ceph/controller/version.go index 82d4565c7..e29c92c87 100644 --- a/pkg/operator/ceph/controller/version.go +++ b/pkg/operator/ceph/controller/version.go @@ -17,13 +17,24 @@ limitations under the License. package controller import ( + "fmt" + "time" + "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/version" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/k8sutil/cmdreporter" + "k8s.io/client-go/kubernetes" ) +const detectCephVersionTimeout = 15 * time.Minute + // ValidateCephVersionsBetweenLocalAndExternalClusters makes sure an external cluster can be connected // by checking the external ceph versions available and comparing it with the local image provided func ValidateCephVersionsBetweenLocalAndExternalClusters(context *clusterd.Context, clusterInfo *client.ClusterInfo) (cephver.CephVersion, error) { @@ -47,3 +58,79 @@ func GetImageVersion(cephCluster cephv1.CephCluster) (*cephver.CephVersion, erro return nil, errors.New("attempt to determine ceph version for the current cluster image timed out") } + +// DetectCephVersion loads the ceph version from the image and checks that it meets the version requirements to +// run in the cluster +func DetectCephVersion(rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*version.CephVersion, error) { + cephImage := cephClusterSpec.CephVersion.Image + logger.Infof("detecting the ceph image version for image %s...", cephImage) + versionReporter, err := cmdreporter.New( + clientset, + ownerInfo, + jobName, + jobName, + namespace, + []string{"ceph"}, + []string{"--version"}, + rookImage, + cephImage, + ) + if err != nil { + return nil, errors.Wrap(err, "failed to set up ceph version job") + } + + job := versionReporter.Job() + job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter" + + // Apply the same placement for the ceph version detection as the mon daemons except for PodAntiAffinity + cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) + job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil + + stdout, stderr, retcode, err := versionReporter.Run(detectCephVersionTimeout) + if err != nil { + return nil, errors.Wrap(err, "failed to complete ceph version job") + } + if retcode != 0 { + return nil, errors.Errorf(`ceph version job returned failure with retcode %d. `+ + `stdout: %s. `+ + `stderr: %s`, + retcode, + stdout, + stderr, + ) + } + + version, err := cephver.ExtractCephVersion(stdout) + if err != nil { + return nil, errors.Wrap(err, "failed to extract ceph version") + } + logger.Infof("detected ceph image version: %q", version) + return version, nil +} + +func CurrentAndDesiredCephVersion(rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + // Detect desired CephCluster version + desiredCephVersion, err := DetectCephVersion(rookImage, namespace, fmt.Sprintf("%s-detect-version", jobName), ownerInfo, context.Clientset, cephClusterSpec) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to detect ceph image version") + } + + // Check the ceph version of the running monitors + runningMonDaemonVersion, err := client.LeastUptodateDaemonVersion(context, clusterInfo, config.MonType) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to retrieve current ceph %q version", config.MonType) + } + + return desiredCephVersion, &runningMonDaemonVersion, nil +} + +func ErrorCephUpgradingRequeue(runningCephVersion, desiredCephVersion *version.CephVersion) error { + return errors.Errorf(`waiting for ceph monitors upgrade to finish. `+ + `current version: %s. `+ + `expected version: %s. `+ + `will reconcile again in %s`, + runningCephVersion.String(), + desiredCephVersion.String(), + WaitForRequeueIfCephClusterIsUpgrading.RequeueAfter.String(), + ) +} diff --git a/pkg/operator/ceph/controller_test.go b/pkg/operator/ceph/controller_test.go new file mode 100644 index 000000000..fec221648 --- /dev/null +++ b/pkg/operator/ceph/controller_test.go @@ -0,0 +1,202 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "context" + "os" + "testing" + "time" + + "github.com/coreos/pkg/capnslog" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/test" + "github.com/rook/rook/pkg/util/exec" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func TestOperatorController(t *testing.T) { + ctx := context.TODO() + // Set DEBUG logging + capnslog.SetGlobalLogLevel(capnslog.DEBUG) + os.Setenv("ROOK_LOG_LEVEL", "DEBUG") + name := controller.OperatorSettingConfigMapName + namespace := "rook-ceph" + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + t.Run("success - normal run", func(t *testing.T) { + fakeClientSet := test.New(t, 1) + test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") + c := &clusterd.Context{ + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), + } + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).Build() + + // Create a ReconcileCSI object with the scheme and fake client. + r := &ReconcileConfig{ + client: cl, + context: c, + config: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + } + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + }) + + t.Run("success - env set for command timeout", func(t *testing.T) { + fakeClientSet := test.New(t, 1) + test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") + c := &clusterd.Context{ + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), + } + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).Build() + + // Create a ReconcileCSI object with the scheme and fake client. + r := &ReconcileConfig{ + client: cl, + context: c, + config: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + } + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "10") + defer os.Unsetenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS") + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + assert.Equal(t, time.Second*10, exec.CephCommandsTimeout) + }) + + t.Run("success - cm set for command timeout", func(t *testing.T) { + fakeClientSet := test.New(t, 1) + test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") + c := &clusterd.Context{ + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), + } + + // Register operator types with the runtime scheme. + // s := scheme.Scheme + s := clientgoscheme.Scheme + // s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}) + + opConfigCM := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.OperatorSettingConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "20", + }, + } + + object := []runtime.Object{ + opConfigCM, + } + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + // Create a ReconcileCSI object with the scheme and fake client. + r := &ReconcileConfig{ + client: cl, + context: c, + config: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + } + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + assert.Equal(t, time.Second*20, exec.CephCommandsTimeout) + }) + + t.Run("success - env set for discovery daemon", func(t *testing.T) { + fakeClientSet := test.New(t, 1) + test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") + c := &clusterd.Context{ + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), + } + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).Build() + + // Create a ReconcileCSI object with the scheme and fake client. + r := &ReconcileConfig{ + client: cl, + context: c, + config: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + } + os.Setenv("ROOK_ENABLE_DISCOVERY_DAEMON", "true") + defer os.Unsetenv("ROOK_ENABLE_DISCOVERY_DAEMON") + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + ds, err := c.Clientset.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + assert.Equal(t, 1, len(ds.Items), ds) + assert.Equal(t, "rook-discover", ds.Items[0].Name, ds) + }) +} diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go index 1055de8ed..051823a7f 100644 --- a/pkg/operator/ceph/cr_manager.go +++ b/pkg/operator/ceph/cr_manager.go @@ -21,8 +21,28 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/agent" + "github.com/rook/rook/pkg/operator/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster" + "github.com/rook/rook/pkg/operator/ceph/cluster/crash" + "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/csi" + "github.com/rook/rook/pkg/operator/ceph/disruption/clusterdisruption" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" + "github.com/rook/rook/pkg/operator/ceph/disruption/machinedisruption" + "github.com/rook/rook/pkg/operator/ceph/disruption/machinelabel" + "github.com/rook/rook/pkg/operator/ceph/file" + "github.com/rook/rook/pkg/operator/ceph/file/mirror" + "github.com/rook/rook/pkg/operator/ceph/nfs" + "github.com/rook/rook/pkg/operator/ceph/object" + "github.com/rook/rook/pkg/operator/ceph/object/bucket" + "github.com/rook/rook/pkg/operator/ceph/object/realm" + objectuser "github.com/rook/rook/pkg/operator/ceph/object/user" + "github.com/rook/rook/pkg/operator/ceph/object/zone" + "github.com/rook/rook/pkg/operator/ceph/object/zonegroup" + "github.com/rook/rook/pkg/operator/ceph/pool" "k8s.io/apimachinery/pkg/runtime" mapiv1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" @@ -52,7 +72,86 @@ var ( webhookResources = []webhook.Validator{&cephv1.CephCluster{}, &cephv1.CephBlockPool{}, &cephv1.CephObjectStore{}} ) -func (o *Operator) startManager(context context.Context, namespaceToWatch string, mgrErrorCh chan error) { +var ( + // EnableMachineDisruptionBudget checks whether machine disruption budget is enabled + EnableMachineDisruptionBudget bool +) + +// AddToManagerFuncsMaintenance is a list of functions to add all Controllers to the Manager (entrypoint for controller) +var AddToManagerFuncsMaintenance = []func(manager.Manager, *controllerconfig.Context) error{ + clusterdisruption.Add, +} + +// MachineDisruptionBudgetAddToManagerFuncs is a list of fencing related functions to add all Controllers to the Manager (entrypoint for controller) +var MachineDisruptionBudgetAddToManagerFuncs = []func(manager.Manager, *controllerconfig.Context) error{ + machinelabel.Add, + machinedisruption.Add, +} + +// AddToManagerFuncs is a list of functions to add all Controllers to the Manager (entrypoint for controller) +var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Context, opcontroller.OperatorConfig) error{ + crash.Add, + pool.Add, + objectuser.Add, + realm.Add, + zonegroup.Add, + zone.Add, + object.Add, + file.Add, + nfs.Add, + rbd.Add, + client.Add, + mirror.Add, + Add, + csi.Add, + agent.Add, + bucket.Add, +} + +// AddToManagerOpFunc is a list of functions to add all Controllers to the Manager (entrypoint for +// controller) +// var AddToManagerOpFunc = []func(manager.Manager, *clusterd.Context, opcontroller.OperatorConfig) error{} + +// AddToManager adds all the registered controllers to the passed manager. +// each controller package will have an Add method listed in AddToManagerFuncs +// which will setup all the necessary watch +func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context) error { + if c == nil { + return errors.New("nil context passed") + } + + // Run CephCluster CR + if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext); err != nil { + return err + } + + // Add Ceph child CR controllers + for _, f := range AddToManagerFuncs { + if err := f(m, c.ClusterdContext, opManagerContext, *o.config); err != nil { + return err + } + } + + // Add maintenance controllers + for _, f := range AddToManagerFuncsMaintenance { + if err := f(m, c); err != nil { + return err + } + } + + // If machine disruption budget is enabled let's add the controllers + if EnableMachineDisruptionBudget { + for _, f := range MachineDisruptionBudgetAddToManagerFuncs { + if err := f(m, c); err != nil { + return err + } + } + } + + return nil +} + +func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan error) { logger.Info("setting up schemes") // Setup Scheme for all resources scheme := runtime.NewScheme() @@ -67,7 +166,7 @@ func (o *Operator) startManager(context context.Context, namespaceToWatch string // Set up a manager mgrOpts := manager.Options{ LeaderElection: false, - Namespace: namespaceToWatch, + Namespace: o.config.NamespaceToWatch, Scheme: scheme, CertDir: certDir, } @@ -79,23 +178,7 @@ func (o *Operator) startManager(context context.Context, namespaceToWatch string return } - // options to pass to the controllers - controllerOpts := &controllerconfig.Context{ - RookImage: o.config.Image, - ClusterdContext: o.context, - OperatorNamespace: o.config.OperatorNamespace, - ReconcileCanaries: &controllerconfig.LockingBool{}, - } - - // Add the registered controllers to the manager (entrypoint for controllers) - err = cluster.AddToManager(mgr, controllerOpts, o.clusterController) - if err != nil { - mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager") - return - } - - // Add Webhooks - // TODO: this needs a callback with a watcher for this secret so we can quickly enable the webhook if it changes + // Add webhook if needed isPresent, err := isSecretPresent(context, o.context) if err != nil { mgrErrorCh <- errors.Wrap(err, "failed to retrieve admission webhook secret") @@ -117,9 +200,25 @@ func (o *Operator) startManager(context context.Context, namespaceToWatch string } } + // options to pass to the controllers + controllerOpts := &controllerconfig.Context{ + ClusterdContext: o.context, + ReconcileCanaries: &controllerconfig.LockingBool{}, + OpManagerContext: context, + } + + // Add the registered controllers to the manager (entrypoint for controllers) + err = o.addToManager(mgr, controllerOpts, context) + if err != nil { + mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager") + return + } + logger.Info("starting the controller-runtime manager") if err := mgr.Start(context); err != nil { mgrErrorCh <- errors.Wrap(err, "failed to run the controller-runtime manager") return } + + logger.Info("successfully started the controller-runtime manager") } diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index 083f309b2..03e76934f 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -87,9 +87,9 @@ func monEndpoints(mons map[string]*cephclient.MonInfo) []string { return endpoints } -// UpdateCsiClusterConfig returns a json-formatted string containing +// updateCsiClusterConfig returns a json-formatted string containing // the cluster-to-mon mapping required to configure ceph csi. -func UpdateCsiClusterConfig( +func updateCsiClusterConfig( curr, clusterKey string, mons map[string]*cephclient.MonInfo) (string, error) { var ( @@ -158,8 +158,6 @@ func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface, ownerI func SaveClusterConfig( clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, l sync.Locker) error { - ctx := context.TODO() - if !CSIEnabled() { return nil } @@ -173,7 +171,7 @@ func SaveClusterConfig( logger.Debugf("Using %+v for CSI ConfigMap Namespace", csiNamespace) // fetch current ConfigMap contents - configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(ctx, + configMap, err := clientset.CoreV1().ConfigMaps(csiNamespace).Get(clusterInfo.Context, ConfigName, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to fetch current csi config map") @@ -184,7 +182,7 @@ func SaveClusterConfig( if currData == "" { currData = "[]" } - newData, err := UpdateCsiClusterConfig( + newData, err := updateCsiClusterConfig( currData, clusterNamespace, clusterInfo.Monitors) if err != nil { return errors.Wrap(err, "failed to update csi config map data") @@ -192,7 +190,7 @@ func SaveClusterConfig( configMap.Data[ConfigKey] = newData // update ConfigMap with new contents - if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(ctx, configMap, metav1.UpdateOptions{}); err != nil { + if _, err := clientset.CoreV1().ConfigMaps(csiNamespace).Update(clusterInfo.Context, configMap, metav1.UpdateOptions{}); err != nil { return errors.Wrapf(err, "failed to update csi config map") } diff --git a/pkg/operator/ceph/csi/cluster_config_test.go b/pkg/operator/ceph/csi/cluster_config_test.go index 2def6e45b..afa53e074 100644 --- a/pkg/operator/ceph/csi/cluster_config_test.go +++ b/pkg/operator/ceph/csi/cluster_config_test.go @@ -28,7 +28,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) { mons := map[string]*cephclient.MonInfo{ "foo": {Name: "foo", Endpoint: "1.2.3.4:5000"}, } - s, err := UpdateCsiClusterConfig("[]", "alpha", mons) + s, err := updateCsiClusterConfig("[]", "alpha", mons) assert.NoError(t, err) assert.Equal(t, s, `[{"clusterID":"alpha","monitors":["1.2.3.4:5000"]}]`) @@ -36,7 +36,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) { // add a 2nd mon to the current cluster mons["bar"] = &cephclient.MonInfo{ Name: "bar", Endpoint: "10.11.12.13:5000"} - s, err = UpdateCsiClusterConfig(s, "alpha", mons) + s, err = updateCsiClusterConfig(s, "alpha", mons) assert.NoError(t, err) cc, err := parseCsiClusterConfig(s) assert.NoError(t, err) @@ -52,7 +52,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) { "flam": {Name: "flam", Endpoint: "20.1.1.2:5000"}, "blam": {Name: "blam", Endpoint: "20.1.1.3:5000"}, } - s, err = UpdateCsiClusterConfig(s, "beta", mons2) + s, err = updateCsiClusterConfig(s, "beta", mons2) assert.NoError(t, err) cc, err = parseCsiClusterConfig(s) assert.NoError(t, err) @@ -69,7 +69,7 @@ func TestUpdateCsiClusterConfig(t *testing.T) { // remove a mon from the 2nd cluster delete(mons2, "blam") - s, err = UpdateCsiClusterConfig(s, "beta", mons2) + s, err = updateCsiClusterConfig(s, "beta", mons2) assert.NoError(t, err) cc, err = parseCsiClusterConfig(s) assert.NoError(t, err) @@ -84,6 +84,6 @@ func TestUpdateCsiClusterConfig(t *testing.T) { assert.Equal(t, len(cc[1].Monitors), 2) // does it return error on garbage input? - _, err = UpdateCsiClusterConfig("qqq", "beta", mons2) + _, err = updateCsiClusterConfig("qqq", "beta", mons2) assert.Error(t, err) } diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go new file mode 100644 index 000000000..40dfcd1f9 --- /dev/null +++ b/pkg/operator/ceph/csi/controller.go @@ -0,0 +1,195 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "os" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/csi/peermap" + "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + controllerName = "rook-ceph-operator-csi-controller" +) + +// ReconcileCSI reconciles a ceph-csi driver +type ReconcileCSI struct { + client client.Client + context *clusterd.Context + opManagerContext context.Context + opConfig opcontroller.OperatorConfig +} + +// Add creates a new Ceph CSI Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { + return &ReconcileCSI{ + client: mgr.GetClient(), + context: context, + opConfig: opConfig, + opManagerContext: opManagerContext, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + logger.Infof("%s successfully started", controllerName) + + // Watch for ConfigMap (operator config) + err = c.Watch(&source.Kind{ + Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController()) + if err != nil { + return err + } + + // Watch for CephCluster + err = c.Watch(&source.Kind{ + Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController()) + if err != nil { + return err + } + + return nil +} + +// Reconcile reads that state of the operator config map and makes changes based on the state read +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface + reconcileResponse, err := r.reconcile(request) + if err != nil { + logger.Errorf("failed to reconcile %v", err) + } + + return reconcileResponse, err +} + +func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) { + // See if there is a CephCluster + cephClusters := &cephv1.CephClusterList{} + err := r.client.List(r.opManagerContext, cephClusters, &client.ListOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("no ceph cluster found not deploying ceph csi driver") + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters") + } + + // // Do not nothing if no ceph cluster is present + if len(cephClusters.Items) == 0 { + logger.Debug("no ceph cluster found not deploying ceph csi driver") + return reconcile.Result{}, nil + } else { + for _, cluster := range cephClusters.Items { + if !cluster.DeletionTimestamp.IsZero() { + logger.Debug("ceph cluster is being deleting, no need to reconcile the csi driver") + return reconcile.Result{}, nil + } + + if !cluster.Spec.External.Enable && cluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() { + logger.Debug("ceph cluster has cleanup policy, the cluster will soon go away, no need to reconcile the csi driver") + return reconcile.Result{}, nil + } + } + } + + // Fetch the operator's configmap + opConfig := &v1.ConfigMap{} + err = r.client.Get(r.opManagerContext, request.NamespacedName, opConfig) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("operator's configmap resource not found. will use default value or env var.") + r.opConfig.Parameters = make(map[string]string) + } else { + // Error reading the object - requeue the request. + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") + } + } else { + // Populate the operator's config + r.opConfig.Parameters = opConfig.Data + } + + serverVersion, err := r.context.Clientset.Discovery().ServerVersion() + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version") + } + + if serverVersion.Major < KubeMinMajor || serverVersion.Major == KubeMinMajor && serverVersion.Minor < ProvDeploymentSuppVersion { + logger.Infof("CSI drivers only supported in K8s 1.14 or newer. version=%s", serverVersion.String()) + // disable csi control variables to disable other csi functions + EnableRBD = false + EnableCephFS = false + return opcontroller.ImmediateRetryResult, nil + } + + ownerRef, err := k8sutil.GetDeploymentOwnerReference(r.context.Clientset, os.Getenv(k8sutil.PodNameEnvVar), r.opConfig.OperatorNamespace) + if err != nil { + logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err) + } + if ownerRef != nil { + blockOwnerDeletion := false + ownerRef.BlockOwnerDeletion = &blockOwnerDeletion + } + + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, r.opConfig.OperatorNamespace) + // create an empty config map. config map will be filled with data + // later when clusters have mons + err = CreateCsiConfigMap(r.opConfig.OperatorNamespace, r.context.Clientset, ownerInfo) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map") + } + + err = peermap.CreateOrUpdateConfig(r.context, &peermap.PeerIDMappings{}) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map") + } + + err = r.validateAndConfigureDrivers(serverVersion, ownerInfo) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed configure ceph csi") + } + + return reconcile.Result{}, nil +} diff --git a/pkg/operator/ceph/csi/controller_test.go b/pkg/operator/ceph/csi/controller_test.go new file mode 100644 index 000000000..821f2c85b --- /dev/null +++ b/pkg/operator/ceph/csi/controller_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "os" + "testing" + + "github.com/coreos/pkg/capnslog" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func TestCephCSIController(t *testing.T) { + ctx := context.TODO() + var ( + name = "rook-ceph" + namespace = "rook-ceph" + ) + // Set DEBUG logging + capnslog.SetGlobalLogLevel(capnslog.DEBUG) + os.Setenv("ROOK_LOG_LEVEL", "DEBUG") + os.Setenv(k8sutil.PodNameEnvVar, "rook-ceph-operator") + os.Setenv(k8sutil.PodNamespaceEnvVar, namespace) + + os.Setenv("ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "true") + CSIParam = Param{ + CSIPluginImage: "image", + RegistrarImage: "image", + ProvisionerImage: "image", + AttacherImage: "image", + SnapshotterImage: "image", + } + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + t.Run("failure because no CephCluster", func(t *testing.T) { + fakeClientSet := test.New(t, 1) + test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") + c := &clusterd.Context{ + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), + } + + _, err := c.Clientset.CoreV1().Pods(namespace).Create(ctx, test.FakeOperatorPod(namespace), metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = c.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), test.FakeReplicaSet(namespace), metav1.CreateOptions{}) + assert.NoError(t, err) + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}, &cephv1.CephClusterList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).Build() + c.Client = cl + + // Create a ReconcileCSI object with the scheme and fake client. + r := &ReconcileCSI{ + client: cl, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + } + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + }) + + t.Run("success ceph csi deployment", func(t *testing.T) { + fakeClientSet := test.New(t, 1) + test.SetFakeKubernetesVersion(fakeClientSet, "v1.21.0") + c := &clusterd.Context{ + Clientset: fakeClientSet, + RookClientset: rookclient.NewSimpleClientset(), + } + _, err := c.Clientset.CoreV1().Pods(namespace).Create(ctx, test.FakeOperatorPod(namespace), metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = c.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), test.FakeReplicaSet(namespace), metav1.CreateOptions{}) + assert.NoError(t, err) + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Status: cephv1.ClusterStatus{ + Phase: "", + CephVersion: &cephv1.ClusterVersion{ + Version: "14.2.9-0", + }, + CephStatus: &cephv1.CephStatus{ + Health: "", + }, + }, + } + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + + object := []runtime.Object{ + cephCluster, + } + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + c.Client = cl + // // Create a ReconcileCSI object with the scheme and fake client. + r := &ReconcileCSI{ + client: cl, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + } + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + + ds, err := c.Clientset.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + assert.Equal(t, 2, len(ds.Items), ds) + }) +} diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index a9e2f0ee2..246d615bf 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -20,136 +20,83 @@ import ( "strconv" "github.com/pkg/errors" - "github.com/rook/rook/pkg/clusterd" - controllerutil "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/apimachinery/pkg/version" - "k8s.io/client-go/kubernetes" ) -func ValidateAndConfigureDrivers(context *clusterd.Context, namespace, rookImage, securityAccount string, serverVersion *version.Info, ownerInfo *k8sutil.OwnerInfo) { - csiLock.Lock() - defer csiLock.Unlock() +func (r *ReconcileCSI) validateAndConfigureDrivers(serverVersion *version.Info, ownerInfo *k8sutil.OwnerInfo) error { var ( v *CephCSIVersion err error ) - if err = setParams(context.Clientset); err != nil { - logger.Errorf("failed to configure CSI parameters. %v", err) - return + if err = r.setParams(); err != nil { + return errors.Wrapf(err, "failed to configure CSI parameters") } if err = validateCSIParam(); err != nil { - logger.Errorf("failed to validate CSI parameters. %v", err) - return + return errors.Wrapf(err, "failed to validate CSI parameters") } if !AllowUnsupported && CSIEnabled() { - if v, err = validateCSIVersion(context.Clientset, namespace, rookImage, securityAccount, ownerInfo); err != nil { - logger.Errorf("invalid csi version. %+v", err) - return + if v, err = r.validateCSIVersion(ownerInfo); err != nil { + return errors.Wrapf(err, "invalid csi version") } } else { - logger.Info("Skipping csi version check, since unsupported versions are allowed or csi is disabled") + logger.Info("skipping csi version check, since unsupported versions are allowed or csi is disabled") } if CSIEnabled() { maxRetries := 3 for i := 0; i < maxRetries; i++ { - if err = startDrivers(context.Clientset, context.RookClientset, namespace, serverVersion, ownerInfo, v); err != nil { + if err = r.startDrivers(serverVersion, ownerInfo, v); err != nil { logger.Errorf("failed to start Ceph csi drivers, will retry starting csi drivers %d more times. %v", maxRetries-i-1, err) } else { break } } + return errors.Wrap(err, "failed to start ceph csi drivers") } - stopDrivers(context.Clientset, namespace, serverVersion) + // Check whether RBD or CephFS needs to be disabled + r.stopDrivers(serverVersion) + + return nil } -func setParams(clientset kubernetes.Interface) error { +func (r *ReconcileCSI) setParams() error { var err error - csiEnableRBD, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ENABLE_RBD", "true") - if err != nil { - return errors.Wrap(err, "unable to determine if CSI driver for RBD is enabled") - } - if EnableRBD, err = strconv.ParseBool(csiEnableRBD); err != nil { + if EnableRBD, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_RBD", "true")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_RBD'") } - csiEnableCephFS, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ENABLE_CEPHFS", "true") - if err != nil { - return errors.Wrap(err, "unable to determine if CSI driver for CephFS is enabled") - } - if EnableCephFS, err = strconv.ParseBool(csiEnableCephFS); err != nil { + if EnableCephFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_CEPHFS'") } - csiAllowUnsupported, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "false") - if err != nil { - return errors.Wrap(err, "unable to determine if unsupported version is allowed") - } - if AllowUnsupported, err = strconv.ParseBool(csiAllowUnsupported); err != nil { + if AllowUnsupported, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "false")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ALLOW_UNSUPPORTED_VERSION'") } - csiEnableCSIGRPCMetrics, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ENABLE_GRPC_METRICS", "false") - if err != nil { - return errors.Wrap(err, "unable to determine if CSI GRPC metrics is enabled") - } - if EnableCSIGRPCMetrics, err = strconv.ParseBool(csiEnableCSIGRPCMetrics); err != nil { + if EnableCSIGRPCMetrics, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_GRPC_METRICS", "false")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_GRPC_METRICS'") } - csiEnableCSIHostNetwork, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_HOST_NETWORK", "true") - if err != nil { - return errors.Wrap(err, "failed to determine if CSI Host Network is enabled") - } - if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(csiEnableCSIHostNetwork); err != nil { + if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") } - CSIParam.CSIPluginImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) - if err != nil { - return errors.Wrap(err, "unable to configure CSI plugin image") - } - CSIParam.RegistrarImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) - if err != nil { - return errors.Wrap(err, "unable to configure CSI registrar image") - } - CSIParam.ProvisionerImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) - if err != nil { - return errors.Wrap(err, "unable to configure CSI provisioner image") - } - CSIParam.AttacherImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) - if err != nil { - return errors.Wrap(err, "unable to configure CSI attacher image") - } - CSIParam.SnapshotterImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) - if err != nil { - return errors.Wrap(err, "unable to configure CSI snapshotter image") - } - CSIParam.KubeletDirPath, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) - if err != nil { - return errors.Wrap(err, "unable to configure CSI kubelet directory path") - } - CSIParam.VolumeReplicationImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage) - if err != nil { - return errors.Wrap(err, "unable to configure Volume replication controller image") - } - - csiCephFSPodLabels, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_CEPHFS_POD_LABELS", "") - if err != nil { - return errors.Wrap(err, "unable to configure CSI CephFS pod labels") - } + CSIParam.CSIPluginImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) + CSIParam.RegistrarImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) + CSIParam.ProvisionerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) + CSIParam.AttacherImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) + CSIParam.SnapshotterImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) + CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) + CSIParam.VolumeReplicationImage = k8sutil.GetValue(r.opConfig.Parameters, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage) + csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "") CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels) - - csiRBDPodLabels, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_RBD_POD_LABELS", "") - if err != nil { - return errors.Wrap(err, "unable to configure CSI RBD pod labels") - } + csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "") CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels) return nil diff --git a/pkg/operator/ceph/csi/peermap/config.go b/pkg/operator/ceph/csi/peermap/config.go index b7a8d6d7d..927c36dd2 100644 --- a/pkg/operator/ceph/csi/peermap/config.go +++ b/pkg/operator/ceph/csi/peermap/config.go @@ -168,7 +168,7 @@ func getClusterPoolIDMap(clusterContext *clusterd.Context, clusterInfo *cephclie logger.Debugf("pool details of local cluster %+v", localPoolDetails) for _, peerSecret := range pool.Spec.Mirroring.Peers.SecretNames { - s, err := clusterContext.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(context.TODO(), peerSecret, metav1.GetOptions{}) + s, err := clusterContext.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(clusterInfo.Context, peerSecret, metav1.GetOptions{}) if err != nil { return mappings, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret) } diff --git a/pkg/operator/ceph/csi/peermap/config_test.go b/pkg/operator/ceph/csi/peermap/config_test.go index 0b1d4fdb7..1878f54b6 100644 --- a/pkg/operator/ceph/csi/peermap/config_test.go +++ b/pkg/operator/ceph/csi/peermap/config_test.go @@ -275,7 +275,7 @@ var mockExecutor = &exectest.MockExecutor{ } func TestSinglePeerMappings(t *testing.T) { - clusterInfo := &cephclient.ClusterInfo{Namespace: ns} + clusterInfo := cephclient.AdminClusterInfo(ns) fakeContext := &clusterd.Context{ Executor: mockExecutor, Clientset: test.New(t, 3), @@ -299,7 +299,7 @@ func TestSinglePeerMappings(t *testing.T) { } func TestMultiPeerMappings(t *testing.T) { - clusterInfo := &cephclient.ClusterInfo{Namespace: ns} + clusterInfo := cephclient.AdminClusterInfo(ns) fakeContext := &clusterd.Context{ Executor: mockExecutor, Clientset: test.New(t, 3), @@ -358,41 +358,8 @@ func TestDecodePeerToken(t *testing.T) { assert.Error(t, err) } -func fakeOperatorPod() *corev1.Pod { - p := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: ns, - OwnerReferences: []metav1.OwnerReference{ - { - Kind: "ReplicaSet", - Name: "testReplicaSet", - }, - }, - }, - Spec: corev1.PodSpec{}, - } - return p -} - -func fakeReplicaSet() *appsv1.ReplicaSet { - r := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testReplicaSet", - Namespace: ns, - OwnerReferences: []metav1.OwnerReference{ - { - Kind: "Deployment", - }, - }, - }, - } - - return r -} - func TestCreateOrUpdateConfig(t *testing.T) { - os.Setenv("POD_NAME", "test") + os.Setenv("POD_NAME", "rook-ceph-operator") defer os.Setenv("POD_NAME", "") os.Setenv("POD_NAMESPACE", ns) defer os.Setenv("POD_NAMESPACE", "") @@ -414,11 +381,11 @@ func TestCreateOrUpdateConfig(t *testing.T) { } // Create fake pod - _, err = fakeContext.Clientset.CoreV1().Pods(ns).Create(context.TODO(), fakeOperatorPod(), metav1.CreateOptions{}) + _, err = fakeContext.Clientset.CoreV1().Pods(ns).Create(context.TODO(), test.FakeOperatorPod(ns), metav1.CreateOptions{}) assert.NoError(t, err) // Create fake replicaset - _, err = fakeContext.Clientset.AppsV1().ReplicaSets(ns).Create(context.TODO(), fakeReplicaSet(), metav1.CreateOptions{}) + _, err = fakeContext.Clientset.AppsV1().ReplicaSets(ns).Create(context.TODO(), test.FakeReplicaSet(ns), metav1.CreateOptions{}) assert.NoError(t, err) // Create empty ID mapping configMap diff --git a/pkg/operator/ceph/csi/predicate.go b/pkg/operator/ceph/csi/predicate.go new file mode 100644 index 000000000..5850e431d --- /dev/null +++ b/pkg/operator/ceph/csi/predicate.go @@ -0,0 +1,93 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "regexp" + + "github.com/google/go-cmp/cmp" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// predicateController is the predicate function to trigger reconcile on operator configuration cm change +func predicateController() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + // if the operator configuration file is created we want to reconcile + if cm, ok := e.Object.(*v1.ConfigMap); ok { + // We don't want to use cm.Generation here, it case the operator was stopped and the + // ConfigMap was created + return cm.Name == opcontroller.OperatorSettingConfigMapName + } + + // If a Ceph Cluster is created we want to reconcile the csi driver + if cephCluster, ok := e.Object.(*cephv1.CephCluster); ok { + // This allows us to avoid a double reconcile of the CSI controller if this is not + // the first generation of the CephCluster. So only return true if this is the very + // first instance of the CephCluster + // Corner case is when the cluster is created but the operator is down AND the cm + // does not exist... However, these days the operator config map is part of the + // operator.yaml so it's probably acceptable? + return cephCluster.Generation == 1 + } + + return false + }, + + UpdateFunc: func(e event.UpdateEvent) bool { + resourceQtyComparer := cmp.Comparer(func(x, y resource.Quantity) bool { return x.Cmp(y) == 0 }) + if old, ok := e.ObjectOld.(*v1.ConfigMap); ok { + if new, ok := e.ObjectNew.(*v1.ConfigMap); ok { + if old.Name == opcontroller.OperatorSettingConfigMapName && new.Name == opcontroller.OperatorSettingConfigMapName { + diff := cmp.Diff(old.Data, new.Data, resourceQtyComparer) + logger.Debugf("operator configmap diff:\n %s", diff) + return findCSIChange(diff) + } + } + } + + return false + }, + + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + } +} + +func findCSIChange(str string) bool { + var re = regexp.MustCompile(`(?m)^(\+|-).*(\"ROOK_CSI_|\"CSI_).*,$`) + found := re.FindAllString(str, -1) + if len(found) > 0 { + for _, match := range found { + // logger.Debugf("ceph csi config changed with: %q", match) + logger.Infof("ceph csi config changed with: %q", match) + } + return true + } + + return false +} diff --git a/pkg/operator/ceph/csi/predicate_test.go b/pkg/operator/ceph/csi/predicate_test.go new file mode 100644 index 000000000..af718e63d --- /dev/null +++ b/pkg/operator/ceph/csi/predicate_test.go @@ -0,0 +1,125 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_findCSIChange(t *testing.T) { + t.Run("no match", func(t *testing.T) { + var str = ` map[string]string{ + "CSI_FORCE_CEPHFS_KERNEL_CLIENT": "true", + "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false", + "ROOK_CSI_ENABLE_GRPC_METRICS": "true", + "ROOK_CSI_ENABLE_RBD": "true", + "ROOK_ENABLE_DISCOVERY_DAEMON": "false", +- "ROOK_LOG_LEVEL": "INFO", ++ "ROOK_LOG_LEVEL": "DEBUG", + "ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", + }` + b := findCSIChange(str) + assert.False(t, b) + }) + + t.Run("match on addition", func(t *testing.T) { + var str = ` map[string]string{ + "CSI_FORCE_CEPHFS_KERNEL_CLIENT": "true", + "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false", ++ "ROOK_CSI_ENABLE_CEPHFS": "false", + "ROOK_CSI_ENABLE_GRPC_METRICS": "true", + "ROOK_CSI_ENABLE_RBD": "true", + "ROOK_ENABLE_DISCOVERY_DAEMON": "false", +- "ROOK_LOG_LEVEL": "INFO", ++ "ROOK_LOG_LEVEL": "DEBUG", + "ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", + }` + b := findCSIChange(str) + assert.True(t, b) + }) + + t.Run("match on deletion", func(t *testing.T) { + var str = ` map[string]string{ + "CSI_FORCE_CEPHFS_KERNEL_CLIENT": "true", + "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false", +- "ROOK_CSI_ENABLE_CEPHFS": "true", + "ROOK_CSI_ENABLE_GRPC_METRICS": "true", + "ROOK_CSI_ENABLE_RBD": "true", + "ROOK_ENABLE_DISCOVERY_DAEMON": "false", +- "ROOK_LOG_LEVEL": "INFO", ++ "ROOK_LOG_LEVEL": "DEBUG", + "ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", + }` + b := findCSIChange(str) + assert.True(t, b) + }) + + t.Run("match on addition and deletion", func(t *testing.T) { + var str = ` map[string]string{ + "CSI_FORCE_CEPHFS_KERNEL_CLIENT": "true", + "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false", +- "ROOK_CSI_ENABLE_CEPHFS": "true", ++ "ROOK_CSI_ENABLE_CEPHFS": "false", + "ROOK_CSI_ENABLE_GRPC_METRICS": "true", + "ROOK_CSI_ENABLE_RBD": "true", + "ROOK_ENABLE_DISCOVERY_DAEMON": "false", +- "ROOK_LOG_LEVEL": "INFO", ++ "ROOK_LOG_LEVEL": "DEBUG", + "ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", + }` + b := findCSIChange(str) + assert.True(t, b) + }) + + t.Run("match with CSI_ name", func(t *testing.T) { + var str = ` map[string]string{ + "CSI_FORCE_CEPHFS_KERNEL_CLIENT": "true", + "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false", +- "CSI_PROVISIONER_TOLERATIONS": "true", ++ "CSI_PROVISIONER_TOLERATIONS": "false", + "ROOK_CSI_ENABLE_GRPC_METRICS": "true", + "ROOK_CSI_ENABLE_RBD": "true", + "ROOK_ENABLE_DISCOVERY_DAEMON": "false", +- "ROOK_LOG_LEVEL": "INFO", ++ "ROOK_LOG_LEVEL": "DEBUG", + "ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", + }` + b := findCSIChange(str) + assert.True(t, b) + }) + + t.Run("match with CSI_ and ROOK_CSI_ name", func(t *testing.T) { + var str = ` map[string]string{ + "CSI_FORCE_CEPHFS_KERNEL_CLIENT": "true", + "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION": "false", +- "CSI_PROVISIONER_TOLERATIONS": "true", ++ "CSI_PROVISIONER_TOLERATIONS": "false", +- "ROOK_CSI_ENABLE_CEPHFS": "true", ++ "ROOK_CSI_ENABLE_CEPHFS": "false", + "ROOK_CSI_ENABLE_GRPC_METRICS": "true", + "ROOK_CSI_ENABLE_RBD": "true", + "ROOK_ENABLE_DISCOVERY_DAEMON": "false", +- "ROOK_LOG_LEVEL": "INFO", ++ "ROOK_LOG_LEVEL": "DEBUG", + "ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", + }` + b := findCSIChange(str) + assert.True(t, b) + }) +} diff --git a/pkg/operator/ceph/csi/secrets.go b/pkg/operator/ceph/csi/secrets.go index 103560790..d2d5a45df 100644 --- a/pkg/operator/ceph/csi/secrets.go +++ b/pkg/operator/ceph/csi/secrets.go @@ -159,7 +159,7 @@ func createOrUpdateCSISecret(clusterInfo *client.ClusterInfo, csiRBDProvisionerS // Create Kubernetes Secret err = k.CreateSecret(s) if err != nil { - return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", secret, clusterInfo.Namespace) + return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace) } } diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index b798c8807..05979b846 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -22,11 +22,8 @@ import ( "fmt" "strconv" "strings" - "sync" "time" - rookclient "github.com/rook/rook/pkg/client/clientset/versioned" - controllerutil "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/k8sutil/cmdreporter" @@ -36,7 +33,6 @@ import ( k8scsi "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" - "k8s.io/client-go/kubernetes" ) type Param struct { @@ -92,7 +88,6 @@ var ( ConfigName = "rook-ceph-csi-config" ConfigKey = "csi-cluster-config-json" - csiLock sync.Mutex csiDriverobj csiDriver ) @@ -208,8 +203,7 @@ func validateCSIParam() error { return nil } -func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Interface, namespace string, ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error { - ctx := context.TODO() +func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error { var ( err error rbdPlugin, cephfsPlugin *apps.DaemonSet @@ -219,12 +213,12 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter tp := templateParam{ Param: CSIParam, - Namespace: namespace, + Namespace: r.opConfig.OperatorNamespace, } // if the user didn't specify a custom DriverNamePrefix use // the namespace (and a dot). if tp.DriverNamePrefix == "" { - tp.DriverNamePrefix = fmt.Sprintf("%s.", namespace) + tp.DriverNamePrefix = fmt.Sprintf("%s.", r.opConfig.OperatorNamespace) } CephFSDriverName = tp.DriverNamePrefix + "cephfs.csi.ceph.com" @@ -239,13 +233,13 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter // than maximum supported version for betav1 object.(unavailable in v1.22+) // Ignore if not found. if EnableRBD && ver.Minor <= kubeMaxVerForBeta1csiDriver { - err = beta1CsiDriver{}.deleteCSIDriverInfo(ctx, clientset, RBDDriverName) + err = beta1CsiDriver{}.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, RBDDriverName) if err != nil { logger.Errorf("failed to delete %q Driver Info. %v", RBDDriverName, err) } } if EnableCephFS && ver.Minor <= kubeMaxVerForBeta1csiDriver { - err = beta1CsiDriver{}.deleteCSIDriverInfo(ctx, clientset, CephFSDriverName) + err = beta1CsiDriver{}.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, CephFSDriverName) if err != nil { logger.Errorf("failed to delete %q Driver Info. %v", CephFSDriverName, err) } @@ -255,46 +249,37 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) // If not set or set to anything but "false", the kernel client will be enabled - kClient, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true") - if err != nil { - return errors.Wrap(err, "failed to load CSI_FORCE_CEPHFS_KERNEL_CLIENT setting") - } - if strings.EqualFold(kClient, "false") { + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") { tp.ForceCephFSKernelClient = "false" } else { tp.ForceCephFSKernelClient = "true" } + // parse GRPC and Liveness ports - tp.CephFSGRPCMetricsPort, err = getPortFromConfig(clientset, "CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) + tp.CephFSGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI CephFS GRPC metrics port.") } - tp.CephFSLivenessMetricsPort, err = getPortFromConfig(clientset, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) + tp.CephFSLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") } - tp.RBDGRPCMetricsPort, err = getPortFromConfig(clientset, "CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort) + tp.RBDGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI RBD GRPC metrics port.") } - tp.RBDLivenessMetricsPort, err = getPortFromConfig(clientset, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) + tp.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") } // default value `system-node-critical` is the highest available priority - tp.PluginPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") - if err != nil { - return errors.Wrap(err, "failed to load CSI_PLUGIN_PRIORITY_CLASSNAME setting") - } + tp.PluginPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") // default value `system-cluster-critical` is applied for some // critical pods in cluster but less priority than plugin pods - tp.ProvisionerPriorityClassName, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "") - if err != nil { - return errors.Wrap(err, "failed to load CSI_PROVISIONER_PRIORITY_CLASSNAME setting") - } + tp.ProvisionerPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "") // OMAP generator will be enabled by default // If AllowUnsupported is set to false and if CSI version is less than @@ -303,11 +288,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter tp.EnableOMAPGenerator = false } - enableOMAPGenerator, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_OMAP_GENERATOR", "false") - if err != nil { - return errors.Wrap(err, "failed to load CSI_ENABLE_OMAP_GENERATOR setting") - } - if strings.EqualFold(enableOMAPGenerator, "true") { + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_OMAP_GENERATOR", "false"), "true") { tp.EnableOMAPGenerator = true } @@ -316,45 +297,27 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter tp.EnableRBDSnapshotter = true tp.EnableCephFSSnapshotter = true } - enableRBDSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_RBD_SNAPSHOTTER", "true") - if err != nil { - return errors.Wrap(err, "failed to load CSI_ENABLE_RBD_SNAPSHOTTER setting") - } - if strings.EqualFold(enableRBDSnapshotter, "false") { + + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { tp.EnableRBDSnapshotter = false } - enableCephFSSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true") - if err != nil { - return errors.Wrap(err, "failed to load CSI_ENABLE_CEPHFS_SNAPSHOTTER setting") - } - if strings.EqualFold(enableCephFSSnapshotter, "false") { + + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") { tp.EnableCephFSSnapshotter = false } tp.EnableVolumeReplicationSideCar = false - enableVolumeReplicationSideCar, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_VOLUME_REPLICATION", "false") - if err != nil { - return errors.Wrap(err, "failed to load CSI_ENABLE_VOLUME_REPLICATION setting") - } - if strings.EqualFold(enableVolumeReplicationSideCar, "true") { + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_REPLICATION", "false"), "true") { tp.EnableVolumeReplicationSideCar = true } - updateStrategy, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate) - if err != nil { - return errors.Wrap(err, "failed to load CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY setting") - } - if strings.EqualFold(updateStrategy, onDelete) { + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { tp.CephFSPluginUpdateStrategy = onDelete } else { tp.CephFSPluginUpdateStrategy = rollingUpdate } - updateStrategy, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate) - if err != nil { - return errors.Wrap(err, "failed to load CSI_RBD_PLUGIN_UPDATE_STRATEGY setting") - } - if strings.EqualFold(updateStrategy, onDelete) { + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { tp.RBDPluginUpdateStrategy = onDelete } else { tp.RBDPluginUpdateStrategy = rollingUpdate @@ -362,13 +325,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter logger.Infof("Kubernetes version is %s.%s", ver.Major, ver.Minor) - tp.ResizerImage, err = k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) - if err != nil { - return errors.Wrap(err, "failed to load ROOK_CSI_RESIZER_IMAGE setting") - } - if tp.ResizerImage == "" { - tp.ResizerImage = DefaultResizerImage - } + tp.ResizerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) if ver.Major == KubeMinMajor && ver.Minor < kubeMinVerForFilesystemRestore { logger.Warning("CSI Filesystem volume expansion requires Kubernetes version >=1.15.0") @@ -377,11 +334,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter logger.Warning("CSI Block volume expansion requires Kubernetes version >=1.16.0") } - logLevel, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_LOG_LEVEL", "") - if err != nil { - // logging a warning and intentionally continuing with the default log level - logger.Warningf("failed to load CSI_LOG_LEVEL. Defaulting to %d. %v", defaultLogLevel, err) - } + logLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LOG_LEVEL", "") tp.LogLevel = defaultLogLevel if logLevel != "" { l, err := strconv.ParseUint(logLevel, 10, 8) @@ -393,7 +346,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter } tp.ProvisionerReplicas = 2 - nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{}) if err == nil { if len(nodes.Items) == 1 { tp.ProvisionerReplicas = 1 @@ -417,7 +370,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if err != nil { return errors.Wrap(err, "failed to load rbd plugin service template") } - rbdService.Namespace = namespace + rbdService.Namespace = r.opConfig.OperatorNamespace } if EnableCephFS { cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath, tp) @@ -434,50 +387,50 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if err != nil { return errors.Wrap(err, "failed to load cephfs plugin service template") } - cephfsService.Namespace = namespace + cephfsService.Namespace = r.opConfig.OperatorNamespace } // get common provisioner tolerations and node affinity - provisionerTolerations := getToleration(clientset, provisionerTolerationsEnv, []corev1.Toleration{}) - provisionerNodeAffinity := getNodeAffinity(clientset, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) + provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}) + provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) // get common plugin tolerations and node affinity - pluginTolerations := getToleration(clientset, pluginTolerationsEnv, []corev1.Toleration{}) - pluginNodeAffinity := getNodeAffinity(clientset, pluginNodeAffinityEnv, &corev1.NodeAffinity{}) + pluginTolerations := getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []corev1.Toleration{}) + pluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &corev1.NodeAffinity{}) if rbdPlugin != nil { // get RBD plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdPluginTolerations := getToleration(clientset, rbdPluginTolerationsEnv, pluginTolerations) - rbdPluginNodeAffinity := getNodeAffinity(clientset, rbdPluginNodeAffinityEnv, pluginNodeAffinity) + rbdPluginTolerations := getToleration(r.opConfig.Parameters, rbdPluginTolerationsEnv, pluginTolerations) + rbdPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdPluginNodeAffinityEnv, pluginNodeAffinity) // apply RBD plugin tolerations and node affinity applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations) // apply resource request and limit to rbdplugin containers - applyResourcesToContainers(clientset, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) + applyResourcesToContainers(r.opConfig.Parameters, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name) } - multusApplied, err := applyCephClusterNetworkConfig(ctx, &rbdPlugin.Spec.Template.ObjectMeta, rookclientset) + multusApplied, err := r.applyCephClusterNetworkConfig(r.opManagerContext, &rbdPlugin.Spec.Template.ObjectMeta) if err != nil { - return errors.Wrapf(err, "failed to apply network config to rbd plugin daemonset: %+v", rbdPlugin) + return errors.Wrapf(err, "failed to apply network config to rbd plugin daemonset %q", rbdPlugin.Name) } if multusApplied { rbdPlugin.Spec.Template.Spec.HostNetwork = false } - err = k8sutil.CreateDaemonSet(csiRBDPlugin, namespace, clientset, rbdPlugin) + err = k8sutil.CreateDaemonSet(csiRBDPlugin, r.opConfig.OperatorNamespace, r.context.Clientset, rbdPlugin) if err != nil { - return errors.Wrapf(err, "failed to start rbdplugin daemonset: %+v", rbdPlugin) + return errors.Wrapf(err, "failed to start rbdplugin daemonset %q", rbdPlugin.Name) } k8sutil.AddRookVersionLabelToDaemonSet(rbdPlugin) } if rbdProvisionerDeployment != nil { // get RBD provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdProvisionerTolerations := getToleration(clientset, rbdProvisionerTolerationsEnv, provisionerTolerations) - rbdProvisionerNodeAffinity := getNodeAffinity(clientset, rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) + rbdProvisionerTolerations := getToleration(r.opConfig.Parameters, rbdProvisionerTolerationsEnv, provisionerTolerations) + rbdProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply RBD provisioner tolerations and node affinity applyToPodSpec(&rbdProvisionerDeployment.Spec.Template.Spec, rbdProvisionerNodeAffinity, rbdProvisionerTolerations) // apply resource request and limit to rbd provisioner containers - applyResourcesToContainers(clientset, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(r.opConfig.Parameters, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd provisioner deployment %q", rbdProvisionerDeployment.Name) @@ -488,65 +441,65 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter Type: apps.RecreateDeploymentStrategyType, } - _, err = applyCephClusterNetworkConfig(ctx, &rbdProvisionerDeployment.Spec.Template.ObjectMeta, rookclientset) + _, err = r.applyCephClusterNetworkConfig(r.opManagerContext, &rbdProvisionerDeployment.Spec.Template.ObjectMeta) if err != nil { - return errors.Wrapf(err, "failed to apply network config to rbd plugin provisioner deployment: %+v", rbdProvisionerDeployment) + return errors.Wrapf(err, "failed to apply network config to rbd plugin provisioner deployment %q", rbdProvisionerDeployment.Name) } - _, err = k8sutil.CreateOrUpdateDeployment(clientset, rbdProvisionerDeployment) + _, err = k8sutil.CreateOrUpdateDeployment(r.context.Clientset, rbdProvisionerDeployment) if err != nil { - return errors.Wrapf(err, "failed to start rbd provisioner deployment: %+v", rbdProvisionerDeployment) + return errors.Wrapf(err, "failed to start rbd provisioner deployment %q", rbdProvisionerDeployment.Name) } k8sutil.AddRookVersionLabelToDeployment(rbdProvisionerDeployment) logger.Info("successfully started CSI Ceph RBD driver") } if rbdService != nil { - rbdService.Namespace = namespace + rbdService.Namespace = r.opConfig.OperatorNamespace err = ownerInfo.SetControllerReference(rbdService) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd service %q", rbdService) } - _, err = k8sutil.CreateOrUpdateService(clientset, namespace, rbdService) + _, err = k8sutil.CreateOrUpdateService(r.context.Clientset, r.opConfig.OperatorNamespace, rbdService) if err != nil { - return errors.Wrapf(err, "failed to create rbd service: %+v", rbdService) + return errors.Wrapf(err, "failed to create rbd service %q", rbdService.Name) } } if cephfsPlugin != nil { // get CephFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSPluginTolerations := getToleration(clientset, cephFSPluginTolerationsEnv, pluginTolerations) - cephFSPluginNodeAffinity := getNodeAffinity(clientset, cephFSPluginNodeAffinityEnv, pluginNodeAffinity) + cephFSPluginTolerations := getToleration(r.opConfig.Parameters, cephFSPluginTolerationsEnv, pluginTolerations) + cephFSPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSPluginNodeAffinityEnv, pluginNodeAffinity) // apply CephFS plugin tolerations and node affinity applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations) // apply resource request and limit to cephfs plugin containers - applyResourcesToContainers(clientset, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) + applyResourcesToContainers(r.opConfig.Parameters, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name) } - multusApplied, err := applyCephClusterNetworkConfig(ctx, &cephfsPlugin.Spec.Template.ObjectMeta, rookclientset) + multusApplied, err := r.applyCephClusterNetworkConfig(r.opManagerContext, &cephfsPlugin.Spec.Template.ObjectMeta) if err != nil { - return errors.Wrapf(err, "failed to apply network config to cephfs plugin daemonset: %+v", cephfsPlugin) + return errors.Wrapf(err, "failed to apply network config to cephfs plugin daemonset %q", cephfsPlugin.Name) } if multusApplied { cephfsPlugin.Spec.Template.Spec.HostNetwork = false } - err = k8sutil.CreateDaemonSet(csiCephFSPlugin, namespace, clientset, cephfsPlugin) + err = k8sutil.CreateDaemonSet(csiCephFSPlugin, r.opConfig.OperatorNamespace, r.context.Clientset, cephfsPlugin) if err != nil { - return errors.Wrapf(err, "failed to start cephfs plugin daemonset: %+v", cephfsPlugin) + return errors.Wrapf(err, "failed to start cephfs plugin daemonset %q", cephfsPlugin.Name) } k8sutil.AddRookVersionLabelToDaemonSet(cephfsPlugin) } if cephfsProvisionerDeployment != nil { // get CephFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSProvisionerTolerations := getToleration(clientset, cephFSProvisionerTolerationsEnv, provisionerTolerations) - cephFSProvisionerNodeAffinity := getNodeAffinity(clientset, cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) + cephFSProvisionerTolerations := getToleration(r.opConfig.Parameters, cephFSProvisionerTolerationsEnv, provisionerTolerations) + cephFSProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply CephFS provisioner tolerations and node affinity applyToPodSpec(&cephfsProvisionerDeployment.Spec.Template.Spec, cephFSProvisionerNodeAffinity, cephFSProvisionerTolerations) // get resource details for cephfs provisioner // apply resource request and limit to cephfs provisioner containers - applyResourcesToContainers(clientset, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(r.opConfig.Parameters, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) @@ -557,13 +510,13 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter Type: apps.RecreateDeploymentStrategyType, } - _, err = applyCephClusterNetworkConfig(ctx, &cephfsProvisionerDeployment.Spec.Template.ObjectMeta, rookclientset) + _, err = r.applyCephClusterNetworkConfig(r.opManagerContext, &cephfsProvisionerDeployment.Spec.Template.ObjectMeta) if err != nil { - return errors.Wrapf(err, "failed to apply network config to cephfs plugin provisioner deployment: %+v", cephfsProvisionerDeployment) + return errors.Wrapf(err, "failed to apply network config to cephfs plugin provisioner deployment %q", cephfsProvisionerDeployment.Name) } - _, err = k8sutil.CreateOrUpdateDeployment(clientset, cephfsProvisionerDeployment) + _, err = k8sutil.CreateOrUpdateDeployment(r.context.Clientset, cephfsProvisionerDeployment) if err != nil { - return errors.Wrapf(err, "failed to start cephfs provisioner deployment: %+v", cephfsProvisionerDeployment) + return errors.Wrapf(err, "failed to start cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) } k8sutil.AddRookVersionLabelToDeployment(cephfsProvisionerDeployment) logger.Info("successfully started CSI CephFS driver") @@ -573,31 +526,20 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs service %q", cephfsService) } - _, err = k8sutil.CreateOrUpdateService(clientset, namespace, cephfsService) + _, err = k8sutil.CreateOrUpdateService(r.context.Clientset, r.opConfig.OperatorNamespace, cephfsService) if err != nil { - return errors.Wrapf(err, "failed to create cephfs service: %+v", cephfsService) + return errors.Wrapf(err, "failed to create cephfs service %q", cephfsService.Name) } } if EnableRBD { - fsGroupPolicyForRBD, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)) - if err != nil { - // logging a warning and intentionally continuing with the default log level - logger.Warningf("failed to parse CSI_RBD_FSGROUPPOLICY. Defaulting to %q. %v", k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy, err) - } - err = csiDriverobj.createCSIDriverInfo(ctx, clientset, RBDDriverName, fsGroupPolicyForRBD) + err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, RBDDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy))) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName) } } if EnableCephFS { - fsGroupPolicyForCephFS, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)) - if err != nil { - // logging a warning and intentionally continuing with the default - // log level - logger.Warningf("failed to parse CSI_CEPHFS_FSGROUPPOLICY. Defaulting to %q. %v", k8scsi.NoneFSGroupPolicy, err) - } - err = csiDriverobj.createCSIDriverInfo(ctx, clientset, CephFSDriverName, fsGroupPolicyForCephFS) + err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, CephFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy))) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName) } @@ -606,10 +548,10 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter return nil } -func stopDrivers(clientset kubernetes.Interface, namespace string, ver *version.Info) { +func (r *ReconcileCSI) stopDrivers(ver *version.Info) { if !EnableRBD { logger.Info("CSI Ceph RBD driver disabled") - succeeded := deleteCSIDriverResources(clientset, ver, namespace, csiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName) + succeeded := r.deleteCSIDriverResources(ver, csiRBDPlugin, csiRBDProvisioner, "csi-rbdplugin-metrics", RBDDriverName) if succeeded { logger.Info("successfully removed CSI Ceph RBD driver") } else { @@ -619,7 +561,7 @@ func stopDrivers(clientset kubernetes.Interface, namespace string, ver *version. if !EnableCephFS { logger.Info("CSI CephFS driver disabled") - succeeded := deleteCSIDriverResources(clientset, ver, namespace, csiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName) + succeeded := r.deleteCSIDriverResources(ver, csiCephFSPlugin, csiCephFSProvisioner, "csi-cephfsplugin-metrics", CephFSDriverName) if succeeded { logger.Info("successfully removed CSI CephFS driver") } else { @@ -628,33 +570,31 @@ func stopDrivers(clientset kubernetes.Interface, namespace string, ver *version. } } -func deleteCSIDriverResources( - clientset kubernetes.Interface, ver *version.Info, namespace, daemonset, deployment, service, driverName string) bool { - ctx := context.TODO() +func (r *ReconcileCSI) deleteCSIDriverResources(ver *version.Info, daemonset, deployment, service, driverName string) bool { succeeded := true csiDriverobj = beta1CsiDriver{} if ver.Major > KubeMinMajor || ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForV1csiDriver { csiDriverobj = v1CsiDriver{} } - err := k8sutil.DeleteDaemonset(clientset, namespace, daemonset) + err := k8sutil.DeleteDaemonset(r.context.Clientset, r.opConfig.OperatorNamespace, daemonset) if err != nil { logger.Errorf("failed to delete the %q. %v", daemonset, err) succeeded = false } - err = k8sutil.DeleteDeployment(clientset, namespace, deployment) + err = k8sutil.DeleteDeployment(r.context.Clientset, r.opConfig.OperatorNamespace, deployment) if err != nil { logger.Errorf("failed to delete the %q. %v", deployment, err) succeeded = false } - err = k8sutil.DeleteService(clientset, namespace, service) + err = k8sutil.DeleteService(r.context.Clientset, r.opConfig.OperatorNamespace, service) if err != nil { logger.Errorf("failed to delete the %q. %v", service, err) succeeded = false } - err = csiDriverobj.deleteCSIDriverInfo(ctx, clientset, driverName) + err = csiDriverobj.deleteCSIDriverInfo(r.opManagerContext, r.context.Clientset, driverName) if err != nil { logger.Errorf("failed to delete %q Driver Info. %v", driverName, err) succeeded = false @@ -662,9 +602,9 @@ func deleteCSIDriverResources( return succeeded } -func applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.ObjectMeta, rookclientset rookclient.Interface) (bool, error) { +func (r *ReconcileCSI) applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.ObjectMeta) (bool, error) { var isMultusApplied bool - cephClusters, err := rookclientset.CephV1().CephClusters(objectMeta.Namespace).List(ctx, metav1.ListOptions{}) + cephClusters, err := r.context.RookClientset.CephV1().CephClusters(objectMeta.Namespace).List(ctx, metav1.ListOptions{}) if err != nil { return false, errors.Wrap(err, "failed to find CephClusters") } @@ -682,27 +622,32 @@ func applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.Objec } // ValidateCSIVersion checks if the configured ceph-csi image is supported -func validateCSIVersion(clientset kubernetes.Interface, namespace, rookImage, serviceAccountName string, ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) { +func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) { timeout := 15 * time.Minute logger.Infof("detecting the ceph csi image version for image %q", CSIParam.CSIPluginImage) versionReporter, err := cmdreporter.New( - clientset, + r.context.Clientset, ownerInfo, - detectCSIVersionName, detectCSIVersionName, namespace, - []string{"cephcsi"}, []string{"--version"}, - rookImage, CSIParam.CSIPluginImage) + detectCSIVersionName, + detectCSIVersionName, + r.opConfig.OperatorNamespace, + []string{"cephcsi"}, + []string{"--version"}, + r.opConfig.Image, + CSIParam.CSIPluginImage, + ) if err != nil { return nil, errors.Wrap(err, "failed to set up ceph CSI version job") } job := versionReporter.Job() - job.Spec.Template.Spec.ServiceAccountName = serviceAccountName + job.Spec.Template.Spec.ServiceAccountName = r.opConfig.ServiceAccount // Apply csi provisioner toleration for csi version check job - job.Spec.Template.Spec.Tolerations = getToleration(clientset, provisionerTolerationsEnv, []corev1.Toleration{}) + job.Spec.Template.Spec.Tolerations = getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}) stdout, _, retcode, err := versionReporter.Run(timeout) if err != nil { return nil, errors.Wrap(err, "failed to complete ceph CSI version job") diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go deleted file mode 100644 index fbd035750..000000000 --- a/pkg/operator/ceph/csi/spec_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2019 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "testing" - - rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake" - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/test" - - "github.com/stretchr/testify/assert" -) - -func TestStartCSI(t *testing.T) { - CSIParam = Param{ - CSIPluginImage: "image", - RegistrarImage: "image", - ProvisionerImage: "image", - AttacherImage: "image", - SnapshotterImage: "image", - } - clientset := test.New(t, 3) - context := &clusterd.Context{ - Clientset: clientset, - RookClientset: rookfake.NewSimpleClientset(), - } - serverVersion, err := clientset.Discovery().ServerVersion() - if err != nil { - assert.Nil(t, err) - } - AllowUnsupported = true - err = startDrivers(context.Clientset, context.RookClientset, "ns", serverVersion, nil, nil) - assert.Nil(t, err) -} diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index ef5fd4ff8..6fa5c5820 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -23,13 +23,11 @@ import ( "text/template" "github.com/pkg/errors" - "github.com/rook/rook/pkg/operator/ceph/controller" k8sutil "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/kubernetes" ) func loadTemplate(name, templateData string, p templateParam) ([]byte, error) { @@ -85,8 +83,8 @@ func templateToDeployment(name, templateData string, p templateParam) (*apps.Dep return &dep, nil } -func applyResourcesToContainers(clientset kubernetes.Interface, key string, podspec *corev1.PodSpec) { - resource := getComputeResource(clientset, key) +func applyResourcesToContainers(opConfig map[string]string, key string, podspec *corev1.PodSpec) { + resource := getComputeResource(opConfig, key) if len(resource) > 0 { for i, c := range podspec.Containers { for _, r := range resource { @@ -98,19 +96,13 @@ func applyResourcesToContainers(clientset kubernetes.Interface, key string, pods } } -func getComputeResource(clientset kubernetes.Interface, key string) []k8sutil.ContainerResource { +func getComputeResource(opConfig map[string]string, key string) []k8sutil.ContainerResource { // Add Resource list if any resource := []k8sutil.ContainerResource{} resourceRaw := "" var err error - resourceRaw, err = k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, key, "") - - if err != nil { - logger.Warningf("resource requirement for %q will not be applied. %v", key, err) - } - - if resourceRaw != "" { + if k8sutil.GetValue(opConfig, key, "") != "" { resource, err = k8sutil.YamlToContainerResource(resourceRaw) if err != nil { logger.Warningf("failed to parse %q. %v", resourceRaw, err) @@ -119,13 +111,9 @@ func getComputeResource(clientset kubernetes.Interface, key string) []k8sutil.Co return resource } -func getToleration(clientset kubernetes.Interface, tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { +func getToleration(opConfig map[string]string, tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { // Add toleration if any, otherwise return defaultTolerations - tolerationsRaw, err := k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, tolerationsName, "") - if err != nil { - logger.Warningf("failed to read %q. %v", tolerationsName, err) - return defaultTolerations - } + tolerationsRaw := k8sutil.GetValue(opConfig, tolerationsName, "") if tolerationsRaw == "" { return defaultTolerations } @@ -146,13 +134,9 @@ func getToleration(clientset kubernetes.Interface, tolerationsName string, defau return tolerations } -func getNodeAffinity(clientset kubernetes.Interface, nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { +func getNodeAffinity(opConfig map[string]string, nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { // Add NodeAffinity if any, otherwise return defaultNodeAffinity - nodeAffinity, err := k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, nodeAffinityName, "") - if err != nil { - logger.Warningf("failed to read %q. %v", nodeAffinityName, err) - return defaultNodeAffinity - } + nodeAffinity := k8sutil.GetValue(opConfig, nodeAffinityName, "") if nodeAffinity == "" { return defaultNodeAffinity } @@ -171,12 +155,9 @@ func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Tole } } -func getPortFromConfig(clientset kubernetes.Interface, env string, defaultPort uint16) (uint16, error) { - port, err := k8sutil.GetOperatorSetting(clientset, controller.OperatorSettingConfigMapName, env, strconv.Itoa(int(defaultPort))) - if err != nil { - return defaultPort, errors.Wrapf(err, "failed to load value for %q.", env) - } - if strings.TrimSpace(port) == "" { +func getPortFromConfig(data map[string]string, env string, defaultPort uint16) (uint16, error) { + port := k8sutil.GetValue(data, env, strconv.Itoa(int(defaultPort))) + if strings.TrimSpace(k8sutil.GetValue(data, env, strconv.Itoa(int(defaultPort)))) == "" { return defaultPort, nil } p, err := strconv.ParseUint(port, 10, 64) diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index 4c24797a1..a5e94ba1e 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/client-go/kubernetes/fake" ) func TestDaemonSetTemplate(t *testing.T) { @@ -43,21 +42,20 @@ func TestDeploymentTemplate(t *testing.T) { assert.Nil(t, err) } -func Test_getPortFromConfig(t *testing.T) { - k8s := fake.NewSimpleClientset() - +func TestGetPortFromConfig(t *testing.T) { var key = "TEST_CSI_PORT_ENV" var defaultPort uint16 = 8000 + data := map[string]string{} // empty env variable - port, err := getPortFromConfig(k8s, key, defaultPort) + port, err := getPortFromConfig(data, key, defaultPort) assert.Nil(t, err) assert.Equal(t, port, defaultPort) // valid port is set in env err = os.Setenv(key, "9000") assert.Nil(t, err) - port, err = getPortFromConfig(k8s, key, defaultPort) + port, err = getPortFromConfig(data, key, defaultPort) assert.Nil(t, err) assert.Equal(t, port, uint16(9000)) @@ -66,7 +64,7 @@ func Test_getPortFromConfig(t *testing.T) { // higher port value is set in env err = os.Setenv(key, "65536") assert.Nil(t, err) - port, err = getPortFromConfig(k8s, key, defaultPort) + port, err = getPortFromConfig(data, key, defaultPort) assert.Error(t, err) assert.Equal(t, port, defaultPort) @@ -75,7 +73,7 @@ func Test_getPortFromConfig(t *testing.T) { // negative port is set in env err = os.Setenv(key, "-1") assert.Nil(t, err) - port, err = getPortFromConfig(k8s, key, defaultPort) + port, err = getPortFromConfig(data, key, defaultPort) assert.Error(t, err) assert.Equal(t, port, defaultPort) diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go index 6c5a9c182..36d2a2564 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go @@ -227,6 +227,7 @@ func TestGetOSDFailureDomains(t *testing.T) { } r := getFakeReconciler(t, objs...) clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, err := r.getOSDFailureDomains(clusterInfo, request, "zone") assert.NoError(t, err) @@ -262,6 +263,7 @@ func TestGetOSDFailureDomainsError(t *testing.T) { r := getFakeReconciler(t, cephCluster, &corev1.ConfigMap{}, tc.osds[1].DeepCopy(), tc.osds[2].DeepCopy(), osd) clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} allfailureDomains, nodeDrainFailureDomains, osdDownFailureDomains, err := r.getOSDFailureDomains(clusterInfo, request, "zone") assert.Error(t, err) @@ -344,6 +346,7 @@ func TestReconcilePDBForOSD(t *testing.T) { t.Run(tc.name, func(t *testing.T) { r := getFakeReconciler(t, cephCluster, tc.configMap) clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { @@ -400,6 +403,7 @@ func TestPGHealthcheckTimeout(t *testing.T) { pdbConfig := fakePDBConfigMap("") r := getFakeReconciler(t, cephCluster, pdbConfig) clusterInfo := getFakeClusterInfo() + clusterInfo.Context = context.TODO() request := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: namespace}} executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { diff --git a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go index 9735c77db..cf3194558 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/reconcile.go @@ -106,6 +106,7 @@ func (r *ReconcileClusterDisruption) reconcile(request reconcile.Request) (recon logger.Infof("clusterName is not known for namespace %q", request.Namespace) return reconcile.Result{Requeue: true, RequeueAfter: 5 * time.Second}, errors.New("clusterName for this namespace not yet known") } + clusterInfo.Context = r.context.OpManagerContext // ensure that the cluster name is populated if request.Name == "" { diff --git a/pkg/operator/ceph/disruption/controllerconfig/context.go b/pkg/operator/ceph/disruption/controllerconfig/context.go index 97df5f7d5..9810fdcc6 100644 --- a/pkg/operator/ceph/disruption/controllerconfig/context.go +++ b/pkg/operator/ceph/disruption/controllerconfig/context.go @@ -17,6 +17,7 @@ limitations under the License. package controllerconfig import ( + "context" "sync" "github.com/rook/rook/pkg/clusterd" @@ -25,9 +26,8 @@ import ( // Context passed to the controller when associating it with the manager. type Context struct { ClusterdContext *clusterd.Context - RookImage string - OperatorNamespace string ReconcileCanaries *LockingBool + OpManagerContext context.Context } // LockingBool is a bool coupled with a sync.Mutex diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index 5825a9ec8..98629a65d 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -29,7 +29,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - opconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/file/mirror" "github.com/rook/rook/pkg/operator/k8sutil" @@ -67,34 +67,41 @@ var controllerTypeMeta = metav1.TypeMeta{ APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } +var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion + // ReconcileCephFilesystem reconciles a CephFilesystem object type ReconcileCephFilesystem struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - cephClusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo - fsChannels map[string]*fsHealth + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + cephClusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + fsContexts map[string]*fsHealth + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } type fsHealth struct { - stopChan chan struct{} - monitoringRunning bool + internalCtx context.Context + internalCancel context.CancelFunc + started bool } // Add creates a new CephFilesystem Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileCephFilesystem{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, - fsChannels: make(map[string]*fsHealth), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + fsContexts: make(map[string]*fsHealth), + opManagerContext: opManagerContext, + opConfig: opConfig, } } @@ -130,18 +137,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } - // Watch for CephCluster Spec changes that we want to propagate to us - err = c.Watch(&source.Kind{Type: &cephv1.CephCluster{ - TypeMeta: metav1.TypeMeta{ - Kind: opcontroller.ClusterResource.Kind, - APIVersion: opcontroller.ClusterResource.APIVersion, - }, - }, - }, handler.EnqueueRequestsFromMapFunc(handlerFunc), opcontroller.WatchCephClusterPredicate()) - if err != nil { - return err - } - // Watch for ConfigMap "rook-ceph-mon-endpoints" update and reconcile, which will reconcile update the bootstrap peer token err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: corev1.SchemeGroupVersion.String()}}}, handler.EnqueueRequestsFromMapFunc(handlerFunc), mon.PredicateMonEndpointChanges()) if err != nil { @@ -168,7 +163,7 @@ func (r *ReconcileCephFilesystem) Reconcile(context context.Context, request rec func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the cephFilesystem instance cephFilesystem := &cephv1.CephFilesystem{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephFilesystem) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephFilesystem) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephFilesystem resource not found. Ignoring since object must be deleted.") @@ -186,7 +181,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil // The CR was just created, initializing status fields if cephFilesystem.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus, nil) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus, nil) } // Make sure a CephCluster is present otherwise do nothing @@ -212,36 +207,33 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } r.cephClusterSpec = &cephCluster.Spec - // Initialize the channel, it allows us to track multiple CephFilesystems in the same namespace - _, fsChannelExists := r.fsChannels[fsChannelKeyName(cephFilesystem)] - if !fsChannelExists { - r.fsChannels[fsChannelKeyName(cephFilesystem)] = &fsHealth{ - stopChan: make(chan struct{}), - monitoringRunning: false, + // Initialize the contexts, they allow us to track multiple CephFilesystems in the same namespace + _, fsContextsExists := r.fsContexts[fsChannelKeyName(cephFilesystem)] + if !fsContextsExists { + internalCtx, internalCancel := context.WithCancel(r.opManagerContext) + r.fsContexts[fsChannelKeyName(cephFilesystem)] = &fsHealth{ + internalCtx: internalCtx, + internalCancel: internalCancel, } } // Populate clusterInfo // Always populate it during each reconcile - clusterInfo, _, _, err := mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } r.clusterInfo = clusterInfo - // Populate CephVersion - currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil - } - return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) - } - r.clusterInfo.CephVersion = currentCephVersion - // DELETE: the CR was deleted if !cephFilesystem.GetDeletionTimestamp().IsZero() { + runningCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, clusterInfo, config.MonType) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", config.MonType) + } + r.clusterInfo.CephVersion = runningCephVersion + + // Detect against running version only logger.Debugf("deleting filesystem %q", cephFilesystem.Name) err = r.reconcileDeleteFilesystem(cephFilesystem) if err != nil { @@ -249,12 +241,12 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } // If the ceph fs still in the map, we must remove it during CR deletion - if fsChannelExists { - // Close the channel to stop the mirroring status - close(r.fsChannels[fsChannelKeyName(cephFilesystem)].stopChan) + if fsContextsExists { + // Cancel the context to stop the mirroring status + r.fsContexts[fsChannelKeyName(cephFilesystem)].internalCancel() // Remove ceph fs from the map - delete(r.fsChannels, fsChannelKeyName(cephFilesystem)) + delete(r.fsContexts, fsChannelKeyName(cephFilesystem)) } // Remove finalizer @@ -267,6 +259,34 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil return reconcile.Result{}, nil } + // Detect desired CephCluster version + runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion( + r.opConfig.Image, + cephFilesystem.Namespace, + controllerName, + k8sutil.NewOwnerInfo(cephFilesystem, r.scheme), + r.context, + r.cephClusterSpec, + r.clusterInfo, + ) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + return reconcile.Result{}, errors.Wrap(err, "failed to detect running and desired ceph version") + } + r.clusterInfo.CephVersion = *runningCephVersion + + // If the version of the Ceph monitor differs from the CephCluster CR image version we assume + // the cluster is being upgraded. So the controller will just wait for the upgrade to finish and + // then versions should match. Obviously using the cmd reporter job adds up to the deployment time + if !reflect.DeepEqual(runningCephVersion, desiredCephVersion) { + // Upgrade is in progress, let's wait for the mons to be done + return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, + opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion) + } + // validate the filesystem settings if err := validateFilesystem(r.context, r.clusterInfo, r.cephClusterSpec, cephFilesystem); err != nil { if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { @@ -280,7 +300,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil logger.Debug("reconciling ceph filesystem store deployments") reconcileResponse, err = r.reconcileCreateFilesystem(cephFilesystem) if err != nil { - updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure, nil) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure, nil) return reconcileResponse, err } @@ -306,7 +326,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil logger.Info("reconciling create cephfs-mirror peer configuration") reconcileResponse, err = opcontroller.CreateBootstrapPeerSecret(r.context, r.clusterInfo, cephFilesystem, k8sutil.NewOwnerInfo(cephFilesystem, r.scheme)) if err != nil { - updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure, nil) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure, nil) return reconcileResponse, errors.Wrapf(err, "failed to create cephfs-mirror bootstrap peer for filesystem %q.", cephFilesystem.Name) } @@ -317,18 +337,18 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, opcontroller.GenerateStatusInfo(cephFilesystem)) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, opcontroller.GenerateStatusInfo(cephFilesystem)) statusUpdated = true // Run go routine check for mirroring status if !cephFilesystem.Spec.StatusCheck.Mirror.Disabled { // Start monitoring cephfs-mirror status - if r.fsChannels[fsChannelKeyName(cephFilesystem)].monitoringRunning { + if r.fsContexts[fsChannelKeyName(cephFilesystem)].started { logger.Debug("ceph filesystem mirror status monitoring go routine already running!") } else { checker := newMirrorChecker(r.context, r.client, r.clusterInfo, request.NamespacedName, &cephFilesystem.Spec, cephFilesystem.Name) - r.fsChannels[fsChannelKeyName(cephFilesystem)].monitoringRunning = true - go checker.checkMirroring(r.fsChannels[fsChannelKeyName(cephFilesystem)].stopChan) + go checker.checkMirroring(r.fsContexts[fsChannelKeyName(cephFilesystem)].internalCtx) + r.fsContexts[fsChannelKeyName(cephFilesystem)].started = true } } } @@ -336,7 +356,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } if !statusUpdated { // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, nil) + r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, nil) } // Return and do not requeue @@ -426,12 +446,11 @@ func (r *ReconcileCephFilesystem) reconcileAddBoostrapPeer(cephFilesystem *cephv if cephFilesystem.Spec.Mirroring.Peers == nil { return nil } - ctx := context.TODO() // List all the peers secret, we can have more than one peer we might want to configure // For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring for _, peerSecret := range cephFilesystem.Spec.Mirroring.Peers.SecretNames { logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret) - s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(ctx, peerSecret, metav1.GetOptions{}) + s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{}) // We don't care about IsNotFound here, we still need to fail if err != nil { return errors.Wrapf(err, "failed to fetch kubernetes secret %q fs-mirror bootstrap peer", peerSecret) diff --git a/pkg/operator/ceph/file/controller_test.go b/pkg/operator/ceph/file/controller_test.go index 1d1570a04..0040df44c 100644 --- a/pkg/operator/ceph/file/controller_test.go +++ b/pkg/operator/ceph/file/controller_test.go @@ -27,6 +27,8 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -152,11 +154,10 @@ func TestCephFilesystemController(t *testing.T) { capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - // - // TEST 1 SETUP - // - // FAILURE because no CephCluster - // + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Pacific, &version.Pacific, nil + } + // A Pool resource with metadata and spec. fs := &cephv1.CephFilesystem{ ObjectMeta: metav1.ObjectMeta{ @@ -203,7 +204,7 @@ func TestCephFilesystemController(t *testing.T) { // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() // Create a ReconcileCephFilesystem object with the scheme and fake client. - r := &ReconcileCephFilesystem{client: cl, scheme: s, context: c, fsChannels: make(map[string]*fsHealth)} + r := &ReconcileCephFilesystem{client: cl, scheme: s, context: c, fsContexts: make(map[string]*fsHealth), opManagerContext: context.TODO()} // Mock request to simulate Reconcile() being called on an event for a // watched resource . @@ -213,17 +214,6 @@ func TestCephFilesystemController(t *testing.T) { Namespace: namespace, }, } - logger.Info("STARTING PHASE 1") - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 1 DONE") - - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // cephCluster := &cephv1.CephCluster{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, @@ -236,75 +226,77 @@ func TestCephFilesystemController(t *testing.T) { }, }, } - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileCephFilesystem object with the scheme and fake client. - r = &ReconcileCephFilesystem{client: cl, scheme: s, context: c} - logger.Info("STARTING PHASE 2") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 2 DONE") - // - // TEST 3: - // - // SUCCESS! The CephCluster is ready - // + t.Run("error - no ceph cluster", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) + t.Run("error - ceph cluster not ready", func(t *testing.T) { + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileCephFilesystem object with the scheme and fake client. + r = &ReconcileCephFilesystem{client: cl, scheme: s, context: c, opManagerContext: context.TODO()} + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + logger.Info("PHASE 2 DONE") + }) - // Add ready status to the CephCluster - cephCluster.Status.Phase = k8sutil.ReadyStatus - cephCluster.Status.CephStatus.Health = "HEALTH_OK" + t.Run("success - ceph cluster ready and mds are running", func(t *testing.T) { + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Add ready status to the CephCluster + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" - executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "status" { - return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil - } - if args[0] == "fs" && args[1] == "get" { - return fsGet, nil - } - if args[0] == "auth" && args[1] == "get-or-create-key" { - return mdsCephAuthGetOrCreateKey, nil - } - if args[0] == "versions" { - return dummyVersionsRaw, nil - } - return "", nil - }, - } - c.Executor = executor + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileCephFilesystem object with the scheme and fake client. - r = &ReconcileCephFilesystem{client: cl, scheme: s, context: c, fsChannels: make(map[string]*fsHealth)} + executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "fs" && args[1] == "get" { + return fsGet, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return mdsCephAuthGetOrCreateKey, nil + } + if args[0] == "versions" { + return dummyVersionsRaw, nil + } + return "", nil + }, + } + c.Executor = executor - logger.Info("STARTING PHASE 3") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, fs) - assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionType("Ready"), fs.Status.Phase, fs) - logger.Info("PHASE 3 DONE") + // Create a ReconcileCephFilesystem object with the scheme and fake client. + r = &ReconcileCephFilesystem{client: cl, scheme: s, context: c, fsContexts: make(map[string]*fsHealth), opManagerContext: context.TODO()} + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, fs) + assert.NoError(t, err) + assert.Equal(t, cephv1.ConditionType("Ready"), fs.Status.Phase, fs) + }) } diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index e53004595..263781038 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -267,7 +267,7 @@ func TestCreateFilesystem(t *testing.T) { ConfigDir: configDir, Clientset: clientset} fs := fsTest(fsName) - clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", CephVersion: version.Octopus} + clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", CephVersion: version.Octopus, Context: ctx} // start a basic cluster ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() @@ -396,7 +396,7 @@ func TestUpgradeFilesystem(t *testing.T) { ConfigDir: configDir, Clientset: clientset} fs := fsTest(fsName) - clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", CephVersion: version.Octopus} + clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", CephVersion: version.Octopus, Context: ctx} // start a basic cluster for upgrade ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() @@ -528,7 +528,7 @@ func TestCreateNopoolFilesystem(t *testing.T) { }, }, } - clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid"} + clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", Context: ctx} // start a basic cluster ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() diff --git a/pkg/operator/ceph/file/health.go b/pkg/operator/ceph/file/health.go index 1f71b98fc..031fa1c29 100644 --- a/pkg/operator/ceph/file/health.go +++ b/pkg/operator/ceph/file/health.go @@ -18,6 +18,7 @@ limitations under the License. package file import ( + "context" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -64,7 +65,7 @@ func newMirrorChecker(context *clusterd.Context, client client.Client, clusterIn } // checkMirroring periodically checks the health of the cluster -func (c *mirrorChecker) checkMirroring(stopCh chan struct{}) { +func (c *mirrorChecker) checkMirroring(context context.Context) { // check the mirroring health immediately before starting the loop err := c.checkMirroringHealth() if err != nil { @@ -74,7 +75,7 @@ func (c *mirrorChecker) checkMirroring(stopCh chan struct{}) { for { select { - case <-stopCh: + case <-context.Done(): logger.Infof("stopping monitoring filesystem mirroring status %q", c.namespacedName.Name) return diff --git a/pkg/operator/ceph/file/mds/config.go b/pkg/operator/ceph/file/mds/config.go index 13cf0a1b7..10c0f0338 100644 --- a/pkg/operator/ceph/file/mds/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -17,7 +17,6 @@ limitations under the License. package mds import ( - "context" "fmt" "strconv" @@ -39,7 +38,6 @@ caps mds = "allow" ) func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) { - ctx := context.TODO() user := fmt.Sprintf("mds.%s", m.DaemonID) access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} @@ -52,7 +50,7 @@ func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) { } // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x - err = c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Delete(ctx, m.ResourceName, metav1.DeleteOptions{}) + err = c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Delete(c.clusterInfo.Context, m.ResourceName, metav1.DeleteOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debugf("legacy mds key %s is already removed", m.ResourceName) diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index 6baa9828d..d43d144b2 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -95,7 +95,6 @@ var UpdateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start starts or updates a Ceph mds cluster in Kubernetes. func (c *Cluster) Start() error { - ctx := context.TODO() // Validate pod's memory if specified err := controller.CheckPodMemory(cephv1.ResourcesKeyMDS, c.fs.Spec.MetadataServer.Resources, cephMdsPodMinimumMemory) if err != nil { @@ -135,7 +134,7 @@ func (c *Cluster) Start() error { desiredDeployments := map[string]bool{} // improvised set // Create/update deployments for i := 0; i < int(replicas); i++ { - deployment, err := c.startDeployment(ctx, k8sutil.IndexToName(i)) + deployment, err := c.startDeployment(c.clusterInfo.Context, k8sutil.IndexToName(i)) if err != nil { return errors.Wrapf(err, "failed to start deployment for MDS %q for filesystem %q", k8sutil.IndexToName(i), c.fs.Name) } @@ -287,7 +286,7 @@ func (c *Cluster) upgradeMDS() error { } // 5. upgrade current active deployment and wait for it come back - _, err = c.startDeployment(context.TODO(), daemonLetterID) + _, err = c.startDeployment(c.clusterInfo.Context, daemonLetterID) if err != nil { return errors.Wrapf(err, "failed to upgrade mds %q", daemonName) } diff --git a/pkg/operator/ceph/file/mirror/config.go b/pkg/operator/ceph/file/mirror/config.go index 2edc5bdf8..0bc0c59d1 100644 --- a/pkg/operator/ceph/file/mirror/config.go +++ b/pkg/operator/ceph/file/mirror/config.go @@ -19,7 +19,6 @@ package mirror import ( "fmt" - "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/version" @@ -51,14 +50,14 @@ type daemonConfig struct { ownerInfo *k8sutil.OwnerInfo } -func (r *ReconcileFilesystemMirror) generateKeyring(clusterInfo *client.ClusterInfo, daemonConfig *daemonConfig) (string, error) { +func (r *ReconcileFilesystemMirror) generateKeyring(daemonConfig *daemonConfig) (string, error) { access := []string{ "mon", "allow profile cephfs-mirror", "mgr", "allow r", "mds", "allow r", "osd", "allow rw tag cephfs metadata=*, allow r tag cephfs data=*", } - s := keyring.GetSecretStore(r.context, clusterInfo, daemonConfig.ownerInfo) + s := keyring.GetSecretStore(r.context, r.clusterInfo, daemonConfig.ownerInfo) key, err := s.GenerateKey(user, access) if err != nil { diff --git a/pkg/operator/ceph/file/mirror/controller.go b/pkg/operator/ceph/file/mirror/controller.go index 78e4dd150..ca5aa9e1b 100644 --- a/pkg/operator/ceph/file/mirror/controller.go +++ b/pkg/operator/ceph/file/mirror/controller.go @@ -28,7 +28,6 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - opconfig "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" @@ -67,27 +66,33 @@ var controllerTypeMeta = metav1.TypeMeta{ APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } +var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion + // ReconcileFilesystemMirror reconciles a CephFilesystemMirror object type ReconcileFilesystemMirror struct { - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - client client.Client - scheme *runtime.Scheme - cephClusterSpec *cephv1.ClusterSpec + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + cephClusterSpec *cephv1.ClusterSpec + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Add creates a new CephFilesystemMirror Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileFilesystemMirror{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opConfig: opConfig, + opManagerContext: opManagerContext, } } @@ -116,25 +121,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } } - // Build Handler function to return the list of ceph object - // This is used by the watchers below - handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephFilesystemMirrorList{}, mgr.GetScheme()) - if err != nil { - return err - } - - // Watch for CephCluster Spec changes that we want to propagate to us - err = c.Watch(&source.Kind{Type: &cephv1.CephCluster{ - TypeMeta: metav1.TypeMeta{ - Kind: opcontroller.ClusterResource.Kind, - APIVersion: opcontroller.ClusterResource.APIVersion, - }, - }, - }, handler.EnqueueRequestsFromMapFunc(handlerFunc), opcontroller.WatchCephClusterPredicate()) - if err != nil { - return err - } - return nil } @@ -146,7 +132,7 @@ func (r *ReconcileFilesystemMirror) Reconcile(context context.Context, request r // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface reconcileResponse, err := r.reconcile(request) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.FailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.FailedStatus) logger.Errorf("failed to reconcile %v", err) } @@ -156,7 +142,7 @@ func (r *ReconcileFilesystemMirror) Reconcile(context context.Context, request r func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephFilesystemMirror instance filesystemMirror := &cephv1.CephFilesystemMirror{} - err := r.client.Get(context.TODO(), request.NamespacedName, filesystemMirror) + err := r.client.Get(r.opManagerContext, request.NamespacedName, filesystemMirror) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephFilesystemMirror resource not found. Ignoring since object must be deleted.") @@ -168,7 +154,7 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc // The CR was just created, initializing status fields if filesystemMirror.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) } // Make sure a CephCluster is present otherwise do nothing @@ -182,26 +168,42 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc r.cephClusterSpec = &cephCluster.Spec // Populate clusterInfo - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to populate cluster info") } - // Populate CephVersion - daemon := string(opconfig.MonType) - currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon) + // Detect desired CephCluster version + runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion( + r.opConfig.Image, + filesystemMirror.Namespace, + controllerName, + k8sutil.NewOwnerInfo(filesystemMirror, r.scheme), + r.context, + r.cephClusterSpec, + r.clusterInfo, + ) if err != nil { if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { logger.Info(opcontroller.OperatorNotInitializedMessage) return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil } - return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon) + return reconcile.Result{}, errors.Wrap(err, "failed to detect running and desired ceph version") } - r.clusterInfo.CephVersion = currentCephVersion + + // If the version of the Ceph monitor differs from the CephCluster CR image version we assume + // the cluster is being upgraded. So the controller will just wait for the upgrade to finish and + // then versions should match. Obviously using the cmd reporter job adds up to the deployment time + if !reflect.DeepEqual(runningCephVersion, desiredCephVersion) { + // Upgrade is in progress, let's wait for the mons to be done + return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, + opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion) + } + r.clusterInfo.CephVersion = *runningCephVersion // Validate Ceph version - if !currentCephVersion.IsAtLeastPacific() { - return opcontroller.ImmediateRetryResult, errors.Errorf("ceph pacific version is required to deploy cephfs mirroring, current cluster runs %q", currentCephVersion.String()) + if !r.clusterInfo.CephVersion.IsAtLeastPacific() { + return opcontroller.ImmediateRetryResult, errors.Errorf("ceph pacific version is required to deploy cephfs mirroring, current cluster runs %q", r.clusterInfo.CephVersion.String()) } // CREATE/UPDATE @@ -212,7 +214,7 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue logger.Debug("done reconciling ceph filesystem mirror") @@ -239,9 +241,9 @@ func (r *ReconcileFilesystemMirror) reconcileFilesystemMirror(filesystemMirror * } // updateStatus updates an object with a given status -func updateStatus(client client.Client, name types.NamespacedName, status string) { +func (r *ReconcileFilesystemMirror) updateStatus(client client.Client, name types.NamespacedName, status string) { fsMirror := &cephv1.CephFilesystemMirror{} - err := client.Get(context.TODO(), name, fsMirror) + err := client.Get(r.opManagerContext, name, fsMirror) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephFilesystemMirror resource not found. Ignoring since object must be deleted.") diff --git a/pkg/operator/ceph/file/mirror/controller_test.go b/pkg/operator/ceph/file/mirror/controller_test.go index f7821faf4..7d38e3a68 100644 --- a/pkg/operator/ceph/file/mirror/controller_test.go +++ b/pkg/operator/ceph/file/mirror/controller_test.go @@ -27,6 +27,9 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -41,18 +44,6 @@ import ( const ( cephAuthGetOrCreateKey = `{"key":"AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="}` - dummyVersionsRaw = ` - { - "mon": { - "ceph version 14.2.8 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable)": 3 - } - }` - pacificVersionsRaw = ` - { - "mon": { - "ceph version 16.2.1 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) pacific (stable)": 3 - } - }` ) func TestCephFilesystemMirrorController(t *testing.T) { @@ -65,12 +56,12 @@ func TestCephFilesystemMirrorController(t *testing.T) { capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - // - // TEST 1 SETUP - // - // FAILURE because no CephCluster - // - // An cephfs-mirror resource with metadata and spec. + // Mock cmd reporter + // pacific := &version.CephVersion{Major: 16, Minor: 2, Extra: 1, Build: 0, CommitID: ""} + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Octopus, &version.Octopus, nil + } + fsMirror := &cephv1.CephFilesystemMirror{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -93,9 +84,6 @@ func TestCephFilesystemMirrorController(t *testing.T) { if args[0] == "auth" && args[1] == "get-or-create-key" { return cephAuthGetOrCreateKey, nil } - if args[0] == "versions" { - return dummyVersionsRaw, nil - } return "", nil }, } @@ -115,32 +103,34 @@ func TestCephFilesystemMirrorController(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() // Create a ReconcileFilesystemMirror object with the scheme and fake client. - r := &ReconcileFilesystemMirror{client: cl, scheme: s, context: c} - - // Mock request to simulate Reconcile() being called on an event for a - // watched resource . + r := &ReconcileFilesystemMirror{ + client: cl, + scheme: s, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + opManagerContext: ctx, + } req := reconcile.Request{ NamespacedName: types.NamespacedName{ Name: name, Namespace: namespace, }, } - logger.Info("STARTING PHASE 1") - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 1 DONE") - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // cephCluster := &cephv1.CephCluster{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, Namespace: namespace, }, + Spec: cephv1.ClusterSpec{ + CephVersion: cephv1.CephVersionSpec{ + Image: "ceph/ceph:16.2.0", + }, + }, Status: cephv1.ClusterStatus{ Phase: "", CephStatus: &cephv1.CephStatus{ @@ -148,81 +138,124 @@ func TestCephFilesystemMirrorController(t *testing.T) { }, }, } - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileFilesystemMirror object with the scheme and fake client. - r = &ReconcileFilesystemMirror{client: cl, scheme: s, context: c} - logger.Info("STARTING PHASE 2") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 2 DONE") - // - // TEST 3: - // - // SUCCESS! The CephCluster is ready but version is too old! - // + t.Run("error - no ceph cluster", func(t *testing.T) { + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Add ready status to the CephCluster - cephCluster.Status.Phase = k8sutil.ReadyStatus - cephCluster.Status.CephStatus.Health = "HEALTH_OK" + t.Run("error - ceph cluster not ready", func(t *testing.T) { + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileFilesystemMirror object with the scheme and fake client. + r = &ReconcileFilesystemMirror{ + client: cl, + scheme: s, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + opManagerContext: ctx, + } + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + t.Run("error - ceph cluster ready but version is too old", func(t *testing.T) { + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) - // Create a ReconcileFilesystemMirror object with the scheme and fake client. - r = &ReconcileFilesystemMirror{ - client: cl, - scheme: s, - context: c, - } - res, err = r.Reconcile(ctx, req) - assert.Error(t, err) - assert.True(t, res.Requeue) + // Add ready status to the CephCluster + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" - // - // TEST 4: - // - // SUCCESS! The CephCluster is ready and running Ceph Pacific! - // - r.context.Executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "status" { - return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil - } - if args[0] == "auth" && args[1] == "get-or-create-key" { - return cephAuthGetOrCreateKey, nil - } - if args[0] == "versions" { - return pacificVersionsRaw, nil - } - return "", nil - }, - } + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, fsMirror) - assert.NoError(t, err) - assert.Equal(t, "Ready", fsMirror.Status.Phase, fsMirror) + // Create a ReconcileFilesystemMirror object with the scheme and fake client. + r = &ReconcileFilesystemMirror{ + client: cl, + scheme: s, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + opManagerContext: ctx, + } + res, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.True(t, res.Requeue) + }) + + t.Run("error - cluster is upgrading", func(t *testing.T) { + r.context.Executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return cephAuthGetOrCreateKey, nil + } + return "", nil + }, + } + + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Octopus, &version.Pacific, nil + } + + res, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.EqualError(t, err, "waiting for ceph monitors upgrade to finish. current version: 16.0.0-0 pacific. expected version: 15.0.0-0 octopus. will reconcile again in 1m0s") + assert.True(t, res.Requeue) + }) + + t.Run("success - running pacific", func(t *testing.T) { + r.context.Executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return cephAuthGetOrCreateKey, nil + } + return "", nil + }, + } + + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Pacific, &version.Pacific, nil + } + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, fsMirror) + assert.NoError(t, err) + assert.Equal(t, "Ready", fsMirror.Status.Phase, fsMirror) + }) } diff --git a/pkg/operator/ceph/file/mirror/mirror.go b/pkg/operator/ceph/file/mirror/mirror.go index a804e2e7e..5b13e9183 100644 --- a/pkg/operator/ceph/file/mirror/mirror.go +++ b/pkg/operator/ceph/file/mirror/mirror.go @@ -18,8 +18,6 @@ limitations under the License. package mirror import ( - "context" - "github.com/banzaicloud/k8s-objectmatcher/patch" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -43,7 +41,6 @@ var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start begins the process of running filesystem mirroring daemons. func (r *ReconcileFilesystemMirror) start(filesystemMirror *cephv1.CephFilesystemMirror) error { - ctx := context.TODO() // Validate pod's memory if specified err := controller.CheckPodMemory(cephv1.ResourcesKeyFilesystemMirror, filesystemMirror.Spec.Resources, cephFilesystemMirrorPodMinimumMemory) if err != nil { @@ -57,7 +54,7 @@ func (r *ReconcileFilesystemMirror) start(filesystemMirror *cephv1.CephFilesyste ownerInfo: ownerInfo, } - _, err = r.generateKeyring(r.clusterInfo, daemonConf) + _, err = r.generateKeyring(daemonConf) if err != nil { return errors.Wrapf(err, "failed to generate keyring for %q", AppName) } @@ -80,7 +77,7 @@ func (r *ReconcileFilesystemMirror) start(filesystemMirror *cephv1.CephFilesyste return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name) } - if _, err := r.context.Clientset.AppsV1().Deployments(filesystemMirror.Namespace).Create(ctx, d, metav1.CreateOptions{}); err != nil { + if _, err := r.context.Clientset.AppsV1().Deployments(filesystemMirror.Namespace).Create(r.opManagerContext, d, metav1.CreateOptions{}); err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create %q deployment", d.Name) } @@ -89,11 +86,11 @@ func (r *ReconcileFilesystemMirror) start(filesystemMirror *cephv1.CephFilesyste if err := updateDeploymentAndWait(r.context, r.clusterInfo, d, config.FilesystemMirrorType, AppName, r.cephClusterSpec.SkipUpgradeChecks, false); err != nil { // fail could be an issue updating label selector (immutable), so try del and recreate logger.Debugf("updateDeploymentAndWait failed for filesystem-mirror %q. Attempting del-and-recreate. %v", d.Name, err) - err = r.context.Clientset.AppsV1().Deployments(filesystemMirror.Namespace).Delete(ctx, filesystemMirror.Name, metav1.DeleteOptions{}) + err = r.context.Clientset.AppsV1().Deployments(filesystemMirror.Namespace).Delete(r.opManagerContext, filesystemMirror.Name, metav1.DeleteOptions{}) if err != nil { return errors.Wrapf(err, "failed to delete filesystem-mirror deployment %q during del-and-recreate update attempt", d.Name) } - if _, err := r.context.Clientset.AppsV1().Deployments(filesystemMirror.Namespace).Create(ctx, d, metav1.CreateOptions{}); err != nil { + if _, err := r.context.Clientset.AppsV1().Deployments(filesystemMirror.Namespace).Create(r.opManagerContext, d, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to recreate filesystem-mirror deployment %q during del-and-recreate update attempt", d.Name) } } diff --git a/pkg/operator/ceph/file/status.go b/pkg/operator/ceph/file/status.go index 7f05afef5..952c98a00 100644 --- a/pkg/operator/ceph/file/status.go +++ b/pkg/operator/ceph/file/status.go @@ -18,7 +18,6 @@ limitations under the License. package file import ( - "context" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -29,9 +28,9 @@ import ( ) // updateStatus updates a fs CR with the given status -func updateStatus(client client.Client, namespacedName types.NamespacedName, status cephv1.ConditionType, info map[string]string) { +func (r *ReconcileCephFilesystem) updateStatus(client client.Client, namespacedName types.NamespacedName, status cephv1.ConditionType, info map[string]string) { fs := &cephv1.CephFilesystem{} - err := client.Get(context.TODO(), namespacedName, fs) + err := client.Get(r.opManagerContext, namespacedName, fs) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephFilesystem resource not found. Ignoring since object must be deleted.") @@ -57,7 +56,7 @@ func updateStatus(client client.Client, namespacedName types.NamespacedName, sta // updateStatusBucket updates an object with a given status func (c *mirrorChecker) updateStatusMirroring(mirrorStatus []cephv1.FilesystemMirroringInfo, snapSchedStatus []cephv1.FilesystemSnapshotSchedulesSpec, details string) { fs := &cephv1.CephFilesystem{} - if err := c.client.Get(context.TODO(), c.namespacedName, fs); err != nil { + if err := c.client.Get(c.clusterInfo.Context, c.namespacedName, fs); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephFilesystem resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index efbc0677e..586d03b77 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -28,7 +28,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - opconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" @@ -67,27 +67,33 @@ var controllerTypeMeta = metav1.TypeMeta{ APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } +var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion + // ReconcileCephNFS reconciles a cephNFS object type ReconcileCephNFS struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - cephClusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + cephClusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Add creates a new cephNFS Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileCephNFS{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, + opConfig: opConfig, } } @@ -136,7 +142,7 @@ func (r *ReconcileCephNFS) Reconcile(context context.Context, request reconcile. func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the cephNFS instance cephNFS := &cephv1.CephNFS{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephNFS) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephNFS) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephNFS resource not found. Ignoring since object must be deleted.") @@ -182,26 +188,23 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul // Populate clusterInfo // Always populate it during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } - // Populate CephVersion - currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil - } - return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) - } - r.clusterInfo.CephVersion = currentCephVersion - // DELETE: the CR was deleted if !cephNFS.GetDeletionTimestamp().IsZero() { logger.Infof("deleting ceph nfs %q", cephNFS.Name) - err := r.removeServersFromDatabase(cephNFS, 0) + + // Detect running Ceph version + runningCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, config.MonType) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", config.MonType) + } + r.clusterInfo.CephVersion = runningCephVersion + + err = r.removeServersFromDatabase(cephNFS, 0) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to delete filesystem %q. ", cephNFS.Name) } @@ -216,6 +219,34 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul return reconcile.Result{}, nil } + // Detect desired CephCluster version + runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion( + r.opConfig.Image, + cephNFS.Namespace, + controllerName, + k8sutil.NewOwnerInfo(cephNFS, r.scheme), + r.context, + r.cephClusterSpec, + r.clusterInfo, + ) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + return reconcile.Result{}, errors.Wrap(err, "failed to detect running and desired ceph version") + } + + // If the version of the Ceph monitor differs from the CephCluster CR image version we assume + // the cluster is being upgraded. So the controller will just wait for the upgrade to finish and + // then versions should match. Obviously using the cmd reporter job adds up to the deployment time + if !reflect.DeepEqual(runningCephVersion, desiredCephVersion) { + // Upgrade is in progress, let's wait for the mons to be done + return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, + opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion) + } + r.clusterInfo.CephVersion = *runningCephVersion + // validate the store settings if err := validateGanesha(r.context, r.clusterInfo, cephNFS); err != nil { return reconcile.Result{}, errors.Wrapf(err, "invalid ceph nfs %q arguments", cephNFS.Name) @@ -239,7 +270,6 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul } func (r *ReconcileCephNFS) reconcileCreateCephNFS(cephNFS *cephv1.CephNFS) (reconcile.Result, error) { - ctx := context.TODO() if r.cephClusterSpec.External.Enable { _, err := opcontroller.ValidateCephVersionsBetweenLocalAndExternalClusters(r.context, r.clusterInfo) if err != nil { @@ -249,7 +279,7 @@ func (r *ReconcileCephNFS) reconcileCreateCephNFS(cephNFS *cephv1.CephNFS) (reco } } - deployments, err := r.context.Clientset.AppsV1().Deployments(cephNFS.Namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}) + deployments, err := r.context.Clientset.AppsV1().Deployments(cephNFS.Namespace).List(r.opManagerContext, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", AppName)}) if err != nil { if kerrors.IsNotFound(err) { logger.Infof("creating ceph nfs %q", cephNFS.Name) diff --git a/pkg/operator/ceph/nfs/controller_test.go b/pkg/operator/ceph/nfs/controller_test.go index c013e26d5..6d2b7841b 100644 --- a/pkg/operator/ceph/nfs/controller_test.go +++ b/pkg/operator/ceph/nfs/controller_test.go @@ -28,7 +28,8 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" - cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -45,13 +46,7 @@ var ( name = "my-nfs" namespace = "rook-ceph" nfsCephAuthGetOrCreateKey = `{"key":"AQCvzWBeIV9lFRAAninzm+8XFxbSfTiPwoX50g=="}` - dummyVersionsRaw = ` - { - "mon": { - "ceph version 14.2.8 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable)": 3 - } - }` - poolDetails = `{ + poolDetails = `{ "pool": "foo", "pool_id": 1, "size": 3, @@ -111,9 +106,6 @@ func TestCephNFSController(t *testing.T) { if args[0] == "status" { return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil } - if args[0] == "versions" { - return dummyVersionsRaw, nil - } return "", nil }, } @@ -132,7 +124,7 @@ func TestCephNFSController(t *testing.T) { // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() // Create a ReconcileCephNFS object with the scheme and fake client. - r := &ReconcileCephNFS{client: cl, scheme: s, context: c} + r := &ReconcileCephNFS{client: cl, scheme: s, context: c, opManagerContext: ctx} // Mock request to simulate Reconcile() being called on an event for a // watched resource . @@ -142,17 +134,7 @@ func TestCephNFSController(t *testing.T) { Namespace: namespace, }, } - logger.Info("STARTING PHASE 1") - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 1 DONE") - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // cephCluster := &cephv1.CephCluster{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, @@ -165,95 +147,96 @@ func TestCephNFSController(t *testing.T) { }, }, } - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileCephNFS object with the scheme and fake client. - r = &ReconcileCephNFS{client: cl, scheme: s, context: c} - logger.Info("STARTING PHASE 2") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 2 DONE") - // - // TEST 3: - // - // SUCCESS! The CephCluster is ready - // - - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Octopus, &version.Octopus, nil } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - // Add ready status to the CephCluster - cephCluster.Status.Phase = k8sutil.ReadyStatus - cephCluster.Status.CephStatus.Health = "HEALTH_OK" + t.Run("error - no ceph cluster", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + t.Run("error - ceph cluster not ready", func(t *testing.T) { + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileCephNFS object with the scheme and fake client. + r = &ReconcileCephNFS{client: cl, scheme: s, context: c, opManagerContext: ctx} + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "status" { - return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil - } - if args[0] == "auth" && args[1] == "get-or-create-key" { - return nfsCephAuthGetOrCreateKey, nil - } - if args[0] == "versions" { - return dummyVersionsRaw, nil - } - if args[0] == "osd" && args[1] == "pool" && args[2] == "get" { - return poolDetails, nil - } - return "", errors.New("unknown command") - }, - MockExecuteCommand: func(command string, args ...string) error { - if command == "rados" { - logger.Infof("mock execute. %s. %s", command, args) - assert.Equal(t, "stat", args[6]) - assert.Equal(t, "conf-my-nfs.a", args[7]) - return nil - } - return errors.New("unknown command") - }, - MockExecuteCommandWithEnv: func(env []string, command string, args ...string) error { - if command == "ganesha-rados-grace" { - logger.Infof("mock execute. %s. %s", command, args) - assert.Equal(t, "add", args[4]) - assert.Len(t, env, 1) - return nil - } - return errors.New("unknown command") - }, - } - c.Executor = executor + t.Run("success - nfs running", func(t *testing.T) { + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) - // Create a ReconcileCephNFS object with the scheme and fake client. - r = &ReconcileCephNFS{client: cl, scheme: s, context: c} + // Add ready status to the CephCluster + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" - logger.Info("STARTING PHASE 3") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, cephNFS) - assert.NoError(t, err) - assert.Equal(t, "Ready", cephNFS.Status.Phase, cephNFS) - logger.Info("PHASE 3 DONE") + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return nfsCephAuthGetOrCreateKey, nil + } + if args[0] == "osd" && args[1] == "pool" && args[2] == "get" { + return poolDetails, nil + } + return "", errors.New("unknown command") + }, + MockExecuteCommand: func(command string, args ...string) error { + if command == "rados" { + logger.Infof("mock execute. %s. %s", command, args) + assert.Equal(t, "stat", args[6]) + assert.Equal(t, "conf-nfs.my-nfs", args[7]) + return nil + } + return errors.New("unknown command") + }, + MockExecuteCommandWithEnv: func(env []string, command string, args ...string) error { + if command == "ganesha-rados-grace" { + logger.Infof("mock execute. %s. %s", command, args) + assert.Equal(t, "add", args[4]) + assert.Len(t, env, 1) + return nil + } + return errors.New("unknown command") + }, + } + c.Executor = executor + + // Create a ReconcileCephNFS object with the scheme and fake client. + r = &ReconcileCephNFS{client: cl, scheme: s, context: c, opManagerContext: ctx} + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, cephNFS) + assert.NoError(t, err) + assert.Equal(t, "Ready", cephNFS.Status.Phase, cephNFS) + }) } func TestGetGaneshaConfigObject(t *testing.T) { @@ -266,15 +249,15 @@ func TestGetGaneshaConfigObject(t *testing.T) { nodeid := "a" expectedName := "conf-nfs.my-nfs" - res := getGaneshaConfigObject(cephNFS, cephver.CephVersion{Major: 16}, nodeid) + res := getGaneshaConfigObject(cephNFS, version.CephVersion{Major: 16}, nodeid) logger.Infof("Config Object for Pacific is %s", res) assert.Equal(t, expectedName, res) - res = getGaneshaConfigObject(cephNFS, cephver.CephVersion{Major: 15, Minor: 2, Extra: 1}, nodeid) + res = getGaneshaConfigObject(cephNFS, version.CephVersion{Major: 15, Minor: 2, Extra: 1}, nodeid) logger.Infof("Config Object for Octopus is %s", res) assert.Equal(t, expectedName, res) - res = getGaneshaConfigObject(cephNFS, cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}, nodeid) + res = getGaneshaConfigObject(cephNFS, version.CephVersion{Major: 14, Minor: 2, Extra: 5}, nodeid) logger.Infof("Config Object for Nautilus is %s", res) assert.Equal(t, "conf-my-nfs.a", res) } diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index 11bfc8f20..3cf1dcd11 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -18,7 +18,6 @@ limitations under the License. package nfs import ( - "context" "fmt" "github.com/banzaicloud/k8s-objectmatcher/patch" @@ -49,7 +48,6 @@ type daemonConfig struct { // Create the ganesha server func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error { - ctx := context.TODO() for i := 0; i < n.Spec.Server.Active; i++ { id := k8sutil.IndexToName(i) @@ -96,7 +94,7 @@ func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error { } // start the deployment - _, err = r.context.Clientset.AppsV1().Deployments(n.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + _, err = r.context.Clientset.AppsV1().Deployments(n.Namespace).Create(r.opManagerContext, deployment, metav1.CreateOptions{}) if err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrap(err, "failed to create ceph nfs deployment") @@ -189,7 +187,6 @@ func (r *ReconcileCephNFS) generateConfigMap(n *cephv1.CephNFS, name string) *v1 } func (r *ReconcileCephNFS) createConfigMap(n *cephv1.CephNFS, name string) (string, error) { - ctx := context.TODO() // Generate configMap configMap := r.generateConfigMap(n, name) @@ -199,13 +196,13 @@ func (r *ReconcileCephNFS) createConfigMap(n *cephv1.CephNFS, name string) (stri return "", errors.Wrapf(err, "failed to set owner reference for ceph ganesha configmap %q", configMap.Name) } - if _, err := r.context.Clientset.CoreV1().ConfigMaps(n.Namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil { + if _, err := r.context.Clientset.CoreV1().ConfigMaps(n.Namespace).Create(r.opManagerContext, configMap, metav1.CreateOptions{}); err != nil { if !kerrors.IsAlreadyExists(err) { return "", errors.Wrap(err, "failed to create ganesha config map") } logger.Debugf("updating config map %q that already exists", configMap.Name) - if _, err = r.context.Clientset.CoreV1().ConfigMaps(n.Namespace).Update(ctx, configMap, metav1.UpdateOptions{}); err != nil { + if _, err = r.context.Clientset.CoreV1().ConfigMaps(n.Namespace).Update(r.opManagerContext, configMap, metav1.UpdateOptions{}); err != nil { return "", errors.Wrap(err, "failed to update ganesha config map") } } @@ -215,7 +212,6 @@ func (r *ReconcileCephNFS) createConfigMap(n *cephv1.CephNFS, name string) (stri // Down scale the ganesha server func (r *ReconcileCephNFS) downCephNFS(n *cephv1.CephNFS, nfsServerListNum int) error { - ctx := context.TODO() diffCount := nfsServerListNum - n.Spec.Server.Active for i := 0; i < diffCount; { depIDToRemove := nfsServerListNum - 1 @@ -225,7 +221,7 @@ func (r *ReconcileCephNFS) downCephNFS(n *cephv1.CephNFS, nfsServerListNum int) // Remove deployment logger.Infof("removing deployment %q", depNameToRemove) - err := r.context.Clientset.AppsV1().Deployments(n.Namespace).Delete(ctx, depNameToRemove, metav1.DeleteOptions{}) + err := r.context.Clientset.AppsV1().Deployments(n.Namespace).Delete(r.opManagerContext, depNameToRemove, metav1.DeleteOptions{}) if err != nil { if !kerrors.IsNotFound(err) { return errors.Wrap(err, "failed to delete ceph nfs deployment") diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 088542a5b..a93cab3cc 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -17,8 +17,6 @@ limitations under the License. package nfs import ( - "context" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" @@ -71,7 +69,6 @@ func (r *ReconcileCephNFS) generateCephNFSService(nfs *cephv1.CephNFS, cfg daemo } func (r *ReconcileCephNFS) createCephNFSService(nfs *cephv1.CephNFS, cfg daemonConfig) error { - ctx := context.TODO() s := r.generateCephNFSService(nfs, cfg) // Set owner ref to the parent object @@ -80,7 +77,7 @@ func (r *ReconcileCephNFS) createCephNFSService(nfs *cephv1.CephNFS, cfg daemonC return errors.Wrapf(err, "failed to set owner reference to ceph nfs %q", s) } - svc, err := r.context.Clientset.CoreV1().Services(nfs.Namespace).Create(ctx, s, metav1.CreateOptions{}) + svc, err := r.context.Clientset.CoreV1().Services(nfs.Namespace).Create(r.opManagerContext, s, metav1.CreateOptions{}) if err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrap(err, "failed to create ganesha service") diff --git a/pkg/operator/ceph/object/bucket/api-handlers.go b/pkg/operator/ceph/object/bucket/api-handlers.go index 7a428d45d..54ac87635 100644 --- a/pkg/operator/ceph/object/bucket/api-handlers.go +++ b/pkg/operator/ceph/object/bucket/api-handlers.go @@ -1,7 +1,6 @@ package bucket import ( - "context" "time" "github.com/pkg/errors" @@ -28,12 +27,11 @@ var backoff = wait.Backoff{ } func (p *Provisioner) getStorageClassWithBackoff(name string) (class *storagev1.StorageClass, err error) { - ctx := context.TODO() logger.Infof("getting storage class %q", name) classClient := p.context.Clientset.StorageV1().StorageClasses() // Retry Get() with backoff. Errors other than IsNotFound are ignored. err = wait.ExponentialBackoff(backoff, func() (done bool, err error) { - class, err = classClient.Get(ctx, name, metav1.GetOptions{}) + class, err = classClient.Get(p.clusterInfo.Context, name, metav1.GetOptions{}) if err == nil { return true, nil } diff --git a/pkg/operator/ceph/object/bucket/controller.go b/pkg/operator/ceph/object/bucket/controller.go new file mode 100644 index 000000000..8a1669f98 --- /dev/null +++ b/pkg/operator/ceph/object/bucket/controller.go @@ -0,0 +1,189 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bucket + +import ( + "context" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + controllerName = "rook-ceph-operator-bucket-controller" +) + +// ReconcileBucket reconciles a ceph-csi driver +type ReconcileBucket struct { + client client.Client + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + opConfig opcontroller.OperatorConfig + opManagerContext context.Context +} + +// Add creates a new Ceph CSI Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { + return &ReconcileBucket{ + client: mgr.GetClient(), + context: context, + opConfig: opConfig, + opManagerContext: opManagerContext, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + logger.Infof("%s successfully started", controllerName) + + // Watch for ConfigMap (operator config) + err = c.Watch(&source.Kind{ + Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController()) + if err != nil { + return err + } + + // Watch for CephCluster + err = c.Watch(&source.Kind{ + Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController()) + if err != nil { + return err + } + + return nil +} + +// Reconcile reads that state of the operator config map and makes changes based on the state read +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileBucket) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + // workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface + reconcileResponse, err := r.reconcile(request) + if err != nil { + logger.Errorf("failed to reconcile %v", err) + } + + return reconcileResponse, err +} + +func (r *ReconcileBucket) reconcile(request reconcile.Request) (reconcile.Result, error) { + // See if there is a CephCluster + cephCluster := &cephv1.CephCluster{} + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Infof("no ceph cluster found in %+v. not deploying the bucket provisioner", request.NamespacedName) + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get the ceph cluster") + } + + if !cephCluster.DeletionTimestamp.IsZero() { + logger.Debug("ceph cluster is being deleted, no need to reconcile the bucket provisioner") + return reconcile.Result{}, nil + } + + if !cephCluster.Spec.External.Enable && cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() { + logger.Debug("ceph cluster has cleanup policy, the cluster will soon go away, no need to reconcile the bucket provisioner") + return reconcile.Result{}, nil + } + + // Fetch the operator's configmap. We force the NamespaceName to the operator since the request + // could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the + // CM won't be found + opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace} + opConfig := &v1.ConfigMap{} + err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig) + if err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("operator's configmap resource not found. will use default value or env var.") + r.opConfig.Parameters = make(map[string]string) + } else { + // Error reading the object - requeue the request. + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") + } + } else { + // Populate the operator's config + r.opConfig.Parameters = opConfig.Data + } + + // Populate clusterInfo during each reconcile + clusterInfo, _, _, err := mon.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace) + if err != nil { + // This avoids a requeue with exponential backoff and allows the controller to reconcile + // more quickly when the cluster is ready. + if errors.Is(err, mon.ClusterInfoNoClusterNoSecret) { + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to populate cluster info") + } + r.clusterInfo = clusterInfo + + // Start the object bucket provisioner + bucketProvisioner := NewProvisioner(r.context, clusterInfo) + // If cluster is external, pass down the user to the bucket controller + + // note: the error return below is ignored and is expected to be removed from the + // bucket library's `NewProvisioner` function + bucketController, _ := NewBucketController(r.context.KubeConfig, bucketProvisioner, r.opConfig.Parameters) + + // We must run this in a go routine since RunWithContext() blocks and waits for the context to + // be Done. However, since it has a context, the go routine will exit on reload with SIGHUP + errChan := make(chan error) + go func() { + err = bucketController.RunWithContext(r.opManagerContext) + if err != nil { + logger.Errorf("failed to run bucket controller. %v", err) + errChan <- err + } + }() + + // Check for errors when running the bucket controller + select { + case <-errChan: + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to run bucket controller") + default: + logger.Info("successfully reconciled bucket provisioner") + return reconcile.Result{}, nil + } +} diff --git a/pkg/operator/ceph/object/bucket/controller_test.go b/pkg/operator/ceph/object/bucket/controller_test.go new file mode 100644 index 000000000..55b0efd9b --- /dev/null +++ b/pkg/operator/ceph/object/bucket/controller_test.go @@ -0,0 +1,181 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bucket + +import ( + "context" + "os" + "testing" + "time" + + "github.com/coreos/pkg/capnslog" + "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func TestCephBucketController(t *testing.T) { + var ( + name = "rook-ceph" + namespace = "rook-ceph" + ) + // Set DEBUG logging + capnslog.SetGlobalLogLevel(capnslog.DEBUG) + os.Setenv("ROOK_LOG_LEVEL", "DEBUG") + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + c := &clusterd.Context{ + Clientset: test.New(t, 1), + } + t.Run("do nothing since there is no CephCluster", func(t *testing.T) { + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).Build() + c.Client = cl + + // Create a ReconcileBucket object with the scheme and fake client. + r := &ReconcileBucket{ + client: cl, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + opManagerContext: context.TODO(), + } + ctx := context.TODO() + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + }) + + t.Run("success bucket prov deployment", func(t *testing.T) { + var err error + + // It's a bit annoying that lib-bucket-prov only supports passing a rest.Config{} which is really + // hard to mock since it's the simplest expression of a kube config. Typically when mocking we pass + // a fake client which is a bit easier to mock. + // Also, mocking the &provisioner.Provisioner{} is not possible since not all the fields + // like informerFactory are exported... I'm leaving the failed attempt below just in + // case lib-bucket-prov changes in the future. For now, the client will start but will fail + // to connect to the API server, we can see that by running the test. + // + // fakeKubeClient := testclient.NewSimpleClientset() + // newBucketController = func(cfg *rest.Config, p *Provisioner, data map[string]string) (*provisioner.Provisioner, error) { + // return &provisioner.Provisioner{ + // Name: "prov", + // informerFactory: informers.NewSharedInformerFactory(fakeKubeClient, 1*time.Second), + + // claimController: provisioner.NewController( + // provisionerName, + // NewProvisioner(c, nil), // change cluster info + // fakeKubeClient, + // libClientset, + // informerFactory.Objectbucket().V1alpha1().ObjectBucketClaims(), + // informerFactory.Objectbucket().V1alpha1().ObjectBuckets()), + // }, nil + // } + + assert.NoError(t, err) + c.KubeConfig = &rest.Config{} + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Status: cephv1.ClusterStatus{ + Phase: "", + CephVersion: &cephv1.ClusterVersion{ + Version: "14.2.9-0", + }, + CephStatus: &cephv1.CephStatus{ + Health: "", + }, + }, + } + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &v1alpha1.ObjectBucketClaim{}, &v1alpha1.ObjectBucket{}) + + object := []runtime.Object{ + cephCluster, + } + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + c.Client = cl + + // Create a ReconcileBucket object with the scheme and fake client. + ctx, cancel := context.WithCancel(context.TODO()) + // defer cancel() + + r := &ReconcileBucket{ + client: cl, + context: c, + opConfig: controller.OperatorConfig{ + OperatorNamespace: namespace, + Image: "rook", + ServiceAccount: "foo", + }, + opManagerContext: ctx, + } + + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + + // wait a few seconds for the manager to start + time.Sleep(2 * time.Second) + cancel() + }) +} diff --git a/pkg/operator/ceph/object/bucket/predicate.go b/pkg/operator/ceph/object/bucket/predicate.go new file mode 100644 index 000000000..f97038d0a --- /dev/null +++ b/pkg/operator/ceph/object/bucket/predicate.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bucket + +import ( + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/operator/ceph/controller" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +const rookOBCWatchOperatorNamespace = "ROOK_OBC_WATCH_OPERATOR_NAMESPACE" + +// predicateController is the predicate function to trigger reconcile on operator configuration cm change +func predicateController() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + // if the operator configuration file is created we want to reconcile + if cm, ok := e.Object.(*v1.ConfigMap); ok { + // It's probably fine to use the Generation value here. The case where the operator was stopped and the + // ConfigMap was created is low since the cm is always present these days + return cm.Name == controller.OperatorSettingConfigMapName && cm.Generation == 1 + } + + // If a Ceph Cluster is created we want to reconcile the bucket provisioner + if _, ok := e.Object.(*cephv1.CephCluster); ok { + // Always return true, so when the controller starts we reconcile too. We don't get + return true + } + + return false + }, + + UpdateFunc: func(e event.UpdateEvent) bool { + if old, ok := e.ObjectOld.(*v1.ConfigMap); ok { + if new, ok := e.ObjectNew.(*v1.ConfigMap); ok { + if old.Name == controller.OperatorSettingConfigMapName && new.Name == controller.OperatorSettingConfigMapName { + if old.Data[rookOBCWatchOperatorNamespace] != new.Data[rookOBCWatchOperatorNamespace] { + logger.Infof("%s changed. reconciling bucket controller", rookOBCWatchOperatorNamespace) + + // We reload the manager so that the controller restarts and goes + // through the CreateFunc again. Then the CephCluster watcher will be triggered + controller.ReloadManager() + } + } + } + } + + return false + }, + + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + } +} diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index 81eaeba4a..31770ede8 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -17,7 +17,6 @@ limitations under the License. package bucket import ( - "context" "fmt" "net/http" "strconv" @@ -98,7 +97,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje } singleBucketQuota := 1 - _, err = p.adminOpsClient.ModifyUser(context.TODO(), admin.User{ID: p.cephUserName, MaxBuckets: &singleBucketQuota}) + _, err = p.adminOpsClient.ModifyUser(p.clusterInfo.Context, admin.User{ID: p.cephUserName, MaxBuckets: &singleBucketQuota}) if err != nil { p.deleteOBCResourceLogError(p.bucketName) return nil, err @@ -140,20 +139,20 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu // need to quota into -1 for restricting creation of new buckets in rgw restrictBucketCreation := -1 - _, err = p.adminOpsClient.ModifyUser(context.TODO(), admin.User{ID: p.cephUserName, MaxBuckets: &restrictBucketCreation}) + _, err = p.adminOpsClient.ModifyUser(p.clusterInfo.Context, admin.User{ID: p.cephUserName, MaxBuckets: &restrictBucketCreation}) if err != nil { p.deleteOBCResourceLogError("") return nil, err } // get the bucket's owner via the bucket metadata - stats, err := p.adminOpsClient.GetBucketInfo(context.TODO(), admin.Bucket{Bucket: p.bucketName}) + stats, err := p.adminOpsClient.GetBucketInfo(p.clusterInfo.Context, admin.Bucket{Bucket: p.bucketName}) if err != nil { p.deleteOBCResourceLogError("") return nil, errors.Wrapf(err, "failed to get bucket %q stats", p.bucketName) } - objectUser, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: stats.Owner}) + objectUser, err := p.adminOpsClient.GetUser(p.clusterInfo.Context, admin.User{ID: stats.Owner}) if err != nil { p.deleteOBCResourceLogError("") return nil, errors.Wrapf(err, "failed to get user %q", stats.Owner) @@ -236,7 +235,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error { } logger.Infof("Revoke: denying access to bucket %q for OB %q", p.bucketName, ob.Name) - bucket, err := p.adminOpsClient.GetBucketInfo(context.TODO(), admin.Bucket{Bucket: p.bucketName}) + bucket, err := p.adminOpsClient.GetBucketInfo(p.clusterInfo.Context, admin.Bucket{Bucket: p.bucketName}) if err != nil { logger.Errorf("%v", err) } else { @@ -244,7 +243,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error { return errors.Errorf("failed to find bucket %q owner", p.bucketName) } - user, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: bucket.Owner}) + user, err := p.adminOpsClient.GetUser(p.clusterInfo.Context, admin.User{ID: bucket.Owner}) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { // The user may not exist. Ignore this in order to ensure the PolicyStatement does not contain the @@ -561,7 +560,7 @@ func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error } // Enabling quota for the user - err := p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, Enabled: "aEnabled}) + err := p.adminOpsClient.SetUserQuota(p.clusterInfo.Context, admin.QuotaSpec{UID: p.cephUserName, Enabled: "aEnabled}) if err != nil { return errors.Wrapf(err, "failed to enable user %q quota for obc", p.cephUserName) } @@ -572,7 +571,7 @@ func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error return errors.Wrap(err, "failed to convert maxObjects to integer") } maxObjectsInt64 := int64(maxObjectsInt) - err = p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, MaxObjects: &maxObjectsInt64}) + err = p.adminOpsClient.SetUserQuota(p.clusterInfo.Context, admin.QuotaSpec{UID: p.cephUserName, MaxObjects: &maxObjectsInt64}) if err != nil { return errors.Wrapf(err, "failed to set MaxObject to user %q", p.cephUserName) } @@ -582,7 +581,7 @@ func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error if err != nil { return errors.Wrapf(err, "failed to parse maxSize quota for user %q", p.cephUserName) } - err = p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, MaxSize: &maxSizeInt}) + err = p.adminOpsClient.SetUserQuota(p.clusterInfo.Context, admin.QuotaSpec{UID: p.cephUserName, MaxSize: &maxSizeInt}) if err != nil { return errors.Wrapf(err, "failed to set MaxSize to user %q", p.cephUserName) } @@ -685,7 +684,7 @@ func (p Provisioner) updateAdditionalSettings(ob *bktv1alpha1.ObjectBucket) erro return errors.Wrapf(err, "failed to parse maxSize quota for user %q", p.cephUserName) } } - objectUser, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: ob.Spec.Connection.AdditionalState[cephUser]}) + objectUser, err := p.adminOpsClient.GetUser(p.clusterInfo.Context, admin.User{ID: ob.Spec.Connection.AdditionalState[cephUser]}) if err != nil { return errors.Wrapf(err, "failed to fetch user %q", p.cephUserName) } @@ -693,7 +692,7 @@ func (p Provisioner) updateAdditionalSettings(ob *bktv1alpha1.ObjectBucket) erro (maxObjects == "" || maxObjectsInt64 < 0) && (maxSize == "" || maxSizeInt64 < 0) { quotaEnabled = false - err = p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, Enabled: "aEnabled}) + err = p.adminOpsClient.SetUserQuota(p.clusterInfo.Context, admin.QuotaSpec{UID: p.cephUserName, Enabled: "aEnabled}) if err != nil { return errors.Wrapf(err, "failed to disable quota to user %q", p.cephUserName) } @@ -712,7 +711,7 @@ func (p Provisioner) updateAdditionalSettings(ob *bktv1alpha1.ObjectBucket) erro if maxSize != "" && (maxSizeInt64 != *objectUser.UserQuota.MaxSize) { quotaSpec.MaxSize = &maxSizeInt64 } - err = p.adminOpsClient.SetUserQuota(context.TODO(), quotaSpec) + err = p.adminOpsClient.SetUserQuota(p.clusterInfo.Context, quotaSpec) if err != nil { return errors.Wrapf(err, "failed to update quota to user %q", p.cephUserName) } diff --git a/pkg/operator/ceph/object/bucket/rgw-handlers.go b/pkg/operator/ceph/object/bucket/rgw-handlers.go index b43b14171..bcce7954e 100644 --- a/pkg/operator/ceph/object/bucket/rgw-handlers.go +++ b/pkg/operator/ceph/object/bucket/rgw-handlers.go @@ -1,7 +1,6 @@ package bucket import ( - "context" "fmt" "github.com/ceph/go-ceph/rgw/admin" @@ -9,7 +8,7 @@ import ( ) func (p *Provisioner) bucketExists(name string) (bool, error) { - _, err := p.adminOpsClient.GetBucketInfo(context.TODO(), admin.Bucket{Bucket: name}) + _, err := p.adminOpsClient.GetBucketInfo(p.clusterInfo.Context, admin.Bucket{Bucket: name}) if err != nil { if errors.Is(err, admin.ErrNoSuchBucket) { return false, nil @@ -20,7 +19,7 @@ func (p *Provisioner) bucketExists(name string) (bool, error) { } func (p *Provisioner) userExists(name string) (bool, error) { - _, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: name}) + _, err := p.adminOpsClient.GetUser(p.clusterInfo.Context, admin.User{ID: name}) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { return false, nil @@ -50,10 +49,10 @@ func (p *Provisioner) createCephUser(username string) (accKey string, secKey str } var u admin.User - u, err = p.adminOpsClient.GetUser(context.TODO(), userConfig) + u, err = p.adminOpsClient.GetUser(p.clusterInfo.Context, userConfig) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { - u, err = p.adminOpsClient.CreateUser(context.TODO(), userConfig) + u, err = p.adminOpsClient.CreateUser(p.clusterInfo.Context, userConfig) if err != nil { return "", "", errors.Wrapf(err, "failed to create ceph object user %v", userConfig.ID) } @@ -96,7 +95,7 @@ func (p *Provisioner) deleteOBCResource(bucketName string) error { if len(bucketName) > 0 { // delete bucket with purge option to remove all objects thePurge := true - err := p.adminOpsClient.RemoveBucket(context.TODO(), admin.Bucket{Bucket: bucketName, PurgeObject: &thePurge}) + err := p.adminOpsClient.RemoveBucket(p.clusterInfo.Context, admin.Bucket{Bucket: bucketName, PurgeObject: &thePurge}) if err == nil { logger.Infof("bucket %q successfully deleted", p.bucketName) } else if errors.Is(err, admin.ErrNoSuchBucket) { @@ -107,7 +106,7 @@ func (p *Provisioner) deleteOBCResource(bucketName string) error { } } if len(p.cephUserName) > 0 { - err := p.adminOpsClient.RemoveUser(context.TODO(), admin.User{ID: p.cephUserName}) + err := p.adminOpsClient.RemoveUser(p.clusterInfo.Context, admin.User{ID: p.cephUserName}) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { logger.Warningf("user %q does not exist, nothing to delete. %v", p.cephUserName, err) diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index 9fd4ed6ef..e9f03b18e 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -17,20 +17,18 @@ limitations under the License. package bucket import ( - "context" "crypto/rand" "github.com/coreos/pkg/capnslog" bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1" "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + cephObject "github.com/rook/rook/pkg/operator/ceph/object" storagev1 "k8s.io/api/storage/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" - - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - cephObject "github.com/rook/rook/pkg/operator/ceph/object" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-bucket-prov") @@ -43,9 +41,9 @@ const ( objectStoreEndpoint = "endpoint" ) -func NewBucketController(cfg *rest.Config, p *Provisioner) (*provisioner.Provisioner, error) { +func NewBucketController(cfg *rest.Config, p *Provisioner, data map[string]string) (*provisioner.Provisioner, error) { const allNamespaces = "" - provName := cephObject.GetObjectBucketProvisioner(p.context, p.clusterInfo.Namespace) + provName := cephObject.GetObjectBucketProvisioner(data, p.clusterInfo.Namespace) logger.Infof("ceph bucket provisioner launched watching for provisioner %q", provName) return provisioner.NewProvisioner(cfg, provName, p, allNamespaces) @@ -78,7 +76,7 @@ func getCephUser(ob *bktv1alpha1.ObjectBucket) string { } func (p *Provisioner) getObjectStore() (*cephv1.CephObjectStore, error) { - ctx := context.TODO() + ctx := p.clusterInfo.Context // Verify the object store API object actually exists store, err := p.context.RookClientset.CephV1().CephObjectStores(p.clusterInfo.Namespace).Get(ctx, p.objectStoreName, metav1.GetOptions{}) if err != nil { @@ -91,7 +89,7 @@ func (p *Provisioner) getObjectStore() (*cephv1.CephObjectStore, error) { } func (p *Provisioner) getCephCluster() (*cephv1.CephCluster, error) { - cephCluster, err := p.context.RookClientset.CephV1().CephClusters(p.clusterInfo.Namespace).List(context.TODO(), metav1.ListOptions{}) + cephCluster, err := p.context.RookClientset.CephV1().CephClusters(p.clusterInfo.Namespace).List(p.clusterInfo.Context, metav1.ListOptions{}) if err != nil { return nil, errors.Wrapf(err, "failed to list ceph clusters in namespace %q", p.clusterInfo.Namespace) } diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 68d8de84c..24e48095b 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -32,7 +32,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" - opconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" "github.com/rook/rook/pkg/operator/k8sutil" @@ -74,6 +74,8 @@ var controllerTypeMeta = metav1.TypeMeta{ APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), } +var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion + // ReconcileCephObjectStore reconciles a cephObjectStore object type ReconcileCephObjectStore struct { client client.Client @@ -82,31 +84,36 @@ type ReconcileCephObjectStore struct { context *clusterd.Context clusterSpec *cephv1.ClusterSpec clusterInfo *cephclient.ClusterInfo - objectStoreChannels map[string]*objectStoreHealth + objectStoreContexts map[string]*objectStoreHealth recorder *k8sutil.EventReporter + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } type objectStoreHealth struct { - stopChan chan struct{} - monitoringRunning bool + internalCtx context.Context + internalCancel context.CancelFunc + started bool } // Add creates a new cephObjectStore Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { context.Client = mgr.GetClient() return &ReconcileCephObjectStore{ client: mgr.GetClient(), scheme: mgr.GetScheme(), context: context, bktclient: bktclient.NewForConfigOrDie(context.KubeConfig), - objectStoreChannels: make(map[string]*objectStoreHealth), + objectStoreContexts: make(map[string]*objectStoreHealth), recorder: k8sutil.NewEventReporter(mgr.GetEventRecorderFor("rook-" + controllerName)), + opManagerContext: opManagerContext, + opConfig: opConfig, } } @@ -135,25 +142,6 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } } - // Build Handler function to return the list of ceph object - // This is used by the watchers below - handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephObjectStoreList{}, mgr.GetScheme()) - if err != nil { - return err - } - - // Watch for CephCluster Spec changes that we want to propagate to us - err = c.Watch(&source.Kind{Type: &cephv1.CephCluster{ - TypeMeta: metav1.TypeMeta{ - Kind: opcontroller.ClusterResource.Kind, - APIVersion: opcontroller.ClusterResource.APIVersion, - }, - }, - }, handler.EnqueueRequestsFromMapFunc(handlerFunc), opcontroller.WatchCephClusterPredicate()) - if err != nil { - return err - } - return nil } @@ -171,7 +159,7 @@ func (r *ReconcileCephObjectStore) Reconcile(context context.Context, request re func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconcile.Result, *cephv1.CephObjectStore, error) { // Fetch the cephObjectStore instance cephObjectStore := &cephv1.CephObjectStore{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephObjectStore) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectStore) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephObjectStore resource not found. Ignoring since object must be deleted.") @@ -219,81 +207,76 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci // Initialize the channel for this object store // This allows us to track multiple ObjectStores in the same namespace - _, ok := r.objectStoreChannels[cephObjectStore.Name] + _, ok := r.objectStoreContexts[cephObjectStore.Name] if !ok { - r.objectStoreChannels[cephObjectStore.Name] = &objectStoreHealth{ - stopChan: make(chan struct{}), - monitoringRunning: false, + internalCtx, internalCancel := context.WithCancel(r.opManagerContext) + r.objectStoreContexts[cephObjectStore.Name] = &objectStoreHealth{ + internalCtx: internalCtx, + internalCancel: internalCancel, } } // Populate clusterInfo during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, cephObjectStore, errors.Wrap(err, "failed to populate cluster info") } - // Populate CephVersion - currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) - if err != nil { - if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { - logger.Info(opcontroller.OperatorNotInitializedMessage) - return opcontroller.WaitForRequeueIfOperatorNotInitialized, cephObjectStore, nil - } - return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) - } - r.clusterInfo.CephVersion = currentCephVersion - // DELETE: the CR was deleted if !cephObjectStore.GetDeletionTimestamp().IsZero() { updateStatus(r.client, request.NamespacedName, cephv1.ConditionDeleting, buildStatusInfo(cephObjectStore)) - if ok { - select { - case <-r.objectStoreChannels[cephObjectStore.Name].stopChan: - // channel was closed - break - default: - // Close the channel to stop the healthcheck of the endpoint - close(r.objectStoreChannels[cephObjectStore.Name].stopChan) - } + // Detect running Ceph version + runningCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, config.MonType) + if err != nil { + return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to retrieve current ceph %q version", config.MonType) + } + r.clusterInfo.CephVersion = runningCephVersion + r.clusterInfo.Context = r.opManagerContext - // get the latest version of the object now that the health checker is stopped - err := r.client.Get(context.TODO(), request.NamespacedName, cephObjectStore) - if err != nil { - return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to get latest CephObjectStore %q", request.NamespacedName.String()) - } + if _, ok := r.objectStoreContexts[cephObjectStore.Name]; ok { + if r.objectStoreContexts[cephObjectStore.Name].internalCtx.Err() == nil { + // get the latest version of the object now that the health checker is stopped + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectStore) + if err != nil { + return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to get latest CephObjectStore %q", request.NamespacedName.String()) + } - objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) - if err != nil { - return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get object context") - } + objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) + if err != nil { + return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get object context") + } - opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) - if err != nil { - return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get admin ops API context") - } + opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) + if err != nil { + return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get admin ops API context") + } - deps, err := CephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx) - if err != nil { - return reconcile.Result{}, cephObjectStore, err - } - if !deps.Empty() { - err := reporting.ReportDeletionBlockedDueToDependents(logger, r.client, cephObjectStore, deps) - return opcontroller.WaitForRequeueIfFinalizerBlocked, cephObjectStore, err - } - reporting.ReportDeletionNotBlockedDueToDependents(logger, r.client, r.recorder, cephObjectStore) + deps, err := CephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx) + if err != nil { + return reconcile.Result{}, cephObjectStore, err + } + if !deps.Empty() { + err := reporting.ReportDeletionBlockedDueToDependents(logger, r.client, cephObjectStore, deps) + return opcontroller.WaitForRequeueIfFinalizerBlocked, cephObjectStore, err + } + reporting.ReportDeletionNotBlockedDueToDependents(logger, r.client, r.recorder, cephObjectStore) - cfg := clusterConfig{ - context: r.context, - store: cephObjectStore, - clusterSpec: r.clusterSpec, - clusterInfo: r.clusterInfo, - } - cfg.deleteStore() + // Cancel the context to stop monitoring the health of the object store + r.objectStoreContexts[cephObjectStore.Name].internalCancel() + r.objectStoreContexts[cephObjectStore.Name].started = false - // Remove object store from the map - delete(r.objectStoreChannels, cephObjectStore.Name) + cfg := clusterConfig{ + context: r.context, + store: cephObjectStore, + clusterSpec: r.clusterSpec, + clusterInfo: r.clusterInfo, + } + cfg.deleteStore() + + // Remove object store from the map + delete(r.objectStoreContexts, cephObjectStore.Name) + } } // Remove finalizer @@ -306,6 +289,35 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci return reconcile.Result{}, cephObjectStore, nil } + // Detect desired CephCluster version + runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion( + r.opConfig.Image, + cephObjectStore.Namespace, + controllerName, + k8sutil.NewOwnerInfo(cephObjectStore, r.scheme), + r.context, + r.clusterSpec, + r.clusterInfo, + ) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, cephObjectStore, nil + } + return reconcile.Result{}, cephObjectStore, errors.Wrap(err, "failed to detect running and desired ceph version") + } + + // If the version of the Ceph monitor differs from the CephCluster CR image version we assume + // the cluster is being upgraded. So the controller will just wait for the upgrade to finish and + // then versions should match. Obviously using the cmd reporter job adds up to the deployment time + if !reflect.DeepEqual(runningCephVersion, desiredCephVersion) { + // Upgrade is in progress, let's wait for the mons to be done + return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, + cephObjectStore, + opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion) + } + r.clusterInfo.CephVersion = *desiredCephVersion + // validate the store settings if err := r.validateStore(cephObjectStore); err != nil { return reconcile.Result{}, cephObjectStore, errors.Wrapf(err, "invalid object store %q arguments", cephObjectStore.Name) @@ -315,7 +327,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci // we force the pg_autoscale_mode to "on" _, propertyExists := cephObjectStore.Spec.DataPool.Parameters[cephclient.PgAutoscaleModeProperty] if mgr.IsModuleInSpec(cephCluster.Spec.Mgr.Modules, mgr.PgautoscalerModuleName) && - !currentCephVersion.IsAtLeastOctopus() && + !desiredCephVersion.IsAtLeastOctopus() && !propertyExists { if len(cephObjectStore.Spec.DataPool.Parameters) == 0 { cephObjectStore.Spec.DataPool.Parameters = make(map[string]string) @@ -349,7 +361,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c store: cephObjectStore, rookVersion: r.clusterSpec.CephVersion.Image, clusterSpec: r.clusterSpec, - DataPathMap: opconfig.NewStatelessDaemonDataPathMap(opconfig.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.clusterSpec.DataDirHostPath), + DataPathMap: config.NewStatelessDaemonDataPathMap(config.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.clusterSpec.DataDirHostPath), client: r.client, ownerInfo: ownerInfo, } @@ -466,7 +478,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 if cephObjectStore.Spec.IsMultisite() { zoneName := cephObjectStore.Spec.Zone.Name zone := &cephv1.CephObjectZone{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: zoneName, Namespace: cephObjectStore.Namespace}, zone) + err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: zoneName, Namespace: cephObjectStore.Namespace}, zone) if err != nil { if kerrors.IsNotFound(err) { return "", "", "", waitForRequeueIfObjectStoreNotReady, err @@ -476,7 +488,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 logger.Debugf("CephObjectZone resource %s found", zone.Name) zonegroup := &cephv1.CephObjectZoneGroup{} - err = r.client.Get(context.TODO(), types.NamespacedName{Name: zone.Spec.ZoneGroup, Namespace: cephObjectStore.Namespace}, zonegroup) + err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: zone.Spec.ZoneGroup, Namespace: cephObjectStore.Namespace}, zonegroup) if err != nil { if kerrors.IsNotFound(err) { return "", "", "", waitForRequeueIfObjectStoreNotReady, err @@ -486,7 +498,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 logger.Debugf("CephObjectZoneGroup resource %s found", zonegroup.Name) realm := &cephv1.CephObjectRealm{} - err = r.client.Get(context.TODO(), types.NamespacedName{Name: zonegroup.Spec.Realm, Namespace: cephObjectStore.Namespace}, realm) + err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: zonegroup.Spec.Realm, Namespace: cephObjectStore.Namespace}, realm) if err != nil { if kerrors.IsNotFound(err) { return "", "", "", waitForRequeueIfObjectStoreNotReady, err @@ -503,8 +515,8 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) { // Start monitoring object store - if r.objectStoreChannels[objectstore.Name].monitoringRunning { - logger.Debug("external rgw endpoint monitoring go routine already running!") + if r.objectStoreContexts[objectstore.Name].started { + logger.Info("external rgw endpoint monitoring go routine already running!") return } @@ -515,8 +527,6 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec } logger.Info("starting rgw healthcheck") - go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan) - - // Set the monitoring flag so we don't start more than one go routine - r.objectStoreChannels[objectstore.Name].monitoringRunning = true + go rgwChecker.checkObjectStore(r.objectStoreContexts[objectstore.Name].internalCtx) + r.objectStoreContexts[objectstore.Name].started = true } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index 25a0e3a86..6c182cae8 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -28,6 +28,8 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -281,11 +283,6 @@ func TestCephObjectStoreController(t *testing.T) { capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - // - // TEST 1 SETUP - // - // FAILURE because no CephCluster - // // A Pool resource with metadata and spec. objectStore := &cephv1.CephObjectStore{ ObjectMeta: metav1.ObjectMeta{ @@ -307,9 +304,6 @@ func TestCephObjectStoreController(t *testing.T) { if args[0] == "status" { return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil } - if args[0] == "versions" { - return dummyVersionsRaw, nil - } return "", nil }, } @@ -332,8 +326,9 @@ func TestCephObjectStoreController(t *testing.T) { client: cl, scheme: s, context: c, - objectStoreChannels: make(map[string]*objectStoreHealth), + objectStoreContexts: make(map[string]*objectStoreHealth), recorder: k8sutil.NewEventReporter(record.NewFakeRecorder(5)), + opManagerContext: context.TODO(), } // Mock request to simulate Reconcile() being called on an event for a @@ -344,17 +339,7 @@ func TestCephObjectStoreController(t *testing.T) { Namespace: namespace, }, } - logger.Info("STARTING PHASE 1") - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 1 DONE") - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // cephCluster := &cephv1.CephCluster{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, @@ -367,68 +352,74 @@ func TestCephObjectStoreController(t *testing.T) { }, }, } - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileCephObjectStore object with the scheme and fake client. - r = &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreChannels: make(map[string]*objectStoreHealth), - recorder: k8sutil.NewEventReporter(record.NewFakeRecorder(5)), + + currentAndDesiredCephVersion = func(rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*cephver.CephVersion, *cephver.CephVersion, error) { + return &cephver.Pacific, &cephver.Pacific, nil } - logger.Info("STARTING PHASE 2") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 2 DONE") - // - // TEST 3: - // - // SUCCESS! The CephCluster is ready - // + t.Run("error - no ceph cluster", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) + t.Run("error - ceph cluster not ready", func(t *testing.T) { + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileCephObjectStore object with the scheme and fake client. + r = &ReconcileCephObjectStore{ + client: cl, + scheme: s, + context: c, + objectStoreContexts: make(map[string]*objectStoreHealth), + recorder: k8sutil.NewEventReporter(record.NewFakeRecorder(5)), + opManagerContext: context.TODO(), + } + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - // Add ready status to the CephCluster - cephCluster.Status.Phase = k8sutil.ReadyStatus - cephCluster.Status.CephStatus.Health = "HEALTH_OK" + t.Run("success - object store is running", func(t *testing.T) { + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Add ready status to the CephCluster + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" - // Override executor with the new ceph status and more content - executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "status" { - return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil - } - if args[0] == "auth" && args[1] == "get-or-create-key" { - return rgwCephAuthGetOrCreateKey, nil - } - if args[0] == "versions" { - return dummyVersionsRaw, nil - } - if args[0] == "osd" && args[1] == "lspools" { - // ceph actually outputs this all on one line, but this parses the same - return `[ + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + // Override executor with the new ceph status and more content + executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return rgwCephAuthGetOrCreateKey, nil + } + if args[0] == "versions" { + return dummyVersionsRaw, nil + } + if args[0] == "osd" && args[1] == "lspools" { + // ceph actually outputs this all on one line, but this parses the same + return `[ {"poolnum":1,"poolname":"replicapool"}, {"poolnum":2,"poolname":"device_health_metrics"}, {"poolnum":3,"poolname":".rgw.root"}, @@ -439,49 +430,50 @@ func TestCephObjectStoreController(t *testing.T) { {"poolnum":8,"poolname":"my-store.rgw.meta"}, {"poolnum":9,"poolname":"my-store.rgw.buckets.data"} ]`, nil - } - return "", nil - }, - MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { - if args[0] == "realm" && args[1] == "list" { - return realmListJSON, nil - } - if args[0] == "realm" && args[1] == "get" { - return realmGetJSON, nil - } - if args[0] == "zonegroup" && args[1] == "get" { - return zoneGroupGetJSON, nil - } - if args[0] == "zone" && args[1] == "get" { - return zoneGetJSON, nil - } - if args[0] == "user" { - return userCreateJSON, nil - } - return "", nil - }, - } - c.Executor = executor + } + return "", nil + }, + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "realm" && args[1] == "list" { + return realmListJSON, nil + } + if args[0] == "realm" && args[1] == "get" { + return realmGetJSON, nil + } + if args[0] == "zonegroup" && args[1] == "get" { + return zoneGroupGetJSON, nil + } + if args[0] == "zone" && args[1] == "get" { + return zoneGetJSON, nil + } + if args[0] == "user" { + return userCreateJSON, nil + } + return "", nil + }, + } + c.Executor = executor - // Create a ReconcileCephObjectStore object with the scheme and fake client. - r = &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreChannels: make(map[string]*objectStoreHealth), - recorder: k8sutil.NewEventReporter(record.NewFakeRecorder(5)), - } + // Create a ReconcileCephObjectStore object with the scheme and fake client. + r = &ReconcileCephObjectStore{ + client: cl, + scheme: s, + context: c, + objectStoreContexts: make(map[string]*objectStoreHealth), + recorder: k8sutil.NewEventReporter(record.NewFakeRecorder(5)), + opManagerContext: context.TODO(), + } + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, objectStore) + assert.NoError(t, err) + assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore) + assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore) + assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore) + }) - logger.Info("STARTING PHASE 3") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, objectStore) - assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore) - assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore) - assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore) - logger.Info("PHASE 3 DONE") } func TestCephObjectStoreControllerMultisite(t *testing.T) { @@ -635,8 +627,9 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { client: cl, scheme: s, context: c, - objectStoreChannels: make(map[string]*objectStoreHealth), + objectStoreContexts: make(map[string]*objectStoreHealth), recorder: k8sutil.NewEventReporter(record.NewFakeRecorder(5)), + opManagerContext: context.TODO(), } _, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) diff --git a/pkg/operator/ceph/object/dependents.go b/pkg/operator/ceph/object/dependents.go index 379660831..cf7d6cb4b 100644 --- a/pkg/operator/ceph/object/dependents.go +++ b/pkg/operator/ceph/object/dependents.go @@ -17,7 +17,6 @@ limitations under the License. package object import ( - "context" "fmt" "github.com/pkg/errors" @@ -53,7 +52,7 @@ func CephObjectStoreDependents( } // CephObjectStoreUsers - users, err := clusterdCtx.RookClientset.CephV1().CephObjectStoreUsers(store.Namespace).List(context.TODO(), metav1.ListOptions{}) + users, err := clusterdCtx.RookClientset.CephV1().CephObjectStoreUsers(store.Namespace).List(clusterInfo.Context, metav1.ListOptions{}) if err != nil { return deps, errors.Wrapf(err, "%s. failed to list CephObjectStoreUsers for CephObjectStore %q", baseErrMsg, nsName) } @@ -93,7 +92,7 @@ func getBucketDependents( } // buckets (including lib-bucket-provisioner buckets and COSI buckets) - buckets, err := opsCtx.AdminOpsClient.ListBuckets(context.TODO()) + buckets, err := opsCtx.AdminOpsClient.ListBuckets(clusterInfo.Context) if err != nil { return errors.Wrapf(err, "failed to list buckets in CephObjectStore %q", nsName) } diff --git a/pkg/operator/ceph/object/health.go b/pkg/operator/ceph/object/health.go index 00f36dea8..b3b051a7d 100644 --- a/pkg/operator/ceph/object/health.go +++ b/pkg/operator/ceph/object/health.go @@ -88,7 +88,7 @@ func newBucketChecker( } // checkObjectStore periodically checks the health of the cluster -func (c *bucketChecker) checkObjectStore(stopCh chan struct{}) { +func (c *bucketChecker) checkObjectStore(context context.Context) { // check the object store health immediately before starting the loop err := c.checkObjectStoreHealth() if err != nil { @@ -98,7 +98,7 @@ func (c *bucketChecker) checkObjectStore(stopCh chan struct{}) { for { select { - case <-stopCh: + case <-context.Done(): // purge bucket and s3 user // Needed for external mode where in converged everything goes away with the CR deletion c.cleanupHealthCheck() @@ -146,10 +146,10 @@ func (c *bucketChecker) checkObjectStoreHealth() error { // Create checker user logger.Debugf("creating s3 user object %q for object store %q health check", userConfig.ID, c.namespacedName.Name) var user admin.User - user, err := c.objContext.AdminOpsClient.GetUser(context.TODO(), userConfig) + user, err := c.objContext.AdminOpsClient.GetUser(c.objContext.clusterInfo.Context, userConfig) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { - user, err = c.objContext.AdminOpsClient.CreateUser(context.TODO(), userConfig) + user, err = c.objContext.AdminOpsClient.CreateUser(c.objContext.clusterInfo.Context, userConfig) if err != nil { return errors.Wrapf(err, "failed to create from ceph object user %v", userConfig.ID) } @@ -202,7 +202,7 @@ func (c *bucketChecker) cleanupHealthCheck() { logger.Infof("deleting object %q from bucket %q in object store %q", s3HealthCheckObjectKey, bucketToDelete, c.namespacedName.Name) thePurge := true - err := c.objContext.AdminOpsClient.RemoveBucket(context.TODO(), admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge}) + err := c.objContext.AdminOpsClient.RemoveBucket(c.objContext.clusterInfo.Context, admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge}) if err != nil { if errors.Is(err, admin.ErrNoSuchBucket) { // opinion: "not found" is not an error @@ -213,7 +213,7 @@ func (c *bucketChecker) cleanupHealthCheck() { } userToDelete := genUserCheckerConfig(c.objContext.UID) - err = c.objContext.AdminOpsClient.RemoveUser(context.TODO(), userToDelete) + err = c.objContext.AdminOpsClient.RemoveUser(c.objContext.clusterInfo.Context, userToDelete) if err != nil && !errors.Is(err, admin.ErrNoSuchUser) { logger.Errorf("failed to delete object user %q for object store %q. %v", userToDelete.ID, c.namespacedName.Name, err) } diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 32f1070a7..007f36f07 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -31,7 +31,6 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/config" - opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/exec" @@ -822,15 +821,11 @@ func poolName(storeName, poolName string) string { } // GetObjectBucketProvisioner returns the bucket provisioner name appended with operator namespace if OBC is watching on it -func GetObjectBucketProvisioner(c *clusterd.Context, namespace string) string { +func GetObjectBucketProvisioner(data map[string]string, namespace string) string { provName := bucketProvisionerName - obcWatchOnNamespace, err := k8sutil.GetOperatorSetting(c.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") - if err != nil { - logger.Warning("failed to verify if obc should watch the operator namespace or all of them, watching all") - } else { - if strings.EqualFold(obcWatchOnNamespace, "true") { - provName = fmt.Sprintf("%s.%s", namespace, bucketProvisionerName) - } + obcWatchOnNamespace := k8sutil.GetValue(data, "ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") + if strings.EqualFold(obcWatchOnNamespace, "true") { + provName = fmt.Sprintf("%s.%s", namespace, bucketProvisionerName) } return provName } diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 2bc03cfe0..c62afaedb 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -31,9 +31,6 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" ) const ( @@ -186,7 +183,7 @@ func deleteStore(t *testing.T, name string, existingStores string, expectedDelet executor.MockExecuteCommandWithTimeout = executorFuncWithTimeout executor.MockExecuteCommandWithOutput = executorFunc executor.MockExecuteCommandWithCombinedOutput = executorFunc - context := &Context{Context: &clusterd.Context{Executor: executor}, Name: "myobj", clusterInfo: &client.ClusterInfo{Namespace: "ns"}} + context := &Context{Context: &clusterd.Context{Executor: executor}, Name: "myobj", clusterInfo: client.AdminClusterInfo("mycluster")} // Delete an object store without deleting the pools spec := cephv1.ObjectStoreSpec{} @@ -217,35 +214,20 @@ func deleteStore(t *testing.T, name string, existingStores string, expectedDelet } func TestGetObjectBucketProvisioner(t *testing.T) { - ctx := context.TODO() - k8s := fake.NewSimpleClientset() - operatorSettingConfigMapName := "rook-ceph-operator-config" testNamespace := "test-namespace" - watchOperatorNamespace := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true"} - ignoreOperatorNamespace := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "false"} - context := &clusterd.Context{Clientset: k8s} os.Setenv(k8sutil.PodNamespaceEnvVar, testNamespace) - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: operatorSettingConfigMapName, - Namespace: testNamespace, - }, - Data: watchOperatorNamespace, - } + t.Run("watch single namespace", func(t *testing.T) { + data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true"} + bktprovisioner := GetObjectBucketProvisioner(data, testNamespace) + assert.Equal(t, fmt.Sprintf("%s.%s", testNamespace, bucketProvisionerName), bktprovisioner) + }) - _, err := k8s.CoreV1().ConfigMaps(testNamespace).Create(ctx, cm, metav1.CreateOptions{}) - assert.NoError(t, err) - - bktprovisioner := GetObjectBucketProvisioner(context, testNamespace) - assert.Equal(t, fmt.Sprintf("%s.%s", testNamespace, bucketProvisionerName), bktprovisioner) - - cm.Data = ignoreOperatorNamespace - _, err = k8s.CoreV1().ConfigMaps(testNamespace).Update(ctx, cm, metav1.UpdateOptions{}) - assert.NoError(t, err) - - bktprovisioner = GetObjectBucketProvisioner(context, testNamespace) - assert.Equal(t, bucketProvisionerName, bktprovisioner) + t.Run("watch all namespaces", func(t *testing.T) { + data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "false"} + bktprovisioner := GetObjectBucketProvisioner(data, testNamespace) + assert.Equal(t, bucketProvisionerName, bktprovisioner) + }) } func TestDashboard(t *testing.T) { @@ -261,10 +243,13 @@ func TestDashboard(t *testing.T) { return "", nil }, } - context := &clusterd.Context{Executor: executor} - objContext := NewContext(context, &client.ClusterInfo{Namespace: "mycluster", - CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 9}}, + objContext := NewContext(&clusterd.Context{Executor: executor}, &client.ClusterInfo{ + Namespace: "mycluster", + CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 9}, + Context: context.TODO(), + }, storeName) + checkdashboard, err := checkDashboardUser(objContext) assert.NoError(t, err) assert.False(t, checkdashboard) @@ -284,9 +269,11 @@ func TestDashboard(t *testing.T) { assert.True(t, checkdashboard) disableRGWDashboard(objContext) - context = &clusterd.Context{Executor: executor} - objContext = NewContext(context, &client.ClusterInfo{Namespace: "mycluster", - CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 10}}, + objContext = NewContext(&clusterd.Context{Executor: executor}, &client.ClusterInfo{ + Namespace: "mycluster", + CephVersion: cephver.CephVersion{Major: 15, Minor: 2, Extra: 10}, + Context: context.TODO(), + }, storeName) err = enableRGWDashboard(objContext) assert.Nil(t, err) diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 85aa8f2c7..2aa3af1a7 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -72,24 +72,26 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileObjectRealm reconciles a ObjectRealm object type ReconcileObjectRealm struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context } // Add creates a new CephObjectRealm Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { return &ReconcileObjectRealm{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, } } @@ -127,7 +129,7 @@ func (r *ReconcileObjectRealm) Reconcile(context context.Context, request reconc func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephObjectRealm instance cephObjectRealm := &cephv1.CephObjectRealm{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephObjectRealm) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectRealm) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.") @@ -139,7 +141,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R // The CR was just created, initializing status fields if cephObjectRealm.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) } // Make sure a CephCluster is present otherwise do nothing @@ -162,7 +164,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R } // Populate clusterInfo during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } @@ -170,12 +172,12 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R // validate the realm settings err = validateRealmCR(cephObjectRealm) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "invalid CephObjectRealm CR %q", cephObjectRealm.Name) } // Start object reconciliation, updating status for this - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcilingStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcilingStatus) // Create/Pull Ceph Realm if cephObjectRealm.Spec.IsPullRealm() { @@ -197,7 +199,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue logger.Debug("realm done reconciling") @@ -251,7 +253,6 @@ func (r *ReconcileObjectRealm) createCephRealm(realm *cephv1.CephObjectRealm) (r } func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (reconcile.Result, error) { - ctx := context.TODO() logger.Debugf("generating access and secret keys for new realm %q", realm.Name) // the realm's secret key and access key are randomly generated and then encoded to base64 @@ -288,7 +289,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of rgw secret %q", secret.Name) } - if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(r.opManagerContext, secret, metav1.CreateOptions{}); err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to save rgw secrets") } logger.Infof("secrets for keys have been created for realm %q", realm.Name) @@ -308,14 +309,14 @@ func validateRealmCR(u *cephv1.CephObjectRealm) error { } func (r *ReconcileObjectRealm) setFailedStatus(name types.NamespacedName, errMessage string, err error) (reconcile.Result, error) { - updateStatus(r.client, name, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, name, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "%s", errMessage) } // updateStatus updates an realm with a given status -func updateStatus(client client.Client, name types.NamespacedName, status string) { +func (r *ReconcileObjectRealm) updateStatus(client client.Client, name types.NamespacedName, status string) { objectRealm := &cephv1.CephObjectRealm{} - if err := client.Get(context.TODO(), name, objectRealm); err != nil { + if err := client.Get(r.opManagerContext, name, objectRealm); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectRealm resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 9865cb5c8..058c5cdb4 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -80,7 +80,6 @@ func (c *clusterConfig) createOrUpdateStore(realmName, zoneGroupName, zoneName s } func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) error { - ctx := context.TODO() // backward compatibility, triggered during updates if c.store.Spec.Gateway.Instances < 1 { // Set the minimum of at least one instance @@ -153,7 +152,7 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) return errors.Wrapf(err, "failed to set annotation for deployment %q", deployment.Name) } - _, createErr := c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + _, createErr := c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Create(c.clusterInfo.Context, deployment, metav1.CreateOptions{}) if createErr != nil { if !kerrors.IsAlreadyExists(createErr) { return errors.Wrap(createErr, "failed to create rgw deployment") @@ -191,7 +190,7 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) // Delete the Secret key secretToRemove := c.generateSecretName(k8sutil.IndexToName(depIDToRemove)) - err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(ctx, secretToRemove, metav1.DeleteOptions{}) + err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.clusterInfo.Context, secretToRemove, metav1.DeleteOptions{}) if err != nil && !kerrors.IsNotFound(err) { logger.Warningf("failed to delete rgw secret %q. %v", secretToRemove, err) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index eadc3dbcd..7fc6dd4fb 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -17,7 +17,6 @@ limitations under the License. package object import ( - "context" "fmt" "path" "reflect" @@ -606,7 +605,7 @@ func (c *clusterConfig) generateVolumeSourceWithCaBundleSecret() (*v1.SecretVolu } func (c *clusterConfig) rgwTLSSecretType(secretName string) (v1.SecretType, error) { - rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, secretName, metav1.GetOptions{}) if rgwTlsSecret != nil { return rgwTlsSecret.Type, nil } diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index 0fb938f5d..1e1c983bf 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -68,27 +68,29 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileObjectStoreUser reconciles a ObjectStoreUser object type ReconcileObjectStoreUser struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - objContext *object.AdminOpsContext - userConfig *admin.User - cephClusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + objContext *object.AdminOpsContext + userConfig *admin.User + cephClusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context } // Add creates a new CephObjectStoreUser Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { return &ReconcileObjectStoreUser{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, } } @@ -135,7 +137,7 @@ func (r *ReconcileObjectStoreUser) Reconcile(context context.Context, request re func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephObjectStoreUser instance cephObjectStoreUser := &cephv1.CephObjectStoreUser{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephObjectStoreUser) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectStoreUser) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectStoreUser resource not found. Ignoring since object must be deleted.") @@ -153,7 +155,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci // The CR was just created, initializing status fields if cephObjectStoreUser.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) } // Make sure a CephCluster is present otherwise do nothing @@ -180,7 +182,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci r.cephClusterSpec = &cephCluster.Spec // Populate clusterInfo during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } @@ -200,7 +202,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci } logger.Debugf("ObjectStore resource not ready in namespace %q, retrying in %q. %v", request.NamespacedName.Namespace, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String(), err) - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return opcontroller.WaitForRequeueIfCephClusterNotReady, nil } @@ -229,26 +231,26 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci // validate the user settings err = r.validateUser(cephObjectStoreUser) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "invalid pool CR %q spec", cephObjectStoreUser.Name) } // CREATE/UPDATE CEPH USER reconcileResponse, err = r.reconcileCephUser(cephObjectStoreUser) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcileResponse, err } // CREATE/UPDATE KUBERNETES SECRET reconcileResponse, err = r.reconcileCephUserSecret(cephObjectStoreUser) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcileResponse, err } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue logger.Debug("done reconciling") @@ -270,10 +272,10 @@ func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectSt logCreateOrUpdate := fmt.Sprintf("retrieved existing ceph object user %q", u.Name) var user admin.User var err error - user, err = r.objContext.AdminOpsClient.GetUser(context.TODO(), *r.userConfig) + user, err = r.objContext.AdminOpsClient.GetUser(r.opManagerContext, *r.userConfig) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { - user, err = r.objContext.AdminOpsClient.CreateUser(context.TODO(), *r.userConfig) + user, err = r.objContext.AdminOpsClient.CreateUser(r.opManagerContext, *r.userConfig) if err != nil { return errors.Wrapf(err, "failed to create ceph object user %v", &r.userConfig.ID) } @@ -283,7 +285,7 @@ func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectSt } } else if *user.MaxBuckets != *r.userConfig.MaxBuckets { // TODO: handle update for user capabilities, depends on https://github.com/ceph/go-ceph/pull/571 - user, err = r.objContext.AdminOpsClient.ModifyUser(context.TODO(), *r.userConfig) + user, err = r.objContext.AdminOpsClient.ModifyUser(r.opManagerContext, *r.userConfig) if err != nil { return errors.Wrapf(err, "failed to create ceph object user %v", &r.userConfig.ID) } @@ -309,7 +311,7 @@ func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectSt MaxSize: &maxSize, MaxObjects: &maxObjects, } - err = r.objContext.AdminOpsClient.SetUserQuota(context.TODO(), userQuota) + err = r.objContext.AdminOpsClient.SetUserQuota(r.opManagerContext, userQuota) if err != nil { return errors.Wrapf(err, "failed to set quotas for user %q", u.Name) } @@ -477,7 +479,7 @@ func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *c func (r *ReconcileObjectStoreUser) getObjectStore(storeName string) (*cephv1.CephObjectStore, error) { // check if CephObjectStore CR is created objectStores := &cephv1.CephObjectStoreList{} - err := r.client.List(context.TODO(), objectStores) + err := r.client.List(r.opManagerContext, objectStores) if err != nil { if kerrors.IsNotFound(err) { return nil, errors.Wrapf(err, "CephObjectStore %q could not be found", storeName) @@ -505,7 +507,7 @@ func (r *ReconcileObjectStoreUser) getRgwPodList(cephObjectStoreUser *cephv1.Cep client.MatchingLabels(labelsForRgw(cephObjectStoreUser.Spec.Store)), } - err := r.client.List(context.TODO(), pods, listOpts...) + err := r.client.List(r.opManagerContext, pods, listOpts...) if err != nil { if kerrors.IsNotFound(err) { return pods, errors.Wrap(err, "no rgw pod could not be found") @@ -518,7 +520,7 @@ func (r *ReconcileObjectStoreUser) getRgwPodList(cephObjectStoreUser *cephv1.Cep // Delete the user func (r *ReconcileObjectStoreUser) deleteUser(u *cephv1.CephObjectStoreUser) error { - err := r.objContext.AdminOpsClient.RemoveUser(context.TODO(), admin.User{ID: u.Name}) + err := r.objContext.AdminOpsClient.RemoveUser(r.opManagerContext, admin.User{ID: u.Name}) if err != nil { if errors.Is(err, admin.ErrNoSuchUser) { logger.Warningf("user %q does not exist, nothing to remove", u.Name) @@ -552,9 +554,9 @@ func labelsForRgw(name string) map[string]string { } // updateStatus updates an object with a given status -func updateStatus(client client.Client, name types.NamespacedName, status string) { +func (r *ReconcileObjectStoreUser) updateStatus(client client.Client, name types.NamespacedName, status string) { user := &cephv1.CephObjectStoreUser{} - if err := client.Get(context.TODO(), name, user); err != nil { + if err := client.Get(r.opManagerContext, name, user); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectStoreUser resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/object/user/controller_test.go b/pkg/operator/ceph/object/user/controller_test.go index ad4550e50..1b429dae1 100644 --- a/pkg/operator/ceph/object/user/controller_test.go +++ b/pkg/operator/ceph/object/user/controller_test.go @@ -101,13 +101,7 @@ func TestCephObjectStoreUserController(t *testing.T) { ctx := context.TODO() // Set DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) - - // - // TEST 1 SETUP - // - // FAILURE because no CephCluster - // - // A Pool resource with metadata and spec. + cephObjectStore := &cephv1.CephObjectStore{} objectUser := &cephv1.CephObjectStoreUser{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -150,7 +144,7 @@ func TestCephObjectStoreUserController(t *testing.T) { // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() // Create a ReconcileObjectStoreUser object with the scheme and fake client. - r := &ReconcileObjectStoreUser{client: cl, scheme: s, context: c} + r := &ReconcileObjectStoreUser{client: cl, scheme: s, context: c, opManagerContext: ctx} // Mock request to simulate Reconcile() being called on an event for a // watched resource . @@ -160,188 +154,167 @@ func TestCephObjectStoreUserController(t *testing.T) { Namespace: namespace, }, } - logger.Info("STARTING PHASE 1") - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 1 DONE") - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // - cephCluster = &cephv1.CephCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Namespace: namespace, - }, - Status: cephv1.ClusterStatus{ - Phase: "", - CephStatus: &cephv1.CephStatus{ - Health: "", + t.Run("failure because no CephCluster", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) + + t.Run("failure CephCluster not ready", func(t *testing.T) { + cephCluster = &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, }, - }, - } - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileObjectStoreUser object with the scheme and fake client. - r = &ReconcileObjectStoreUser{client: cl, scheme: s, context: c} - logger.Info("STARTING PHASE 2") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 2 DONE") - - // - // TEST 3: - // - // FAILURE! The CephCluster is ready but NO rgw object - // - - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - - // Add ready status to the CephCluster - cephCluster.Status.Phase = k8sutil.ReadyStatus - cephCluster.Status.CephStatus.Health = "HEALTH_OK" - - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - - executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "status" { - return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil - } - return "", nil - }, - MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { - if args[0] == "user" { - return userCreateJSON, nil - } - return "", nil - }, - } - c.Executor = executor - - // Create a ReconcileObjectStoreUser object with the scheme and fake client. - r = &ReconcileObjectStoreUser{client: cl, scheme: s, context: c} - - logger.Info("STARTING PHASE 3") - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 3 DONE") - - // - // TEST 4: - // - // FAILURE! The CephCluster is ready - // Rgw object exists but NO pod are running - // - cephObjectStore := &cephv1.CephObjectStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: store, - Namespace: namespace, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "CephObjectStore", - }, - Spec: cephv1.ObjectStoreSpec{ - Gateway: cephv1.GatewaySpec{ - Port: 80, - }, - }, - Status: &cephv1.ObjectStoreStatus{ - Info: map[string]string{"endpoint": "http://rook-ceph-rgw-my-store.rook-ceph:80"}, - }, - } - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{}) - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStoreList{}) - object = append(object, cephObjectStore) - - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - // Create a ReconcileObjectStoreUser object with the scheme and fake client. - r = &ReconcileObjectStoreUser{client: cl, scheme: s, context: c} - - logger.Info("STARTING PHASE 4") - err = r.client.Get(context.TODO(), types.NamespacedName{Name: store, Namespace: namespace}, cephObjectStore) - assert.NoError(t, err, cephObjectStore) - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) - logger.Info("PHASE 4 DONE") - - // - // TEST 5: - // - // SUCCESS! The CephCluster is ready - // Rgw object exists and pods are running - // - rgwPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-rgw-my-store-a-5fd6fb4489-xv65v", - Namespace: namespace, - Labels: map[string]string{k8sutil.AppAttr: appName, "rgw": "my-store"}}} - - // Get the updated object. - logger.Info("STARTING PHASE 5") - // Create RGW pod - err = r.client.Create(context.TODO(), rgwPod) - assert.NoError(t, err) - - // Mock client - newMultisiteAdminOpsCtxFunc = func(objContext *cephobject.Context, spec *cephv1.ObjectStoreSpec) (*cephobject.AdminOpsContext, error) { - mockClient := &cephobject.MockClient{ - MockDo: func(req *http.Request) (*http.Response, error) { - if (req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=1000&uid=my-user" && (req.Method == http.MethodGet || req.Method == http.MethodPost) && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user") || - (req.URL.RawQuery == "enabled=false&format=json&max-objects=-1&max-size=-1"a="a-type=user&uid=my-user" && req.Method == http.MethodPut && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user") { - return &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(bytes.NewReader([]byte(userCreateJSON))), - }, nil - } - return nil, fmt.Errorf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, req.URL.Path) + Status: cephv1.ClusterStatus{ + Phase: "", + CephStatus: &cephv1.CephStatus{ + Health: "", + }, }, } - context, err := cephobject.NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileObjectStoreUser object with the scheme and fake client. + r = &ReconcileObjectStoreUser{client: cl, scheme: s, context: c, opManagerContext: ctx} + res, err := r.Reconcile(ctx, req) assert.NoError(t, err) - adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient) + assert.True(t, res.Requeue) + }) + + t.Run("failure CephCluster is ready but NO rgw object", func(t *testing.T) { + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) assert.NoError(t, err) - return &cephobject.AdminOpsContext{ - Context: *context, - AdminOpsUserAccessKey: "53S6B9S809NUP19IJ2K3", - AdminOpsUserSecretKey: "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", - AdminOpsClient: adminClient, - }, nil - } + // Add ready status to the CephCluster + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" - // Run reconcile - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, objectUser) - assert.NoError(t, err) - assert.Equal(t, "Ready", objectUser.Status.Phase, objectUser) - logger.Info("PHASE 5 DONE") + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + return "", nil + }, + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "user" { + return userCreateJSON, nil + } + return "", nil + }, + } + c.Executor = executor + + // Create a ReconcileObjectStoreUser object with the scheme and fake client. + r = &ReconcileObjectStoreUser{client: cl, scheme: s, context: c, opManagerContext: ctx} + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) + + t.Run("failure Rgw object exists but NO pod are running", func(t *testing.T) { + cephObjectStore = &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: store, + Namespace: namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "CephObjectStore", + }, + Spec: cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{ + Port: 80, + }, + }, + Status: &cephv1.ObjectStoreStatus{ + Info: map[string]string{"endpoint": "http://rook-ceph-rgw-my-store.rook-ceph:80"}, + }, + } + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStoreList{}) + object = append(object, cephObjectStore) + + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + // Create a ReconcileObjectStoreUser object with the scheme and fake client. + r = &ReconcileObjectStoreUser{client: cl, scheme: s, context: c, opManagerContext: ctx} + + err := r.client.Get(context.TODO(), types.NamespacedName{Name: store, Namespace: namespace}, cephObjectStore) + assert.NoError(t, err, cephObjectStore) + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) + + t.Run("success Rgw object exists and pods are running", func(t *testing.T) { + rgwPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-rgw-my-store-a-5fd6fb4489-xv65v", + Namespace: namespace, + Labels: map[string]string{k8sutil.AppAttr: appName, "rgw": "my-store"}}} + + // Get the updated object. + // Create RGW pod + err := r.client.Create(context.TODO(), rgwPod) + assert.NoError(t, err) + + // Mock client + newMultisiteAdminOpsCtxFunc = func(objContext *cephobject.Context, spec *cephv1.ObjectStoreSpec) (*cephobject.AdminOpsContext, error) { + mockClient := &cephobject.MockClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + if (req.URL.RawQuery == "display-name=my-user&format=json&max-buckets=1000&uid=my-user" && (req.Method == http.MethodGet || req.Method == http.MethodPost) && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user") || + (req.URL.RawQuery == "enabled=false&format=json&max-objects=-1&max-size=-1"a="a-type=user&uid=my-user" && req.Method == http.MethodPut && req.URL.Path == "rook-ceph-rgw-my-store.mycluster.svc/admin/user") { + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader([]byte(userCreateJSON))), + }, nil + } + return nil, fmt.Errorf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, req.URL.Path) + }, + } + + context, err := cephobject.NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) + assert.NoError(t, err) + adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient) + assert.NoError(t, err) + + return &cephobject.AdminOpsContext{ + Context: *context, + AdminOpsUserAccessKey: "53S6B9S809NUP19IJ2K3", + AdminOpsUserSecretKey: "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", + AdminOpsClient: adminClient, + }, nil + } + + // Run reconcile + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, objectUser) + assert.NoError(t, err) + assert.Equal(t, "Ready", objectUser.Status.Phase, objectUser) + }) } func TestBuildUpdateStatusInfo(t *testing.T) { @@ -415,7 +388,8 @@ func TestCreateorUpdateCephUser(t *testing.T) { objContext: &cephobject.AdminOpsContext{ AdminOpsClient: adminClient, }, - userConfig: &userConfig, + userConfig: &userConfig, + opManagerContext: context.TODO(), } maxsize, err := resource.ParseQuantity(maxsizestr) assert.NoError(t, err) diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index 70ed39460..4baed0d29 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -67,25 +67,27 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileObjectZone reconciles a ObjectZone object type ReconcileObjectZone struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - clusterSpec *cephv1.ClusterSpec + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + clusterSpec *cephv1.ClusterSpec + opManagerContext context.Context } // Add creates a new CephObjectZone Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { return &ReconcileObjectZone{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, } } @@ -123,7 +125,7 @@ func (r *ReconcileObjectZone) Reconcile(context context.Context, request reconci func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephObjectZone instance cephObjectZone := &cephv1.CephObjectZone{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephObjectZone) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectZone) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectZone resource not found. Ignoring since object must be deleted.") @@ -135,7 +137,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re // The CR was just created, initializing status fields if cephObjectZone.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) } // Make sure a CephCluster is present otherwise do nothing @@ -160,7 +162,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re } // Populate clusterInfo during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } @@ -168,12 +170,12 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re // validate the zone settings err = r.validateZoneCR(cephObjectZone) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "invalid CephObjectZone CR %q", cephObjectZone.Name) } // Start object reconciliation, updating status for this - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcilingStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcilingStatus) // Make sure an ObjectZoneGroup is present realmName, reconcileResponse, err := r.reconcileObjectZoneGroup(cephObjectZone) @@ -194,7 +196,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue logger.Debug("zone done reconciling") @@ -287,7 +289,7 @@ func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zon func (r *ReconcileObjectZone) reconcileObjectZoneGroup(zone *cephv1.CephObjectZone) (string, reconcile.Result, error) { // empty zoneGroup gets filled by r.client.Get() zoneGroup := &cephv1.CephObjectZoneGroup{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: zone.Spec.ZoneGroup, Namespace: zone.Namespace}, zoneGroup) + err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: zone.Spec.ZoneGroup, Namespace: zone.Namespace}, zoneGroup) if err != nil { if kerrors.IsNotFound(err) { return "", waitForRequeueIfObjectZoneGroupNotReady, err @@ -338,14 +340,14 @@ func (r *ReconcileObjectZone) validateZoneCR(z *cephv1.CephObjectZone) error { } func (r *ReconcileObjectZone) setFailedStatus(name types.NamespacedName, errMessage string, err error) (reconcile.Result, error) { - updateStatus(r.client, name, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, name, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "%s", errMessage) } // updateStatus updates an zone with a given status -func updateStatus(client client.Client, name types.NamespacedName, status string) { +func (r *ReconcileObjectZone) updateStatus(client client.Client, name types.NamespacedName, status string) { objectZone := &cephv1.CephObjectZone{} - if err := client.Get(context.TODO(), name, objectZone); err != nil { + if err := client.Get(r.opManagerContext, name, objectZone); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectZone resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 5c65f68e4..5d5a42c64 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -66,24 +66,26 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileObjectZoneGroup reconciles a ObjectZoneGroup object type ReconcileObjectZoneGroup struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context } // Add creates a new CephObjectZoneGroup Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { return &ReconcileObjectZoneGroup{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, } } @@ -121,7 +123,7 @@ func (r *ReconcileObjectZoneGroup) Reconcile(context context.Context, request re func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephObjectZoneGroup instance cephObjectZoneGroup := &cephv1.CephObjectZoneGroup{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephObjectZoneGroup) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephObjectZoneGroup) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectZoneGroup resource not found. Ignoring since object must be deleted.") @@ -133,7 +135,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci // The CR was just created, initializing status fields if cephObjectZoneGroup.Status == nil { - updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.EmptyStatus) } // Make sure a CephCluster is present otherwise do nothing @@ -156,7 +158,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci } // Populate clusterInfo during each reconcile - r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + r.clusterInfo, _, _, err = mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } @@ -164,12 +166,12 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci // validate the zone group settings err = validateZoneGroup(cephObjectZoneGroup) if err != nil { - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "invalid CephObjectZoneGroup CR %q", cephObjectZoneGroup.Name) } // Start object reconciliation, updating status for this - updateStatus(r.client, request.NamespacedName, k8sutil.ReconcilingStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReconcilingStatus) // Make sure an ObjectRealm Resource is present reconcileResponse, err = r.reconcileObjectRealm(cephObjectZoneGroup) @@ -190,7 +192,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci } // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) + r.updateStatus(r.client, request.NamespacedName, k8sutil.ReadyStatus) // Return and do not requeue logger.Debug("zone group done reconciling") @@ -259,7 +261,7 @@ func (r *ReconcileObjectZoneGroup) createCephZoneGroup(zoneGroup *cephv1.CephObj func (r *ReconcileObjectZoneGroup) reconcileObjectRealm(zoneGroup *cephv1.CephObjectZoneGroup) (reconcile.Result, error) { // Verify the object realm API object actually exists cephObjectRealm := &cephv1.CephObjectRealm{} - err := r.client.Get(context.TODO(), types.NamespacedName{Name: zoneGroup.Spec.Realm, Namespace: zoneGroup.Namespace}, cephObjectRealm) + err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: zoneGroup.Spec.Realm, Namespace: zoneGroup.Namespace}, cephObjectRealm) if err != nil { if kerrors.IsNotFound(err) { return waitForRequeueIfObjectRealmNotReady, errors.Wrapf(err, "realm %q not found", zoneGroup.Spec.Realm) @@ -289,14 +291,14 @@ func (r *ReconcileObjectZoneGroup) reconcileCephRealm(zoneGroup *cephv1.CephObje } func (r *ReconcileObjectZoneGroup) setFailedStatus(name types.NamespacedName, errMessage string, err error) (reconcile.Result, error) { - updateStatus(r.client, name, k8sutil.ReconcileFailedStatus) + r.updateStatus(r.client, name, k8sutil.ReconcileFailedStatus) return reconcile.Result{}, errors.Wrapf(err, "%s", errMessage) } // updateStatus updates an zone group with a given status -func updateStatus(client client.Client, name types.NamespacedName, status string) { +func (r *ReconcileObjectZoneGroup) updateStatus(client client.Client, name types.NamespacedName, status string) { objectZoneGroup := &cephv1.CephObjectZoneGroup{} - if err := client.Get(context.TODO(), name, objectZoneGroup); err != nil { + if err := client.Get(r.opManagerContext, name, objectZoneGroup); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephObjectZoneGroup resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index 7f4e6cdd7..cdf8cd8c6 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -21,65 +21,48 @@ import ( "context" "os" "os/signal" - "strings" + "runtime" "syscall" + "time" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" - "github.com/rook/rook/pkg/operator/ceph/agent" "github.com/rook/rook/pkg/operator/ceph/cluster" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" - "github.com/rook/rook/pkg/operator/ceph/csi" - "github.com/rook/rook/pkg/operator/ceph/csi/peermap" - "github.com/rook/rook/pkg/operator/ceph/provisioner" - "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/sig-storage-lib-external-provisioner/v6/controller" -) - -// volume provisioner constant -const ( - provisionerName = "ceph.rook.io/block" - provisionerNameLegacy = "rook.io/block" ) var ( logger = capnslog.NewPackageLogger("github.com/rook/rook", "operator") - // The supported configurations for the volume provisioner - provisionerConfigs = map[string]string{ - provisionerName: flexvolume.FlexvolumeVendor, - provisionerNameLegacy: flexvolume.FlexvolumeVendorLegacy, - } - // ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object. ImmediateRetryResult = reconcile.Result{Requeue: true} // Signals to watch for to terminate the operator gracefully // Using os.Interrupt is more portable across platforms instead of os.SIGINT shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + + // Placeholder for the CRD manager life cycle, first we have the context to manage cancellation + // of the manager. + // Then we have the cancel function that we can call anything to terminate the context + // Finally the channel to receive errors from the manager from within the go routine + opManagerContext context.Context + opManagerStop context.CancelFunc + mgrCRDErrorChan chan error ) // Operator type for managing storage type Operator struct { context *clusterd.Context resources []k8sutil.CustomResource - config *Config + config *opcontroller.OperatorConfig // The custom resource that is global to the kubernetes cluster. // The cluster is global because you create multiple clusters in k8s - clusterController *cluster.ClusterController - delayedDaemonsStarted bool -} - -type Config struct { - OperatorNamespace string - Image string - ServiceAccount string + clusterController *cluster.ClusterController } // New creates an operator instance @@ -89,192 +72,90 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen o := &Operator{ context: context, resources: schemes, - config: &Config{ + config: &opcontroller.OperatorConfig{ OperatorNamespace: os.Getenv(k8sutil.PodNamespaceEnvVar), Image: rookImage, ServiceAccount: serviceAccount, }, } - operatorConfigCallbacks := []func() error{ - o.updateDrivers, - o.updateOperatorLogLevel, - } - addCallbacks := []func() error{ - o.startDrivers, - } - o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper, operatorConfigCallbacks, addCallbacks) + o.clusterController = cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper) return o } -func (o *Operator) cleanup(stopCh chan struct{}) { - close(stopCh) - o.clusterController.StopWatch() -} - -func (o *Operator) updateOperatorLogLevel() error { - rookLogLevel, err := k8sutil.GetOperatorSetting(o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_LOG_LEVEL", "INFO") - if err != nil { - logger.Warningf("failed to load ROOK_LOG_LEVEL. Defaulting to INFO. %v", err) - rookLogLevel = "INFO" - } - - logLevel, err := capnslog.ParseLevel(strings.ToUpper(rookLogLevel)) - if err != nil { - return errors.Wrapf(err, "failed to load ROOK_LOG_LEVEL %q.", rookLogLevel) - } - - capnslog.SetGlobalLogLevel(logLevel) - return nil -} - // Run the operator instance func (o *Operator) Run() error { + // Initialize signal handler and context for the operator process life cycle + // This context is used to handle the graceful termination of the operator + operatorContext, operatorStop := signal.NotifyContext(context.Background(), shutdownSignals...) + defer operatorStop() - if o.config.OperatorNamespace == "" { - return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %q", k8sutil.PodNamespaceEnvVar) - } + // Start the CRD manager + o.runCRDManager() - // TODO: needs a callback? - opcontroller.SetCephCommandsTimeout(o.context) + // Used to watch for operator's config changes + configChan := make(chan os.Signal, 1) + signal.Notify(configChan, syscall.SIGHUP) - // Initialize signal handler and context - stopContext, stopFunc := signal.NotifyContext(context.Background(), shutdownSignals...) - defer stopFunc() - - rookDiscover := discover.New(o.context.Clientset) - if opcontroller.DiscoveryDaemonEnabled(o.context) { - if err := rookDiscover.Start(o.config.OperatorNamespace, o.config.Image, o.config.ServiceAccount, true); err != nil { - return errors.Wrap(err, "failed to start device discovery daemonset") - } - } else { - if err := rookDiscover.Stop(stopContext, o.config.OperatorNamespace); err != nil { - return errors.Wrap(err, "failed to stop device discovery daemonset") - } - } - - serverVersion, err := o.context.Clientset.Discovery().ServerVersion() - if err != nil { - return errors.Wrap(err, "failed to get server version") - } - - // Initialize stop channel for watchers - stopChan := make(chan struct{}) - - // For Flex Driver, run volume provisioner for each of the supported configurations - if opcontroller.FlexDriverEnabled(o.context) { - for name, vendor := range provisionerConfigs { - volumeProvisioner := provisioner.New(o.context, vendor) - pc := controller.NewProvisionController( - o.context.Clientset, - name, - volumeProvisioner, - serverVersion.GitVersion, - ) - go pc.Run(stopContext) - logger.Infof("rook-provisioner %q started using %q flex vendor dir", name, vendor) - } - } - - var namespaceToWatch string - if os.Getenv("ROOK_CURRENT_NAMESPACE_ONLY") == "true" { - logger.Infof("watching the current namespace for a ceph cluster CR") - namespaceToWatch = o.config.OperatorNamespace - } else { - logger.Infof("watching all namespaces for ceph cluster CRs") - namespaceToWatch = v1.NamespaceAll - } - - // Start the controller-runtime Manager. - mgrErrorChan := make(chan error) - go o.startManager(stopContext, namespaceToWatch, mgrErrorChan) - - // Start the operator setting watcher - go o.clusterController.StartOperatorSettingsWatch(stopChan) - - // Signal handler to stop the operator + // Main infinite loop to watch for channel events for { select { - case <-stopContext.Done(): - logger.Infof("shutdown signal received, exiting... %v", stopContext.Err()) - o.cleanup(stopChan) + case <-operatorContext.Done(): + // Terminate the operator CRD manager, we cannot use "defer opManagerStop()" since + // earlier in this code the function has not been populated yet. So explicitly calling + // it here during the main context termination. + opManagerStop() + + logger.Infof("shutdown signal received, exiting... %v", operatorContext.Err()) return nil - case err := <-mgrErrorChan: - o.cleanup(stopChan) - return err + case <-configChan: + logger.Info("reloading operator's CRDs manager, cancelling all orchestrations!") + + // Stop the operator CRD manager + opManagerStop() + + // Run the operator CRD manager again + o.runCRDManager() + + case err := <-mgrCRDErrorChan: + return errors.Wrapf(err, "gave up to run the operator manager") } } } -func (o *Operator) startDrivers() error { - if o.delayedDaemonsStarted { - return nil - } +func (o *Operator) runCRDManager() { + // Create the error channel so that the go routine can return an error + mgrCRDErrorChan = make(chan error) - o.delayedDaemonsStarted = true - if err := o.updateDrivers(); err != nil { - o.delayedDaemonsStarted = false // unset because failed to updateDrivers - return err - } + // Create the context and the cancellation function + opManagerContext, opManagerStop = context.WithCancel(context.Background()) - return nil + // The operator config manager is also watching for changes here so if the operator config map + // content changes for ROOK_CURRENT_NAMESPACE_ONLY we must reload the operator CRD manager + o.namespaceToWatch(opManagerContext) + + // Pass the parent context to the cluster controller so that the monitoring go routines can + // consume it to terminate gracefully + o.clusterController.OpManagerCtx = opManagerContext + + // Run the operator CRD manager + go o.startCRDManager(opManagerContext, mgrCRDErrorChan) + + // Run an informative go routine that prints the number of goroutines + go func() { + // Let's wait a bit to make sure most of the reconcilers are done + time.Sleep(time.Minute) + logger.Debugf("number of goroutines %d", runtime.NumGoroutine()) + }() } -func (o *Operator) updateDrivers() error { - var err error - - // Skipping CSI driver update since the first cluster hasn't been started yet - if !o.delayedDaemonsStarted { - return nil +func (o *Operator) namespaceToWatch(context context.Context) { + currentNamespaceOnly, _ := k8sutil.GetOperatorSetting(opManagerContext, o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_CURRENT_NAMESPACE_ONLY", "true") + if currentNamespaceOnly == "true" { + o.config.NamespaceToWatch = o.config.OperatorNamespace + logger.Infof("watching the current namespace %q for a Ceph CRs", o.config.OperatorNamespace) + } else { + o.config.NamespaceToWatch = v1.NamespaceAll + logger.Infof("watching all namespaces for Ceph CRs") } - - if o.config.OperatorNamespace == "" { - return errors.Errorf("rook operator namespace is not provided. expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar) - } - - if opcontroller.FlexDriverEnabled(o.context) { - rookAgent := agent.New(o.context.Clientset) - if err := rookAgent.Start(o.config.OperatorNamespace, o.config.Image, o.config.ServiceAccount); err != nil { - return errors.Wrap(err, "error starting agent daemonset") - } - } - - serverVersion, err := o.context.Clientset.Discovery().ServerVersion() - if err != nil { - return errors.Wrap(err, "error getting server version") - } - - if serverVersion.Major < csi.KubeMinMajor || serverVersion.Major == csi.KubeMinMajor && serverVersion.Minor < csi.ProvDeploymentSuppVersion { - logger.Infof("CSI drivers only supported in K8s 1.14 or newer. version=%s", serverVersion.String()) - // disable csi control variables to disable other csi functions - csi.EnableRBD = false - csi.EnableCephFS = false - return nil - } - - operatorPodName := os.Getenv(k8sutil.PodNameEnvVar) - ownerRef, err := k8sutil.GetDeploymentOwnerReference(o.context.Clientset, operatorPodName, o.config.OperatorNamespace) - if err != nil { - logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err) - } - if ownerRef != nil { - blockOwnerDeletion := false - ownerRef.BlockOwnerDeletion = &blockOwnerDeletion - } - - ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, o.config.OperatorNamespace) - // create an empty config map. config map will be filled with data - // later when clusters have mons - err = csi.CreateCsiConfigMap(o.config.OperatorNamespace, o.context.Clientset, ownerInfo) - if err != nil { - return errors.Wrap(err, "failed creating csi config map") - } - - err = peermap.CreateOrUpdateConfig(o.context, &peermap.PeerIDMappings{}) - if err != nil { - return errors.Wrap(err, "failed to create pool ID mapping config map") - } - - go csi.ValidateAndConfigureDrivers(o.context, o.config.OperatorNamespace, o.config.Image, o.config.ServiceAccount, serverVersion, ownerInfo) - return nil } diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 2f69a52c8..f0214164a 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -70,27 +70,30 @@ type ReconcileCephBlockPool struct { scheme *runtime.Scheme context *clusterd.Context clusterInfo *cephclient.ClusterInfo - blockPoolChannels map[string]*blockPoolHealth + blockPoolContexts map[string]*blockPoolHealth + opManagerContext context.Context } type blockPoolHealth struct { - stopChan chan struct{} - monitoringRunning bool + internalCtx context.Context + internalCancel context.CancelFunc + started bool } // Add creates a new CephBlockPool Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, context *clusterd.Context) error { - return add(mgr, newReconciler(mgr, context)) +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext)) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, context *clusterd.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler { return &ReconcileCephBlockPool{ client: mgr.GetClient(), scheme: mgr.GetScheme(), context: context, - blockPoolChannels: make(map[string]*blockPoolHealth), + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: opManagerContext, } } @@ -141,7 +144,7 @@ func (r *ReconcileCephBlockPool) Reconcile(context context.Context, request reco func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the CephBlockPool instance cephBlockPool := &cephv1.CephBlockPool{} - err := r.client.Get(context.TODO(), request.NamespacedName, cephBlockPool) + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephBlockPool) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephBlockPool resource not found. Ignoring since object must be deleted.") @@ -185,7 +188,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile } // Populate clusterInfo during each reconcile - clusterInfo, _, _, err := mon.LoadClusterInfo(r.context, request.NamespacedName.Namespace) + clusterInfo, _, _, err := mon.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to populate cluster info") } @@ -195,11 +198,12 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile // Initialize the channel for this pool // This allows us to track multiple CephBlockPool in the same namespace blockPoolChannelKey := fmt.Sprintf("%s-%s", cephBlockPool.Namespace, cephBlockPool.Name) - _, poolChannelExists := r.blockPoolChannels[blockPoolChannelKey] - if !poolChannelExists { - r.blockPoolChannels[blockPoolChannelKey] = &blockPoolHealth{ - stopChan: make(chan struct{}), - monitoringRunning: false, + _, blockPoolContextsExists := r.blockPoolContexts[blockPoolChannelKey] + if !blockPoolContextsExists { + internalCtx, internalCancel := context.WithCancel(r.opManagerContext) + r.blockPoolContexts[blockPoolChannelKey] = &blockPoolHealth{ + internalCtx: internalCtx, + internalCancel: internalCancel, } } @@ -207,7 +211,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile if !cephBlockPool.GetDeletionTimestamp().IsZero() { // If the ceph block pool is still in the map, we must remove it during CR deletion // We must remove it first otherwise the checker will panic since the status/info will be nil - if poolChannelExists { + if blockPoolContextsExists { r.cancelMirrorMonitoring(blockPoolChannelKey) } @@ -292,11 +296,11 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile // Run the goroutine to update the mirroring status if !cephBlockPool.Spec.StatusCheck.Mirror.Disabled { // Start monitoring of the pool - if r.blockPoolChannels[blockPoolChannelKey].monitoringRunning { + if r.blockPoolContexts[blockPoolChannelKey].started { logger.Debug("pool monitoring go routine already running!") } else { - r.blockPoolChannels[blockPoolChannelKey].monitoringRunning = true - go checker.checkMirroring(r.blockPoolChannels[blockPoolChannelKey].stopChan) + go checker.checkMirroring(r.blockPoolContexts[blockPoolChannelKey].internalCtx) + r.blockPoolContexts[blockPoolChannelKey].started = true } } @@ -322,7 +326,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, nil) // Stop monitoring the mirroring status of this pool - if poolChannelExists && r.blockPoolChannels[blockPoolChannelKey].monitoringRunning { + if blockPoolContextsExists && r.blockPoolContexts[blockPoolChannelKey].started { r.cancelMirrorMonitoring(blockPoolChannelKey) // Reset the MirrorHealthCheckSpec checker.updateStatusMirroring(nil, nil, nil, "") @@ -405,9 +409,9 @@ func configureRBDStats(clusterContext *clusterd.Context, clusterInfo *cephclient } func (r *ReconcileCephBlockPool) cancelMirrorMonitoring(cephBlockPoolName string) { - // Close the channel to stop the mirroring status - close(r.blockPoolChannels[cephBlockPoolName].stopChan) + // Cancel the context to stop the go routine + r.blockPoolContexts[cephBlockPoolName].internalCancel() // Remove ceph block pool from the map - delete(r.blockPoolChannels, cephBlockPoolName) + delete(r.blockPoolContexts, cephBlockPoolName) } diff --git a/pkg/operator/ceph/pool/controller_test.go b/pkg/operator/ceph/pool/controller_test.go index 276e79c14..c51eaa097 100644 --- a/pkg/operator/ceph/pool/controller_test.go +++ b/pkg/operator/ceph/pool/controller_test.go @@ -26,15 +26,14 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - "github.com/tevino/abool" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -45,7 +44,7 @@ import ( ) func TestCreatePool(t *testing.T) { - clusterInfo := &cephclient.ClusterInfo{Namespace: "myns"} + clusterInfo := client.AdminClusterInfo("mycluster") executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) @@ -75,7 +74,7 @@ func TestCreatePool(t *testing.T) { func TestDeletePool(t *testing.T) { failOnDelete := false - clusterInfo := &cephclient.ClusterInfo{Namespace: "myns"} + clusterInfo := cephclient.AdminClusterInfo("mycluster") executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { emptyPool := "{\"images\":{\"count\":0,\"provisioned_bytes\":0,\"snap_count\":0},\"trash\":{\"count\":1,\"provisioned_bytes\":2048,\"snap_count\":0}}" @@ -125,13 +124,6 @@ func TestCephBlockPoolController(t *testing.T) { // Set DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) os.Setenv("ROOK_LOG_LEVEL", "DEBUG") - - // - // TEST 1 SETUP - // - // FAILURE because no CephCluster - // - logger.Info("RUN 1") var ( name = "my-pool" namespace = "rook-ceph" @@ -178,10 +170,9 @@ func TestCephBlockPoolController(t *testing.T) { }, } c := &clusterd.Context{ - Executor: executor, - Clientset: testop.New(t, 1), - RookClientset: rookclient.NewSimpleClientset(), - RequestCancelOrchestration: abool.New(), + Executor: executor, + Clientset: testop.New(t, 1), + RookClientset: rookclient.NewSimpleClientset(), } // Register operator types with the runtime scheme. @@ -196,7 +187,8 @@ func TestCephBlockPoolController(t *testing.T) { client: cl, scheme: s, context: c, - blockPoolChannels: make(map[string]*blockPoolHealth), + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: context.TODO(), } // Mock request to simulate Reconcile() being called on an event for a @@ -208,152 +200,131 @@ func TestCephBlockPoolController(t *testing.T) { }, } - // Create pool for updateCephBlockPoolStatus() - _, err := c.RookClientset.CephV1().CephBlockPools(namespace).Create(ctx, pool, metav1.CreateOptions{}) - assert.NoError(t, err) - res, err := r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.True(t, res.Requeue) + cephCluster := &cephv1.CephCluster{} - // - // TEST 2: - // - // FAILURE we have a cluster but it's not ready - // - logger.Info("RUN 2") - cephCluster := &cephv1.CephCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Namespace: namespace, - }, - Status: cephv1.ClusterStatus{ - Phase: "", - CephVersion: &cephv1.ClusterVersion{ - Version: "14.2.9-0", + t.Run("failure no CephCluster", func(t *testing.T) { + // Create pool for updateCephBlockPoolStatus() + _, err := c.RookClientset.CephV1().CephBlockPools(namespace).Create(ctx, pool, metav1.CreateOptions{}) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) + + t.Run("failure CephCluster not ready", func(t *testing.T) { + cephCluster = &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, }, - CephStatus: &cephv1.CephStatus{ - Health: "", + Status: cephv1.ClusterStatus{ + Phase: "", + CephVersion: &cephv1.ClusterVersion{ + Version: "14.2.9-0", + }, + CephStatus: &cephv1.CephStatus{ + Health: "", + }, }, - }, - } + } - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) - // Create CephCluster for updateCephBlockPoolStatus() - _, err = c.RookClientset.CephV1().CephClusters(namespace).Create(ctx, cephCluster, metav1.CreateOptions{}) - assert.NoError(t, err) + // Create CephCluster for updateCephBlockPoolStatus() + _, err := c.RookClientset.CephV1().CephClusters(namespace).Create(ctx, cephCluster, metav1.CreateOptions{}) + assert.NoError(t, err) - object = append(object, cephCluster) - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithRuntimeObjects(object...).Build() - // Create a ReconcileCephBlockPool object with the scheme and fake client. - r = &ReconcileCephBlockPool{ - client: cl, - scheme: s, - context: c, - blockPoolChannels: make(map[string]*blockPoolHealth), - } - assert.True(t, res.Requeue) + object = append(object, cephCluster) + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithRuntimeObjects(object...).Build() + // Create a ReconcileCephBlockPool object with the scheme and fake client. + r = &ReconcileCephBlockPool{ + client: cl, + scheme: s, + context: c, + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: context.TODO(), + } - // - // TEST 3: - // - // SUCCESS! The CephCluster is ready - // - logger.Info("RUN 3") - cephCluster.Status.Phase = cephv1.ConditionReady - cephCluster.Status.CephStatus.Health = "HEALTH_OK" + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.True(t, res.Requeue) + }) - objects := []runtime.Object{ - pool, - cephCluster, - } - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithRuntimeObjects(objects...).Build() - c.Client = cl + t.Run("failure CephCluster is ready", func(t *testing.T) { + cephCluster.Status.Phase = cephv1.ConditionReady + cephCluster.Status.CephStatus.Health = "HEALTH_OK" + objects := []runtime.Object{ + pool, + cephCluster, + } + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithRuntimeObjects(objects...).Build() + c.Client = cl + + executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "config" && args[2] == "mgr." && args[3] == "mgr/prometheus/rbd_stats_pools" { + return "", nil + } - executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "status" { - return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil - } - if args[0] == "config" && args[2] == "mgr." && args[3] == "mgr/prometheus/rbd_stats_pools" { return "", nil - } + }, + } + c.Executor = executor - return "", nil - }, - } - c.Executor = executor + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte(name), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: namespace, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) + // Create a ReconcileCephBlockPool object with the scheme and fake client. + r = &ReconcileCephBlockPool{ + client: cl, + scheme: s, + context: c, + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: context.TODO(), + } + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) - s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) - // Create a ReconcileCephBlockPool object with the scheme and fake client. - r = &ReconcileCephBlockPool{ - client: cl, - scheme: s, - context: c, - blockPoolChannels: make(map[string]*blockPoolHealth), - } - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, pool) + assert.NoError(t, err) + assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase) + }) - err = r.client.Get(context.TODO(), req.NamespacedName, pool) - assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase) - - // - // TEST 4: Mirroring - // No mirror mode set: failure - logger.Info("RUN 4") - pool.Spec.Mirroring.Enabled = true - err = r.client.Update(context.TODO(), pool) - assert.NoError(t, err) - res, err = r.Reconcile(ctx, req) - assert.Error(t, err) - assert.True(t, res.Requeue) - - // - // TEST 5: Mirroring - // mirror mode set: Success - executor = &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - if args[0] == "mirror" && args[1] == "pool" && args[2] == "peer" && args[3] == "bootstrap" && args[4] == "create" { - return `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==`, nil - } - return "", nil - }, - } - c.Executor = executor - r = &ReconcileCephBlockPool{ - client: cl, - scheme: s, - context: c, - blockPoolChannels: make(map[string]*blockPoolHealth), - } + t.Run("failure no mirror mode", func(t *testing.T) { + pool.Spec.Mirroring.Enabled = true + err := r.client.Update(context.TODO(), pool) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.True(t, res.Requeue) + }) os.Setenv("POD_NAME", "test") defer os.Setenv("POD_NAME", "") os.Setenv("POD_NAMESPACE", namespace) defer os.Setenv("POD_NAMESPACE", "") - p := &corev1.Pod{ + p := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: namespace, @@ -366,7 +337,7 @@ func TestCephBlockPoolController(t *testing.T) { }, } // Create fake pod - _, err = r.context.Clientset.CoreV1().Pods(namespace).Create(context.TODO(), p, metav1.CreateOptions{}) + _, err := r.context.Clientset.CoreV1().Pods(namespace).Create(context.TODO(), p, metav1.CreateOptions{}) assert.NoError(t, err) replicaSet := &appsv1.ReplicaSet{ @@ -385,92 +356,113 @@ func TestCephBlockPoolController(t *testing.T) { _, err = r.context.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), replicaSet, metav1.CreateOptions{}) assert.NoError(t, err) - pool.Spec.Mirroring.Mode = "image" - pool.Spec.Mirroring.Peers.SecretNames = []string{} - err = r.client.Update(context.TODO(), pool) - assert.NoError(t, err) - for i := 0; i < 5; i++ { - res, err = r.Reconcile(ctx, req) + t.Run("success - mirroring set", func(t *testing.T) { + executor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if args[0] == "mirror" && args[1] == "pool" && args[2] == "peer" && args[3] == "bootstrap" && args[4] == "create" { + return `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==`, nil + } + return "", nil + }, + } + c.Executor = executor + r = &ReconcileCephBlockPool{ + client: cl, + scheme: s, + context: c, + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: context.TODO(), + } + + pool.Spec.Mirroring.Mode = "image" + pool.Spec.Mirroring.Peers.SecretNames = []string{} + err = r.client.Update(context.TODO(), pool) + assert.NoError(t, err) + for i := 0; i < 5; i++ { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, pool) + assert.NoError(t, err) + assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase) + if _, ok := pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName]; ok { + break + } + logger.Infof("FIX: trying again to update the mirroring status") + } + assert.NotEmpty(t, pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName], pool.Status.Info) + + // fetch the secret + myPeerSecret, err := c.Clientset.CoreV1().Secrets(namespace).Get(ctx, pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName], metav1.GetOptions{}) + assert.NoError(t, err) + if myPeerSecret != nil { + assert.NotEmpty(t, myPeerSecret.Data["token"], myPeerSecret.Data) + assert.NotEmpty(t, myPeerSecret.Data["pool"]) + } + }) + + peerSecretName := "peer-secret" + t.Run("failure - import peer token but was not created", func(t *testing.T) { + // Create a fake client to mock API calls. + cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + // Create a ReconcileCephBlockPool object with the scheme and fake client. + r = &ReconcileCephBlockPool{ + client: cl, + scheme: s, + context: c, + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: context.TODO(), + } + + pool.Spec.Mirroring.Peers.SecretNames = []string{peerSecretName} + err := r.client.Update(context.TODO(), pool) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + // assert reconcile failure because peer token secert was not created + assert.Error(t, err) + assert.True(t, res.Requeue) + }) + + t.Run("success - import peer token but was created", func(t *testing.T) { + bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token + peerSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: peerSecretName, + Namespace: namespace, + }, + Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")}, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.False(t, res.Requeue) + err = r.client.Get(context.TODO(), req.NamespacedName, pool) + assert.NoError(t, err) + }) + + t.Run("failure - mirroring disabled", func(t *testing.T) { + r = &ReconcileCephBlockPool{ + client: cl, + scheme: s, + context: c, + blockPoolContexts: make(map[string]*blockPoolHealth), + opManagerContext: context.TODO(), + } + pool.Spec.Mirroring.Enabled = false + pool.Spec.Mirroring.Mode = "image" + err := r.client.Update(context.TODO(), pool) + assert.NoError(t, err) + res, err := r.Reconcile(ctx, req) assert.NoError(t, err) assert.False(t, res.Requeue) err = r.client.Get(context.TODO(), req.NamespacedName, pool) assert.NoError(t, err) assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase) - if _, ok := pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName]; ok { - break - } - logger.Infof("FIX: trying again to update the mirroring status") - } - assert.NotEmpty(t, pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName], pool.Status.Info) - - // fetch the secret - myPeerSecret, err := c.Clientset.CoreV1().Secrets(namespace).Get(ctx, pool.Status.Info[opcontroller.RBDMirrorBootstrapPeerSecretName], metav1.GetOptions{}) - assert.NoError(t, err) - if myPeerSecret != nil { - assert.NotEmpty(t, myPeerSecret.Data["token"], myPeerSecret.Data) - assert.NotEmpty(t, myPeerSecret.Data["pool"]) - } - - // - // TEST 6: Import peer token - - // Create a fake client to mock API calls. - cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() - - // Create a ReconcileCephBlockPool object with the scheme and fake client. - r = &ReconcileCephBlockPool{ - client: cl, - scheme: s, - context: c, - blockPoolChannels: make(map[string]*blockPoolHealth), - } - - peerSecretName := "peer-secret" - pool.Spec.Mirroring.Peers.SecretNames = []string{peerSecretName} - err = r.client.Update(context.TODO(), pool) - assert.NoError(t, err) - res, err = r.Reconcile(ctx, req) - // assert reconcile failure because peer token secert was not created - assert.Error(t, err) - assert.True(t, res.Requeue) - - bootstrapPeerToken := `eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==` //nolint:gosec // This is just a var name, not a real token - peerSecret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: peerSecretName, - Namespace: namespace, - }, - Data: map[string][]byte{"token": []byte(bootstrapPeerToken), "pool": []byte("goo")}, - Type: k8sutil.RookType, - } - _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, peerSecret, metav1.CreateOptions{}) - assert.NoError(t, err) - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, pool) - assert.NoError(t, err) - - // - // TEST 7: Mirroring disabled - r = &ReconcileCephBlockPool{ - client: cl, - scheme: s, - context: c, - blockPoolChannels: make(map[string]*blockPoolHealth), - } - pool.Spec.Mirroring.Enabled = false - pool.Spec.Mirroring.Mode = "image" - err = r.client.Update(context.TODO(), pool) - assert.NoError(t, err) - res, err = r.Reconcile(ctx, req) - assert.NoError(t, err) - assert.False(t, res.Requeue) - err = r.client.Get(context.TODO(), req.NamespacedName, pool) - assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionReady, pool.Status.Phase) - - assert.Nil(t, pool.Status.MirroringStatus) + assert.Nil(t, pool.Status.MirroringStatus) + }) } func TestConfigureRBDStats(t *testing.T) { @@ -501,7 +493,7 @@ func TestConfigureRBDStats(t *testing.T) { context.Executor = executor context.Client = fake.NewClientBuilder().WithScheme(s).Build() - clusterInfo := &cephclient.ClusterInfo{Namespace: namespace} + clusterInfo := cephclient.AdminClusterInfo(namespace) // Case 1: CephBlockPoolList is not registered in scheme. // So, an error is expected as List() operation would fail. diff --git a/pkg/operator/ceph/pool/health.go b/pkg/operator/ceph/pool/health.go index 1dd90d27b..5d9dd473e 100644 --- a/pkg/operator/ceph/pool/health.go +++ b/pkg/operator/ceph/pool/health.go @@ -17,6 +17,7 @@ limitations under the License. package pool import ( + "context" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -63,7 +64,7 @@ func newMirrorChecker(context *clusterd.Context, client client.Client, clusterIn } // checkMirroring periodically checks the health of the cluster -func (c *mirrorChecker) checkMirroring(stopCh chan struct{}) { +func (c *mirrorChecker) checkMirroring(context context.Context) { // check the mirroring health immediately before starting the loop err := c.checkMirroringHealth() if err != nil { @@ -73,7 +74,7 @@ func (c *mirrorChecker) checkMirroring(stopCh chan struct{}) { for { select { - case <-stopCh: + case <-context.Done(): logger.Infof("stopping monitoring pool mirroring status %q", c.namespacedName.Name) return diff --git a/pkg/operator/ceph/pool/peers.go b/pkg/operator/ceph/pool/peers.go index 496a4bcca..ceadbb2d3 100644 --- a/pkg/operator/ceph/pool/peers.go +++ b/pkg/operator/ceph/pool/peers.go @@ -17,8 +17,6 @@ limitations under the License. package pool import ( - "context" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" @@ -39,7 +37,7 @@ func (r *ReconcileCephBlockPool) reconcileAddBoostrapPeer(pool *cephv1.CephBlock // For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring for _, peerSecret := range pool.Spec.Mirroring.Peers.SecretNames { logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret) - s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(context.TODO(), peerSecret, metav1.GetOptions{}) + s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(r.opManagerContext, peerSecret, metav1.GetOptions{}) // We don't care about IsNotFound here, we still need to fail if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret) diff --git a/pkg/operator/ceph/pool/status.go b/pkg/operator/ceph/pool/status.go index 9c8b5b4d3..ffef8b1db 100644 --- a/pkg/operator/ceph/pool/status.go +++ b/pkg/operator/ceph/pool/status.go @@ -57,7 +57,7 @@ func updateStatus(client client.Client, poolName types.NamespacedName, status ce // updateStatusBucket updates an object with a given status func (c *mirrorChecker) updateStatusMirroring(mirrorStatus *cephv1.PoolMirroringStatusSummarySpec, mirrorInfo *cephv1.PoolMirroringInfo, snapSchedStatus []cephv1.SnapshotSchedulesSpec, details string) { blockPool := &cephv1.CephBlockPool{} - if err := c.client.Get(context.TODO(), c.namespacedName, blockPool); err != nil { + if err := c.client.Get(c.clusterInfo.Context, c.namespacedName, blockPool); err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephBlockPool resource not found. Ignoring since object must be deleted.") return diff --git a/pkg/operator/ceph/pool/validate_test.go b/pkg/operator/ceph/pool/validate_test.go index 8bd1faf5b..1366b7fc9 100644 --- a/pkg/operator/ceph/pool/validate_test.go +++ b/pkg/operator/ceph/pool/validate_test.go @@ -181,7 +181,7 @@ func TestValidatePool(t *testing.T) { func TestValidateCrushProperties(t *testing.T) { executor := &exectest.MockExecutor{} context := &clusterd.Context{Executor: executor} - clusterInfo := &cephclient.ClusterInfo{Namespace: "myns"} + clusterInfo := cephclient.AdminClusterInfo("mycluster") executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { logger.Infof("Command: %s %v", command, args) if args[1] == "crush" && args[2] == "dump" { @@ -267,7 +267,7 @@ func TestValidateDeviceClasses(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - clusterInfo := &cephclient.ClusterInfo{Namespace: "myns"} + clusterInfo := cephclient.AdminClusterInfo("mycluster") executor := &exectest.MockExecutor{} context := &clusterd.Context{Executor: executor} executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { diff --git a/pkg/operator/ceph/predicate.go b/pkg/operator/ceph/predicate.go new file mode 100644 index 000000000..2b159f4f7 --- /dev/null +++ b/pkg/operator/ceph/predicate.go @@ -0,0 +1,128 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package operator to manage Kubernetes storage. +package operator + +import ( + "context" + + "github.com/rook/rook/pkg/operator/ceph/controller" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// predicateOpController is the predicate function to trigger reconcile on operator configuration cm change +func predicateController(client client.Client) predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + if cm, ok := e.Object.(*v1.ConfigMap); ok { + return cm.Name == controller.OperatorSettingConfigMapName + } else if s, ok := e.Object.(*v1.Secret); ok { + if s.Name == admissionControllerAppName { + err := client.Get(context.TODO(), types.NamespacedName{Name: admissionControllerAppName, Namespace: e.Object.GetNamespace()}, &v1.Service{}) + if err != nil { + if kerrors.IsNotFound(err) { + // If the service is present we don't need to reload again. If we don't perform + // this check it will result in an infinite + // reconcile loop. CREATE event is only triggered when the controller is started + // no matter what. + // No need to ask for reconciliation since the context is going to be terminated when + // the signal is caught and the reconcile will run when the controller + // starts. + logger.Debug("webhook secret created reloading the manager to enable the webhook server") + controller.ReloadManager() + } + } else { + logger.Debug("webhook service already set up, not reloading the manager") + } + + return false + } + } + + return false + }, + + UpdateFunc: func(e event.UpdateEvent) bool { + if old, ok := e.ObjectOld.(*v1.ConfigMap); ok { + if new, ok := e.ObjectNew.(*v1.ConfigMap); ok { + if old.Name == controller.OperatorSettingConfigMapName && new.Name == controller.OperatorSettingConfigMapName { + if old.Data["ROOK_CURRENT_NAMESPACE_ONLY"] != new.Data["ROOK_CURRENT_NAMESPACE_ONLY"] { + logger.Debug("ROOK_CURRENT_NAMESPACE_ONLY config updated, reloading the manager") + controller.ReloadManager() + + // No need to ask for reconciliation since the context is going to be terminated when + // the signal is caught and the reconcile will run when the controller starts. + return false + } + + // We still want to reconcile the operator manager if the configmap is updated + return true + } + } else if s, ok := e.ObjectNew.(*v1.Secret); ok { + if s.Name == admissionControllerAppName { + logger.Debug("webhook secret updated, reloading the manager") + controller.ReloadManager() + + // No need to ask for reconciliation since the context is going to be terminated when + // the signal is caught and the reconcile will run when the controller starts. + // If the admission controller secret is created or deleted we still need to reload and + // the webhook might be enabled or disabled + // + // The same goes the update, the secret changes we still need to reload the webhook + return false + } + } + } + + return false + }, + + DeleteFunc: func(e event.DeleteEvent) bool { + if cm, ok := e.Object.(*v1.ConfigMap); ok { + if cm.Name == controller.OperatorSettingConfigMapName { + logger.Debug("operator configmap deleted, not reconciling") + return false + } + } + if s, ok := e.Object.(*v1.Secret); ok { + if s.Name == admissionControllerAppName { + logger.Debug("webhook secret deleted, reloading the manager") + controller.ReloadManager() + + // No need to ask for reconciliation since the context is going to be terminated when + // the signal is caught and the reconcile will run when the controller starts. + // If the admission controller secret is created or deleted we still need to reload and + // the webhook might be enabled or disabled + // + // The same goes the update, the secret changes we still need to reload the webhook + return false + } + } + + return false + }, + + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + } +} diff --git a/pkg/operator/ceph/webhook.go b/pkg/operator/ceph/webhook.go index d67577a77..f49a3a842 100644 --- a/pkg/operator/ceph/webhook.go +++ b/pkg/operator/ceph/webhook.go @@ -33,8 +33,8 @@ import ( ) const ( - appName = "rook-ceph-admission-controller" - tlsPort int32 = 443 + admissionControllerAppName = "rook-ceph-admission-controller" + tlsPort int32 = 443 ) var ( @@ -42,12 +42,12 @@ var ( ) func isSecretPresent(ctx context.Context, context *clusterd.Context) (bool, error) { - logger.Infof("looking for admission webhook secret %q", appName) - s, err := context.Clientset.CoreV1().Secrets(namespace).Get(ctx, appName, metav1.GetOptions{}) + logger.Infof("looking for admission webhook secret %q", admissionControllerAppName) + s, err := context.Clientset.CoreV1().Secrets(namespace).Get(ctx, admissionControllerAppName, metav1.GetOptions{}) if err != nil { // If secret is not found. All good ! Proceed with rook without admission controllers if apierrors.IsNotFound(err) { - logger.Infof("admission webhook secret %q not found. proceeding without the admission controller", appName) + logger.Infof("admission webhook secret %q not found. proceeding without the admission controller", admissionControllerAppName) return false, nil } return false, err @@ -57,7 +57,7 @@ func isSecretPresent(ctx context.Context, context *clusterd.Context) (bool, erro logger.Debug("searching for old admission controller deployment") removeOldAdmissionControllerDeployment(ctx, context) - logger.Infof("admission webhook secret %q found", appName) + logger.Infof("admission webhook secret %q found", admissionControllerAppName) for k, data := range s.Data { path := fmt.Sprintf("%s/%s", certDir, k) err := ioutil.WriteFile(path, data, 0400) @@ -72,7 +72,7 @@ func isSecretPresent(ctx context.Context, context *clusterd.Context) (bool, erro func createWebhookService(context *clusterd.Context) error { webhookService := corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: appName, + Name: admissionControllerAppName, Namespace: namespace, }, Spec: corev1.ServiceSpec{ @@ -99,7 +99,7 @@ func createWebhookService(context *clusterd.Context) error { } func removeOldAdmissionControllerDeployment(ctx context.Context, context *clusterd.Context) { - opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", appName)} + opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", admissionControllerAppName)} d, err := context.Clientset.AppsV1().Deployments(namespace).List(ctx, opts) if err != nil { logger.Warningf("failed to get old admission controller deployment. %v", err) diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index e46224f52..90eb3f280 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -70,17 +70,15 @@ func New(clientset kubernetes.Interface) *Discover { } // Start the discover -func (d *Discover) Start(namespace, discoverImage, securityAccount string, useCephVolume bool) error { - - err := d.createDiscoverDaemonSet(namespace, discoverImage, securityAccount, useCephVolume) +func (d *Discover) Start(ctx context.Context, namespace, discoverImage, securityAccount string, useCephVolume bool) error { + err := d.createDiscoverDaemonSet(ctx, namespace, discoverImage, securityAccount, useCephVolume) if err != nil { - return fmt.Errorf("Error starting discover daemonset: %v", err) + return fmt.Errorf("failed to start discover daemonset. %v", err) } return nil } -func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage, securityAccount string, useCephVolume bool) error { - ctx := context.TODO() +func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, discoverImage, securityAccount string, useCephVolume bool) error { privileged := true discovery_parameters := []string{"discover", "--discover-interval", getEnvVar(discoverIntervalEnv, defaultDiscoverInterval)} diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go index 92472548b..ba7a86e78 100644 --- a/pkg/operator/discover/discover_test.go +++ b/pkg/operator/discover/discover_test.go @@ -68,7 +68,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) { _, err := clientset.CoreV1().Pods("rook-system").Create(ctx, &pod, metav1.CreateOptions{}) assert.NoError(t, err) // start a basic cluster - err = a.Start(namespace, "rook/rook:myversion", "mysa", false) + err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", false) assert.Nil(t, err) // check daemonset parameters diff --git a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go index 2053bee48..45f213982 100644 --- a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go +++ b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go @@ -31,6 +31,7 @@ import ( batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" watch "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" ) @@ -214,7 +215,7 @@ func (cr *CmdReporter) newWatcher() (watch.Interface, error) { TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", }, - FieldSelector: fmt.Sprintf("metadata.name=%s", jobName), + FieldSelector: fields.OneTermEqualSelector("metadata.name", jobName).String(), } list, err := cr.clientset.CoreV1().ConfigMaps(namespace).List(ctx, listOpts) diff --git a/pkg/operator/k8sutil/configmap.go b/pkg/operator/k8sutil/configmap.go index 44489d5c7..9a10537b4 100644 --- a/pkg/operator/k8sutil/configmap.go +++ b/pkg/operator/k8sutil/configmap.go @@ -43,11 +43,10 @@ func DeleteConfigMap(clientset kubernetes.Interface, cmName, namespace string, o // GetOperatorSetting gets the operator setting from ConfigMap or Env Var // returns defaultValue if setting is not found -func GetOperatorSetting(clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) { +func GetOperatorSetting(context context.Context, clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) { // config must be in operator pod namespace namespace := os.Getenv(PodNamespaceEnvVar) - ctx := context.TODO() - cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configMapName, metav1.GetOptions{}) + cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(context, configMapName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { if settingValue, ok := os.LookupEnv(settingName); ok { @@ -59,13 +58,18 @@ func GetOperatorSetting(clientset kubernetes.Interface, configMapName, settingNa } return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", configMapName, err) } - if settingValue, ok := cm.Data[settingName]; ok { + + return GetValue(cm.Data, settingName, defaultValue), nil +} + +func GetValue(data map[string]string, settingName, defaultValue string) string { + if settingValue, ok := data[settingName]; ok { logger.Infof("%s=%q (configmap)", settingName, settingValue) - return settingValue, nil + return settingValue } else if settingValue, ok := os.LookupEnv(settingName); ok { logger.Infof("%s=%q (env var)", settingName, settingValue) - return settingValue, nil + return settingValue } logger.Infof("%s=%q (default)", settingName, defaultValue) - return defaultValue, nil + return defaultValue } diff --git a/pkg/operator/k8sutil/configmap_test.go b/pkg/operator/k8sutil/configmap_test.go index 2c8e3d980..eb8f6ea55 100644 --- a/pkg/operator/k8sutil/configmap_test.go +++ b/pkg/operator/k8sutil/configmap_test.go @@ -82,7 +82,7 @@ func TestGetOperatorSetting(t *testing.T) { defaultValue := "" // ConfigMap is not found - setting, err := GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) + setting, err := GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) assert.NoError(t, err) // Env Var doesn't exist @@ -90,7 +90,7 @@ func TestGetOperatorSetting(t *testing.T) { // Env Var exists err = os.Setenv(nodeAffinity, envSettingValue) assert.NoError(t, err) - setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) + setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) assert.NoError(t, err) assert.Equal(t, envSettingValue, setting) @@ -100,7 +100,7 @@ func TestGetOperatorSetting(t *testing.T) { assert.NoError(t, err) // Setting exists in ConfigMap - setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) + setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) assert.NoError(t, err) // Env Var exists assert.Equal(t, cmSettingValue, setting) @@ -110,14 +110,14 @@ func TestGetOperatorSetting(t *testing.T) { assert.Equal(t, cmSettingValue, setting) // Setting doesn't exist in ConfigMap - setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue) + setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, podAffinity, defaultValue) assert.NoError(t, err) // Env Var doesn't exist assert.Equal(t, defaultValue, setting) // Env Var exists err = os.Setenv(podAffinity, envSettingValue) assert.NoError(t, err) - setting, err = GetOperatorSetting(k8s, operatorSettingConfigMapName, podAffinity, defaultValue) + setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, podAffinity, defaultValue) assert.NoError(t, err) assert.Equal(t, envSettingValue, setting) } diff --git a/pkg/operator/k8sutil/k8sutil.go b/pkg/operator/k8sutil/k8sutil.go index f3a44c76b..20c6fbec2 100644 --- a/pkg/operator/k8sutil/k8sutil.go +++ b/pkg/operator/k8sutil/k8sutil.go @@ -27,15 +27,12 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" - "github.com/rook/rook/pkg/clusterd" rookversion "github.com/rook/rook/pkg/version" - v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-k8sutil") @@ -192,22 +189,6 @@ func validateLabelValue(value string) string { return sanitized } -// StartOperatorSettingsWatch starts the watch for Operator Settings ConfigMap -func StartOperatorSettingsWatch(context *clusterd.Context, operatorNamespace, operatorSettingConfigMapName string, - addFunc func(obj interface{}), updateFunc func(oldObj, newObj interface{}), deleteFunc func(obj interface{}), stopCh chan struct{}) { - _, cacheController := cache.NewInformer(cache.NewFilteredListWatchFromClient(context.Clientset.CoreV1().RESTClient(), - "configmaps", operatorNamespace, func(options *metav1.ListOptions) { - options.FieldSelector = fmt.Sprintf("%s=%s", "metadata.name", operatorSettingConfigMapName) - }), &v1.ConfigMap{}, - 0, - cache.ResourceEventHandlerFuncs{ - AddFunc: addFunc, - UpdateFunc: updateFunc, - DeleteFunc: deleteFunc, - }) - go cacheController.Run(stopCh) -} - func UsePDBV1Beta1Version(Clientset kubernetes.Interface) (bool, error) { k8sVersion, err := GetK8SVersion(Clientset) if err != nil { diff --git a/pkg/operator/test/client.go b/pkg/operator/test/client.go index b97c8e2db..c5da9dd7b 100644 --- a/pkg/operator/test/client.go +++ b/pkg/operator/test/client.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/google/uuid" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -264,3 +265,36 @@ func pickNode(clientset *fake.Clientset) string { pickNodeIdx++ return name } + +func FakeOperatorPod(ns string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-operator", + Namespace: ns, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testReplicaSet", + }, + }, + }, + Spec: corev1.PodSpec{}, + } + return p +} + +func FakeReplicaSet(ns string) *appsv1.ReplicaSet { + r := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testReplicaSet", + Namespace: ns, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + }, + }, + }, + } + + return r +} diff --git a/pkg/util/flags/flags.go b/pkg/util/flags/flags.go index 2dbcc8463..53b7f1619 100644 --- a/pkg/util/flags/flags.go +++ b/pkg/util/flags/flags.go @@ -60,6 +60,8 @@ func SetLoggingFlags(flags *pflag.FlagSet) { //and not to a file by default flags.AddGoFlagSet(flag.CommandLine) if err := flags.Set("logtostderr", "true"); err != nil { + // TODO: fix me + // 2021-09-02 09:41:29.645366 I | op-flags: failed to set flag "logtostderr". no such flag -logtostderr logger.Infof("failed to set flag %q. %v", "logtostderr", err) } if err := flags.Parse(nil); err != nil { diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index a85bd9284..234c225ea 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -47,8 +47,11 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(values map[string]interface{}) if err := h.k8shelper.CreateNamespace(h.settings.OperatorNamespace); err != nil { return errors.Errorf("failed to create namespace %s. %v", h.settings.Namespace, err) } + if err := h.k8shelper.CreateOpConfigMap(h.settings.OperatorNamespace); err != nil { + return errors.Errorf("failed to create operator config map. %v", err) + } if err := h.startAdmissionController(); err != nil { - return errors.Errorf("Failed to start admission controllers: %v", err) + return errors.Errorf("failed to start admission controllers. %v", err) } if err := h.helmHelper.InstallLocalRookHelmChart(h.settings.OperatorNamespace, OperatorChartName, values); err != nil { return errors.Errorf("failed to install rook operator via helm, err : %v", err) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 3fbbc847e..f5b364d2d 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -36,6 +36,7 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/cluster/crash" + "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" "github.com/stretchr/testify/require" @@ -368,6 +369,22 @@ func (k8sh *K8sHelper) CreateNamespace(namespace string) error { return nil } +func (k8sh *K8sHelper) CreateOpConfigMap(namespace string) error { + ctx := context.TODO() + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.OperatorSettingConfigMapName, + Namespace: namespace, + }, + } + _, err := k8sh.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{}) + if err != nil && !kerrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create operator config map %s. %+v", controller.OperatorSettingConfigMapName, err) + } + + return nil +} + func (k8sh *K8sHelper) CountPodsWithLabel(label string, namespace string) (int, error) { ctx := context.TODO() options := metav1.ListOptions{LabelSelector: label} diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index d3fba44d9..a688086c7 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -429,7 +429,8 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh * store := &cephv1.CephObjectStore{} i := 0 - for i = 0; i < 4; i++ { + retry := 10 + for i = 0; i < retry; i++ { storeStr, err := k8sh.GetResource("-n", namespace, "CephObjectStore", storeName, "-o", "json") assert.NoError(t, err) logger.Infof("store: \n%s", storeStr) @@ -438,13 +439,15 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh * assert.NoError(t, err) cond := cephv1.FindStatusCondition(store.Status.Conditions, cephv1.ConditionDeletionIsBlocked) - if cond.Status == v1.ConditionFalse { + if cond.Status == v1.ConditionFalse && cond.Reason == cephv1.ObjectHasNoDependentsReason { + // Let's give some time to the object to be updated + time.Sleep(5 * time.Second) break } logger.Info("waiting 3 more seconds for CephObjectStore to be unblocked by dependents") time.Sleep(3 * time.Second) } - assert.NotEqual(t, 4, i) + assert.NotEqual(t, retry, i) assert.Equal(t, cephv1.ConditionDeleting, store.Status.Phase) // phase == "Deleting" // verify deletion is NOT blocked b/c object has dependents