From 388ff3e78bbfde4d040d8628fd7808e28c66a71a Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 12 Feb 2021 17:41:40 -0700 Subject: [PATCH] ceph: start sidecar to monitor active mgr The mgr daemon may be failed over by ceph if the active mgr is not responding and the standby mgr is available. If the active mgr changes the services for the dashboard and metrics will be updated with a label selector for the new active mgr. The services cannot direct traffic to the standby mgr or else they will be incorrectly redirected to the active mgr. Signed-off-by: Travis Nielsen --- PendingReleaseNotes.md | 3 +- cluster/charts/rook-ceph/templates/role.yaml | 2 + cluster/examples/kubernetes/ceph/common.yaml | 2 + cmd/rook/ceph/ceph.go | 1 + cmd/rook/ceph/mgr.go | 92 ++++++++++++ cmd/rook/ceph/osd.go | 6 +- pkg/daemon/ceph/client/config.go | 32 +++++ pkg/daemon/ceph/client/info.go | 1 + pkg/daemon/ceph/client/mgr.go | 10 +- pkg/daemon/ceph/osd/daemon.go | 36 +---- pkg/daemon/ceph/osd/remove.go | 2 +- pkg/operator/ceph/cluster/mgr/mgr.go | 74 +++++++--- pkg/operator/ceph/cluster/mgr/mgr_test.go | 136 +++++++++++++++--- pkg/operator/ceph/cluster/mgr/spec.go | 75 +++++----- pkg/operator/ceph/cluster/mon/spec.go | 2 +- pkg/operator/ceph/cluster/osd/envs.go | 1 + pkg/operator/ceph/cluster/osd/osd_pvc_test.go | 1 + pkg/operator/ceph/cluster/osd/osd_test.go | 3 + pkg/operator/ceph/cluster/osd/spec.go | 4 +- pkg/operator/ceph/cluster/osd/spec_test.go | 5 + pkg/operator/k8sutil/deployment.go | 123 ++++++++-------- pkg/operator/k8sutil/pod.go | 10 -- pkg/operator/k8sutil/pod_test.go | 5 - pkg/operator/k8sutil/service.go | 3 + tests/framework/installer/ceph_manifests.go | 2 + 25 files changed, 432 insertions(+), 199 deletions(-) create mode 100644 cmd/rook/ceph/mgr.go diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c119e6bf7..fa407a7a2 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,7 +9,7 @@ v1.6... ## Breaking Changes ### Ceph -* Support for adding OSDs via Drive Groups was removed. Please refer to the +* Support for adding OSDs via Drive Groups was removed. Please refer to the [Ceph upgrade guide](Documentation/ceph-upgrade.md#migrate-the-drive-group-spec) for migration instructions. See https://github.com/rook/rook/issues/7275 for more information. @@ -26,5 +26,6 @@ v1.6... * Extending the support of vault KMS configuration for Ceph RGW * Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons * Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon +* Multiple Ceph mgr daemons are supported for stretch clusters and other clusters where HA of the mgr is more critical * Ceph OSD: as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore * Disable CSI GRPC metrics by default diff --git a/cluster/charts/rook-ceph/templates/role.yaml b/cluster/charts/rook-ceph/templates/role.yaml index e1a594690..00cb93c36 100644 --- a/cluster/charts/rook-ceph/templates/role.yaml +++ b/cluster/charts/rook-ceph/templates/role.yaml @@ -92,6 +92,8 @@ rules: - get - list - watch + - create + - update - delete - apiGroups: - batch diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 7e61d27cc..ee19a1b80 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -465,6 +465,8 @@ rules: - get - list - watch + - create + - update - delete - apiGroups: - batch diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index ba311eb64..8b099ad1d 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -61,6 +61,7 @@ func init() { agentCmd, admissionCmd, osdCmd, + mgrCmd, configCmd) } diff --git a/cmd/rook/ceph/mgr.go b/cmd/rook/ceph/mgr.go new file mode 100644 index 000000000..2b8ba012a --- /dev/null +++ b/cmd/rook/ceph/mgr.go @@ -0,0 +1,92 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ceph + +import ( + "time" + + "github.com/rook/rook/cmd/rook/rook" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" +) + +var mgrCmd = &cobra.Command{ + Use: "mgr", +} +var mgrSidecarCmd = &cobra.Command{ + Use: "watch-active", +} +var ( + updateMgrServicesInterval string + daemonName string + clusterSpec cephv1.ClusterSpec +) + +func init() { + addCephFlags(mgrCmd) + + // add the subcommands to the parent mgr command + mgrCmd.AddCommand(mgrSidecarCmd) + + mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Dashboard.Enabled, "dashboard-enabled", false, "whether the dashboard is enabled") + mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Monitoring.Enabled, "monitoring-enabled", false, "whether the monitoring is enabled") + mgrSidecarCmd.Flags().StringVar(&updateMgrServicesInterval, "update-interval", "", "the interval at which to update the mgr services") + mgrSidecarCmd.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster") + mgrSidecarCmd.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster") + mgrSidecarCmd.Flags().StringVar(&daemonName, "daemon-name", "", "the name of the local mgr daemon") + + flags.SetFlagsFromEnv(mgrCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetFlagsFromEnv(mgrSidecarCmd.Flags(), rook.RookEnvVarPrefix) + mgrSidecarCmd.RunE = runMgrSidecar +} + +// Start the mgr daemon sidecar +func runMgrSidecar(cmd *cobra.Command, args []string) error { + rook.SetLogLevel() + + context := createContext() + clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) + rook.LogStartupInfo(mgrSidecarCmd.Flags()) + + ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) + clusterInfo.OwnerRef = ownerRef + + if err := client.WriteCephConfig(context, &clusterInfo); err != nil { + rook.TerminateFatal(err) + } + + interval, err := time.ParseDuration(updateMgrServicesInterval) + if err != nil { + rook.TerminateFatal(err) + } + + m := mgr.New(context, &clusterInfo, clusterSpec, "") + for { + err := m.ReconcileMultipleServices(daemonName) + if err != nil { + logger.Errorf("failed to reconcile services. %v", err) + } else { + logger.Infof("successfully reconciled services. checking again in %ds", (int)(interval.Seconds())) + } + time.Sleep(interval) + } +} diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 886f860a6..38fa7b4d1 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -60,6 +60,7 @@ var ( osdDataDeviceFilter string osdDataDevicePathFilter string ownerRefID string + clusterName string osdID int osdStoreType string osdStringID string @@ -106,7 +107,8 @@ func addOSDFlags(command *cobra.Command) { } func addOSDConfigFlags(command *cobra.Command) { - command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster") + command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster") + command.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster") command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") command.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node") @@ -218,7 +220,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { logger.Infof("crush location of osd: %s", crushLocation) forceFormat := false - ownerRef := opcontroller.ClusterOwnerRef(clusterInfo.Namespace, ownerRefID) + ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) clusterInfo.OwnerRef = ownerRef kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerRef) agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat, diff --git a/pkg/daemon/ceph/client/config.go b/pkg/daemon/ceph/client/config.go index ff668e4a1..c87b8dd92 100644 --- a/pkg/daemon/ceph/client/config.go +++ b/pkg/daemon/ceph/client/config.go @@ -20,9 +20,11 @@ package client import ( "context" "fmt" + "io/ioutil" "net" "os" "path" + "path/filepath" "strconv" "strings" @@ -270,3 +272,33 @@ func PopulateMonHostMembers(monitors map[string]*MonInfo) ([]string, []string) { return monMembers, monHosts } + +// WriteCephConfig writes the ceph config so ceph commands can be executed +func WriteCephConfig(context *clusterd.Context, clusterInfo *ClusterInfo) error { + // create the ceph.conf with the default settings + cephConfig, err := CreateDefaultCephConfig(context, clusterInfo) + if err != nil { + return errors.Wrap(err, "failed to create default ceph config") + } + + // write the latest config to the config dir + confFilePath, err := GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig) + if err != nil { + return errors.Wrap(err, "failed to write connection config") + } + src, err := ioutil.ReadFile(filepath.Clean(confFilePath)) + if err != nil { + return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config") + } + err = ioutil.WriteFile(DefaultConfigFilePath(), src, 0444) + if err != nil { + return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", DefaultConfigFilePath()) + } + dst, err := ioutil.ReadFile(DefaultConfigFilePath()) + if err == nil { + logger.Debugf("config file @ %s: %s", DefaultConfigFilePath(), dst) + } else { + logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", DefaultConfigFilePath(), err) + } + return nil +} diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index 8eaf54b08..fae7c254e 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -81,6 +81,7 @@ func AdminClusterInfo(namespace string) *ClusterInfo { CephCred: CephCred{ Username: AdminUsername, }, + name: "testing", } } diff --git a/pkg/daemon/ceph/client/mgr.go b/pkg/daemon/ceph/client/mgr.go index 98640b0b6..21c5932ab 100755 --- a/pkg/daemon/ceph/client/mgr.go +++ b/pkg/daemon/ceph/client/mgr.go @@ -30,22 +30,22 @@ var ( moduleEnableWaitTime = 5 * time.Second ) -func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (MgrMap, error) { +func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (*MgrMap, error) { args := []string{"mgr", "dump"} buf, err := NewCephCommand(context, clusterInfo, args).Run() if err != nil { if len(buf) > 0 { - return MgrMap{}, errors.Wrapf(err, "failed to get status. %s", string(buf)) + return nil, errors.Wrapf(err, "failed to get mgr dump. %s", string(buf)) } - return MgrMap{}, errors.Wrap(err, "failed to get ceph status") + return nil, errors.Wrap(err, "failed to get mgr dump") } var mgrMap MgrMap if err := json.Unmarshal([]byte(buf), &mgrMap); err != nil { - return MgrMap{}, errors.Wrap(err, "failed to unmarshal status response") + return nil, errors.Wrap(err, "failed to unmarshal mgr dump") } - return mgrMap, nil + return &mgrMap, nil } // MgrEnableModule enables a mgr module diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index b8415608e..96b48aa4f 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -18,7 +18,6 @@ package osd import ( "fmt" - "io/ioutil" "os" "os/signal" "path/filepath" @@ -29,7 +28,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" - cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/daemon/ceph/client" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/util/sys" ) @@ -163,37 +162,6 @@ func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, er return rawDevice, nil } -// writeCephConfig writes the ceph config so ceph commands can be executed -func writeCephConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error { - - // create the ceph.conf with the default settings - cephConfig, err := cephclient.CreateDefaultCephConfig(context, clusterInfo) - if err != nil { - return errors.Wrap(err, "failed to create default ceph config") - } - - // write the latest config to the config dir - confFilePath, err := cephclient.GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig) - if err != nil { - return errors.Wrap(err, "failed to write connection config") - } - src, err := ioutil.ReadFile(filepath.Clean(confFilePath)) - if err != nil { - return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config") - } - err = ioutil.WriteFile(cephclient.DefaultConfigFilePath(), src, 0444) - if err != nil { - return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", cephclient.DefaultConfigFilePath()) - } - dst, err := ioutil.ReadFile(cephclient.DefaultConfigFilePath()) - if err == nil { - logger.Debugf("config file @ %s: %s", cephclient.DefaultConfigFilePath(), dst) - } else { - logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", cephclient.DefaultConfigFilePath(), err) - } - return nil -} - // Provision provisions an OSD func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error { if agent.pvcBacked { @@ -214,7 +182,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating} oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) - if err := writeCephConfig(context, agent.clusterInfo); err != nil { + if err := client.WriteCephConfig(context, agent.clusterInfo); err != nil { return errors.Wrap(err, "failed to generate ceph config") } diff --git a/pkg/daemon/ceph/osd/remove.go b/pkg/daemon/ceph/osd/remove.go index 7d436afe8..26c6a940b 100644 --- a/pkg/daemon/ceph/osd/remove.go +++ b/pkg/daemon/ceph/osd/remove.go @@ -35,7 +35,7 @@ import ( func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error { // Generate the ceph config for running ceph commands similar to the operator - if err := writeCephConfig(context, clusterInfo); err != nil { + if err := client.WriteCephConfig(context, clusterInfo); err != nil { return errors.Wrap(err, "failed to write the ceph config") } diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 4b6cee553..be1eea4ab 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -37,6 +37,7 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" + v1 "k8s.io/api/apps/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -67,13 +68,11 @@ const ( // Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs. type Cluster struct { - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - Replicas int - rookVersion string - exitCode func(err error) (int, bool) - appliedHttpBind bool - spec cephv1.ClusterSpec + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + rookVersion string + exitCode func(err error) (int, bool) + spec cephv1.ClusterSpec } // New creates an instance of the mgr @@ -83,16 +82,16 @@ func New(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, spec ce clusterInfo: clusterInfo, spec: spec, rookVersion: rookVersion, - Replicas: spec.Mgr.Count, exitCode: exec.ExitStatus, } } +var waitForDeploymentToStart = k8sutil.WaitForDeploymentToStart var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // for backward compatibility, default to 1 mgr func (c *Cluster) getReplicas() int { - replicas := c.Replicas + replicas := c.spec.Mgr.Count if replicas == 0 { replicas = 1 } @@ -118,6 +117,8 @@ func (c *Cluster) Start() error { logger.Infof("start running mgr") daemonIDs := c.getDaemonIDs() + var deploymentsToWaitFor []*v1.Deployment + for _, daemonID := range daemonIDs { // Check whether we need to cancel the orchestration if err := controller.CheckForCancelledOrchestration(c.context); err != nil { @@ -150,7 +151,7 @@ func (c *Cluster) Start() error { return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name) } - _, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{}) + newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{}) if err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName) @@ -160,11 +161,26 @@ func (c *Cluster) Start() error { if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, false); err != nil { logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err) } + } else { + // wait for the new deployment + deploymentsToWaitFor = append(deploymentsToWaitFor, newDeployment) } } - if err := c.reconcileServices(); err != nil { - return errors.Wrap(err, "failed to enable mgr services") + // If the mgr is newly created, wait for it to start before continuing with the service and + // module configuration + for _, d := range deploymentsToWaitFor { + if err := waitForDeploymentToStart(c.context, d); err != nil { + return errors.Wrapf(err, "failed to wait for mgr %q to start", d.Name) + } + } + + if len(daemonIDs) == 1 { + // Only reconcile the mgr services in the operator if there is a single mgr. + // If there are multiple mgrs they will be managed by the mgr sidecar. + if err := c.reconcileService(daemonIDs[0]); err != nil { + return errors.Wrap(err, "failed to enable mgr services") + } } // configure the mgr modules @@ -188,17 +204,40 @@ func (c *Cluster) Start() error { return nil } -func (c *Cluster) reconcileServices() error { +// ReconcileMultipleServices reconciles the services if the active mgr is the one running +// in the sidecar +func (c *Cluster) ReconcileMultipleServices(daemonNameToUpdate string) error { + // If the services are already set to this daemon, no need to attempt to update + svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), AppName, metav1.GetOptions{}) + if err != nil { + logger.Errorf("failed to check current mgr service, proceeding to update. %v", err) + } else { + currentDaemon := svc.Spec.Selector[controller.DaemonIDLabel] + if currentDaemon == daemonNameToUpdate { + logger.Infof("mgr services already set to daemon %q, no need to update", daemonNameToUpdate) + return nil + } + logger.Infof("mgr service currently set to %q, checking if need to update to %q", currentDaemon, daemonNameToUpdate) + } mgrMap, err := cephclient.CephMgrMap(c.context, c.clusterInfo) if err != nil { return errors.Wrap(err, "failed to get ceph status for the active mgr") } - activeDaemon := mgrMap.ActiveName - if activeDaemon == "" { - return errors.Errorf("active mgr not found") + if mgrMap.ActiveName == "" { + return errors.New("active mgr not found") } - logger.Infof("Active mgr is %q", activeDaemon) + if daemonNameToUpdate != mgrMap.ActiveName { + logger.Infof("no need for the mgr update since the active mgr is %q, rather than the local mgr %q", mgrMap.ActiveName, daemonNameToUpdate) + return nil + } + + return c.reconcileService(mgrMap.ActiveName) +} + +// reconcile the services, if the active mgr is not detected, use the default mgr +func (c *Cluster) reconcileService(activeDaemon string) error { + logger.Infof("setting services to point to mgr %q", activeDaemon) if err := c.configureDashboardService(activeDaemon); err != nil { return errors.Wrap(err, "failed to configure dashboard svc") @@ -222,7 +261,6 @@ func (c *Cluster) reconcileServices() error { func (c *Cluster) configureModules(daemonIDs []string) { // Configure the modules asynchronously so we can complete all the configuration much sooner. - startModuleConfiguration("http bind settings", c.clearHTTPBindFix) startModuleConfiguration("prometheus", c.enablePrometheusModule) startModuleConfiguration("dashboard", c.configureDashboardModules) // "crash" is part of the "always_on_modules" list as of Octopus diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index 0386a7012..b65f4619a 100755 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -35,12 +35,13 @@ import ( "github.com/stretchr/testify/require" "github.com/tevino/abool" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestStartMgr(t *testing.T) { - ctx := context.TODO() var deploymentsUpdated *[]*apps.Deployment updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() @@ -49,11 +50,15 @@ func TestStartMgr(t *testing.T) { return "{\"key\":\"mysecurekey\"}", nil }, } + waitForDeploymentToStart = func(clusterdContext *clusterd.Context, deployment *v1.Deployment) error { + logger.Infof("simulated mgr deployment starting") + return nil + } clientset := testop.New(t, 3) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - context := &clusterd.Context{ + ctx := &clusterd.Context{ Executor: executor, ConfigDir: configDir, Clientset: clientset, @@ -68,13 +73,13 @@ func TestStartMgr(t *testing.T) { PriorityClassNames: map[rookv1.KeyType]string{cephv1.KeyMgr: "my-priority-class"}, DataDirHostPath: "/var/lib/rook/", } - c := New(context, clusterInfo, clusterSpec, "myversion") + c := New(ctx, clusterInfo, clusterSpec, "myversion") defer os.RemoveAll(c.spec.DataDirHostPath) // start a basic service err := c.Start() assert.Nil(t, err) - validateStart(ctx, t, c) + validateStart(t, c) assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) @@ -82,40 +87,64 @@ func TestStartMgr(t *testing.T) { c.spec.Dashboard.Port = 12345 err = c.Start() assert.Nil(t, err) - validateStart(ctx, t, c) + validateStart(t, c) assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) - // starting again with more replicas - c.Replicas = 3 + // starting with more replicas + c.spec.Mgr.Count = 2 c.spec.Dashboard.Enabled = false + // delete the previous mgr since the mocked test won't update the existing one + err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{}) + assert.Nil(t, err) err = c.Start() assert.Nil(t, err) - validateStart(ctx, t, c) - assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) - testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) + // trigger the sidecar reconcile since the operator didn't do it so we can perform the full validation + err = c.reconcileService("a") + assert.Nil(t, err) + validateStart(t, c) + + // the dashboard service is only deleted by the operator reconcile if the replicas are 1, + // otherwise the sidecar has the responsibility + c.spec.Mgr.Count = 1 + c.spec.Dashboard.Enabled = false + // clean the previous deployments + err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{}) + assert.Nil(t, err) + err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-b", metav1.DeleteOptions{}) + assert.Nil(t, err) + err = c.Start() + assert.Nil(t, err) + validateStart(t, c) } -func validateStart(ctx context.Context, t *testing.T, c *Cluster) { +func validateStart(t *testing.T, c *Cluster) { mgrNames := []string{"a", "b"} - for i := 0; i < c.Replicas; i++ { - if i == 2 { - logger.Warning("cannot have more than 2 mgrs") - break - } + for i := 0; i < c.spec.Mgr.Count; i++ { logger.Infof("Looking for cephmgr replica %d", i) daemonName := mgrNames[i] - d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(ctx, fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{}) + d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(context.TODO(), fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, map[string]string{"my": "annotation"}, d.Spec.Template.Annotations) assert.Contains(t, d.Spec.Template.Labels, "my-label-key") assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName) + if c.spec.Mgr.Count == 1 { + assert.Equal(t, 1, len(d.Spec.Template.Spec.Containers)) + } else { + // The sidecar container is only there when multiple mgrs are enabled + assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers)) + assert.Equal(t, "watch-active", d.Spec.Template.Spec.Containers[1].Name) + } } - _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr", metav1.GetOptions{}) + validateServices(t, c) +} + +func validateServices(t *testing.T, c *Cluster) { + _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{}) assert.Nil(t, err) - ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{}) + ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr-dashboard", metav1.GetOptions{}) if c.spec.Dashboard.Enabled { assert.NoError(t, err) if c.spec.Dashboard.Port == 0 { @@ -130,6 +159,65 @@ func validateStart(ctx context.Context, t *testing.T, c *Cluster) { } } +func TestMgrSidecarReconcile(t *testing.T) { + activeMgr := "a" + spec := cephv1.ClusterSpec{ + Mgr: cephv1.MgrSpec{Count: 1}, + Dashboard: cephv1.DashboardSpec{ + Enabled: true, + Port: 7000, + }, + } + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutputFile: func(command, outFile string, args ...string) (string, error) { + return fmt.Sprintf(`{"active_name":"%s"}`, activeMgr), nil + }, + } + clientset := testop.New(t, 3) + configDir, _ := ioutil.TempDir("", "") + defer os.RemoveAll(configDir) + ctx := &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + Clientset: clientset, + } + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") + c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo} + + // Update services according to the active mgr + err := c.ReconcileMultipleServices(activeMgr) + assert.NoError(t, err) + validateServices(t, c) + validateServiceMatches(t, c, "a") + + // nothing is created or updated when the requested mgr is not the active mgr + err = c.ReconcileMultipleServices("b") + assert.NoError(t, err) + _, err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{}) + assert.True(t, kerrors.IsNotFound(err)) + + // nothing is updated when the requested mgr is not the active mgr + activeMgr = "b" + err = c.ReconcileMultipleServices("b") + assert.NoError(t, err) + validateServices(t, c) + validateServiceMatches(t, c, "b") +} + +func validateServiceMatches(t *testing.T, c *Cluster, expectedActive string) { + // The service labels should match the active mgr + svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{}) + assert.NoError(t, err) + matchDaemon, ok := svc.Spec.Selector["ceph_daemon_id"] + assert.True(t, ok) + assert.Equal(t, expectedActive, matchDaemon) + + // clean up the service for the next test + err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr", metav1.DeleteOptions{}) + assert.NoError(t, err) +} + func TestConfigureModules(t *testing.T) { modulesEnabled := 0 modulesDisabled := 0 @@ -205,8 +293,16 @@ func TestConfigureModules(t *testing.T) { } func TestMgrDaemons(t *testing.T) { - c := &Cluster{Replicas: 2} + spec := cephv1.ClusterSpec{ + Mgr: cephv1.MgrSpec{Count: 1}, + } + c := &Cluster{spec: spec} daemons := c.getDaemonIDs() + require.Equal(t, 1, len(daemons)) + assert.Equal(t, "a", daemons[0]) + + c.spec.Mgr.Count = 2 + daemons = c.getDaemonIDs() require.Equal(t, 2, len(daemons)) assert.Equal(t, "a", daemons[0]) assert.Equal(t, "b", daemons[1]) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 598df59a5..4f221f3c2 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -21,11 +21,10 @@ import ( "os" "strconv" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" - "github.com/rook/rook/pkg/daemon/ceph/client" + opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" @@ -63,6 +62,9 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) PriorityClassName: cephv1.GetMgrPriorityClassName(c.spec.PriorityClassNames), }, } + if c.spec.Mgr.Count > 1 { + podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig)) + } // If the log collector is enabled we add the side-car container if c.spec.LogCollector.Enabled { @@ -113,42 +115,6 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) return d, nil } -// if we do not need the http bind fix, then we need to be careful. if we are -// upgrading from a cluster that had the fix applied, then the fix is no longer -// needed, and furthermore, needs to be removed so that there is not a lingering -// ceph configuration option that contains an old ip. by clearing the option, -// we let ceph bind to its default ANYADDR address. However, since we don't -// know which version of Ceph we are may be upgrading _from_ we need to (a) -// always do this and (b) make sure that all forms of the configuration option -// are removed (see the init container factory method). Once the minimum -// supported version of Rook contains the fix, all of this can be removed. -func (c *Cluster) clearHTTPBindFix() error { - // We only need to apply these changes once. No harm in once each time the operator restarts. - if c.appliedHttpBind { - return nil - } - for _, daemonID := range c.getDaemonIDs() { - for _, module := range []string{"dashboard", "prometheus"} { - // there are two forms of the configuration key that might exist which - // depends not on the current version, but on the version that may be - // the version being upgraded from. - if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID, - fmt.Sprintf("mgr/%s/server_addr", module), "", false); err != nil { - return errors.Wrap(err, "failed to set config for an mgr daemon using v2 format") - } - - // this is for the format used in v1.0 - // https://github.com/rook/rook/commit/11d318fb2f77a6ac9a8f2b9be42c826d3b4a93c3 - if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID, - fmt.Sprintf("mgr/%s/%s/server_addr", module, daemonID), "", false); err != nil { - return errors.Wrap(err, "failed to set config for an mgr daemon using v1 format") - } - } - } - c.appliedHttpBind = true - return nil -} - func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container { return controller.ChownCephDataDirsInitContainer( *mgrConfig.DataPathMap, @@ -217,6 +183,39 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { return container } +func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container { + envVars := []v1.EnvVar{ + {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)}, + {Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)}, + k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), + opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace), + opmon.EndpointEnvVar(), + opmon.SecretEnvVar(), + opmon.CephUsernameEnvVar(), + opmon.CephSecretEnvVar(), + k8sutil.ConfigOverrideEnvVar(), + {Name: "ROOK_FSID", ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{Name: "rook-ceph-mon"}, + Key: "fsid", + }, + }}, + {Name: "ROOK_DASHBOARD_ENABLED", Value: strconv.FormatBool(c.spec.Dashboard.Enabled)}, + {Name: "ROOK_MONITORING_ENABLED", Value: strconv.FormatBool(c.spec.Monitoring.Enabled)}, + {Name: "ROOK_UPDATE_INTERVAL", Value: "15s"}, + {Name: "ROOK_DAEMON_NAME", Value: mgrConfig.DaemonID}, + } + + return v1.Container{ + Args: []string{"ceph", "mgr", "watch-active"}, + Name: "watch-active", + Image: c.rookVersion, + Env: envVars, + //Resources: **TODO**, + } +} + func getDefaultMgrLivenessProbe() *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index d003ef025..092c497e8 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -362,6 +362,6 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client. return nil } - _, err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback) + err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback) return err } diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index 04c0bea82..db7ef3ad5 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -57,6 +57,7 @@ func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1. envVars := []v1.EnvVar{ nodeNameEnvVar(osdProps.crushHostname), {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)}, + {Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace), diff --git a/pkg/operator/ceph/cluster/osd/osd_pvc_test.go b/pkg/operator/ceph/cluster/osd/osd_pvc_test.go index 18a1eef18..3e920b6b9 100644 --- a/pkg/operator/ceph/cluster/osd/osd_pvc_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_pvc_test.go @@ -224,6 +224,7 @@ func testOSDsOnPVC(t *testing.T) { Namespace: namespace, CephVersion: cephver.Nautilus, } + clusterInfo.SetName("mycluster") executor := osdPVCTestExecutor(t, clientset, namespace) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: executor, RequestCancelOrchestration: abool.New()} diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index ce4da39b4..78b85cfa7 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -295,6 +295,7 @@ func TestAddNodeFailure(t *testing.T) { Namespace: "ns-add-remove", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("testcluster") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -381,6 +382,7 @@ func TestGetPVCHostName(t *testing.T) { func TestGetOSDInfo(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") context := &clusterd.Context{} spec := cephv1.ClusterSpec{DataDirHostPath: "/rook"} c := New(context, clusterInfo, spec, "myversion") @@ -517,6 +519,7 @@ func TestDetectCrushLocation(t *testing.T) { func TestGetOSDInfoWithCustomRoot(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") context := &clusterd.Context{} spec := cephv1.ClusterSpec{ DataDirHostPath: "/rook", diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index b2bb746bf..473c277e3 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -488,7 +488,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC v1.Container{ Args: []string{"ceph", "osd", "init"}, Name: controller.ConfigInitContainerName, - Image: k8sutil.MakeRookImage(c.rookVersion), + Image: c.rookVersion, VolumeMounts: configVolumeMounts, Env: configEnvVars, SecurityContext: securityContext, @@ -708,7 +708,7 @@ func (c *Cluster) getCopyBinariesContainer() (v1.Volume, *v1.Container) { "copy-binaries", "--copy-to-dir", rookBinariesMountPath}, Name: "copy-bins", - Image: k8sutil.MakeRookImage(c.rookVersion), + Image: c.rookVersion, VolumeMounts: []v1.VolumeMount{mount}, } } diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 40ce50153..60dcfe45f 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -88,6 +88,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { Namespace: "ns", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("test") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"}, @@ -430,6 +431,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { Namespace: "ns", CephVersion: cephver.Octopus, } + clusterInfo.SetName("test") c := New(context, clusterInfo, spec, "rook/rook:myversion") deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.NoError(t, err) @@ -464,6 +466,7 @@ func TestStorageSpecConfig(t *testing.T) { Namespace: "ns", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("testing") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -545,6 +548,7 @@ func TestHostNetwork(t *testing.T) { Namespace: "ns", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("test") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ Storage: storageSpec, @@ -583,6 +587,7 @@ func TestOsdPrepareResources(t *testing.T) { context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") spec := cephv1.ClusterSpec{ Resources: map[string]v1.ResourceRequirements{"prepareosd": { Limits: v1.ResourceList{ diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index 3338d71c2..c3e1fa458 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -61,11 +61,11 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c // 2. verify that we can continue the update procedure // Basically, we go one resource by one and check if we can stop and then if the resource has been successfully updated // we check if we can go ahead and move to the next one. -func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) (*v1.Deployment, error) { +func UpdateDeploymentAndWait(clusterContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) error { ctx := context.TODO() - currentDeployment, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{}) + currentDeployment, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err) + return fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err) } // Check whether the current deployment and newly generated one are identical @@ -78,67 +78,66 @@ func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployme patchChanged = true } - // If deployments are different, let's update! - if patchChanged { - logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name) - - // Let's verify the deployment can be stopped - err = verifyCallback("stop") - if err != nil { - return nil, fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err) - } - - // Set hash annotation to the newly generated deployment - err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment) - if err != nil { - return nil, fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err) - } - - if _, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil { - return nil, fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err) - } - - // wait for the deployment to be restarted - sleepTime := 2 - attempts := 30 - if currentDeployment.Spec.ProgressDeadlineSeconds != nil { - // make the attempts double the progress deadline since the pod is both stopping and starting - attempts = 2 * (int(*currentDeployment.Spec.ProgressDeadlineSeconds) / sleepTime) - } - for i := 0; i < attempts; i++ { - // check for the status of the deployment - d, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to get deployment %q. %v", modifiedDeployment.Name, err) - } - if d.Status.ObservedGeneration != currentDeployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { - logger.Infof("finished waiting for updated deployment %q", d.Name) - - // Now we check if we can go to the next daemon - err = verifyCallback("continue") - if err != nil { - return nil, fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err) - } - - return d, nil - } - - // If ProgressDeadlineExceeded is reached let's fail earlier - // This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state - for _, condition := range d.Status.Conditions { - if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" { - return nil, fmt.Errorf("gave up waiting for deployment %q to update because %q", modifiedDeployment.Name, condition.Reason) - } - } - - logger.Debugf("deployment %q status=%+v", d.Name, d.Status) - time.Sleep(time.Duration(sleepTime) * time.Second) - } - return nil, fmt.Errorf("gave up waiting for deployment %q to update", modifiedDeployment.Name) + if !patchChanged { + logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name) + return nil } - logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name) - return nil, nil + // If deployments are different, let's update! + logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name) + + // Let's verify the deployment can be stopped + if err := verifyCallback("stop"); err != nil { + return fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err) + } + + // Set hash annotation to the newly generated deployment + if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment); err != nil { + return fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err) + } + + if _, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err) + } + + if err := WaitForDeploymentToStart(clusterContext, currentDeployment); err != nil { + return err + } + + // Now we check if we can go to the next daemon + if err := verifyCallback("continue"); err != nil { + return fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err) + } + return nil +} + +func WaitForDeploymentToStart(clusterdContext *clusterd.Context, deployment *v1.Deployment) error { + // wait for the deployment to be restarted up to 300s + sleepTime := 3 + attempts := 100 + for i := 0; i < attempts; i++ { + // check for the status of the deployment + d, err := clusterdContext.Clientset.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %q. %v", deployment.Name, err) + } + if d.Status.ObservedGeneration != deployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { + logger.Infof("finished waiting for updated deployment %q", d.Name) + return nil + } + + // If ProgressDeadlineExceeded is reached let's fail earlier + // This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state + for _, condition := range d.Status.Conditions { + if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" { + return fmt.Errorf("gave up waiting for deployment %q to update because %q", deployment.Name, condition.Reason) + } + } + + logger.Debugf("deployment %q status=%+v", d.Name, d.Status) + time.Sleep(time.Duration(sleepTime) * time.Second) + } + return fmt.Errorf("gave up waiting for deployment %q to update", deployment.Name) } // GetDeployments returns a list of deployment names labels matching a given selector diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 04e19bfa5..f8e34557b 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -51,7 +51,6 @@ const ( ConfigOverrideName = "rook-config-override" // ConfigOverrideVal config override value ConfigOverrideVal = "config" - defaultVersion = "rook/rook:latest" configMountDir = "/etc/rook/config" overrideFilename = "override.conf" ) @@ -203,15 +202,6 @@ func GetMatchingContainer(containers []v1.Container, name string) (v1.Container, return *result, nil } -// MakeRookImage formats the container name -func MakeRookImage(version string) string { - if version == "" { - return defaultVersion - } - - return version -} - // PodsRunningWithLabel returns the number of running pods with the given label func PodsRunningWithLabel(clientset kubernetes.Interface, namespace, label string) (int, error) { running, _, err := podStatusWithLabel(clientset, namespace, label) diff --git a/pkg/operator/k8sutil/pod_test.go b/pkg/operator/k8sutil/pod_test.go index 459d1e52b..9a875c742 100644 --- a/pkg/operator/k8sutil/pod_test.go +++ b/pkg/operator/k8sutil/pod_test.go @@ -26,11 +26,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestMakeRookImage(t *testing.T) { - assert.Equal(t, "rook/rook:v1", MakeRookImage("rook/rook:v1")) - assert.Equal(t, defaultVersion, MakeRookImage("")) -} - func TestGetContainerInPod(t *testing.T) { expectedName := "mycontainer" imageName := "myimage" diff --git a/pkg/operator/k8sutil/service.go b/pkg/operator/k8sutil/service.go index a9f8ab081..51b73f2b6 100644 --- a/pkg/operator/k8sutil/service.go +++ b/pkg/operator/k8sutil/service.go @@ -33,6 +33,7 @@ func CreateOrUpdateService( ctx := context.TODO() name := serviceDefinition.Name logger.Debugf("creating service %s", name) + s, err := clientset.CoreV1().Services(namespace).Create(ctx, serviceDefinition, metav1.CreateOptions{}) if err != nil { if !errors.IsAlreadyExists(err) { @@ -42,6 +43,8 @@ func CreateOrUpdateService( if err != nil { return nil, fmt.Errorf("failed to update service %s. %+v", name, err) } + } else { + logger.Debugf("created service %s", s.Name) } return s, err } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index aa5cdad04..a249d471c 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -1061,6 +1061,8 @@ rules: - get - list - watch + - create + - update - delete - apiGroups: - batch