diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c119e6bf7..fa407a7a2 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,7 +9,7 @@ v1.6... ## Breaking Changes ### Ceph -* Support for adding OSDs via Drive Groups was removed. Please refer to the +* Support for adding OSDs via Drive Groups was removed. Please refer to the [Ceph upgrade guide](Documentation/ceph-upgrade.md#migrate-the-drive-group-spec) for migration instructions. See https://github.com/rook/rook/issues/7275 for more information. @@ -26,5 +26,6 @@ v1.6... * Extending the support of vault KMS configuration for Ceph RGW * Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons * Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon +* Multiple Ceph mgr daemons are supported for stretch clusters and other clusters where HA of the mgr is more critical * Ceph OSD: as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore * Disable CSI GRPC metrics by default diff --git a/cluster/charts/rook-ceph/templates/role.yaml b/cluster/charts/rook-ceph/templates/role.yaml index e1a594690..00cb93c36 100644 --- a/cluster/charts/rook-ceph/templates/role.yaml +++ b/cluster/charts/rook-ceph/templates/role.yaml @@ -92,6 +92,8 @@ rules: - get - list - watch + - create + - update - delete - apiGroups: - batch diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 7e61d27cc..ee19a1b80 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -465,6 +465,8 @@ rules: - get - list - watch + - create + - update - delete - apiGroups: - batch diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index ba311eb64..8b099ad1d 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -61,6 +61,7 @@ func init() { agentCmd, admissionCmd, osdCmd, + mgrCmd, configCmd) } diff --git a/cmd/rook/ceph/mgr.go b/cmd/rook/ceph/mgr.go new file mode 100644 index 000000000..2b8ba012a --- /dev/null +++ b/cmd/rook/ceph/mgr.go @@ -0,0 +1,92 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ceph + +import ( + "time" + + "github.com/rook/rook/cmd/rook/rook" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" +) + +var mgrCmd = &cobra.Command{ + Use: "mgr", +} +var mgrSidecarCmd = &cobra.Command{ + Use: "watch-active", +} +var ( + updateMgrServicesInterval string + daemonName string + clusterSpec cephv1.ClusterSpec +) + +func init() { + addCephFlags(mgrCmd) + + // add the subcommands to the parent mgr command + mgrCmd.AddCommand(mgrSidecarCmd) + + mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Dashboard.Enabled, "dashboard-enabled", false, "whether the dashboard is enabled") + mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Monitoring.Enabled, "monitoring-enabled", false, "whether the monitoring is enabled") + mgrSidecarCmd.Flags().StringVar(&updateMgrServicesInterval, "update-interval", "", "the interval at which to update the mgr services") + mgrSidecarCmd.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster") + mgrSidecarCmd.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster") + mgrSidecarCmd.Flags().StringVar(&daemonName, "daemon-name", "", "the name of the local mgr daemon") + + flags.SetFlagsFromEnv(mgrCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetFlagsFromEnv(mgrSidecarCmd.Flags(), rook.RookEnvVarPrefix) + mgrSidecarCmd.RunE = runMgrSidecar +} + +// Start the mgr daemon sidecar +func runMgrSidecar(cmd *cobra.Command, args []string) error { + rook.SetLogLevel() + + context := createContext() + clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) + rook.LogStartupInfo(mgrSidecarCmd.Flags()) + + ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) + clusterInfo.OwnerRef = ownerRef + + if err := client.WriteCephConfig(context, &clusterInfo); err != nil { + rook.TerminateFatal(err) + } + + interval, err := time.ParseDuration(updateMgrServicesInterval) + if err != nil { + rook.TerminateFatal(err) + } + + m := mgr.New(context, &clusterInfo, clusterSpec, "") + for { + err := m.ReconcileMultipleServices(daemonName) + if err != nil { + logger.Errorf("failed to reconcile services. %v", err) + } else { + logger.Infof("successfully reconciled services. checking again in %ds", (int)(interval.Seconds())) + } + time.Sleep(interval) + } +} diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 886f860a6..38fa7b4d1 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -60,6 +60,7 @@ var ( osdDataDeviceFilter string osdDataDevicePathFilter string ownerRefID string + clusterName string osdID int osdStoreType string osdStringID string @@ -106,7 +107,8 @@ func addOSDFlags(command *cobra.Command) { } func addOSDConfigFlags(command *cobra.Command) { - command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster") + command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster") + command.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster") command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") command.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node") @@ -218,7 +220,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { logger.Infof("crush location of osd: %s", crushLocation) forceFormat := false - ownerRef := opcontroller.ClusterOwnerRef(clusterInfo.Namespace, ownerRefID) + ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID) clusterInfo.OwnerRef = ownerRef kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerRef) agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat, diff --git a/pkg/daemon/ceph/client/config.go b/pkg/daemon/ceph/client/config.go index ff668e4a1..c87b8dd92 100644 --- a/pkg/daemon/ceph/client/config.go +++ b/pkg/daemon/ceph/client/config.go @@ -20,9 +20,11 @@ package client import ( "context" "fmt" + "io/ioutil" "net" "os" "path" + "path/filepath" "strconv" "strings" @@ -270,3 +272,33 @@ func PopulateMonHostMembers(monitors map[string]*MonInfo) ([]string, []string) { return monMembers, monHosts } + +// WriteCephConfig writes the ceph config so ceph commands can be executed +func WriteCephConfig(context *clusterd.Context, clusterInfo *ClusterInfo) error { + // create the ceph.conf with the default settings + cephConfig, err := CreateDefaultCephConfig(context, clusterInfo) + if err != nil { + return errors.Wrap(err, "failed to create default ceph config") + } + + // write the latest config to the config dir + confFilePath, err := GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig) + if err != nil { + return errors.Wrap(err, "failed to write connection config") + } + src, err := ioutil.ReadFile(filepath.Clean(confFilePath)) + if err != nil { + return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config") + } + err = ioutil.WriteFile(DefaultConfigFilePath(), src, 0444) + if err != nil { + return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", DefaultConfigFilePath()) + } + dst, err := ioutil.ReadFile(DefaultConfigFilePath()) + if err == nil { + logger.Debugf("config file @ %s: %s", DefaultConfigFilePath(), dst) + } else { + logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", DefaultConfigFilePath(), err) + } + return nil +} diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index 8eaf54b08..fae7c254e 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -81,6 +81,7 @@ func AdminClusterInfo(namespace string) *ClusterInfo { CephCred: CephCred{ Username: AdminUsername, }, + name: "testing", } } diff --git a/pkg/daemon/ceph/client/mgr.go b/pkg/daemon/ceph/client/mgr.go index 98640b0b6..21c5932ab 100755 --- a/pkg/daemon/ceph/client/mgr.go +++ b/pkg/daemon/ceph/client/mgr.go @@ -30,22 +30,22 @@ var ( moduleEnableWaitTime = 5 * time.Second ) -func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (MgrMap, error) { +func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (*MgrMap, error) { args := []string{"mgr", "dump"} buf, err := NewCephCommand(context, clusterInfo, args).Run() if err != nil { if len(buf) > 0 { - return MgrMap{}, errors.Wrapf(err, "failed to get status. %s", string(buf)) + return nil, errors.Wrapf(err, "failed to get mgr dump. %s", string(buf)) } - return MgrMap{}, errors.Wrap(err, "failed to get ceph status") + return nil, errors.Wrap(err, "failed to get mgr dump") } var mgrMap MgrMap if err := json.Unmarshal([]byte(buf), &mgrMap); err != nil { - return MgrMap{}, errors.Wrap(err, "failed to unmarshal status response") + return nil, errors.Wrap(err, "failed to unmarshal mgr dump") } - return mgrMap, nil + return &mgrMap, nil } // MgrEnableModule enables a mgr module diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index b8415608e..96b48aa4f 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -18,7 +18,6 @@ package osd import ( "fmt" - "io/ioutil" "os" "os/signal" "path/filepath" @@ -29,7 +28,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" - cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/daemon/ceph/client" oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/util/sys" ) @@ -163,37 +162,6 @@ func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, er return rawDevice, nil } -// writeCephConfig writes the ceph config so ceph commands can be executed -func writeCephConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error { - - // create the ceph.conf with the default settings - cephConfig, err := cephclient.CreateDefaultCephConfig(context, clusterInfo) - if err != nil { - return errors.Wrap(err, "failed to create default ceph config") - } - - // write the latest config to the config dir - confFilePath, err := cephclient.GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig) - if err != nil { - return errors.Wrap(err, "failed to write connection config") - } - src, err := ioutil.ReadFile(filepath.Clean(confFilePath)) - if err != nil { - return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config") - } - err = ioutil.WriteFile(cephclient.DefaultConfigFilePath(), src, 0444) - if err != nil { - return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", cephclient.DefaultConfigFilePath()) - } - dst, err := ioutil.ReadFile(cephclient.DefaultConfigFilePath()) - if err == nil { - logger.Debugf("config file @ %s: %s", cephclient.DefaultConfigFilePath(), dst) - } else { - logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", cephclient.DefaultConfigFilePath(), err) - } - return nil -} - // Provision provisions an OSD func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error { if agent.pvcBacked { @@ -214,7 +182,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating} oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status) - if err := writeCephConfig(context, agent.clusterInfo); err != nil { + if err := client.WriteCephConfig(context, agent.clusterInfo); err != nil { return errors.Wrap(err, "failed to generate ceph config") } diff --git a/pkg/daemon/ceph/osd/remove.go b/pkg/daemon/ceph/osd/remove.go index 7d436afe8..26c6a940b 100644 --- a/pkg/daemon/ceph/osd/remove.go +++ b/pkg/daemon/ceph/osd/remove.go @@ -35,7 +35,7 @@ import ( func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error { // Generate the ceph config for running ceph commands similar to the operator - if err := writeCephConfig(context, clusterInfo); err != nil { + if err := client.WriteCephConfig(context, clusterInfo); err != nil { return errors.Wrap(err, "failed to write the ceph config") } diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 4b6cee553..be1eea4ab 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -37,6 +37,7 @@ import ( cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" + v1 "k8s.io/api/apps/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -67,13 +68,11 @@ const ( // Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs. type Cluster struct { - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - Replicas int - rookVersion string - exitCode func(err error) (int, bool) - appliedHttpBind bool - spec cephv1.ClusterSpec + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + rookVersion string + exitCode func(err error) (int, bool) + spec cephv1.ClusterSpec } // New creates an instance of the mgr @@ -83,16 +82,16 @@ func New(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, spec ce clusterInfo: clusterInfo, spec: spec, rookVersion: rookVersion, - Replicas: spec.Mgr.Count, exitCode: exec.ExitStatus, } } +var waitForDeploymentToStart = k8sutil.WaitForDeploymentToStart var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // for backward compatibility, default to 1 mgr func (c *Cluster) getReplicas() int { - replicas := c.Replicas + replicas := c.spec.Mgr.Count if replicas == 0 { replicas = 1 } @@ -118,6 +117,8 @@ func (c *Cluster) Start() error { logger.Infof("start running mgr") daemonIDs := c.getDaemonIDs() + var deploymentsToWaitFor []*v1.Deployment + for _, daemonID := range daemonIDs { // Check whether we need to cancel the orchestration if err := controller.CheckForCancelledOrchestration(c.context); err != nil { @@ -150,7 +151,7 @@ func (c *Cluster) Start() error { return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name) } - _, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{}) + newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{}) if err != nil { if !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName) @@ -160,11 +161,26 @@ func (c *Cluster) Start() error { if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, false); err != nil { logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err) } + } else { + // wait for the new deployment + deploymentsToWaitFor = append(deploymentsToWaitFor, newDeployment) } } - if err := c.reconcileServices(); err != nil { - return errors.Wrap(err, "failed to enable mgr services") + // If the mgr is newly created, wait for it to start before continuing with the service and + // module configuration + for _, d := range deploymentsToWaitFor { + if err := waitForDeploymentToStart(c.context, d); err != nil { + return errors.Wrapf(err, "failed to wait for mgr %q to start", d.Name) + } + } + + if len(daemonIDs) == 1 { + // Only reconcile the mgr services in the operator if there is a single mgr. + // If there are multiple mgrs they will be managed by the mgr sidecar. + if err := c.reconcileService(daemonIDs[0]); err != nil { + return errors.Wrap(err, "failed to enable mgr services") + } } // configure the mgr modules @@ -188,17 +204,40 @@ func (c *Cluster) Start() error { return nil } -func (c *Cluster) reconcileServices() error { +// ReconcileMultipleServices reconciles the services if the active mgr is the one running +// in the sidecar +func (c *Cluster) ReconcileMultipleServices(daemonNameToUpdate string) error { + // If the services are already set to this daemon, no need to attempt to update + svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), AppName, metav1.GetOptions{}) + if err != nil { + logger.Errorf("failed to check current mgr service, proceeding to update. %v", err) + } else { + currentDaemon := svc.Spec.Selector[controller.DaemonIDLabel] + if currentDaemon == daemonNameToUpdate { + logger.Infof("mgr services already set to daemon %q, no need to update", daemonNameToUpdate) + return nil + } + logger.Infof("mgr service currently set to %q, checking if need to update to %q", currentDaemon, daemonNameToUpdate) + } mgrMap, err := cephclient.CephMgrMap(c.context, c.clusterInfo) if err != nil { return errors.Wrap(err, "failed to get ceph status for the active mgr") } - activeDaemon := mgrMap.ActiveName - if activeDaemon == "" { - return errors.Errorf("active mgr not found") + if mgrMap.ActiveName == "" { + return errors.New("active mgr not found") } - logger.Infof("Active mgr is %q", activeDaemon) + if daemonNameToUpdate != mgrMap.ActiveName { + logger.Infof("no need for the mgr update since the active mgr is %q, rather than the local mgr %q", mgrMap.ActiveName, daemonNameToUpdate) + return nil + } + + return c.reconcileService(mgrMap.ActiveName) +} + +// reconcile the services, if the active mgr is not detected, use the default mgr +func (c *Cluster) reconcileService(activeDaemon string) error { + logger.Infof("setting services to point to mgr %q", activeDaemon) if err := c.configureDashboardService(activeDaemon); err != nil { return errors.Wrap(err, "failed to configure dashboard svc") @@ -222,7 +261,6 @@ func (c *Cluster) reconcileServices() error { func (c *Cluster) configureModules(daemonIDs []string) { // Configure the modules asynchronously so we can complete all the configuration much sooner. - startModuleConfiguration("http bind settings", c.clearHTTPBindFix) startModuleConfiguration("prometheus", c.enablePrometheusModule) startModuleConfiguration("dashboard", c.configureDashboardModules) // "crash" is part of the "always_on_modules" list as of Octopus diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index 0386a7012..b65f4619a 100755 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -35,12 +35,13 @@ import ( "github.com/stretchr/testify/require" "github.com/tevino/abool" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestStartMgr(t *testing.T) { - ctx := context.TODO() var deploymentsUpdated *[]*apps.Deployment updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() @@ -49,11 +50,15 @@ func TestStartMgr(t *testing.T) { return "{\"key\":\"mysecurekey\"}", nil }, } + waitForDeploymentToStart = func(clusterdContext *clusterd.Context, deployment *v1.Deployment) error { + logger.Infof("simulated mgr deployment starting") + return nil + } clientset := testop.New(t, 3) configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - context := &clusterd.Context{ + ctx := &clusterd.Context{ Executor: executor, ConfigDir: configDir, Clientset: clientset, @@ -68,13 +73,13 @@ func TestStartMgr(t *testing.T) { PriorityClassNames: map[rookv1.KeyType]string{cephv1.KeyMgr: "my-priority-class"}, DataDirHostPath: "/var/lib/rook/", } - c := New(context, clusterInfo, clusterSpec, "myversion") + c := New(ctx, clusterInfo, clusterSpec, "myversion") defer os.RemoveAll(c.spec.DataDirHostPath) // start a basic service err := c.Start() assert.Nil(t, err) - validateStart(ctx, t, c) + validateStart(t, c) assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) @@ -82,40 +87,64 @@ func TestStartMgr(t *testing.T) { c.spec.Dashboard.Port = 12345 err = c.Start() assert.Nil(t, err) - validateStart(ctx, t, c) + validateStart(t, c) assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) - // starting again with more replicas - c.Replicas = 3 + // starting with more replicas + c.spec.Mgr.Count = 2 c.spec.Dashboard.Enabled = false + // delete the previous mgr since the mocked test won't update the existing one + err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{}) + assert.Nil(t, err) err = c.Start() assert.Nil(t, err) - validateStart(ctx, t, c) - assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) - testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) + // trigger the sidecar reconcile since the operator didn't do it so we can perform the full validation + err = c.reconcileService("a") + assert.Nil(t, err) + validateStart(t, c) + + // the dashboard service is only deleted by the operator reconcile if the replicas are 1, + // otherwise the sidecar has the responsibility + c.spec.Mgr.Count = 1 + c.spec.Dashboard.Enabled = false + // clean the previous deployments + err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{}) + assert.Nil(t, err) + err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-b", metav1.DeleteOptions{}) + assert.Nil(t, err) + err = c.Start() + assert.Nil(t, err) + validateStart(t, c) } -func validateStart(ctx context.Context, t *testing.T, c *Cluster) { +func validateStart(t *testing.T, c *Cluster) { mgrNames := []string{"a", "b"} - for i := 0; i < c.Replicas; i++ { - if i == 2 { - logger.Warning("cannot have more than 2 mgrs") - break - } + for i := 0; i < c.spec.Mgr.Count; i++ { logger.Infof("Looking for cephmgr replica %d", i) daemonName := mgrNames[i] - d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(ctx, fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{}) + d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(context.TODO(), fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, map[string]string{"my": "annotation"}, d.Spec.Template.Annotations) assert.Contains(t, d.Spec.Template.Labels, "my-label-key") assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName) + if c.spec.Mgr.Count == 1 { + assert.Equal(t, 1, len(d.Spec.Template.Spec.Containers)) + } else { + // The sidecar container is only there when multiple mgrs are enabled + assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers)) + assert.Equal(t, "watch-active", d.Spec.Template.Spec.Containers[1].Name) + } } - _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr", metav1.GetOptions{}) + validateServices(t, c) +} + +func validateServices(t *testing.T, c *Cluster) { + _, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{}) assert.Nil(t, err) - ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{}) + ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr-dashboard", metav1.GetOptions{}) if c.spec.Dashboard.Enabled { assert.NoError(t, err) if c.spec.Dashboard.Port == 0 { @@ -130,6 +159,65 @@ func validateStart(ctx context.Context, t *testing.T, c *Cluster) { } } +func TestMgrSidecarReconcile(t *testing.T) { + activeMgr := "a" + spec := cephv1.ClusterSpec{ + Mgr: cephv1.MgrSpec{Count: 1}, + Dashboard: cephv1.DashboardSpec{ + Enabled: true, + Port: 7000, + }, + } + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutputFile: func(command, outFile string, args ...string) (string, error) { + return fmt.Sprintf(`{"active_name":"%s"}`, activeMgr), nil + }, + } + clientset := testop.New(t, 3) + configDir, _ := ioutil.TempDir("", "") + defer os.RemoveAll(configDir) + ctx := &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + Clientset: clientset, + } + clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") + c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo} + + // Update services according to the active mgr + err := c.ReconcileMultipleServices(activeMgr) + assert.NoError(t, err) + validateServices(t, c) + validateServiceMatches(t, c, "a") + + // nothing is created or updated when the requested mgr is not the active mgr + err = c.ReconcileMultipleServices("b") + assert.NoError(t, err) + _, err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{}) + assert.True(t, kerrors.IsNotFound(err)) + + // nothing is updated when the requested mgr is not the active mgr + activeMgr = "b" + err = c.ReconcileMultipleServices("b") + assert.NoError(t, err) + validateServices(t, c) + validateServiceMatches(t, c, "b") +} + +func validateServiceMatches(t *testing.T, c *Cluster, expectedActive string) { + // The service labels should match the active mgr + svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{}) + assert.NoError(t, err) + matchDaemon, ok := svc.Spec.Selector["ceph_daemon_id"] + assert.True(t, ok) + assert.Equal(t, expectedActive, matchDaemon) + + // clean up the service for the next test + err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr", metav1.DeleteOptions{}) + assert.NoError(t, err) +} + func TestConfigureModules(t *testing.T) { modulesEnabled := 0 modulesDisabled := 0 @@ -205,8 +293,16 @@ func TestConfigureModules(t *testing.T) { } func TestMgrDaemons(t *testing.T) { - c := &Cluster{Replicas: 2} + spec := cephv1.ClusterSpec{ + Mgr: cephv1.MgrSpec{Count: 1}, + } + c := &Cluster{spec: spec} daemons := c.getDaemonIDs() + require.Equal(t, 1, len(daemons)) + assert.Equal(t, "a", daemons[0]) + + c.spec.Mgr.Count = 2 + daemons = c.getDaemonIDs() require.Equal(t, 2, len(daemons)) assert.Equal(t, "a", daemons[0]) assert.Equal(t, "b", daemons[1]) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 598df59a5..4f221f3c2 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -21,11 +21,10 @@ import ( "os" "strconv" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" - "github.com/rook/rook/pkg/daemon/ceph/client" + opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" @@ -63,6 +62,9 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) PriorityClassName: cephv1.GetMgrPriorityClassName(c.spec.PriorityClassNames), }, } + if c.spec.Mgr.Count > 1 { + podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig)) + } // If the log collector is enabled we add the side-car container if c.spec.LogCollector.Enabled { @@ -113,42 +115,6 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) return d, nil } -// if we do not need the http bind fix, then we need to be careful. if we are -// upgrading from a cluster that had the fix applied, then the fix is no longer -// needed, and furthermore, needs to be removed so that there is not a lingering -// ceph configuration option that contains an old ip. by clearing the option, -// we let ceph bind to its default ANYADDR address. However, since we don't -// know which version of Ceph we are may be upgrading _from_ we need to (a) -// always do this and (b) make sure that all forms of the configuration option -// are removed (see the init container factory method). Once the minimum -// supported version of Rook contains the fix, all of this can be removed. -func (c *Cluster) clearHTTPBindFix() error { - // We only need to apply these changes once. No harm in once each time the operator restarts. - if c.appliedHttpBind { - return nil - } - for _, daemonID := range c.getDaemonIDs() { - for _, module := range []string{"dashboard", "prometheus"} { - // there are two forms of the configuration key that might exist which - // depends not on the current version, but on the version that may be - // the version being upgraded from. - if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID, - fmt.Sprintf("mgr/%s/server_addr", module), "", false); err != nil { - return errors.Wrap(err, "failed to set config for an mgr daemon using v2 format") - } - - // this is for the format used in v1.0 - // https://github.com/rook/rook/commit/11d318fb2f77a6ac9a8f2b9be42c826d3b4a93c3 - if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID, - fmt.Sprintf("mgr/%s/%s/server_addr", module, daemonID), "", false); err != nil { - return errors.Wrap(err, "failed to set config for an mgr daemon using v1 format") - } - } - } - c.appliedHttpBind = true - return nil -} - func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container { return controller.ChownCephDataDirsInitContainer( *mgrConfig.DataPathMap, @@ -217,6 +183,39 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { return container } +func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container { + envVars := []v1.EnvVar{ + {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)}, + {Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)}, + k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), + opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace), + opmon.EndpointEnvVar(), + opmon.SecretEnvVar(), + opmon.CephUsernameEnvVar(), + opmon.CephSecretEnvVar(), + k8sutil.ConfigOverrideEnvVar(), + {Name: "ROOK_FSID", ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{Name: "rook-ceph-mon"}, + Key: "fsid", + }, + }}, + {Name: "ROOK_DASHBOARD_ENABLED", Value: strconv.FormatBool(c.spec.Dashboard.Enabled)}, + {Name: "ROOK_MONITORING_ENABLED", Value: strconv.FormatBool(c.spec.Monitoring.Enabled)}, + {Name: "ROOK_UPDATE_INTERVAL", Value: "15s"}, + {Name: "ROOK_DAEMON_NAME", Value: mgrConfig.DaemonID}, + } + + return v1.Container{ + Args: []string{"ceph", "mgr", "watch-active"}, + Name: "watch-active", + Image: c.rookVersion, + Env: envVars, + //Resources: **TODO**, + } +} + func getDefaultMgrLivenessProbe() *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index d003ef025..092c497e8 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -362,6 +362,6 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client. return nil } - _, err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback) + err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback) return err } diff --git a/pkg/operator/ceph/cluster/osd/envs.go b/pkg/operator/ceph/cluster/osd/envs.go index 04c0bea82..db7ef3ad5 100644 --- a/pkg/operator/ceph/cluster/osd/envs.go +++ b/pkg/operator/ceph/cluster/osd/envs.go @@ -57,6 +57,7 @@ func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1. envVars := []v1.EnvVar{ nodeNameEnvVar(osdProps.crushHostname), {Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)}, + {Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace), diff --git a/pkg/operator/ceph/cluster/osd/osd_pvc_test.go b/pkg/operator/ceph/cluster/osd/osd_pvc_test.go index 18a1eef18..3e920b6b9 100644 --- a/pkg/operator/ceph/cluster/osd/osd_pvc_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_pvc_test.go @@ -224,6 +224,7 @@ func testOSDsOnPVC(t *testing.T) { Namespace: namespace, CephVersion: cephver.Nautilus, } + clusterInfo.SetName("mycluster") executor := osdPVCTestExecutor(t, clientset, namespace) context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: executor, RequestCancelOrchestration: abool.New()} diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index ce4da39b4..78b85cfa7 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -295,6 +295,7 @@ func TestAddNodeFailure(t *testing.T) { Namespace: "ns-add-remove", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("testcluster") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -381,6 +382,7 @@ func TestGetPVCHostName(t *testing.T) { func TestGetOSDInfo(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") context := &clusterd.Context{} spec := cephv1.ClusterSpec{DataDirHostPath: "/rook"} c := New(context, clusterInfo, spec, "myversion") @@ -517,6 +519,7 @@ func TestDetectCrushLocation(t *testing.T) { func TestGetOSDInfoWithCustomRoot(t *testing.T) { clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") context := &clusterd.Context{} spec := cephv1.ClusterSpec{ DataDirHostPath: "/rook", diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index b2bb746bf..473c277e3 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -488,7 +488,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC v1.Container{ Args: []string{"ceph", "osd", "init"}, Name: controller.ConfigInitContainerName, - Image: k8sutil.MakeRookImage(c.rookVersion), + Image: c.rookVersion, VolumeMounts: configVolumeMounts, Env: configEnvVars, SecurityContext: securityContext, @@ -708,7 +708,7 @@ func (c *Cluster) getCopyBinariesContainer() (v1.Volume, *v1.Container) { "copy-binaries", "--copy-to-dir", rookBinariesMountPath}, Name: "copy-bins", - Image: k8sutil.MakeRookImage(c.rookVersion), + Image: c.rookVersion, VolumeMounts: []v1.VolumeMount{mount}, } } diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 40ce50153..60dcfe45f 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -88,6 +88,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { Namespace: "ns", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("test") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"}, @@ -430,6 +431,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { Namespace: "ns", CephVersion: cephver.Octopus, } + clusterInfo.SetName("test") c := New(context, clusterInfo, spec, "rook/rook:myversion") deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.NoError(t, err) @@ -464,6 +466,7 @@ func TestStorageSpecConfig(t *testing.T) { Namespace: "ns", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("testing") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ DataDirHostPath: context.ConfigDir, @@ -545,6 +548,7 @@ func TestHostNetwork(t *testing.T) { Namespace: "ns", CephVersion: cephver.Nautilus, } + clusterInfo.SetName("test") context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} spec := cephv1.ClusterSpec{ Storage: storageSpec, @@ -583,6 +587,7 @@ func TestOsdPrepareResources(t *testing.T) { context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}} clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"} + clusterInfo.SetName("test") spec := cephv1.ClusterSpec{ Resources: map[string]v1.ResourceRequirements{"prepareosd": { Limits: v1.ResourceList{ diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index 3338d71c2..c3e1fa458 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -61,11 +61,11 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c // 2. verify that we can continue the update procedure // Basically, we go one resource by one and check if we can stop and then if the resource has been successfully updated // we check if we can go ahead and move to the next one. -func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) (*v1.Deployment, error) { +func UpdateDeploymentAndWait(clusterContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) error { ctx := context.TODO() - currentDeployment, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{}) + currentDeployment, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err) + return fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err) } // Check whether the current deployment and newly generated one are identical @@ -78,67 +78,66 @@ func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployme patchChanged = true } - // If deployments are different, let's update! - if patchChanged { - logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name) - - // Let's verify the deployment can be stopped - err = verifyCallback("stop") - if err != nil { - return nil, fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err) - } - - // Set hash annotation to the newly generated deployment - err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment) - if err != nil { - return nil, fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err) - } - - if _, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil { - return nil, fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err) - } - - // wait for the deployment to be restarted - sleepTime := 2 - attempts := 30 - if currentDeployment.Spec.ProgressDeadlineSeconds != nil { - // make the attempts double the progress deadline since the pod is both stopping and starting - attempts = 2 * (int(*currentDeployment.Spec.ProgressDeadlineSeconds) / sleepTime) - } - for i := 0; i < attempts; i++ { - // check for the status of the deployment - d, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to get deployment %q. %v", modifiedDeployment.Name, err) - } - if d.Status.ObservedGeneration != currentDeployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { - logger.Infof("finished waiting for updated deployment %q", d.Name) - - // Now we check if we can go to the next daemon - err = verifyCallback("continue") - if err != nil { - return nil, fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err) - } - - return d, nil - } - - // If ProgressDeadlineExceeded is reached let's fail earlier - // This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state - for _, condition := range d.Status.Conditions { - if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" { - return nil, fmt.Errorf("gave up waiting for deployment %q to update because %q", modifiedDeployment.Name, condition.Reason) - } - } - - logger.Debugf("deployment %q status=%+v", d.Name, d.Status) - time.Sleep(time.Duration(sleepTime) * time.Second) - } - return nil, fmt.Errorf("gave up waiting for deployment %q to update", modifiedDeployment.Name) + if !patchChanged { + logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name) + return nil } - logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name) - return nil, nil + // If deployments are different, let's update! + logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name) + + // Let's verify the deployment can be stopped + if err := verifyCallback("stop"); err != nil { + return fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err) + } + + // Set hash annotation to the newly generated deployment + if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment); err != nil { + return fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err) + } + + if _, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err) + } + + if err := WaitForDeploymentToStart(clusterContext, currentDeployment); err != nil { + return err + } + + // Now we check if we can go to the next daemon + if err := verifyCallback("continue"); err != nil { + return fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err) + } + return nil +} + +func WaitForDeploymentToStart(clusterdContext *clusterd.Context, deployment *v1.Deployment) error { + // wait for the deployment to be restarted up to 300s + sleepTime := 3 + attempts := 100 + for i := 0; i < attempts; i++ { + // check for the status of the deployment + d, err := clusterdContext.Clientset.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %q. %v", deployment.Name, err) + } + if d.Status.ObservedGeneration != deployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { + logger.Infof("finished waiting for updated deployment %q", d.Name) + return nil + } + + // If ProgressDeadlineExceeded is reached let's fail earlier + // This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state + for _, condition := range d.Status.Conditions { + if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" { + return fmt.Errorf("gave up waiting for deployment %q to update because %q", deployment.Name, condition.Reason) + } + } + + logger.Debugf("deployment %q status=%+v", d.Name, d.Status) + time.Sleep(time.Duration(sleepTime) * time.Second) + } + return fmt.Errorf("gave up waiting for deployment %q to update", deployment.Name) } // GetDeployments returns a list of deployment names labels matching a given selector diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 04e19bfa5..f8e34557b 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -51,7 +51,6 @@ const ( ConfigOverrideName = "rook-config-override" // ConfigOverrideVal config override value ConfigOverrideVal = "config" - defaultVersion = "rook/rook:latest" configMountDir = "/etc/rook/config" overrideFilename = "override.conf" ) @@ -203,15 +202,6 @@ func GetMatchingContainer(containers []v1.Container, name string) (v1.Container, return *result, nil } -// MakeRookImage formats the container name -func MakeRookImage(version string) string { - if version == "" { - return defaultVersion - } - - return version -} - // PodsRunningWithLabel returns the number of running pods with the given label func PodsRunningWithLabel(clientset kubernetes.Interface, namespace, label string) (int, error) { running, _, err := podStatusWithLabel(clientset, namespace, label) diff --git a/pkg/operator/k8sutil/pod_test.go b/pkg/operator/k8sutil/pod_test.go index 459d1e52b..9a875c742 100644 --- a/pkg/operator/k8sutil/pod_test.go +++ b/pkg/operator/k8sutil/pod_test.go @@ -26,11 +26,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestMakeRookImage(t *testing.T) { - assert.Equal(t, "rook/rook:v1", MakeRookImage("rook/rook:v1")) - assert.Equal(t, defaultVersion, MakeRookImage("")) -} - func TestGetContainerInPod(t *testing.T) { expectedName := "mycontainer" imageName := "myimage" diff --git a/pkg/operator/k8sutil/service.go b/pkg/operator/k8sutil/service.go index a9f8ab081..51b73f2b6 100644 --- a/pkg/operator/k8sutil/service.go +++ b/pkg/operator/k8sutil/service.go @@ -33,6 +33,7 @@ func CreateOrUpdateService( ctx := context.TODO() name := serviceDefinition.Name logger.Debugf("creating service %s", name) + s, err := clientset.CoreV1().Services(namespace).Create(ctx, serviceDefinition, metav1.CreateOptions{}) if err != nil { if !errors.IsAlreadyExists(err) { @@ -42,6 +43,8 @@ func CreateOrUpdateService( if err != nil { return nil, fmt.Errorf("failed to update service %s. %+v", name, err) } + } else { + logger.Debugf("created service %s", s.Name) } return s, err } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index aa5cdad04..a249d471c 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -1061,6 +1061,8 @@ rules: - get - list - watch + - create + - update - delete - apiGroups: - batch