From cba9a359a0b546aa84e852936d11d3a2c5eb2f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 13 Jun 2019 19:32:53 +0200 Subject: [PATCH] ceph: upgrade apply osd nautilus flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When OSDs are running on Nautilus we always disable old osd features and aplpy the onces for Nautilus as described in the upgrade doc. During an upgrade or the next time an orchestration will be called the command will be applied. The command is idempotent so we can run it each time. This can be backported for 1.0.3 Closes: https://github.com/rook/rook/issues/2960 Signed-off-by: Sébastien Han --- PendingReleaseNotes.md | 1 + pkg/daemon/ceph/client/upgrade.go | 12 ++++++++++ pkg/daemon/ceph/client/upgrade_test.go | 15 +++++++++++++ pkg/operator/ceph/cluster/osd/osd.go | 31 ++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c6abc3661..2a236ada0 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -12,6 +12,7 @@ an example usage - Rook can now be configured to read "region" and "zone" labels on Kubernetes nodes and use that information as part of the CRUSH location for the OSDs. - Rgw pods have liveness probe enabled - Rgw is now configured with the Beast backend as of the Nautilus release +- OSD: newly updated cluster from 0.9 to 1.0.3 and thus Ceph Nautilus will have their OSDs allowing new features for Nautilus ## Breaking Changes diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 550ba1af4..9208f9f70 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -27,6 +27,7 @@ import ( // CephDaemonsVersions is a structure that can be used to parsed the output of the 'ceph versions' command type CephDaemonsVersions struct { Mon map[string]int `json:"mon,omitempty"` + Osd map[string]int `json:"osd,omitempty"` Mgr map[string]int `json:"mgr,omitempty"` Mds map[string]int `json:"mds,omitempty"` Overall map[string]int `json:"overall,omitempty"` @@ -95,3 +96,14 @@ func EnableMessenger2(context *clusterd.Context) error { return nil } + +// EnableNautilusOSD disallows pre-Nautilus OSDs and enables all new Nautilus-only functionality +func EnableNautilusOSD(context *clusterd.Context) error { + _, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "osd", "require-osd-release", "nautilus") + if err != nil { + return fmt.Errorf("failed to disallow pre-nautilus osds and enable all new nautilus-only functionality: %+v", err) + } + logger.Infof("successfully disallowed pre-nautilus osds and enabled all new nautilus-only functionality") + + return nil +} diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index 17de5d973..af2ede739 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -63,3 +63,18 @@ func TestEnableMessenger2(t *testing.T) { err := EnableMessenger2(context) assert.Nil(t, err) } + +func TestEnableNautilusOSD(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + assert.Equal(t, "osd", args[0]) + assert.Equal(t, "require-osd-release", args[1]) + assert.Equal(t, "nautilus", args[2]) + assert.Equal(t, 3, len(args)) + return "", nil + } + context := &clusterd.Context{Executor: executor} + + err := EnableNautilusOSD(context) + assert.Nil(t, err) +} diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index e7410dc52..5ef63ce86 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -30,6 +30,7 @@ import ( cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" osdconfig "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/display" apps "k8s.io/api/apps/v1" @@ -187,6 +188,36 @@ func (c *Cluster) Start() error { len(config.errorMessages), c.Namespace, strings.Join(config.errorMessages, "\n")) } + // The following block is used to apply any command(s) required by an upgrade + // The block below handles the upgrade from Mimic to Nautilus. + if c.clusterInfo.CephVersion.IsAtLeastNautilus() { + versions, err := client.GetCephVersions(c.context) + if err != nil { + logger.Warningf("failed to get ceph daemons versions. this likely means there are no osds yet. %+v", err) + } else { + // If length is one, this clearly indicates that all the osds are running the same version + logger.Infof("len of version.Osd is %d", len(versions.Osd)) + // If this is the first time we are creating a cluster length will be 0 + // On an initial OSD boostrap, by the time we reach this code, the OSDs haven't registered yet + // Basically, this task is happening too quickly and OSD pods are not running yet. + // That's not an issue since it's an initial bootstrap and not an update. + if len(versions.Osd) == 1 { + for v := range versions.Osd { + logger.Infof("v is %s", v) + osdVersion, err := cephver.ExtractCephVersion(v) + if err != nil { + return fmt.Errorf("failed to extract ceph version. %+v", err) + } + logger.Infof("osdVersion is: %v", osdVersion) + // if the version of these OSDs is Nautilus then we run the command + if osdVersion.IsAtLeastNautilus() { + client.EnableNautilusOSD(c.context) + } + } + } + } + } + logger.Infof("completed running osds in namespace %s", c.Namespace) return nil }