From fca761c950db617e25cd476eb8402af77a381f3d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 6 Apr 2021 12:07:51 -0600 Subject: [PATCH] ceph: set the location on the mon daemon The mon daemon in a stretch cluster now can have its location set as a CLI param instead of setting it with a separate command. This enables mon failover to set the location of a mon immediately when it is joining quorum instead of having a delayed command to set the location. Signed-off-by: Travis Nielsen --- PendingReleaseNotes.md | 4 ++- pkg/daemon/ceph/client/mon.go | 14 -------- pkg/daemon/ceph/client/mon_test.go | 22 ------------- pkg/operator/ceph/cluster/mon/health.go | 7 ---- pkg/operator/ceph/cluster/mon/mon.go | 43 ------------------------- pkg/operator/ceph/cluster/mon/spec.go | 9 ++++++ 6 files changed, 12 insertions(+), 87 deletions(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index b878635e6..d7e2bb33d 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -31,4 +31,6 @@ v1.7... - Checkout the [ceph docs](https://docs.ceph.com/en/latest/rados/operations/crush-map/#custom-crush-rules) for detailed information. - Add support cephfs mirroring peer configuration, refer to the [configuration](Documentation/ceph-filesystem-crd.md#mirroring) for more details -- Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server. +- Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server. +- Stretch clusters are considered stable + - Ceph v16.2.5 or greater is required for stretch clusters diff --git a/pkg/daemon/ceph/client/mon.go b/pkg/daemon/ceph/client/mon.go index c5dcd675a..dec00f07b 100644 --- a/pkg/daemon/ceph/client/mon.go +++ b/pkg/daemon/ceph/client/mon.go @@ -17,7 +17,6 @@ package client import ( "encoding/json" - "fmt" "strings" "syscall" @@ -133,19 +132,6 @@ func CreateDefaultStretchCrushRule(context *clusterd.Context, clusterInfo *Clust return nil } -// SetMonStretchZone sets the location of a mon in the stretch cluster -func SetMonStretchZone(context *clusterd.Context, clusterInfo *ClusterInfo, monName, failureDomain, zone string) error { - args := []string{"mon", "set_location", monName, fmt.Sprintf("%s=%s", failureDomain, zone)} - buf, err := NewCephCommand(context, clusterInfo, args).Run() - if err != nil { - return errors.Wrap(err, "failed to set mon stretch zone") - } - output := string(buf) - logger.Debug(output) - logger.Infof("successfully set mon %q stretch zone to %q", monName, zone) - return nil -} - // SetMonStretchTiebreaker sets the tiebreaker mon in the stretch cluster func SetMonStretchTiebreaker(context *clusterd.Context, clusterInfo *ClusterInfo, monName, bucketType string) error { logger.Infof("enabling stretch mode with mon arbiter %q with crush rule %q in failure domain %q", monName, defaultStretchCrushRuleName, bucketType) diff --git a/pkg/daemon/ceph/client/mon_test.go b/pkg/daemon/ceph/client/mon_test.go index 4177a96b8..a610a1299 100644 --- a/pkg/daemon/ceph/client/mon_test.go +++ b/pkg/daemon/ceph/client/mon_test.go @@ -82,28 +82,6 @@ func TestStretchElectionStrategy(t *testing.T) { assert.NoError(t, err) } -func TestStretchClusterSettings(t *testing.T) { - monName := "a" - failureDomain := "rack" - zone := "rack-x" - executor := &exectest.MockExecutor{} - executor.MockExecuteCommandWithOutputFile = func(command, outputFile string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - switch { - case args[0] == "mon" && args[1] == "set_location": - assert.Equal(t, monName, args[2]) - assert.Equal(t, fmt.Sprintf("%s=%s", failureDomain, zone), args[3]) - return "", nil - } - return "", errors.Errorf("unexpected ceph command %q", args) - } - context := &clusterd.Context{Executor: executor} - clusterInfo := AdminClusterInfo("mycluster") - - err := SetMonStretchZone(context, clusterInfo, monName, failureDomain, zone) - assert.NoError(t, err) -} - func TestStretchClusterMonTiebreaker(t *testing.T) { monName := "a" failureDomain := "rack" diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 205a72961..4deb8f916 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -426,14 +426,7 @@ func (c *Cluster) failoverMon(name string) error { // Assign to a zone if a stretch cluster if c.spec.IsStretchCluster() { - updateArbiter := false if name == c.arbiterMon { - updateArbiter = true - } - if err := c.assignStretchMonsToZones([]*monConfig{m}); err != nil { - return errors.Wrap(err, "failed to assign mons to zones") - } - if updateArbiter { // Update the arbiter mon for the stretch cluster if it changed if err := c.ConfigureArbiter(); err != nil { return errors.Wrap(err, "failed to configure stretch arbiter") diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 0f66f168d..c6c6566b7 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -309,10 +309,6 @@ func (c *Cluster) startMons(targetCount int) error { } func (c *Cluster) configureStretchCluster(mons []*monConfig) error { - if err := c.assignStretchMonsToZones(mons); err != nil { - return errors.Wrap(err, "failed to assign mons to zones") - } - // Enable the mon connectivity strategy if err := cephclient.EnableStretchElectionStrategy(c.context, c.ClusterInfo); err != nil { return errors.Wrap(err, "failed to enable stretch cluster") @@ -342,45 +338,6 @@ func (c *Cluster) isArbiterZone(zone string) bool { return c.getArbiterZone() == zone } -func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error { - arbiterZone := c.getArbiterZone() - - // Get the mon dump to see if the zones are already applied to the mons - monDump, err := cephclient.GetMonDump(c.context, c.ClusterInfo) - if err != nil { - return errors.Wrap(err, "failed to detect if stretch mons are already assigned to zones") - } - - // Set the location for each mon - domainName := c.stretchFailureDomainName() - for _, mon := range mons { - if mon.Zone == arbiterZone { - // remember the arbiter mon to be set later in the reconcile after the OSDs are configured - c.arbiterMon = mon.DaemonName - } - // Check if the zone is already set on the mon - alreadySet := false - for _, monInfo := range monDump.Mons { - if monInfo.Name == mon.DaemonName { - desiredLocation := fmt.Sprintf("{%s=%s}", domainName, mon.Zone) - if monInfo.CrushLocation == desiredLocation { - alreadySet = true - } - break - } - } - if alreadySet { - logger.Infof("mon %q stretch domain %q is already set to %q", mon.DaemonName, domainName, mon.Zone) - continue - } - logger.Infof("setting mon %q to stretch %s=%s", mon.DaemonName, domainName, mon.Zone) - if err := cephclient.SetMonStretchZone(c.context, c.ClusterInfo, mon.DaemonName, domainName, mon.Zone); err != nil { - return errors.Wrapf(err, "failed to set mon %q zone", mon.DaemonName) - } - } - return nil -} - func (c *Cluster) ConfigureArbiter() error { if c.arbiterMon == "" { return errors.New("arbiter not specified for the stretch cluster") diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 0586ad62d..5bdbbc91e 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -317,6 +317,15 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container WorkingDir: config.VarLogCephDir, } + if monConfig.Zone != "" { + desiredLocation := fmt.Sprintf("%s=%s", c.stretchFailureDomainName(), monConfig.Zone) + container.Args = append(container.Args, []string{"--set-crush-location", desiredLocation}...) + if monConfig.Zone == c.getArbiterZone() { + // remember the arbiter mon to be set later in the reconcile after the OSDs are configured + c.arbiterMon = monConfig.DaemonName + } + } + // If the liveness probe is enabled container = config.ConfigureLivenessProbe(cephv1.KeyMon, container, c.spec.HealthCheck)