ceph: set the location on the mon daemon

The mon daemon in a stretch cluster now can have its location set
as a CLI param instead of setting it with a separate command.
This enables mon failover to set the location of a mon immediately
when it is joining quorum instead of having a delayed command
to set the location.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2021-07-19 14:30:43 -06:00
1 parent ce5777e528
commit fca761c950
6 files changed
+12 -87

No files matched your search

+3 -1
View File
@@ -31,4 +31,6 @@ v1.7...
- Checkout the [ceph docs](https://docs.ceph.com/en/latest/rados/operations/crush-map/#custom-crush-rules)
for detailed information.
- Add support cephfs mirroring peer configuration, refer to the [configuration](Documentation/ceph-filesystem-crd.md#mirroring) for more details
- Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server.
- Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server.
- Stretch clusters are considered stable
- Ceph v16.2.5 or greater is required for stretch clusters
-14
View File
@@ -17,7 +17,6 @@ package client
import (
"encoding/json"
"fmt"
"strings"
"syscall"
@@ -133,19 +132,6 @@ func CreateDefaultStretchCrushRule(context *clusterd.Context, clusterInfo *Clust
return nil
}
// SetMonStretchZone sets the location of a mon in the stretch cluster
func SetMonStretchZone(context *clusterd.Context, clusterInfo *ClusterInfo, monName, failureDomain, zone string) error {
args := []string{"mon", "set_location", monName, fmt.Sprintf("%s=%s", failureDomain, zone)}
buf, err := NewCephCommand(context, clusterInfo, args).Run()
if err != nil {
return errors.Wrap(err, "failed to set mon stretch zone")
}
output := string(buf)
logger.Debug(output)
logger.Infof("successfully set mon %q stretch zone to %q", monName, zone)
return nil
}
// SetMonStretchTiebreaker sets the tiebreaker mon in the stretch cluster
func SetMonStretchTiebreaker(context *clusterd.Context, clusterInfo *ClusterInfo, monName, bucketType string) error {
logger.Infof("enabling stretch mode with mon arbiter %q with crush rule %q in failure domain %q", monName, defaultStretchCrushRuleName, bucketType)
-22
View File
@@ -82,28 +82,6 @@ func TestStretchElectionStrategy(t *testing.T) {
assert.NoError(t, err)
}
func TestStretchClusterSettings(t *testing.T) {
monName := "a"
failureDomain := "rack"
zone := "rack-x"
executor := &exectest.MockExecutor{}
executor.MockExecuteCommandWithOutputFile = func(command, outputFile string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
switch {
case args[0] == "mon" && args[1] == "set_location":
assert.Equal(t, monName, args[2])
assert.Equal(t, fmt.Sprintf("%s=%s", failureDomain, zone), args[3])
return "", nil
}
return "", errors.Errorf("unexpected ceph command %q", args)
}
context := &clusterd.Context{Executor: executor}
clusterInfo := AdminClusterInfo("mycluster")
err := SetMonStretchZone(context, clusterInfo, monName, failureDomain, zone)
assert.NoError(t, err)
}
func TestStretchClusterMonTiebreaker(t *testing.T) {
monName := "a"
failureDomain := "rack"
-7
View File
@@ -426,14 +426,7 @@ func (c *Cluster) failoverMon(name string) error {
// Assign to a zone if a stretch cluster
if c.spec.IsStretchCluster() {
updateArbiter := false
if name == c.arbiterMon {
updateArbiter = true
}
if err := c.assignStretchMonsToZones([]*monConfig{m}); err != nil {
return errors.Wrap(err, "failed to assign mons to zones")
}
if updateArbiter {
// Update the arbiter mon for the stretch cluster if it changed
if err := c.ConfigureArbiter(); err != nil {
return errors.Wrap(err, "failed to configure stretch arbiter")
-43
View File
@@ -309,10 +309,6 @@ func (c *Cluster) startMons(targetCount int) error {
}
func (c *Cluster) configureStretchCluster(mons []*monConfig) error {
if err := c.assignStretchMonsToZones(mons); err != nil {
return errors.Wrap(err, "failed to assign mons to zones")
}
// Enable the mon connectivity strategy
if err := cephclient.EnableStretchElectionStrategy(c.context, c.ClusterInfo); err != nil {
return errors.Wrap(err, "failed to enable stretch cluster")
@@ -342,45 +338,6 @@ func (c *Cluster) isArbiterZone(zone string) bool {
return c.getArbiterZone() == zone
}
func (c *Cluster) assignStretchMonsToZones(mons []*monConfig) error {
arbiterZone := c.getArbiterZone()
// Get the mon dump to see if the zones are already applied to the mons
monDump, err := cephclient.GetMonDump(c.context, c.ClusterInfo)
if err != nil {
return errors.Wrap(err, "failed to detect if stretch mons are already assigned to zones")
}
// Set the location for each mon
domainName := c.stretchFailureDomainName()
for _, mon := range mons {
if mon.Zone == arbiterZone {
// remember the arbiter mon to be set later in the reconcile after the OSDs are configured
c.arbiterMon = mon.DaemonName
}
// Check if the zone is already set on the mon
alreadySet := false
for _, monInfo := range monDump.Mons {
if monInfo.Name == mon.DaemonName {
desiredLocation := fmt.Sprintf("{%s=%s}", domainName, mon.Zone)
if monInfo.CrushLocation == desiredLocation {
alreadySet = true
}
break
}
}
if alreadySet {
logger.Infof("mon %q stretch domain %q is already set to %q", mon.DaemonName, domainName, mon.Zone)
continue
}
logger.Infof("setting mon %q to stretch %s=%s", mon.DaemonName, domainName, mon.Zone)
if err := cephclient.SetMonStretchZone(c.context, c.ClusterInfo, mon.DaemonName, domainName, mon.Zone); err != nil {
return errors.Wrapf(err, "failed to set mon %q zone", mon.DaemonName)
}
}
return nil
}
func (c *Cluster) ConfigureArbiter() error {
if c.arbiterMon == "" {
return errors.New("arbiter not specified for the stretch cluster")
+9
View File
@@ -317,6 +317,15 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
WorkingDir: config.VarLogCephDir,
}
if monConfig.Zone != "" {
desiredLocation := fmt.Sprintf("%s=%s", c.stretchFailureDomainName(), monConfig.Zone)
container.Args = append(container.Args, []string{"--set-crush-location", desiredLocation}...)
if monConfig.Zone == c.getArbiterZone() {
// remember the arbiter mon to be set later in the reconcile after the OSDs are configured
c.arbiterMon = monConfig.DaemonName
}
}
// If the liveness probe is enabled
container = config.ConfigureLivenessProbe(cephv1.KeyMon, container, c.spec.HealthCheck)