mgr: use k8s readiness probe to implement mgr HA

The idea behind this change is to use the readiness probe to implement
the mgr HA mechanism. In the current ceph mgr implementation only the
active instance offers the command 'mgr_status' through the admin
socket. We use this command combined with a Readiness Exec Probe to
detect which manager is active. Kubernetes will automatically mark
it as ready and redirect any service traffic to the active instance.

Closes: https://github.com/rook/rook/issues/11640
Closes: https://github.com/rook/rook/issues/11638
Signed-off-by: Redouane Kachach <rkachach@redhat.com>

pkg/operator/ceph/cluster/mgr/spec.go

@@ -190,6 +190,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
 		SecurityContext: controller.PodSecurityContext(),
 		StartupProbe:    controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
 		LivenessProbe:   controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
+		ReadinessProbe:  controller.GenerateMgrReadinessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
 		WorkingDir:      config.VarLogCephDir,
 	}
 

pkg/operator/ceph/controller/spec.go

@@ -57,6 +57,8 @@ const (
 	CephUserID                              = 167
 	livenessProbeTimeoutSeconds       int32 = 5
 	livenessProbeInitialDelaySeconds  int32 = 10
+	readinessProbePeriodSeconds       int32 = 5
+	readinessProbeInitialDelaySeconds int32 = 5
 	startupProbeFailuresDaemonDefault int32 = 6 // multiply by 10 = effective startup timeout
 	// The OSD requires a long timeout in case the OSD is taking extra time to
 	// scrub data during startup. We don't want the probe to disrupt the OSD update
@@ -563,44 +565,81 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1
 	}
 }
 
+// Generate an exec ProbeHandler
+func GenProbeHandler(command []string) v1.ProbeHandler {
+	return v1.ProbeHandler{
+		Exec: &v1.ExecAction{
+			Command: command,
+		},
+	}
+}
+
 // GenerateLivenessProbeExecDaemon generates a liveness probe that makes sure a daemon has a socket,
 // that it can be called, and that it returns 0
 func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
+	// Run with env -i to clean env variables in the exec context
+	// This avoids conflict with the CEPH_ARGS env
+	//
+	// Example:
+	// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status"
+	//
+	// Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails.
+	// Add a clear message after Ceph's to help.
+	// ref: https://github.com/rook/rook/issues/9846
 	confDaemon := getDaemonConfig(daemonType, daemonID)
-
-	return &v1.Probe{
-		ProbeHandler: v1.ProbeHandler{
-			Exec: &v1.ExecAction{
-				// Run with env -i to clean env variables in the exec context
-				// This avoids conflict with the CEPH_ARGS env
-				//
-				// Example:
-				// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status"
-				//
-				// Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails.
-				// Add a clear message after Ceph's to help.
-				// ref: https://github.com/rook/rook/issues/9846
-				Command: []string{
-					"env",
-					"-i",
-					"sh",
-					"-c",
-					fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)"
+	command := []string{
+		"env",
+		"-i",
+		"sh",
+		"-c",
+		fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)"
 rc=$?
 if [ $rc -ne 0 ]; then
   echo "ceph daemon health check failed with the following output:"
   echo "$outp" | sed -e 's/^/> /g'
   exit $rc
 fi`,
-						confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()),
-				},
-			},
-		},
+			confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()),
+	}
+
+	return &v1.Probe{
+		ProbeHandler:        GenProbeHandler(command),
 		InitialDelaySeconds: livenessProbeInitialDelaySeconds,
 		TimeoutSeconds:      livenessProbeTimeoutSeconds,
 	}
 }
 
+// GenerateMgrReadinessProbeExecDaemon generates a readiness probe that makes sure
+// a mgr daemon is ready to process traffic
+func GenerateMgrReadinessProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
+	// Run with env -i to clean env variables in the exec context
+	// This avoids conflict with the CEPH_ARGS env
+	//
+	// Example:
+	// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status"
+	//
+	confDaemon := getDaemonConfig(daemonType, daemonID)
+	command := []string{
+		"env",
+		"-i",
+		"sh",
+		"-c",
+		fmt.Sprintf(`ceph --admin-daemon %s mgr_status &>/dev/null
+rc=$?
+if [ $rc -ne 0 ]; then
+  echo "Ceph standby manager does not process requests."
+  exit $rc
+fi`,
+			confDaemon.buildSocketPath()),
+	}
+
+	return &v1.Probe{
+		ProbeHandler:        GenProbeHandler(command),
+		InitialDelaySeconds: readinessProbeInitialDelaySeconds,
+		PeriodSeconds:       readinessProbePeriodSeconds,
+	}
+}
+
 // GenerateStartupProbeExecDaemon generates a startup probe that makes sure a daemon has a socket,
 // that it can be called, and that it returns 0
 func GenerateStartupProbeExecDaemon(daemonType, daemonID string) *v1.Probe {

pkg/operator/ceph/controller/spec_test.go

@@ -171,6 +171,24 @@ fi`}
 	assert.Equal(t, livenessProbeTimeoutSeconds, probe.TimeoutSeconds)
 }
 
+func TestGenerateReadinessProbeExecDaemon(t *testing.T) {
+	daemonID := "a"
+	probe := GenerateMgrReadinessProbeExecDaemon(config.MgrType, daemonID)
+	expectedCommand := []string{"env",
+		"-i",
+		"sh",
+		"-c",
+		`ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status &>/dev/null
+rc=$?
+if [ $rc -ne 0 ]; then
+  echo "Ceph standby manager does not process requests."
+  exit $rc
+fi`}
+
+	assert.Equal(t, expectedCommand, probe.ProbeHandler.Exec.Command)
+	assert.Equal(t, readinessProbeInitialDelaySeconds, probe.InitialDelaySeconds)
+}
+
 func TestDaemonFlags(t *testing.T) {
 	testcases := []struct {
 		label       string