From dc76f81feaf3bc7a2a87f2cdbe006355050a14a9 Mon Sep 17 00:00:00 2001 From: Redouane Kachach Date: Thu, 9 Feb 2023 16:06:00 +0100 Subject: [PATCH] mgr: use k8s readiness probe to implement mgr HA The idea behind this change is to use the readiness probe to implement the mgr HA mechanism. In the current ceph mgr implementation only the active instance offers the command 'mgr_status' through the admin socket. We use this command combined with a Readiness Exec Probe to detect which manager is active. Kubernetes will automatically mark it as ready and redirect any service traffic to the active instance. Closes: https://github.com/rook/rook/issues/11640 Closes: https://github.com/rook/rook/issues/11638 Signed-off-by: Redouane Kachach --- pkg/operator/ceph/cluster/mgr/spec.go | 1 + pkg/operator/ceph/controller/spec.go | 85 +++++++++++++++++------ pkg/operator/ceph/controller/spec_test.go | 18 +++++ 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 0536908bf..d2d29602f 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -190,6 +190,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { SecurityContext: controller.PodSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), + ReadinessProbe: controller.GenerateMgrReadinessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), WorkingDir: config.VarLogCephDir, } diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index a972c466f..bb984b8be 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -57,6 +57,8 @@ const ( CephUserID = 167 livenessProbeTimeoutSeconds int32 = 5 livenessProbeInitialDelaySeconds int32 = 10 + readinessProbePeriodSeconds int32 = 5 + readinessProbeInitialDelaySeconds int32 = 5 startupProbeFailuresDaemonDefault int32 = 6 // multiply by 10 = effective startup timeout // The OSD requires a long timeout in case the OSD is taking extra time to // scrub data during startup. We don't want the probe to disrupt the OSD update @@ -563,44 +565,81 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1 } } +// Generate an exec ProbeHandler +func GenProbeHandler(command []string) v1.ProbeHandler { + return v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: command, + }, + } +} + // GenerateLivenessProbeExecDaemon generates a liveness probe that makes sure a daemon has a socket, // that it can be called, and that it returns 0 func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { + // Run with env -i to clean env variables in the exec context + // This avoids conflict with the CEPH_ARGS env + // + // Example: + // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status" + // + // Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails. + // Add a clear message after Ceph's to help. + // ref: https://github.com/rook/rook/issues/9846 confDaemon := getDaemonConfig(daemonType, daemonID) - - return &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - Exec: &v1.ExecAction{ - // Run with env -i to clean env variables in the exec context - // This avoids conflict with the CEPH_ARGS env - // - // Example: - // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status" - // - // Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails. - // Add a clear message after Ceph's to help. - // ref: https://github.com/rook/rook/issues/9846 - Command: []string{ - "env", - "-i", - "sh", - "-c", - fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)" + command := []string{ + "env", + "-i", + "sh", + "-c", + fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)" rc=$? if [ $rc -ne 0 ]; then echo "ceph daemon health check failed with the following output:" echo "$outp" | sed -e 's/^/> /g' exit $rc fi`, - confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()), - }, - }, - }, + confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()), + } + + return &v1.Probe{ + ProbeHandler: GenProbeHandler(command), InitialDelaySeconds: livenessProbeInitialDelaySeconds, TimeoutSeconds: livenessProbeTimeoutSeconds, } } +// GenerateMgrReadinessProbeExecDaemon generates a readiness probe that makes sure +// a mgr daemon is ready to process traffic +func GenerateMgrReadinessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { + // Run with env -i to clean env variables in the exec context + // This avoids conflict with the CEPH_ARGS env + // + // Example: + // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status" + // + confDaemon := getDaemonConfig(daemonType, daemonID) + command := []string{ + "env", + "-i", + "sh", + "-c", + fmt.Sprintf(`ceph --admin-daemon %s mgr_status &>/dev/null +rc=$? +if [ $rc -ne 0 ]; then + echo "Ceph standby manager does not process requests." + exit $rc +fi`, + confDaemon.buildSocketPath()), + } + + return &v1.Probe{ + ProbeHandler: GenProbeHandler(command), + InitialDelaySeconds: readinessProbeInitialDelaySeconds, + PeriodSeconds: readinessProbePeriodSeconds, + } +} + // GenerateStartupProbeExecDaemon generates a startup probe that makes sure a daemon has a socket, // that it can be called, and that it returns 0 func GenerateStartupProbeExecDaemon(daemonType, daemonID string) *v1.Probe { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 12498a04c..7c1d8c249 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -171,6 +171,24 @@ fi`} assert.Equal(t, livenessProbeTimeoutSeconds, probe.TimeoutSeconds) } +func TestGenerateReadinessProbeExecDaemon(t *testing.T) { + daemonID := "a" + probe := GenerateMgrReadinessProbeExecDaemon(config.MgrType, daemonID) + expectedCommand := []string{"env", + "-i", + "sh", + "-c", + `ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status &>/dev/null +rc=$? +if [ $rc -ne 0 ]; then + echo "Ceph standby manager does not process requests." + exit $rc +fi`} + + assert.Equal(t, expectedCommand, probe.ProbeHandler.Exec.Command) + assert.Equal(t, readinessProbeInitialDelaySeconds, probe.InitialDelaySeconds) +} + func TestDaemonFlags(t *testing.T) { testcases := []struct { label string