diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 0536908bf..d2d29602f 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -190,6 +190,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { SecurityContext: controller.PodSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), + ReadinessProbe: controller.GenerateMgrReadinessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID), WorkingDir: config.VarLogCephDir, } diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index a972c466f..bb984b8be 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -57,6 +57,8 @@ const ( CephUserID = 167 livenessProbeTimeoutSeconds int32 = 5 livenessProbeInitialDelaySeconds int32 = 10 + readinessProbePeriodSeconds int32 = 5 + readinessProbeInitialDelaySeconds int32 = 5 startupProbeFailuresDaemonDefault int32 = 6 // multiply by 10 = effective startup timeout // The OSD requires a long timeout in case the OSD is taking extra time to // scrub data during startup. We don't want the probe to disrupt the OSD update @@ -563,44 +565,81 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1 } } +// Generate an exec ProbeHandler +func GenProbeHandler(command []string) v1.ProbeHandler { + return v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: command, + }, + } +} + // GenerateLivenessProbeExecDaemon generates a liveness probe that makes sure a daemon has a socket, // that it can be called, and that it returns 0 func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { + // Run with env -i to clean env variables in the exec context + // This avoids conflict with the CEPH_ARGS env + // + // Example: + // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status" + // + // Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails. + // Add a clear message after Ceph's to help. + // ref: https://github.com/rook/rook/issues/9846 confDaemon := getDaemonConfig(daemonType, daemonID) - - return &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - Exec: &v1.ExecAction{ - // Run with env -i to clean env variables in the exec context - // This avoids conflict with the CEPH_ARGS env - // - // Example: - // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status" - // - // Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails. - // Add a clear message after Ceph's to help. - // ref: https://github.com/rook/rook/issues/9846 - Command: []string{ - "env", - "-i", - "sh", - "-c", - fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)" + command := []string{ + "env", + "-i", + "sh", + "-c", + fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)" rc=$? if [ $rc -ne 0 ]; then echo "ceph daemon health check failed with the following output:" echo "$outp" | sed -e 's/^/> /g' exit $rc fi`, - confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()), - }, - }, - }, + confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()), + } + + return &v1.Probe{ + ProbeHandler: GenProbeHandler(command), InitialDelaySeconds: livenessProbeInitialDelaySeconds, TimeoutSeconds: livenessProbeTimeoutSeconds, } } +// GenerateMgrReadinessProbeExecDaemon generates a readiness probe that makes sure +// a mgr daemon is ready to process traffic +func GenerateMgrReadinessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { + // Run with env -i to clean env variables in the exec context + // This avoids conflict with the CEPH_ARGS env + // + // Example: + // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status" + // + confDaemon := getDaemonConfig(daemonType, daemonID) + command := []string{ + "env", + "-i", + "sh", + "-c", + fmt.Sprintf(`ceph --admin-daemon %s mgr_status &>/dev/null +rc=$? +if [ $rc -ne 0 ]; then + echo "Ceph standby manager does not process requests." + exit $rc +fi`, + confDaemon.buildSocketPath()), + } + + return &v1.Probe{ + ProbeHandler: GenProbeHandler(command), + InitialDelaySeconds: readinessProbeInitialDelaySeconds, + PeriodSeconds: readinessProbePeriodSeconds, + } +} + // GenerateStartupProbeExecDaemon generates a startup probe that makes sure a daemon has a socket, // that it can be called, and that it returns 0 func GenerateStartupProbeExecDaemon(daemonType, daemonID string) *v1.Probe { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 12498a04c..7c1d8c249 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -171,6 +171,24 @@ fi`} assert.Equal(t, livenessProbeTimeoutSeconds, probe.TimeoutSeconds) } +func TestGenerateReadinessProbeExecDaemon(t *testing.T) { + daemonID := "a" + probe := GenerateMgrReadinessProbeExecDaemon(config.MgrType, daemonID) + expectedCommand := []string{"env", + "-i", + "sh", + "-c", + `ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status &>/dev/null +rc=$? +if [ $rc -ne 0 ]; then + echo "Ceph standby manager does not process requests." + exit $rc +fi`} + + assert.Equal(t, expectedCommand, probe.ProbeHandler.Exec.Command) + assert.Equal(t, readinessProbeInitialDelaySeconds, probe.InitialDelaySeconds) +} + func TestDaemonFlags(t *testing.T) { testcases := []struct { label string