Merge pull request #11643 from rkachach/fix_issue_11640

mgr: use k8s readiness probe to implement mgr HA
This commit is contained in:
Travis Nielsen authored and GitHub committed 2023-02-15 09:51:48 -07:00
commit b008c5753f
3 files changed
+81 -23

No files matched your search

+1
View File
@@ -190,6 +190,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
SecurityContext: controller.PodSecurityContext(),
StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
ReadinessProbe: controller.GenerateMgrReadinessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
WorkingDir: config.VarLogCephDir,
}
+62 -23
View File
@@ -57,6 +57,8 @@ const (
CephUserID = 167
livenessProbeTimeoutSeconds int32 = 5
livenessProbeInitialDelaySeconds int32 = 10
readinessProbePeriodSeconds int32 = 5
readinessProbeInitialDelaySeconds int32 = 5
startupProbeFailuresDaemonDefault int32 = 6 // multiply by 10 = effective startup timeout
// The OSD requires a long timeout in case the OSD is taking extra time to
// scrub data during startup. We don't want the probe to disrupt the OSD update
@@ -567,44 +569,81 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1
}
}
// Generate an exec ProbeHandler
func GenProbeHandler(command []string) v1.ProbeHandler {
return v1.ProbeHandler{
Exec: &v1.ExecAction{
Command: command,
},
}
}
// GenerateLivenessProbeExecDaemon generates a liveness probe that makes sure a daemon has a socket,
// that it can be called, and that it returns 0
func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
// Run with env -i to clean env variables in the exec context
// This avoids conflict with the CEPH_ARGS env
//
// Example:
// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status"
//
// Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails.
// Add a clear message after Ceph's to help.
// ref: https://github.com/rook/rook/issues/9846
confDaemon := getDaemonConfig(daemonType, daemonID)
return &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
// Run with env -i to clean env variables in the exec context
// This avoids conflict with the CEPH_ARGS env
//
// Example:
// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status"
//
// Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails.
// Add a clear message after Ceph's to help.
// ref: https://github.com/rook/rook/issues/9846
Command: []string{
"env",
"-i",
"sh",
"-c",
fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)"
command := []string{
"env",
"-i",
"sh",
"-c",
fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)"
rc=$?
if [ $rc -ne 0 ]; then
echo "ceph daemon health check failed with the following output:"
echo "$outp" | sed -e 's/^/> /g'
exit $rc
fi`,
confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()),
},
},
},
confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()),
}
return &v1.Probe{
ProbeHandler: GenProbeHandler(command),
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
TimeoutSeconds: livenessProbeTimeoutSeconds,
}
}
// GenerateMgrReadinessProbeExecDaemon generates a readiness probe that makes sure
// a mgr daemon is ready to process traffic
func GenerateMgrReadinessProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
// Run with env -i to clean env variables in the exec context
// This avoids conflict with the CEPH_ARGS env
//
// Example:
// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status"
//
confDaemon := getDaemonConfig(daemonType, daemonID)
command := []string{
"env",
"-i",
"sh",
"-c",
fmt.Sprintf(`ceph --admin-daemon %s mgr_status &>/dev/null
rc=$?
if [ $rc -ne 0 ]; then
echo "Ceph standby manager does not process requests."
exit $rc
fi`,
confDaemon.buildSocketPath()),
}
return &v1.Probe{
ProbeHandler: GenProbeHandler(command),
InitialDelaySeconds: readinessProbeInitialDelaySeconds,
PeriodSeconds: readinessProbePeriodSeconds,
}
}
// GenerateStartupProbeExecDaemon generates a startup probe that makes sure a daemon has a socket,
// that it can be called, and that it returns 0
func GenerateStartupProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
+18
View File
@@ -171,6 +171,24 @@ fi`}
assert.Equal(t, livenessProbeTimeoutSeconds, probe.TimeoutSeconds)
}
func TestGenerateReadinessProbeExecDaemon(t *testing.T) {
daemonID := "a"
probe := GenerateMgrReadinessProbeExecDaemon(config.MgrType, daemonID)
expectedCommand := []string{"env",
"-i",
"sh",
"-c",
`ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status &>/dev/null
rc=$?
if [ $rc -ne 0 ]; then
echo "Ceph standby manager does not process requests."
exit $rc
fi`}
assert.Equal(t, expectedCommand, probe.ProbeHandler.Exec.Command)
assert.Equal(t, readinessProbeInitialDelaySeconds, probe.InitialDelaySeconds)
}
func TestDaemonFlags(t *testing.T) {
testcases := []struct {
label string