Merge pull request #11643 from rkachach/fix_issue_11640
mgr: use k8s readiness probe to implement mgr HA
This commit is contained in:
3 files changed
+81
-23
No files matched your search
@@ -190,6 +190,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
StartupProbe: controller.GenerateStartupProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
|
||||
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
|
||||
ReadinessProbe: controller.GenerateMgrReadinessProbeExecDaemon(config.MgrType, mgrConfig.DaemonID),
|
||||
WorkingDir: config.VarLogCephDir,
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,8 @@ const (
|
||||
CephUserID = 167
|
||||
livenessProbeTimeoutSeconds int32 = 5
|
||||
livenessProbeInitialDelaySeconds int32 = 10
|
||||
readinessProbePeriodSeconds int32 = 5
|
||||
readinessProbeInitialDelaySeconds int32 = 5
|
||||
startupProbeFailuresDaemonDefault int32 = 6 // multiply by 10 = effective startup timeout
|
||||
// The OSD requires a long timeout in case the OSD is taking extra time to
|
||||
// scrub data during startup. We don't want the probe to disrupt the OSD update
|
||||
@@ -567,44 +569,81 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1
|
||||
}
|
||||
}
|
||||
|
||||
// Generate an exec ProbeHandler
|
||||
func GenProbeHandler(command []string) v1.ProbeHandler {
|
||||
return v1.ProbeHandler{
|
||||
Exec: &v1.ExecAction{
|
||||
Command: command,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateLivenessProbeExecDaemon generates a liveness probe that makes sure a daemon has a socket,
|
||||
// that it can be called, and that it returns 0
|
||||
func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
|
||||
// Run with env -i to clean env variables in the exec context
|
||||
// This avoids conflict with the CEPH_ARGS env
|
||||
//
|
||||
// Example:
|
||||
// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status"
|
||||
//
|
||||
// Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails.
|
||||
// Add a clear message after Ceph's to help.
|
||||
// ref: https://github.com/rook/rook/issues/9846
|
||||
confDaemon := getDaemonConfig(daemonType, daemonID)
|
||||
|
||||
return &v1.Probe{
|
||||
ProbeHandler: v1.ProbeHandler{
|
||||
Exec: &v1.ExecAction{
|
||||
// Run with env -i to clean env variables in the exec context
|
||||
// This avoids conflict with the CEPH_ARGS env
|
||||
//
|
||||
// Example:
|
||||
// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status"
|
||||
//
|
||||
// Ceph gives pretty un-diagnostic error message when `ceph status` or `ceph mon_status` command fails.
|
||||
// Add a clear message after Ceph's to help.
|
||||
// ref: https://github.com/rook/rook/issues/9846
|
||||
Command: []string{
|
||||
"env",
|
||||
"-i",
|
||||
"sh",
|
||||
"-c",
|
||||
fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)"
|
||||
command := []string{
|
||||
"env",
|
||||
"-i",
|
||||
"sh",
|
||||
"-c",
|
||||
fmt.Sprintf(`outp="$(ceph --admin-daemon %s %s 2>&1)"
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo "ceph daemon health check failed with the following output:"
|
||||
echo "$outp" | sed -e 's/^/> /g'
|
||||
exit $rc
|
||||
fi`,
|
||||
confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()),
|
||||
},
|
||||
},
|
||||
},
|
||||
confDaemon.buildSocketPath(), confDaemon.buildAdminSocketCommand()),
|
||||
}
|
||||
|
||||
return &v1.Probe{
|
||||
ProbeHandler: GenProbeHandler(command),
|
||||
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
|
||||
TimeoutSeconds: livenessProbeTimeoutSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateMgrReadinessProbeExecDaemon generates a readiness probe that makes sure
|
||||
// a mgr daemon is ready to process traffic
|
||||
func GenerateMgrReadinessProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
|
||||
// Run with env -i to clean env variables in the exec context
|
||||
// This avoids conflict with the CEPH_ARGS env
|
||||
//
|
||||
// Example:
|
||||
// env -i sh -c "ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status"
|
||||
//
|
||||
confDaemon := getDaemonConfig(daemonType, daemonID)
|
||||
command := []string{
|
||||
"env",
|
||||
"-i",
|
||||
"sh",
|
||||
"-c",
|
||||
fmt.Sprintf(`ceph --admin-daemon %s mgr_status &>/dev/null
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo "Ceph standby manager does not process requests."
|
||||
exit $rc
|
||||
fi`,
|
||||
confDaemon.buildSocketPath()),
|
||||
}
|
||||
|
||||
return &v1.Probe{
|
||||
ProbeHandler: GenProbeHandler(command),
|
||||
InitialDelaySeconds: readinessProbeInitialDelaySeconds,
|
||||
PeriodSeconds: readinessProbePeriodSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateStartupProbeExecDaemon generates a startup probe that makes sure a daemon has a socket,
|
||||
// that it can be called, and that it returns 0
|
||||
func GenerateStartupProbeExecDaemon(daemonType, daemonID string) *v1.Probe {
|
||||
|
||||
@@ -171,6 +171,24 @@ fi`}
|
||||
assert.Equal(t, livenessProbeTimeoutSeconds, probe.TimeoutSeconds)
|
||||
}
|
||||
|
||||
func TestGenerateReadinessProbeExecDaemon(t *testing.T) {
|
||||
daemonID := "a"
|
||||
probe := GenerateMgrReadinessProbeExecDaemon(config.MgrType, daemonID)
|
||||
expectedCommand := []string{"env",
|
||||
"-i",
|
||||
"sh",
|
||||
"-c",
|
||||
`ceph --admin-daemon /run/ceph/ceph-mgr.a.asok mgr_status &>/dev/null
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo "Ceph standby manager does not process requests."
|
||||
exit $rc
|
||||
fi`}
|
||||
|
||||
assert.Equal(t, expectedCommand, probe.ProbeHandler.Exec.Command)
|
||||
assert.Equal(t, readinessProbeInitialDelaySeconds, probe.InitialDelaySeconds)
|
||||
}
|
||||
|
||||
func TestDaemonFlags(t *testing.T) {
|
||||
testcases := []struct {
|
||||
label string
|
||||
|
||||
Reference in new issue
Block a user