From c8de7009de053feeee100fc0d5408f140ea9cb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 5 Jun 2020 15:49:50 +0200 Subject: [PATCH] ceph: increase liveness probe start delay on OSD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the cluster is loaded and we restart an OSD, it will need some time to respond to socket calls, basically more to be ready. Increasing the initialDelaySeconds of the liveness probe fixes that issue. For OSD, it waits for 45 sec, where other daemons 10 sec. Closes: https://github.com/rook/rook/issues/5492 Signed-off-by: Sébastien Han --- pkg/operator/ceph/controller/spec.go | 20 +++++++++++++------- pkg/operator/ceph/controller/spec_test.go | 6 ++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 7cbe2439e..0dc660394 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -34,11 +34,13 @@ import ( const ( // ConfigInitContainerName is the name which is given to the config initialization container // in all Ceph pods. - ConfigInitContainerName = "config-init" - logVolumeName = "rook-ceph-log" - volumeMountSubPath = "data" - crashVolumeName = "rook-ceph-crash" - daemonSocketDir = "/run/ceph" + ConfigInitContainerName = "config-init" + logVolumeName = "rook-ceph-log" + volumeMountSubPath = "data" + crashVolumeName = "rook-ceph-crash" + daemonSocketDir = "/run/ceph" + initialDelaySecondsNonOSDDaemon int32 = 10 + initialDelaySecondsOSDDaemon int32 = 45 ) type daemonConfig struct { @@ -467,6 +469,10 @@ func StoredLogAndCrashVolumeMount(varLogCephDir, varLibCephCrashDir string) []v1 // GenerateLivenessProbeExecDaemon makes sure a daemon has a socket and that it can be called and returns 0 func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { confDaemon := getDaemonConfig(daemonType, daemonID) + initialDelaySeconds := initialDelaySecondsNonOSDDaemon + if daemonType == config.OsdType { + initialDelaySeconds = initialDelaySecondsOSDDaemon + } return &v1.Probe{ Handler: v1.Handler{ @@ -475,7 +481,7 @@ func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { // This avoids conflict with the CEPH_ARGS env // // Example: - // env -i sh -c ceph --admin-daemon /run/ceph/ceph-osd.0.asok status + // env -i sh -c "ceph --admin-daemon /run/ceph/ceph-osd.0.asok status" Command: []string{ "env", "-i", @@ -485,7 +491,7 @@ func GenerateLivenessProbeExecDaemon(daemonType, daemonID string) *v1.Probe { }, }, }, - InitialDelaySeconds: 10, + InitialDelaySeconds: initialDelaySeconds, } } diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 42e75dcc5..153777181 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -146,4 +146,10 @@ func TestGenerateLivenessProbeExecDaemon(t *testing.T) { } assert.Equal(t, expectedCommand, probe.Handler.Exec.Command) + // it's an OSD the delay must be 45 + assert.Equal(t, initialDelaySecondsOSDDaemon, probe.InitialDelaySeconds) + + // test with a mon so the delay should be 10 + probe = GenerateLivenessProbeExecDaemon(config.MonType, "a") + assert.Equal(t, initialDelaySecondsNonOSDDaemon, probe.InitialDelaySeconds) }