rgw: change the way to livenessProbe and introduce readinessProbe

rgw doesn't respond `livenessProbe` if the number of connection reaches its
limit (by default, 1000). Then rgw is out of service but still live.
Hense the current `livenessProbe` logic is suitiable for `readinessProbe`.
`tcpSocket` is enough for `livenessProbe`.

Closes: #8407

Signed-off-by: Yuzuki Mimura <yuzuki725.m@gmail.com>
Signed-off-by: Satoru Takeuchi <satoru.takeuchi@gmail.com>
This commit is contained in:
Yuzuki Mimura authored and Satoru Takeuchi committed 2021-10-29 15:07:29 +00:00
1 parent e60e970c15
commit 536b59ef0f
14 files changed
+342 -38

No files matched your search

@@ -1055,7 +1055,7 @@ jobs:
tests/scripts/github-action-helper.sh deploy_first_rook_cluster
kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml
# wait for multisite store to be created
tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph
tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph
- name: prep second cluster pull realm config
run: |
@@ -1072,7 +1072,7 @@ jobs:
tests/scripts/github-action-helper.sh deploy_second_rook_cluster
kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml
# wait for realms to be pulled and zone-b-multisite-store to be created
tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary
tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary
- name: write an object to one cluster, read from the other
run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2
@@ -7560,6 +7560,102 @@ spec:
type: integer
type: object
type: object
readinessProbe:
description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
properties:
disabled:
description: Disabled determines whether probe is disable or not
type: boolean
probe:
description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.
properties:
exec:
description: One and only one of the following should be specified. Exec specifies the action to take.
properties:
command:
description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
items:
type: string
type: array
type: object
failureThreshold:
description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.
format: int32
type: integer
httpGet:
description: HTTPGet specifies the http request to perform.
properties:
host:
description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead.
type: string
httpHeaders:
description: Custom headers to set in the request. HTTP allows repeated headers.
items:
description: HTTPHeader describes a custom header to be used in HTTP probes
properties:
name:
description: The header field name
type: string
value:
description: The header field value
type: string
required:
- name
- value
type: object
type: array
path:
description: Path to access on the HTTP server.
type: string
port:
anyOf:
- type: integer
- type: string
description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
scheme:
description: Scheme to use for connecting to the host. Defaults to HTTP.
type: string
required:
- port
type: object
initialDelaySeconds:
description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
periodSeconds:
description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.
format: int32
type: integer
successThreshold:
description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
format: int32
type: integer
tcpSocket:
description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'
properties:
host:
description: 'Optional: Host name to connect to, defaults to the pod IP.'
type: string
port:
anyOf:
- type: integer
- type: string
description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
required:
- port
type: object
terminationGracePeriodSeconds:
description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.
format: int64
type: integer
timeoutSeconds:
description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
type: object
type: object
type: object
metadataPool:
description: The metadata pool settings
@@ -7556,6 +7556,102 @@ spec:
type: integer
type: object
type: object
readinessProbe:
description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
properties:
disabled:
description: Disabled determines whether probe is disable or not
type: boolean
probe:
description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.
properties:
exec:
description: One and only one of the following should be specified. Exec specifies the action to take.
properties:
command:
description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
items:
type: string
type: array
type: object
failureThreshold:
description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.
format: int32
type: integer
httpGet:
description: HTTPGet specifies the http request to perform.
properties:
host:
description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead.
type: string
httpHeaders:
description: Custom headers to set in the request. HTTP allows repeated headers.
items:
description: HTTPHeader describes a custom header to be used in HTTP probes
properties:
name:
description: The header field name
type: string
value:
description: The header field value
type: string
required:
- name
- value
type: object
type: array
path:
description: Path to access on the HTTP server.
type: string
port:
anyOf:
- type: integer
- type: string
description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
scheme:
description: Scheme to use for connecting to the host. Defaults to HTTP.
type: string
required:
- port
type: object
initialDelaySeconds:
description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
periodSeconds:
description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.
format: int32
type: integer
successThreshold:
description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
format: int32
type: integer
tcpSocket:
description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'
properties:
host:
description: 'Optional: Host name to connect to, defaults to the pod IP.'
type: string
port:
anyOf:
- type: integer
- type: string
description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
required:
- port
type: object
terminationGracePeriodSeconds:
description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.
format: int64
type: integer
timeoutSeconds:
description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
type: object
type: object
type: object
metadataPool:
description: The metadata pool settings
@@ -87,6 +87,8 @@ spec:
bucket:
disabled: false
interval: 60s
# Configure the pod liveness probe for the rgw daemon
# Configure the pod probes for the rgw daemon
livenessProbe:
disabled: false
readinessProbe:
disabled: false
@@ -101,9 +101,11 @@ spec:
bucket:
disabled: false
interval: 60s
# Configure the pod liveness probe for the rgw daemon
# Configure the pod probes for the rgw daemon
livenessProbe:
disabled: false
readinessProbe:
disabled: false
# security oriented settings
# security:
# To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file
+3 -1
View File
@@ -107,9 +107,11 @@ spec:
bucket:
disabled: false
interval: 60s
# Configure the pod liveness probe for the rgw daemon
# Configure the pod probes for the rgw daemon
livenessProbe:
disabled: false
readinessProbe:
disabled: false
# security oriented settings
# security:
# To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file
+2
View File
@@ -1285,6 +1285,8 @@ type BucketHealthCheckSpec struct {
Bucket HealthCheckSpec `json:"bucket,omitempty"`
// +optional
LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
// +optional
ReadinessProbe *ProbeSpec `json:"readinessProbe,omitempty"`
}
// HealthCheckSpec represents the health check of an object store bucket
@@ -89,6 +89,11 @@ func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) {
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.ReadinessProbe != nil {
in, out := &in.ReadinessProbe, &out.ReadinessProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
return
}
+2 -2
View File
@@ -43,7 +43,7 @@ func ConfigureLivenessProbe(daemon cephv1.KeyType, container v1.Container, healt
// If the spec value is not empty, let's apply it along with default when some fields are not specified
if probe != nil {
// Set the liveness probe on the container to overwrite the default probe created by Rook
container.LivenessProbe = GetLivenessProbeWithDefaults(probe, container.LivenessProbe)
container.LivenessProbe = GetProbeWithDefaults(probe, container.LivenessProbe)
}
}
}
@@ -51,7 +51,7 @@ func ConfigureLivenessProbe(daemon cephv1.KeyType, container v1.Container, healt
return container
}
func GetLivenessProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe {
func GetProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe {
newProbe := *desiredProbe
// Do not replace the handler with the previous one!
@@ -74,7 +74,7 @@ func configLivenessProbeHelper(t *testing.T, keyType cephv1.KeyType) {
}
}
func TestGetLivenessProbeWithDefaults(t *testing.T) {
func TestGetProbeWithDefaults(t *testing.T) {
t.Run("using default probe", func(t *testing.T) {
currentProb := &v1.Probe{
Handler: v1.Handler{
@@ -93,7 +93,7 @@ func TestGetLivenessProbeWithDefaults(t *testing.T) {
}
// in case of default probe
desiredProbe := &v1.Probe{}
desiredProbe = GetLivenessProbeWithDefaults(desiredProbe, currentProb)
desiredProbe = GetProbeWithDefaults(desiredProbe, currentProb)
assert.Equal(t, desiredProbe, currentProb)
})
@@ -133,7 +133,7 @@ func TestGetLivenessProbeWithDefaults(t *testing.T) {
SuccessThreshold: 4,
TimeoutSeconds: 5,
}
desiredProbe = GetLivenessProbeWithDefaults(desiredProbe, currentProb)
desiredProbe = GetProbeWithDefaults(desiredProbe, currentProb)
assert.Equal(t, desiredProbe.Exec.Command, []string{"env", "-i", "sh", "-c", "ceph --admin-daemon /run/ceph/ceph-mon.c.asok mon_status"})
assert.Equal(t, desiredProbe.InitialDelaySeconds, int32(1))
assert.Equal(t, desiredProbe.FailureThreshold, int32(2))
+41 -11
View File
@@ -37,7 +37,7 @@ import (
)
const (
livenessProbePath = "/swift/healthcheck"
readinessProbePath = "/swift/healthcheck"
// #nosec G101 since this is not leaking any hardcoded details
setupVaultTokenFile = `
set -e
@@ -270,13 +270,16 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
),
Env: controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image),
Resources: c.store.Spec.Gateway.Resources,
LivenessProbe: c.generateLiveProbe(),
LivenessProbe: c.defaultLivenessProbe(),
ReadinessProbe: c.defaultReadinessProbe(),
SecurityContext: controller.PodSecurityContext(),
WorkingDir: cephconfig.VarLogCephDir,
}
// If the liveness probe is enabled
configureLivenessProbe(&container, c.store.Spec.HealthCheck)
// If the readiness probe is enabled
configureReadinessProbe(&container, c.store.Spec.HealthCheck)
if c.store.Spec.IsTLSEnabled() {
// Add a volume mount for the ssl certificate
mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true}
@@ -320,7 +323,7 @@ func configureLivenessProbe(container *v1.Container, healthCheck cephv1.BucketHe
// If the spec value is empty, let's use a default
if probe != nil {
// Set the liveness probe on the container to overwrite the default probe created by Rook
container.LivenessProbe = cephconfig.GetLivenessProbeWithDefaults(probe, container.LivenessProbe)
container.LivenessProbe = cephconfig.GetProbeWithDefaults(probe, container.LivenessProbe)
}
} else {
container.LivenessProbe = nil
@@ -328,20 +331,47 @@ func configureLivenessProbe(container *v1.Container, healthCheck cephv1.BucketHe
}
}
func (c *clusterConfig) generateLiveProbe() *v1.Probe {
// configureReadinessProbe returns the desired readiness probe for a given daemon
func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) {
if ok := healthCheck.ReadinessProbe; ok != nil {
if !healthCheck.ReadinessProbe.Disabled {
probe := healthCheck.ReadinessProbe.Probe
// If the spec value is empty, let's use a default
if probe != nil {
// Set the readiness probe on the container to overwrite the default probe created by Rook
container.ReadinessProbe = cephconfig.GetProbeWithDefaults(probe, container.ReadinessProbe)
}
} else {
container.ReadinessProbe = nil
}
}
}
func (c *clusterConfig) defaultLivenessProbe() *v1.Probe {
return &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: livenessProbePath,
Port: c.generateLiveProbePort(),
Scheme: c.generateLiveProbeScheme(),
TCPSocket: &v1.TCPSocketAction{
Port: c.generateProbePort(),
},
},
InitialDelaySeconds: 10,
}
}
func (c *clusterConfig) generateLiveProbeScheme() v1.URIScheme {
func (c *clusterConfig) defaultReadinessProbe() *v1.Probe {
return &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: readinessProbePath,
Port: c.generateProbePort(),
Scheme: c.generateReadinessProbeScheme(),
},
},
InitialDelaySeconds: 10,
}
}
func (c *clusterConfig) generateReadinessProbeScheme() v1.URIScheme {
// Default to HTTP
uriScheme := v1.URISchemeHTTP
@@ -354,7 +384,7 @@ func (c *clusterConfig) generateLiveProbeScheme() v1.URIScheme {
return uriScheme
}
func (c *clusterConfig) generateLiveProbePort() intstr.IntOrString {
func (c *clusterConfig) generateProbePort() intstr.IntOrString {
// The port the liveness probe needs to probe
// Assume we run on SDN by default
port := intstr.FromInt(int(rgwPortInternalPort))
@@ -386,7 +416,7 @@ func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore)
svc.Spec.ClusterIP = v1.ClusterIPNone
}
destPort := c.generateLiveProbePort()
destPort := c.generateProbePort()
// When the cluster is external we must use the same one as the gateways are listening on
if cephObjectStore.Spec.IsExternal() {
+73 -12
View File
@@ -35,6 +35,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
func TestPodSpecs(t *testing.T) {
@@ -282,7 +283,7 @@ func TestValidateSpec(t *testing.T) {
assert.Nil(t, err)
}
func TestGenerateLiveProbe(t *testing.T) {
func TestDefaultLivenessProbe(t *testing.T) {
store := simpleStore()
c := &clusterConfig{
store: store,
@@ -293,33 +294,93 @@ func TestGenerateLiveProbe(t *testing.T) {
},
}
desiredProbe := &v1.Probe{
Handler: v1.Handler{
TCPSocket: &v1.TCPSocketAction{
Port: intstr.FromInt(8080),
},
},
InitialDelaySeconds: 10,
}
// No SSL - HostNetwork is disabled - using internal port
p := c.generateLiveProbe()
assert.Equal(t, int32(8080), p.Handler.HTTPGet.Port.IntVal)
assert.Equal(t, v1.URISchemeHTTP, p.Handler.HTTPGet.Scheme)
p := c.defaultLivenessProbe()
assert.Equal(t, desiredProbe, p)
// No SSL - HostNetwork is enabled
c.store.Spec.Gateway.Port = 123
c.store.Spec.Gateway.SecurePort = 0
c.clusterSpec.Network.HostNetwork = true
p = c.generateLiveProbe()
assert.Equal(t, int32(123), p.Handler.HTTPGet.Port.IntVal)
p = c.defaultLivenessProbe()
desiredProbe.Handler.TCPSocket.Port = intstr.FromInt(123)
assert.Equal(t, desiredProbe, p)
// SSL - HostNetwork is enabled
c.store.Spec.Gateway.Port = 0
c.store.Spec.Gateway.SecurePort = 321
c.store.Spec.Gateway.SSLCertificateRef = "foo"
p = c.generateLiveProbe()
assert.Equal(t, int32(321), p.Handler.HTTPGet.Port.IntVal)
p = c.defaultLivenessProbe()
desiredProbe.Handler.TCPSocket.Port = intstr.FromInt(321)
assert.Equal(t, desiredProbe, p)
// Both Non-SSL and SSL are enabled
// liveprobe just on Non-SSL
// livenessProbe just on Non-SSL
c.store.Spec.Gateway.Port = 123
c.store.Spec.Gateway.SecurePort = 321
p = c.defaultLivenessProbe()
desiredProbe.Handler.TCPSocket.Port = intstr.FromInt(123)
assert.Equal(t, desiredProbe, p)
}
func TestDefaultReadinessProbe(t *testing.T) {
store := simpleStore()
c := &clusterConfig{
store: store,
clusterSpec: &cephv1.ClusterSpec{
Network: cephv1.NetworkSpec{
HostNetwork: false,
},
},
}
desiredProbe := &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: readinessProbePath,
Port: intstr.FromInt(8080),
Scheme: v1.URISchemeHTTP,
},
},
InitialDelaySeconds: 10,
}
// No SSL - HostNetwork is disabled - using internal port
p := c.defaultReadinessProbe()
assert.Equal(t, desiredProbe, p)
// No SSL - HostNetwork is enabled
c.store.Spec.Gateway.Port = 123
c.store.Spec.Gateway.SecurePort = 0
c.clusterSpec.Network.HostNetwork = true
p = c.defaultReadinessProbe()
desiredProbe.Handler.HTTPGet.Port = intstr.FromInt(123)
assert.Equal(t, desiredProbe, p)
// SSL - HostNetwork is enabled
c.store.Spec.Gateway.Port = 0
c.store.Spec.Gateway.SecurePort = 321
c.store.Spec.Gateway.SSLCertificateRef = "foo"
p = c.generateLiveProbe()
assert.Equal(t, v1.URISchemeHTTP, p.Handler.HTTPGet.Scheme)
assert.Equal(t, int32(123), p.Handler.HTTPGet.Port.IntVal)
p = c.defaultReadinessProbe()
desiredProbe.Handler.HTTPGet.Port = intstr.FromInt(321)
desiredProbe.Handler.HTTPGet.Scheme = v1.URISchemeHTTPS
assert.Equal(t, desiredProbe, p)
// Both Non-SSL and SSL are enabled
// readinessProbe just on Non-SSL
c.store.Spec.Gateway.Port = 123
c.store.Spec.Gateway.SecurePort = 321
p = c.defaultReadinessProbe()
desiredProbe.Handler.HTTPGet.Port = intstr.FromInt(123)
desiredProbe.Handler.HTTPGet.Scheme = v1.URISchemeHTTP
assert.Equal(t, desiredProbe, p)
}
func TestCheckRGWKMS(t *testing.T) {
@@ -104,6 +104,7 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils
}
assert.True(t, k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"))
logger.Info("RGW pods are running")
assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=rook-ceph-rgw", namespace))
logger.Infof("Object store %q created successfully", storeName)
})
+12 -5
View File
@@ -286,16 +286,23 @@ function deploy_second_rook_cluster() {
deploy_manifest_with_local_build toolbox.yaml toolbox.yaml
}
function wait_for_rgw_pods() {
function wait_for_rgw() {
for _ in {1..120}; do
if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --field-selector=status.phase=Running|wc -l)" -gt 1 ] ; then
echo "rgw pods found"
if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --no-headers --field-selector=status.phase=Running|wc -l)" -ge 1 ] ; then
echo "rgw pod is found"
break
fi
echo "waiting for rgw pods"
sleep 5;
sleep 5
done
for _ in {1..120}; do
if [ "$(kubectl -n "$1" get deployment -l app=rook-ceph-rgw -o yaml | yq read - 'items[0].status.readyReplicas')" -ge 1 ] ; then
echo "rgw is ready"
break
fi
echo "waiting for rgw becomes ready"
sleep 5
done
}
function verify_operator_log_message() {