From 536b59ef0f7cd20f60634eceb23b6e0dbb8ca5e8 Mon Sep 17 00:00:00 2001 From: Yuzuki Mimura Date: Fri, 3 Sep 2021 14:14:51 +0900 Subject: [PATCH] rgw: change the way to livenessProbe and introduce readinessProbe rgw doesn't respond `livenessProbe` if the number of connection reaches its limit (by default, 1000). Then rgw is out of service but still live. Hense the current `livenessProbe` logic is suitiable for `readinessProbe`. `tcpSocket` is enough for `livenessProbe`. Closes: #8407 Signed-off-by: Yuzuki Mimura Signed-off-by: Satoru Takeuchi --- .github/workflows/canary-integration-test.yml | 4 +- .../charts/rook-ceph/templates/resources.yaml | 96 +++++++++++++++++++ cluster/examples/kubernetes/ceph/crds.yaml | 96 +++++++++++++++++++ .../examples/kubernetes/ceph/object-ec.yaml | 4 +- .../kubernetes/ceph/object-openshift.yaml | 4 +- cluster/examples/kubernetes/ceph/object.yaml | 4 +- pkg/apis/ceph.rook.io/v1/types.go | 2 + .../ceph.rook.io/v1/zz_generated.deepcopy.go | 5 + pkg/operator/ceph/config/livenessprobe.go | 4 +- .../ceph/config/livenessprobe_test.go | 6 +- pkg/operator/ceph/object/spec.go | 52 +++++++--- pkg/operator/ceph/object/spec_test.go | 85 +++++++++++++--- tests/integration/ceph_base_object_test.go | 1 + tests/scripts/github-action-helper.sh | 17 +++- 14 files changed, 342 insertions(+), 38 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 0c88de727..fcd7f3303 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1055,7 +1055,7 @@ jobs: tests/scripts/github-action-helper.sh deploy_first_rook_cluster kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-test.yaml # wait for multisite store to be created - tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph + tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph - name: prep second cluster pull realm config run: | @@ -1072,7 +1072,7 @@ jobs: tests/scripts/github-action-helper.sh deploy_second_rook_cluster kubectl create -f cluster/examples/kubernetes/ceph/object-multisite-pull-realm-test.yaml # wait for realms to be pulled and zone-b-multisite-store to be created - tests/scripts/github-action-helper.sh wait_for_rgw_pods rook-ceph-secondary + tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary - name: write an object to one cluster, read from the other run: tests/scripts/github-action-helper.sh write_object_to_cluster1_read_from_cluster2 diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 4867edb85..0ccc119cd 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -7560,6 +7560,102 @@ spec: type: integer type: object type: object + readinessProbe: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object type: object metadataPool: description: The metadata pool settings diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index e06e1162a..db06d5176 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -7556,6 +7556,102 @@ spec: type: integer type: object type: object + readinessProbe: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object type: object metadataPool: description: The metadata pool settings diff --git a/cluster/examples/kubernetes/ceph/object-ec.yaml b/cluster/examples/kubernetes/ceph/object-ec.yaml index 08347cf3e..8dabb8fde 100644 --- a/cluster/examples/kubernetes/ceph/object-ec.yaml +++ b/cluster/examples/kubernetes/ceph/object-ec.yaml @@ -87,6 +87,8 @@ spec: bucket: disabled: false interval: 60s - # Configure the pod liveness probe for the rgw daemon + # Configure the pod probes for the rgw daemon livenessProbe: disabled: false + readinessProbe: + disabled: false diff --git a/cluster/examples/kubernetes/ceph/object-openshift.yaml b/cluster/examples/kubernetes/ceph/object-openshift.yaml index 6fa870446..1ad50edc1 100644 --- a/cluster/examples/kubernetes/ceph/object-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/object-openshift.yaml @@ -101,9 +101,11 @@ spec: bucket: disabled: false interval: 60s - # Configure the pod liveness probe for the rgw daemon + # Configure the pod probes for the rgw daemon livenessProbe: disabled: false + readinessProbe: + disabled: false # security oriented settings # security: # To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file diff --git a/cluster/examples/kubernetes/ceph/object.yaml b/cluster/examples/kubernetes/ceph/object.yaml index 4fd04a387..e0c4d64d7 100644 --- a/cluster/examples/kubernetes/ceph/object.yaml +++ b/cluster/examples/kubernetes/ceph/object.yaml @@ -107,9 +107,11 @@ spec: bucket: disabled: false interval: 60s - # Configure the pod liveness probe for the rgw daemon + # Configure the pod probes for the rgw daemon livenessProbe: disabled: false + readinessProbe: + disabled: false # security oriented settings # security: # To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index a8fbdf2dc..9ebe6feeb 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1285,6 +1285,8 @@ type BucketHealthCheckSpec struct { Bucket HealthCheckSpec `json:"bucket,omitempty"` // +optional LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` + // +optional + ReadinessProbe *ProbeSpec `json:"readinessProbe,omitempty"` } // HealthCheckSpec represents the health check of an object store bucket diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 82e3ec605..f61eddbbb 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -89,6 +89,11 @@ func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) { *out = new(ProbeSpec) (*in).DeepCopyInto(*out) } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/operator/ceph/config/livenessprobe.go b/pkg/operator/ceph/config/livenessprobe.go index 5039797ff..558c7d919 100644 --- a/pkg/operator/ceph/config/livenessprobe.go +++ b/pkg/operator/ceph/config/livenessprobe.go @@ -43,7 +43,7 @@ func ConfigureLivenessProbe(daemon cephv1.KeyType, container v1.Container, healt // If the spec value is not empty, let's apply it along with default when some fields are not specified if probe != nil { // Set the liveness probe on the container to overwrite the default probe created by Rook - container.LivenessProbe = GetLivenessProbeWithDefaults(probe, container.LivenessProbe) + container.LivenessProbe = GetProbeWithDefaults(probe, container.LivenessProbe) } } } @@ -51,7 +51,7 @@ func ConfigureLivenessProbe(daemon cephv1.KeyType, container v1.Container, healt return container } -func GetLivenessProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe { +func GetProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe { newProbe := *desiredProbe // Do not replace the handler with the previous one! diff --git a/pkg/operator/ceph/config/livenessprobe_test.go b/pkg/operator/ceph/config/livenessprobe_test.go index adb4ccce2..90c558ea9 100644 --- a/pkg/operator/ceph/config/livenessprobe_test.go +++ b/pkg/operator/ceph/config/livenessprobe_test.go @@ -74,7 +74,7 @@ func configLivenessProbeHelper(t *testing.T, keyType cephv1.KeyType) { } } -func TestGetLivenessProbeWithDefaults(t *testing.T) { +func TestGetProbeWithDefaults(t *testing.T) { t.Run("using default probe", func(t *testing.T) { currentProb := &v1.Probe{ Handler: v1.Handler{ @@ -93,7 +93,7 @@ func TestGetLivenessProbeWithDefaults(t *testing.T) { } // in case of default probe desiredProbe := &v1.Probe{} - desiredProbe = GetLivenessProbeWithDefaults(desiredProbe, currentProb) + desiredProbe = GetProbeWithDefaults(desiredProbe, currentProb) assert.Equal(t, desiredProbe, currentProb) }) @@ -133,7 +133,7 @@ func TestGetLivenessProbeWithDefaults(t *testing.T) { SuccessThreshold: 4, TimeoutSeconds: 5, } - desiredProbe = GetLivenessProbeWithDefaults(desiredProbe, currentProb) + desiredProbe = GetProbeWithDefaults(desiredProbe, currentProb) assert.Equal(t, desiredProbe.Exec.Command, []string{"env", "-i", "sh", "-c", "ceph --admin-daemon /run/ceph/ceph-mon.c.asok mon_status"}) assert.Equal(t, desiredProbe.InitialDelaySeconds, int32(1)) assert.Equal(t, desiredProbe.FailureThreshold, int32(2)) diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index e8b3ad0ca..065a813fa 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -37,7 +37,7 @@ import ( ) const ( - livenessProbePath = "/swift/healthcheck" + readinessProbePath = "/swift/healthcheck" // #nosec G101 since this is not leaking any hardcoded details setupVaultTokenFile = ` set -e @@ -270,13 +270,16 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { ), Env: controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), Resources: c.store.Spec.Gateway.Resources, - LivenessProbe: c.generateLiveProbe(), + LivenessProbe: c.defaultLivenessProbe(), + ReadinessProbe: c.defaultReadinessProbe(), SecurityContext: controller.PodSecurityContext(), WorkingDir: cephconfig.VarLogCephDir, } // If the liveness probe is enabled configureLivenessProbe(&container, c.store.Spec.HealthCheck) + // If the readiness probe is enabled + configureReadinessProbe(&container, c.store.Spec.HealthCheck) if c.store.Spec.IsTLSEnabled() { // Add a volume mount for the ssl certificate mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true} @@ -320,7 +323,7 @@ func configureLivenessProbe(container *v1.Container, healthCheck cephv1.BucketHe // If the spec value is empty, let's use a default if probe != nil { // Set the liveness probe on the container to overwrite the default probe created by Rook - container.LivenessProbe = cephconfig.GetLivenessProbeWithDefaults(probe, container.LivenessProbe) + container.LivenessProbe = cephconfig.GetProbeWithDefaults(probe, container.LivenessProbe) } } else { container.LivenessProbe = nil @@ -328,20 +331,47 @@ func configureLivenessProbe(container *v1.Container, healthCheck cephv1.BucketHe } } -func (c *clusterConfig) generateLiveProbe() *v1.Probe { +// configureReadinessProbe returns the desired readiness probe for a given daemon +func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) { + if ok := healthCheck.ReadinessProbe; ok != nil { + if !healthCheck.ReadinessProbe.Disabled { + probe := healthCheck.ReadinessProbe.Probe + // If the spec value is empty, let's use a default + if probe != nil { + // Set the readiness probe on the container to overwrite the default probe created by Rook + container.ReadinessProbe = cephconfig.GetProbeWithDefaults(probe, container.ReadinessProbe) + } + } else { + container.ReadinessProbe = nil + } + } +} + +func (c *clusterConfig) defaultLivenessProbe() *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: livenessProbePath, - Port: c.generateLiveProbePort(), - Scheme: c.generateLiveProbeScheme(), + TCPSocket: &v1.TCPSocketAction{ + Port: c.generateProbePort(), }, }, InitialDelaySeconds: 10, } } -func (c *clusterConfig) generateLiveProbeScheme() v1.URIScheme { +func (c *clusterConfig) defaultReadinessProbe() *v1.Probe { + return &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: readinessProbePath, + Port: c.generateProbePort(), + Scheme: c.generateReadinessProbeScheme(), + }, + }, + InitialDelaySeconds: 10, + } +} + +func (c *clusterConfig) generateReadinessProbeScheme() v1.URIScheme { // Default to HTTP uriScheme := v1.URISchemeHTTP @@ -354,7 +384,7 @@ func (c *clusterConfig) generateLiveProbeScheme() v1.URIScheme { return uriScheme } -func (c *clusterConfig) generateLiveProbePort() intstr.IntOrString { +func (c *clusterConfig) generateProbePort() intstr.IntOrString { // The port the liveness probe needs to probe // Assume we run on SDN by default port := intstr.FromInt(int(rgwPortInternalPort)) @@ -386,7 +416,7 @@ func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) svc.Spec.ClusterIP = v1.ClusterIPNone } - destPort := c.generateLiveProbePort() + destPort := c.generateProbePort() // When the cluster is external we must use the same one as the gateways are listening on if cephObjectStore.Spec.IsExternal() { diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 70c7c9e78..8d0096e02 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -35,6 +35,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) func TestPodSpecs(t *testing.T) { @@ -282,7 +283,7 @@ func TestValidateSpec(t *testing.T) { assert.Nil(t, err) } -func TestGenerateLiveProbe(t *testing.T) { +func TestDefaultLivenessProbe(t *testing.T) { store := simpleStore() c := &clusterConfig{ store: store, @@ -293,33 +294,93 @@ func TestGenerateLiveProbe(t *testing.T) { }, } + desiredProbe := &v1.Probe{ + Handler: v1.Handler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 10, + } // No SSL - HostNetwork is disabled - using internal port - p := c.generateLiveProbe() - assert.Equal(t, int32(8080), p.Handler.HTTPGet.Port.IntVal) - assert.Equal(t, v1.URISchemeHTTP, p.Handler.HTTPGet.Scheme) + p := c.defaultLivenessProbe() + assert.Equal(t, desiredProbe, p) // No SSL - HostNetwork is enabled c.store.Spec.Gateway.Port = 123 c.store.Spec.Gateway.SecurePort = 0 c.clusterSpec.Network.HostNetwork = true - p = c.generateLiveProbe() - assert.Equal(t, int32(123), p.Handler.HTTPGet.Port.IntVal) + p = c.defaultLivenessProbe() + desiredProbe.Handler.TCPSocket.Port = intstr.FromInt(123) + assert.Equal(t, desiredProbe, p) // SSL - HostNetwork is enabled c.store.Spec.Gateway.Port = 0 c.store.Spec.Gateway.SecurePort = 321 c.store.Spec.Gateway.SSLCertificateRef = "foo" - p = c.generateLiveProbe() - assert.Equal(t, int32(321), p.Handler.HTTPGet.Port.IntVal) + p = c.defaultLivenessProbe() + desiredProbe.Handler.TCPSocket.Port = intstr.FromInt(321) + assert.Equal(t, desiredProbe, p) // Both Non-SSL and SSL are enabled - // liveprobe just on Non-SSL + // livenessProbe just on Non-SSL c.store.Spec.Gateway.Port = 123 c.store.Spec.Gateway.SecurePort = 321 + p = c.defaultLivenessProbe() + desiredProbe.Handler.TCPSocket.Port = intstr.FromInt(123) + assert.Equal(t, desiredProbe, p) +} + +func TestDefaultReadinessProbe(t *testing.T) { + store := simpleStore() + c := &clusterConfig{ + store: store, + clusterSpec: &cephv1.ClusterSpec{ + Network: cephv1.NetworkSpec{ + HostNetwork: false, + }, + }, + } + + desiredProbe := &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: readinessProbePath, + Port: intstr.FromInt(8080), + Scheme: v1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + } + // No SSL - HostNetwork is disabled - using internal port + p := c.defaultReadinessProbe() + assert.Equal(t, desiredProbe, p) + + // No SSL - HostNetwork is enabled + c.store.Spec.Gateway.Port = 123 + c.store.Spec.Gateway.SecurePort = 0 + c.clusterSpec.Network.HostNetwork = true + p = c.defaultReadinessProbe() + desiredProbe.Handler.HTTPGet.Port = intstr.FromInt(123) + assert.Equal(t, desiredProbe, p) + + // SSL - HostNetwork is enabled + c.store.Spec.Gateway.Port = 0 + c.store.Spec.Gateway.SecurePort = 321 c.store.Spec.Gateway.SSLCertificateRef = "foo" - p = c.generateLiveProbe() - assert.Equal(t, v1.URISchemeHTTP, p.Handler.HTTPGet.Scheme) - assert.Equal(t, int32(123), p.Handler.HTTPGet.Port.IntVal) + p = c.defaultReadinessProbe() + desiredProbe.Handler.HTTPGet.Port = intstr.FromInt(321) + desiredProbe.Handler.HTTPGet.Scheme = v1.URISchemeHTTPS + assert.Equal(t, desiredProbe, p) + + // Both Non-SSL and SSL are enabled + // readinessProbe just on Non-SSL + c.store.Spec.Gateway.Port = 123 + c.store.Spec.Gateway.SecurePort = 321 + p = c.defaultReadinessProbe() + desiredProbe.Handler.HTTPGet.Port = intstr.FromInt(123) + desiredProbe.Handler.HTTPGet.Scheme = v1.URISchemeHTTP + assert.Equal(t, desiredProbe, p) } func TestCheckRGWKMS(t *testing.T) { diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 4ae3a4687..9f1c8cc86 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -104,6 +104,7 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils } assert.True(t, k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running")) logger.Info("RGW pods are running") + assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=rook-ceph-rgw", namespace)) logger.Infof("Object store %q created successfully", storeName) }) diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 8574b2c28..12ec6feff 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -286,16 +286,23 @@ function deploy_second_rook_cluster() { deploy_manifest_with_local_build toolbox.yaml toolbox.yaml } -function wait_for_rgw_pods() { +function wait_for_rgw() { for _ in {1..120}; do - if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --field-selector=status.phase=Running|wc -l)" -gt 1 ] ; then - echo "rgw pods found" + if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --no-headers --field-selector=status.phase=Running|wc -l)" -ge 1 ] ; then + echo "rgw pod is found" break fi echo "waiting for rgw pods" - sleep 5; + sleep 5 + done + for _ in {1..120}; do + if [ "$(kubectl -n "$1" get deployment -l app=rook-ceph-rgw -o yaml | yq read - 'items[0].status.readyReplicas')" -ge 1 ] ; then + echo "rgw is ready" + break + fi + echo "waiting for rgw becomes ready" + sleep 5 done - } function verify_operator_log_message() {