From 7c3e91390ed3ea57975e2704674c4d4103bff9b7 Mon Sep 17 00:00:00 2001 From: Shachar Sharon Date: Wed, 25 Oct 2023 13:38:44 +0300 Subject: [PATCH 1/2] nfs: add optional LivenessProbe to GaneshaServerSpec Allow end-user to have full control on NFS-Ganesha liveness probe: - Disabled - Enabled, using user-provided definition - Enabled, using Rook's default liveness probe for Ganesha. Signed-off-by: Shachar Sharon --- Documentation/CRDs/specification.md | 17 ++- .../charts/rook-ceph/templates/resources.yaml | 109 ++++++++++++++++++ deploy/examples/crds.yaml | 109 ++++++++++++++++++ deploy/examples/nfs.yaml | 4 + pkg/apis/ceph.rook.io/v1/types.go | 5 + 5 files changed, 243 insertions(+), 1 deletion(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 37bc3bae5..8f311b93a 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -6274,6 +6274,21 @@ bool

Whether host networking is enabled for the Ganesha server. If not set, the network settings from the cluster CR will be applied.

+ + +livenessProbe
+ + +ProbeSpec + + + + +(Optional) +

A liveness-probe to verify that Ganesha server has valid run-time state. +If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.

+ +

GatewaySpec @@ -10014,7 +10029,7 @@ QuotaSpec

ProbeSpec

-(Appears on:MetadataServerSpec, ObjectHealthCheckSpec) +(Appears on:GaneshaServerSpec, MetadataServerSpec, ObjectHealthCheckSpec)

ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon

diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index a1c2b07ff..64216fd98 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -9161,6 +9161,115 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + livenessProbe: + description: A liveness-probe to verify that Ganesha server has valid run-time state. If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object logLevel: description: LogLevel set logging level type: string diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index b557dc0ea..f94a22380 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -9154,6 +9154,115 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + livenessProbe: + description: A liveness-probe to verify that Ganesha server has valid run-time state. If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object logLevel: description: LogLevel set logging level type: string diff --git a/deploy/examples/nfs.yaml b/deploy/examples/nfs.yaml index f5b755f43..b43ae3033 100644 --- a/deploy/examples/nfs.yaml +++ b/deploy/examples/nfs.yaml @@ -59,6 +59,10 @@ spec: # The logging levels: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG |NIV_FULL_DEBUG |NB_LOG_LEVEL logLevel: NIV_INFO + # Allow liveness-probe via pod's nfs port (TCP 2049) + # livenessProbe: + # disabled: false + # Configure security options for the NFS cluster. See docs for more information: # https://rook.github.io/docs/rook/latest/Storage-Configuration/NFS/nfs-security/ security: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 5db6c1b1e..3dc6edaa1 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2118,6 +2118,11 @@ type GaneshaServerSpec struct { // +nullable // +optional HostNetwork *bool `json:"hostNetwork,omitempty"` + + // A liveness-probe to verify that Ganesha server has valid run-time state. + // If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + // +optional + LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` } // NFSSecuritySpec represents security configurations for an NFS server pod From e05184d0a79f657c4ddedcafa6316dc769f67065 Mon Sep 17 00:00:00 2001 From: Shachar Sharon Date: Wed, 30 Aug 2023 17:14:17 +0300 Subject: [PATCH 2/2] nfs: allow livness-probe for nfs-ganesha container Use K8s LivenessProbe mechanism to check OK-status of nfs-ganesha container. A user may define his own lineness-probe, or a default one which expects NFS TCP-port 2049 to be active; that is, willing to accept new connections: for Ceph>=18.2.1 issue 'rpcinfo' call on local pod; otherwise use standard K8s TCP-socket liveness probe mechanism. Define permissive values to liveness-probe to ensure that the NFS service is defined in failed-state only when it has non-recoverable error. The current default definition of LivenessProbe is expected to guard the nfs pod from at least the following two cases: - Deadlocks: where an nfs-ganesha server is running, but unable serve new connections due to internal bad-state. - Resource exhaustion on the host node (e.g. OOM) which prevents the server from accepting new connections and reply to NULL RPC request. In both cases we expect K8s to reschedule the nfs pod, most likely on different host node. Refs rook issue #12719 Signed-off-by: Shachar Sharon --- pkg/operator/ceph/controller/spec.go | 36 ++++++++++++++++++++ pkg/operator/ceph/nfs/spec.go | 17 +++++++++- pkg/operator/ceph/nfs/spec_test.go | 51 ++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index e9598ab8b..7f63415f0 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -18,6 +18,7 @@ limitations under the License. package controller import ( + "encoding/binary" "fmt" "os" "path" @@ -39,6 +40,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull return containerImagePullPolicy } + +// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has +// TCP a socket binded to specific port, and may create new connection. +func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe { + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{IntVal: port}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} + +// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell +// command which checks that the local NFS daemon has TCP a socket binded to +// specific port, and it has valid reply to NULL RPC request. +func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe { + bb := make([]byte, 2) + binary.BigEndian.PutUint16(bb, port) // port-num in network-order + servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1]) + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 8c857edf4..4c4bcbf45 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -22,8 +22,10 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig logLevel = nfs.Spec.Server.LogLevel } - return v1.Container{ + container := v1.Container{ Name: "nfs-ganesha", Command: []string{ "ganesha.nfsd", @@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), + LivenessProbe: r.defaultGaneshaLivenessProbe(nfs), } + return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe) +} + +func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe { + failureThreshold := int32(10) + cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1} + if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) { + // liveness-probe using rpcinfo utility + return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold) + } + // liveness-probe using K8s builtin TCP-socket action + return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold) } func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { diff --git a/pkg/operator/ceph/nfs/spec_test.go b/pkg/operator/ceph/nfs/spec_test.go index 0e729a2bf..548faeb83 100644 --- a/pkg/operator/ceph/nfs/spec_test.go +++ b/pkg/operator/ceph/nfs/spec_test.go @@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) { } assert.Contains(t, contNames, "sssd") }) + + t.Run("basic with default liveness-probe", func(t *testing.T) { + nfs := &cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NFSGaneshaSpec{ + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "myfs-data0", + Namespace: "nfs-test-ns", + }, + Server: cephv1.GaneshaServerSpec{ + Active: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + PriorityClassName: "my-priority-class", + LivenessProbe: &cephv1.ProbeSpec{ + Disabled: false, + }, + }, + }, + } + + r, cfg := newDeploymentSpecTest(t) + d, err := r.makeDeployment(nfs, cfg) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + + // Expects valid settings to default livness-probe + var ganeshaCont *v1.Container = nil + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" { + ganeshaCont = &d.Spec.Template.Spec.Containers[i] + break + } + } + assert.NotNil(t, ganeshaCont) + assert.NotNil(t, ganeshaCont.LivenessProbe) + assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10)) + assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10)) + assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5)) + }) }