From 47839e46dcc660e7d3d98c7ed615aa2aaf28b268 Mon Sep 17 00:00:00 2001 From: Shachar Sharon Date: Wed, 30 Aug 2023 17:14:17 +0300 Subject: [PATCH] nfs: allow livness-probe for nfs-ganesha container Use K8s LivenessProbe mechanism to check OK-status of nfs-ganesha container. A user may define his own lineness-probe, or a default one which expects NFS TCP-port 2049 to be active; that is, willing to accept new connections: for Ceph>=18.2.1 issue 'rpcinfo' call on local pod; otherwise use standard K8s TCP-socket liveness probe mechanism. Define permissive values to liveness-probe to ensure that the NFS service is defined in failed-state only when it has non-recoverable error. The current default definition of LivenessProbe is expected to guard the nfs pod from at least the following two cases: - Deadlocks: where an nfs-ganesha server is running, but unable serve new connections due to internal bad-state. - Resource exhaustion on the host node (e.g. OOM) which prevents the server from accepting new connections and reply to NULL RPC request. In both cases we expect K8s to reschedule the nfs pod, most likely on different host node. Refs rook issue #12719 Signed-off-by: Shachar Sharon (cherry picked from commit e05184d0a79f657c4ddedcafa6316dc769f67065) --- pkg/operator/ceph/controller/spec.go | 36 ++++++++++++++++++++ pkg/operator/ceph/nfs/spec.go | 17 +++++++++- pkg/operator/ceph/nfs/spec_test.go | 51 ++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index e9598ab8b..7f63415f0 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -18,6 +18,7 @@ limitations under the License. package controller import ( + "encoding/binary" "fmt" "os" "path" @@ -39,6 +40,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull return containerImagePullPolicy } + +// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has +// TCP a socket binded to specific port, and may create new connection. +func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe { + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{IntVal: port}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} + +// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell +// command which checks that the local NFS daemon has TCP a socket binded to +// specific port, and it has valid reply to NULL RPC request. +func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe { + bb := make([]byte, 2) + binary.BigEndian.PutUint16(bb, port) // port-num in network-order + servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1]) + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 8c857edf4..4c4bcbf45 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -22,8 +22,10 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig logLevel = nfs.Spec.Server.LogLevel } - return v1.Container{ + container := v1.Container{ Name: "nfs-ganesha", Command: []string{ "ganesha.nfsd", @@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), + LivenessProbe: r.defaultGaneshaLivenessProbe(nfs), } + return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe) +} + +func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe { + failureThreshold := int32(10) + cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1} + if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) { + // liveness-probe using rpcinfo utility + return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold) + } + // liveness-probe using K8s builtin TCP-socket action + return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold) } func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { diff --git a/pkg/operator/ceph/nfs/spec_test.go b/pkg/operator/ceph/nfs/spec_test.go index 0e729a2bf..548faeb83 100644 --- a/pkg/operator/ceph/nfs/spec_test.go +++ b/pkg/operator/ceph/nfs/spec_test.go @@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) { } assert.Contains(t, contNames, "sssd") }) + + t.Run("basic with default liveness-probe", func(t *testing.T) { + nfs := &cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NFSGaneshaSpec{ + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "myfs-data0", + Namespace: "nfs-test-ns", + }, + Server: cephv1.GaneshaServerSpec{ + Active: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + PriorityClassName: "my-priority-class", + LivenessProbe: &cephv1.ProbeSpec{ + Disabled: false, + }, + }, + }, + } + + r, cfg := newDeploymentSpecTest(t) + d, err := r.makeDeployment(nfs, cfg) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + + // Expects valid settings to default livness-probe + var ganeshaCont *v1.Container = nil + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" { + ganeshaCont = &d.Spec.Template.Spec.Containers[i] + break + } + } + assert.NotNil(t, ganeshaCont) + assert.NotNil(t, ganeshaCont.LivenessProbe) + assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10)) + assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10)) + assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5)) + }) }