diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index e9598ab8b..7f63415f0 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -18,6 +18,7 @@ limitations under the License. package controller import ( + "encoding/binary" "fmt" "os" "path" @@ -39,6 +40,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull return containerImagePullPolicy } + +// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has +// TCP a socket binded to specific port, and may create new connection. +func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe { + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{IntVal: port}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} + +// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell +// command which checks that the local NFS daemon has TCP a socket binded to +// specific port, and it has valid reply to NULL RPC request. +func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe { + bb := make([]byte, 2) + binary.BigEndian.PutUint16(bb, port) // port-num in network-order + servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1]) + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 8c857edf4..4c4bcbf45 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -22,8 +22,10 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig logLevel = nfs.Spec.Server.LogLevel } - return v1.Container{ + container := v1.Container{ Name: "nfs-ganesha", Command: []string{ "ganesha.nfsd", @@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), + LivenessProbe: r.defaultGaneshaLivenessProbe(nfs), } + return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe) +} + +func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe { + failureThreshold := int32(10) + cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1} + if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) { + // liveness-probe using rpcinfo utility + return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold) + } + // liveness-probe using K8s builtin TCP-socket action + return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold) } func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { diff --git a/pkg/operator/ceph/nfs/spec_test.go b/pkg/operator/ceph/nfs/spec_test.go index 0e729a2bf..548faeb83 100644 --- a/pkg/operator/ceph/nfs/spec_test.go +++ b/pkg/operator/ceph/nfs/spec_test.go @@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) { } assert.Contains(t, contNames, "sssd") }) + + t.Run("basic with default liveness-probe", func(t *testing.T) { + nfs := &cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NFSGaneshaSpec{ + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "myfs-data0", + Namespace: "nfs-test-ns", + }, + Server: cephv1.GaneshaServerSpec{ + Active: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + PriorityClassName: "my-priority-class", + LivenessProbe: &cephv1.ProbeSpec{ + Disabled: false, + }, + }, + }, + } + + r, cfg := newDeploymentSpecTest(t) + d, err := r.makeDeployment(nfs, cfg) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + + // Expects valid settings to default livness-probe + var ganeshaCont *v1.Container = nil + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" { + ganeshaCont = &d.Spec.Template.Spec.Containers[i] + break + } + } + assert.NotNil(t, ganeshaCont) + assert.NotNil(t, ganeshaCont.LivenessProbe) + assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10)) + assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10)) + assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5)) + }) }