nfs: allow livness-probe for nfs-ganesha container

Use K8s LivenessProbe mechanism to check OK-status of nfs-ganesha
container. A user may define his own lineness-probe, or a default one
which expects NFS TCP-port 2049 to be active; that is, willing to accept
new connections: for Ceph>=18.2.1 issue 'rpcinfo' call on local pod;
otherwise use standard K8s TCP-socket liveness probe mechanism.
Define permissive values to liveness-probe to ensure that the NFS
service is defined in failed-state only when it has non-recoverable
error.

The current default definition of LivenessProbe is expected to guard the
nfs pod from at least the following two cases:

  - Deadlocks: where an nfs-ganesha server is running, but unable serve
    new connections due to internal bad-state.

  - Resource exhaustion on the host node (e.g. OOM) which prevents the
    server from accepting new connections and reply to NULL RPC request.

In both cases we expect K8s to reschedule the nfs pod, most likely on
different host node.

Refs rook issue #12719

Signed-off-by: Shachar Sharon <ssharon@redhat.com>
(cherry picked from commit e05184d0a7)
This commit is contained in:
Shachar Sharon authored and Mergify committed 2023-11-13 19:45:28 +00:00
1 parent f2420162bc
commit 47839e46dc
3 files changed
+103 -1

No files matched your search

+36
View File
@@ -18,6 +18,7 @@ limitations under the License.
package controller
import (
"encoding/binary"
"fmt"
"os"
"path"
@@ -39,6 +40,7 @@ import (
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
@@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull
return containerImagePullPolicy
}
// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has
// TCP a socket binded to specific port, and may create new connection.
func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe {
return &v1.Probe{
ProbeHandler: v1.ProbeHandler{
TCPSocket: &v1.TCPSocketAction{
Port: intstr.IntOrString{IntVal: port},
},
},
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
TimeoutSeconds: livenessProbeTimeoutSeconds,
FailureThreshold: failureThreshold,
}
}
// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell
// command which checks that the local NFS daemon has TCP a socket binded to
// specific port, and it has valid reply to NULL RPC request.
func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe {
bb := make([]byte, 2)
binary.BigEndian.PutUint16(bb, port) // port-num in network-order
servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1])
return &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"},
},
},
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
TimeoutSeconds: livenessProbeTimeoutSeconds,
FailureThreshold: failureThreshold,
}
}
+16 -1
View File
@@ -22,8 +22,10 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
logLevel = nfs.Spec.Server.LogLevel
}
return v1.Container{
container := v1.Container{
Name: "nfs-ganesha",
Command: []string{
"ganesha.nfsd",
@@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
Env: controller.DaemonEnvVars(r.cephClusterSpec),
Resources: nfs.Spec.Server.Resources,
SecurityContext: controller.PodSecurityContext(),
LivenessProbe: r.defaultGaneshaLivenessProbe(nfs),
}
return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe)
}
func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe {
failureThreshold := int32(10)
cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1}
if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) {
// liveness-probe using rpcinfo utility
return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold)
}
// liveness-probe using K8s builtin TCP-socket action
return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold)
}
func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container {
+51
View File
@@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) {
}
assert.Contains(t, contNames, "sssd")
})
t.Run("basic with default liveness-probe", func(t *testing.T) {
nfs := &cephv1.CephNFS{
ObjectMeta: metav1.ObjectMeta{
Name: "my-nfs",
Namespace: "rook-ceph-test-ns",
},
Spec: cephv1.NFSGaneshaSpec{
RADOS: cephv1.GaneshaRADOSSpec{
Pool: "myfs-data0",
Namespace: "nfs-test-ns",
},
Server: cephv1.GaneshaServerSpec{
Active: 3,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
},
},
PriorityClassName: "my-priority-class",
LivenessProbe: &cephv1.ProbeSpec{
Disabled: false,
},
},
},
}
r, cfg := newDeploymentSpecTest(t)
d, err := r.makeDeployment(nfs, cfg)
assert.NoError(t, err)
assert.NotEmpty(t, d.Spec.Template.Annotations)
// Expects valid settings to default livness-probe
var ganeshaCont *v1.Container = nil
for i := range d.Spec.Template.Spec.Containers {
if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" {
ganeshaCont = &d.Spec.Template.Spec.Containers[i]
break
}
}
assert.NotNil(t, ganeshaCont)
assert.NotNil(t, ganeshaCont.LivenessProbe)
assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10))
assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10))
assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5))
})
}