nfs: allow livness-probe for nfs-ganesha container
Use K8s LivenessProbe mechanism to check OK-status of nfs-ganesha
container. A user may define his own lineness-probe, or a default one
which expects NFS TCP-port 2049 to be active; that is, willing to accept
new connections: for Ceph>=18.2.1 issue 'rpcinfo' call on local pod;
otherwise use standard K8s TCP-socket liveness probe mechanism.
Define permissive values to liveness-probe to ensure that the NFS
service is defined in failed-state only when it has non-recoverable
error.
The current default definition of LivenessProbe is expected to guard the
nfs pod from at least the following two cases:
- Deadlocks: where an nfs-ganesha server is running, but unable serve
new connections due to internal bad-state.
- Resource exhaustion on the host node (e.g. OOM) which prevents the
server from accepting new connections and reply to NULL RPC request.
In both cases we expect K8s to reschedule the nfs pod, most likely on
different host node.
Refs rook issue #12719
Signed-off-by: Shachar Sharon <ssharon@redhat.com>
(cherry picked from commit e05184d0a7)
This commit is contained in:
1 parent
f2420162bc
commit
47839e46dc
3 files changed
+103
-1
No files matched your search
@@ -18,6 +18,7 @@ limitations under the License.
|
||||
package controller
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
@@ -39,6 +40,7 @@ import (
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull
|
||||
|
||||
return containerImagePullPolicy
|
||||
}
|
||||
|
||||
// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has
|
||||
// TCP a socket binded to specific port, and may create new connection.
|
||||
func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe {
|
||||
return &v1.Probe{
|
||||
ProbeHandler: v1.ProbeHandler{
|
||||
TCPSocket: &v1.TCPSocketAction{
|
||||
Port: intstr.IntOrString{IntVal: port},
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
|
||||
TimeoutSeconds: livenessProbeTimeoutSeconds,
|
||||
FailureThreshold: failureThreshold,
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell
|
||||
// command which checks that the local NFS daemon has TCP a socket binded to
|
||||
// specific port, and it has valid reply to NULL RPC request.
|
||||
func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe {
|
||||
bb := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(bb, port) // port-num in network-order
|
||||
servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1])
|
||||
return &v1.Probe{
|
||||
ProbeHandler: v1.ProbeHandler{
|
||||
Exec: &v1.ExecAction{
|
||||
Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"},
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
|
||||
TimeoutSeconds: livenessProbeTimeoutSeconds,
|
||||
FailureThreshold: failureThreshold,
|
||||
}
|
||||
}
|
||||
@@ -22,8 +22,10 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
|
||||
logLevel = nfs.Spec.Server.LogLevel
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
container := v1.Container{
|
||||
Name: "nfs-ganesha",
|
||||
Command: []string{
|
||||
"ganesha.nfsd",
|
||||
@@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
|
||||
Env: controller.DaemonEnvVars(r.cephClusterSpec),
|
||||
Resources: nfs.Spec.Server.Resources,
|
||||
SecurityContext: controller.PodSecurityContext(),
|
||||
LivenessProbe: r.defaultGaneshaLivenessProbe(nfs),
|
||||
}
|
||||
return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe)
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe {
|
||||
failureThreshold := int32(10)
|
||||
cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1}
|
||||
if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) {
|
||||
// liveness-probe using rpcinfo utility
|
||||
return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold)
|
||||
}
|
||||
// liveness-probe using K8s builtin TCP-socket action
|
||||
return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold)
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container {
|
||||
|
||||
@@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
}
|
||||
assert.Contains(t, contNames, "sssd")
|
||||
})
|
||||
|
||||
t.Run("basic with default liveness-probe", func(t *testing.T) {
|
||||
nfs := &cephv1.CephNFS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nfs",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
RADOS: cephv1.GaneshaRADOSSpec{
|
||||
Pool: "myfs-data0",
|
||||
Namespace: "nfs-test-ns",
|
||||
},
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 3,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
PriorityClassName: "my-priority-class",
|
||||
LivenessProbe: &cephv1.ProbeSpec{
|
||||
Disabled: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
r, cfg := newDeploymentSpecTest(t)
|
||||
d, err := r.makeDeployment(nfs, cfg)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, d.Spec.Template.Annotations)
|
||||
|
||||
// Expects valid settings to default livness-probe
|
||||
var ganeshaCont *v1.Container = nil
|
||||
for i := range d.Spec.Template.Spec.Containers {
|
||||
if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" {
|
||||
ganeshaCont = &d.Spec.Template.Spec.Containers[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.NotNil(t, ganeshaCont)
|
||||
assert.NotNil(t, ganeshaCont.LivenessProbe)
|
||||
assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10))
|
||||
assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10))
|
||||
assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5))
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user