feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #12845 from synarete/nfs-liveness-probe

Browse Source 
nfs: add livness-probe to nfs-ganesha container
This commit is contained in:
Blaine Gardner
2023-11-13 13:44:57 -06:00
committed by GitHub
parent 6f1a3eee01 e05184d0a7
commit 5230fcd60d
8 changed files with 346 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/CRDs/specification.md
+16 -1
View File
@@ -6274,6 +6274,21 @@ bool
<p>Whether host networking is enabled for the Ganesha server. If not set, the network settings from the cluster CR will be applied.</p>
</td>
</tr>
<tr>
<td>
<code>livenessProbe</code><br/>
<em>
<a href="#ceph.rook.io/v1.ProbeSpec">
ProbeSpec
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>A liveness-probe to verify that Ganesha server has valid run-time state.
If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.</p>
</td>
</tr>
</tbody>
</table>
<h3 id="ceph.rook.io/v1.GatewaySpec">GatewaySpec
@@ -10014,7 +10029,7 @@ QuotaSpec
<h3 id="ceph.rook.io/v1.ProbeSpec">ProbeSpec
</h3>
<p>
(<em>Appears on:</em><a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.ObjectHealthCheckSpec">ObjectHealthCheckSpec</a>)
(<em>Appears on:</em><a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.ObjectHealthCheckSpec">ObjectHealthCheckSpec</a>)
</p>
<div>
<p>ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon</p>
deploy/charts/rook-ceph/templates/resources.yaml
+109
View File
@@ -9161,6 +9161,115 @@ spec:
nullable: true
type: object
x-kubernetes-preserve-unknown-fields: true
livenessProbe:
description: A liveness-probe to verify that Ganesha server has valid run-time state. If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
properties:
disabled:
description: Disabled determines whether probe is disable or not
type: boolean
probe:
description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.
properties:
exec:
description: Exec specifies the action to take.
properties:
command:
description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
items:
type: string
type: array
type: object
failureThreshold:
description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.
format: int32
type: integer
grpc:
description: GRPC specifies an action involving a GRPC port.
properties:
port:
description: Port number of the gRPC service. Number must be in the range 1 to 65535.
format: int32
type: integer
service:
description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC."
type: string
required:
- port
type: object
httpGet:
description: HTTPGet specifies the http request to perform.
properties:
host:
description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead.
type: string
httpHeaders:
description: Custom headers to set in the request. HTTP allows repeated headers.
items:
description: HTTPHeader describes a custom header to be used in HTTP probes
properties:
name:
description: The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.
type: string
value:
description: The header field value
type: string
required:
- name
- value
type: object
type: array
path:
description: Path to access on the HTTP server.
type: string
port:
anyOf:
- type: integer
- type: string
description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
scheme:
description: Scheme to use for connecting to the host. Defaults to HTTP.
type: string
required:
- port
type: object
initialDelaySeconds:
description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
periodSeconds:
description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.
format: int32
type: integer
successThreshold:
description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
format: int32
type: integer
tcpSocket:
description: TCPSocket specifies an action involving a TCP port.
properties:
host:
description: 'Optional: Host name to connect to, defaults to the pod IP.'
type: string
port:
anyOf:
- type: integer
- type: string
description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
required:
- port
type: object
terminationGracePeriodSeconds:
description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.
format: int64
type: integer
timeoutSeconds:
description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
type: object
type: object
logLevel:
description: LogLevel set logging level
type: string
deploy/examples/crds.yaml
+109
View File
@@ -9154,6 +9154,115 @@ spec:
nullable: true
type: object
x-kubernetes-preserve-unknown-fields: true
livenessProbe:
description: A liveness-probe to verify that Ganesha server has valid run-time state. If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
properties:
disabled:
description: Disabled determines whether probe is disable or not
type: boolean
probe:
description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.
properties:
exec:
description: Exec specifies the action to take.
properties:
command:
description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
items:
type: string
type: array
type: object
failureThreshold:
description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.
format: int32
type: integer
grpc:
description: GRPC specifies an action involving a GRPC port.
properties:
port:
description: Port number of the gRPC service. Number must be in the range 1 to 65535.
format: int32
type: integer
service:
description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC."
type: string
required:
- port
type: object
httpGet:
description: HTTPGet specifies the http request to perform.
properties:
host:
description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead.
type: string
httpHeaders:
description: Custom headers to set in the request. HTTP allows repeated headers.
items:
description: HTTPHeader describes a custom header to be used in HTTP probes
properties:
name:
description: The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.
type: string
value:
description: The header field value
type: string
required:
- name
- value
type: object
type: array
path:
description: Path to access on the HTTP server.
type: string
port:
anyOf:
- type: integer
- type: string
description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
scheme:
description: Scheme to use for connecting to the host. Defaults to HTTP.
type: string
required:
- port
type: object
initialDelaySeconds:
description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
periodSeconds:
description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.
format: int32
type: integer
successThreshold:
description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
format: int32
type: integer
tcpSocket:
description: TCPSocket specifies an action involving a TCP port.
properties:
host:
description: 'Optional: Host name to connect to, defaults to the pod IP.'
type: string
port:
anyOf:
- type: integer
- type: string
description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
x-kubernetes-int-or-string: true
required:
- port
type: object
terminationGracePeriodSeconds:
description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.
format: int64
type: integer
timeoutSeconds:
description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
format: int32
type: integer
type: object
type: object
logLevel:
description: LogLevel set logging level
type: string
deploy/examples/nfs.yaml
+4
View File
@@ -59,6 +59,10 @@ spec:
# The logging levels: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG |NIV_FULL_DEBUG |NB_LOG_LEVEL
logLevel: NIV_INFO
# Allow liveness-probe via pod's nfs port (TCP 2049)
# livenessProbe:
# disabled: false
# Configure security options for the NFS cluster. See docs for more information:
# https://rook.github.io/docs/rook/latest/Storage-Configuration/NFS/nfs-security/
security:
pkg/apis/ceph.rook.io/v1/types.go
+5
View File
@@ -2118,6 +2118,11 @@ type GaneshaServerSpec struct {
// +nullable
// +optional
HostNetwork *bool `json:"hostNetwork,omitempty"`
// A liveness-probe to verify that Ganesha server has valid run-time state.
// If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
// +optional
LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
}
// NFSSecuritySpec represents security configurations for an NFS server pod
pkg/operator/ceph/controller/spec.go
+36
View File
@@ -18,6 +18,7 @@ limitations under the License.
package controller
import (
"encoding/binary"
"fmt"
"os"
"path"
@@ -39,6 +40,7 @@ import (
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
@@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull
return containerImagePullPolicy
}
// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has
// TCP a socket binded to specific port, and may create new connection.
func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe {
return &v1.Probe{
ProbeHandler: v1.ProbeHandler{
TCPSocket: &v1.TCPSocketAction{
Port: intstr.IntOrString{IntVal: port},
},
},
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
TimeoutSeconds: livenessProbeTimeoutSeconds,
FailureThreshold: failureThreshold,
}
}
// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell
// command which checks that the local NFS daemon has TCP a socket binded to
// specific port, and it has valid reply to NULL RPC request.
func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe {
bb := make([]byte, 2)
binary.BigEndian.PutUint16(bb, port) // port-num in network-order
servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1])
return &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"},
},
},
InitialDelaySeconds: livenessProbeInitialDelaySeconds,
TimeoutSeconds: livenessProbeTimeoutSeconds,
FailureThreshold: failureThreshold,
}
}
pkg/operator/ceph/nfs/spec.go
+16 -1
View File
@@ -22,8 +22,10 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
logLevel = nfs.Spec.Server.LogLevel
}
return v1.Container{
container := v1.Container{
Name: "nfs-ganesha",
Command: []string{
"ganesha.nfsd",
@@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
Env: controller.DaemonEnvVars(r.cephClusterSpec),
Resources: nfs.Spec.Server.Resources,
SecurityContext: controller.PodSecurityContext(),
LivenessProbe: r.defaultGaneshaLivenessProbe(nfs),
}
return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe)
}
func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe {
failureThreshold := int32(10)
cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1}
if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) {
// liveness-probe using rpcinfo utility
return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold)
}
// liveness-probe using K8s builtin TCP-socket action
return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold)
}
func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container {
pkg/operator/ceph/nfs/spec_test.go
+51
View File
@@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) {
}
assert.Contains(t, contNames, "sssd")
})
t.Run("basic with default liveness-probe", func(t *testing.T) {
nfs := &cephv1.CephNFS{
ObjectMeta: metav1.ObjectMeta{
Name: "my-nfs",
Namespace: "rook-ceph-test-ns",
},
Spec: cephv1.NFSGaneshaSpec{
RADOS: cephv1.GaneshaRADOSSpec{
Pool: "myfs-data0",
Namespace: "nfs-test-ns",
},
Server: cephv1.GaneshaServerSpec{
Active: 3,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
},
},
PriorityClassName: "my-priority-class",
LivenessProbe: &cephv1.ProbeSpec{
Disabled: false,
},
},
},
}
r, cfg := newDeploymentSpecTest(t)
d, err := r.makeDeployment(nfs, cfg)
assert.NoError(t, err)
assert.NotEmpty(t, d.Spec.Template.Annotations)
// Expects valid settings to default livness-probe
var ganeshaCont *v1.Container = nil
for i := range d.Spec.Template.Spec.Containers {
if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" {
ganeshaCont = &d.Spec.Template.Spec.Containers[i]
break
}
}
assert.NotNil(t, ganeshaCont)
assert.NotNil(t, ganeshaCont.LivenessProbe)
assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10))
assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10))
assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5))
})
}