feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

mds: improve the mds liveness probe calls

Browse Source 
Instead of checking the socket files for mds daemon,
check the mds daemon in fs maps

Closes: https://github.com/rook/rook/issues/12789

Signed-off-by: parth-gr <paarora@redhat.com>
This commit is contained in:
parth-gr committed 2023-10-05 00:54:06 +05:30
1 parent a64f9bccf7
commit 2dd9b00ca3
13 files changed
+2069 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/operator/ceph/file/mds/livenessprobe.go
+92
View File
@@ -0,0 +1,92 @@
package mds
import (
"bytes"
_ "embed"
"fmt"
"html/template"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
)
// mds probe constants
const (
// set this to be close to the probe period to avoid routine timeouts, also avoids
// unlikely routine leaks in the case of some really unexpected failure mode in the probe script
mdsTimeoutSeconds int32 = 25
// timeout for the mds cmd, should be less than mdsTimeoutSeconds
mdsCmdTimeout int32 = 20
// set this equal to (or greater than) the probe period so we allow the daemon time
// to stabilize a bit before querying it
mdsInitialDelaySeconds int32 = 30
// we want to balance checking too often with getting the cluster back to healthy in a reasonable time-frame
mdsPeriodSeconds int32 = 30
// this should also prevent unnecessary MDS restarts when Rook is initially creating
// the filesystem and waiting for MDSes to join for the first time
mdsFailureThreshold int32 = 5
// we have strong certainty of success, and a low value keeps from restarting the MDS pod too often in flaky systems
mdsSuccessThreshold int32 = 1
)
var (
//go:embed livenessprobe.sh
mdsLivenessProbeCmdScript string
)
type mdsLivenessProbeConfig struct {
MdsId string
FilesystemName string
Keyring string
}
func renderProbe(mdsLivenessProbeConfigValue mdsLivenessProbeConfig) (string, error) {
var writer bytes.Buffer
name := mdsLivenessProbeConfigValue.FilesystemName + mdsLivenessProbeConfigValue.MdsId + "-probe"
t := template.New(name)
t, err := t.Parse(mdsLivenessProbeCmdScript)
if err != nil {
return "", errors.Wrapf(err, "failed to parse template %q", name)
}
if err := t.Execute(&writer, mdsLivenessProbeConfigValue); err != nil {
return "", errors.Wrapf(err, "failed to render template %q", name)
}
return writer.String(), nil
}
// GenerateMDSLivenessProbeExecDaemon generates a liveness probe that makes sure mds daemon is present in fs map,
// that it can be called, and that it returns 0
func generateMDSLivenessProbeExecDaemon(daemonID, filesystemName, keyring string) *v1.Probe {
mdsLivenessProbeConfigValue := mdsLivenessProbeConfig{
MdsId: daemonID,
FilesystemName: filesystemName,
Keyring: keyring,
}
mdsLivenessProbeCmd, err := renderProbe(mdsLivenessProbeConfigValue)
if err != nil {
logger.Warning(err)
}
return &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Command: []string{
"timeout",
fmt.Sprintf("%d", mdsCmdTimeout),
"sh",
"-c",
mdsLivenessProbeCmd,
},
},
},
InitialDelaySeconds: mdsInitialDelaySeconds,
TimeoutSeconds: mdsTimeoutSeconds,
PeriodSeconds: mdsPeriodSeconds,
SuccessThreshold: mdsSuccessThreshold,
FailureThreshold: mdsFailureThreshold,
}
}
pkg/operator/ceph/file/mds/livenessprobe.sh
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
# do not use 'set -e -u' etc. because it is important to only fail this probe when failure is certain
# spurious failures risk destabilizing ceph or the filesystem
MDS_ID="{{ .MdsId }}"
FILESYSTEM_NAME="{{ .FilesystemName }}"
KEYRING="{{ .Keyring }}"
outp="$(ceph fs dump --mon-host="$ROOK_CEPH_MON_HOST" --mon-initial-members="$ROOK_CEPH_MON_INITIAL_MEMBERS" --keyring "$KEYRING" --format json)"
rc=$?
if [ $rc -ne 0 ]; then
echo "ceph MDS dump check failed with the following output:"
echo "$outp"
echo "passing probe to avoid restarting MDS. cannot determine if MDS is unhealthy. restarting MDS risks destabilizing ceph/filesystem, which is likely unreachable or in error state"
exit 0
fi
# get the active and standby MDS in the fs map
standbyMds=$(echo "$outp" | jq ".standbys | map(.name) | any(.[]; . == \"$MDS_ID\")")
activeMds=$(echo "$outp" | jq ".filesystems[] | select(.mdsmap.fs_name == \"$FILESYSTEM_NAME\") | .mdsmap.info | map(.name) | any(.[]; . == \"$MDS_ID\")")
if [[ $standbyMds == true || $activeMds == true ]]; then
echo "MDS ID present in MDS map, no need to re-start the container"
exit 0
fi
echo "Error: MDS ID not present in MDS map"
exit 1
pkg/operator/ceph/file/mds/livenessprobe_test.go
+647
View File
@@ -0,0 +1,647 @@
package mds
import (
"bufio"
"bytes"
_ "embed"
"fmt"
"log"
"os"
"os/exec"
"testing"
"github.com/stretchr/testify/assert"
)
// simple bash wrapper that provides mock behavior for ceph CLI call(s)
// acts as shim between go test definitions and bash code under test
// define as string because embed doesn't work in _test.go files
var probeShimWrapper string = `
#!/usr/bin/env bash
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
export SCRIPT_DIR
function ceph() {
# IMPORTANT: tests should always define behavior for specific commands being called to ensure code
# changes in the future do not inherit invalid success/failure behavior of existing unit tests
if [[ "$1" == "fs" ]] && [[ "$2" == "dump" ]] && [[ "$3" == "--mon-host=fake-mon-host" ]] && [[ "$4" == "--mon-initial-members=fake-mon-members" ]]; then
[[ -n "$CEPH_MDS_DUMP_MOCK_STDOUT" ]] && echo "$CEPH_MDS_DUMP_MOCK_STDOUT"
[[ -n "$CEPH_MDS_DUMP_MOCK_STDERR" ]] && echo "$CEPH_MDS_DUMP_MOCK_STDERR" >/dev/stderr
exit $CEPH_MDS_DUMP_MOCK_RETCODE
fi
echo "command unexpected: ceph $*" >/dev/stderr
exit 111 # random return that probably won't overlap with anything else
}
export -f ceph
function jq() {
$JQ "$@" # env should specify jq binary location
}
export -f jq
bash "${@}"
`
var (
//go:embed mdsLivenessProbeTestsFiles/0FS.json
testJsonData0FS string
//go:embed mdsLivenessProbeTestsFiles/0FS-2MDS.json
testJsonData0FS2MDS string
//go:embed mdsLivenessProbeTestsFiles/1FS-2MDS.json
testJsonData1FS2MDS string
//go:embed mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json
testJsonData2FS00MDS string
//go:embed mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json
testJsonData2FS02MDS string
//go:embed mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json
testJsonData2FS12MDS string
//go:embed mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json
testJsonData2FS22MDS string
)
// writeLines writes the lines to the given file.
func writeLines(lines []string, path string) (*os.File, error) {
file, err := os.Create(path)
if err != nil {
return file, err
}
defer file.Close()
w := bufio.NewWriter(file)
for _, line := range lines {
fmt.Fprintln(w, line)
}
return file, w.Flush()
}
func Test_liveness_probe_script(t *testing.T) {
// liveness probe behavior has nuances that are critical to unit test, but there are challenges:
// - want to test the script in unit testing so tests live close to implementation
// - the probe script is written in bash, so shimming is required to test with go framework
// - the script processes output with 'jq'
// -> effective tests will treat 'jq' as a black box to ensure 'jq' commands are correct
// -> it is against good unit testing practice to require CLI tools for tests
// - a decent compromise is to skip this test in environments where 'jq' is not present (users)
// BUT ensure the test always runs in CI
// TODO: set ROOK_UNIT_JQ_PATH in CI
jqPath := ""
jqPathOverride := os.Getenv("ROOK_UNIT_JQ_PATH") // set this env var to ensure this test runs in CI
if jqPathOverride != "" {
jqPath = jqPathOverride
} else {
// convenience behavior: allow unit test to run in user environments that have jq
cmd := exec.Command("which", "jq")
cmd.Env = os.Environ() // this is the ONLY circumstance where unit tests are allowed to use host's env
stdout := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
err := cmd.Run()
if err != nil {
jqPath = ""
} else {
jqPath = stdout.String()
}
}
if jqPath == "" {
t.Skip("skipping mds liveness probe script unit tests because jq binary location is not known")
}
t.Run("0 FS in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-c", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData0FS, // no fs and mds
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("0 FS AND 2 MDS in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData0FS2MDS, // no fs but mds present but myfs-a-a will be in standby list so result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("1 filesystem and 2 mds in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-a == myfs-a should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("mds not in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-c", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-c != myfs-a should result in failure
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("filesystem not in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs1", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs1 filesystem not present should result in failure
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("2 filesystem and 2 MDS for each filesystem in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS22MDS, // myfs1-a second mds and myfs1 second filesystem is present should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("2 filesystem with 0 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a-a second mds and myfs-a second filesystem is present should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
probe = generateMDSLivenessProbeExecDaemon("myfs-a-b", "myfs-a", "/etc/ceph/keyring-store/keyring")
file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd = []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd = exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a-b second mds and myfs-a second filesystem is present should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout = bytes.NewBuffer([]byte{})
stderr = bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
// similar to above test, this time testing for unhealthy daemon
t.Run("2 filesystem with 0 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a second mds of filesystem myfs is not present should result in failure
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
probe = generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring")
file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd = []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd = exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-b second mds of filesystem myfs is not present should result in failure
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout = bytes.NewBuffer([]byte{})
stderr = bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("2 filesystem with 1 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a-a second mds and myfs-a second filesystem is present should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
probe = generateMDSLivenessProbeExecDaemon("myfs-a-b", "myfs-a", "/etc/ceph/keyring-store/keyring")
file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd = []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd = exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a-b second mds and myfs-a second filesystem is present should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout = bytes.NewBuffer([]byte{})
stderr = bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
// similar to above test, this time testing for unhealthy daemon
t.Run("2 filesystem with 1 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a second mds of filesystem myfs is not present should result in failure
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
probe = generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring")
file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd = []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd = exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-b second mds and myfs filesystem is present should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout = bytes.NewBuffer([]byte{})
stderr = bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("ceph mds dump error", func(t *testing.T) {
cmd := exec.Command("bash", "-c", probeShimWrapper)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDERR='mock failure'",
"CEPH_MDS_DUMP_MOCK_RETCODE=1",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err := cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("ceph mds dump invalid json", func(t *testing.T) {
cmd := exec.Command("bash", "-c", probeShimWrapper)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
`CEPH_MDS_DUMP_MOCK_STDOUT={"active": "mds.a"`, // missing end brace
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err := cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("ceph mds in standby list", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-b == myfs-b should result in success
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.NoError(t, err)
assert.Equal(t, 0, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
t.Run("2 filesystem with 0 mds for each filesystem", func(t *testing.T) {
probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring")
file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh")
if err != nil {
log.Fatalf("writeLines: %s", err)
}
defer os.Remove(file.Name())
shellcmd := []string{"-c", probeShimWrapper}
shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh")
cmd := exec.Command("bash", shellcmd...)
cmd.Env = []string{
"ROOK_CEPH_MON_HOST=fake-mon-host",
"ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members",
"CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS00MDS, // myfs-a != "" should result in failure
"CEPH_MDS_DUMP_MOCK_RETCODE=0",
"JQ=" + jqPath,
}
stdout := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
assert.Error(t, err)
assert.Equal(t, 1, cmd.ProcessState.ExitCode())
if t.Failed() {
t.Log("stdout:", stdout.String())
t.Log("stderr:", stderr.String())
}
})
}
pkg/operator/ceph/file/mds/mds.go
+3 -1
View File
@@ -38,6 +38,7 @@ import (
"github.com/rook/rook/pkg/util/exec"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds")
@@ -179,7 +180,8 @@ func (c *Cluster) startDeployment(ctx context.Context, daemonLetterID string) (s
}
// start the deployment
d, err := c.makeDeployment(mdsConfig, c.fs.Namespace)
fsNamespacedname := types.NamespacedName{Name: c.fs.Name, Namespace: c.fs.Namespace}
d, err := c.makeDeployment(mdsConfig, fsNamespacedname)
if err != nil {
return "", errors.Wrap(err, "failed to create deployment")
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS-2MDS.json
+114
View File
@@ -0,0 +1,114 @@
{
"epoch": 128,
"default_fscid": 3,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [
{
"gid": 62980,
"name": "myfs-a-a",
"rank": -1,
"incarnation": 0,
"state": "up:standby",
"state_seq": 2,
"addr": "10.244.2.46:6801/4048246440",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.2.46:6800",
"nonce": 4048246440
},
{
"type": "v1",
"addr": "10.244.2.46:6801",
"nonce": 4048246440
}
]
},
"join_fscid": -1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"epoch": 105
},
{
"gid": 84093,
"name": "myfs-a-b",
"rank": -1,
"incarnation": 0,
"state": "up:standby",
"state_seq": 1,
"addr": "10.244.0.19:6801/1646791331",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.0.19:6800",
"nonce": 1646791331
},
{
"type": "v1",
"addr": "10.244.0.19:6801",
"nonce": 1646791331
}
]
},
"join_fscid": -1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"epoch": 113
}
],
"filesystems": []
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS.json
+25
View File
@@ -0,0 +1,25 @@
{
"epoch": 26,
"default_fscid": -1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [],
"filesystems": []
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/1FS-2MDS.json
+175
View File
@@ -0,0 +1,175 @@
{
"epoch": 4534,
"default_fscid": 1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [
{
"gid": 36824,
"name": "myfs-b",
"rank": -1,
"incarnation": 0,
"state": "up:standby",
"state_seq": 1,
"addr": "10.244.1.24:6801/2525614038",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.1.24:6800",
"nonce": 2525614038
},
{
"type": "v1",
"addr": "10.244.1.24:6801",
"nonce": 2525614038
}
]
},
"join_fscid": 1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"epoch": 2551
}
],
"filesystems": [
{
"mdsmap": {
"epoch": 4534,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-09-25T08:42:40.606222+0000",
"modified": "2023-09-25T16:35:44.806094+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 42,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 36794
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_36794": {
"gid": 36794,
"name": "myfs-a",
"rank": 0,
"incarnation": 2550,
"state": "up:creating",
"state_seq": 2490,
"addr": "10.244.2.28:6801/3456990210",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.2.28:6800",
"nonce": 3456990210
},
{
"type": "v1",
"addr": "10.244.2.28:6801",
"nonce": 3456990210
}
]
},
"join_fscid": 1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
}
},
"data_pools": [
2
],
"metadata_pool": 1,
"enabled": true,
"fs_name": "myfs",
"balancer": "",
"standby_count_wanted": 0
},
"id": 1
}
]
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json
+138
View File
@@ -0,0 +1,138 @@
{
"epoch": 4369,
"default_fscid": 1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [],
"filesystems": [
{
"mdsmap": {
"epoch": 4367,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-09-28T06:51:11.262949+0000",
"modified": "2023-09-29T15:54:44.408889+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 45,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [],
"up": {},
"failed": [],
"damaged": [],
"stopped": [],
"info": {},
"data_pools": [
2
],
"metadata_pool": 1,
"enabled": true,
"fs_name": "myfs",
"balancer": "",
"standby_count_wanted": 0
},
"id": 1
},
{
"mdsmap": {
"epoch": 4369,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-09-29T15:38:54.538991+0000",
"modified": "2023-09-29T15:54:47.429406+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 46,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [],
"up": {},
"failed": [],
"damaged": [],
"stopped": [],
"info": {},
"data_pools": [
4
],
"metadata_pool": 3,
"enabled": true,
"fs_name": "myfs1",
"balancer": "",
"standby_count_wanted": 0
},
"id": 2
}
]
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json
+233
View File
@@ -0,0 +1,233 @@
{
"epoch": 15,
"default_fscid": 1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [],
"filesystems": [
{
"mdsmap": {
"epoch": 15,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-10-04T11:52:39.450914+0000",
"modified": "2023-10-04T13:04:45.435992+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 85,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {},
"failed": [
0
],
"damaged": [],
"stopped": [],
"info": {},
"data_pools": [
3
],
"metadata_pool": 2,
"enabled": true,
"fs_name": "myfs",
"balancer": "",
"standby_count_wanted": 1
},
"id": 1
},
{
"mdsmap": {
"epoch": 13,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-10-04T11:53:12.291488+0000",
"modified": "2023-10-04T12:10:56.711212+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 24755
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_24755": {
"gid": 24755,
"name": "myfs-a-a",
"rank": 0,
"incarnation": 10,
"state": "up:active",
"state_seq": 5,
"addr": "10.244.1.14:6801/1909720117",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.1.14:6800",
"nonce": 1909720117
},
{
"type": "v1",
"addr": "10.244.1.14:6801",
"nonce": 1909720117
}
]
},
"join_fscid": 2,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
},
"gid_26507": {
"gid": 26507,
"name": "myfs-a-b",
"rank": 0,
"incarnation": 0,
"state": "up:standby-replay",
"state_seq": 1,
"addr": "10.244.3.2:6801/1455891216",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.3.2:6800",
"nonce": 1455891216
},
{
"type": "v1",
"addr": "10.244.3.2:6801",
"nonce": 1455891216
}
]
},
"join_fscid": 2,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
}
},
"data_pools": [
5
],
"metadata_pool": 4,
"enabled": true,
"fs_name": "myfs-a",
"balancer": "",
"standby_count_wanted": 1
},
"id": 2
}
]
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json
+277
View File
@@ -0,0 +1,277 @@
{
"epoch": 19,
"default_fscid": 1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [],
"filesystems": [
{
"mdsmap": {
"epoch": 19,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-10-04T11:52:39.450914+0000",
"modified": "2023-10-04T13:36:07.867946+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 85,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 12534
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_12534": {
"gid": 12534,
"name": "myfs-b",
"rank": 0,
"incarnation": 17,
"state": "up:rejoin",
"state_seq": 3,
"addr": "10.244.2.20:6801/2640787256",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.2.20:6800",
"nonce": 2640787256
},
{
"type": "v1",
"addr": "10.244.2.20:6801",
"nonce": 2640787256
}
]
},
"join_fscid": 1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
}
},
"data_pools": [
3
],
"metadata_pool": 2,
"enabled": true,
"fs_name": "myfs",
"balancer": "",
"standby_count_wanted": 1
},
"id": 1
},
{
"mdsmap": {
"epoch": 13,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-10-04T11:53:12.291488+0000",
"modified": "2023-10-04T12:10:56.711212+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 24755
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_24755": {
"gid": 24755,
"name": "myfs-a-a",
"rank": 0,
"incarnation": 10,
"state": "up:active",
"state_seq": 5,
"addr": "10.244.1.14:6801/1909720117",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.1.14:6800",
"nonce": 1909720117
},
{
"type": "v1",
"addr": "10.244.1.14:6801",
"nonce": 1909720117
}
]
},
"join_fscid": 2,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
},
"gid_26507": {
"gid": 26507,
"name": "myfs-a-b",
"rank": 0,
"incarnation": 0,
"state": "up:standby-replay",
"state_seq": 1,
"addr": "10.244.3.2:6801/1455891216",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.3.2:6800",
"nonce": 1455891216
},
{
"type": "v1",
"addr": "10.244.3.2:6801",
"nonce": 1455891216
}
]
},
"join_fscid": 2,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
}
},
"data_pools": [
5
],
"metadata_pool": 4,
"enabled": true,
"fs_name": "myfs-a",
"balancer": "",
"standby_count_wanted": 1
},
"id": 2
}
]
}
pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json
+320
View File
@@ -0,0 +1,320 @@
{
"epoch": 13,
"default_fscid": 1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [],
"filesystems": [
{
"mdsmap": {
"epoch": 7,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-10-04T11:52:39.450914+0000",
"modified": "2023-10-04T11:52:40.460825+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 14695
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_4692": {
"gid": 4692,
"name": "myfs-a",
"rank": 0,
"incarnation": 0,
"state": "up:standby-replay",
"state_seq": 5,
"addr": "10.244.0.11:6801/2412019386",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.0.11:6800",
"nonce": 2412019386
},
{
"type": "v1",
"addr": "10.244.0.11:6801",
"nonce": 2412019386
}
]
},
"join_fscid": 1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
},
"gid_14695": {
"gid": 14695,
"name": "myfs-b",
"rank": 0,
"incarnation": 5,
"state": "up:active",
"state_seq": 4,
"addr": "10.244.2.15:6801/3023911818",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.2.15:6800",
"nonce": 3023911818
},
{
"type": "v1",
"addr": "10.244.2.15:6801",
"nonce": 3023911818
}
]
},
"join_fscid": 1,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
}
},
"data_pools": [
3
],
"metadata_pool": 2,
"enabled": true,
"fs_name": "myfs",
"balancer": "",
"standby_count_wanted": 1
},
"id": 1
},
{
"mdsmap": {
"epoch": 13,
"flags": 50,
"flags_state": {
"joinable": true,
"allow_snaps": true,
"allow_multimds_snaps": true,
"allow_standby_replay": true
},
"ever_allowed_features": 32,
"explicitly_allowed_features": 32,
"created": "2023-10-04T11:53:12.291488+0000",
"modified": "2023-10-04T12:10:56.711212+0000",
"tableserver": 0,
"root": 0,
"session_timeout": 60,
"session_autoclose": 300,
"required_client_features": {},
"max_file_size": 1099511627776,
"last_failure": 0,
"last_failure_osd_epoch": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"max_mds": 1,
"in": [
0
],
"up": {
"mds_0": 24755
},
"failed": [],
"damaged": [],
"stopped": [],
"info": {
"gid_24755": {
"gid": 24755,
"name": "myfs-a-a",
"rank": 0,
"incarnation": 10,
"state": "up:active",
"state_seq": 5,
"addr": "10.244.1.14:6801/1909720117",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.1.14:6800",
"nonce": 1909720117
},
{
"type": "v1",
"addr": "10.244.1.14:6801",
"nonce": 1909720117
}
]
},
"join_fscid": 2,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
},
"gid_26507": {
"gid": 26507,
"name": "myfs-a-b",
"rank": 0,
"incarnation": 0,
"state": "up:standby-replay",
"state_seq": 1,
"addr": "10.244.3.2:6801/1455891216",
"addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.244.3.2:6800",
"nonce": 1455891216
},
{
"type": "v1",
"addr": "10.244.3.2:6801",
"nonce": 1455891216
}
]
},
"join_fscid": 2,
"export_targets": [],
"features": 4540138320759226367,
"flags": 0,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_7": "mds uses inline data",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
}
}
},
"data_pools": [
5
],
"metadata_pool": 4,
"enabled": true,
"fs_name": "myfs-a",
"balancer": "",
"standby_count_wanted": 1
},
"id": 2
}
]
}
pkg/operator/ceph/file/mds/spec.go
+9 -7
View File
@@ -23,12 +23,14 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
const (
@@ -40,16 +42,16 @@ const (
mdsCacheMemoryResourceFactor = 0.8
)
func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, namespace string) (*apps.Deployment, error) {
func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, fsNamespacedname types.NamespacedName) (*apps.Deployment, error) {
mdsContainer := c.makeMdsDaemonContainer(mdsConfig)
mdsContainer := c.makeMdsDaemonContainer(mdsConfig, fsNamespacedname.Name)
mdsContainer = cephconfig.ConfigureStartupProbe(mdsContainer, c.fs.Spec.MetadataServer.StartupProbe)
mdsContainer = cephconfig.ConfigureLivenessProbe(mdsContainer, c.fs.Spec.MetadataServer.LivenessProbe)
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: mdsConfig.ResourceName,
Namespace: namespace,
Namespace: fsNamespacedname.Namespace,
Labels: c.podLabels(mdsConfig, true),
},
Spec: v1.PodSpec{
@@ -127,7 +129,7 @@ func (c *Cluster) makeChownInitContainer(mdsConfig *mdsConfig) v1.Container {
)
}
func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig, fsName string) v1.Container {
args := append(
controller.DaemonFlags(c.clusterInfo, c.clusterSpec, mdsConfig.DaemonID),
"--foreground",
@@ -150,9 +152,9 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)),
Resources: c.fs.Spec.MetadataServer.Resources,
SecurityContext: controller.PodSecurityContext(),
StartupProbe: controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID),
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID),
WorkingDir: cephconfig.VarLogCephDir,
// StartupProbe time for MDS is covered liveness probe
LivenessProbe: generateMDSLivenessProbeExecDaemon(mdsConfig.DaemonID, fsName, keyring.VolumeMount().KeyringFilePath()),
WorkingDir: cephconfig.VarLogCephDir,
}
return container
pkg/operator/ceph/file/mds/spec_test.go
+8 -5
View File
@@ -35,6 +35,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
func testDeploymentObject(t *testing.T, network cephv1.NetworkSpec) (*apps.Deployment, error) {
@@ -81,18 +82,20 @@ func testDeploymentObject(t *testing.T, network cephv1.NetworkSpec) (*apps.Deplo
ResourceName: "rook-ceph-mds-myfs-a",
DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, "myfs-a", "rook-ceph", "/var/lib/rook/"),
}
fsNamespacedname := types.NamespacedName{Name: "myfs-a", Namespace: "ns"}
t.Run(("check mds ConfigureProbe"), func(t *testing.T) {
c.fs.Spec.MetadataServer.StartupProbe = &cephv1.ProbeSpec{Disabled: false, Probe: &v1.Probe{InitialDelaySeconds: 1000}}
c.fs.Spec.MetadataServer.LivenessProbe = &cephv1.ProbeSpec{Disabled: false, Probe: &v1.Probe{InitialDelaySeconds: 900}}
deployment, err := c.makeDeployment(mdsTestConfig, "ns")
deployment, err := c.makeDeployment(mdsTestConfig, fsNamespacedname)
assert.Nil(t, err)
assert.NotNil(t, deployment)
assert.NotNil(t, c.fs.Spec.MetadataServer.LivenessProbe)
assert.NotNil(t, c.fs.Spec.MetadataServer.StartupProbe)
assert.Equal(t, "myfs-a", mdsTestConfig.DaemonID)
assert.Equal(t, "myfs-a", fsNamespacedname.Name)
assert.Equal(t, int32(900), deployment.Spec.Template.Spec.Containers[0].LivenessProbe.InitialDelaySeconds)
assert.Equal(t, int32(1000), deployment.Spec.Template.Spec.Containers[0].StartupProbe.InitialDelaySeconds)
assert.NotNil(t, deployment.Spec.Template.Spec.Containers[0].LivenessProbe.ProbeHandler.Exec)
})
return c.makeDeployment(mdsTestConfig, "ns")
return c.makeDeployment(mdsTestConfig, fsNamespacedname)
}
func TestPodSpecs(t *testing.T) {