From 2dd9b00ca30d8a16b46af3b98e0bfedee45896aa Mon Sep 17 00:00:00 2001 From: parth-gr Date: Wed, 6 Sep 2023 18:59:52 +0530 Subject: [PATCH] mds: improve the mds liveness probe calls Instead of checking the socket files for mds daemon, check the mds daemon in fs maps Closes: https://github.com/rook/rook/issues/12789 Signed-off-by: parth-gr --- pkg/operator/ceph/file/mds/livenessprobe.go | 92 +++ pkg/operator/ceph/file/mds/livenessprobe.sh | 28 + .../ceph/file/mds/livenessprobe_test.go | 647 ++++++++++++++++++ pkg/operator/ceph/file/mds/mds.go | 4 +- .../mdsLivenessProbeTestsFiles/0FS-2MDS.json | 114 +++ .../mds/mdsLivenessProbeTestsFiles/0FS.json | 25 + .../mdsLivenessProbeTestsFiles/1FS-2MDS.json | 175 +++++ .../2FS-(0,0)MDS.json | 138 ++++ .../2FS-(0,2)MDS.json | 233 +++++++ .../2FS-(1,2)MDS.json | 277 ++++++++ .../2FS-(2,2)MDS.json | 320 +++++++++ pkg/operator/ceph/file/mds/spec.go | 16 +- pkg/operator/ceph/file/mds/spec_test.go | 13 +- 13 files changed, 2069 insertions(+), 13 deletions(-) create mode 100644 pkg/operator/ceph/file/mds/livenessprobe.go create mode 100644 pkg/operator/ceph/file/mds/livenessprobe.sh create mode 100644 pkg/operator/ceph/file/mds/livenessprobe_test.go create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS-2MDS.json create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS.json create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/1FS-2MDS.json create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json create mode 100644 pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json diff --git a/pkg/operator/ceph/file/mds/livenessprobe.go b/pkg/operator/ceph/file/mds/livenessprobe.go new file mode 100644 index 000000000..beec3882a --- /dev/null +++ b/pkg/operator/ceph/file/mds/livenessprobe.go @@ -0,0 +1,92 @@ +package mds + +import ( + "bytes" + _ "embed" + "fmt" + "html/template" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" +) + +// mds probe constants +const ( + // set this to be close to the probe period to avoid routine timeouts, also avoids + // unlikely routine leaks in the case of some really unexpected failure mode in the probe script + mdsTimeoutSeconds int32 = 25 + // timeout for the mds cmd, should be less than mdsTimeoutSeconds + mdsCmdTimeout int32 = 20 + // set this equal to (or greater than) the probe period so we allow the daemon time + // to stabilize a bit before querying it + mdsInitialDelaySeconds int32 = 30 + // we want to balance checking too often with getting the cluster back to healthy in a reasonable time-frame + mdsPeriodSeconds int32 = 30 + // this should also prevent unnecessary MDS restarts when Rook is initially creating + // the filesystem and waiting for MDSes to join for the first time + mdsFailureThreshold int32 = 5 + // we have strong certainty of success, and a low value keeps from restarting the MDS pod too often in flaky systems + mdsSuccessThreshold int32 = 1 +) + +var ( + //go:embed livenessprobe.sh + mdsLivenessProbeCmdScript string +) + +type mdsLivenessProbeConfig struct { + MdsId string + FilesystemName string + Keyring string +} + +func renderProbe(mdsLivenessProbeConfigValue mdsLivenessProbeConfig) (string, error) { + var writer bytes.Buffer + name := mdsLivenessProbeConfigValue.FilesystemName + mdsLivenessProbeConfigValue.MdsId + "-probe" + + t := template.New(name) + t, err := t.Parse(mdsLivenessProbeCmdScript) + if err != nil { + return "", errors.Wrapf(err, "failed to parse template %q", name) + } + + if err := t.Execute(&writer, mdsLivenessProbeConfigValue); err != nil { + return "", errors.Wrapf(err, "failed to render template %q", name) + } + + return writer.String(), nil +} + +// GenerateMDSLivenessProbeExecDaemon generates a liveness probe that makes sure mds daemon is present in fs map, +// that it can be called, and that it returns 0 +func generateMDSLivenessProbeExecDaemon(daemonID, filesystemName, keyring string) *v1.Probe { + mdsLivenessProbeConfigValue := mdsLivenessProbeConfig{ + MdsId: daemonID, + FilesystemName: filesystemName, + Keyring: keyring, + } + + mdsLivenessProbeCmd, err := renderProbe(mdsLivenessProbeConfigValue) + if err != nil { + logger.Warning(err) + } + + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{ + "timeout", + fmt.Sprintf("%d", mdsCmdTimeout), + "sh", + "-c", + mdsLivenessProbeCmd, + }, + }, + }, + InitialDelaySeconds: mdsInitialDelaySeconds, + TimeoutSeconds: mdsTimeoutSeconds, + PeriodSeconds: mdsPeriodSeconds, + SuccessThreshold: mdsSuccessThreshold, + FailureThreshold: mdsFailureThreshold, + } +} diff --git a/pkg/operator/ceph/file/mds/livenessprobe.sh b/pkg/operator/ceph/file/mds/livenessprobe.sh new file mode 100644 index 000000000..97ad139bd --- /dev/null +++ b/pkg/operator/ceph/file/mds/livenessprobe.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# do not use 'set -e -u' etc. because it is important to only fail this probe when failure is certain +# spurious failures risk destabilizing ceph or the filesystem + +MDS_ID="{{ .MdsId }}" +FILESYSTEM_NAME="{{ .FilesystemName }}" +KEYRING="{{ .Keyring }}" + +outp="$(ceph fs dump --mon-host="$ROOK_CEPH_MON_HOST" --mon-initial-members="$ROOK_CEPH_MON_INITIAL_MEMBERS" --keyring "$KEYRING" --format json)" +rc=$? +if [ $rc -ne 0 ]; then + echo "ceph MDS dump check failed with the following output:" + echo "$outp" + echo "passing probe to avoid restarting MDS. cannot determine if MDS is unhealthy. restarting MDS risks destabilizing ceph/filesystem, which is likely unreachable or in error state" + exit 0 +fi + +# get the active and standby MDS in the fs map +standbyMds=$(echo "$outp" | jq ".standbys | map(.name) | any(.[]; . == \"$MDS_ID\")") +activeMds=$(echo "$outp" | jq ".filesystems[] | select(.mdsmap.fs_name == \"$FILESYSTEM_NAME\") | .mdsmap.info | map(.name) | any(.[]; . == \"$MDS_ID\")") + +if [[ $standbyMds == true || $activeMds == true ]]; then + echo "MDS ID present in MDS map, no need to re-start the container" + exit 0 +fi + +echo "Error: MDS ID not present in MDS map" +exit 1 diff --git a/pkg/operator/ceph/file/mds/livenessprobe_test.go b/pkg/operator/ceph/file/mds/livenessprobe_test.go new file mode 100644 index 000000000..e2d729b1e --- /dev/null +++ b/pkg/operator/ceph/file/mds/livenessprobe_test.go @@ -0,0 +1,647 @@ +package mds + +import ( + "bufio" + "bytes" + _ "embed" + "fmt" + "log" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" +) + +// simple bash wrapper that provides mock behavior for ceph CLI call(s) +// acts as shim between go test definitions and bash code under test +// define as string because embed doesn't work in _test.go files +var probeShimWrapper string = ` +#!/usr/bin/env bash +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +export SCRIPT_DIR +function ceph() { + # IMPORTANT: tests should always define behavior for specific commands being called to ensure code + # changes in the future do not inherit invalid success/failure behavior of existing unit tests + if [[ "$1" == "fs" ]] && [[ "$2" == "dump" ]] && [[ "$3" == "--mon-host=fake-mon-host" ]] && [[ "$4" == "--mon-initial-members=fake-mon-members" ]]; then + [[ -n "$CEPH_MDS_DUMP_MOCK_STDOUT" ]] && echo "$CEPH_MDS_DUMP_MOCK_STDOUT" + [[ -n "$CEPH_MDS_DUMP_MOCK_STDERR" ]] && echo "$CEPH_MDS_DUMP_MOCK_STDERR" >/dev/stderr + exit $CEPH_MDS_DUMP_MOCK_RETCODE + fi + echo "command unexpected: ceph $*" >/dev/stderr + exit 111 # random return that probably won't overlap with anything else +} +export -f ceph +function jq() { + $JQ "$@" # env should specify jq binary location +} +export -f jq +bash "${@}" +` + +var ( + //go:embed mdsLivenessProbeTestsFiles/0FS.json + testJsonData0FS string + //go:embed mdsLivenessProbeTestsFiles/0FS-2MDS.json + testJsonData0FS2MDS string + //go:embed mdsLivenessProbeTestsFiles/1FS-2MDS.json + testJsonData1FS2MDS string + //go:embed mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json + testJsonData2FS00MDS string + //go:embed mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json + testJsonData2FS02MDS string + //go:embed mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json + testJsonData2FS12MDS string + //go:embed mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json + testJsonData2FS22MDS string +) + +// writeLines writes the lines to the given file. +func writeLines(lines []string, path string) (*os.File, error) { + file, err := os.Create(path) + if err != nil { + return file, err + } + defer file.Close() + + w := bufio.NewWriter(file) + for _, line := range lines { + fmt.Fprintln(w, line) + } + return file, w.Flush() +} + +func Test_liveness_probe_script(t *testing.T) { + // liveness probe behavior has nuances that are critical to unit test, but there are challenges: + // - want to test the script in unit testing so tests live close to implementation + // - the probe script is written in bash, so shimming is required to test with go framework + // - the script processes output with 'jq' + // -> effective tests will treat 'jq' as a black box to ensure 'jq' commands are correct + // -> it is against good unit testing practice to require CLI tools for tests + // - a decent compromise is to skip this test in environments where 'jq' is not present (users) + // BUT ensure the test always runs in CI + + // TODO: set ROOK_UNIT_JQ_PATH in CI + + jqPath := "" + jqPathOverride := os.Getenv("ROOK_UNIT_JQ_PATH") // set this env var to ensure this test runs in CI + if jqPathOverride != "" { + jqPath = jqPathOverride + } else { + // convenience behavior: allow unit test to run in user environments that have jq + cmd := exec.Command("which", "jq") + cmd.Env = os.Environ() // this is the ONLY circumstance where unit tests are allowed to use host's env + stdout := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + err := cmd.Run() + if err != nil { + jqPath = "" + } else { + jqPath = stdout.String() + } + } + if jqPath == "" { + t.Skip("skipping mds liveness probe script unit tests because jq binary location is not known") + } + + t.Run("0 FS in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-c", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData0FS, // no fs and mds + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("0 FS AND 2 MDS in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData0FS2MDS, // no fs but mds present but myfs-a-a will be in standby list so result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("1 filesystem and 2 mds in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-a == myfs-a should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("mds not in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-c", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-c != myfs-a should result in failure + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("filesystem not in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs1", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs1 filesystem not present should result in failure + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("2 filesystem and 2 MDS for each filesystem in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS22MDS, // myfs1-a second mds and myfs1 second filesystem is present should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("2 filesystem with 0 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a-a second mds and myfs-a second filesystem is present should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + + probe = generateMDSLivenessProbeExecDaemon("myfs-a-b", "myfs-a", "/etc/ceph/keyring-store/keyring") + file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd = []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd = exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a-b second mds and myfs-a second filesystem is present should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout = bytes.NewBuffer([]byte{}) + stderr = bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + // similar to above test, this time testing for unhealthy daemon + t.Run("2 filesystem with 0 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a second mds of filesystem myfs is not present should result in failure + + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + + probe = generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring") + file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd = []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd = exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-b second mds of filesystem myfs is not present should result in failure + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout = bytes.NewBuffer([]byte{}) + stderr = bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("2 filesystem with 1 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a-a second mds and myfs-a second filesystem is present should result in success + + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + + probe = generateMDSLivenessProbeExecDaemon("myfs-a-b", "myfs-a", "/etc/ceph/keyring-store/keyring") + file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd = []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd = exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a-b second mds and myfs-a second filesystem is present should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout = bytes.NewBuffer([]byte{}) + stderr = bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + // similar to above test, this time testing for unhealthy daemon + t.Run("2 filesystem with 1 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a second mds of filesystem myfs is not present should result in failure + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + + probe = generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring") + file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd = []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd = exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-b second mds and myfs filesystem is present should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout = bytes.NewBuffer([]byte{}) + stderr = bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("ceph mds dump error", func(t *testing.T) { + cmd := exec.Command("bash", "-c", probeShimWrapper) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDERR='mock failure'", + "CEPH_MDS_DUMP_MOCK_RETCODE=1", + "JQ=" + jqPath, + } + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err := cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("ceph mds dump invalid json", func(t *testing.T) { + cmd := exec.Command("bash", "-c", probeShimWrapper) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + `CEPH_MDS_DUMP_MOCK_STDOUT={"active": "mds.a"`, // missing end brace + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err := cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("ceph mds in standby list", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-b == myfs-b should result in success + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.NoError(t, err) + assert.Equal(t, 0, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) + t.Run("2 filesystem with 0 mds for each filesystem", func(t *testing.T) { + probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") + file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") + if err != nil { + log.Fatalf("writeLines: %s", err) + } + defer os.Remove(file.Name()) + shellcmd := []string{"-c", probeShimWrapper} + shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") + cmd := exec.Command("bash", shellcmd...) + cmd.Env = []string{ + "ROOK_CEPH_MON_HOST=fake-mon-host", + "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", + "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS00MDS, // myfs-a != "" should result in failure + "CEPH_MDS_DUMP_MOCK_RETCODE=0", + "JQ=" + jqPath, + } + + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cmd.Stdout = stdout + cmd.Stderr = stderr + + err = cmd.Run() + assert.Error(t, err) + assert.Equal(t, 1, cmd.ProcessState.ExitCode()) + if t.Failed() { + t.Log("stdout:", stdout.String()) + t.Log("stderr:", stderr.String()) + } + }) +} diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index b95372d06..d1aef87af 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -38,6 +38,7 @@ import ( "github.com/rook/rook/pkg/util/exec" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds") @@ -179,7 +180,8 @@ func (c *Cluster) startDeployment(ctx context.Context, daemonLetterID string) (s } // start the deployment - d, err := c.makeDeployment(mdsConfig, c.fs.Namespace) + fsNamespacedname := types.NamespacedName{Name: c.fs.Name, Namespace: c.fs.Namespace} + d, err := c.makeDeployment(mdsConfig, fsNamespacedname) if err != nil { return "", errors.Wrap(err, "failed to create deployment") } diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS-2MDS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS-2MDS.json new file mode 100644 index 000000000..280760008 --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS-2MDS.json @@ -0,0 +1,114 @@ +{ + "epoch": 128, + "default_fscid": 3, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [ + { + "gid": 62980, + "name": "myfs-a-a", + "rank": -1, + "incarnation": 0, + "state": "up:standby", + "state_seq": 2, + "addr": "10.244.2.46:6801/4048246440", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.2.46:6800", + "nonce": 4048246440 + }, + { + "type": "v1", + "addr": "10.244.2.46:6801", + "nonce": 4048246440 + } + ] + }, + "join_fscid": -1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "epoch": 105 + }, + { + "gid": 84093, + "name": "myfs-a-b", + "rank": -1, + "incarnation": 0, + "state": "up:standby", + "state_seq": 1, + "addr": "10.244.0.19:6801/1646791331", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.0.19:6800", + "nonce": 1646791331 + }, + { + "type": "v1", + "addr": "10.244.0.19:6801", + "nonce": 1646791331 + } + ] + }, + "join_fscid": -1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "epoch": 113 + } + ], + "filesystems": [] +} diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS.json new file mode 100644 index 000000000..e9e52b765 --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/0FS.json @@ -0,0 +1,25 @@ +{ + "epoch": 26, + "default_fscid": -1, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [], + "filesystems": [] +} diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/1FS-2MDS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/1FS-2MDS.json new file mode 100644 index 000000000..f4996b0ad --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/1FS-2MDS.json @@ -0,0 +1,175 @@ +{ + "epoch": 4534, + "default_fscid": 1, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [ + { + "gid": 36824, + "name": "myfs-b", + "rank": -1, + "incarnation": 0, + "state": "up:standby", + "state_seq": 1, + "addr": "10.244.1.24:6801/2525614038", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.1.24:6800", + "nonce": 2525614038 + }, + { + "type": "v1", + "addr": "10.244.1.24:6801", + "nonce": 2525614038 + } + ] + }, + "join_fscid": 1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "epoch": 2551 + } + ], + "filesystems": [ + { + "mdsmap": { + "epoch": 4534, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-09-25T08:42:40.606222+0000", + "modified": "2023-09-25T16:35:44.806094+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 42, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": { + "mds_0": 36794 + }, + "failed": [], + "damaged": [], + "stopped": [], + "info": { + "gid_36794": { + "gid": 36794, + "name": "myfs-a", + "rank": 0, + "incarnation": 2550, + "state": "up:creating", + "state_seq": 2490, + "addr": "10.244.2.28:6801/3456990210", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.2.28:6800", + "nonce": 3456990210 + }, + { + "type": "v1", + "addr": "10.244.2.28:6801", + "nonce": 3456990210 + } + ] + }, + "join_fscid": 1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + } + }, + "data_pools": [ + 2 + ], + "metadata_pool": 1, + "enabled": true, + "fs_name": "myfs", + "balancer": "", + "standby_count_wanted": 0 + }, + "id": 1 + } + ] +} diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json new file mode 100644 index 000000000..f8dc563eb --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,0)MDS.json @@ -0,0 +1,138 @@ +{ + "epoch": 4369, + "default_fscid": 1, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [], + "filesystems": [ + { + "mdsmap": { + "epoch": 4367, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-09-28T06:51:11.262949+0000", + "modified": "2023-09-29T15:54:44.408889+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 45, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [], + "up": {}, + "failed": [], + "damaged": [], + "stopped": [], + "info": {}, + "data_pools": [ + 2 + ], + "metadata_pool": 1, + "enabled": true, + "fs_name": "myfs", + "balancer": "", + "standby_count_wanted": 0 + }, + "id": 1 + }, + { + "mdsmap": { + "epoch": 4369, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-09-29T15:38:54.538991+0000", + "modified": "2023-09-29T15:54:47.429406+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 46, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [], + "up": {}, + "failed": [], + "damaged": [], + "stopped": [], + "info": {}, + "data_pools": [ + 4 + ], + "metadata_pool": 3, + "enabled": true, + "fs_name": "myfs1", + "balancer": "", + "standby_count_wanted": 0 + }, + "id": 2 + } + ] +} diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json new file mode 100644 index 000000000..c6652f4c9 --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(0,2)MDS.json @@ -0,0 +1,233 @@ +{ + "epoch": 15, + "default_fscid": 1, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [], + "filesystems": [ + { + "mdsmap": { + "epoch": 15, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-10-04T11:52:39.450914+0000", + "modified": "2023-10-04T13:04:45.435992+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 85, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": {}, + "failed": [ + 0 + ], + "damaged": [], + "stopped": [], + "info": {}, + "data_pools": [ + 3 + ], + "metadata_pool": 2, + "enabled": true, + "fs_name": "myfs", + "balancer": "", + "standby_count_wanted": 1 + }, + "id": 1 + }, + { + "mdsmap": { + "epoch": 13, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-10-04T11:53:12.291488+0000", + "modified": "2023-10-04T12:10:56.711212+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": { + "mds_0": 24755 + }, + "failed": [], + "damaged": [], + "stopped": [], + "info": { + "gid_24755": { + "gid": 24755, + "name": "myfs-a-a", + "rank": 0, + "incarnation": 10, + "state": "up:active", + "state_seq": 5, + "addr": "10.244.1.14:6801/1909720117", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.1.14:6800", + "nonce": 1909720117 + }, + { + "type": "v1", + "addr": "10.244.1.14:6801", + "nonce": 1909720117 + } + ] + }, + "join_fscid": 2, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + }, + "gid_26507": { + "gid": 26507, + "name": "myfs-a-b", + "rank": 0, + "incarnation": 0, + "state": "up:standby-replay", + "state_seq": 1, + "addr": "10.244.3.2:6801/1455891216", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.3.2:6800", + "nonce": 1455891216 + }, + { + "type": "v1", + "addr": "10.244.3.2:6801", + "nonce": 1455891216 + } + ] + }, + "join_fscid": 2, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + } + }, + "data_pools": [ + 5 + ], + "metadata_pool": 4, + "enabled": true, + "fs_name": "myfs-a", + "balancer": "", + "standby_count_wanted": 1 + }, + "id": 2 + } + ] +} diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json new file mode 100644 index 000000000..d6843db0f --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(1,2)MDS.json @@ -0,0 +1,277 @@ +{ + "epoch": 19, + "default_fscid": 1, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [], + "filesystems": [ + { + "mdsmap": { + "epoch": 19, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-10-04T11:52:39.450914+0000", + "modified": "2023-10-04T13:36:07.867946+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 85, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": { + "mds_0": 12534 + }, + "failed": [], + "damaged": [], + "stopped": [], + "info": { + "gid_12534": { + "gid": 12534, + "name": "myfs-b", + "rank": 0, + "incarnation": 17, + "state": "up:rejoin", + "state_seq": 3, + "addr": "10.244.2.20:6801/2640787256", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.2.20:6800", + "nonce": 2640787256 + }, + { + "type": "v1", + "addr": "10.244.2.20:6801", + "nonce": 2640787256 + } + ] + }, + "join_fscid": 1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + } + }, + "data_pools": [ + 3 + ], + "metadata_pool": 2, + "enabled": true, + "fs_name": "myfs", + "balancer": "", + "standby_count_wanted": 1 + }, + "id": 1 + }, + { + "mdsmap": { + "epoch": 13, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-10-04T11:53:12.291488+0000", + "modified": "2023-10-04T12:10:56.711212+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": { + "mds_0": 24755 + }, + "failed": [], + "damaged": [], + "stopped": [], + "info": { + "gid_24755": { + "gid": 24755, + "name": "myfs-a-a", + "rank": 0, + "incarnation": 10, + "state": "up:active", + "state_seq": 5, + "addr": "10.244.1.14:6801/1909720117", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.1.14:6800", + "nonce": 1909720117 + }, + { + "type": "v1", + "addr": "10.244.1.14:6801", + "nonce": 1909720117 + } + ] + }, + "join_fscid": 2, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + }, + "gid_26507": { + "gid": 26507, + "name": "myfs-a-b", + "rank": 0, + "incarnation": 0, + "state": "up:standby-replay", + "state_seq": 1, + "addr": "10.244.3.2:6801/1455891216", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.3.2:6800", + "nonce": 1455891216 + }, + { + "type": "v1", + "addr": "10.244.3.2:6801", + "nonce": 1455891216 + } + ] + }, + "join_fscid": 2, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + } + }, + "data_pools": [ + 5 + ], + "metadata_pool": 4, + "enabled": true, + "fs_name": "myfs-a", + "balancer": "", + "standby_count_wanted": 1 + }, + "id": 2 + } + ] +} diff --git a/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json new file mode 100644 index 000000000..532394b5c --- /dev/null +++ b/pkg/operator/ceph/file/mds/mdsLivenessProbeTestsFiles/2FS-(2,2)MDS.json @@ -0,0 +1,320 @@ +{ + "epoch": 13, + "default_fscid": 1, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "feature_flags": { + "enable_multiple": true, + "ever_enabled_multiple": true + }, + "standbys": [], + "filesystems": [ + { + "mdsmap": { + "epoch": 7, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-10-04T11:52:39.450914+0000", + "modified": "2023-10-04T11:52:40.460825+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": { + "mds_0": 14695 + }, + "failed": [], + "damaged": [], + "stopped": [], + "info": { + "gid_4692": { + "gid": 4692, + "name": "myfs-a", + "rank": 0, + "incarnation": 0, + "state": "up:standby-replay", + "state_seq": 5, + "addr": "10.244.0.11:6801/2412019386", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.0.11:6800", + "nonce": 2412019386 + }, + { + "type": "v1", + "addr": "10.244.0.11:6801", + "nonce": 2412019386 + } + ] + }, + "join_fscid": 1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + }, + "gid_14695": { + "gid": 14695, + "name": "myfs-b", + "rank": 0, + "incarnation": 5, + "state": "up:active", + "state_seq": 4, + "addr": "10.244.2.15:6801/3023911818", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.2.15:6800", + "nonce": 3023911818 + }, + { + "type": "v1", + "addr": "10.244.2.15:6801", + "nonce": 3023911818 + } + ] + }, + "join_fscid": 1, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + } + }, + "data_pools": [ + 3 + ], + "metadata_pool": 2, + "enabled": true, + "fs_name": "myfs", + "balancer": "", + "standby_count_wanted": 1 + }, + "id": 1 + }, + { + "mdsmap": { + "epoch": 13, + "flags": 50, + "flags_state": { + "joinable": true, + "allow_snaps": true, + "allow_multimds_snaps": true, + "allow_standby_replay": true + }, + "ever_allowed_features": 32, + "explicitly_allowed_features": 32, + "created": "2023-10-04T11:53:12.291488+0000", + "modified": "2023-10-04T12:10:56.711212+0000", + "tableserver": 0, + "root": 0, + "session_timeout": 60, + "session_autoclose": 300, + "required_client_features": {}, + "max_file_size": 1099511627776, + "last_failure": 0, + "last_failure_osd_epoch": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + }, + "max_mds": 1, + "in": [ + 0 + ], + "up": { + "mds_0": 24755 + }, + "failed": [], + "damaged": [], + "stopped": [], + "info": { + "gid_24755": { + "gid": 24755, + "name": "myfs-a-a", + "rank": 0, + "incarnation": 10, + "state": "up:active", + "state_seq": 5, + "addr": "10.244.1.14:6801/1909720117", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.1.14:6800", + "nonce": 1909720117 + }, + { + "type": "v1", + "addr": "10.244.1.14:6801", + "nonce": 1909720117 + } + ] + }, + "join_fscid": 2, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + }, + "gid_26507": { + "gid": 26507, + "name": "myfs-a-b", + "rank": 0, + "incarnation": 0, + "state": "up:standby-replay", + "state_seq": 1, + "addr": "10.244.3.2:6801/1455891216", + "addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "10.244.3.2:6800", + "nonce": 1455891216 + }, + { + "type": "v1", + "addr": "10.244.3.2:6801", + "nonce": 1455891216 + } + ] + }, + "join_fscid": 2, + "export_targets": [], + "features": 4540138320759226367, + "flags": 0, + "compat": { + "compat": {}, + "ro_compat": {}, + "incompat": { + "feature_1": "base v0.20", + "feature_2": "client writeable ranges", + "feature_3": "default file layouts on dirs", + "feature_4": "dir inode in separate object", + "feature_5": "mds uses versioned encoding", + "feature_6": "dirfrag is stored in omap", + "feature_7": "mds uses inline data", + "feature_8": "no anchor table", + "feature_9": "file layout v2", + "feature_10": "snaprealm v2" + } + } + } + }, + "data_pools": [ + 5 + ], + "metadata_pool": 4, + "enabled": true, + "fs_name": "myfs-a", + "balancer": "", + "standby_count_wanted": 1 + }, + "id": 2 + } + ] +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index f1d25cc1c..83d38dab2 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -23,12 +23,14 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) const ( @@ -40,16 +42,16 @@ const ( mdsCacheMemoryResourceFactor = 0.8 ) -func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, namespace string) (*apps.Deployment, error) { +func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, fsNamespacedname types.NamespacedName) (*apps.Deployment, error) { - mdsContainer := c.makeMdsDaemonContainer(mdsConfig) + mdsContainer := c.makeMdsDaemonContainer(mdsConfig, fsNamespacedname.Name) mdsContainer = cephconfig.ConfigureStartupProbe(mdsContainer, c.fs.Spec.MetadataServer.StartupProbe) mdsContainer = cephconfig.ConfigureLivenessProbe(mdsContainer, c.fs.Spec.MetadataServer.LivenessProbe) podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, - Namespace: namespace, + Namespace: fsNamespacedname.Namespace, Labels: c.podLabels(mdsConfig, true), }, Spec: v1.PodSpec{ @@ -127,7 +129,7 @@ func (c *Cluster) makeChownInitContainer(mdsConfig *mdsConfig) v1.Container { ) } -func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { +func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig, fsName string) v1.Container { args := append( controller.DaemonFlags(c.clusterInfo, c.clusterSpec, mdsConfig.DaemonID), "--foreground", @@ -150,9 +152,9 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, SecurityContext: controller.PodSecurityContext(), - StartupProbe: controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID), - LivenessProbe: controller.GenerateLivenessProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID), - WorkingDir: cephconfig.VarLogCephDir, + // StartupProbe time for MDS is covered liveness probe + LivenessProbe: generateMDSLivenessProbeExecDaemon(mdsConfig.DaemonID, fsName, keyring.VolumeMount().KeyringFilePath()), + WorkingDir: cephconfig.VarLogCephDir, } return container diff --git a/pkg/operator/ceph/file/mds/spec_test.go b/pkg/operator/ceph/file/mds/spec_test.go index d75fe842e..18445edc1 100644 --- a/pkg/operator/ceph/file/mds/spec_test.go +++ b/pkg/operator/ceph/file/mds/spec_test.go @@ -35,6 +35,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) func testDeploymentObject(t *testing.T, network cephv1.NetworkSpec) (*apps.Deployment, error) { @@ -81,18 +82,20 @@ func testDeploymentObject(t *testing.T, network cephv1.NetworkSpec) (*apps.Deplo ResourceName: "rook-ceph-mds-myfs-a", DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, "myfs-a", "rook-ceph", "/var/lib/rook/"), } + fsNamespacedname := types.NamespacedName{Name: "myfs-a", Namespace: "ns"} + t.Run(("check mds ConfigureProbe"), func(t *testing.T) { - c.fs.Spec.MetadataServer.StartupProbe = &cephv1.ProbeSpec{Disabled: false, Probe: &v1.Probe{InitialDelaySeconds: 1000}} c.fs.Spec.MetadataServer.LivenessProbe = &cephv1.ProbeSpec{Disabled: false, Probe: &v1.Probe{InitialDelaySeconds: 900}} - deployment, err := c.makeDeployment(mdsTestConfig, "ns") + deployment, err := c.makeDeployment(mdsTestConfig, fsNamespacedname) assert.Nil(t, err) assert.NotNil(t, deployment) assert.NotNil(t, c.fs.Spec.MetadataServer.LivenessProbe) - assert.NotNil(t, c.fs.Spec.MetadataServer.StartupProbe) + assert.Equal(t, "myfs-a", mdsTestConfig.DaemonID) + assert.Equal(t, "myfs-a", fsNamespacedname.Name) assert.Equal(t, int32(900), deployment.Spec.Template.Spec.Containers[0].LivenessProbe.InitialDelaySeconds) - assert.Equal(t, int32(1000), deployment.Spec.Template.Spec.Containers[0].StartupProbe.InitialDelaySeconds) + assert.NotNil(t, deployment.Spec.Template.Spec.Containers[0].LivenessProbe.ProbeHandler.Exec) }) - return c.makeDeployment(mdsTestConfig, "ns") + return c.makeDeployment(mdsTestConfig, fsNamespacedname) } func TestPodSpecs(t *testing.T) {