package mds import ( "bufio" "bytes" _ "embed" "fmt" "log" "os" "os/exec" "testing" "github.com/stretchr/testify/assert" ) // simple bash wrapper that provides mock behavior for ceph CLI call(s) // acts as shim between go test definitions and bash code under test // define as string because embed doesn't work in _test.go files var probeShimWrapper string = ` #!/usr/bin/env bash SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) export SCRIPT_DIR function ceph() { # IMPORTANT: tests should always define behavior for specific commands being called to ensure code # changes in the future do not inherit invalid success/failure behavior of existing unit tests if [[ "$1" == "fs" ]] && [[ "$2" == "dump" ]] && [[ "$3" == "--mon-host=fake-mon-host" ]] && [[ "$4" == "--mon-initial-members=fake-mon-members" ]]; then [[ -n "$CEPH_MDS_DUMP_MOCK_STDOUT" ]] && echo "$CEPH_MDS_DUMP_MOCK_STDOUT" [[ -n "$CEPH_MDS_DUMP_MOCK_STDERR" ]] && echo "$CEPH_MDS_DUMP_MOCK_STDERR" >/dev/stderr exit $CEPH_MDS_DUMP_MOCK_RETCODE fi echo "command unexpected: ceph $*" >/dev/stderr exit 111 # random return that probably won't overlap with anything else } export -f ceph function jq() { $JQ "$@" # env should specify jq binary location } export -f jq bash "${@}" ` var ( //go:embed test/0FS.json testJsonData0FS string //go:embed test/0FS-2MDS.json testJsonData0FS2MDS string //go:embed test/1FS-2MDS.json testJsonData1FS2MDS string //go:embed test/2FS-(0,0)MDS.json testJsonData2FS00MDS string //go:embed test/2FS-(0,2)MDS.json testJsonData2FS02MDS string //go:embed test/2FS-(1,2)MDS.json testJsonData2FS12MDS string //go:embed test/2FS-(2,2)MDS.json testJsonData2FS22MDS string ) // writeLines writes the lines to the given file. func writeLines(lines []string, path string) (*os.File, error) { file, err := os.Create(path) if err != nil { return file, err } defer file.Close() w := bufio.NewWriter(file) for _, line := range lines { fmt.Fprintln(w, line) } return file, w.Flush() } func Test_liveness_probe_script(t *testing.T) { // liveness probe behavior has nuances that are critical to unit test, but there are challenges: // - want to test the script in unit testing so tests live close to implementation // - the probe script is written in bash, so shimming is required to test with go framework // - the script processes output with 'jq' // -> effective tests will treat 'jq' as a black box to ensure 'jq' commands are correct // -> it is against good unit testing practice to require CLI tools for tests // - a decent compromise is to skip this test in environments where 'jq' is not present (users) // BUT ensure the test always runs in CI // TODO: set ROOK_UNIT_JQ_PATH in CI jqPath := "" jqPathOverride := os.Getenv("ROOK_UNIT_JQ_PATH") // set this env var to ensure this test runs in CI if jqPathOverride != "" { jqPath = jqPathOverride } else { // convenience behavior: allow unit test to run in user environments that have jq cmd := exec.Command("which", "jq") cmd.Env = os.Environ() // this is the ONLY circumstance where unit tests are allowed to use host's env stdout := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout err := cmd.Run() if err != nil { jqPath = "" } else { jqPath = stdout.String() } } if jqPath == "" { t.Skip("skipping mds liveness probe script unit tests because jq binary location is not known") } t.Run("0 FS in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-c", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData0FS, // no fs and mds "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("0 FS AND 2 MDS in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData0FS2MDS, // no fs but mds present but myfs-a-a will be in standby list so result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("1 filesystem and 2 mds in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-a == myfs-a should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("mds not in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-c", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-c != myfs-a should result in failure "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("filesystem not in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs1", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs1 filesystem not present should result in failure "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("2 filesystem and 2 MDS for each filesystem in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS22MDS, // myfs1-a second mds and myfs1 second filesystem is present should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("2 filesystem with 0 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a-a second mds and myfs-a second filesystem is present should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } probe = generateMDSLivenessProbeExecDaemon("myfs-a-b", "myfs-a", "/etc/ceph/keyring-store/keyring") file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd = []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd = exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a-b second mds and myfs-a second filesystem is present should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout = bytes.NewBuffer([]byte{}) stderr = bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) // similar to above test, this time testing for unhealthy daemon t.Run("2 filesystem with 0 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-a second mds of filesystem myfs is not present should result in failure "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } probe = generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring") file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd = []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd = exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS02MDS, // myfs-b second mds of filesystem myfs is not present should result in failure "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout = bytes.NewBuffer([]byte{}) stderr = bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("2 filesystem with 1 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a-a", "myfs-a", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a-a second mds and myfs-a second filesystem is present should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } probe = generateMDSLivenessProbeExecDaemon("myfs-a-b", "myfs-a", "/etc/ceph/keyring-store/keyring") file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd = []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd = exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a-b second mds and myfs-a second filesystem is present should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout = bytes.NewBuffer([]byte{}) stderr = bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) // similar to above test, this time testing for unhealthy daemon t.Run("2 filesystem with 1 MDS for first and 2 MDS for second filesystem in map", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-a second mds of filesystem myfs is not present should result in failure "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } probe = generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring") file, err = writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd = []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd = exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS12MDS, // myfs-b second mds and myfs filesystem is present should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout = bytes.NewBuffer([]byte{}) stderr = bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("ceph mds dump error", func(t *testing.T) { cmd := exec.Command("bash", "-c", probeShimWrapper) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDERR='mock failure'", "CEPH_MDS_DUMP_MOCK_RETCODE=1", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err := cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("ceph mds dump invalid json", func(t *testing.T) { cmd := exec.Command("bash", "-c", probeShimWrapper) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", `CEPH_MDS_DUMP_MOCK_STDOUT={"active": "mds.a"`, // missing end brace "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err := cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("ceph mds in standby list", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-b", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData1FS2MDS, // myfs-b == myfs-b should result in success "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.NoError(t, err) assert.Equal(t, 0, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) t.Run("2 filesystem with 0 mds for each filesystem", func(t *testing.T) { probe := generateMDSLivenessProbeExecDaemon("myfs-a", "myfs", "/etc/ceph/keyring-store/keyring") file, err := writeLines(probe.Exec.Command, "livenessprobeSample.sh") if err != nil { log.Fatalf("writeLines: %s", err) } defer os.Remove(file.Name()) shellcmd := []string{"-c", probeShimWrapper} shellcmd = append(shellcmd, "-x", "livenessprobeSample.sh") cmd := exec.Command("bash", shellcmd...) cmd.Env = []string{ "ROOK_CEPH_MON_HOST=fake-mon-host", "ROOK_CEPH_MON_INITIAL_MEMBERS=fake-mon-members", "CEPH_MDS_DUMP_MOCK_STDOUT=" + testJsonData2FS00MDS, // myfs-a != "" should result in failure "CEPH_MDS_DUMP_MOCK_RETCODE=0", "JQ=" + jqPath, } stdout := bytes.NewBuffer([]byte{}) stderr := bytes.NewBuffer([]byte{}) cmd.Stdout = stdout cmd.Stderr = stderr err = cmd.Run() assert.Error(t, err) assert.Equal(t, 1, cmd.ProcessState.ExitCode()) if t.Failed() { t.Log("stdout:", stdout.String()) t.Log("stderr:", stderr.String()) } }) }