From 35ee1af42a0169a71a766f6b7235bd7351e1071e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 15 Jan 2020 10:51:53 +0100 Subject: [PATCH] ceph: expose c-v log per pvc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now use a defined directory for each prepare job so that logs won't be entangled with each other. Also, if an error occurs, we print the log file too so we don't have to ssh onto a host to find the c-v debug log. Closes: https://github.com/rook/rook/issues/3888 Signed-off-by: Sébastien Han --- pkg/daemon/ceph/osd/volume.go | 71 +++++++++++++++++++++++++----- pkg/daemon/ceph/osd/volume_test.go | 29 ++++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 338a0fc84..69a591726 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path" "strconv" "strings" @@ -39,7 +40,9 @@ import ( // These are not constants because they are used by the tests var ( cephConfigDir = "/var/lib/ceph" + cephLogDir = "/var/log/ceph" lvmConfPath = "/etc/lvm/lvm.conf" + cvLogDir = "" ) const ( @@ -102,7 +105,24 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, error) { baseCommand := "stdbuf" - baseArgs := []string{"-oL", cephVolumeCmd, "lvm", "prepare"} + var baseArgs []string + + // Create a specific log directory so that each prepare command will have its own log + // Only do this if nothing is present so that we don't override existing logs + cvLogDir = path.Join(cephLogDir, a.nodeName) + err := os.MkdirAll(cvLogDir, 0755) + if err != nil { + logger.Errorf("failed to create ceph-volume log directory %q, continue with default %q. %v", cvLogDir, cephLogDir, err) + baseArgs = []string{"-oL", cephVolumeCmd} + } else { + baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir} + } + + baseArgs = append(baseArgs, []string{ + "lvm", + "prepare", + }...) + var lvpath string for name, device := range devices.Entries { if device.LegacyPartitionsFound { @@ -127,21 +147,31 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device "--data", deviceArg, }...) - // execute ceph-volume with the device - if op, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", baseCommand, immediateExecuteArgs...); err != nil { + // execute ceph-volume with the device + op, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", baseCommand, immediateExecuteArgs...) + if err != nil { + cvLogFilePath := path.Join(cvLogDir, "ceph-volume.log") + + // Print c-v log before exiting + cvLog := readCVLogContent(cvLogFilePath) + if cvLog != "" { + logger.Errorf("%s", cvLog) + } + + // Return failure return "", errors.Wrapf(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume + } + logger.Infof("%v", op) + if lvBackedPV { + lvpath = deviceArg } else { - logger.Infof("%v", op) - if lvBackedPV { - lvpath = deviceArg - } else { - lvpath = getLVPath(op) - if lvpath == "" { - return "", errors.New("failed to get lvpath from ceph-volume lvm prepare output") - } + lvpath = getLVPath(op) + if lvpath == "" { + return "", errors.New("failed to get lvpath from ceph-volume lvm prepare output") } } + } else { logger.Infof("skipping device %s with osd %d already configured", name, device.Data) } @@ -499,6 +529,25 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid s return osds, nil } +func readCVLogContent(cvLogFilePath string) string { + // Open c-v log file + cvLogFile, err := os.Open(cvLogFilePath) + if err != nil { + logger.Errorf("failed to open ceph-volume log file %q. %v", cvLogFilePath, err) + return "" + } + defer cvLogFile.Close() + + // Read c-v log file + b, err := ioutil.ReadAll(cvLogFile) + if err != nil { + logger.Errorf("failed to read ceph-volume log file %q. %v", cvLogFilePath, err) + return "" + } + + return string(b) +} + type osdInfo struct { Name string `json:"name"` Path string `json:"path"` diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 5523439d2..1f78150cc 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -17,6 +17,9 @@ limitations under the License. package osd import ( + "io/ioutil" + "os" + "path" "testing" "github.com/pkg/errors" @@ -311,3 +314,29 @@ func TestGetDatabaseSize(t *testing.T) { assert.Equal(t, 0, getDatabaseSize(0, 0)) assert.Equal(t, 2048, getDatabaseSize(4096, 2048)) } + +func TestPrintCVLogContent(t *testing.T) { + tmp, err := ioutil.TempFile("", "cv-log") + assert.Nil(t, err) + + defer os.Remove(tmp.Name()) + + nodeName := "set1-2-data-jmxdx" + cvLogDir = path.Join(tmp.Name(), nodeName) + assert.Equal(t, path.Join(tmp.Name(), nodeName), cvLogDir) + + cvLogFilePath := path.Join(cvLogDir, "ceph-volume.log") + assert.Equal(t, path.Join(cvLogDir, "ceph-volume.log"), cvLogFilePath) + + // Print c-v log, it is empty so this is similating a failure (e,g: the file does not exist) + cvLog := readCVLogContent(tmp.Name()) + assert.Empty(t, cvLog, cvLog) + + // Write content in the file + cvDummyLog := []byte(`dummy log`) + _, err = tmp.Write(cvDummyLog) + assert.NoError(t, err) + // Print again, now there is content + cvLog = readCVLogContent(tmp.Name()) + assert.NotEmpty(t, cvLog, cvLog) +}