diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index 338a0fc84..69a591726 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path" "strconv" "strings" @@ -39,7 +40,9 @@ import ( // These are not constants because they are used by the tests var ( cephConfigDir = "/var/lib/ceph" + cephLogDir = "/var/log/ceph" lvmConfPath = "/etc/lvm/lvm.conf" + cvLogDir = "" ) const ( @@ -102,7 +105,24 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, error) { baseCommand := "stdbuf" - baseArgs := []string{"-oL", cephVolumeCmd, "lvm", "prepare"} + var baseArgs []string + + // Create a specific log directory so that each prepare command will have its own log + // Only do this if nothing is present so that we don't override existing logs + cvLogDir = path.Join(cephLogDir, a.nodeName) + err := os.MkdirAll(cvLogDir, 0755) + if err != nil { + logger.Errorf("failed to create ceph-volume log directory %q, continue with default %q. %v", cvLogDir, cephLogDir, err) + baseArgs = []string{"-oL", cephVolumeCmd} + } else { + baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir} + } + + baseArgs = append(baseArgs, []string{ + "lvm", + "prepare", + }...) + var lvpath string for name, device := range devices.Entries { if device.LegacyPartitionsFound { @@ -127,21 +147,31 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device "--data", deviceArg, }...) - // execute ceph-volume with the device - if op, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", baseCommand, immediateExecuteArgs...); err != nil { + // execute ceph-volume with the device + op, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", baseCommand, immediateExecuteArgs...) + if err != nil { + cvLogFilePath := path.Join(cvLogDir, "ceph-volume.log") + + // Print c-v log before exiting + cvLog := readCVLogContent(cvLogFilePath) + if cvLog != "" { + logger.Errorf("%s", cvLog) + } + + // Return failure return "", errors.Wrapf(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume + } + logger.Infof("%v", op) + if lvBackedPV { + lvpath = deviceArg } else { - logger.Infof("%v", op) - if lvBackedPV { - lvpath = deviceArg - } else { - lvpath = getLVPath(op) - if lvpath == "" { - return "", errors.New("failed to get lvpath from ceph-volume lvm prepare output") - } + lvpath = getLVPath(op) + if lvpath == "" { + return "", errors.New("failed to get lvpath from ceph-volume lvm prepare output") } } + } else { logger.Infof("skipping device %s with osd %d already configured", name, device.Data) } @@ -499,6 +529,25 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid s return osds, nil } +func readCVLogContent(cvLogFilePath string) string { + // Open c-v log file + cvLogFile, err := os.Open(cvLogFilePath) + if err != nil { + logger.Errorf("failed to open ceph-volume log file %q. %v", cvLogFilePath, err) + return "" + } + defer cvLogFile.Close() + + // Read c-v log file + b, err := ioutil.ReadAll(cvLogFile) + if err != nil { + logger.Errorf("failed to read ceph-volume log file %q. %v", cvLogFilePath, err) + return "" + } + + return string(b) +} + type osdInfo struct { Name string `json:"name"` Path string `json:"path"` diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 5523439d2..1f78150cc 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -17,6 +17,9 @@ limitations under the License. package osd import ( + "io/ioutil" + "os" + "path" "testing" "github.com/pkg/errors" @@ -311,3 +314,29 @@ func TestGetDatabaseSize(t *testing.T) { assert.Equal(t, 0, getDatabaseSize(0, 0)) assert.Equal(t, 2048, getDatabaseSize(4096, 2048)) } + +func TestPrintCVLogContent(t *testing.T) { + tmp, err := ioutil.TempFile("", "cv-log") + assert.Nil(t, err) + + defer os.Remove(tmp.Name()) + + nodeName := "set1-2-data-jmxdx" + cvLogDir = path.Join(tmp.Name(), nodeName) + assert.Equal(t, path.Join(tmp.Name(), nodeName), cvLogDir) + + cvLogFilePath := path.Join(cvLogDir, "ceph-volume.log") + assert.Equal(t, path.Join(cvLogDir, "ceph-volume.log"), cvLogFilePath) + + // Print c-v log, it is empty so this is similating a failure (e,g: the file does not exist) + cvLog := readCVLogContent(tmp.Name()) + assert.Empty(t, cvLog, cvLog) + + // Write content in the file + cvDummyLog := []byte(`dummy log`) + _, err = tmp.Write(cvDummyLog) + assert.NoError(t, err) + // Print again, now there is content + cvLog = readCVLogContent(tmp.Name()) + assert.NotEmpty(t, cvLog, cvLog) +}