Merge pull request #4687 from leseb/cv-log-prepare-per-dev

ceph: expose c-v log per pvc

pkg/daemon/ceph/osd/volume.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -39,7 +40,9 @@ import (
 // These are not constants because they are used by the tests
 var (
 	cephConfigDir = "/var/lib/ceph"
+	cephLogDir    = "/var/log/ceph"
 	lvmConfPath   = "/etc/lvm/lvm.conf"
+	cvLogDir      = ""
 )
 
 const (
@@ -102,7 +105,24 @@ func (a *OsdAgent) configureCVDevices(context *clusterd.Context, devices *Device
 
 func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *DeviceOsdMapping, lvBackedPV bool) (string, error) {
 	baseCommand := "stdbuf"
-	baseArgs := []string{"-oL", cephVolumeCmd, "lvm", "prepare"}
+	var baseArgs []string
+
+	// Create a specific log directory so that each prepare command will have its own log
+	// Only do this if nothing is present so that we don't override existing logs
+	cvLogDir = path.Join(cephLogDir, a.nodeName)
+	err := os.MkdirAll(cvLogDir, 0755)
+	if err != nil {
+		logger.Errorf("failed to create ceph-volume log directory %q, continue with default %q. %v", cvLogDir, cephLogDir, err)
+		baseArgs = []string{"-oL", cephVolumeCmd}
+	} else {
+		baseArgs = []string{"-oL", cephVolumeCmd, "--log-path", cvLogDir}
+	}
+
+	baseArgs = append(baseArgs, []string{
+		"lvm",
+		"prepare",
+	}...)
+
 	var lvpath string
 	for name, device := range devices.Entries {
 		if device.LegacyPartitionsFound {
@@ -127,21 +147,31 @@ func (a *OsdAgent) initializeBlockPVC(context *clusterd.Context, devices *Device
 				"--data",
 				deviceArg,
 			}...)
-			// execute ceph-volume with the device
 
-			if op, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", baseCommand, immediateExecuteArgs...); err != nil {
+			// execute ceph-volume with the device
+			op, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", baseCommand, immediateExecuteArgs...)
+			if err != nil {
+				cvLogFilePath := path.Join(cvLogDir, "ceph-volume.log")
+
+				// Print c-v log before exiting
+				cvLog := readCVLogContent(cvLogFilePath)
+				if cvLog != "" {
+					logger.Errorf("%s", cvLog)
+				}
+
+				// Return failure
 				return "", errors.Wrapf(err, "failed ceph-volume") // fail return here as validation provided by ceph-volume
+			}
+			logger.Infof("%v", op)
+			if lvBackedPV {
+				lvpath = deviceArg
 			} else {
-				logger.Infof("%v", op)
-				if lvBackedPV {
-					lvpath = deviceArg
-				} else {
-					lvpath = getLVPath(op)
-					if lvpath == "" {
-						return "", errors.New("failed to get lvpath from ceph-volume lvm prepare output")
-					}
+				lvpath = getLVPath(op)
+				if lvpath == "" {
+					return "", errors.New("failed to get lvpath from ceph-volume lvm prepare output")
 				}
 			}
+
 		} else {
 			logger.Infof("skipping device %s with osd %d already configured", name, device.Data)
 		}
@@ -499,6 +529,25 @@ func getCephVolumeOSDs(context *clusterd.Context, clusterName string, cephfsid s
 	return osds, nil
 }
 
+func readCVLogContent(cvLogFilePath string) string {
+	// Open c-v log file
+	cvLogFile, err := os.Open(cvLogFilePath)
+	if err != nil {
+		logger.Errorf("failed to open ceph-volume log file %q. %v", cvLogFilePath, err)
+		return ""
+	}
+	defer cvLogFile.Close()
+
+	// Read c-v log file
+	b, err := ioutil.ReadAll(cvLogFile)
+	if err != nil {
+		logger.Errorf("failed to read ceph-volume log file %q. %v", cvLogFilePath, err)
+		return ""
+	}
+
+	return string(b)
+}
+
 type osdInfo struct {
 	Name string  `json:"name"`
 	Path string  `json:"path"`

pkg/daemon/ceph/osd/volume_test.go

@@ -17,6 +17,9 @@ limitations under the License.
 package osd
 
 import (
+	"io/ioutil"
+	"os"
+	"path"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -311,3 +314,29 @@ func TestGetDatabaseSize(t *testing.T) {
 	assert.Equal(t, 0, getDatabaseSize(0, 0))
 	assert.Equal(t, 2048, getDatabaseSize(4096, 2048))
 }
+
+func TestPrintCVLogContent(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "cv-log")
+	assert.Nil(t, err)
+
+	defer os.Remove(tmp.Name())
+
+	nodeName := "set1-2-data-jmxdx"
+	cvLogDir = path.Join(tmp.Name(), nodeName)
+	assert.Equal(t, path.Join(tmp.Name(), nodeName), cvLogDir)
+
+	cvLogFilePath := path.Join(cvLogDir, "ceph-volume.log")
+	assert.Equal(t, path.Join(cvLogDir, "ceph-volume.log"), cvLogFilePath)
+
+	// Print c-v log, it is empty so this is similating a failure (e,g: the file does not exist)
+	cvLog := readCVLogContent(tmp.Name())
+	assert.Empty(t, cvLog, cvLog)
+
+	// Write content in the file
+	cvDummyLog := []byte(`dummy log`)
+	_, err = tmp.Write(cvDummyLog)
+	assert.NoError(t, err)
+	// Print again, now there is content
+	cvLog = readCVLogContent(tmp.Name())
+	assert.NotEmpty(t, cvLog, cvLog)
+}