From aae4d7c7f6fdda7409b92a9c54d7ce9e3d40c4f6 Mon Sep 17 00:00:00 2001 From: travisn Date: Mon, 15 Apr 2019 15:31:32 -0600 Subject: [PATCH] osds: write ceph-volume log entries as they occur When ceph-volume is called, the default python logging will buffer everything until the process exits, which means the pod log doesnt show what is happening until ceph-volume is done. This change will allow logs to be written immediately as tasks are performed inside ceph-volume Signed-off-by: travisn --- pkg/daemon/ceph/osd/daemon.go | 2 +- pkg/daemon/ceph/osd/volume.go | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 45ce54814..beefe0254 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -50,7 +50,7 @@ func StartOSD(context *clusterd.Context, osdType, osdID, osdUUID string, cephArg // activate the osd with ceph-volume storeFlag := "--" + osdType - if err := context.Executor.ExecuteCommand(false, "", "ceph-volume", "lvm", "activate", "--no-systemd", storeFlag, osdID, osdUUID); err != nil { + if err := context.Executor.ExecuteCommand(false, "", "stdbuf", "-oL", "ceph-volume", "lvm", "activate", "--no-systemd", storeFlag, osdID, osdUUID); err != nil { return fmt.Errorf("failed to activate osd. %+v", err) } diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index edef2202f..849c060a1 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -69,7 +69,10 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO storeFlag = "--filestore" } - baseArgs := []string{"lvm", "batch", "--prepare", storeFlag, "--yes"} + // Use stdbuf to capture the python output buffer such that we can write to the pod log as the logging happens + // instead of using the default buffering that will log everything after ceph-volume exits + baseCommand := "stdbuf" + baseArgs := []string{"-oL", cephVolumeCmd, "lvm", "batch", "--prepare", storeFlag, "--yes"} if a.storeConfig.EncryptedDevice { baseArgs = append(baseArgs, encryptedFlag) } @@ -105,7 +108,7 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO sanitizeOSDsPerDevice(device.Config.OSDsPerDevice), }...) - if err := context.Executor.ExecuteCommand(false, "", cephVolumeCmd, immediateExecuteArgs...); err != nil { + if err := context.Executor.ExecuteCommand(false, "", baseCommand, immediateExecuteArgs...); err != nil { return fmt.Errorf("failed ceph-volume. %+v", err) } @@ -116,7 +119,8 @@ func (a *OsdAgent) initializeDevices(context *clusterd.Context, devices *DeviceO } if configured > 0 { - if err := context.Executor.ExecuteCommand(false, "", cephVolumeCmd, batchArgs...); err != nil { + // Run "stdbuf -oL ceph-volume" so we can get more frequent updates in the container logs + if err := context.Executor.ExecuteCommand(false, "", baseCommand, batchArgs...); err != nil { return fmt.Errorf("failed ceph-volume. %+v", err) } }