From 51cc7c2e8587026e1d07bfb05d72b1871d8f99ca Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 27 Nov 2018 10:44:03 -0700 Subject: [PATCH] Ceph integration tests: add more verbose logging Signed-off-by: Blaine Gardner --- pkg/daemon/ceph/osd/agent.go | 4 ++-- pkg/daemon/ceph/osd/agent_test.go | 24 ++++++++++----------- pkg/daemon/ceph/osd/daemon.go | 4 ++-- pkg/daemon/ceph/osd/device.go | 2 +- tests/framework/installer/ceph_installer.go | 6 ++++++ tests/framework/utils/k8s_helper.go | 3 +++ 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/pkg/daemon/ceph/osd/agent.go b/pkg/daemon/ceph/osd/agent.go index 62642f76f..fe1495e47 100644 --- a/pkg/daemon/ceph/osd/agent.go +++ b/pkg/daemon/ceph/osd/agent.go @@ -139,7 +139,7 @@ func (a *OsdAgent) removeDirs(context *clusterd.Context, removedDirs map[string] if len(failedDirs) > 0 { // at least one OSD failed, return an overall error - return fmt.Errorf("failed to cleanup directories: %v", failedDirs) + return fmt.Errorf("failed to cleanup directories: %+v", failedDirs) } return nil @@ -478,7 +478,7 @@ func (a *OsdAgent) prepareOSD(context *clusterd.Context, cfg *osdConfig) (*oposd } } osdInfo := getOSDInfo(a.cluster.Name, cfg, devPartInfo) - logger.Infof("completed preparing osd %v", osdInfo) + logger.Infof("completed preparing osd %+v", osdInfo) if devPartInfo != nil { sys.UnmountDevice(devPartInfo.pathToUnmount, context.Executor) diff --git a/pkg/daemon/ceph/osd/agent_test.go b/pkg/daemon/ceph/osd/agent_test.go index 30b8849e5..936a4f29a 100644 --- a/pkg/daemon/ceph/osd/agent_test.go +++ b/pkg/daemon/ceph/osd/agent_test.go @@ -285,7 +285,7 @@ func TestOSDAgentNoDevices(t *testing.T) { startCount := 0 executor.MockStartExecuteCommand = func(debug bool, name string, command string, args ...string) (*exec.Cmd, error) { - logger.Infof("StartExecuteCommand: %s %v", command, args) + logger.Infof("StartExecuteCommand: %s %+v", command, args) startCount++ cmd := &exec.Cmd{Args: append([]string{command}, args...)} return cmd, nil @@ -293,7 +293,7 @@ func TestOSDAgentNoDevices(t *testing.T) { runCount := 0 executor.MockExecuteCommand = func(debug bool, name string, command string, args ...string) error { - logger.Infof("ExecuteCommand: %s %v", command, args) + logger.Infof("ExecuteCommand: %s %+v", command, args) runCount++ createTestKeyring(t, configDir, args) return nil @@ -301,14 +301,14 @@ func TestOSDAgentNoDevices(t *testing.T) { execWithOutputFileCount := 0 executor.MockExecuteCommandWithOutputFile = func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { - logger.Infof("ExecuteCommandWithOutputFile: %s %v", command, args) + logger.Infof("ExecuteCommandWithOutputFile: %s %+v", command, args) execWithOutputFileCount++ return "{\"key\":\"mysecurekey\", \"osdid\":3.0}", nil } execWithOutputCount := 0 executor.MockExecuteCommandWithOutput = func(debug bool, actionName string, command string, arg ...string) (string, error) { - logger.Infof("ExecuteCommandWithOutput: %s %v", command, arg) + logger.Infof("ExecuteCommandWithOutput: %s %+v", command, arg) execWithOutputCount++ return "", nil } @@ -344,7 +344,7 @@ func TestRemoveDevices(t *testing.T) { osdUsageCallCount := 0 mockExec.MockExecuteCommandWithOutputFile = func(debug bool, actionName, command, outputFile string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) + logger.Infof("Command: %s %+v", command, args) if args[0] == "osd" && args[1] == "df" { osdUsageCallCount++ if osdUsageCallCount == 1 { @@ -439,10 +439,10 @@ func TestGetPartitionPerfScheme(t *testing.T) { currOsdID++ return fmt.Sprintf(`{"osdid": %d}`, currOsdID), nil } - return "", fmt.Errorf("unexpected command '%v'", args) + return "", fmt.Errorf("unexpected command '%+v'", args) }, MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) + logger.Infof("Command: %s %+v", command, args) if command == "lsblk" { if args[0] == "/dev/sda" { return `NAME="sda" SIZE="107374182400" TYPE="disk" PKNAME=""`, nil @@ -460,7 +460,7 @@ func TestGetPartitionPerfScheme(t *testing.T) { if command == "udevadm" { return "", nil } - return "", fmt.Errorf("unexpected command %s %v", command, args) + return "", fmt.Errorf("unexpected command %s %+v", command, args) }, } context.Executor = executor @@ -510,7 +510,7 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) { executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) + logger.Infof("Command: %s %+v", command, args) if command == "lsblk" { if args[0] == "/dev/sda" { return `NAME="sda" SIZE="20971520000" TYPE="disk" PKNAME="" @@ -524,7 +524,7 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) { if command == "udevadm" { return "", nil } - return "", fmt.Errorf("unexpected command %s %v", command, args) + return "", fmt.Errorf("unexpected command %s %+v", command, args) }, } context := &clusterd.Context{ @@ -569,7 +569,7 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) { executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) + logger.Infof("Command: %s %+v", command, args) if command == "lsblk" { if args[0] == "/dev/sda-changed" { return `NAME="sda" SIZE="20971520000" TYPE="disk" PKNAME="" @@ -588,7 +588,7 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) { if command == "udevadm" { return "", nil } - return "", fmt.Errorf("unexpected command %s %v", command, args) + return "", fmt.Errorf("unexpected command %s %+v", command, args) }, } context := &clusterd.Context{ diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index e7fbd7503..b7cf2d959 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -142,7 +142,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error { logger.Infof("configuring osd dirs: %+v", dirs) dirOSDs, err := agent.configureDirs(context, dirs) if err != nil { - return fmt.Errorf("failed to configure dirs %v. %+v", dirs, err) + return fmt.Errorf("failed to configure dirs %+v. %+v", dirs, err) } // now we can start removing OSDs from devices and directories @@ -161,7 +161,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent) error { return fmt.Errorf("failed to save osd dir map. %+v", err) } - logger.Infof("device osds:%v\ndir osds: %v", deviceOSDs, dirOSDs) + logger.Infof("device osds:%+v\ndir osds: %+v", deviceOSDs, dirOSDs) osds := append(deviceOSDs, dirOSDs...) // orchestration is completed, update the status diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 17b305d68..ac9a758f5 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -139,7 +139,7 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool if !dangerousToFormat || forceFormat { devPartInfo, err = partitionOSD(context, config) if err != nil { - return nil, fmt.Errorf("failed to partion device %s. %v", dataDetails.Device, err) + return nil, fmt.Errorf("failed to partion device %s. %+v", dataDetails.Device, err) } } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 376726ca0..5e6575040 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -350,6 +350,12 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNa logger.Infof("Uninstalling Rook") var err error for _, namespace := range namespaces { + if h.T().Failed() { + // When the test has failed, it's sometimes useful to have pod descriptions to check + // that pods are configured as expected. + h.k8shelper.PrintPodDescribeForNamespace(namespace) + } + roles := h.Manifests.GetClusterRoles(namespace, systemNamespace) _, err = h.k8shelper.KubectlWithStdin(roles, deleteFromStdinArgs...) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index e6f2d8b8b..c84872f95 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -321,6 +321,7 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio // ResourceOperation performs a kubectl action on a pod definition func (k8sh *K8sHelper) ResourceOperation(action string, manifest string) error { args := []string{action, "-f", "-"} + logger.Infof("kubectl create manifest:\n%s", manifest) _, err := k8sh.KubectlWithStdin(manifest, args...) if err == nil { return nil @@ -1144,6 +1145,7 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m } actualPodCount = len(podList.Items) if actualPodCount >= minExpected { + logger.Infof("%d of %d pods with label app=%s were found", actualPodCount, minExpected, podName) podCountCheck = true break } @@ -1221,6 +1223,7 @@ func (k8sh *K8sHelper) WaitUntilPVCIsBound(namespace string, pvcname string) boo out, err := k8sh.GetPVCStatus(namespace, pvcname) if err == nil { if out == v1.PersistentVolumeClaimPhase(v1.ClaimBound) { + logger.Infof("PVC %s is bound", pvcname) return true } }