diff --git a/tests/framework/installer/cockroachdb.go b/tests/framework/installer/cockroachdb.go index af92c9acc..db1e96ed8 100644 --- a/tests/framework/installer/cockroachdb.go +++ b/tests/framework/installer/cockroachdb.go @@ -107,7 +107,7 @@ func (h *InstallHelper) CreateCockroachDBCluster(namespace string, count int) er func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string) { logger.Infof("uninstalling cockroachdb from namespace %s", namespace) - _, err := h.k8shelper.DeleteResource("-n", namespace, "cluster.cockroachdb.rook.io", namespace) + _, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace) h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace)) crdCheckerFunc := func() error { @@ -117,7 +117,7 @@ func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string) err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc) h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) - _, err = h.k8shelper.DeleteResource("namespace", namespace) + _, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace)) logger.Infof("removing the operator from namespace %s", systemNamespace) diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index 6ac973122..21920e065 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -415,7 +415,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err) } - _, err = h.k8shelper.DeleteResource("-n", namespace, "cluster.ceph.rook.io", namespace) + _, err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.ceph.rook.io", namespace) h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace)) crdCheckerFunc := func() error { @@ -425,7 +425,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc) h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace)) - _, err = h.k8shelper.DeleteResource("namespace", namespace) + _, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace) h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace)) } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index a0ec76bf9..2d39cc86a 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -116,7 +116,6 @@ func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { return result, fmt.Errorf("Failed to run: kubectl %v : %v", args, err) } return result, nil - } //KubectlWithStdin is wrapper for executing kubectl commands in stdin @@ -277,7 +276,6 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio } logger.Errorf("Failed to execute kubectl %v %v -- %v", args, podDef, err) return "", fmt.Errorf("Could Not create resource in args : %v %v-- %v", args, podDef, err) - } //ResourceOperation performs a kubectl action on a pod definition @@ -290,18 +288,29 @@ func (k8sh *K8sHelper) ResourceOperation(action string, podDefinition string) (s } logger.Errorf("Failed to execute kubectl %v -- %v", args, err) return "", fmt.Errorf("Could Not create resource in args : %v -- %v", args, err) - } //DeleteResource performs a kubectl delete on give args func (k8sh *K8sHelper) DeleteResource(args ...string) (string, error) { + return k8sh.DeleteResourceAndWait(true, args...) +} + +// DeleteResource performs a kubectl delete on give args. +// If wait is false, a flag will be passed to indicate the delete should return immediately +func (k8sh *K8sHelper) DeleteResourceAndWait(wait bool, args ...string) (string, error) { + if !wait { + // new flag in k8s 1.11 + v := version.MustParseSemantic(k8sh.GetK8sServerVersion()) + if v.AtLeast(version.MustParseSemantic("1.11.0")) { + args = append(args, "--wait=false") + } + } args = append([]string{"delete"}, args...) result, err := k8sh.Kubectl(args...) if err == nil { return result, nil } return "", fmt.Errorf("Could Not delete resource in k8s -- %v", err) - } //GetResource performs a kubectl get on give args @@ -370,28 +379,35 @@ func (k8sh *K8sHelper) IsPodWithLabelPresent(label string, namespace string) boo //WaitForLabeledPodToRun returns true if a Pod is running status or goes to Running status within 90s else returns false func (k8sh *K8sHelper) WaitForLabeledPodToRun(label string, namespace string) error { options := metav1.ListOptions{LabelSelector: label} - inc := 0 - for inc < RetryLoop { + var lastPod v1.Pod + for i := 0; i < RetryLoop; i++ { pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options) + lastStatus := "" if err == nil && len(pods.Items) > 0 { for _, pod := range pods.Items { if pod.Status.Phase == "Running" { return nil } + lastPod = pod + lastStatus = string(pod.Status.Phase) } } - inc++ - logger.Infof("waiting for pod with label %s in namespace %s to be running. err=%+v", label, namespace, err) + logger.Infof("waiting for pod with label %s in namespace %s to be running. status=%s, err=%+v", label, namespace, lastStatus, err) time.Sleep(RetryInterval * time.Second) } + + if len(lastPod.Name) == 0 { + logger.Infof("no pod was found with label %s", label) + } else { + k8sh.PrintPodDescribe(namespace, lastPod.Name) + } return fmt.Errorf("Giving up waiting for pod with label %s in namespace %s to be running", label, namespace) } //WaitUntilPodWithLabelDeleted returns true if a Pod is deleted within 90s else returns false func (k8sh *K8sHelper) WaitUntilPodWithLabelDeleted(label string, namespace string) bool { options := metav1.ListOptions{LabelSelector: label} - inc := 0 - for inc < RetryLoop { + for i := 0; i < RetryLoop; i++ { pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options) if errors.IsNotFound(err) { logger.Infof("error Found err %v", err) @@ -401,7 +417,6 @@ func (k8sh *K8sHelper) WaitUntilPodWithLabelDeleted(label string, namespace stri return true } - inc++ time.Sleep(RetryInterval * time.Second) logger.Infof("waiting for pod with label %s in namespace %s to be deleted", label, namespace) @@ -437,15 +452,14 @@ func (k8sh *K8sHelper) PrintPodDescribeForNamespace(namespace string) { k8sh.PrintEventsForNamespace(namespace) } -func (k8sh *K8sHelper) PrintPodDescribe(name, namespace string) { - pod, err := k8sh.Clientset.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) +func (k8sh *K8sHelper) PrintPodDescribe(namespace string, args ...string) { + args = append([]string{"describe", "pod", "-n", namespace}, args...) + description, err := k8sh.Kubectl(args...) if err != nil { - logger.Warningf("failed to get pod %s in namespace %s. %+v", name, namespace) - return + logger.Errorf("failed to describe pod. %v %+v", args, err) + } else { + logger.Infof("pod description:\n%s\n", description) } - logger.Infof("pod %s in namespace %s: %+v", name, namespace, pod) - - k8sh.PrintEventsForNamespace(namespace) } func (k8sh *K8sHelper) PrintEventsForNamespace(namespace string) { @@ -932,7 +946,7 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m } inc++ - logger.Infof("waiting for %d pods with label app=%s,found %d", minExpected, podName, actualPodCount) + logger.Infof("waiting for %d pods with label app=%s, found %d", minExpected, podName, actualPodCount) time.Sleep(RetryInterval * time.Second) } if !podCountCheck { @@ -940,8 +954,7 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m return false } - inc = 0 - for inc < RetryLoop { + for i := 0; i < RetryLoop; i++ { checkAllPodsStatus := true pl, _ := k8sh.Clientset.CoreV1().Pods(namespace).List(listOpts) for _, pod := range pl.Items { @@ -953,11 +966,11 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m if checkAllPodsStatus { return true } - inc++ time.Sleep(RetryInterval * time.Second) - } + logger.Errorf("All pods with app Name %v not in %v phase ", podName, expectedPhase) + k8sh.PrintPodDescribe(namespace, "-l", listOpts.LabelSelector) return false } @@ -1160,6 +1173,10 @@ func (k8sh *K8sHelper) GetRookLogs(podAppName string, hostType string, namespace return } + if len(podList.Items) == 0 { + logger.Infof("no logs found for pod %s in namespace %s", podAppName, namespace) + } + for _, pod := range podList.Items { podName := pod.Name logger.Infof("getting logs for pod : %v", podName) diff --git a/tests/integration/base_file_test.go b/tests/integration/base_file_test.go index e530dac46..fba30687a 100644 --- a/tests/integration/base_file_test.go +++ b/tests/integration/base_file_test.go @@ -55,7 +55,7 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S require.Nil(s.T(), mtfsErr) filePodRunning := k8sh.IsPodRunning(filePodName, namespace) if !filePodRunning { - k8sh.PrintPodDescribe(filePodName, namespace) + k8sh.PrintPodDescribe(namespace, filePodName) k8sh.PrintPodStatus(namespace) k8sh.PrintPodStatus(installer.SystemNamespace(namespace)) } diff --git a/tests/integration/block_createBlock_k8s_test.go b/tests/integration/block_createBlock_k8s_test.go index c8036d00a..e35ab1281 100644 --- a/tests/integration/block_createBlock_k8s_test.go +++ b/tests/integration/block_createBlock_k8s_test.go @@ -160,20 +160,20 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() { _, err2 := s.kh.Clientset.AppsV1beta1().StatefulSets(defaultNamespace).Create(statefulset) assert.Nil(s.T(), err1) assert.Nil(s.T(), err2) - assert.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running")) - assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 1, "Bound")) + require.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running")) + require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 1, "Bound")) logger.Info("Step 3 : Scale up replication on statefulSet") scaleerr := s.kh.ScaleStatefulSet(statefulPodsName, defaultNamespace, 2) assert.NoError(s.T(), scaleerr, "make sure scale up is successful") - assert.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 2, "Running")) - assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound")) + require.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 2, "Running")) + require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound")) logger.Info("Step 4 : Scale down replication on statefulSet") scaleerr = s.kh.ScaleStatefulSet(statefulPodsName, defaultNamespace, 1) assert.NoError(s.T(), scaleerr, "make sure scale down is successful") - assert.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running")) - assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound")) + require.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running")) + require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound")) logger.Info("Step 5 : Delete statefulSet") delOpts := metav1.DeleteOptions{} @@ -184,8 +184,8 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() { assert.Nil(s.T(), err1) assert.Nil(s.T(), err2) assert.Nil(s.T(), err3) - assert.True(s.T(), s.kh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("app=%s", statefulSetName), defaultNamespace)) - assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound")) + require.True(s.T(), s.kh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("app=%s", statefulSetName), defaultNamespace)) + require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound")) } func (s *BlockCreateSuite) statefulSetDataCleanup(namespace, poolName, storageClassName, statefulSetName, statefulPodsName string) { diff --git a/tests/integration/zblock_mount_unmount_test.go b/tests/integration/zblock_mount_unmount_test.go index 963d087e7..802d96b10 100644 --- a/tests/integration/zblock_mount_unmount_test.go +++ b/tests/integration/zblock_mount_unmount_test.go @@ -113,7 +113,7 @@ func (s *BlockMountUnMountSuite) setupPVCs() { require.Nil(s.T(), err) rwoVolumePresent := s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName) if !rwoVolumePresent { - s.kh.PrintPodDescribe("setup-block-rwo", defaultNamespace) + s.kh.PrintPodDescribe(defaultNamespace, "setup-block-rwo") s.kh.PrintPodStatus(s.namespace) s.kh.PrintPodStatus(installer.SystemNamespace(s.namespace)) }