feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #1983 from travisn/test-delete-resources

Browse Source 
Continue with tests while namespace and crds are being deleted
This commit is contained in:
Alexander Trost authored and GitHub committed 2018-08-07 12:33:45 +02:00
2 parents 6dbae677f0 4f9286f227
commit b33e64dfce
6 files changed
+54 -37
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/framework/installer/cockroachdb.go
+2 -2
View File
@@ -107,7 +107,7 @@ func (h *InstallHelper) CreateCockroachDBCluster(namespace string, count int) er
func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string) {
logger.Infof("uninstalling cockroachdb from namespace %s", namespace)
_, err := h.k8shelper.DeleteResource("-n", namespace, "cluster.cockroachdb.rook.io", namespace)
_, err := h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.cockroachdb.rook.io", namespace)
h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace))
crdCheckerFunc := func() error {
@@ -117,7 +117,7 @@ func (h *InstallHelper) UninstallCockroachDB(systemNamespace, namespace string)
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
_, err = h.k8shelper.DeleteResource("namespace", namespace)
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
logger.Infof("removing the operator from namespace %s", systemNamespace)
tests/framework/installer/install_helper.go
+2 -2
View File
@@ -415,7 +415,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err)
}
_, err = h.k8shelper.DeleteResource("-n", namespace, "cluster.ceph.rook.io", namespace)
_, err = h.k8shelper.DeleteResourceAndWait(false, "-n", namespace, "cluster.ceph.rook.io", namespace)
h.checkError(err, fmt.Sprintf("cannot remove cluster %s", namespace))
crdCheckerFunc := func() error {
@@ -425,7 +425,7 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
err = h.waitForCustomResourceDeletion(namespace, crdCheckerFunc)
h.checkError(err, fmt.Sprintf("failed to wait for crd %s deletion", namespace))
_, err = h.k8shelper.DeleteResource("namespace", namespace)
_, err = h.k8shelper.DeleteResourceAndWait(false, "namespace", namespace)
h.checkError(err, fmt.Sprintf("cannot delete namespace %s", namespace))
}
tests/framework/utils/k8s_helper.go
+40 -23
View File
@@ -116,7 +116,6 @@ func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) {
return result, fmt.Errorf("Failed to run: kubectl %v : %v", args, err)
}
return result, nil
}
//KubectlWithStdin is wrapper for executing kubectl commands in stdin
@@ -277,7 +276,6 @@ func (k8sh *K8sHelper) ResourceOperationFromTemplate(action string, podDefinitio
}
logger.Errorf("Failed to execute kubectl %v %v -- %v", args, podDef, err)
return "", fmt.Errorf("Could Not create resource in args : %v %v-- %v", args, podDef, err)
}
//ResourceOperation performs a kubectl action on a pod definition
@@ -290,18 +288,29 @@ func (k8sh *K8sHelper) ResourceOperation(action string, podDefinition string) (s
}
logger.Errorf("Failed to execute kubectl %v -- %v", args, err)
return "", fmt.Errorf("Could Not create resource in args : %v -- %v", args, err)
}
//DeleteResource performs a kubectl delete on give args
func (k8sh *K8sHelper) DeleteResource(args ...string) (string, error) {
return k8sh.DeleteResourceAndWait(true, args...)
}
// DeleteResource performs a kubectl delete on give args.
// If wait is false, a flag will be passed to indicate the delete should return immediately
func (k8sh *K8sHelper) DeleteResourceAndWait(wait bool, args ...string) (string, error) {
if !wait {
// new flag in k8s 1.11
v := version.MustParseSemantic(k8sh.GetK8sServerVersion())
if v.AtLeast(version.MustParseSemantic("1.11.0")) {
args = append(args, "--wait=false")
}
}
args = append([]string{"delete"}, args...)
result, err := k8sh.Kubectl(args...)
if err == nil {
return result, nil
}
return "", fmt.Errorf("Could Not delete resource in k8s -- %v", err)
}
//GetResource performs a kubectl get on give args
@@ -370,28 +379,35 @@ func (k8sh *K8sHelper) IsPodWithLabelPresent(label string, namespace string) boo
//WaitForLabeledPodToRun returns true if a Pod is running status or goes to Running status within 90s else returns false
func (k8sh *K8sHelper) WaitForLabeledPodToRun(label string, namespace string) error {
options := metav1.ListOptions{LabelSelector: label}
inc := 0
for inc < RetryLoop {
var lastPod v1.Pod
for i := 0; i < RetryLoop; i++ {
pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options)
lastStatus := ""
if err == nil && len(pods.Items) > 0 {
for _, pod := range pods.Items {
if pod.Status.Phase == "Running" {
return nil
}
lastPod = pod
lastStatus = string(pod.Status.Phase)
}
}
inc++
logger.Infof("waiting for pod with label %s in namespace %s to be running. err=%+v", label, namespace, err)
logger.Infof("waiting for pod with label %s in namespace %s to be running. status=%s, err=%+v", label, namespace, lastStatus, err)
time.Sleep(RetryInterval * time.Second)
}
if len(lastPod.Name) == 0 {
logger.Infof("no pod was found with label %s", label)
} else {
k8sh.PrintPodDescribe(namespace, lastPod.Name)
}
return fmt.Errorf("Giving up waiting for pod with label %s in namespace %s to be running", label, namespace)
}
//WaitUntilPodWithLabelDeleted returns true if a Pod is deleted within 90s else returns false
func (k8sh *K8sHelper) WaitUntilPodWithLabelDeleted(label string, namespace string) bool {
options := metav1.ListOptions{LabelSelector: label}
inc := 0
for inc < RetryLoop {
for i := 0; i < RetryLoop; i++ {
pods, err := k8sh.Clientset.CoreV1().Pods(namespace).List(options)
if errors.IsNotFound(err) {
logger.Infof("error Found err %v", err)
@@ -401,7 +417,6 @@ func (k8sh *K8sHelper) WaitUntilPodWithLabelDeleted(label string, namespace stri
return true
}
inc++
time.Sleep(RetryInterval * time.Second)
logger.Infof("waiting for pod with label %s in namespace %s to be deleted", label, namespace)
@@ -437,15 +452,14 @@ func (k8sh *K8sHelper) PrintPodDescribeForNamespace(namespace string) {
k8sh.PrintEventsForNamespace(namespace)
}
func (k8sh *K8sHelper) PrintPodDescribe(name, namespace string) {
pod, err := k8sh.Clientset.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
func (k8sh *K8sHelper) PrintPodDescribe(namespace string, args ...string) {
args = append([]string{"describe", "pod", "-n", namespace}, args...)
description, err := k8sh.Kubectl(args...)
if err != nil {
logger.Warningf("failed to get pod %s in namespace %s. %+v", name, namespace)
return
logger.Errorf("failed to describe pod. %v %+v", args, err)
} else {
logger.Infof("pod description:\n%s\n", description)
}
logger.Infof("pod %s in namespace %s: %+v", name, namespace, pod)
k8sh.PrintEventsForNamespace(namespace)
}
func (k8sh *K8sHelper) PrintEventsForNamespace(namespace string) {
@@ -932,7 +946,7 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m
}
inc++
logger.Infof("waiting for %d pods with label app=%s,found %d", minExpected, podName, actualPodCount)
logger.Infof("waiting for %d pods with label app=%s, found %d", minExpected, podName, actualPodCount)
time.Sleep(RetryInterval * time.Second)
}
if !podCountCheck {
@@ -940,8 +954,7 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m
return false
}
inc = 0
for inc < RetryLoop {
for i := 0; i < RetryLoop; i++ {
checkAllPodsStatus := true
pl, _ := k8sh.Clientset.CoreV1().Pods(namespace).List(listOpts)
for _, pod := range pl.Items {
@@ -953,11 +966,11 @@ func (k8sh *K8sHelper) CheckPodCountAndState(podName string, namespace string, m
if checkAllPodsStatus {
return true
}
inc++
time.Sleep(RetryInterval * time.Second)
}
logger.Errorf("All pods with app Name %v not in %v phase ", podName, expectedPhase)
k8sh.PrintPodDescribe(namespace, "-l", listOpts.LabelSelector)
return false
}
@@ -1160,6 +1173,10 @@ func (k8sh *K8sHelper) GetRookLogs(podAppName string, hostType string, namespace
return
}
if len(podList.Items) == 0 {
logger.Infof("no logs found for pod %s in namespace %s", podAppName, namespace)
}
for _, pod := range podList.Items {
podName := pod.Name
logger.Infof("getting logs for pod : %v", podName)
tests/integration/base_file_test.go
+1 -1
View File
@@ -55,7 +55,7 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S
require.Nil(s.T(), mtfsErr)
filePodRunning := k8sh.IsPodRunning(filePodName, namespace)
if !filePodRunning {
k8sh.PrintPodDescribe(filePodName, namespace)
k8sh.PrintPodDescribe(namespace, filePodName)
k8sh.PrintPodStatus(namespace)
k8sh.PrintPodStatus(installer.SystemNamespace(namespace))
}
tests/integration/block_createBlock_k8s_test.go
+8 -8
View File
@@ -160,20 +160,20 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() {
_, err2 := s.kh.Clientset.AppsV1beta1().StatefulSets(defaultNamespace).Create(statefulset)
assert.Nil(s.T(), err1)
assert.Nil(s.T(), err2)
assert.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running"))
assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 1, "Bound"))
require.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running"))
require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 1, "Bound"))
logger.Info("Step 3 : Scale up replication on statefulSet")
scaleerr := s.kh.ScaleStatefulSet(statefulPodsName, defaultNamespace, 2)
assert.NoError(s.T(), scaleerr, "make sure scale up is successful")
assert.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 2, "Running"))
assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound"))
require.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 2, "Running"))
require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound"))
logger.Info("Step 4 : Scale down replication on statefulSet")
scaleerr = s.kh.ScaleStatefulSet(statefulPodsName, defaultNamespace, 1)
assert.NoError(s.T(), scaleerr, "make sure scale down is successful")
assert.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running"))
assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound"))
require.True(s.T(), s.kh.CheckPodCountAndState(statefulSetName, defaultNamespace, 1, "Running"))
require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound"))
logger.Info("Step 5 : Delete statefulSet")
delOpts := metav1.DeleteOptions{}
@@ -184,8 +184,8 @@ func (s *BlockCreateSuite) TestBlockStorageMountUnMountForStatefulSets() {
assert.Nil(s.T(), err1)
assert.Nil(s.T(), err2)
assert.Nil(s.T(), err3)
assert.True(s.T(), s.kh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("app=%s", statefulSetName), defaultNamespace))
assert.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound"))
require.True(s.T(), s.kh.WaitUntilPodWithLabelDeleted(fmt.Sprintf("app=%s", statefulSetName), defaultNamespace))
require.True(s.T(), s.kh.CheckPvcCountAndStatus(statefulSetName, defaultNamespace, 2, "Bound"))
}
func (s *BlockCreateSuite) statefulSetDataCleanup(namespace, poolName, storageClassName, statefulSetName, statefulPodsName string) {
tests/integration/zblock_mount_unmount_test.go
+1 -1
View File
@@ -113,7 +113,7 @@ func (s *BlockMountUnMountSuite) setupPVCs() {
require.Nil(s.T(), err)
rwoVolumePresent := s.kh.IsVolumeResourcePresent(installer.SystemNamespace(s.namespace), crdName)
if !rwoVolumePresent {
s.kh.PrintPodDescribe("setup-block-rwo", defaultNamespace)
s.kh.PrintPodDescribe(defaultNamespace, "setup-block-rwo")
s.kh.PrintPodStatus(s.namespace)
s.kh.PrintPodStatus(installer.SystemNamespace(s.namespace))
}