From 0ebf7a4bb0218eb6b0b8ae62b6c9903fe868ea44 Mon Sep 17 00:00:00 2001 From: Divesh Angula Date: Fri, 10 Nov 2017 12:03:43 -0800 Subject: [PATCH] recover tests from panic and capture logs. --- tests/framework/contracts/operations.go | 4 ++ tests/framework/installer/install_helper.go | 12 ++++ tests/framework/utils/k8s_helper.go | 2 +- tests/integration/base_deploy_test.go | 43 ++++++++++--- .../integration/block_createBlock_api_test.go | 17 +++-- .../integration/block_createBlock_k8s_test.go | 15 +++-- tests/integration/helm_test.go | 15 +++-- .../integration/multi_cluster_deploy_test.go | 64 ++++++++++++++----- tests/integration/smoke_test.go | 17 +++-- 9 files changed, 140 insertions(+), 49 deletions(-) diff --git a/tests/framework/contracts/operations.go b/tests/framework/contracts/operations.go index e3756f981..7602598b8 100644 --- a/tests/framework/contracts/operations.go +++ b/tests/framework/contracts/operations.go @@ -85,3 +85,7 @@ type RestAPIOperator interface { UpdateObjectUser(storeName string, user model.ObjectUser) (*model.ObjectUser, error) DeleteObjectUser(storeName, id string) error } + +type TestOperator interface { + TearDown() +} diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index c14884e76..7bebdc41a 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -307,6 +307,18 @@ func (h *InstallHelper) CleanupCluster(clusterName string) { } } +func (h *InstallHelper) GatherAllRookLogs(nameSpace string, testName string) { + logger.Infof("Gathering all logs from Rook Cluster %s", nameSpace) + h.k8shelper.GetRookLogs("rook-operator", h.Env.HostType, SystemNamespace(nameSpace), testName) + h.k8shelper.GetRookLogs("rook-agent", h.Env.HostType, SystemNamespace(nameSpace), testName) + h.k8shelper.GetRookLogs("rook-api", h.Env.HostType, nameSpace, testName) + h.k8shelper.GetRookLogs("rook-ceph-mgr", h.Env.HostType, nameSpace, testName) + h.k8shelper.GetRookLogs("rook-ceph-mon", h.Env.HostType, nameSpace, testName) + h.k8shelper.GetRookLogs("rook-ceph-osd", h.Env.HostType, nameSpace, testName) + h.k8shelper.GetRookLogs("rook-ceph-rgw", h.Env.HostType, nameSpace, testName) + h.k8shelper.GetRookLogs("rook-ceph-mds", h.Env.HostType, nameSpace, testName) +} + //NewK8sRookhelper creates new instance of InstallHelper func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *InstallHelper { diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index b5d4529b0..1e130814c 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -601,7 +601,7 @@ func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (st return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) } - if podList.Size() < 1 { + if len(podList.Items) < 1 { logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err) } diff --git a/tests/integration/base_deploy_test.go b/tests/integration/base_deploy_test.go index 28e33f070..540e22b59 100644 --- a/tests/integration/base_deploy_test.go +++ b/tests/integration/base_deploy_test.go @@ -23,10 +23,13 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/model" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/contracts" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "testing" ) var ( @@ -73,14 +76,34 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, require.Nil(s.T(), err) } -func gatherAllRookLogs(k8sh *utils.K8sHelper, s suite.Suite, hostType string, opNamespace string, clusterNamespace string) { - logger.Infof("Gathering all logs from Rook Cluster %s", clusterNamespace) - k8sh.GetRookLogs("rook-operator", hostType, opNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-agent", hostType, opNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-api", hostType, clusterNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-ceph-mgr", hostType, clusterNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-ceph-mon", hostType, clusterNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-ceph-osd", hostType, clusterNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-ceph-rgw", hostType, clusterNamespace, s.T().Name()) - k8sh.GetRookLogs("rook-ceph-mds", hostType, clusterNamespace, s.T().Name()) +func HandlePanics(r interface{}, o contracts.TestOperator, t func() *testing.T) { + if r != nil { + logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r) + t().Fail() + o.TearDown() + t().FailNow() + } + +} + +//BaseTestOperations struct for handling panic and test suite tear down +type BaseTestOperations struct { + installer *installer.InstallHelper + T func() *testing.T + namespace string + helmInstalled bool +} + +//NewBaseTestOperations creates new instance of BaseTestOperations struct +func NewBaseTestOperations(i *installer.InstallHelper, t func() *testing.T, namespace string, helmInstalled bool) BaseTestOperations { + return BaseTestOperations{i, t, namespace, helmInstalled} +} + +//TearDown is a wrapper for tearDown after Sutie +func (o BaseTestOperations) TearDown() { + if o.installer.T().Failed() { + o.installer.GatherAllRookLogs(o.namespace, o.installer.T().Name()) + } + o.installer.UninstallRookFromK8s(o.namespace, o.helmInstalled) + } diff --git a/tests/integration/block_createBlock_api_test.go b/tests/integration/block_createBlock_api_test.go index 806e9719b..e69c3b1bc 100644 --- a/tests/integration/block_createBlock_api_test.go +++ b/tests/integration/block_createBlock_api_test.go @@ -37,7 +37,11 @@ var ( ) func TestBlockImageCreateSuite(t *testing.T) { - suite.Run(t, new(BlockImageCreateSuite)) + s := new(BlockImageCreateSuite) + defer func(s *BlockImageCreateSuite) { + HandlePanics(recover(), s.o, s.T) + }(s) + suite.Run(t, s) } type BlockImageCreateSuite struct { @@ -45,9 +49,10 @@ type BlockImageCreateSuite struct { testClient *clients.TestClient kh *utils.K8sHelper rc contracts.RestAPIOperator + installer *installer.InstallHelper + o contracts.TestOperator initBlockCount int namespace string - installer *installer.InstallHelper } func (s *BlockImageCreateSuite) SetupSuite() { @@ -57,10 +62,12 @@ func (s *BlockImageCreateSuite) SetupSuite() { assert.NoError(s.T(), err) s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T) + s.o = NewBaseTestOperations(s.installer, s.T, s.namespace, false) isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1) if !isRookInstalled { logger.Errorf("Rook Was not installed successfully") + s.T().Fail() s.TearDownSuite() s.T().FailNow() } @@ -68,6 +75,7 @@ func (s *BlockImageCreateSuite) SetupSuite() { s.testClient, err = clients.CreateTestClient(s.kh, s.namespace) if err != nil { logger.Errorf("Cannot create rook test client, er -> %v", err) + s.T().Fail() s.TearDownSuite() s.T().FailNow() } @@ -168,9 +176,6 @@ func (s *BlockImageCreateSuite) TearDownTest() { } } func (s *BlockImageCreateSuite) TearDownSuite() { - if s.T().Failed() { - gatherAllRookLogs(s.kh, s.Suite, s.installer.Env.HostType, s.namespace, s.namespace) - } - s.installer.UninstallRookFromK8s(s.namespace, false) + s.o.TearDown() } diff --git a/tests/integration/block_createBlock_k8s_test.go b/tests/integration/block_createBlock_k8s_test.go index a797686c7..88e08f07e 100644 --- a/tests/integration/block_createBlock_k8s_test.go +++ b/tests/integration/block_createBlock_k8s_test.go @@ -32,7 +32,11 @@ import ( // Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes func TestK8sBlockImageCreateSuite(t *testing.T) { - suite.Run(t, new(K8sBlockImageCreateSuite)) + s := new(K8sBlockImageCreateSuite) + defer func(s *K8sBlockImageCreateSuite) { + HandlePanics(recover(), s.o, s.T) + }(s) + suite.Run(t, s) } type K8sBlockImageCreateSuite struct { @@ -43,6 +47,7 @@ type K8sBlockImageCreateSuite struct { initBlockCount int namespace string installer *installer.InstallHelper + o contracts.TestOperator } func (s *K8sBlockImageCreateSuite) SetupSuite() { @@ -53,11 +58,13 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() { assert.NoError(s.T(), err) s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T) + s.o = NewBaseTestOperations(s.installer, s.T, s.namespace, false) isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1) assert.NoError(s.T(), err) if !isRookInstalled { logger.Errorf("Rook Was not installed successfully") + s.T().Fail() s.TearDownSuite() s.T().FailNow() } @@ -65,6 +72,7 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() { s.testClient, err = clients.CreateTestClient(s.kh, s.namespace) if err != nil { logger.Errorf("Cannot create rook test client, er -> %v", err) + s.T().Fail() s.TearDownSuite() s.T().FailNow() } @@ -193,8 +201,5 @@ func (s *K8sBlockImageCreateSuite) tearDownTest(claimName string, poolName strin } func (s *K8sBlockImageCreateSuite) TearDownSuite() { - if s.T().Failed() { - gatherAllRookLogs(s.kh, s.Suite, s.installer.Env.HostType, s.namespace, s.namespace) - } - s.installer.UninstallRookFromK8s(s.namespace, false) + s.o.TearDown() } diff --git a/tests/integration/helm_test.go b/tests/integration/helm_test.go index 48fd85f88..66e407a81 100644 --- a/tests/integration/helm_test.go +++ b/tests/integration/helm_test.go @@ -7,6 +7,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" @@ -34,7 +35,11 @@ var ( // - Create the object store via the CRD // *************************************************** func TestHelmSuite(t *testing.T) { - suite.Run(t, new(HelmSuite)) + s := new(HelmSuite) + defer func(s *HelmSuite) { + HandlePanics(recover(), s.o, s.T) + }(s) + suite.Run(t, s) } type HelmSuite struct { @@ -43,6 +48,7 @@ type HelmSuite struct { k8sh *utils.K8sHelper installer *installer.InstallHelper hh *utils.HelmHelper + o contracts.TestOperator namespace string } @@ -55,6 +61,7 @@ func (hs *HelmSuite) SetupSuite() { hs.hh = utils.NewHelmHelper() hs.installer = installer.NewK8sRookhelper(kh.Clientset, hs.T) + hs.o = NewBaseTestOperations(hs.installer, hs.T, hs.namespace, true) err = hs.installer.CreateK8sRookOperatorViaHelm(hs.namespace) require.NoError(hs.T(), err) @@ -78,11 +85,7 @@ func (hs *HelmSuite) SetupSuite() { } func (hs *HelmSuite) TearDownSuite() { - if hs.T().Failed() { - gatherAllRookLogs(hs.k8sh, hs.Suite, hs.installer.Env.HostType, hs.namespace, hs.namespace) - } - hs.installer.UninstallRookFromK8s(hs.namespace, true) - + hs.o.TearDown() } //Test to make sure all rook components are installed and Running diff --git a/tests/integration/multi_cluster_deploy_test.go b/tests/integration/multi_cluster_deploy_test.go index 84bad8c9e..970f67cb2 100644 --- a/tests/integration/multi_cluster_deploy_test.go +++ b/tests/integration/multi_cluster_deploy_test.go @@ -8,6 +8,7 @@ import ( "github.com/rook/rook/pkg/model" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" @@ -31,7 +32,11 @@ import ( // - Create the object store via the CRD // ************************************************************* func TestMultiClusterDeploySuite(t *testing.T) { - suite.Run(t, new(MultiClusterDeploySuite)) + s := new(MultiClusterDeploySuite) + defer func(s *MultiClusterDeploySuite) { + HandlePanics(recover(), s.o, s.T) + }(s) + suite.Run(t, s) } type MultiClusterDeploySuite struct { @@ -43,6 +48,7 @@ type MultiClusterDeploySuite struct { installData *installer.InstallData namespace1 string namespace2 string + o contracts.TestOperator } //Deploy Multiple Rook clusters @@ -57,6 +63,7 @@ func (mrc *MultiClusterDeploySuite) SetupSuite() { mrc.k8sh = kh mrc.installer = installer.NewK8sRookhelper(kh.Clientset, mrc.T) mrc.installData = installer.NewK8sInstallData() + mrc.o = NewMCTestOperations(mrc.installer, mrc.installData, mrc.k8sh, mrc.T, mrc.namespace1, mrc.namespace2) err = mrc.installer.CreateK8sRookOperator(installer.SystemNamespace(mrc.namespace1)) require.NoError(mrc.T(), err) @@ -113,39 +120,65 @@ func (mrc *MultiClusterDeploySuite) startCluster(namespace, store string, errCh } func (mrc *MultiClusterDeploySuite) TearDownSuite() { - if mrc.T().Failed() { - gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace1) - gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace2) + mrc.o.TearDown() +} + +//MCTestOperations struct for handling panic and test suite tear down +type MCTestOperations struct { + installer *installer.InstallHelper + installData *installer.InstallData + k8sh *utils.K8sHelper + T func() *testing.T + namespace1 string + namespace2 string +} + +//NewMCTestOperations creates new instance of BaseTestOperations struct +func NewMCTestOperations(i *installer.InstallHelper, d *installer.InstallData, k8sh *utils.K8sHelper, t func() *testing.T, namespace1 string, namespace2 string) MCTestOperations { + return MCTestOperations{i, d, k8sh, t, namespace1, namespace2} +} + +//TearDown is a wrapper for tearDown after suite +func (o MCTestOperations) TearDown() { + defer func() { + if r := recover(); r != nil { + logger.Infof("Unexpected Errors while cleaning up MultiCluster test --> %v", r) + o.T().FailNow() + } + }() + if o.T().Failed() { + o.installer.GatherAllRookLogs(o.namespace1, o.T().Name()) + o.installer.GatherAllRookLogs(o.namespace2, o.T().Name()) } deleteArgs := []string{"delete", "-f", "-"} - skipRookInstall := strings.EqualFold(mrc.installer.Env.SkipInstallRook, "true") + skipRookInstall := strings.EqualFold(o.installer.Env.SkipInstallRook, "true") if skipRookInstall { return } logger.Infof("Uninstalling All Rook Clusters") - k8sHelp, err := utils.CreateK8sHelper(mrc.T) + k8sHelp, err := utils.CreateK8sHelper(o.T) if err != nil { panic(err) } - rookOperator := mrc.installData.GetRookOperator(installer.SystemNamespace(mrc.namespace1)) - mrc.k8sh.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-agent", nil) - mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-agent", nil) + rookOperator := o.installData.GetRookOperator(installer.SystemNamespace(o.namespace1)) + o.k8sh.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-agent", nil) + o.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-agent", nil) //Delete rook operator - _, err = mrc.k8sh.KubectlWithStdin(rookOperator, deleteArgs...) + _, err = o.k8sh.KubectlWithStdin(rookOperator, deleteArgs...) if err != nil { logger.Errorf("Rook operator cannot be deleted,err -> %v", err) panic(err) } //delete rook cluster - mrc.installer.CleanupCluster(mrc.namespace1) - mrc.installer.CleanupCluster(mrc.namespace2) + o.installer.CleanupCluster(o.namespace1) + o.installer.CleanupCluster(o.namespace2) // Delete crd/tpr - if mrc.k8sh.VersionAtLeast("v1.7.0") { + if o.k8sh.VersionAtLeast("v1.7.0") { _, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io"}) if err != nil { logger.Errorf("crd cannot be deleted") @@ -159,8 +192,9 @@ func (mrc *MultiClusterDeploySuite) TearDownSuite() { } } - mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) - logger.Infof("Rook clusters %s and %s uninstalled", mrc.namespace1, mrc.namespace2) + o.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) + logger.Infof("Rook clusters %s and %s uninstalled", o.namespace1, o.namespace2) + } //Test to make sure all rook components are installed and Running diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 16ba52d66..680f8218d 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/contracts" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/assert" @@ -52,7 +53,11 @@ import ( // - PUT/GET objects // ************************************************ func TestSmokeSuite(t *testing.T) { - suite.Run(t, new(SmokeSuite)) + s := new(SmokeSuite) + defer func(s *SmokeSuite) { + HandlePanics(recover(), s.o, s.T) + }(s) + suite.Run(t, s) } type SmokeSuite struct { @@ -60,6 +65,7 @@ type SmokeSuite struct { helper *clients.TestClient k8sh *utils.K8sHelper installer *installer.InstallHelper + o contracts.TestOperator namespace string } @@ -71,11 +77,13 @@ func (suite *SmokeSuite) SetupSuite() { suite.k8sh = kh suite.installer = installer.NewK8sRookhelper(kh.Clientset, suite.T) + suite.o = NewBaseTestOperations(suite.installer, suite.T, suite.namespace, false) isRookInstalled, err := suite.installer.InstallRookOnK8sWithHostPathAndDevices(suite.namespace, "bluestore", "", true, 3) assert.NoError(suite.T(), err) if !isRookInstalled { logger.Errorf("Rook Was not installed successfully") + suite.T().Fail() suite.TearDownSuite() suite.T().FailNow() } @@ -83,17 +91,14 @@ func (suite *SmokeSuite) SetupSuite() { suite.helper, err = clients.CreateTestClient(kh, suite.namespace) if err != nil { logger.Errorf("Cannot create rook test client, er -> %v", err) + suite.T().Fail() suite.TearDownSuite() suite.T().FailNow() } } func (suite *SmokeSuite) TearDownSuite() { - if suite.T().Failed() { - gatherAllRookLogs(suite.k8sh, suite.Suite, suite.installer.Env.HostType, installer.SystemNamespace(suite.namespace), suite.namespace) - - } - suite.installer.UninstallRookFromK8s(suite.namespace, false) + suite.o.TearDown() } func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {