Merge pull request #1223 from dangula/pr_recoverTestAfterPanic

recover tests from panic and capture logs.
This commit is contained in:
Jared Watts authored and GitHub committed 2017-11-20 15:44:55 -08:00
commit df22444045
9 files changed
+140 -49

No files matched your search

+4
View File
@@ -86,3 +86,7 @@ type RestAPIOperator interface {
UpdateObjectUser(storeName string, user model.ObjectUser) (*model.ObjectUser, error)
DeleteObjectUser(storeName, id string) error
}
type TestOperator interface {
TearDown()
}
@@ -307,6 +307,18 @@ func (h *InstallHelper) CleanupCluster(clusterName string) {
}
}
func (h *InstallHelper) GatherAllRookLogs(nameSpace string, testName string) {
logger.Infof("Gathering all logs from Rook Cluster %s", nameSpace)
h.k8shelper.GetRookLogs("rook-operator", h.Env.HostType, SystemNamespace(nameSpace), testName)
h.k8shelper.GetRookLogs("rook-agent", h.Env.HostType, SystemNamespace(nameSpace), testName)
h.k8shelper.GetRookLogs("rook-api", h.Env.HostType, nameSpace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mgr", h.Env.HostType, nameSpace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mon", h.Env.HostType, nameSpace, testName)
h.k8shelper.GetRookLogs("rook-ceph-osd", h.Env.HostType, nameSpace, testName)
h.k8shelper.GetRookLogs("rook-ceph-rgw", h.Env.HostType, nameSpace, testName)
h.k8shelper.GetRookLogs("rook-ceph-mds", h.Env.HostType, nameSpace, testName)
}
//NewK8sRookhelper creates new instance of InstallHelper
func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *InstallHelper {
+1 -1
View File
@@ -601,7 +601,7 @@ func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (st
return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
}
if podList.Size() < 1 {
if len(podList.Items) < 1 {
logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
}
+33 -10
View File
@@ -23,10 +23,13 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"testing"
)
var (
@@ -73,14 +76,34 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
require.Nil(s.T(), err)
}
func gatherAllRookLogs(k8sh *utils.K8sHelper, s suite.Suite, hostType string, opNamespace string, clusterNamespace string) {
logger.Infof("Gathering all logs from Rook Cluster %s", clusterNamespace)
k8sh.GetRookLogs("rook-operator", hostType, opNamespace, s.T().Name())
k8sh.GetRookLogs("rook-agent", hostType, opNamespace, s.T().Name())
k8sh.GetRookLogs("rook-api", hostType, clusterNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-mgr", hostType, clusterNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-mon", hostType, clusterNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-osd", hostType, clusterNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-rgw", hostType, clusterNamespace, s.T().Name())
k8sh.GetRookLogs("rook-ceph-mds", hostType, clusterNamespace, s.T().Name())
func HandlePanics(r interface{}, o contracts.TestOperator, t func() *testing.T) {
if r != nil {
logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)
t().Fail()
o.TearDown()
t().FailNow()
}
}
//BaseTestOperations struct for handling panic and test suite tear down
type BaseTestOperations struct {
installer *installer.InstallHelper
T func() *testing.T
namespace string
helmInstalled bool
}
//NewBaseTestOperations creates new instance of BaseTestOperations struct
func NewBaseTestOperations(i *installer.InstallHelper, t func() *testing.T, namespace string, helmInstalled bool) BaseTestOperations {
return BaseTestOperations{i, t, namespace, helmInstalled}
}
//TearDown is a wrapper for tearDown after Sutie
func (o BaseTestOperations) TearDown() {
if o.installer.T().Failed() {
o.installer.GatherAllRookLogs(o.namespace, o.installer.T().Name())
}
o.installer.UninstallRookFromK8s(o.namespace, o.helmInstalled)
}
@@ -37,7 +37,11 @@ var (
)
func TestBlockImageCreateSuite(t *testing.T) {
suite.Run(t, new(BlockImageCreateSuite))
s := new(BlockImageCreateSuite)
defer func(s *BlockImageCreateSuite) {
HandlePanics(recover(), s.o, s.T)
}(s)
suite.Run(t, s)
}
type BlockImageCreateSuite struct {
@@ -45,9 +49,10 @@ type BlockImageCreateSuite struct {
testClient *clients.TestClient
kh *utils.K8sHelper
rc contracts.RestAPIOperator
installer *installer.InstallHelper
o contracts.TestOperator
initBlockCount int
namespace string
installer *installer.InstallHelper
}
func (s *BlockImageCreateSuite) SetupSuite() {
@@ -57,10 +62,12 @@ func (s *BlockImageCreateSuite) SetupSuite() {
assert.NoError(s.T(), err)
s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T)
s.o = NewBaseTestOperations(s.installer, s.T, s.namespace, false)
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1)
if !isRookInstalled {
logger.Errorf("Rook Was not installed successfully")
s.T().Fail()
s.TearDownSuite()
s.T().FailNow()
}
@@ -68,6 +75,7 @@ func (s *BlockImageCreateSuite) SetupSuite() {
s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
if err != nil {
logger.Errorf("Cannot create rook test client, er -> %v", err)
s.T().Fail()
s.TearDownSuite()
s.T().FailNow()
}
@@ -168,9 +176,6 @@ func (s *BlockImageCreateSuite) TearDownTest() {
}
}
func (s *BlockImageCreateSuite) TearDownSuite() {
if s.T().Failed() {
gatherAllRookLogs(s.kh, s.Suite, s.installer.Env.HostType, s.namespace, s.namespace)
}
s.installer.UninstallRookFromK8s(s.namespace, false)
s.o.TearDown()
}
@@ -32,7 +32,11 @@ import (
// Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes
func TestK8sBlockImageCreateSuite(t *testing.T) {
suite.Run(t, new(K8sBlockImageCreateSuite))
s := new(K8sBlockImageCreateSuite)
defer func(s *K8sBlockImageCreateSuite) {
HandlePanics(recover(), s.o, s.T)
}(s)
suite.Run(t, s)
}
type K8sBlockImageCreateSuite struct {
@@ -43,6 +47,7 @@ type K8sBlockImageCreateSuite struct {
initBlockCount int
namespace string
installer *installer.InstallHelper
o contracts.TestOperator
}
func (s *K8sBlockImageCreateSuite) SetupSuite() {
@@ -53,11 +58,13 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() {
assert.NoError(s.T(), err)
s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T)
s.o = NewBaseTestOperations(s.installer, s.T, s.namespace, false)
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1)
assert.NoError(s.T(), err)
if !isRookInstalled {
logger.Errorf("Rook Was not installed successfully")
s.T().Fail()
s.TearDownSuite()
s.T().FailNow()
}
@@ -65,6 +72,7 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() {
s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
if err != nil {
logger.Errorf("Cannot create rook test client, er -> %v", err)
s.T().Fail()
s.TearDownSuite()
s.T().FailNow()
}
@@ -193,8 +201,5 @@ func (s *K8sBlockImageCreateSuite) tearDownTest(claimName string, poolName strin
}
func (s *K8sBlockImageCreateSuite) TearDownSuite() {
if s.T().Failed() {
gatherAllRookLogs(s.kh, s.Suite, s.installer.Env.HostType, s.namespace, s.namespace)
}
s.installer.UninstallRookFromK8s(s.namespace, false)
s.o.TearDown()
}
+9 -6
View File
@@ -7,6 +7,7 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
@@ -34,7 +35,11 @@ var (
// - Create the object store via the CRD
// ***************************************************
func TestHelmSuite(t *testing.T) {
suite.Run(t, new(HelmSuite))
s := new(HelmSuite)
defer func(s *HelmSuite) {
HandlePanics(recover(), s.o, s.T)
}(s)
suite.Run(t, s)
}
type HelmSuite struct {
@@ -43,6 +48,7 @@ type HelmSuite struct {
k8sh *utils.K8sHelper
installer *installer.InstallHelper
hh *utils.HelmHelper
o contracts.TestOperator
namespace string
}
@@ -55,6 +61,7 @@ func (hs *HelmSuite) SetupSuite() {
hs.hh = utils.NewHelmHelper()
hs.installer = installer.NewK8sRookhelper(kh.Clientset, hs.T)
hs.o = NewBaseTestOperations(hs.installer, hs.T, hs.namespace, true)
err = hs.installer.CreateK8sRookOperatorViaHelm(hs.namespace)
require.NoError(hs.T(), err)
@@ -78,11 +85,7 @@ func (hs *HelmSuite) SetupSuite() {
}
func (hs *HelmSuite) TearDownSuite() {
if hs.T().Failed() {
gatherAllRookLogs(hs.k8sh, hs.Suite, hs.installer.Env.HostType, hs.namespace, hs.namespace)
}
hs.installer.UninstallRookFromK8s(hs.namespace, true)
hs.o.TearDown()
}
//Test to make sure all rook components are installed and Running
+49 -15
View File
@@ -8,6 +8,7 @@ import (
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
@@ -31,7 +32,11 @@ import (
// - Create the object store via the CRD
// *************************************************************
func TestMultiClusterDeploySuite(t *testing.T) {
suite.Run(t, new(MultiClusterDeploySuite))
s := new(MultiClusterDeploySuite)
defer func(s *MultiClusterDeploySuite) {
HandlePanics(recover(), s.o, s.T)
}(s)
suite.Run(t, s)
}
type MultiClusterDeploySuite struct {
@@ -43,6 +48,7 @@ type MultiClusterDeploySuite struct {
installData *installer.InstallData
namespace1 string
namespace2 string
o contracts.TestOperator
}
//Deploy Multiple Rook clusters
@@ -57,6 +63,7 @@ func (mrc *MultiClusterDeploySuite) SetupSuite() {
mrc.k8sh = kh
mrc.installer = installer.NewK8sRookhelper(kh.Clientset, mrc.T)
mrc.installData = installer.NewK8sInstallData()
mrc.o = NewMCTestOperations(mrc.installer, mrc.installData, mrc.k8sh, mrc.T, mrc.namespace1, mrc.namespace2)
err = mrc.installer.CreateK8sRookOperator(installer.SystemNamespace(mrc.namespace1))
require.NoError(mrc.T(), err)
@@ -113,39 +120,65 @@ func (mrc *MultiClusterDeploySuite) startCluster(namespace, store string, errCh
}
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
if mrc.T().Failed() {
gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace1)
gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace2)
mrc.o.TearDown()
}
//MCTestOperations struct for handling panic and test suite tear down
type MCTestOperations struct {
installer *installer.InstallHelper
installData *installer.InstallData
k8sh *utils.K8sHelper
T func() *testing.T
namespace1 string
namespace2 string
}
//NewMCTestOperations creates new instance of BaseTestOperations struct
func NewMCTestOperations(i *installer.InstallHelper, d *installer.InstallData, k8sh *utils.K8sHelper, t func() *testing.T, namespace1 string, namespace2 string) MCTestOperations {
return MCTestOperations{i, d, k8sh, t, namespace1, namespace2}
}
//TearDown is a wrapper for tearDown after suite
func (o MCTestOperations) TearDown() {
defer func() {
if r := recover(); r != nil {
logger.Infof("Unexpected Errors while cleaning up MultiCluster test --> %v", r)
o.T().FailNow()
}
}()
if o.T().Failed() {
o.installer.GatherAllRookLogs(o.namespace1, o.T().Name())
o.installer.GatherAllRookLogs(o.namespace2, o.T().Name())
}
deleteArgs := []string{"delete", "-f", "-"}
skipRookInstall := strings.EqualFold(mrc.installer.Env.SkipInstallRook, "true")
skipRookInstall := strings.EqualFold(o.installer.Env.SkipInstallRook, "true")
if skipRookInstall {
return
}
logger.Infof("Uninstalling All Rook Clusters")
k8sHelp, err := utils.CreateK8sHelper(mrc.T)
k8sHelp, err := utils.CreateK8sHelper(o.T)
if err != nil {
panic(err)
}
rookOperator := mrc.installData.GetRookOperator(installer.SystemNamespace(mrc.namespace1))
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-agent", nil)
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-agent", nil)
rookOperator := o.installData.GetRookOperator(installer.SystemNamespace(o.namespace1))
o.k8sh.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-agent", nil)
o.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-agent", nil)
//Delete rook operator
_, err = mrc.k8sh.KubectlWithStdin(rookOperator, deleteArgs...)
_, err = o.k8sh.KubectlWithStdin(rookOperator, deleteArgs...)
if err != nil {
logger.Errorf("Rook operator cannot be deleted,err -> %v", err)
panic(err)
}
//delete rook cluster
mrc.installer.CleanupCluster(mrc.namespace1)
mrc.installer.CleanupCluster(mrc.namespace2)
o.installer.CleanupCluster(o.namespace1)
o.installer.CleanupCluster(o.namespace2)
// Delete crd/tpr
if mrc.k8sh.VersionAtLeast("v1.7.0") {
if o.k8sh.VersionAtLeast("v1.7.0") {
_, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io"})
if err != nil {
logger.Errorf("crd cannot be deleted")
@@ -159,8 +192,9 @@ func (mrc *MultiClusterDeploySuite) TearDownSuite() {
}
}
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
logger.Infof("Rook clusters %s and %s uninstalled", mrc.namespace1, mrc.namespace2)
o.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
logger.Infof("Rook clusters %s and %s uninstalled", o.namespace1, o.namespace2)
}
//Test to make sure all rook components are installed and Running
+11 -6
View File
@@ -20,6 +20,7 @@ import (
"testing"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/contracts"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
@@ -52,7 +53,11 @@ import (
// - PUT/GET objects
// ************************************************
func TestSmokeSuite(t *testing.T) {
suite.Run(t, new(SmokeSuite))
s := new(SmokeSuite)
defer func(s *SmokeSuite) {
HandlePanics(recover(), s.o, s.T)
}(s)
suite.Run(t, s)
}
type SmokeSuite struct {
@@ -60,6 +65,7 @@ type SmokeSuite struct {
helper *clients.TestClient
k8sh *utils.K8sHelper
installer *installer.InstallHelper
o contracts.TestOperator
namespace string
}
@@ -71,11 +77,13 @@ func (suite *SmokeSuite) SetupSuite() {
suite.k8sh = kh
suite.installer = installer.NewK8sRookhelper(kh.Clientset, suite.T)
suite.o = NewBaseTestOperations(suite.installer, suite.T, suite.namespace, false)
isRookInstalled, err := suite.installer.InstallRookOnK8sWithHostPathAndDevices(suite.namespace, "bluestore", "", true, 3)
assert.NoError(suite.T(), err)
if !isRookInstalled {
logger.Errorf("Rook Was not installed successfully")
suite.T().Fail()
suite.TearDownSuite()
suite.T().FailNow()
}
@@ -83,17 +91,14 @@ func (suite *SmokeSuite) SetupSuite() {
suite.helper, err = clients.CreateTestClient(kh, suite.namespace)
if err != nil {
logger.Errorf("Cannot create rook test client, er -> %v", err)
suite.T().Fail()
suite.TearDownSuite()
suite.T().FailNow()
}
}
func (suite *SmokeSuite) TearDownSuite() {
if suite.T().Failed() {
gatherAllRookLogs(suite.k8sh, suite.Suite, suite.installer.Env.HostType, installer.SystemNamespace(suite.namespace), suite.namespace)
}
suite.installer.UninstallRookFromK8s(suite.namespace, false)
suite.o.TearDown()
}
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {