Merge pull request #1223 from dangula/pr_recoverTestAfterPanic
recover tests from panic and capture logs.
This commit is contained in:
9 files changed
+140
-49
No files matched your search
@@ -86,3 +86,7 @@ type RestAPIOperator interface {
|
||||
UpdateObjectUser(storeName string, user model.ObjectUser) (*model.ObjectUser, error)
|
||||
DeleteObjectUser(storeName, id string) error
|
||||
}
|
||||
|
||||
type TestOperator interface {
|
||||
TearDown()
|
||||
}
|
||||
@@ -307,6 +307,18 @@ func (h *InstallHelper) CleanupCluster(clusterName string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (h *InstallHelper) GatherAllRookLogs(nameSpace string, testName string) {
|
||||
logger.Infof("Gathering all logs from Rook Cluster %s", nameSpace)
|
||||
h.k8shelper.GetRookLogs("rook-operator", h.Env.HostType, SystemNamespace(nameSpace), testName)
|
||||
h.k8shelper.GetRookLogs("rook-agent", h.Env.HostType, SystemNamespace(nameSpace), testName)
|
||||
h.k8shelper.GetRookLogs("rook-api", h.Env.HostType, nameSpace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mgr", h.Env.HostType, nameSpace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mon", h.Env.HostType, nameSpace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-osd", h.Env.HostType, nameSpace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-rgw", h.Env.HostType, nameSpace, testName)
|
||||
h.k8shelper.GetRookLogs("rook-ceph-mds", h.Env.HostType, nameSpace, testName)
|
||||
}
|
||||
|
||||
//NewK8sRookhelper creates new instance of InstallHelper
|
||||
func NewK8sRookhelper(clientset *kubernetes.Clientset, t func() *testing.T) *InstallHelper {
|
||||
|
||||
|
||||
@@ -601,7 +601,7 @@ func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (st
|
||||
return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
|
||||
}
|
||||
|
||||
if podList.Size() < 1 {
|
||||
if len(podList.Items) < 1 {
|
||||
logger.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
|
||||
return "", fmt.Errorf("Cannot get hostIp for app : %v in namespace %v, err: %v", podNamePattern, namespace, err)
|
||||
}
|
||||
|
||||
@@ -23,10 +23,13 @@ import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/pkg/model"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -73,14 +76,34 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient,
|
||||
require.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func gatherAllRookLogs(k8sh *utils.K8sHelper, s suite.Suite, hostType string, opNamespace string, clusterNamespace string) {
|
||||
logger.Infof("Gathering all logs from Rook Cluster %s", clusterNamespace)
|
||||
k8sh.GetRookLogs("rook-operator", hostType, opNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-agent", hostType, opNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-api", hostType, clusterNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-ceph-mgr", hostType, clusterNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-ceph-mon", hostType, clusterNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-ceph-osd", hostType, clusterNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-ceph-rgw", hostType, clusterNamespace, s.T().Name())
|
||||
k8sh.GetRookLogs("rook-ceph-mds", hostType, clusterNamespace, s.T().Name())
|
||||
func HandlePanics(r interface{}, o contracts.TestOperator, t func() *testing.T) {
|
||||
if r != nil {
|
||||
logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r)
|
||||
t().Fail()
|
||||
o.TearDown()
|
||||
t().FailNow()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//BaseTestOperations struct for handling panic and test suite tear down
|
||||
type BaseTestOperations struct {
|
||||
installer *installer.InstallHelper
|
||||
T func() *testing.T
|
||||
namespace string
|
||||
helmInstalled bool
|
||||
}
|
||||
|
||||
//NewBaseTestOperations creates new instance of BaseTestOperations struct
|
||||
func NewBaseTestOperations(i *installer.InstallHelper, t func() *testing.T, namespace string, helmInstalled bool) BaseTestOperations {
|
||||
return BaseTestOperations{i, t, namespace, helmInstalled}
|
||||
}
|
||||
|
||||
//TearDown is a wrapper for tearDown after Sutie
|
||||
func (o BaseTestOperations) TearDown() {
|
||||
if o.installer.T().Failed() {
|
||||
o.installer.GatherAllRookLogs(o.namespace, o.installer.T().Name())
|
||||
}
|
||||
o.installer.UninstallRookFromK8s(o.namespace, o.helmInstalled)
|
||||
|
||||
}
|
||||
@@ -37,7 +37,11 @@ var (
|
||||
)
|
||||
|
||||
func TestBlockImageCreateSuite(t *testing.T) {
|
||||
suite.Run(t, new(BlockImageCreateSuite))
|
||||
s := new(BlockImageCreateSuite)
|
||||
defer func(s *BlockImageCreateSuite) {
|
||||
HandlePanics(recover(), s.o, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type BlockImageCreateSuite struct {
|
||||
@@ -45,9 +49,10 @@ type BlockImageCreateSuite struct {
|
||||
testClient *clients.TestClient
|
||||
kh *utils.K8sHelper
|
||||
rc contracts.RestAPIOperator
|
||||
installer *installer.InstallHelper
|
||||
o contracts.TestOperator
|
||||
initBlockCount int
|
||||
namespace string
|
||||
installer *installer.InstallHelper
|
||||
}
|
||||
|
||||
func (s *BlockImageCreateSuite) SetupSuite() {
|
||||
@@ -57,10 +62,12 @@ func (s *BlockImageCreateSuite) SetupSuite() {
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T)
|
||||
s.o = NewBaseTestOperations(s.installer, s.T, s.namespace, false)
|
||||
|
||||
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1)
|
||||
if !isRookInstalled {
|
||||
logger.Errorf("Rook Was not installed successfully")
|
||||
s.T().Fail()
|
||||
s.TearDownSuite()
|
||||
s.T().FailNow()
|
||||
}
|
||||
@@ -68,6 +75,7 @@ func (s *BlockImageCreateSuite) SetupSuite() {
|
||||
s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
|
||||
if err != nil {
|
||||
logger.Errorf("Cannot create rook test client, er -> %v", err)
|
||||
s.T().Fail()
|
||||
s.TearDownSuite()
|
||||
s.T().FailNow()
|
||||
}
|
||||
@@ -168,9 +176,6 @@ func (s *BlockImageCreateSuite) TearDownTest() {
|
||||
}
|
||||
}
|
||||
func (s *BlockImageCreateSuite) TearDownSuite() {
|
||||
if s.T().Failed() {
|
||||
gatherAllRookLogs(s.kh, s.Suite, s.installer.Env.HostType, s.namespace, s.namespace)
|
||||
}
|
||||
s.installer.UninstallRookFromK8s(s.namespace, false)
|
||||
s.o.TearDown()
|
||||
|
||||
}
|
||||
@@ -32,7 +32,11 @@ import (
|
||||
// Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes
|
||||
|
||||
func TestK8sBlockImageCreateSuite(t *testing.T) {
|
||||
suite.Run(t, new(K8sBlockImageCreateSuite))
|
||||
s := new(K8sBlockImageCreateSuite)
|
||||
defer func(s *K8sBlockImageCreateSuite) {
|
||||
HandlePanics(recover(), s.o, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type K8sBlockImageCreateSuite struct {
|
||||
@@ -43,6 +47,7 @@ type K8sBlockImageCreateSuite struct {
|
||||
initBlockCount int
|
||||
namespace string
|
||||
installer *installer.InstallHelper
|
||||
o contracts.TestOperator
|
||||
}
|
||||
|
||||
func (s *K8sBlockImageCreateSuite) SetupSuite() {
|
||||
@@ -53,11 +58,13 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() {
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T)
|
||||
s.o = NewBaseTestOperations(s.installer, s.T, s.namespace, false)
|
||||
|
||||
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1)
|
||||
assert.NoError(s.T(), err)
|
||||
if !isRookInstalled {
|
||||
logger.Errorf("Rook Was not installed successfully")
|
||||
s.T().Fail()
|
||||
s.TearDownSuite()
|
||||
s.T().FailNow()
|
||||
}
|
||||
@@ -65,6 +72,7 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() {
|
||||
s.testClient, err = clients.CreateTestClient(s.kh, s.namespace)
|
||||
if err != nil {
|
||||
logger.Errorf("Cannot create rook test client, er -> %v", err)
|
||||
s.T().Fail()
|
||||
s.TearDownSuite()
|
||||
s.T().FailNow()
|
||||
}
|
||||
@@ -193,8 +201,5 @@ func (s *K8sBlockImageCreateSuite) tearDownTest(claimName string, poolName strin
|
||||
}
|
||||
|
||||
func (s *K8sBlockImageCreateSuite) TearDownSuite() {
|
||||
if s.T().Failed() {
|
||||
gatherAllRookLogs(s.kh, s.Suite, s.installer.Env.HostType, s.namespace, s.namespace)
|
||||
}
|
||||
s.installer.UninstallRookFromK8s(s.namespace, false)
|
||||
s.o.TearDown()
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -34,7 +35,11 @@ var (
|
||||
// - Create the object store via the CRD
|
||||
// ***************************************************
|
||||
func TestHelmSuite(t *testing.T) {
|
||||
suite.Run(t, new(HelmSuite))
|
||||
s := new(HelmSuite)
|
||||
defer func(s *HelmSuite) {
|
||||
HandlePanics(recover(), s.o, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type HelmSuite struct {
|
||||
@@ -43,6 +48,7 @@ type HelmSuite struct {
|
||||
k8sh *utils.K8sHelper
|
||||
installer *installer.InstallHelper
|
||||
hh *utils.HelmHelper
|
||||
o contracts.TestOperator
|
||||
namespace string
|
||||
}
|
||||
|
||||
@@ -55,6 +61,7 @@ func (hs *HelmSuite) SetupSuite() {
|
||||
hs.hh = utils.NewHelmHelper()
|
||||
|
||||
hs.installer = installer.NewK8sRookhelper(kh.Clientset, hs.T)
|
||||
hs.o = NewBaseTestOperations(hs.installer, hs.T, hs.namespace, true)
|
||||
|
||||
err = hs.installer.CreateK8sRookOperatorViaHelm(hs.namespace)
|
||||
require.NoError(hs.T(), err)
|
||||
@@ -78,11 +85,7 @@ func (hs *HelmSuite) SetupSuite() {
|
||||
}
|
||||
|
||||
func (hs *HelmSuite) TearDownSuite() {
|
||||
if hs.T().Failed() {
|
||||
gatherAllRookLogs(hs.k8sh, hs.Suite, hs.installer.Env.HostType, hs.namespace, hs.namespace)
|
||||
}
|
||||
hs.installer.UninstallRookFromK8s(hs.namespace, true)
|
||||
|
||||
hs.o.TearDown()
|
||||
}
|
||||
|
||||
//Test to make sure all rook components are installed and Running
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/model"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -31,7 +32,11 @@ import (
|
||||
// - Create the object store via the CRD
|
||||
// *************************************************************
|
||||
func TestMultiClusterDeploySuite(t *testing.T) {
|
||||
suite.Run(t, new(MultiClusterDeploySuite))
|
||||
s := new(MultiClusterDeploySuite)
|
||||
defer func(s *MultiClusterDeploySuite) {
|
||||
HandlePanics(recover(), s.o, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type MultiClusterDeploySuite struct {
|
||||
@@ -43,6 +48,7 @@ type MultiClusterDeploySuite struct {
|
||||
installData *installer.InstallData
|
||||
namespace1 string
|
||||
namespace2 string
|
||||
o contracts.TestOperator
|
||||
}
|
||||
|
||||
//Deploy Multiple Rook clusters
|
||||
@@ -57,6 +63,7 @@ func (mrc *MultiClusterDeploySuite) SetupSuite() {
|
||||
mrc.k8sh = kh
|
||||
mrc.installer = installer.NewK8sRookhelper(kh.Clientset, mrc.T)
|
||||
mrc.installData = installer.NewK8sInstallData()
|
||||
mrc.o = NewMCTestOperations(mrc.installer, mrc.installData, mrc.k8sh, mrc.T, mrc.namespace1, mrc.namespace2)
|
||||
|
||||
err = mrc.installer.CreateK8sRookOperator(installer.SystemNamespace(mrc.namespace1))
|
||||
require.NoError(mrc.T(), err)
|
||||
@@ -113,39 +120,65 @@ func (mrc *MultiClusterDeploySuite) startCluster(namespace, store string, errCh
|
||||
}
|
||||
|
||||
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
|
||||
if mrc.T().Failed() {
|
||||
gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace1)
|
||||
gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace2)
|
||||
mrc.o.TearDown()
|
||||
}
|
||||
|
||||
//MCTestOperations struct for handling panic and test suite tear down
|
||||
type MCTestOperations struct {
|
||||
installer *installer.InstallHelper
|
||||
installData *installer.InstallData
|
||||
k8sh *utils.K8sHelper
|
||||
T func() *testing.T
|
||||
namespace1 string
|
||||
namespace2 string
|
||||
}
|
||||
|
||||
//NewMCTestOperations creates new instance of BaseTestOperations struct
|
||||
func NewMCTestOperations(i *installer.InstallHelper, d *installer.InstallData, k8sh *utils.K8sHelper, t func() *testing.T, namespace1 string, namespace2 string) MCTestOperations {
|
||||
return MCTestOperations{i, d, k8sh, t, namespace1, namespace2}
|
||||
}
|
||||
|
||||
//TearDown is a wrapper for tearDown after suite
|
||||
func (o MCTestOperations) TearDown() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Infof("Unexpected Errors while cleaning up MultiCluster test --> %v", r)
|
||||
o.T().FailNow()
|
||||
}
|
||||
}()
|
||||
if o.T().Failed() {
|
||||
o.installer.GatherAllRookLogs(o.namespace1, o.T().Name())
|
||||
o.installer.GatherAllRookLogs(o.namespace2, o.T().Name())
|
||||
}
|
||||
deleteArgs := []string{"delete", "-f", "-"}
|
||||
|
||||
skipRookInstall := strings.EqualFold(mrc.installer.Env.SkipInstallRook, "true")
|
||||
skipRookInstall := strings.EqualFold(o.installer.Env.SkipInstallRook, "true")
|
||||
if skipRookInstall {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("Uninstalling All Rook Clusters")
|
||||
k8sHelp, err := utils.CreateK8sHelper(mrc.T)
|
||||
k8sHelp, err := utils.CreateK8sHelper(o.T)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rookOperator := mrc.installData.GetRookOperator(installer.SystemNamespace(mrc.namespace1))
|
||||
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-agent", nil)
|
||||
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-agent", nil)
|
||||
rookOperator := o.installData.GetRookOperator(installer.SystemNamespace(o.namespace1))
|
||||
o.k8sh.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-agent", nil)
|
||||
o.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-agent", nil)
|
||||
|
||||
//Delete rook operator
|
||||
_, err = mrc.k8sh.KubectlWithStdin(rookOperator, deleteArgs...)
|
||||
_, err = o.k8sh.KubectlWithStdin(rookOperator, deleteArgs...)
|
||||
if err != nil {
|
||||
logger.Errorf("Rook operator cannot be deleted,err -> %v", err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
//delete rook cluster
|
||||
mrc.installer.CleanupCluster(mrc.namespace1)
|
||||
mrc.installer.CleanupCluster(mrc.namespace2)
|
||||
o.installer.CleanupCluster(o.namespace1)
|
||||
o.installer.CleanupCluster(o.namespace2)
|
||||
|
||||
// Delete crd/tpr
|
||||
if mrc.k8sh.VersionAtLeast("v1.7.0") {
|
||||
if o.k8sh.VersionAtLeast("v1.7.0") {
|
||||
_, err = k8sHelp.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io"})
|
||||
if err != nil {
|
||||
logger.Errorf("crd cannot be deleted")
|
||||
@@ -159,8 +192,9 @@ func (mrc *MultiClusterDeploySuite) TearDownSuite() {
|
||||
}
|
||||
}
|
||||
|
||||
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
|
||||
logger.Infof("Rook clusters %s and %s uninstalled", mrc.namespace1, mrc.namespace2)
|
||||
o.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
|
||||
logger.Infof("Rook clusters %s and %s uninstalled", o.namespace1, o.namespace2)
|
||||
|
||||
}
|
||||
|
||||
//Test to make sure all rook components are installed and Running
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/contracts"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -52,7 +53,11 @@ import (
|
||||
// - PUT/GET objects
|
||||
// ************************************************
|
||||
func TestSmokeSuite(t *testing.T) {
|
||||
suite.Run(t, new(SmokeSuite))
|
||||
s := new(SmokeSuite)
|
||||
defer func(s *SmokeSuite) {
|
||||
HandlePanics(recover(), s.o, s.T)
|
||||
}(s)
|
||||
suite.Run(t, s)
|
||||
}
|
||||
|
||||
type SmokeSuite struct {
|
||||
@@ -60,6 +65,7 @@ type SmokeSuite struct {
|
||||
helper *clients.TestClient
|
||||
k8sh *utils.K8sHelper
|
||||
installer *installer.InstallHelper
|
||||
o contracts.TestOperator
|
||||
namespace string
|
||||
}
|
||||
|
||||
@@ -71,11 +77,13 @@ func (suite *SmokeSuite) SetupSuite() {
|
||||
suite.k8sh = kh
|
||||
|
||||
suite.installer = installer.NewK8sRookhelper(kh.Clientset, suite.T)
|
||||
suite.o = NewBaseTestOperations(suite.installer, suite.T, suite.namespace, false)
|
||||
|
||||
isRookInstalled, err := suite.installer.InstallRookOnK8sWithHostPathAndDevices(suite.namespace, "bluestore", "", true, 3)
|
||||
assert.NoError(suite.T(), err)
|
||||
if !isRookInstalled {
|
||||
logger.Errorf("Rook Was not installed successfully")
|
||||
suite.T().Fail()
|
||||
suite.TearDownSuite()
|
||||
suite.T().FailNow()
|
||||
}
|
||||
@@ -83,17 +91,14 @@ func (suite *SmokeSuite) SetupSuite() {
|
||||
suite.helper, err = clients.CreateTestClient(kh, suite.namespace)
|
||||
if err != nil {
|
||||
logger.Errorf("Cannot create rook test client, er -> %v", err)
|
||||
suite.T().Fail()
|
||||
suite.TearDownSuite()
|
||||
suite.T().FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TearDownSuite() {
|
||||
if suite.T().Failed() {
|
||||
gatherAllRookLogs(suite.k8sh, suite.Suite, suite.installer.Env.HostType, installer.SystemNamespace(suite.namespace), suite.namespace)
|
||||
|
||||
}
|
||||
suite.installer.UninstallRookFromK8s(suite.namespace, false)
|
||||
suite.o.TearDown()
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
|
||||
|
||||
Reference in new issue
Block a user