feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

tests: more efficient cluster creation and teardown

Browse Source
This commit is contained in:
Travis Nielsen committed 2017-10-27 14:37:56 -07:00
1 parent 305c3f93cb
commit 66efebf28a
11 files changed
+128 -75
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/framework/clients/cluster.go
+1 -1
View File
@@ -31,7 +31,7 @@ func IsClusterHealthy(testClient *TestClient) (model.StatusDetails, error) {
}
// verify all mons are in quorum
if len(status.Monitors) < 3 {
if len(status.Monitors) == 0 {
return status, fmt.Errorf("too few monitors: %+v", status)
}
for _, m := range status.Monitors {
tests/framework/installer/install_data.go
+2 -1
View File
@@ -185,7 +185,7 @@ spec:
}
//GetRookCluster returns rook-cluster manifest
func (i *InstallData) GetRookCluster(namespace string, storeType string, dataDirHostPath string, useAllDevices bool) string {
func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string {
return `apiVersion: v1
kind: Namespace
metadata:
@@ -200,6 +200,7 @@ spec:
versionTag: master
dataDirHostPath: ` + dataDirHostPath + `
hostNetwork: false
monCount: ` + strconv.Itoa(mons) + `
storage:
useAllNodes: true
useAllDevices: ` + strconv.FormatBool(useAllDevices) + `
tests/framework/installer/install_helper.go
+14 -23
View File
@@ -139,18 +139,18 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
}
func (h *InstallHelper) CreateK8sRookCluster(namespace string, storeType string) (err error) {
return h.CreateK8sRookClusterWithHostPath(namespace, storeType, "")
return h.CreateK8sRookClusterWithHostPath(namespace, storeType, "", 1)
}
func (h *InstallHelper) CreateK8sRookClusterWithHostPath(namespace string, storeType string, dataDirHostPath string) (err error) {
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", IsAdditionalDeviceAvailableOnCluster())
func (h *InstallHelper) CreateK8sRookClusterWithHostPath(namespace string, storeType string, dataDirHostPath string, mons int) (err error) {
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", false, mons)
}
//CreateK8sRookCluster creates rook cluster via kubectl
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace string, storeType string, dataDirHostPath string, useAllDevices bool) (err error) {
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) (err error) {
logger.Infof("Starting Rook Cluster")
rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices)
rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices, mons)
_, err = h.k8shelper.KubectlWithStdin(rookCluster, createArgs...)
@@ -171,12 +171,12 @@ func SystemNamespace(namespace string) string {
return fmt.Sprintf("%s-system", namespace)
}
func (h *InstallHelper) InstallRookOnK8s(namespace string, storeType string) (bool, error) {
return h.InstallRookOnK8sWithHostPath(namespace, storeType, "")
func (h *InstallHelper) InstallRookOnK8s(namespace, storeType string, mons int) (bool, error) {
return h.InstallRookOnK8sWithHostPathAndDevices(namespace, storeType, "", false, mons)
}
//InstallRookOnK8s installs rook on k8s
func (h *InstallHelper) InstallRookOnK8sWithHostPath(namespace string, storeType string, dataDirHostPath string) (bool, error) {
func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, useDevices bool, mons int) (bool, error) {
//flag used for local debuggin purpose, when rook is pre-installed
skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true")
@@ -204,8 +204,12 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPath(namespace string, storeType
time.Sleep(10 * time.Second)
if useDevices {
useDevices = IsAdditionalDeviceAvailableOnCluster()
}
//Create rook cluster
err = h.CreateK8sRookClusterWithHostPath(namespace, storeType, dataDirHostPath)
err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath, useDevices, mons)
if err != nil {
logger.Errorf("Rook cluster %s not installed ,error -> %v", namespace, err)
return false, err
@@ -271,18 +275,9 @@ func (h *InstallHelper) UninstallRookFromK8s(namespace string, helmInstalled boo
_, err = k8sHelp.DeleteResource([]string{"namespace", namespace})
assert.NoError(h.T(), err, "cannot delete namespace %s err -> %+v", namespace, err)
isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", namespace)
isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted(namespace)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
assert.True(h.T(), isRookUninstalled, "make sure rook is uninstalled")
assert.True(h.T(), isNameSpaceDeleted, "make sure rook is uninstalled")
if isRookUninstalled && isNameSpaceDeleted {
logger.Infof("Rook cluster %s uninstalled successfully", namespace)
return
}
logger.Errorf("Rook cluster %s not uninstalled", namespace)
logger.Infof("Rook cluster %s uninstalled", namespace)
}
//CleanupCluster deletes a rook cluster for a namespace
@@ -308,10 +303,6 @@ func (h *InstallHelper) CleanupCluster(clusterName string) {
if err != nil {
logger.Errorf("namespace %s cannot be deleted,err -> %v", clusterName, err)
}
h.k8shelper.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterName)
h.k8shelper.WaitUntilNameSpaceIsDeleted(clusterName)
}
//NewK8sRookhelper creates new instance of InstallHelper
tests/integration/base_deploy_test.go
+4 -3
View File
@@ -17,6 +17,7 @@ limitations under the License.
package integration
import (
"fmt"
"time"
"github.com/coreos/pkg/capnslog"
@@ -35,7 +36,7 @@ var (
)
//Test to make sure all rook components are installed and Running
func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNamespace string, clusterNamespace string) {
func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNamespace, clusterNamespace string, mons int) {
logger.Infof("Make sure all Pods in Rook Cluster %s are running", clusterNamespace)
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-operator", opNamespace, 1, "Running"),
"Make sure there is 1 rook-operator present in Running state")
@@ -47,8 +48,8 @@ func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNames
"Make sure there is 1 rook-ceph-mgr present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", clusterNamespace, 1, "Running"),
"Make sure there is at lest 1 rook-ceph-osd present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mon", clusterNamespace, 3, "Running"),
"Make sure there are 3 rook-ceph-mon present in Running state")
assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mon", clusterNamespace, mons, "Running"),
fmt.Sprintf("Make sure there are %d rook-ceph-mon present in Running state", mons))
}
func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, clusterNamespace string) {
tests/integration/base_file_test.go
-1
View File
@@ -95,7 +95,6 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
logger.Infof("Cleaning up file system")
podWithFilesystem(k8sh, s, podname, namespace, filesystemName, fileMountPath, "delete")
helper.GetFileSystemClient().FSDelete(filesystemName)
}
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error {
tests/integration/block_createBlock_api_test.go
+4 -3
View File
@@ -36,7 +36,7 @@ var (
blockImageName = "testImage"
)
func TestBlockCreateAPI(t *testing.T) {
func TestBlockImageCreateSuite(t *testing.T) {
suite.Run(t, new(BlockImageCreateSuite))
}
@@ -58,7 +58,7 @@ func (s *BlockImageCreateSuite) SetupSuite() {
s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T)
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore")
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1)
if !isRookInstalled {
logger.Errorf("Rook Was not installed successfully")
s.TearDownSuite()
@@ -158,7 +158,8 @@ func (s *BlockImageCreateSuite) TestRecreatingBlockImageForDifferentPool() {
// Delete all Block images that have the word Test in their name
func (s *BlockImageCreateSuite) TearDownTest() {
blocks, _ := s.rc.GetBlockImages()
blocks, err := s.rc.GetBlockImages()
assert.NoError(s.T(), err)
for _, b := range blocks {
if strings.Contains(b.Name, "test") {
tests/integration/block_createBlock_k8s_test.go
+2 -2
View File
@@ -31,7 +31,7 @@ import (
// Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes
func TestK8sBlockCreate(t *testing.T) {
func TestK8sBlockImageCreateSuite(t *testing.T) {
suite.Run(t, new(K8sBlockImageCreateSuite))
}
@@ -54,7 +54,7 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() {
s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T)
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore")
isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1)
assert.NoError(s.T(), err)
if !isRookInstalled {
logger.Errorf("Rook Was not installed successfully")
tests/integration/helm_test.go
+18 -2
View File
@@ -17,7 +17,23 @@ var (
hslogger = capnslog.NewPackageLogger("github.com/rook/rook", "helmSmokeTest")
)
func TestHelmIntegrationSuite(t *testing.T) {
// ***************************************************
// *** Major scenarios tested by the TestHelmSuite ***
// Setup
// - A cluster created via the Helm chart
// Monitors
// - One mon
// OSDs
// - Bluestore running on a directory
// Block
// - Create a pool in each cluster
// - Mount/unmount a block device through the dynamic provisioner
// File system
// - Create a file system via the REST API
// Object
// - Create the object store via the CRD
// ***************************************************
func TestHelmSuite(t *testing.T) {
suite.Run(t, new(HelmSuite))
}
@@ -71,7 +87,7 @@ func (hs *HelmSuite) TearDownSuite() {
//Test to make sure all rook components are installed and Running
func (hs *HelmSuite) TestRookInstallViaHelm() {
checkIfRookClusterIsInstalled(hs.Suite, hs.k8sh, hs.namespace, hs.namespace)
checkIfRookClusterIsInstalled(hs.Suite, hs.k8sh, hs.namespace, hs.namespace, 1)
}
//Test BlockCreation on Rook that was installed via Helm
tests/integration/multi_cluster_deploy_test.go
+55 -35
View File
@@ -1,6 +1,7 @@
package integration
import (
"fmt"
"strings"
"testing"
"time"
@@ -13,11 +14,27 @@ import (
"github.com/stretchr/testify/suite"
)
func TestInstallingMultipleRookCluster(t *testing.T) {
suite.Run(t, new(MultiRookClusterDeploySuite))
// *************************************************************
// *** Major scenarios tested by the MultiClusterDeploySuite ***
// Setup
// - Two clusters started in different namespaces via the CRD
// Monitors
// - One mon in each cluster
// OSDs
// - Bluestore running on a directory
// Block
// - Create a pool in each cluster
// - Mount/unmount a block device through the dynamic provisioner
// File system
// - Create a file system via the REST API
// Object
// - Create the object store via the CRD
// *************************************************************
func TestMultiClusterDeploySuite(t *testing.T) {
suite.Run(t, new(MultiClusterDeploySuite))
}
type MultiRookClusterDeploySuite struct {
type MultiClusterDeploySuite struct {
suite.Suite
helper1 *clients.TestClient
helper2 *clients.TestClient
@@ -29,7 +46,7 @@ type MultiRookClusterDeploySuite struct {
}
//Deploy Multiple Rook clusters
func (mrc *MultiRookClusterDeploySuite) SetupSuite() {
func (mrc *MultiClusterDeploySuite) SetupSuite() {
mrc.namespace1 = "mrc-n1"
mrc.namespace2 = "mrc-n2"
@@ -52,22 +69,20 @@ func (mrc *MultiRookClusterDeploySuite) SetupSuite() {
time.Sleep(10 * time.Second)
err = mrc.installer.CreateK8sRookClusterWithHostPathAndDevices(mrc.namespace1, "bluestore", "", installer.IsAdditionalDeviceAvailableOnCluster())
require.NoError(mrc.T(), err)
// start the two clusters in parallel
logger.Infof("starting two clusters in parallel")
errCh1 := make(chan error, 1)
errCh2 := make(chan error, 1)
go mrc.startCluster(mrc.namespace1, "bluestore", errCh1)
go mrc.startCluster(mrc.namespace2, "filestore", errCh2)
require.NoError(mrc.T(), <-errCh1)
require.NoError(mrc.T(), <-errCh2)
logger.Infof("finished starting clusters")
err = mrc.installer.CreateK8sRookToolbox(mrc.namespace1)
require.NoError(mrc.T(), err)
err = mrc.installer.CreateK8sRookClusterWithHostPathAndDevices(mrc.namespace2, "filestore", "", false)
require.NoError(mrc.T(), err)
err = mrc.installer.CreateK8sRookToolbox(mrc.namespace2)
require.NoError(mrc.T(), err)
mrc.helper1, err = clients.CreateTestClient(kh, mrc.namespace1)
mrc.helper1, err = clients.CreateTestClient(mrc.k8sh, mrc.namespace1)
require.Nil(mrc.T(), err)
mrc.helper2, err = clients.CreateTestClient(kh, mrc.namespace2)
mrc.helper2, err = clients.CreateTestClient(mrc.k8sh, mrc.namespace2)
require.Nil(mrc.T(), err)
// create a test pool in each cluster so that we get some PGs
@@ -82,7 +97,22 @@ func (mrc *MultiRookClusterDeploySuite) SetupSuite() {
require.Nil(mrc.T(), err)
}
func (mrc *MultiRookClusterDeploySuite) TearDownSuite() {
func (mrc *MultiClusterDeploySuite) startCluster(namespace, store string, errCh chan error) {
logger.Infof("starting cluster %s", namespace)
if err := mrc.installer.CreateK8sRookCluster(namespace, store); err != nil {
errCh <- fmt.Errorf("failed to create cluster %s. %+v", namespace, err)
return
}
if err := mrc.installer.CreateK8sRookToolbox(namespace); err != nil {
errCh <- fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err)
return
}
logger.Infof("succeeded starting cluster %s", namespace)
errCh <- nil
}
func (mrc *MultiClusterDeploySuite) TearDownSuite() {
if mrc.T().Failed() {
gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace1)
gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace2)
@@ -127,45 +157,35 @@ func (mrc *MultiRookClusterDeploySuite) TearDownSuite() {
}
}
isRookUninstalled1 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", mrc.namespace1)
isNameSpaceDeleted1 := k8sHelp.WaitUntilNameSpaceIsDeleted(mrc.namespace1)
isRookUninstalled2 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", mrc.namespace2)
isNameSpaceDeleted2 := k8sHelp.WaitUntilNameSpaceIsDeleted(mrc.namespace2)
mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
if isRookUninstalled1 && isNameSpaceDeleted1 && isRookUninstalled2 && isNameSpaceDeleted2 {
logger.Infof("Rook clusters %s and %s uninstalled successfully", mrc.namespace1, mrc.namespace2)
return
}
logger.Infof("Rook clusters %s and %s not uninstalled successfully", mrc.namespace1, mrc.namespace2)
logger.Infof("Rook clusters %s and %s uninstalled", mrc.namespace1, mrc.namespace2)
}
//Test to make sure all rook components are installed and Running
func (mrc *MultiRookClusterDeploySuite) TestInstallingMultipleRookClusters() {
func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() {
//Check if Rook cluster 1 is deployed successfully
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1)
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper1, mrc.namespace1)
//Check if Rook cluster 2 is deployed successfully
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2)
checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2, 1)
checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper2, mrc.namespace2)
}
//Test Block Store Creation on multiple rook clusters
func (mrc *MultiRookClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() {
func (mrc *MultiClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() {
runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1)
runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2)
}
//Test Filesystem Creation on multiple rook clusters
func (mrc *MultiRookClusterDeploySuite) TestFileStoreOnMultiRookCluster() {
func (mrc *MultiClusterDeploySuite) TestFileStoreOnMultiRookCluster() {
runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1")
runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2")
}
//Test Object Store Creation on multiple rook clusters
func (mrc *MultiRookClusterDeploySuite) TestObjectStoreOnMultiRookCluster() {
func (mrc *MultiClusterDeploySuite) TestObjectStoreOnMultiRookCluster() {
runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2)
runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1)
}
tests/integration/smoke_test.go
+27 -3
View File
@@ -27,7 +27,31 @@ import (
"github.com/stretchr/testify/suite"
)
func TestSmokeSuiteK8s(t *testing.T) {
// ************************************************
// *** Major scenarios tested by the SmokeSuite ***
// Setup
// - via the cluster CRD
// Monitors
// - Three mons in the cluster
// - Failover of an unhealthy monitor
// OSDs
// - Bluestore running on devices
// Block
// - Mount/unmount a block device through the dynamic provisioner
// - Fencing of the block device
// - Read/write to the device
// File system
// - Create the file system via the CRD
// - Mount/unmount a file system in pod
// - Read/write to the file system
// - Delete the file system
// Object
// - Create the object store via the REST API
// - Create/delete buckets
// - Create/delete users
// - PUT/GET objects
// ************************************************
func TestSmokeSuite(t *testing.T) {
suite.Run(t, new(SmokeSuite))
}
@@ -48,7 +72,7 @@ func (suite *SmokeSuite) SetupSuite() {
suite.installer = installer.NewK8sRookhelper(kh.Clientset, suite.T)
isRookInstalled, err := suite.installer.InstallRookOnK8s(suite.namespace, "bluestore")
isRookInstalled, err := suite.installer.InstallRookOnK8sWithHostPathAndDevices(suite.namespace, "bluestore", "", true, 3)
assert.NoError(suite.T(), err)
if !isRookInstalled {
logger.Errorf("Rook Was not installed successfully")
@@ -84,5 +108,5 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
//Test to make sure all rook components are installed and Running
func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() {
checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace)
checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3)
}
tests/longhaul/base_block_test.go
+1 -1
View File
@@ -22,7 +22,7 @@ func setUpRookAndPoolInNamespace(t func() *testing.T, namespace string, storageC
i := installer.NewK8sRookhelper(kh.Clientset, t)
if !kh.IsRookInstalled(namespace) {
isRookInstalled, err := i.InstallRookOnK8sWithHostPath(namespace, "bluestore", "/temp/rookBackup")
isRookInstalled, err := i.InstallRookOnK8sWithHostPathAndDevices(namespace, "bluestore", "/temp/rookBackup", true, 3)
require.NoError(t(), err)
require.True(t(), isRookInstalled)
}