diff --git a/tests/framework/clients/cluster.go b/tests/framework/clients/cluster.go index a392fbf6b..0683edb33 100644 --- a/tests/framework/clients/cluster.go +++ b/tests/framework/clients/cluster.go @@ -31,7 +31,7 @@ func IsClusterHealthy(testClient *TestClient) (model.StatusDetails, error) { } // verify all mons are in quorum - if len(status.Monitors) < 3 { + if len(status.Monitors) == 0 { return status, fmt.Errorf("too few monitors: %+v", status) } for _, m := range status.Monitors { diff --git a/tests/framework/installer/install_data.go b/tests/framework/installer/install_data.go index 02ec08079..564784a0f 100644 --- a/tests/framework/installer/install_data.go +++ b/tests/framework/installer/install_data.go @@ -185,7 +185,7 @@ spec: } //GetRookCluster returns rook-cluster manifest -func (i *InstallData) GetRookCluster(namespace string, storeType string, dataDirHostPath string, useAllDevices bool) string { +func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string { return `apiVersion: v1 kind: Namespace metadata: @@ -200,6 +200,7 @@ spec: versionTag: master dataDirHostPath: ` + dataDirHostPath + ` hostNetwork: false + monCount: ` + strconv.Itoa(mons) + ` storage: useAllNodes: true useAllDevices: ` + strconv.FormatBool(useAllDevices) + ` diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index 21107f7a8..0ab852e78 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -139,18 +139,18 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) { } func (h *InstallHelper) CreateK8sRookCluster(namespace string, storeType string) (err error) { - return h.CreateK8sRookClusterWithHostPath(namespace, storeType, "") + return h.CreateK8sRookClusterWithHostPath(namespace, storeType, "", 1) } -func (h *InstallHelper) CreateK8sRookClusterWithHostPath(namespace string, storeType string, dataDirHostPath string) (err error) { - return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", IsAdditionalDeviceAvailableOnCluster()) +func (h *InstallHelper) CreateK8sRookClusterWithHostPath(namespace string, storeType string, dataDirHostPath string, mons int) (err error) { + return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", false, mons) } //CreateK8sRookCluster creates rook cluster via kubectl -func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace string, storeType string, dataDirHostPath string, useAllDevices bool) (err error) { +func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) (err error) { logger.Infof("Starting Rook Cluster") - rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices) + rookCluster := h.installData.GetRookCluster(namespace, storeType, dataDirHostPath, useAllDevices, mons) _, err = h.k8shelper.KubectlWithStdin(rookCluster, createArgs...) @@ -171,12 +171,12 @@ func SystemNamespace(namespace string) string { return fmt.Sprintf("%s-system", namespace) } -func (h *InstallHelper) InstallRookOnK8s(namespace string, storeType string) (bool, error) { - return h.InstallRookOnK8sWithHostPath(namespace, storeType, "") +func (h *InstallHelper) InstallRookOnK8s(namespace, storeType string, mons int) (bool, error) { + return h.InstallRookOnK8sWithHostPathAndDevices(namespace, storeType, "", false, mons) } //InstallRookOnK8s installs rook on k8s -func (h *InstallHelper) InstallRookOnK8sWithHostPath(namespace string, storeType string, dataDirHostPath string) (bool, error) { +func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, useDevices bool, mons int) (bool, error) { //flag used for local debuggin purpose, when rook is pre-installed skipRookInstall := strings.EqualFold(h.Env.SkipInstallRook, "true") @@ -204,8 +204,12 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPath(namespace string, storeType time.Sleep(10 * time.Second) + if useDevices { + useDevices = IsAdditionalDeviceAvailableOnCluster() + } + //Create rook cluster - err = h.CreateK8sRookClusterWithHostPath(namespace, storeType, dataDirHostPath) + err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath, useDevices, mons) if err != nil { logger.Errorf("Rook cluster %s not installed ,error -> %v", namespace, err) return false, err @@ -271,18 +275,9 @@ func (h *InstallHelper) UninstallRookFromK8s(namespace string, helmInstalled boo _, err = k8sHelp.DeleteResource([]string{"namespace", namespace}) assert.NoError(h.T(), err, "cannot delete namespace %s err -> %+v", namespace, err) - isRookUninstalled := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", namespace) - isNameSpaceDeleted := k8sHelp.WaitUntilNameSpaceIsDeleted(namespace) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) - assert.True(h.T(), isRookUninstalled, "make sure rook is uninstalled") - assert.True(h.T(), isNameSpaceDeleted, "make sure rook is uninstalled") - - if isRookUninstalled && isNameSpaceDeleted { - logger.Infof("Rook cluster %s uninstalled successfully", namespace) - return - } - logger.Errorf("Rook cluster %s not uninstalled", namespace) + logger.Infof("Rook cluster %s uninstalled", namespace) } //CleanupCluster deletes a rook cluster for a namespace @@ -308,10 +303,6 @@ func (h *InstallHelper) CleanupCluster(clusterName string) { if err != nil { logger.Errorf("namespace %s cannot be deleted,err -> %v", clusterName, err) } - - h.k8shelper.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", clusterName) - h.k8shelper.WaitUntilNameSpaceIsDeleted(clusterName) - } //NewK8sRookhelper creates new instance of InstallHelper diff --git a/tests/integration/base_deploy_test.go b/tests/integration/base_deploy_test.go index 9b67b308e..28e33f070 100644 --- a/tests/integration/base_deploy_test.go +++ b/tests/integration/base_deploy_test.go @@ -17,6 +17,7 @@ limitations under the License. package integration import ( + "fmt" "time" "github.com/coreos/pkg/capnslog" @@ -35,7 +36,7 @@ var ( ) //Test to make sure all rook components are installed and Running -func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNamespace string, clusterNamespace string) { +func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNamespace, clusterNamespace string, mons int) { logger.Infof("Make sure all Pods in Rook Cluster %s are running", clusterNamespace) assert.True(s.T(), k8sh.CheckPodCountAndState("rook-operator", opNamespace, 1, "Running"), "Make sure there is 1 rook-operator present in Running state") @@ -47,8 +48,8 @@ func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNames "Make sure there is 1 rook-ceph-mgr present in Running state") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", clusterNamespace, 1, "Running"), "Make sure there is at lest 1 rook-ceph-osd present in Running state") - assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mon", clusterNamespace, 3, "Running"), - "Make sure there are 3 rook-ceph-mon present in Running state") + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mon", clusterNamespace, mons, "Running"), + fmt.Sprintf("Make sure there are %d rook-ceph-mon present in Running state", mons)) } func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, clusterNamespace string) { diff --git a/tests/integration/base_file_test.go b/tests/integration/base_file_test.go index 891108d8c..7a297c99c 100644 --- a/tests/integration/base_file_test.go +++ b/tests/integration/base_file_test.go @@ -95,7 +95,6 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su logger.Infof("Cleaning up file system") podWithFilesystem(k8sh, s, podname, namespace, filesystemName, fileMountPath, "delete") helper.GetFileSystemClient().FSDelete(filesystemName) - } func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error { diff --git a/tests/integration/block_createBlock_api_test.go b/tests/integration/block_createBlock_api_test.go index 1508cbd51..806e9719b 100644 --- a/tests/integration/block_createBlock_api_test.go +++ b/tests/integration/block_createBlock_api_test.go @@ -36,7 +36,7 @@ var ( blockImageName = "testImage" ) -func TestBlockCreateAPI(t *testing.T) { +func TestBlockImageCreateSuite(t *testing.T) { suite.Run(t, new(BlockImageCreateSuite)) } @@ -58,7 +58,7 @@ func (s *BlockImageCreateSuite) SetupSuite() { s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T) - isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore") + isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1) if !isRookInstalled { logger.Errorf("Rook Was not installed successfully") s.TearDownSuite() @@ -158,7 +158,8 @@ func (s *BlockImageCreateSuite) TestRecreatingBlockImageForDifferentPool() { // Delete all Block images that have the word Test in their name func (s *BlockImageCreateSuite) TearDownTest() { - blocks, _ := s.rc.GetBlockImages() + blocks, err := s.rc.GetBlockImages() + assert.NoError(s.T(), err) for _, b := range blocks { if strings.Contains(b.Name, "test") { diff --git a/tests/integration/block_createBlock_k8s_test.go b/tests/integration/block_createBlock_k8s_test.go index 576d82b8c..a797686c7 100644 --- a/tests/integration/block_createBlock_k8s_test.go +++ b/tests/integration/block_createBlock_k8s_test.go @@ -31,7 +31,7 @@ import ( // Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes -func TestK8sBlockCreate(t *testing.T) { +func TestK8sBlockImageCreateSuite(t *testing.T) { suite.Run(t, new(K8sBlockImageCreateSuite)) } @@ -54,7 +54,7 @@ func (s *K8sBlockImageCreateSuite) SetupSuite() { s.installer = installer.NewK8sRookhelper(s.kh.Clientset, s.T) - isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore") + isRookInstalled, err := s.installer.InstallRookOnK8s(s.namespace, "bluestore", 1) assert.NoError(s.T(), err) if !isRookInstalled { logger.Errorf("Rook Was not installed successfully") diff --git a/tests/integration/helm_test.go b/tests/integration/helm_test.go index 7212c9046..48fd85f88 100644 --- a/tests/integration/helm_test.go +++ b/tests/integration/helm_test.go @@ -17,7 +17,23 @@ var ( hslogger = capnslog.NewPackageLogger("github.com/rook/rook", "helmSmokeTest") ) -func TestHelmIntegrationSuite(t *testing.T) { +// *************************************************** +// *** Major scenarios tested by the TestHelmSuite *** +// Setup +// - A cluster created via the Helm chart +// Monitors +// - One mon +// OSDs +// - Bluestore running on a directory +// Block +// - Create a pool in each cluster +// - Mount/unmount a block device through the dynamic provisioner +// File system +// - Create a file system via the REST API +// Object +// - Create the object store via the CRD +// *************************************************** +func TestHelmSuite(t *testing.T) { suite.Run(t, new(HelmSuite)) } @@ -71,7 +87,7 @@ func (hs *HelmSuite) TearDownSuite() { //Test to make sure all rook components are installed and Running func (hs *HelmSuite) TestRookInstallViaHelm() { - checkIfRookClusterIsInstalled(hs.Suite, hs.k8sh, hs.namespace, hs.namespace) + checkIfRookClusterIsInstalled(hs.Suite, hs.k8sh, hs.namespace, hs.namespace, 1) } //Test BlockCreation on Rook that was installed via Helm diff --git a/tests/integration/multi_cluster_deploy_test.go b/tests/integration/multi_cluster_deploy_test.go index 291f45cd2..4b00d111f 100644 --- a/tests/integration/multi_cluster_deploy_test.go +++ b/tests/integration/multi_cluster_deploy_test.go @@ -1,6 +1,7 @@ package integration import ( + "fmt" "strings" "testing" "time" @@ -13,11 +14,27 @@ import ( "github.com/stretchr/testify/suite" ) -func TestInstallingMultipleRookCluster(t *testing.T) { - suite.Run(t, new(MultiRookClusterDeploySuite)) +// ************************************************************* +// *** Major scenarios tested by the MultiClusterDeploySuite *** +// Setup +// - Two clusters started in different namespaces via the CRD +// Monitors +// - One mon in each cluster +// OSDs +// - Bluestore running on a directory +// Block +// - Create a pool in each cluster +// - Mount/unmount a block device through the dynamic provisioner +// File system +// - Create a file system via the REST API +// Object +// - Create the object store via the CRD +// ************************************************************* +func TestMultiClusterDeploySuite(t *testing.T) { + suite.Run(t, new(MultiClusterDeploySuite)) } -type MultiRookClusterDeploySuite struct { +type MultiClusterDeploySuite struct { suite.Suite helper1 *clients.TestClient helper2 *clients.TestClient @@ -29,7 +46,7 @@ type MultiRookClusterDeploySuite struct { } //Deploy Multiple Rook clusters -func (mrc *MultiRookClusterDeploySuite) SetupSuite() { +func (mrc *MultiClusterDeploySuite) SetupSuite() { mrc.namespace1 = "mrc-n1" mrc.namespace2 = "mrc-n2" @@ -52,22 +69,20 @@ func (mrc *MultiRookClusterDeploySuite) SetupSuite() { time.Sleep(10 * time.Second) - err = mrc.installer.CreateK8sRookClusterWithHostPathAndDevices(mrc.namespace1, "bluestore", "", installer.IsAdditionalDeviceAvailableOnCluster()) - require.NoError(mrc.T(), err) + // start the two clusters in parallel + logger.Infof("starting two clusters in parallel") + errCh1 := make(chan error, 1) + errCh2 := make(chan error, 1) + go mrc.startCluster(mrc.namespace1, "bluestore", errCh1) + go mrc.startCluster(mrc.namespace2, "filestore", errCh2) + require.NoError(mrc.T(), <-errCh1) + require.NoError(mrc.T(), <-errCh2) + logger.Infof("finished starting clusters") - err = mrc.installer.CreateK8sRookToolbox(mrc.namespace1) - require.NoError(mrc.T(), err) - - err = mrc.installer.CreateK8sRookClusterWithHostPathAndDevices(mrc.namespace2, "filestore", "", false) - require.NoError(mrc.T(), err) - - err = mrc.installer.CreateK8sRookToolbox(mrc.namespace2) - require.NoError(mrc.T(), err) - - mrc.helper1, err = clients.CreateTestClient(kh, mrc.namespace1) + mrc.helper1, err = clients.CreateTestClient(mrc.k8sh, mrc.namespace1) require.Nil(mrc.T(), err) - mrc.helper2, err = clients.CreateTestClient(kh, mrc.namespace2) + mrc.helper2, err = clients.CreateTestClient(mrc.k8sh, mrc.namespace2) require.Nil(mrc.T(), err) // create a test pool in each cluster so that we get some PGs @@ -82,7 +97,22 @@ func (mrc *MultiRookClusterDeploySuite) SetupSuite() { require.Nil(mrc.T(), err) } -func (mrc *MultiRookClusterDeploySuite) TearDownSuite() { +func (mrc *MultiClusterDeploySuite) startCluster(namespace, store string, errCh chan error) { + logger.Infof("starting cluster %s", namespace) + if err := mrc.installer.CreateK8sRookCluster(namespace, store); err != nil { + errCh <- fmt.Errorf("failed to create cluster %s. %+v", namespace, err) + return + } + + if err := mrc.installer.CreateK8sRookToolbox(namespace); err != nil { + errCh <- fmt.Errorf("failed to create toolbox for %s. %+v", namespace, err) + return + } + logger.Infof("succeeded starting cluster %s", namespace) + errCh <- nil +} + +func (mrc *MultiClusterDeploySuite) TearDownSuite() { if mrc.T().Failed() { gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace1) gatherAllRookLogs(mrc.k8sh, mrc.Suite, mrc.installer.Env.HostType, installer.SystemNamespace(mrc.namespace1), mrc.namespace2) @@ -127,45 +157,35 @@ func (mrc *MultiRookClusterDeploySuite) TearDownSuite() { } } - isRookUninstalled1 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", mrc.namespace1) - isNameSpaceDeleted1 := k8sHelp.WaitUntilNameSpaceIsDeleted(mrc.namespace1) - isRookUninstalled2 := k8sHelp.WaitUntilPodInNamespaceIsDeleted("rook-ceph-mon", mrc.namespace2) - isNameSpaceDeleted2 := k8sHelp.WaitUntilNameSpaceIsDeleted(mrc.namespace2) mrc.k8sh.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) - - if isRookUninstalled1 && isNameSpaceDeleted1 && isRookUninstalled2 && isNameSpaceDeleted2 { - logger.Infof("Rook clusters %s and %s uninstalled successfully", mrc.namespace1, mrc.namespace2) - return - } - logger.Infof("Rook clusters %s and %s not uninstalled successfully", mrc.namespace1, mrc.namespace2) - + logger.Infof("Rook clusters %s and %s uninstalled", mrc.namespace1, mrc.namespace2) } //Test to make sure all rook components are installed and Running -func (mrc *MultiRookClusterDeploySuite) TestInstallingMultipleRookClusters() { +func (mrc *MultiClusterDeploySuite) TestInstallingMultipleRookClusters() { //Check if Rook cluster 1 is deployed successfully - checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1) + checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace1, 1) checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper1, mrc.namespace1) //Check if Rook cluster 2 is deployed successfully - checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2) + checkIfRookClusterIsInstalled(mrc.Suite, mrc.k8sh, installer.SystemNamespace(mrc.namespace1), mrc.namespace2, 1) checkIfRookClusterIsHealthy(mrc.Suite, mrc.helper2, mrc.namespace2) } //Test Block Store Creation on multiple rook clusters -func (mrc *MultiRookClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() { +func (mrc *MultiClusterDeploySuite) TestBlockStoreOnMultipleRookCluster() { runBlockE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1) runBlockE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2) } //Test Filesystem Creation on multiple rook clusters -func (mrc *MultiRookClusterDeploySuite) TestFileStoreOnMultiRookCluster() { +func (mrc *MultiClusterDeploySuite) TestFileStoreOnMultiRookCluster() { runFileE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "test-fs-1") runFileE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "test-fs-2") } //Test Object Store Creation on multiple rook clusters -func (mrc *MultiRookClusterDeploySuite) TestObjectStoreOnMultiRookCluster() { +func (mrc *MultiClusterDeploySuite) TestObjectStoreOnMultiRookCluster() { runObjectE2ETestLite(mrc.helper1, mrc.k8sh, mrc.Suite, mrc.namespace1, "default-c1", 2) runObjectE2ETestLite(mrc.helper2, mrc.k8sh, mrc.Suite, mrc.namespace2, "default-c2", 1) } diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 978ed26bb..16ba52d66 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -27,7 +27,31 @@ import ( "github.com/stretchr/testify/suite" ) -func TestSmokeSuiteK8s(t *testing.T) { +// ************************************************ +// *** Major scenarios tested by the SmokeSuite *** +// Setup +// - via the cluster CRD +// Monitors +// - Three mons in the cluster +// - Failover of an unhealthy monitor +// OSDs +// - Bluestore running on devices +// Block +// - Mount/unmount a block device through the dynamic provisioner +// - Fencing of the block device +// - Read/write to the device +// File system +// - Create the file system via the CRD +// - Mount/unmount a file system in pod +// - Read/write to the file system +// - Delete the file system +// Object +// - Create the object store via the REST API +// - Create/delete buckets +// - Create/delete users +// - PUT/GET objects +// ************************************************ +func TestSmokeSuite(t *testing.T) { suite.Run(t, new(SmokeSuite)) } @@ -48,7 +72,7 @@ func (suite *SmokeSuite) SetupSuite() { suite.installer = installer.NewK8sRookhelper(kh.Clientset, suite.T) - isRookInstalled, err := suite.installer.InstallRookOnK8s(suite.namespace, "bluestore") + isRookInstalled, err := suite.installer.InstallRookOnK8sWithHostPathAndDevices(suite.namespace, "bluestore", "", true, 3) assert.NoError(suite.T(), err) if !isRookInstalled { logger.Errorf("Rook Was not installed successfully") @@ -84,5 +108,5 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { //Test to make sure all rook components are installed and Running func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() { - checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace) + checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3) } diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index 7798c05c3..cbe0a624d 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -22,7 +22,7 @@ func setUpRookAndPoolInNamespace(t func() *testing.T, namespace string, storageC i := installer.NewK8sRookhelper(kh.Clientset, t) if !kh.IsRookInstalled(namespace) { - isRookInstalled, err := i.InstallRookOnK8sWithHostPath(namespace, "bluestore", "/temp/rookBackup") + isRookInstalled, err := i.InstallRookOnK8sWithHostPathAndDevices(namespace, "bluestore", "/temp/rookBackup", true, 3) require.NoError(t(), err) require.True(t(), isRookInstalled) }