diff --git a/Jenkinsfile b/Jenkinsfile index bf1844b8d..52d64f676 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,6 +31,9 @@ pipeline { env.getLogs = "all" } + // When running in a PR we assuming it's not an official build + env.isOfficialBuild = "false" + if (!body.contains("[test full]")) { // By default run the min test matrix (all tests run, but will be distributed on different versions of K8s). // This only affects the PR builds. The master and release builds will always run the full matrix. @@ -111,10 +114,16 @@ pipeline { return env.shouldBuild != "false" && env.shouldTest != "false" && !params.skipIntegrationTests } } - steps{ + steps { + // If it's not a PR assume it is an "official" master or release build + script { + if (env.isOfficialBuild == "") { + env.isOfficialBuild = "true" + } + } sh 'cat _output/version | xargs tests/scripts/makeTestImages.sh save amd64' stash name: 'repo-amd64',includes: 'ceph-amd64.tar,cockroachdb-amd64.tar,cassandra-amd64.tar,nfs-amd64.tar,yugabytedb-amd64.tar,build/common.sh,_output/tests/linux_amd64/,_output/charts/,tests/scripts/' - script{ + script { def data = [ "aws_1.14.x": "v1.14.10", "aws_1.15.x": "v1.15.11", @@ -203,6 +212,7 @@ def RunIntegrationTest(k, v) { TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \ STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' \ TEST_ARGUMENTS='''+"${env.testArgs}"+''' \ + TEST_IS_OFFICIAL_BUILD='''+"${env.isOfficialBuild}"+''' \ TEST_SCRATCH_DEVICE=/dev/xvdc kubectl config view _output/tests/linux_amd64/integration -test.v -test.timeout 7200s 2>&1 | tee _output/tests/integrationTests.log''' diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 84f438fe6..f92a79f21 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -45,7 +45,9 @@ const ( // test with the latest nautilus build nautilusTestImage = "ceph/ceph:v14" // test with the latest octopus build - octopusTestImage = "ceph/ceph:v15" + octopusTestImage = "ceph/ceph:v15" + // test with the latest master image + masterTestImage = "ceph/daemon-base:latest-master-devel" helmChartName = "local/rook-ceph" helmDeployName = "rook-ceph" cephOperatorLabel = "app=rook-ceph-operator" @@ -56,6 +58,7 @@ var ( MimicVersion = cephv1.CephVersionSpec{Image: mimicTestImage} NautilusVersion = cephv1.CephVersionSpec{Image: nautilusTestImage} OctopusVersion = cephv1.CephVersionSpec{Image: octopusTestImage} + MasterVersion = cephv1.CephVersionSpec{Image: masterTestImage, AllowUnsupported: true} globalTestJobIndex = 0 ) @@ -101,6 +104,10 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { } } + if err = h.WipeDisks(namespace, 2); err != nil { + return err + } + rookOperator := h.Manifests.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) if err != nil { @@ -129,6 +136,10 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin } + if err = h.WipeDisks(namespace, 2); err != nil { + return err + } + return nil } @@ -152,9 +163,20 @@ func (h *CephInstaller) CreateRookToolbox(namespace string) (err error) { return nil } +// Execute a command in the ceph toolbox +func (h *CephInstaller) Execute(command string, parameters []string, namespace string) (error, string) { + cmd, args := client.FinalizeCephCommandArgs(command, parameters, h.k8shelper.MakeContext().ConfigDir, namespace, client.AdminUsername) + result, err := h.k8shelper.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...) + if err != nil { + logger.Warningf("Error executing command %q: <%v>", command, err) + return err, result + } + return nil, result +} + // CreateRookCluster creates rook cluster via kubectl func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, cephVersion cephv1.CephVersionSpec) error { + mon cephv1.MonSpec, startWithAllNodes bool, skipOSDCreation bool, cephVersion cephv1.CephVersionSpec) error { dataDirHostPath, err := h.initTestDir(namespace) if err != nil { @@ -195,15 +217,8 @@ osd_pool_default_size = 1 } } - if err := h.WipeClusterDisks(namespace); err != nil { - logger.Warningf("failed to wipe cluster disks. %+v. trying again...", err) - if err = h.WipeClusterDisks(namespace); err != nil { - return fmt.Errorf("failed to wipe cluster disks. %+v", err) - } - } - logger.Infof("Starting Rook Cluster with yaml") - settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, cephVersion} + settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, skipOSDCreation, cephVersion} rookCluster := h.Manifests.GetRookCluster(settings) if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil { return fmt.Errorf("Failed to create rook cluster : %v ", err) @@ -217,13 +232,19 @@ osd_pool_default_size = 1 return err } - if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", namespace, 1); err != nil { - return err + if !skipOSDCreation { + if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", namespace, 1); err != nil { + return err + } } logger.Infof("Rook Cluster started") - err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", namespace) - return err + if !skipOSDCreation { + err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", namespace) + return err + } + + return nil } // CreateRookExternalCluster creates rook external cluster via kubectl @@ -367,7 +388,7 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) { // InstallRook installs rook on k8s func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int) (bool, error) { + mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int, skipOSDCreation bool) (bool, error) { var err error k8sversion := h.k8shelper.GetK8sServerVersion() @@ -405,7 +426,8 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st // Create rook cluster err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName, - cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, h.CephVersion) + cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, + skipOSDCreation, h.CephVersion) if err != nil { logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) return false, err @@ -719,8 +741,21 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH return h } -// WipeClusterDisks runs a disk wipe job on all nodes in the k8s cluster. -func (h *CephInstaller) WipeClusterDisks(namespace string) error { +// Wipe cluster disks +func (h *CephInstaller) WipeDisks(namespace string, retries int) (err error) { + for i := 1; i <= retries; i++ { + if err = h.wipeClusterDisks(namespace); err == nil { + logger.Info("disks wiped successfully") + return nil + } else { + logger.Warningf("failed to wipe cluster disks. %+v. Attemp %d/%d ...", err, i, retries) + } + } + return fmt.Errorf("failed to wipe cluster disks. %+v", err) +} + +// wipeClusterDisks runs a disk wipe job on all nodes in the k8s cluster. +func (h *CephInstaller) wipeClusterDisks(namespace string) error { // Wipe clean disks on all nodes nodes, err := h.GetNodeHostnames() if err != nil { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 19bc0e763..406042228 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -62,6 +62,7 @@ type clusterSettings struct { RBDMirrorWorkers int UsePVCs bool StorageClassName string + skipOSDCreation bool CephVersion cephv1.CephVersionSpec } @@ -1924,9 +1925,9 @@ spec: skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: true - useAllDevices: true - deviceFilter: '' + useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + deviceFilter: '' config: ` + store + ` databaseSizeMB: "1024" diff --git a/tests/framework/installer/environment.go b/tests/framework/installer/environment.go index e593af28d..6a53353f1 100644 --- a/tests/framework/installer/environment.go +++ b/tests/framework/installer/environment.go @@ -40,6 +40,11 @@ func testStorageProvider() string { return getEnvVarWithDefault("STORAGE_PROVIDER_TESTS", "") } +// TestIsOfficialBuild gets the storage provider for which tests should be run +func TestIsOfficialBuild() bool { + return os.Getenv("TEST_IS_OFFICIAL_BUILD") == "true" +} + // baseTestDir gets the base test directory func baseTestDir() string { // If the base test directory is actively set to empty (as in CI), we use the current working directory. diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index 701b40919..84271e7e9 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -40,11 +40,12 @@ const ( // These versions are for running a minimal test suite for more efficient tests across different versions of K8s // instead of running all suites on all versions // To run on multiple versions, add a comma separate list such as 1.16.0,1.17.0 - flexDriverMinimalTestVersion = "1.14.0" - multiClusterMinimalTestVersion = "1.15.0" - helmMinimalTestVersion = "1.16.0" - upgradeMinimalTestVersion = "1.17.0" - smokeSuiteMinimalTestVersion = "1.18.0" + flexDriverMinimalTestVersion = "1.14.0" + cephMasterSuiteMinimalTestVersion = "1.15.0" + multiClusterMinimalTestVersion = "1.15.0" + helmMinimalTestVersion = "1.16.0" + upgradeMinimalTestVersion = "1.17.0" + smokeSuiteMinimalTestVersion = "1.18.0" ) var ( @@ -110,6 +111,7 @@ type TestCluster struct { mons int rbdMirrorWorkers int rookCephCleanup bool + skipOSDCreation bool minimalMatrixK8sVersion string rookVersion string cephVersion cephv1.CephVersionSpec @@ -166,7 +168,7 @@ func (op *TestCluster) Setup() { capnslog.SetGlobalLogLevel(capnslog.DEBUG) isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName, - cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers) + cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation) if !isRookInstalled || err != nil { logger.Errorf("Rook was not installed successfully: %v", err) diff --git a/tests/integration/ceph_flex_test.go b/tests/integration/ceph_flex_test.go index 41d602ed6..dd526050b 100644 --- a/tests/integration/ceph_flex_test.go +++ b/tests/integration/ceph_flex_test.go @@ -91,6 +91,7 @@ func (s *CephFlexDriverSuite) SetupSuite() { mons: 1, rbdMirrorWorkers: 1, rookCephCleanup: false, + skipOSDCreation: false, minimalMatrixK8sVersion: flexDriverMinimalTestVersion, rookVersion: installer.VersionMaster, cephVersion: installer.OctopusVersion, diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 829320aaf..278118ece 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -77,6 +77,7 @@ func (hs *HelmSuite) SetupSuite() { mons: 1, rbdMirrorWorkers: 1, rookCephCleanup: true, + skipOSDCreation: false, minimalMatrixK8sVersion: helmMinimalTestVersion, rookVersion: installer.VersionMaster, cephVersion: installer.NautilusVersion, diff --git a/tests/integration/ceph_mgr_test.go b/tests/integration/ceph_mgr_test.go new file mode 100644 index 000000000..a72b2fb31 --- /dev/null +++ b/tests/integration/ceph_mgr_test.go @@ -0,0 +1,250 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "testing" + "time" + + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// ************************************************** +// *** Mgr operations covered by TestMgrSmokeSuite *** +// +// Ceph orchestrator device ls +// Ceph orchestrator status +// Ceph orchestrator host ls +// Ceph orchestrator create OSD +// Ceph orchestrator ls +// ************************************************** +func TestCephMgrSuite(t *testing.T) { + if installer.SkipTestSuite(installer.CephTestSuite) { + t.Skip() + } + // Skip this test suite in master and release builds. If there is an issue + // running against Ceph master we don't want to block the official builds. + if installer.TestIsOfficialBuild() { + t.Skip() + } + + s := new(CephMgrSuite) + defer func(s *CephMgrSuite) { + HandlePanics(recover(), s.cluster, s.T) + }(s) + suite.Run(t, s) +} + +type CephMgrSuite struct { + suite.Suite + cluster *TestCluster + k8sh *utils.K8sHelper + namespace string +} + +type host struct { + Addr string + Hostname string + Labels []string + Status string +} + +type serviceStatus struct { + ContainerImageName string `json:"Container_image_name"` + LastRefresh string `json:"Last_refresh"` + Running int + Size int +} + +type service struct { + placement map[string]string + ServiceName string `json:"Service_name"` + ServiceType string `json:"Service_type"` + Status serviceStatus +} + +func (suite *CephMgrSuite) SetupSuite() { + suite.namespace = "mgr-ns" + + mgrTestCluster := TestCluster{ + clusterName: suite.namespace, + namespace: suite.namespace, + storeType: "bluestore", + storageClassName: "", + useHelm: false, + usePVC: false, + mons: 1, + rbdMirrorWorkers: 0, + rookCephCleanup: true, + skipOSDCreation: true, + minimalMatrixK8sVersion: cephMasterSuiteMinimalTestVersion, + rookVersion: installer.VersionMaster, + cephVersion: installer.MasterVersion, + } + + suite.cluster, suite.k8sh = StartTestCluster(suite.T, &mgrTestCluster) + suite.waitForOrchestrationModule() +} + +func (suite *CephMgrSuite) AfterTest(suiteName, testName string) { + suite.cluster.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace)) +} + +func (suite *CephMgrSuite) TearDownSuite() { + suite.cluster.Teardown() +} + +func (suite *CephMgrSuite) execute(command []string) (error, string) { + orchCommand := append([]string{"orch"}, command...) + return suite.cluster.installer.Execute("ceph", orchCommand, suite.namespace) +} + +func (suite *CephMgrSuite) waitForOrchestrationModule() { + for timeout := 0; timeout < 30; timeout++ { + err, output := suite.execute([]string{"status"}) + logger.Info("%s", output) + if err == nil { + logger.Info("Rook Toolbox ready to execute commands") + break + } + time.Sleep(2 * time.Second) + } +} +func (suite *CephMgrSuite) TestDeviceLs() { + logger.Info("Testing .... ") + err, device_list := suite.execute([]string{"device", "ls"}) + assert.Nil(suite.T(), err) + logger.Infof("output = %s", device_list) +} + +func (suite *CephMgrSuite) TestStatus() { + logger.Info("Testing .... ") + err, status := suite.execute([]string{"status"}) + assert.Nil(suite.T(), err) + logger.Infof("output = %s", status) + + assert.Equal(suite.T(), status, "Backend: rook\nAvailable: True") +} + +func (suite *CephMgrSuite) TestHostLs() { + logger.Info("Testing .... ") + + // Get the orchestrator hosts + err, output := suite.execute([]string{"host", "ls", "json"}) + assert.Nil(suite.T(), err) + logger.Infof("output = %s", output) + + hosts := []byte(output) + var hostsList []host + + err = json.Unmarshal(hosts, &hostsList) + if err != nil { + assert.Nil(suite.T(), err) + } + + var hostOutput []string + for _, hostItem := range hostsList { + hostOutput = append(hostOutput, hostItem.Addr) + } + sort.Strings(hostOutput) + + // get the k8s nodes + nodes, err := k8sutil.GetNodeHostNames(suite.k8sh.Clientset) + assert.Nil(suite.T(), err) + + k8sNodes := make([]string, 0, len(nodes)) + for k := range nodes { + k8sNodes = append(k8sNodes, k) + } + sort.Strings(k8sNodes) + + // nodes and hosts must be the same + assert.Equal(suite.T(), hostOutput, k8sNodes) +} + +func (suite *CephMgrSuite) TestCreateOSD() { + logger.Info("Testing .... ") + + // Get the first available device + err, deviceList := suite.execute([]string{"device", "ls", "--format", "json"}) + assert.Nil(suite.T(), err) + logger.Infof("output = %s", deviceList) + + inventory := make([]map[string]interface{}, 0) + + err = json.Unmarshal([]byte(deviceList), &inventory) + assert.Nil(suite.T(), err) + + selectedNode := "" + selectedDevice := "" + for _, node := range inventory { + for _, device := range node["devices"].([]interface{}) { + if device.(map[string]interface{})["available"].(bool) { + selectedNode = node["name"].(string) + selectedDevice = strings.TrimPrefix(device.(map[string]interface{})["path"].(string), "/dev/") + break + } + } + if selectedDevice != "" { + break + } + } + assert.NotEqual(suite.T(), "", selectedDevice, "No devices available to create test OSD") + assert.NotEqual(suite.T(), "", selectedNode, "No nodes available to create test OSD") + + if selectedDevice == "" || selectedNode == "" { + return + } + // Create the OSD + err, output := suite.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)}) + + assert.Nil(suite.T(), err) + logger.Infof("output = %s", output) + + err = suite.k8sh.WaitForPodCount("app=rook-ceph-osd", suite.namespace, 1) + assert.Nil(suite.T(), err) +} + +func (suite *CephMgrSuite) TestServiceLs() { + logger.Info("Testing .... ") + err, output := suite.execute([]string{"ls", "--format", "json"}) + assert.Nil(suite.T(), err) + logger.Infof("output = %s", output) + + services := []byte(output) + var servicesList []service + + err = json.Unmarshal(services, &servicesList) + assert.Nil(suite.T(), err) + + for _, svc := range servicesList { + labelFilter := fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName) + k8sPods, err := k8sutil.PodsRunningWithLabel(suite.k8sh.Clientset, suite.namespace, labelFilter) + logger.Infof("Service: %+v", svc) + logger.Infof("k8s pods for svc %q: %d", svc.ServiceName, k8sPods) + assert.Nil(suite.T(), err) + assert.Equal(suite.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName)) + } +} diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index 6c1863fda..13512d556 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -184,7 +184,7 @@ func (o MCTestOperations) Teardown() { func (o MCTestOperations) startCluster(namespace, store string) error { logger.Infof("starting cluster %s", namespace) err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName, - cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, installer.NautilusVersion) + cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, installer.NautilusVersion) if err != nil { o.T().Fail() o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace) diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 3cfafd28a..09a19e47e 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -87,6 +87,7 @@ func (suite *SmokeSuite) SetupSuite() { mons: 3, rbdMirrorWorkers: 1, rookCephCleanup: false, + skipOSDCreation: false, minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion, rookVersion: installer.VersionMaster, cephVersion: installer.OctopusVersion, diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 622cd671c..cdf1f0d64 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -82,6 +82,7 @@ func (s *UpgradeSuite) SetupSuite() { mons: 1, rbdMirrorWorkers: 0, rookCephCleanup: false, + skipOSDCreation: false, minimalMatrixK8sVersion: upgradeMinimalTestVersion, rookVersion: installer.Version1_2, cephVersion: installer.NautilusVersion,