Merge pull request #4913 from jmolmo/mgr_integration_tests

Ceph: manager integration tests
This commit is contained in:
Travis Nielsen authored and GitHub committed 2020-05-11 09:19:29 -06:00
commit b95b89f838
11 files changed
+337 -30

No files matched your search

Vendored
+12 -2
View File
@@ -31,6 +31,9 @@ pipeline {
env.getLogs = "all"
}
// When running in a PR we assuming it's not an official build
env.isOfficialBuild = "false"
if (!body.contains("[test full]")) {
// By default run the min test matrix (all tests run, but will be distributed on different versions of K8s).
// This only affects the PR builds. The master and release builds will always run the full matrix.
@@ -111,10 +114,16 @@ pipeline {
return env.shouldBuild != "false" && env.shouldTest != "false" && !params.skipIntegrationTests
}
}
steps{
steps {
// If it's not a PR assume it is an "official" master or release build
script {
if (env.isOfficialBuild == "") {
env.isOfficialBuild = "true"
}
}
sh 'cat _output/version | xargs tests/scripts/makeTestImages.sh save amd64'
stash name: 'repo-amd64',includes: 'ceph-amd64.tar,cockroachdb-amd64.tar,cassandra-amd64.tar,nfs-amd64.tar,yugabytedb-amd64.tar,build/common.sh,_output/tests/linux_amd64/,_output/charts/,tests/scripts/'
script{
script {
def data = [
"aws_1.14.x": "v1.14.10",
"aws_1.15.x": "v1.15.11",
@@ -203,6 +212,7 @@ def RunIntegrationTest(k, v) {
TEST_LOG_COLLECTION_LEVEL='''+"${env.getLogs}"+''' \
STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+''' \
TEST_ARGUMENTS='''+"${env.testArgs}"+''' \
TEST_IS_OFFICIAL_BUILD='''+"${env.isOfficialBuild}"+''' \
TEST_SCRATCH_DEVICE=/dev/xvdc
kubectl config view
_output/tests/linux_amd64/integration -test.v -test.timeout 7200s 2>&1 | tee _output/tests/integrationTests.log'''
+53 -18
View File
@@ -45,7 +45,9 @@ const (
// test with the latest nautilus build
nautilusTestImage = "ceph/ceph:v14"
// test with the latest octopus build
octopusTestImage = "ceph/ceph:v15"
octopusTestImage = "ceph/ceph:v15"
// test with the latest master image
masterTestImage = "ceph/daemon-base:latest-master-devel"
helmChartName = "local/rook-ceph"
helmDeployName = "rook-ceph"
cephOperatorLabel = "app=rook-ceph-operator"
@@ -56,6 +58,7 @@ var (
MimicVersion = cephv1.CephVersionSpec{Image: mimicTestImage}
NautilusVersion = cephv1.CephVersionSpec{Image: nautilusTestImage}
OctopusVersion = cephv1.CephVersionSpec{Image: octopusTestImage}
MasterVersion = cephv1.CephVersionSpec{Image: masterTestImage, AllowUnsupported: true}
globalTestJobIndex = 0
)
@@ -101,6 +104,10 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) {
}
}
if err = h.WipeDisks(namespace, 2); err != nil {
return err
}
rookOperator := h.Manifests.GetRookOperator(namespace)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...)
if err != nil {
@@ -129,6 +136,10 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin
}
if err = h.WipeDisks(namespace, 2); err != nil {
return err
}
return nil
}
@@ -152,9 +163,20 @@ func (h *CephInstaller) CreateRookToolbox(namespace string) (err error) {
return nil
}
// Execute a command in the ceph toolbox
func (h *CephInstaller) Execute(command string, parameters []string, namespace string) (error, string) {
cmd, args := client.FinalizeCephCommandArgs(command, parameters, h.k8shelper.MakeContext().ConfigDir, namespace, client.AdminUsername)
result, err := h.k8shelper.MakeContext().Executor.ExecuteCommandWithOutput(cmd, args...)
if err != nil {
logger.Warningf("Error executing command %q: <%v>", command, err)
return err, result
}
return nil, result
}
// CreateRookCluster creates rook cluster via kubectl
func (h *CephInstaller) CreateRookCluster(namespace, systemNamespace, storeType string, usePVC bool, storageClassName string,
mon cephv1.MonSpec, startWithAllNodes bool, cephVersion cephv1.CephVersionSpec) error {
mon cephv1.MonSpec, startWithAllNodes bool, skipOSDCreation bool, cephVersion cephv1.CephVersionSpec) error {
dataDirHostPath, err := h.initTestDir(namespace)
if err != nil {
@@ -195,15 +217,8 @@ osd_pool_default_size = 1
}
}
if err := h.WipeClusterDisks(namespace); err != nil {
logger.Warningf("failed to wipe cluster disks. %+v. trying again...", err)
if err = h.WipeClusterDisks(namespace); err != nil {
return fmt.Errorf("failed to wipe cluster disks. %+v", err)
}
}
logger.Infof("Starting Rook Cluster with yaml")
settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, cephVersion}
settings := &clusterSettings{h.clusterName, namespace, storeType, dataDirHostPath, mon.Count, 0, usePVC, storageClassName, skipOSDCreation, cephVersion}
rookCluster := h.Manifests.GetRookCluster(settings)
if _, err := h.k8shelper.KubectlWithStdin(rookCluster, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create rook cluster : %v ", err)
@@ -217,13 +232,19 @@ osd_pool_default_size = 1
return err
}
if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", namespace, 1); err != nil {
return err
if !skipOSDCreation {
if err := h.k8shelper.WaitForPodCount("app=rook-ceph-osd", namespace, 1); err != nil {
return err
}
}
logger.Infof("Rook Cluster started")
err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", namespace)
return err
if !skipOSDCreation {
err = h.k8shelper.WaitForLabeledPodsToRun("app=rook-ceph-osd", namespace)
return err
}
return nil
}
// CreateRookExternalCluster creates rook external cluster via kubectl
@@ -367,7 +388,7 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) {
// InstallRook installs rook on k8s
func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, storageClassName string,
mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int) (bool, error) {
mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int, skipOSDCreation bool) (bool, error) {
var err error
k8sversion := h.k8shelper.GetK8sServerVersion()
@@ -405,7 +426,8 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st
// Create rook cluster
err = h.CreateRookCluster(namespace, onamespace, storeType, usePVC, storageClassName,
cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes, h.CephVersion)
cephv1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes,
skipOSDCreation, h.CephVersion)
if err != nil {
logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err)
return false, err
@@ -719,8 +741,21 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH
return h
}
// WipeClusterDisks runs a disk wipe job on all nodes in the k8s cluster.
func (h *CephInstaller) WipeClusterDisks(namespace string) error {
// Wipe cluster disks
func (h *CephInstaller) WipeDisks(namespace string, retries int) (err error) {
for i := 1; i <= retries; i++ {
if err = h.wipeClusterDisks(namespace); err == nil {
logger.Info("disks wiped successfully")
return nil
} else {
logger.Warningf("failed to wipe cluster disks. %+v. Attemp %d/%d ...", err, i, retries)
}
}
return fmt.Errorf("failed to wipe cluster disks. %+v", err)
}
// wipeClusterDisks runs a disk wipe job on all nodes in the k8s cluster.
func (h *CephInstaller) wipeClusterDisks(namespace string) error {
// Wipe clean disks on all nodes
nodes, err := h.GetNodeHostnames()
if err != nil {
+4 -3
View File
@@ -62,6 +62,7 @@ type clusterSettings struct {
RBDMirrorWorkers int
UsePVCs bool
StorageClassName string
skipOSDCreation bool
CephVersion cephv1.CephVersionSpec
}
@@ -1924,9 +1925,9 @@ spec:
skipUpgradeChecks: true
metadataDevice:
storage:
useAllNodes: true
useAllDevices: true
deviceFilter: ''
useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + `
useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + `
deviceFilter: ''
config:
` + store + `
databaseSizeMB: "1024"
+5
View File
@@ -40,6 +40,11 @@ func testStorageProvider() string {
return getEnvVarWithDefault("STORAGE_PROVIDER_TESTS", "")
}
// TestIsOfficialBuild gets the storage provider for which tests should be run
func TestIsOfficialBuild() bool {
return os.Getenv("TEST_IS_OFFICIAL_BUILD") == "true"
}
// baseTestDir gets the base test directory
func baseTestDir() string {
// If the base test directory is actively set to empty (as in CI), we use the current working directory.
+8 -6
View File
@@ -40,11 +40,12 @@ const (
// These versions are for running a minimal test suite for more efficient tests across different versions of K8s
// instead of running all suites on all versions
// To run on multiple versions, add a comma separate list such as 1.16.0,1.17.0
flexDriverMinimalTestVersion = "1.14.0"
multiClusterMinimalTestVersion = "1.15.0"
helmMinimalTestVersion = "1.16.0"
upgradeMinimalTestVersion = "1.17.0"
smokeSuiteMinimalTestVersion = "1.18.0"
flexDriverMinimalTestVersion = "1.14.0"
cephMasterSuiteMinimalTestVersion = "1.15.0"
multiClusterMinimalTestVersion = "1.15.0"
helmMinimalTestVersion = "1.16.0"
upgradeMinimalTestVersion = "1.17.0"
smokeSuiteMinimalTestVersion = "1.18.0"
)
var (
@@ -110,6 +111,7 @@ type TestCluster struct {
mons int
rbdMirrorWorkers int
rookCephCleanup bool
skipOSDCreation bool
minimalMatrixK8sVersion string
rookVersion string
cephVersion cephv1.CephVersionSpec
@@ -166,7 +168,7 @@ func (op *TestCluster) Setup() {
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName,
cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers)
cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation)
if !isRookInstalled || err != nil {
logger.Errorf("Rook was not installed successfully: %v", err)
+1
View File
@@ -91,6 +91,7 @@ func (s *CephFlexDriverSuite) SetupSuite() {
mons: 1,
rbdMirrorWorkers: 1,
rookCephCleanup: false,
skipOSDCreation: false,
minimalMatrixK8sVersion: flexDriverMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.OctopusVersion,
+1
View File
@@ -77,6 +77,7 @@ func (hs *HelmSuite) SetupSuite() {
mons: 1,
rbdMirrorWorkers: 1,
rookCephCleanup: true,
skipOSDCreation: false,
minimalMatrixK8sVersion: helmMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.NautilusVersion,
+250
View File
@@ -0,0 +1,250 @@
/*
Copyright 2020 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"encoding/json"
"fmt"
"sort"
"strings"
"testing"
"time"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
// **************************************************
// *** Mgr operations covered by TestMgrSmokeSuite ***
//
// Ceph orchestrator device ls
// Ceph orchestrator status
// Ceph orchestrator host ls
// Ceph orchestrator create OSD
// Ceph orchestrator ls
// **************************************************
func TestCephMgrSuite(t *testing.T) {
if installer.SkipTestSuite(installer.CephTestSuite) {
t.Skip()
}
// Skip this test suite in master and release builds. If there is an issue
// running against Ceph master we don't want to block the official builds.
if installer.TestIsOfficialBuild() {
t.Skip()
}
s := new(CephMgrSuite)
defer func(s *CephMgrSuite) {
HandlePanics(recover(), s.cluster, s.T)
}(s)
suite.Run(t, s)
}
type CephMgrSuite struct {
suite.Suite
cluster *TestCluster
k8sh *utils.K8sHelper
namespace string
}
type host struct {
Addr string
Hostname string
Labels []string
Status string
}
type serviceStatus struct {
ContainerImageName string `json:"Container_image_name"`
LastRefresh string `json:"Last_refresh"`
Running int
Size int
}
type service struct {
placement map[string]string
ServiceName string `json:"Service_name"`
ServiceType string `json:"Service_type"`
Status serviceStatus
}
func (suite *CephMgrSuite) SetupSuite() {
suite.namespace = "mgr-ns"
mgrTestCluster := TestCluster{
clusterName: suite.namespace,
namespace: suite.namespace,
storeType: "bluestore",
storageClassName: "",
useHelm: false,
usePVC: false,
mons: 1,
rbdMirrorWorkers: 0,
rookCephCleanup: true,
skipOSDCreation: true,
minimalMatrixK8sVersion: cephMasterSuiteMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.MasterVersion,
}
suite.cluster, suite.k8sh = StartTestCluster(suite.T, &mgrTestCluster)
suite.waitForOrchestrationModule()
}
func (suite *CephMgrSuite) AfterTest(suiteName, testName string) {
suite.cluster.installer.CollectOperatorLog(suiteName, testName, installer.SystemNamespace(suite.namespace))
}
func (suite *CephMgrSuite) TearDownSuite() {
suite.cluster.Teardown()
}
func (suite *CephMgrSuite) execute(command []string) (error, string) {
orchCommand := append([]string{"orch"}, command...)
return suite.cluster.installer.Execute("ceph", orchCommand, suite.namespace)
}
func (suite *CephMgrSuite) waitForOrchestrationModule() {
for timeout := 0; timeout < 30; timeout++ {
err, output := suite.execute([]string{"status"})
logger.Info("%s", output)
if err == nil {
logger.Info("Rook Toolbox ready to execute commands")
break
}
time.Sleep(2 * time.Second)
}
}
func (suite *CephMgrSuite) TestDeviceLs() {
logger.Info("Testing .... <ceph orch device ls>")
err, device_list := suite.execute([]string{"device", "ls"})
assert.Nil(suite.T(), err)
logger.Infof("output = %s", device_list)
}
func (suite *CephMgrSuite) TestStatus() {
logger.Info("Testing .... <ceph orch status>")
err, status := suite.execute([]string{"status"})
assert.Nil(suite.T(), err)
logger.Infof("output = %s", status)
assert.Equal(suite.T(), status, "Backend: rook\nAvailable: True")
}
func (suite *CephMgrSuite) TestHostLs() {
logger.Info("Testing .... <ceph orch host ls>")
// Get the orchestrator hosts
err, output := suite.execute([]string{"host", "ls", "json"})
assert.Nil(suite.T(), err)
logger.Infof("output = %s", output)
hosts := []byte(output)
var hostsList []host
err = json.Unmarshal(hosts, &hostsList)
if err != nil {
assert.Nil(suite.T(), err)
}
var hostOutput []string
for _, hostItem := range hostsList {
hostOutput = append(hostOutput, hostItem.Addr)
}
sort.Strings(hostOutput)
// get the k8s nodes
nodes, err := k8sutil.GetNodeHostNames(suite.k8sh.Clientset)
assert.Nil(suite.T(), err)
k8sNodes := make([]string, 0, len(nodes))
for k := range nodes {
k8sNodes = append(k8sNodes, k)
}
sort.Strings(k8sNodes)
// nodes and hosts must be the same
assert.Equal(suite.T(), hostOutput, k8sNodes)
}
func (suite *CephMgrSuite) TestCreateOSD() {
logger.Info("Testing .... <ceph orch create OSD>")
// Get the first available device
err, deviceList := suite.execute([]string{"device", "ls", "--format", "json"})
assert.Nil(suite.T(), err)
logger.Infof("output = %s", deviceList)
inventory := make([]map[string]interface{}, 0)
err = json.Unmarshal([]byte(deviceList), &inventory)
assert.Nil(suite.T(), err)
selectedNode := ""
selectedDevice := ""
for _, node := range inventory {
for _, device := range node["devices"].([]interface{}) {
if device.(map[string]interface{})["available"].(bool) {
selectedNode = node["name"].(string)
selectedDevice = strings.TrimPrefix(device.(map[string]interface{})["path"].(string), "/dev/")
break
}
}
if selectedDevice != "" {
break
}
}
assert.NotEqual(suite.T(), "", selectedDevice, "No devices available to create test OSD")
assert.NotEqual(suite.T(), "", selectedNode, "No nodes available to create test OSD")
if selectedDevice == "" || selectedNode == "" {
return
}
// Create the OSD
err, output := suite.execute([]string{"daemon", "add", "osd", fmt.Sprintf("%s:%s", selectedNode, selectedDevice)})
assert.Nil(suite.T(), err)
logger.Infof("output = %s", output)
err = suite.k8sh.WaitForPodCount("app=rook-ceph-osd", suite.namespace, 1)
assert.Nil(suite.T(), err)
}
func (suite *CephMgrSuite) TestServiceLs() {
logger.Info("Testing .... <ceph orch ls>")
err, output := suite.execute([]string{"ls", "--format", "json"})
assert.Nil(suite.T(), err)
logger.Infof("output = %s", output)
services := []byte(output)
var servicesList []service
err = json.Unmarshal(services, &servicesList)
assert.Nil(suite.T(), err)
for _, svc := range servicesList {
labelFilter := fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName)
k8sPods, err := k8sutil.PodsRunningWithLabel(suite.k8sh.Clientset, suite.namespace, labelFilter)
logger.Infof("Service: %+v", svc)
logger.Infof("k8s pods for svc %q: %d", svc.ServiceName, k8sPods)
assert.Nil(suite.T(), err)
assert.Equal(suite.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceName))
}
}
+1 -1
View File
@@ -184,7 +184,7 @@ func (o MCTestOperations) Teardown() {
func (o MCTestOperations) startCluster(namespace, store string) error {
logger.Infof("starting cluster %s", namespace)
err := o.installer.CreateRookCluster(namespace, o.systemNamespace, store, o.testOverPVC, o.storageClassName,
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, installer.NautilusVersion)
cephv1.MonSpec{Count: 1, AllowMultiplePerNode: true}, true, false, installer.NautilusVersion)
if err != nil {
o.T().Fail()
o.installer.GatherAllRookLogs(o.T().Name(), namespace, o.systemNamespace)
+1
View File
@@ -87,6 +87,7 @@ func (suite *SmokeSuite) SetupSuite() {
mons: 3,
rbdMirrorWorkers: 1,
rookCephCleanup: false,
skipOSDCreation: false,
minimalMatrixK8sVersion: smokeSuiteMinimalTestVersion,
rookVersion: installer.VersionMaster,
cephVersion: installer.OctopusVersion,
+1
View File
@@ -82,6 +82,7 @@ func (s *UpgradeSuite) SetupSuite() {
mons: 1,
rbdMirrorWorkers: 0,
rookCephCleanup: false,
skipOSDCreation: false,
minimalMatrixK8sVersion: upgradeMinimalTestVersion,
rookVersion: installer.Version1_2,
cephVersion: installer.NautilusVersion,