Rook orchestrator is going to use PVs as <devices> for OSDs. These PVs must use one storage class present in the cluster and configured properly in the manager. With this change we include this new requirement in the Rook manager module test Signed-off-by: Juan Miguel Olmo Martínez <jolmomar@redhat.com>
286 lines
8.3 KiB
Go
286 lines
8.3 KiB
Go
/*
|
|
Copyright 2020 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package integration
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"github.com/stretchr/testify/require"
|
|
"os/exec"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
"github.com/rook/rook/tests/framework/installer"
|
|
"github.com/rook/rook/tests/framework/utils"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/suite"
|
|
)
|
|
|
|
// **************************************************
|
|
// *** Mgr operations covered by TestMgrSmokeSuite ***
|
|
//
|
|
// Ceph orchestrator device ls
|
|
// Ceph orchestrator status
|
|
// Ceph orchestrator host ls
|
|
// Ceph orchestrator ls
|
|
// **************************************************
|
|
func TestCephMgrSuite(t *testing.T) {
|
|
if installer.SkipTestSuite(installer.CephTestSuite) {
|
|
t.Skip()
|
|
}
|
|
|
|
s := new(CephMgrSuite)
|
|
defer func(s *CephMgrSuite) {
|
|
HandlePanics(recover(), s.TearDownSuite, s.T)
|
|
}(s)
|
|
suite.Run(t, s)
|
|
}
|
|
|
|
type CephMgrSuite struct {
|
|
suite.Suite
|
|
settings *installer.TestCephSettings
|
|
k8sh *utils.K8sHelper
|
|
installer *installer.CephInstaller
|
|
namespace string
|
|
}
|
|
|
|
type host struct {
|
|
Addr string
|
|
Hostname string
|
|
Labels []string
|
|
Status string
|
|
}
|
|
|
|
type serviceStatus struct {
|
|
ContainerImageName string `json:"Container_image_name"`
|
|
LastRefresh string `json:"Last_refresh"`
|
|
Running int
|
|
Size int
|
|
}
|
|
|
|
type service struct {
|
|
ServiceName string `json:"Service_name"`
|
|
ServiceType string `json:"Service_type"`
|
|
Status serviceStatus
|
|
}
|
|
|
|
func (s *CephMgrSuite) SetupSuite() {
|
|
s.namespace = "mgr-ns"
|
|
|
|
s.settings = &installer.TestCephSettings{
|
|
ClusterName: s.namespace,
|
|
OperatorNamespace: installer.SystemNamespace(s.namespace),
|
|
Namespace: s.namespace,
|
|
StorageClassName: "",
|
|
UseHelm: false,
|
|
UsePVC: false,
|
|
Mons: 1,
|
|
UseCSI: true,
|
|
SkipOSDCreation: true,
|
|
EnableDiscovery: true,
|
|
RookVersion: installer.VersionMaster,
|
|
CephVersion: installer.MasterVersion,
|
|
}
|
|
s.settings.ApplyEnvVars()
|
|
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion)
|
|
s.waitForOrchestrationModule()
|
|
s.prepareLocalStorageClass("local-storage")
|
|
}
|
|
|
|
func (s *CephMgrSuite) AfterTest(suiteName, testName string) {
|
|
s.installer.CollectOperatorLog(suiteName, testName)
|
|
}
|
|
|
|
func (s *CephMgrSuite) TearDownSuite() {
|
|
_ = s.k8sh.DeleteResource("sc", "local-storage")
|
|
s.installer.UninstallRook()
|
|
}
|
|
|
|
func (s *CephMgrSuite) execute(command []string) (error, string) {
|
|
orchCommand := append([]string{"orch"}, command...)
|
|
return s.installer.Execute("ceph", orchCommand, s.namespace)
|
|
}
|
|
|
|
func (s *CephMgrSuite) prepareLocalStorageClass(storageClassName string) {
|
|
// Rook orchestrator use PVs based in this storage class to create OSDs
|
|
// It is also needed to list "devices"
|
|
localStorageClass := `
|
|
kind: StorageClass
|
|
apiVersion: storage.k8s.io/v1
|
|
metadata:
|
|
name: ` + storageClassName + `
|
|
provisioner: kubernetes.io/no-provisioner
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
`
|
|
err := s.k8sh.ResourceOperation("apply", localStorageClass)
|
|
if err == nil {
|
|
err, _ = s.installer.Execute("ceph", []string{"config", "set", "mgr", "mgr/rook/storage_class", storageClassName}, s.namespace)
|
|
if err == nil {
|
|
logger.Infof("Storage class %q set in manager config", storageClassName)
|
|
} else {
|
|
assert.Fail(s.T(), fmt.Sprintf("Error configuring local storage class in manager config: %q", err))
|
|
}
|
|
} else {
|
|
assert.Fail(s.T(), fmt.Sprintf("Error creating local storage class: %q ", err))
|
|
}
|
|
}
|
|
|
|
func (s *CephMgrSuite) enableOrchestratorModule() {
|
|
logger.Info("Enabling Rook orchestrator module: <ceph mgr module enable rook --force>")
|
|
err, output := s.installer.Execute("ceph", []string{"mgr", "module", "enable", "rook", "--force"}, s.namespace)
|
|
logger.Infof("output: %s", output)
|
|
if err != nil {
|
|
logger.Infof("Failed to enable rook orchestrator module: %q", err)
|
|
return
|
|
}
|
|
|
|
logger.Info("Setting orchestrator backend to Rook .... <ceph orch set backend rook>")
|
|
err, output = s.execute([]string{"set", "backend", "rook"})
|
|
logger.Infof("output: %s", output)
|
|
if err != nil {
|
|
logger.Infof("Not possible to set rook as backend orchestrator module: %q", err)
|
|
}
|
|
}
|
|
|
|
func (s *CephMgrSuite) waitForOrchestrationModule() {
|
|
var err error
|
|
|
|
// Status struct
|
|
type orchStatus struct {
|
|
Available bool `json:"available"`
|
|
Backend string `json:"backend"`
|
|
}
|
|
|
|
for timeout := 0; timeout < 30; timeout++ {
|
|
logger.Info("Waiting for rook orchestrator module enabled and ready ...")
|
|
err, output := s.execute([]string{"status", "--format", "json"})
|
|
logger.Infof("%s", output)
|
|
if err == nil {
|
|
logger.Info("Ceph orchestrator ready to execute commands")
|
|
|
|
// Get status information
|
|
bytes := []byte(output)
|
|
var status orchStatus
|
|
err := json.Unmarshal(bytes, &status)
|
|
if err != nil {
|
|
logger.Error("Error getting ceph orch status")
|
|
continue
|
|
}
|
|
|
|
if status.Backend != "rook" {
|
|
assert.Fail(s.T(), fmt.Sprintf("Orchestrator backend is <%q>. Setting it to <Rook>", status.Backend))
|
|
s.enableOrchestratorModule()
|
|
} else {
|
|
logger.Info("Orchestrator backend is <Rook>")
|
|
return
|
|
}
|
|
} else {
|
|
exitError, _ := err.(*exec.ExitError)
|
|
if exitError.ExitCode() == 22 { // The <ceph orch> commands are still not recognized
|
|
logger.Info("Ceph manager modules still not ready ... ")
|
|
} else if exitError.ExitCode() == 2 { // The rook orchestrator is not the orchestrator backend
|
|
s.enableOrchestratorModule()
|
|
}
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
}
|
|
if err != nil {
|
|
logger.Error("Giving up waiting for manager module to be ready")
|
|
}
|
|
require.Nil(s.T(), err)
|
|
}
|
|
func (s *CephMgrSuite) TestDeviceLs() {
|
|
logger.Info("Testing .... <ceph orch device ls>")
|
|
err, device_list := s.execute([]string{"device", "ls"})
|
|
assert.Nil(s.T(), err)
|
|
logger.Infof("output = %s", device_list)
|
|
}
|
|
|
|
func (s *CephMgrSuite) TestStatus() {
|
|
logger.Info("Testing .... <ceph orch status>")
|
|
err, status := s.execute([]string{"status"})
|
|
assert.Nil(s.T(), err)
|
|
logger.Infof("output = %s", status)
|
|
|
|
assert.Equal(s.T(), status, "Backend: rook\nAvailable: Yes")
|
|
}
|
|
|
|
func (s *CephMgrSuite) TestHostLs() {
|
|
logger.Info("Testing .... <ceph orch host ls>")
|
|
|
|
// Get the orchestrator hosts
|
|
err, output := s.execute([]string{"host", "ls", "json"})
|
|
assert.Nil(s.T(), err)
|
|
logger.Infof("output = %s", output)
|
|
|
|
hosts := []byte(output)
|
|
var hostsList []host
|
|
|
|
err = json.Unmarshal(hosts, &hostsList)
|
|
if err != nil {
|
|
assert.Nil(s.T(), err)
|
|
}
|
|
|
|
var hostOutput []string
|
|
for _, hostItem := range hostsList {
|
|
hostOutput = append(hostOutput, hostItem.Addr)
|
|
}
|
|
sort.Strings(hostOutput)
|
|
|
|
// get the k8s nodes
|
|
nodes, err := k8sutil.GetNodeHostNames(s.k8sh.Clientset)
|
|
assert.Nil(s.T(), err)
|
|
|
|
k8sNodes := make([]string, 0, len(nodes))
|
|
for k := range nodes {
|
|
k8sNodes = append(k8sNodes, k)
|
|
}
|
|
sort.Strings(k8sNodes)
|
|
|
|
// nodes and hosts must be the same
|
|
assert.Equal(s.T(), hostOutput, k8sNodes)
|
|
}
|
|
|
|
func (s *CephMgrSuite) TestServiceLs() {
|
|
logger.Info("Testing .... <ceph orch ls --format json>")
|
|
err, output := s.execute([]string{"ls", "--format", "json"})
|
|
assert.Nil(s.T(), err)
|
|
logger.Infof("output = %s", output)
|
|
|
|
services := []byte(output)
|
|
var servicesList []service
|
|
|
|
err = json.Unmarshal(services, &servicesList)
|
|
assert.Nil(s.T(), err)
|
|
|
|
labelFilter := ""
|
|
for _, svc := range servicesList {
|
|
if svc.ServiceName != "crash" {
|
|
labelFilter = fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName)
|
|
} else {
|
|
labelFilter = "app=rook-ceph-crashcollector"
|
|
}
|
|
k8sPods, err := k8sutil.PodsRunningWithLabel(s.k8sh.Clientset, s.namespace, labelFilter)
|
|
logger.Infof("Service: %+v", svc)
|
|
logger.Infof("k8s pods for svc %q using label <%q>: %d", svc.ServiceName, labelFilter, k8sPods)
|
|
assert.Nil(s.T(), err)
|
|
assert.Equal(s.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceType))
|
|
}
|
|
}
|