Files
my-rook-config/tests/integration/ceph_mgr_test.go
T
Juan Miguel Olmo Martínez 9edff582c3 ceph: enable again the Rook orchestrator mgr test
Enable again the mgr test:
- Now is more reliable and robust the start of the test.
- Minor fixes to adapt the <service ls>  to the new name of the crash daemon
deployed by rook

- The creation of OSDs is disabled in the orchestrator, so i have removed
this test until we will have the functionality ready again in the orchestrator
part

My plan is to provide in the orchestrator two different ways to create OSDs:
- Creation of OSD using specific devices if discovery daemon is running
- Creation of OSds using PVS (if we have LSO/other LS operator running)

Signed-off-by: Juan Miguel Olmo Martínez <jolmomar@redhat.com>
2021-07-28 16:27:42 +02:00

260 lines
7.3 KiB
Go

/*
Copyright 2020 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"encoding/json"
"fmt"
"github.com/stretchr/testify/require"
"os/exec"
"sort"
"testing"
"time"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
// **************************************************
// *** Mgr operations covered by TestMgrSmokeSuite ***
//
// Ceph orchestrator device ls
// Ceph orchestrator status
// Ceph orchestrator host ls
// Ceph orchestrator ls
// **************************************************
func TestCephMgrSuite(t *testing.T) {
if installer.SkipTestSuite(installer.CephTestSuite) {
t.Skip()
}
s := new(CephMgrSuite)
defer func(s *CephMgrSuite) {
HandlePanics(recover(), s.TearDownSuite, s.T)
}(s)
suite.Run(t, s)
}
type CephMgrSuite struct {
suite.Suite
settings *installer.TestCephSettings
k8sh *utils.K8sHelper
installer *installer.CephInstaller
namespace string
}
type host struct {
Addr string
Hostname string
Labels []string
Status string
}
type serviceStatus struct {
ContainerImageName string `json:"Container_image_name"`
LastRefresh string `json:"Last_refresh"`
Running int
Size int
}
type service struct {
ServiceName string `json:"Service_name"`
ServiceType string `json:"Service_type"`
Status serviceStatus
}
func (s *CephMgrSuite) SetupSuite() {
s.namespace = "mgr-ns"
s.settings = &installer.TestCephSettings{
ClusterName: s.namespace,
OperatorNamespace: installer.SystemNamespace(s.namespace),
Namespace: s.namespace,
StorageClassName: "",
UseHelm: false,
UsePVC: false,
Mons: 1,
UseCSI: true,
SkipOSDCreation: true,
EnableDiscovery: true,
RookVersion: installer.VersionMaster,
CephVersion: installer.MasterVersion,
}
s.settings.ApplyEnvVars()
s.installer, s.k8sh = StartTestCluster(s.T, s.settings, cephMasterSuiteMinimalTestVersion)
s.waitForOrchestrationModule()
}
func (s *CephMgrSuite) AfterTest(suiteName, testName string) {
s.installer.CollectOperatorLog(suiteName, testName)
}
func (s *CephMgrSuite) TearDownSuite() {
s.installer.UninstallRook()
}
func (s *CephMgrSuite) execute(command []string) (error, string) {
orchCommand := append([]string{"orch"}, command...)
return s.installer.Execute("ceph", orchCommand, s.namespace)
}
func (s *CephMgrSuite) enableOrchestratorModule() {
logger.Info("Enabling Rook orchestrator module: <ceph mgr module enable rook --force>")
err, output := s.installer.Execute("ceph", []string{"mgr", "module", "enable", "rook", "--force"}, s.namespace)
logger.Infof("output: %s", output)
if err != nil {
logger.Infof("Failed to enable rook orchestrator module: %q", err)
return
}
logger.Info("Setting orchestrator backend to Rook .... <ceph orch set backend rook>")
err, output = s.execute([]string{"set", "backend", "rook"})
logger.Infof("output: %s", output)
if err != nil {
logger.Infof("Not possible to set rook as backend orchestrator module: %q", err)
}
}
func (s *CephMgrSuite) waitForOrchestrationModule() {
var err error
// Status struct
type orchStatus struct {
Available bool `json:"available"`
Backend string `json:"backend"`
}
for timeout := 0; timeout < 30; timeout++ {
logger.Info("Waiting for rook orchestrator module enabled and ready ...")
err, output := s.execute([]string{"status", "--format", "json"})
logger.Infof("%s", output)
if err == nil {
logger.Info("Ceph orchestrator ready to execute commands")
// Get status information
bytes := []byte(output)
var status orchStatus
err := json.Unmarshal(bytes, &status)
if err != nil {
logger.Error("Error getting ceph orch status")
continue
}
if status.Backend != "rook" {
assert.Fail(s.T(), fmt.Sprintf("Orchestrator backend is <%q>. Setting it to <Rook>", status.Backend))
s.enableOrchestratorModule()
} else {
logger.Info("Orchestrator backend is <Rook>")
return
}
} else {
exitError, _ := err.(*exec.ExitError)
if exitError.ExitCode() == 22 { // The <ceph orch> commands are still not recognized
logger.Info("Ceph manager modules still not ready ... ")
} else if exitError.ExitCode() == 2 { // The rook orchestrator is not the orchestrator backend
s.enableOrchestratorModule()
}
}
time.Sleep(5 * time.Second)
}
if err != nil {
logger.Error("Giving up waiting for manager module to be ready")
}
require.Nil(s.T(), err)
}
func (s *CephMgrSuite) TestDeviceLs() {
logger.Info("Testing .... <ceph orch device ls>")
err, device_list := s.execute([]string{"device", "ls"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", device_list)
}
func (s *CephMgrSuite) TestStatus() {
logger.Info("Testing .... <ceph orch status>")
err, status := s.execute([]string{"status"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", status)
assert.Equal(s.T(), status, "Backend: rook\nAvailable: Yes")
}
func (s *CephMgrSuite) TestHostLs() {
logger.Info("Testing .... <ceph orch host ls>")
// Get the orchestrator hosts
err, output := s.execute([]string{"host", "ls", "json"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)
hosts := []byte(output)
var hostsList []host
err = json.Unmarshal(hosts, &hostsList)
if err != nil {
assert.Nil(s.T(), err)
}
var hostOutput []string
for _, hostItem := range hostsList {
hostOutput = append(hostOutput, hostItem.Addr)
}
sort.Strings(hostOutput)
// get the k8s nodes
nodes, err := k8sutil.GetNodeHostNames(s.k8sh.Clientset)
assert.Nil(s.T(), err)
k8sNodes := make([]string, 0, len(nodes))
for k := range nodes {
k8sNodes = append(k8sNodes, k)
}
sort.Strings(k8sNodes)
// nodes and hosts must be the same
assert.Equal(s.T(), hostOutput, k8sNodes)
}
func (s *CephMgrSuite) TestServiceLs() {
logger.Info("Testing .... <ceph orch ls --format json>")
err, output := s.execute([]string{"ls", "--format", "json"})
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)
services := []byte(output)
var servicesList []service
err = json.Unmarshal(services, &servicesList)
assert.Nil(s.T(), err)
labelFilter := ""
for _, svc := range servicesList {
if svc.ServiceName != "crash" {
labelFilter = fmt.Sprintf("app=rook-ceph-%s", svc.ServiceName)
} else {
labelFilter = "app=rook-ceph-crashcollector"
}
k8sPods, err := k8sutil.PodsRunningWithLabel(s.k8sh.Clientset, s.namespace, labelFilter)
logger.Infof("Service: %+v", svc)
logger.Infof("k8s pods for svc %q using label <%q>: %d", svc.ServiceName, labelFilter, k8sPods)
assert.Nil(s.T(), err)
assert.Equal(s.T(), svc.Status.Running, k8sPods, fmt.Sprintf("Wrong number of pods for kind of service <%s>", svc.ServiceType))
}
}