forked from rook/rook
Blaine Gardner
2fd9c9a88c
Cleanup up overall usage of CephX statuses in CephCluster. Convert all CephX statuses to non-pointer types. There is no specific need for pointer types, and keeping these as non-pointers means there is not a risk of nil pointer exceptions in all the various Rook controllers for upgraded clusters. Move initialization of CephCluster CephX status items to `preMonStartupActions()`. This helps resolve 2 issues: 1. 'Uninitialized' state was being set for external CephClusters where CephX statuses are not relevant. 2. 'Uninitialized' state was being set for upgraded clusters whose key status cannot be determined. Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
619 lines
22 KiB
Go
619 lines
22 KiB
Go
/*
|
|
Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package mgr
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
|
"github.com/rook/rook/pkg/operator/ceph/controller"
|
|
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
|
|
testop "github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
apps "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
policyv1 "k8s.io/api/policy/v1"
|
|
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
)
|
|
|
|
const (
|
|
clusterName = "test"
|
|
clusterNamespace = "ns"
|
|
dummyVersionsRaw = `
|
|
{
|
|
"osd": {
|
|
"ceph version 20.2.0 (0000000000000000) tentacle (stable)": 3
|
|
}
|
|
}`
|
|
)
|
|
|
|
func createNewCluster(t *testing.T) *Cluster {
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("Execute: %s %v", command, args)
|
|
if args[0] == "mgr" && args[1] == "stat" {
|
|
return `{"active_name": "a"}`, nil
|
|
} else if args[0] == "auth" && args[1] == "get-or-create-key" {
|
|
return "{\"key\":\"mysecurekey\"}", nil
|
|
} else if args[0] == "auth" && args[1] == "rotate" {
|
|
return `[{"key":"myrotatedkey"}]`, nil
|
|
} else if args[0] == "versions" {
|
|
return dummyVersionsRaw, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
waitForDeploymentToStart = func(ctx context.Context, clusterdContext *clusterd.Context, deployment *apps.Deployment) error {
|
|
logger.Infof("simulated mgr deployment starting")
|
|
return nil
|
|
}
|
|
|
|
uninitializedStatus := keyring.UninitializedCephxStatus()
|
|
fakeCluster := &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: clusterName,
|
|
Namespace: clusterNamespace,
|
|
},
|
|
Spec: cephv1.ClusterSpec{
|
|
Security: cephv1.ClusterSecuritySpec{
|
|
CephX: cephv1.ClusterCephxConfig{
|
|
Daemon: cephv1.CephxConfig{},
|
|
},
|
|
},
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Cephx: cephv1.ClusterCephxStatus{
|
|
Mgr: uninitializedStatus,
|
|
},
|
|
},
|
|
}
|
|
|
|
clientset := testop.New(t, 3)
|
|
configDir := t.TempDir()
|
|
scheme := scheme.Scheme
|
|
err := policyv1.AddToScheme(scheme)
|
|
assert.NoError(t, err)
|
|
cl := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(fakeCluster).Build()
|
|
|
|
ctx := &clusterd.Context{
|
|
Executor: executor,
|
|
ConfigDir: configDir,
|
|
Clientset: clientset,
|
|
Client: cl,
|
|
}
|
|
ownerInfo := cephclient.NewMinimumOwnerInfo(t)
|
|
clusterInfo := &cephclient.ClusterInfo{Namespace: clusterNamespace, FSID: "myfsid", OwnerInfo: ownerInfo, CephVersion: cephver.CephVersion{Major: 17, Minor: 2, Build: 0}, Context: context.TODO()}
|
|
clusterInfo.SetName(clusterName)
|
|
clusterSpec := cephv1.ClusterSpec{
|
|
Annotations: map[cephv1.KeyType]cephv1.Annotations{cephv1.KeyMgr: {"my": "annotation"}},
|
|
Labels: map[cephv1.KeyType]cephv1.Labels{cephv1.KeyMgr: {"my-label-key": "value"}},
|
|
Dashboard: cephv1.DashboardSpec{Enabled: true, SSL: true},
|
|
Mgr: cephv1.MgrSpec{Count: 1},
|
|
PriorityClassNames: map[cephv1.KeyType]string{cephv1.KeyMgr: "my-priority-class"},
|
|
DataDirHostPath: "/var/lib/rook/",
|
|
}
|
|
c := New(ctx, clusterInfo, clusterSpec, "myversion")
|
|
defer os.RemoveAll(c.spec.DataDirHostPath)
|
|
|
|
return c
|
|
}
|
|
|
|
func TestStartMgr(t *testing.T) {
|
|
var deploymentsUpdated *[]*apps.Deployment
|
|
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
|
|
|
c := createNewCluster(t)
|
|
|
|
// start a basic service
|
|
err := c.Start()
|
|
assert.NoError(t, err)
|
|
validateStart(t, c)
|
|
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
|
|
c.spec.Dashboard.URLPrefix = "/test"
|
|
c.spec.Dashboard.Port = 12345
|
|
err = c.Start()
|
|
assert.NoError(t, err)
|
|
validateStart(t, c)
|
|
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
|
|
// starting with more replicas
|
|
c.spec.Mgr.Count = 2
|
|
c.spec.Dashboard.Enabled = false
|
|
// delete the previous mgr since the mocked test won't update the existing one
|
|
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{})
|
|
assert.NoError(t, err)
|
|
err = c.Start()
|
|
assert.NoError(t, err)
|
|
validateStart(t, c)
|
|
|
|
c.spec.Mgr.Count = 1
|
|
c.spec.Dashboard.Enabled = false
|
|
// clean the previous deployments
|
|
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{})
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, err)
|
|
err = c.Start()
|
|
assert.NoError(t, err)
|
|
validateStart(t, c)
|
|
}
|
|
|
|
func validateStart(t *testing.T, c *Cluster) {
|
|
mgrNames := []string{"a", "b"}
|
|
for i := 0; i < c.spec.Mgr.Count; i++ {
|
|
logger.Infof("Looking for cephmgr replica %d", i)
|
|
daemonName := mgrNames[i]
|
|
d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(context.TODO(), fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "annotation", d.Spec.Template.Annotations["my"])
|
|
assert.Contains(t, d.Spec.Template.Labels, "my-label-key")
|
|
assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName)
|
|
if c.spec.Mgr.Count == 1 {
|
|
assert.Equal(t, 1, len(d.Spec.Template.Spec.Containers))
|
|
} else {
|
|
// The sidecar container is only there when multiple mgrs are enabled
|
|
assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers))
|
|
assert.Equal(t, "watch-active", d.Spec.Template.Spec.Containers[1].Name)
|
|
}
|
|
}
|
|
|
|
// verify we have exactly the expected number of deployments and not extra
|
|
// the expected deployments were already retrieved above, but now we check for no extra deployments
|
|
options := metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"}
|
|
deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(context.TODO(), options)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, c.spec.Mgr.Count, len(deployments.Items))
|
|
|
|
validateServices(t, c)
|
|
}
|
|
|
|
func validateServices(t *testing.T, c *Cluster) {
|
|
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr-dashboard", metav1.GetOptions{})
|
|
if c.spec.Dashboard.Enabled {
|
|
assert.NoError(t, err)
|
|
if c.spec.Dashboard.Port == 0 {
|
|
// port=0 -> default port
|
|
assert.Equal(t, ds.Spec.Ports[0].Port, int32(dashboardPortHTTPS))
|
|
} else {
|
|
// non-zero ports are configured as-is
|
|
// nolint:gosec // G115 no overflow expected in the test
|
|
assert.Equal(t, ds.Spec.Ports[0].Port, int32(c.spec.Dashboard.Port))
|
|
}
|
|
} else {
|
|
assert.True(t, kerrors.IsNotFound(err))
|
|
}
|
|
}
|
|
|
|
func TestActiveMgrLabels(t *testing.T) {
|
|
var deploymentsUpdated *[]*apps.Deployment
|
|
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
|
|
|
c := createNewCluster(t)
|
|
err := c.Start()
|
|
assert.NoError(t, err)
|
|
validateStart(t, c)
|
|
|
|
mgrNames := []string{"a", "b"}
|
|
for i := 0; i < len(mgrNames); i++ {
|
|
// Simulate the Mgr pod having been created
|
|
daemonName := mgrNames[i]
|
|
mgrPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{
|
|
Name: fmt.Sprintf("rook-ceph-mgr-%s", daemonName),
|
|
Labels: map[string]string{
|
|
k8sutil.AppAttr: "rook-ceph-mgr",
|
|
"instance": daemonName,
|
|
"ceph_daemon_id": daemonName,
|
|
"mgr": daemonName,
|
|
"rook_cluster": clusterName,
|
|
},
|
|
}}
|
|
_, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, mgrPod, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
}
|
|
mgrA, mgrB := mgrNames[0], mgrNames[1]
|
|
|
|
err = c.SetMgrRoleLabel(mgrA, true)
|
|
assert.NoError(t, err, "set mgr a to active")
|
|
err = c.SetMgrRoleLabel(mgrB, false)
|
|
assert.NoError(t, err, "set mgr b to standby")
|
|
|
|
pods, err := c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
|
|
assert.NoError(t, err, "lookup all mgr pods")
|
|
assert.Len(t, pods.Items, 2)
|
|
sort.Slice(pods.Items, func(i, j int) bool {
|
|
// sort pods by name, so pod for mgrA will be 0 and for B - 1.
|
|
return pods.Items[i].GetName() < pods.Items[j].GetName()
|
|
})
|
|
|
|
assert.Equal(t, pods.Items[0].Labels["mgr"], mgrA)
|
|
assert.Equal(t, pods.Items[0].Labels["mgr_role"], "active", "mgr a is active")
|
|
|
|
assert.Equal(t, pods.Items[1].Labels["mgr"], mgrB)
|
|
assert.Equal(t, pods.Items[1].Labels["mgr_role"], "standby", "mgr b is standby")
|
|
|
|
err = c.SetMgrRoleLabel(mgrA, false)
|
|
assert.NoError(t, err, "set mgr a to standby")
|
|
|
|
pods, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
|
|
assert.NoError(t, err, "lookup all mgr pods")
|
|
assert.Len(t, pods.Items, 2)
|
|
sort.Slice(pods.Items, func(i, j int) bool {
|
|
// sort pods by name, so pod for mgrA will be 0 and for B - 1.
|
|
return pods.Items[i].GetName() < pods.Items[j].GetName()
|
|
})
|
|
|
|
assert.Equal(t, pods.Items[0].Labels["mgr"], mgrA)
|
|
assert.Equal(t, pods.Items[0].Labels["mgr_role"], "standby", "mgr a is now standby")
|
|
|
|
assert.Equal(t, pods.Items[1].Labels["mgr"], mgrB)
|
|
assert.Equal(t, pods.Items[1].Labels["mgr_role"], "standby", "mgr b is still standby")
|
|
|
|
err = c.SetMgrRoleLabel(mgrB, true)
|
|
assert.NoError(t, err, "set mgr b to active")
|
|
|
|
pods, err = c.context.Clientset.CoreV1().Pods(c.clusterInfo.Namespace).List(c.clusterInfo.Context, metav1.ListOptions{LabelSelector: "app=rook-ceph-mgr"})
|
|
assert.NoError(t, err, "lookup all mgr pods")
|
|
assert.Len(t, pods.Items, 2)
|
|
sort.Slice(pods.Items, func(i, j int) bool {
|
|
// sort pods by name, so pod for mgrA will be 0 and for B - 1.
|
|
return pods.Items[i].GetName() < pods.Items[j].GetName()
|
|
})
|
|
|
|
assert.Equal(t, pods.Items[0].Labels["mgr"], mgrA)
|
|
assert.Equal(t, pods.Items[0].Labels["mgr_role"], "standby", "mgr a is still standby")
|
|
|
|
assert.Equal(t, pods.Items[1].Labels["mgr"], mgrB)
|
|
assert.Equal(t, pods.Items[1].Labels["mgr_role"], "active", "mgr b is now active")
|
|
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
}
|
|
|
|
func TestUpdateServiceSelectors(t *testing.T) {
|
|
clientset := testop.New(t, 3)
|
|
ctx := &clusterd.Context{Clientset: clientset}
|
|
clusterInfo := cephclient.AdminTestClusterInfo(clusterNamespace)
|
|
spec := cephv1.ClusterSpec{
|
|
Dashboard: cephv1.DashboardSpec{
|
|
Enabled: true,
|
|
Port: 7000,
|
|
},
|
|
}
|
|
c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo}
|
|
|
|
// Make sure we remove the daemon_id label from the selector
|
|
// of all services with a label "app=rook-ceph-mgr"
|
|
t.Run("remove daemon_id from mgr services", func(t *testing.T) {
|
|
svc1 := corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "svc1",
|
|
Labels: map[string]string{
|
|
"app": "rook-ceph-mgr",
|
|
"svc": "rook-ceph-mgr",
|
|
controller.DaemonIDLabel: "a",
|
|
},
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Selector: map[string]string{
|
|
"app": "rook-ceph-mgr",
|
|
controller.DaemonIDLabel: "a",
|
|
},
|
|
},
|
|
}
|
|
svc2 := corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "svc2"},
|
|
Spec: corev1.ServiceSpec{
|
|
Selector: map[string]string{
|
|
"app": "rook-ceph-mgr",
|
|
controller.DaemonIDLabel: "a",
|
|
},
|
|
},
|
|
}
|
|
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Create(clusterInfo.Context, &svc1, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
_, err2 := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Create(clusterInfo.Context, &svc2, metav1.CreateOptions{})
|
|
assert.NoError(t, err2)
|
|
|
|
// Check the the label has been only removed from svc1
|
|
c.updateServiceSelectors()
|
|
|
|
// Make sure we remove controller.DaemonIDLabel label from all services tagged as 'app: rook-ceph-mgr'
|
|
// and we add the selector label "mgr_role" set to "active"
|
|
updatedService1, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(clusterInfo.Context, "svc1", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.NotContains(t, updatedService1.Spec.Selector, controller.DaemonIDLabel)
|
|
assert.Contains(t, updatedService1.Spec.Selector, "mgr_role")
|
|
assert.Equal(t, updatedService1.Spec.Selector["mgr_role"], "active")
|
|
|
|
// Make sure services not tagged as 'app: rook-ceph-mgr' are not modified
|
|
updatedService2, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(clusterInfo.Context, "svc2", metav1.GetOptions{})
|
|
assert.NoError(t, err)
|
|
assert.Contains(t, updatedService2.Spec.Selector, controller.DaemonIDLabel)
|
|
assert.NotContains(t, updatedService2.Spec.Selector, "mgr_role")
|
|
})
|
|
}
|
|
|
|
func TestConfigureModules(t *testing.T) {
|
|
modulesEnabled := 0
|
|
modulesDisabled := 0
|
|
configSettings := map[string]string{}
|
|
lastModuleConfigured := ""
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if command == "ceph" && len(args) > 3 {
|
|
if args[0] == "mgr" && args[1] == "module" {
|
|
if args[2] == "enable" {
|
|
modulesEnabled++
|
|
}
|
|
if args[2] == "disable" {
|
|
modulesDisabled++
|
|
}
|
|
lastModuleConfigured = args[3]
|
|
}
|
|
}
|
|
return "", nil // return "{\"key\":\"mysecurekey\"}", nil
|
|
},
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
if args[0] == "config" && args[1] == "set" && args[2] == "global" {
|
|
configSettings[args[3]] = args[4]
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
clientset := testop.New(t, 3)
|
|
context := &clusterd.Context{Executor: executor, Clientset: clientset}
|
|
clusterInfo := cephclient.AdminTestClusterInfo(clusterNamespace)
|
|
c := &Cluster{
|
|
context: context,
|
|
clusterInfo: clusterInfo,
|
|
}
|
|
|
|
// one module without any special configuration
|
|
c.spec.Mgr.Modules = []cephv1.Module{
|
|
{Name: "mymodule", Enabled: true},
|
|
}
|
|
assert.NoError(t, c.configureMgrModules())
|
|
assert.Equal(t, 1, modulesEnabled)
|
|
assert.Equal(t, 0, modulesDisabled)
|
|
assert.Equal(t, "mymodule", lastModuleConfigured)
|
|
|
|
// disable the module
|
|
modulesEnabled = 0
|
|
lastModuleConfigured = ""
|
|
configSettings = map[string]string{}
|
|
c.spec.Mgr.Modules[0].Enabled = false
|
|
assert.NoError(t, c.configureMgrModules())
|
|
assert.Equal(t, 0, modulesEnabled)
|
|
assert.Equal(t, 1, modulesDisabled)
|
|
assert.Equal(t, "mymodule", lastModuleConfigured)
|
|
assert.Equal(t, 0, len(configSettings))
|
|
}
|
|
|
|
func TestMgrDaemons(t *testing.T) {
|
|
spec := cephv1.ClusterSpec{
|
|
Mgr: cephv1.MgrSpec{Count: 1},
|
|
}
|
|
c := &Cluster{spec: spec}
|
|
daemons := c.getDaemonIDs()
|
|
require.Equal(t, 1, len(daemons))
|
|
assert.Equal(t, "a", daemons[0])
|
|
|
|
c.spec.Mgr.Count = 2
|
|
daemons = c.getDaemonIDs()
|
|
require.Equal(t, 2, len(daemons))
|
|
assert.Equal(t, "a", daemons[0])
|
|
assert.Equal(t, "b", daemons[1])
|
|
}
|
|
|
|
func TestApplyMonitoringLabels(t *testing.T) {
|
|
clusterSpec := cephv1.ClusterSpec{
|
|
Labels: cephv1.LabelsSpec{},
|
|
}
|
|
c := &Cluster{spec: clusterSpec}
|
|
sm := &monitoringv1.ServiceMonitor{Spec: monitoringv1.ServiceMonitorSpec{
|
|
Endpoints: []monitoringv1.Endpoint{{}},
|
|
}}
|
|
|
|
// Service Monitor RelabelConfigs updated when 'rook.io/managedBy' monitoring label is found
|
|
monitoringLabels := cephv1.LabelsSpec{
|
|
cephv1.KeyMonitoring: map[string]string{
|
|
"rook.io/managedBy": "storagecluster",
|
|
},
|
|
}
|
|
c.spec.Labels = monitoringLabels
|
|
applyMonitoringLabels(c, sm)
|
|
assert.Equal(t, "managedBy", sm.Spec.Endpoints[0].RelabelConfigs[0].TargetLabel)
|
|
assert.Equal(t, "storagecluster", *sm.Spec.Endpoints[0].RelabelConfigs[0].Replacement)
|
|
|
|
// Service Monitor RelabelConfigs not updated when the required monitoring label is not found
|
|
monitoringLabels = cephv1.LabelsSpec{
|
|
cephv1.KeyMonitoring: map[string]string{
|
|
"wrongLabelKey": "storagecluster",
|
|
},
|
|
}
|
|
c.spec.Labels = monitoringLabels
|
|
sm.Spec.Endpoints[0].RelabelConfigs = nil
|
|
applyMonitoringLabels(c, sm)
|
|
assert.Nil(t, sm.Spec.Endpoints[0].RelabelConfigs)
|
|
|
|
// Service Monitor RelabelConfigs not updated when no monitoring labels are found
|
|
c.spec.Labels = cephv1.LabelsSpec{}
|
|
sm.Spec.Endpoints[0].RelabelConfigs = nil
|
|
applyMonitoringLabels(c, sm)
|
|
assert.Nil(t, sm.Spec.Endpoints[0].RelabelConfigs)
|
|
}
|
|
|
|
func TestCluster_configurePrometheusModule(t *testing.T) {
|
|
modulesEnabled := 0
|
|
modulesDisabled := 0
|
|
configSettings := map[string]string{}
|
|
lastModuleConfigured := ""
|
|
modulesGetOutput := ""
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if command == "ceph" && len(args) > 3 {
|
|
if args[0] == "mgr" && args[1] == "module" {
|
|
if args[2] == "enable" {
|
|
modulesEnabled++
|
|
}
|
|
if args[2] == "disable" {
|
|
modulesDisabled++
|
|
}
|
|
lastModuleConfigured = args[3]
|
|
}
|
|
}
|
|
return "", nil // return "{\"key\":\"mysecurekey\"}", nil
|
|
},
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
if args[0] == "config" && args[1] == "set" && args[2] == "mgr" {
|
|
configSettings[args[3]] = args[4]
|
|
}
|
|
if args[0] == "config" && args[1] == "get" && args[2] == "mgr" {
|
|
return modulesGetOutput, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
c := &Cluster{
|
|
context: &clusterd.Context{Executor: executor, Clientset: testop.New(t, 3)},
|
|
clusterInfo: cephclient.AdminTestClusterInfo(clusterNamespace),
|
|
spec: cephv1.ClusterSpec{
|
|
Monitoring: cephv1.MonitoringSpec{
|
|
Enabled: false,
|
|
MetricsDisabled: true,
|
|
},
|
|
},
|
|
}
|
|
// Disable prometheus module
|
|
err := c.configurePrometheusModule()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 0, modulesEnabled)
|
|
assert.Equal(t, 1, modulesDisabled)
|
|
assert.Equal(t, PrometheusModuleName, lastModuleConfigured)
|
|
|
|
// Enable prometheus module, no changed
|
|
modulesDisabled = 0
|
|
c.spec.Monitoring.MetricsDisabled = false
|
|
err = c.configurePrometheusModule()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, modulesEnabled)
|
|
assert.Equal(t, 0, modulesDisabled)
|
|
|
|
// Enable prometheus module, port changed
|
|
modulesEnabled = 0
|
|
c.spec.Monitoring.Port = 30001
|
|
err = c.configurePrometheusModule()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 2, modulesEnabled)
|
|
assert.Equal(t, 1, modulesDisabled)
|
|
assert.Equal(t, "30001", configSettings["mgr/prometheus/server_port"])
|
|
assert.Equal(t, 1, len(configSettings))
|
|
|
|
modulesEnabled = 0
|
|
modulesDisabled = 0
|
|
c.spec.Monitoring.Interval = &metav1.Duration{
|
|
Duration: 30 * time.Second,
|
|
}
|
|
modulesGetOutput = "30"
|
|
configSettings = make(map[string]string)
|
|
err = c.configurePrometheusModule()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 2, modulesEnabled)
|
|
assert.Equal(t, 1, modulesDisabled)
|
|
assert.Equal(t, "30001", configSettings["mgr/prometheus/server_port"])
|
|
assert.Equal(t, 1, len(configSettings))
|
|
|
|
// Enable prometheus module, port and interval changed
|
|
modulesEnabled = 0
|
|
modulesDisabled = 0
|
|
c.spec.Monitoring.Port = 30002
|
|
c.spec.Monitoring.Interval = &metav1.Duration{
|
|
Duration: 60 * time.Second,
|
|
}
|
|
err = c.configurePrometheusModule()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 2, modulesEnabled)
|
|
assert.Equal(t, 1, modulesDisabled)
|
|
assert.Equal(t, "30002", configSettings["mgr/prometheus/server_port"])
|
|
assert.Equal(t, "60", configSettings["mgr/prometheus/scrape_interval"])
|
|
}
|
|
|
|
func TestMgrKeyRotation(t *testing.T) {
|
|
var deploymentsUpdated *[]*apps.Deployment
|
|
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
|
c := createNewCluster(t)
|
|
c.clusterInfo.CephVersion = keyring.CephAuthRotateSupportedVersion
|
|
|
|
// verify keyGeneration is set to 1 when the keys are created for the first time.
|
|
err := c.Start()
|
|
assert.NoError(t, err)
|
|
cluster := &cephv1.CephCluster{}
|
|
err = c.context.Client.Get(context.TODO(), c.clusterInfo.NamespacedName(), cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(1), cluster.Status.Cephx.Mgr.KeyGeneration)
|
|
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
|
|
// verify that keyGeneration is set to 2 when the mgr key is rotated.
|
|
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
|
|
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 2
|
|
err = c.context.Client.Update(context.TODO(), cluster)
|
|
assert.NoError(t, err)
|
|
|
|
err = c.Start()
|
|
assert.NoError(t, err)
|
|
err = c.context.Client.Get(context.TODO(), c.clusterInfo.NamespacedName(), cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(2), cluster.Status.Cephx.Mgr.KeyGeneration)
|
|
|
|
// verify that keyGeneration is set to 3 when the mgr key is rotated again.
|
|
cluster.Spec.Security.CephX.Daemon.KeyRotationPolicy = cephv1.KeyGenerationCephxKeyRotationPolicy
|
|
cluster.Spec.Security.CephX.Daemon.KeyGeneration = 3
|
|
err = c.context.Client.Update(context.TODO(), cluster)
|
|
assert.NoError(t, err)
|
|
|
|
err = c.Start()
|
|
assert.NoError(t, err)
|
|
err = c.context.Client.Get(context.TODO(), c.clusterInfo.NamespacedName(), cluster)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, uint32(3), cluster.Status.Cephx.Mgr.KeyGeneration)
|
|
}
|