We can now prioritize orchestrations on certain event. Today only two
events will cancel on-going orchestrations (if any):
* request for cluster deletion
* request for cluster upgrade
If one of the two are caught by the watcher we will cancel the on-going
orchestration.
For that we implemented a simple approach based on check points, where
we will check for a cancellation request in certain part of the
orchestration. Mainly before each mons/mgr/osds orchestration loops.
This solution is not perfect, but we are waiting for the
controller-runtime to release its 0.7 version which will embed context
support. With that we will be able to cancel reconciles more precisely
and rapidly.
Operator log example:
```
2020-11-24 13:54:59.499719 I | op-mon: parsing mon endpoints: a=10.109.126.120:6789
2020-11-24 13:54:59.499719 I | op-mon: parsing mon endpoints: a=10.109.126.120:6789
2020-11-25 12:59:12.986264 I | ceph-cluster-controller: done reconciling ceph cluster in namespace "rook-ceph"
2020-11-25 13:07:33.776947 I | ceph-cluster-controller: CR has changed for "rook-ceph". diff= v1.ClusterSpec{
CephVersion: v1.CephVersionSpec{
Image: "ceph/ceph:v15.2.5",
- AllowUnsupported: true,
+ AllowUnsupported: false,
},
DriveGroups: nil,
Storage: {UseAllNodes: true, Selection: {UseAllDevices: &true}},
... // 20 identical fields
}
2020-11-25 13:07:33.777039 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph"
2020-11-25 13:07:33.785088 I | op-mon: parsing mon endpoints: a=10.107.242.49:6789,b=10.109.71.30:6789,c=10.98.93.224:6789
2020-11-25 13:07:33.788626 I | ceph-cluster-controller: detecting the ceph image version for image ceph/ceph:v15.2.5...
2020-11-25 13:07:35.280789 I | ceph-cluster-controller: detected ceph image version: "15.2.5-0 octopus"
2020-11-25 13:07:35.280806 I | ceph-cluster-controller: validating ceph version from provided image
2020-11-25 13:07:35.285888 I | op-mon: parsing mon endpoints: a=10.107.242.49:6789,b=10.109.71.30:6789,c=10.98.93.224:6789
2020-11-25 13:07:35.287828 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config
2020-11-25 13:07:35.288082 I | cephclient: generated admin config in /var/lib/rook/rook-ceph
2020-11-25 13:07:35.621625 I | ceph-cluster-controller: cluster "rook-ceph": version "15.2.5-0 octopus" detected for image "ceph/ceph:v15.2.5"
2020-11-25 13:07:35.642688 I | op-mon: start running mons
2020-11-25 13:07:35.646323 I | op-mon: parsing mon endpoints: a=10.107.242.49:6789,b=10.109.71.30:6789,c=10.98.93.224:6789
2020-11-25 13:07:35.654070 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.107.242.49:6789","10.109.71.30:6789","10.98.93.224:6789"]}] data:a=10.107.242.49:6789,b=10.109.71.30:6789,c=10.98.93.224:6789 mapping:{"node":{"a":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.3"},"b":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.3"},"c":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.3"}}} maxMonId:2]
2020-11-25 13:07:35.868253 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config
2020-11-25 13:07:35.868573 I | cephclient: generated admin config in /var/lib/rook/rook-ceph
2020-11-25 13:07:37.074353 I | op-mon: targeting the mon count 3
2020-11-25 13:07:38.153435 I | op-mon: checking for basic quorum with existing mons
2020-11-25 13:07:38.178029 I | op-mon: mon "a" endpoint is [v2:10.107.242.49:3300,v1:10.107.242.49:6789]
2020-11-25 13:07:38.670191 I | op-mon: mon "b" endpoint is [v2:10.109.71.30:3300,v1:10.109.71.30:6789]
2020-11-25 13:07:39.477820 I | op-mon: mon "c" endpoint is [v2:10.98.93.224:3300,v1:10.98.93.224:6789]
2020-11-25 13:07:39.874094 I | op-mon: saved mon endpoints to config map map[csi-cluster-config-json:[{"clusterID":"rook-ceph","monitors":["10.107.242.49:6789","10.109.71.30:6789","10.98.93.224:6789"]}] data:a=10.107.242.49:6789,b=10.109.71.30:6789,c=10.98.93.224:6789 mapping:{"node":{"a":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.3"},"b":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.3"},"c":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.3"}}} maxMonId:2]
2020-11-25 13:07:40.467999 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config
2020-11-25 13:07:40.469733 I | cephclient: generated admin config in /var/lib/rook/rook-ceph
2020-11-25 13:07:41.071710 I | cephclient: writing config file /var/lib/rook/rook-ceph/rook-ceph.config
2020-11-25 13:07:41.078903 I | cephclient: generated admin config in /var/lib/rook/rook-ceph
2020-11-25 13:07:41.125233 I | op-mon: deployment for mon rook-ceph-mon-a already exists. updating if needed
2020-11-25 13:07:41.327778 I | op-k8sutil: updating deployment "rook-ceph-mon-a" after verifying it is safe to stop
2020-11-25 13:07:41.327895 I | op-mon: checking if we can stop the deployment rook-ceph-mon-a
2020-11-25 13:07:44.045644 I | op-k8sutil: finished waiting for updated deployment "rook-ceph-mon-a"
2020-11-25 13:07:44.045706 I | op-mon: checking if we can continue the deployment rook-ceph-mon-a
2020-11-25 13:07:44.045740 I | op-mon: waiting for mon quorum with [a b c]
2020-11-25 13:07:44.109159 I | op-mon: mons running: [a b c]
2020-11-25 13:07:44.474596 I | op-mon: Monitors in quorum: [a b c]
2020-11-25 13:07:44.478565 I | op-mon: deployment for mon rook-ceph-mon-b already exists. updating if needed
2020-11-25 13:07:44.493374 I | op-k8sutil: updating deployment "rook-ceph-mon-b" after verifying it is safe to stop
2020-11-25 13:07:44.493403 I | op-mon: checking if we can stop the deployment rook-ceph-mon-b
2020-11-25 13:07:47.135524 I | op-k8sutil: finished waiting for updated deployment "rook-ceph-mon-b"
2020-11-25 13:07:47.135542 I | op-mon: checking if we can continue the deployment rook-ceph-mon-b
2020-11-25 13:07:47.135551 I | op-mon: waiting for mon quorum with [a b c]
2020-11-25 13:07:47.148820 I | op-mon: mons running: [a b c]
2020-11-25 13:07:47.445946 I | op-mon: Monitors in quorum: [a b c]
2020-11-25 13:07:47.448991 I | op-mon: deployment for mon rook-ceph-mon-c already exists. updating if needed
2020-11-25 13:07:47.462041 I | op-k8sutil: updating deployment "rook-ceph-mon-c" after verifying it is safe to stop
2020-11-25 13:07:47.462060 I | op-mon: checking if we can stop the deployment rook-ceph-mon-c
2020-11-25 13:07:48.853118 I | ceph-cluster-controller: CR has changed for "rook-ceph". diff= v1.ClusterSpec{
CephVersion: v1.CephVersionSpec{
- Image: "ceph/ceph:v15.2.5",
+ Image: "ceph/ceph:v15.2.6",
AllowUnsupported: false,
},
DriveGroups: nil,
Storage: {UseAllNodes: true, Selection: {UseAllDevices: &true}},
... // 20 identical fields
}
2020-11-25 13:07:48.853140 I | ceph-cluster-controller: upgrade requested, cancelling any ongoing orchestration
2020-11-25 13:07:50.119584 I | op-k8sutil: finished waiting for updated deployment "rook-ceph-mon-c"
2020-11-25 13:07:50.119606 I | op-mon: checking if we can continue the deployment rook-ceph-mon-c
2020-11-25 13:07:50.119619 I | op-mon: waiting for mon quorum with [a b c]
2020-11-25 13:07:50.130860 I | op-mon: mons running: [a b c]
2020-11-25 13:07:50.431341 I | op-mon: Monitors in quorum: [a b c]
2020-11-25 13:07:50.431361 I | op-mon: mons created: 3
2020-11-25 13:07:50.734156 I | op-mon: waiting for mon quorum with [a b c]
2020-11-25 13:07:50.745763 I | op-mon: mons running: [a b c]
2020-11-25 13:07:51.045108 I | op-mon: Monitors in quorum: [a b c]
2020-11-25 13:07:51.054497 E | ceph-cluster-controller: failed to reconcile. failed to reconcile cluster "rook-ceph": failed to configure local ceph cluster: failed to create cluster: CANCELLING CURRENT ORCHESTATION
2020-11-25 13:07:52.055208 I | ceph-cluster-controller: reconciling ceph cluster in namespace "rook-ceph"
2020-11-25 13:07:52.070690 I | op-mon: parsing mon endpoints: a=10.107.242.49:6789,b=10.109.71.30:6789,c=10.98.93.224:6789
2020-11-25 13:07:52.088979 I | ceph-cluster-controller: detecting the ceph image version for image ceph/ceph:v15.2.6...
2020-11-25 13:07:53.904811 I | ceph-cluster-controller: detected ceph image version: "15.2.6-0 octopus"
2020-11-25 13:07:53.904862 I | ceph-cluster-controller: validating ceph version from provided image
```
Closes: https://github.com/rook/rook/issues/6587
Signed-off-by: Sébastien Han <seb@redhat.com>
213 lines
7.3 KiB
Go
Executable File
213 lines
7.3 KiB
Go
Executable File
/*
|
|
Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package mgr
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"testing"
|
|
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
|
testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
|
|
testop "github.com/rook/rook/pkg/operator/test"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/tevino/abool"
|
|
apps "k8s.io/api/apps/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
func TestStartMGR(t *testing.T) {
|
|
ctx := context.TODO()
|
|
var deploymentsUpdated *[]*apps.Deployment
|
|
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
|
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutputFile: func(command string, outFileArg string, args ...string) (string, error) {
|
|
return "{\"key\":\"mysecurekey\"}", nil
|
|
},
|
|
}
|
|
|
|
clientset := testop.New(t, 3)
|
|
configDir, _ := ioutil.TempDir("", "")
|
|
defer os.RemoveAll(configDir)
|
|
context := &clusterd.Context{
|
|
Executor: executor,
|
|
ConfigDir: configDir,
|
|
Clientset: clientset,
|
|
RequestCancelOrchestration: abool.New()}
|
|
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"}
|
|
clusterInfo.SetName("test")
|
|
clusterSpec := cephv1.ClusterSpec{
|
|
Annotations: map[rookv1.KeyType]rookv1.Annotations{cephv1.KeyMgr: {"my": "annotation"}},
|
|
Labels: map[rookv1.KeyType]rookv1.Labels{cephv1.KeyMgr: {"my-label-key": "value"}},
|
|
Dashboard: cephv1.DashboardSpec{Enabled: true, SSL: true},
|
|
Monitoring: cephv1.MonitoringSpec{Enabled: true, RulesNamespace: ""},
|
|
PriorityClassNames: map[rookv1.KeyType]string{cephv1.KeyMgr: "my-priority-class"},
|
|
DataDirHostPath: "/var/lib/rook/",
|
|
}
|
|
c := New(context, clusterInfo, clusterSpec, "myversion")
|
|
defer os.RemoveAll(c.spec.DataDirHostPath)
|
|
|
|
// start a basic service
|
|
err := c.Start()
|
|
assert.Nil(t, err)
|
|
validateStart(ctx, t, c)
|
|
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
|
|
c.spec.Dashboard.UrlPrefix = "/test"
|
|
c.spec.Dashboard.Port = 12345
|
|
err = c.Start()
|
|
assert.Nil(t, err)
|
|
validateStart(ctx, t, c)
|
|
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
|
|
// starting again with more replicas
|
|
c.Replicas = 3
|
|
c.spec.Dashboard.Enabled = false
|
|
err = c.Start()
|
|
assert.Nil(t, err)
|
|
validateStart(ctx, t, c)
|
|
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
|
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
|
}
|
|
|
|
func validateStart(ctx context.Context, t *testing.T, c *Cluster) {
|
|
mgrNames := []string{"a", "b"}
|
|
for i := 0; i < c.Replicas; i++ {
|
|
if i == 2 {
|
|
break
|
|
}
|
|
logger.Infof("Looking for cephmgr replica %d", i)
|
|
daemonName := mgrNames[i]
|
|
d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(ctx, fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, map[string]string{"my": "annotation"}, d.Spec.Template.Annotations)
|
|
assert.Contains(t, d.Spec.Template.Labels, "my-label-key")
|
|
assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName)
|
|
}
|
|
|
|
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr", metav1.GetOptions{})
|
|
assert.Nil(t, err)
|
|
|
|
ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{})
|
|
if c.spec.Dashboard.Enabled {
|
|
assert.NoError(t, err)
|
|
if c.spec.Dashboard.Port == 0 {
|
|
// port=0 -> default port
|
|
assert.Equal(t, ds.Spec.Ports[0].Port, int32(dashboardPortHTTPS))
|
|
} else {
|
|
// non-zero ports are configured as-is
|
|
assert.Equal(t, ds.Spec.Ports[0].Port, int32(c.spec.Dashboard.Port))
|
|
}
|
|
} else {
|
|
assert.True(t, errors.IsNotFound(err))
|
|
}
|
|
}
|
|
|
|
func TestConfigureModules(t *testing.T) {
|
|
modulesEnabled := 0
|
|
modulesDisabled := 0
|
|
configSettings := map[string]string{}
|
|
lastModuleConfigured := ""
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutputFile: func(command string, outFileArg string, args ...string) (string, error) {
|
|
logger.Infof("Command: %s %v", command, args)
|
|
if command == "ceph" && len(args) > 3 {
|
|
if args[0] == "mgr" && args[1] == "module" {
|
|
if args[2] == "enable" {
|
|
modulesEnabled++
|
|
}
|
|
if args[2] == "disable" {
|
|
modulesDisabled++
|
|
}
|
|
lastModuleConfigured = args[3]
|
|
}
|
|
if args[0] == "config" && args[1] == "set" && args[2] == "global" {
|
|
configSettings[args[3]] = args[4]
|
|
}
|
|
}
|
|
return "", nil //return "{\"key\":\"mysecurekey\"}", nil
|
|
},
|
|
}
|
|
|
|
clientset := testop.New(t, 3)
|
|
context := &clusterd.Context{Executor: executor, Clientset: clientset}
|
|
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
|
|
c := &Cluster{
|
|
context: context,
|
|
clusterInfo: clusterInfo,
|
|
}
|
|
|
|
// one module without any special configuration
|
|
c.spec.Mgr.Modules = []cephv1.Module{
|
|
{Name: "mymodule", Enabled: true},
|
|
}
|
|
assert.NoError(t, c.configureMgrModules())
|
|
assert.Equal(t, 1, modulesEnabled)
|
|
assert.Equal(t, 0, modulesDisabled)
|
|
assert.Equal(t, "mymodule", lastModuleConfigured)
|
|
|
|
// one module that has a min version that is not met
|
|
c.spec.Mgr.Modules = []cephv1.Module{
|
|
{Name: "pg_autoscaler", Enabled: true},
|
|
}
|
|
|
|
// one module that has a min version that is met
|
|
c.spec.Mgr.Modules = []cephv1.Module{
|
|
{Name: "pg_autoscaler", Enabled: true},
|
|
}
|
|
c.clusterInfo.CephVersion = cephver.CephVersion{Major: 14}
|
|
modulesEnabled = 0
|
|
assert.NoError(t, c.configureMgrModules())
|
|
assert.Equal(t, 1, modulesEnabled)
|
|
assert.Equal(t, 0, modulesDisabled)
|
|
assert.Equal(t, "pg_autoscaler", lastModuleConfigured)
|
|
assert.Equal(t, 2, len(configSettings))
|
|
assert.Equal(t, "on", configSettings["osd_pool_default_pg_autoscale_mode"])
|
|
assert.Equal(t, "0", configSettings["mon_pg_warn_min_per_osd"])
|
|
|
|
// disable the module
|
|
modulesEnabled = 0
|
|
lastModuleConfigured = ""
|
|
configSettings = map[string]string{}
|
|
c.spec.Mgr.Modules[0].Enabled = false
|
|
assert.NoError(t, c.configureMgrModules())
|
|
assert.Equal(t, 0, modulesEnabled)
|
|
assert.Equal(t, 1, modulesDisabled)
|
|
assert.Equal(t, "pg_autoscaler", lastModuleConfigured)
|
|
assert.Equal(t, 0, len(configSettings))
|
|
}
|
|
|
|
func TestMgrDaemons(t *testing.T) {
|
|
c := &Cluster{Replicas: 3}
|
|
daemons := c.getDaemonIDs()
|
|
require.Equal(t, 2, len(daemons))
|
|
assert.Equal(t, "a", daemons[0])
|
|
assert.Equal(t, "b", daemons[1])
|
|
}
|