ceph: start sidecar to monitor active mgr

The mgr daemon may be failed over by ceph if the active mgr is not
responding and the standby mgr is available. If the active mgr changes
the services for the dashboard and metrics will be updated with a
label selector for the new active mgr. The services cannot direct
traffic to the standby mgr or else they will be incorrectly redirected
to the active mgr.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2021-03-10 08:37:17 -07:00
1 parent d93ff428ec
commit 388ff3e78b
25 files changed
+432 -199

No files matched your search

+2 -1
View File
@@ -9,7 +9,7 @@ v1.6...
## Breaking Changes
### Ceph
* Support for adding OSDs via Drive Groups was removed. Please refer to the
* Support for adding OSDs via Drive Groups was removed. Please refer to the
[Ceph upgrade guide](Documentation/ceph-upgrade.md#migrate-the-drive-group-spec) for migration
instructions.
See https://github.com/rook/rook/issues/7275 for more information.
@@ -26,5 +26,6 @@ v1.6...
* Extending the support of vault KMS configuration for Ceph RGW
* Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons
* Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon
* Multiple Ceph mgr daemons are supported for stretch clusters and other clusters where HA of the mgr is more critical
* Ceph OSD: as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore
* Disable CSI GRPC metrics by default
@@ -92,6 +92,8 @@ rules:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- batch
@@ -465,6 +465,8 @@ rules:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- batch
+1
View File
@@ -61,6 +61,7 @@ func init() {
agentCmd,
admissionCmd,
osdCmd,
mgrCmd,
configCmd)
}
+92
View File
@@ -0,0 +1,92 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ceph
import (
"time"
"github.com/rook/rook/cmd/rook/rook"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
)
var mgrCmd = &cobra.Command{
Use: "mgr",
}
var mgrSidecarCmd = &cobra.Command{
Use: "watch-active",
}
var (
updateMgrServicesInterval string
daemonName string
clusterSpec cephv1.ClusterSpec
)
func init() {
addCephFlags(mgrCmd)
// add the subcommands to the parent mgr command
mgrCmd.AddCommand(mgrSidecarCmd)
mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Dashboard.Enabled, "dashboard-enabled", false, "whether the dashboard is enabled")
mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Monitoring.Enabled, "monitoring-enabled", false, "whether the monitoring is enabled")
mgrSidecarCmd.Flags().StringVar(&updateMgrServicesInterval, "update-interval", "", "the interval at which to update the mgr services")
mgrSidecarCmd.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster")
mgrSidecarCmd.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster")
mgrSidecarCmd.Flags().StringVar(&daemonName, "daemon-name", "", "the name of the local mgr daemon")
flags.SetFlagsFromEnv(mgrCmd.Flags(), rook.RookEnvVarPrefix)
flags.SetFlagsFromEnv(mgrSidecarCmd.Flags(), rook.RookEnvVarPrefix)
mgrSidecarCmd.RunE = runMgrSidecar
}
// Start the mgr daemon sidecar
func runMgrSidecar(cmd *cobra.Command, args []string) error {
rook.SetLogLevel()
context := createContext()
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
rook.LogStartupInfo(mgrSidecarCmd.Flags())
ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID)
clusterInfo.OwnerRef = ownerRef
if err := client.WriteCephConfig(context, &clusterInfo); err != nil {
rook.TerminateFatal(err)
}
interval, err := time.ParseDuration(updateMgrServicesInterval)
if err != nil {
rook.TerminateFatal(err)
}
m := mgr.New(context, &clusterInfo, clusterSpec, "")
for {
err := m.ReconcileMultipleServices(daemonName)
if err != nil {
logger.Errorf("failed to reconcile services. %v", err)
} else {
logger.Infof("successfully reconciled services. checking again in %ds", (int)(interval.Seconds()))
}
time.Sleep(interval)
}
}
+4 -2
View File
@@ -60,6 +60,7 @@ var (
osdDataDeviceFilter string
osdDataDevicePathFilter string
ownerRefID string
clusterName string
osdID int
osdStoreType string
osdStringID string
@@ -106,7 +107,8 @@ func addOSDFlags(command *cobra.Command) {
}
func addOSDConfigFlags(command *cobra.Command) {
command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster")
command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster")
command.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster")
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
command.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node")
@@ -218,7 +220,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
logger.Infof("crush location of osd: %s", crushLocation)
forceFormat := false
ownerRef := opcontroller.ClusterOwnerRef(clusterInfo.Namespace, ownerRefID)
ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID)
clusterInfo.OwnerRef = ownerRef
kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerRef)
agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat,
+32
View File
@@ -20,9 +20,11 @@ package client
import (
"context"
"fmt"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@@ -270,3 +272,33 @@ func PopulateMonHostMembers(monitors map[string]*MonInfo) ([]string, []string) {
return monMembers, monHosts
}
// WriteCephConfig writes the ceph config so ceph commands can be executed
func WriteCephConfig(context *clusterd.Context, clusterInfo *ClusterInfo) error {
// create the ceph.conf with the default settings
cephConfig, err := CreateDefaultCephConfig(context, clusterInfo)
if err != nil {
return errors.Wrap(err, "failed to create default ceph config")
}
// write the latest config to the config dir
confFilePath, err := GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig)
if err != nil {
return errors.Wrap(err, "failed to write connection config")
}
src, err := ioutil.ReadFile(filepath.Clean(confFilePath))
if err != nil {
return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config")
}
err = ioutil.WriteFile(DefaultConfigFilePath(), src, 0444)
if err != nil {
return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", DefaultConfigFilePath())
}
dst, err := ioutil.ReadFile(DefaultConfigFilePath())
if err == nil {
logger.Debugf("config file @ %s: %s", DefaultConfigFilePath(), dst)
} else {
logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", DefaultConfigFilePath(), err)
}
return nil
}
+1
View File
@@ -81,6 +81,7 @@ func AdminClusterInfo(namespace string) *ClusterInfo {
CephCred: CephCred{
Username: AdminUsername,
},
name: "testing",
}
}
+5 -5
View File
@@ -30,22 +30,22 @@ var (
moduleEnableWaitTime = 5 * time.Second
)
func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (MgrMap, error) {
func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (*MgrMap, error) {
args := []string{"mgr", "dump"}
buf, err := NewCephCommand(context, clusterInfo, args).Run()
if err != nil {
if len(buf) > 0 {
return MgrMap{}, errors.Wrapf(err, "failed to get status. %s", string(buf))
return nil, errors.Wrapf(err, "failed to get mgr dump. %s", string(buf))
}
return MgrMap{}, errors.Wrap(err, "failed to get ceph status")
return nil, errors.Wrap(err, "failed to get mgr dump")
}
var mgrMap MgrMap
if err := json.Unmarshal([]byte(buf), &mgrMap); err != nil {
return MgrMap{}, errors.Wrap(err, "failed to unmarshal status response")
return nil, errors.Wrap(err, "failed to unmarshal mgr dump")
}
return mgrMap, nil
return &mgrMap, nil
}
// MgrEnableModule enables a mgr module
+2 -34
View File
@@ -18,7 +18,6 @@ package osd
import (
"fmt"
"io/ioutil"
"os"
"os/signal"
"path/filepath"
@@ -29,7 +28,7 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/daemon/ceph/client"
oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
"github.com/rook/rook/pkg/util/sys"
)
@@ -163,37 +162,6 @@ func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, er
return rawDevice, nil
}
// writeCephConfig writes the ceph config so ceph commands can be executed
func writeCephConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
// create the ceph.conf with the default settings
cephConfig, err := cephclient.CreateDefaultCephConfig(context, clusterInfo)
if err != nil {
return errors.Wrap(err, "failed to create default ceph config")
}
// write the latest config to the config dir
confFilePath, err := cephclient.GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig)
if err != nil {
return errors.Wrap(err, "failed to write connection config")
}
src, err := ioutil.ReadFile(filepath.Clean(confFilePath))
if err != nil {
return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config")
}
err = ioutil.WriteFile(cephclient.DefaultConfigFilePath(), src, 0444)
if err != nil {
return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", cephclient.DefaultConfigFilePath())
}
dst, err := ioutil.ReadFile(cephclient.DefaultConfigFilePath())
if err == nil {
logger.Debugf("config file @ %s: %s", cephclient.DefaultConfigFilePath(), dst)
} else {
logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", cephclient.DefaultConfigFilePath(), err)
}
return nil
}
// Provision provisions an OSD
func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error {
if agent.pvcBacked {
@@ -214,7 +182,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo
status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating}
oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status)
if err := writeCephConfig(context, agent.clusterInfo); err != nil {
if err := client.WriteCephConfig(context, agent.clusterInfo); err != nil {
return errors.Wrap(err, "failed to generate ceph config")
}
+1 -1
View File
@@ -35,7 +35,7 @@ import (
func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error {
// Generate the ceph config for running ceph commands similar to the operator
if err := writeCephConfig(context, clusterInfo); err != nil {
if err := client.WriteCephConfig(context, clusterInfo); err != nil {
return errors.Wrap(err, "failed to write the ceph config")
}
+56 -18
View File
@@ -37,6 +37,7 @@ import (
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/exec"
v1 "k8s.io/api/apps/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -67,13 +68,11 @@ const (
// Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs.
type Cluster struct {
context *clusterd.Context
clusterInfo *cephclient.ClusterInfo
Replicas int
rookVersion string
exitCode func(err error) (int, bool)
appliedHttpBind bool
spec cephv1.ClusterSpec
context *clusterd.Context
clusterInfo *cephclient.ClusterInfo
rookVersion string
exitCode func(err error) (int, bool)
spec cephv1.ClusterSpec
}
// New creates an instance of the mgr
@@ -83,16 +82,16 @@ func New(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, spec ce
clusterInfo: clusterInfo,
spec: spec,
rookVersion: rookVersion,
Replicas: spec.Mgr.Count,
exitCode: exec.ExitStatus,
}
}
var waitForDeploymentToStart = k8sutil.WaitForDeploymentToStart
var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait
// for backward compatibility, default to 1 mgr
func (c *Cluster) getReplicas() int {
replicas := c.Replicas
replicas := c.spec.Mgr.Count
if replicas == 0 {
replicas = 1
}
@@ -118,6 +117,8 @@ func (c *Cluster) Start() error {
logger.Infof("start running mgr")
daemonIDs := c.getDaemonIDs()
var deploymentsToWaitFor []*v1.Deployment
for _, daemonID := range daemonIDs {
// Check whether we need to cancel the orchestration
if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
@@ -150,7 +151,7 @@ func (c *Cluster) Start() error {
return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name)
}
_, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{})
newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{})
if err != nil {
if !kerrors.IsAlreadyExists(err) {
return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName)
@@ -160,11 +161,26 @@ func (c *Cluster) Start() error {
if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, false); err != nil {
logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err)
}
} else {
// wait for the new deployment
deploymentsToWaitFor = append(deploymentsToWaitFor, newDeployment)
}
}
if err := c.reconcileServices(); err != nil {
return errors.Wrap(err, "failed to enable mgr services")
// If the mgr is newly created, wait for it to start before continuing with the service and
// module configuration
for _, d := range deploymentsToWaitFor {
if err := waitForDeploymentToStart(c.context, d); err != nil {
return errors.Wrapf(err, "failed to wait for mgr %q to start", d.Name)
}
}
if len(daemonIDs) == 1 {
// Only reconcile the mgr services in the operator if there is a single mgr.
// If there are multiple mgrs they will be managed by the mgr sidecar.
if err := c.reconcileService(daemonIDs[0]); err != nil {
return errors.Wrap(err, "failed to enable mgr services")
}
}
// configure the mgr modules
@@ -188,17 +204,40 @@ func (c *Cluster) Start() error {
return nil
}
func (c *Cluster) reconcileServices() error {
// ReconcileMultipleServices reconciles the services if the active mgr is the one running
// in the sidecar
func (c *Cluster) ReconcileMultipleServices(daemonNameToUpdate string) error {
// If the services are already set to this daemon, no need to attempt to update
svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), AppName, metav1.GetOptions{})
if err != nil {
logger.Errorf("failed to check current mgr service, proceeding to update. %v", err)
} else {
currentDaemon := svc.Spec.Selector[controller.DaemonIDLabel]
if currentDaemon == daemonNameToUpdate {
logger.Infof("mgr services already set to daemon %q, no need to update", daemonNameToUpdate)
return nil
}
logger.Infof("mgr service currently set to %q, checking if need to update to %q", currentDaemon, daemonNameToUpdate)
}
mgrMap, err := cephclient.CephMgrMap(c.context, c.clusterInfo)
if err != nil {
return errors.Wrap(err, "failed to get ceph status for the active mgr")
}
activeDaemon := mgrMap.ActiveName
if activeDaemon == "" {
return errors.Errorf("active mgr not found")
if mgrMap.ActiveName == "" {
return errors.New("active mgr not found")
}
logger.Infof("Active mgr is %q", activeDaemon)
if daemonNameToUpdate != mgrMap.ActiveName {
logger.Infof("no need for the mgr update since the active mgr is %q, rather than the local mgr %q", mgrMap.ActiveName, daemonNameToUpdate)
return nil
}
return c.reconcileService(mgrMap.ActiveName)
}
// reconcile the services, if the active mgr is not detected, use the default mgr
func (c *Cluster) reconcileService(activeDaemon string) error {
logger.Infof("setting services to point to mgr %q", activeDaemon)
if err := c.configureDashboardService(activeDaemon); err != nil {
return errors.Wrap(err, "failed to configure dashboard svc")
@@ -222,7 +261,6 @@ func (c *Cluster) reconcileServices() error {
func (c *Cluster) configureModules(daemonIDs []string) {
// Configure the modules asynchronously so we can complete all the configuration much sooner.
startModuleConfiguration("http bind settings", c.clearHTTPBindFix)
startModuleConfiguration("prometheus", c.enablePrometheusModule)
startModuleConfiguration("dashboard", c.configureDashboardModules)
// "crash" is part of the "always_on_modules" list as of Octopus
+116 -20
View File
@@ -35,12 +35,13 @@ import (
"github.com/stretchr/testify/require"
"github.com/tevino/abool"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/errors"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestStartMgr(t *testing.T) {
ctx := context.TODO()
var deploymentsUpdated *[]*apps.Deployment
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
@@ -49,11 +50,15 @@ func TestStartMgr(t *testing.T) {
return "{\"key\":\"mysecurekey\"}", nil
},
}
waitForDeploymentToStart = func(clusterdContext *clusterd.Context, deployment *v1.Deployment) error {
logger.Infof("simulated mgr deployment starting")
return nil
}
clientset := testop.New(t, 3)
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
context := &clusterd.Context{
ctx := &clusterd.Context{
Executor: executor,
ConfigDir: configDir,
Clientset: clientset,
@@ -68,13 +73,13 @@ func TestStartMgr(t *testing.T) {
PriorityClassNames: map[rookv1.KeyType]string{cephv1.KeyMgr: "my-priority-class"},
DataDirHostPath: "/var/lib/rook/",
}
c := New(context, clusterInfo, clusterSpec, "myversion")
c := New(ctx, clusterInfo, clusterSpec, "myversion")
defer os.RemoveAll(c.spec.DataDirHostPath)
// start a basic service
err := c.Start()
assert.Nil(t, err)
validateStart(ctx, t, c)
validateStart(t, c)
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
@@ -82,40 +87,64 @@ func TestStartMgr(t *testing.T) {
c.spec.Dashboard.Port = 12345
err = c.Start()
assert.Nil(t, err)
validateStart(ctx, t, c)
validateStart(t, c)
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
// starting again with more replicas
c.Replicas = 3
// starting with more replicas
c.spec.Mgr.Count = 2
c.spec.Dashboard.Enabled = false
// delete the previous mgr since the mocked test won't update the existing one
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{})
assert.Nil(t, err)
err = c.Start()
assert.Nil(t, err)
validateStart(ctx, t, c)
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
// trigger the sidecar reconcile since the operator didn't do it so we can perform the full validation
err = c.reconcileService("a")
assert.Nil(t, err)
validateStart(t, c)
// the dashboard service is only deleted by the operator reconcile if the replicas are 1,
// otherwise the sidecar has the responsibility
c.spec.Mgr.Count = 1
c.spec.Dashboard.Enabled = false
// clean the previous deployments
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{})
assert.Nil(t, err)
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-b", metav1.DeleteOptions{})
assert.Nil(t, err)
err = c.Start()
assert.Nil(t, err)
validateStart(t, c)
}
func validateStart(ctx context.Context, t *testing.T, c *Cluster) {
func validateStart(t *testing.T, c *Cluster) {
mgrNames := []string{"a", "b"}
for i := 0; i < c.Replicas; i++ {
if i == 2 {
logger.Warning("cannot have more than 2 mgrs")
break
}
for i := 0; i < c.spec.Mgr.Count; i++ {
logger.Infof("Looking for cephmgr replica %d", i)
daemonName := mgrNames[i]
d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(ctx, fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(context.TODO(), fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, map[string]string{"my": "annotation"}, d.Spec.Template.Annotations)
assert.Contains(t, d.Spec.Template.Labels, "my-label-key")
assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName)
if c.spec.Mgr.Count == 1 {
assert.Equal(t, 1, len(d.Spec.Template.Spec.Containers))
} else {
// The sidecar container is only there when multiple mgrs are enabled
assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers))
assert.Equal(t, "watch-active", d.Spec.Template.Spec.Containers[1].Name)
}
}
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr", metav1.GetOptions{})
validateServices(t, c)
}
func validateServices(t *testing.T, c *Cluster) {
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
assert.Nil(t, err)
ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{})
ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr-dashboard", metav1.GetOptions{})
if c.spec.Dashboard.Enabled {
assert.NoError(t, err)
if c.spec.Dashboard.Port == 0 {
@@ -130,6 +159,65 @@ func validateStart(ctx context.Context, t *testing.T, c *Cluster) {
}
}
func TestMgrSidecarReconcile(t *testing.T) {
activeMgr := "a"
spec := cephv1.ClusterSpec{
Mgr: cephv1.MgrSpec{Count: 1},
Dashboard: cephv1.DashboardSpec{
Enabled: true,
Port: 7000,
},
}
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutputFile: func(command, outFile string, args ...string) (string, error) {
return fmt.Sprintf(`{"active_name":"%s"}`, activeMgr), nil
},
}
clientset := testop.New(t, 3)
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
ctx := &clusterd.Context{
Executor: executor,
ConfigDir: configDir,
Clientset: clientset,
}
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
clusterInfo.SetName("test")
c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo}
// Update services according to the active mgr
err := c.ReconcileMultipleServices(activeMgr)
assert.NoError(t, err)
validateServices(t, c)
validateServiceMatches(t, c, "a")
// nothing is created or updated when the requested mgr is not the active mgr
err = c.ReconcileMultipleServices("b")
assert.NoError(t, err)
_, err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
assert.True(t, kerrors.IsNotFound(err))
// nothing is updated when the requested mgr is not the active mgr
activeMgr = "b"
err = c.ReconcileMultipleServices("b")
assert.NoError(t, err)
validateServices(t, c)
validateServiceMatches(t, c, "b")
}
func validateServiceMatches(t *testing.T, c *Cluster, expectedActive string) {
// The service labels should match the active mgr
svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
assert.NoError(t, err)
matchDaemon, ok := svc.Spec.Selector["ceph_daemon_id"]
assert.True(t, ok)
assert.Equal(t, expectedActive, matchDaemon)
// clean up the service for the next test
err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr", metav1.DeleteOptions{})
assert.NoError(t, err)
}
func TestConfigureModules(t *testing.T) {
modulesEnabled := 0
modulesDisabled := 0
@@ -205,8 +293,16 @@ func TestConfigureModules(t *testing.T) {
}
func TestMgrDaemons(t *testing.T) {
c := &Cluster{Replicas: 2}
spec := cephv1.ClusterSpec{
Mgr: cephv1.MgrSpec{Count: 1},
}
c := &Cluster{spec: spec}
daemons := c.getDaemonIDs()
require.Equal(t, 1, len(daemons))
assert.Equal(t, "a", daemons[0])
c.spec.Mgr.Count = 2
daemons = c.getDaemonIDs()
require.Equal(t, 2, len(daemons))
assert.Equal(t, "a", daemons[0])
assert.Equal(t, "b", daemons[1])
+37 -38
View File
@@ -21,11 +21,10 @@ import (
"os"
"strconv"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -63,6 +62,9 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
PriorityClassName: cephv1.GetMgrPriorityClassName(c.spec.PriorityClassNames),
},
}
if c.spec.Mgr.Count > 1 {
podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig))
}
// If the log collector is enabled we add the side-car container
if c.spec.LogCollector.Enabled {
@@ -113,42 +115,6 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
return d, nil
}
// if we do not need the http bind fix, then we need to be careful. if we are
// upgrading from a cluster that had the fix applied, then the fix is no longer
// needed, and furthermore, needs to be removed so that there is not a lingering
// ceph configuration option that contains an old ip. by clearing the option,
// we let ceph bind to its default ANYADDR address. However, since we don't
// know which version of Ceph we are may be upgrading _from_ we need to (a)
// always do this and (b) make sure that all forms of the configuration option
// are removed (see the init container factory method). Once the minimum
// supported version of Rook contains the fix, all of this can be removed.
func (c *Cluster) clearHTTPBindFix() error {
// We only need to apply these changes once. No harm in once each time the operator restarts.
if c.appliedHttpBind {
return nil
}
for _, daemonID := range c.getDaemonIDs() {
for _, module := range []string{"dashboard", "prometheus"} {
// there are two forms of the configuration key that might exist which
// depends not on the current version, but on the version that may be
// the version being upgraded from.
if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID,
fmt.Sprintf("mgr/%s/server_addr", module), "", false); err != nil {
return errors.Wrap(err, "failed to set config for an mgr daemon using v2 format")
}
// this is for the format used in v1.0
// https://github.com/rook/rook/commit/11d318fb2f77a6ac9a8f2b9be42c826d3b4a93c3
if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID,
fmt.Sprintf("mgr/%s/%s/server_addr", module, daemonID), "", false); err != nil {
return errors.Wrap(err, "failed to set config for an mgr daemon using v1 format")
}
}
}
c.appliedHttpBind = true
return nil
}
func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container {
return controller.ChownCephDataDirsInitContainer(
*mgrConfig.DataPathMap,
@@ -217,6 +183,39 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
return container
}
func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
envVars := []v1.EnvVar{
{Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)},
{Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)},
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace),
opmon.EndpointEnvVar(),
opmon.SecretEnvVar(),
opmon.CephUsernameEnvVar(),
opmon.CephSecretEnvVar(),
k8sutil.ConfigOverrideEnvVar(),
{Name: "ROOK_FSID", ValueFrom: &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{Name: "rook-ceph-mon"},
Key: "fsid",
},
}},
{Name: "ROOK_DASHBOARD_ENABLED", Value: strconv.FormatBool(c.spec.Dashboard.Enabled)},
{Name: "ROOK_MONITORING_ENABLED", Value: strconv.FormatBool(c.spec.Monitoring.Enabled)},
{Name: "ROOK_UPDATE_INTERVAL", Value: "15s"},
{Name: "ROOK_DAEMON_NAME", Value: mgrConfig.DaemonID},
}
return v1.Container{
Args: []string{"ceph", "mgr", "watch-active"},
Name: "watch-active",
Image: c.rookVersion,
Env: envVars,
//Resources: **TODO**,
}
}
func getDefaultMgrLivenessProbe() *v1.Probe {
return &v1.Probe{
Handler: v1.Handler{
+1 -1
View File
@@ -362,6 +362,6 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client.
return nil
}
_, err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback)
err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback)
return err
}
+1
View File
@@ -57,6 +57,7 @@ func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1.
envVars := []v1.EnvVar{
nodeNameEnvVar(osdProps.crushHostname),
{Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)},
{Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)},
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace),
@@ -224,6 +224,7 @@ func testOSDsOnPVC(t *testing.T) {
Namespace: namespace,
CephVersion: cephver.Nautilus,
}
clusterInfo.SetName("mycluster")
executor := osdPVCTestExecutor(t, clientset, namespace)
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: executor, RequestCancelOrchestration: abool.New()}
@@ -295,6 +295,7 @@ func TestAddNodeFailure(t *testing.T) {
Namespace: "ns-add-remove",
CephVersion: cephver.Nautilus,
}
clusterInfo.SetName("testcluster")
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()}
spec := cephv1.ClusterSpec{
DataDirHostPath: context.ConfigDir,
@@ -381,6 +382,7 @@ func TestGetPVCHostName(t *testing.T) {
func TestGetOSDInfo(t *testing.T) {
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
clusterInfo.SetName("test")
context := &clusterd.Context{}
spec := cephv1.ClusterSpec{DataDirHostPath: "/rook"}
c := New(context, clusterInfo, spec, "myversion")
@@ -517,6 +519,7 @@ func TestDetectCrushLocation(t *testing.T) {
func TestGetOSDInfoWithCustomRoot(t *testing.T) {
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
clusterInfo.SetName("test")
context := &clusterd.Context{}
spec := cephv1.ClusterSpec{
DataDirHostPath: "/rook",
+2 -2
View File
@@ -488,7 +488,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
v1.Container{
Args: []string{"ceph", "osd", "init"},
Name: controller.ConfigInitContainerName,
Image: k8sutil.MakeRookImage(c.rookVersion),
Image: c.rookVersion,
VolumeMounts: configVolumeMounts,
Env: configEnvVars,
SecurityContext: securityContext,
@@ -708,7 +708,7 @@ func (c *Cluster) getCopyBinariesContainer() (v1.Volume, *v1.Container) {
"copy-binaries",
"--copy-to-dir", rookBinariesMountPath},
Name: "copy-bins",
Image: k8sutil.MakeRookImage(c.rookVersion),
Image: c.rookVersion,
VolumeMounts: []v1.VolumeMount{mount},
}
}
@@ -88,6 +88,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
Namespace: "ns",
CephVersion: cephver.Nautilus,
}
clusterInfo.SetName("test")
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
spec := cephv1.ClusterSpec{
CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"},
@@ -430,6 +431,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
Namespace: "ns",
CephVersion: cephver.Octopus,
}
clusterInfo.SetName("test")
c := New(context, clusterInfo, spec, "rook/rook:myversion")
deployment, err = c.makeDeployment(osdProp, osd, dataPathMap)
assert.NoError(t, err)
@@ -464,6 +466,7 @@ func TestStorageSpecConfig(t *testing.T) {
Namespace: "ns",
CephVersion: cephver.Nautilus,
}
clusterInfo.SetName("testing")
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
spec := cephv1.ClusterSpec{
DataDirHostPath: context.ConfigDir,
@@ -545,6 +548,7 @@ func TestHostNetwork(t *testing.T) {
Namespace: "ns",
CephVersion: cephver.Nautilus,
}
clusterInfo.SetName("test")
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
spec := cephv1.ClusterSpec{
Storage: storageSpec,
@@ -583,6 +587,7 @@ func TestOsdPrepareResources(t *testing.T) {
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
clusterInfo.SetName("test")
spec := cephv1.ClusterSpec{
Resources: map[string]v1.ResourceRequirements{"prepareosd": {
Limits: v1.ResourceList{
+61 -62
View File
@@ -61,11 +61,11 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c
// 2. verify that we can continue the update procedure
// Basically, we go one resource by one and check if we can stop and then if the resource has been successfully updated
// we check if we can go ahead and move to the next one.
func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) (*v1.Deployment, error) {
func UpdateDeploymentAndWait(clusterContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) error {
ctx := context.TODO()
currentDeployment, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{})
currentDeployment, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err)
return fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err)
}
// Check whether the current deployment and newly generated one are identical
@@ -78,67 +78,66 @@ func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployme
patchChanged = true
}
// If deployments are different, let's update!
if patchChanged {
logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name)
// Let's verify the deployment can be stopped
err = verifyCallback("stop")
if err != nil {
return nil, fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err)
}
// Set hash annotation to the newly generated deployment
err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment)
if err != nil {
return nil, fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err)
}
if _, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil {
return nil, fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err)
}
// wait for the deployment to be restarted
sleepTime := 2
attempts := 30
if currentDeployment.Spec.ProgressDeadlineSeconds != nil {
// make the attempts double the progress deadline since the pod is both stopping and starting
attempts = 2 * (int(*currentDeployment.Spec.ProgressDeadlineSeconds) / sleepTime)
}
for i := 0; i < attempts; i++ {
// check for the status of the deployment
d, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get deployment %q. %v", modifiedDeployment.Name, err)
}
if d.Status.ObservedGeneration != currentDeployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 {
logger.Infof("finished waiting for updated deployment %q", d.Name)
// Now we check if we can go to the next daemon
err = verifyCallback("continue")
if err != nil {
return nil, fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err)
}
return d, nil
}
// If ProgressDeadlineExceeded is reached let's fail earlier
// This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state
for _, condition := range d.Status.Conditions {
if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" {
return nil, fmt.Errorf("gave up waiting for deployment %q to update because %q", modifiedDeployment.Name, condition.Reason)
}
}
logger.Debugf("deployment %q status=%+v", d.Name, d.Status)
time.Sleep(time.Duration(sleepTime) * time.Second)
}
return nil, fmt.Errorf("gave up waiting for deployment %q to update", modifiedDeployment.Name)
if !patchChanged {
logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name)
return nil
}
logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name)
return nil, nil
// If deployments are different, let's update!
logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name)
// Let's verify the deployment can be stopped
if err := verifyCallback("stop"); err != nil {
return fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err)
}
// Set hash annotation to the newly generated deployment
if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment); err != nil {
return fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err)
}
if _, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err)
}
if err := WaitForDeploymentToStart(clusterContext, currentDeployment); err != nil {
return err
}
// Now we check if we can go to the next daemon
if err := verifyCallback("continue"); err != nil {
return fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err)
}
return nil
}
func WaitForDeploymentToStart(clusterdContext *clusterd.Context, deployment *v1.Deployment) error {
// wait for the deployment to be restarted up to 300s
sleepTime := 3
attempts := 100
for i := 0; i < attempts; i++ {
// check for the status of the deployment
d, err := clusterdContext.Clientset.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get deployment %q. %v", deployment.Name, err)
}
if d.Status.ObservedGeneration != deployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 {
logger.Infof("finished waiting for updated deployment %q", d.Name)
return nil
}
// If ProgressDeadlineExceeded is reached let's fail earlier
// This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state
for _, condition := range d.Status.Conditions {
if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" {
return fmt.Errorf("gave up waiting for deployment %q to update because %q", deployment.Name, condition.Reason)
}
}
logger.Debugf("deployment %q status=%+v", d.Name, d.Status)
time.Sleep(time.Duration(sleepTime) * time.Second)
}
return fmt.Errorf("gave up waiting for deployment %q to update", deployment.Name)
}
// GetDeployments returns a list of deployment names labels matching a given selector
-10
View File
@@ -51,7 +51,6 @@ const (
ConfigOverrideName = "rook-config-override"
// ConfigOverrideVal config override value
ConfigOverrideVal = "config"
defaultVersion = "rook/rook:latest"
configMountDir = "/etc/rook/config"
overrideFilename = "override.conf"
)
@@ -203,15 +202,6 @@ func GetMatchingContainer(containers []v1.Container, name string) (v1.Container,
return *result, nil
}
// MakeRookImage formats the container name
func MakeRookImage(version string) string {
if version == "" {
return defaultVersion
}
return version
}
// PodsRunningWithLabel returns the number of running pods with the given label
func PodsRunningWithLabel(clientset kubernetes.Interface, namespace, label string) (int, error) {
running, _, err := podStatusWithLabel(clientset, namespace, label)
-5
View File
@@ -26,11 +26,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestMakeRookImage(t *testing.T) {
assert.Equal(t, "rook/rook:v1", MakeRookImage("rook/rook:v1"))
assert.Equal(t, defaultVersion, MakeRookImage(""))
}
func TestGetContainerInPod(t *testing.T) {
expectedName := "mycontainer"
imageName := "myimage"
+3
View File
@@ -33,6 +33,7 @@ func CreateOrUpdateService(
ctx := context.TODO()
name := serviceDefinition.Name
logger.Debugf("creating service %s", name)
s, err := clientset.CoreV1().Services(namespace).Create(ctx, serviceDefinition, metav1.CreateOptions{})
if err != nil {
if !errors.IsAlreadyExists(err) {
@@ -42,6 +43,8 @@ func CreateOrUpdateService(
if err != nil {
return nil, fmt.Errorf("failed to update service %s. %+v", name, err)
}
} else {
logger.Debugf("created service %s", s.Name)
}
return s, err
}
@@ -1061,6 +1061,8 @@ rules:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- batch