ceph: start sidecar to monitor active mgr
The mgr daemon may be failed over by ceph if the active mgr is not responding and the standby mgr is available. If the active mgr changes the services for the dashboard and metrics will be updated with a label selector for the new active mgr. The services cannot direct traffic to the standby mgr or else they will be incorrectly redirected to the active mgr. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
25 files changed
+432
-199
No files matched your search
@@ -9,7 +9,7 @@ v1.6...
|
||||
## Breaking Changes
|
||||
|
||||
### Ceph
|
||||
* Support for adding OSDs via Drive Groups was removed. Please refer to the
|
||||
* Support for adding OSDs via Drive Groups was removed. Please refer to the
|
||||
[Ceph upgrade guide](Documentation/ceph-upgrade.md#migrate-the-drive-group-spec) for migration
|
||||
instructions.
|
||||
See https://github.com/rook/rook/issues/7275 for more information.
|
||||
@@ -26,5 +26,6 @@ v1.6...
|
||||
* Extending the support of vault KMS configuration for Ceph RGW
|
||||
* Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons
|
||||
* Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon
|
||||
* Multiple Ceph mgr daemons are supported for stretch clusters and other clusters where HA of the mgr is more critical
|
||||
* Ceph OSD: as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore
|
||||
* Disable CSI GRPC metrics by default
|
||||
@@ -92,6 +92,8 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- batch
|
||||
|
||||
@@ -465,6 +465,8 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- batch
|
||||
|
||||
@@ -61,6 +61,7 @@ func init() {
|
||||
agentCmd,
|
||||
admissionCmd,
|
||||
osdCmd,
|
||||
mgrCmd,
|
||||
configCmd)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/cmd/rook/rook"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var mgrCmd = &cobra.Command{
|
||||
Use: "mgr",
|
||||
}
|
||||
var mgrSidecarCmd = &cobra.Command{
|
||||
Use: "watch-active",
|
||||
}
|
||||
var (
|
||||
updateMgrServicesInterval string
|
||||
daemonName string
|
||||
clusterSpec cephv1.ClusterSpec
|
||||
)
|
||||
|
||||
func init() {
|
||||
addCephFlags(mgrCmd)
|
||||
|
||||
// add the subcommands to the parent mgr command
|
||||
mgrCmd.AddCommand(mgrSidecarCmd)
|
||||
|
||||
mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Dashboard.Enabled, "dashboard-enabled", false, "whether the dashboard is enabled")
|
||||
mgrSidecarCmd.Flags().BoolVar(&clusterSpec.Monitoring.Enabled, "monitoring-enabled", false, "whether the monitoring is enabled")
|
||||
mgrSidecarCmd.Flags().StringVar(&updateMgrServicesInterval, "update-interval", "", "the interval at which to update the mgr services")
|
||||
mgrSidecarCmd.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster")
|
||||
mgrSidecarCmd.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster")
|
||||
mgrSidecarCmd.Flags().StringVar(&daemonName, "daemon-name", "", "the name of the local mgr daemon")
|
||||
|
||||
flags.SetFlagsFromEnv(mgrCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
flags.SetFlagsFromEnv(mgrSidecarCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
mgrSidecarCmd.RunE = runMgrSidecar
|
||||
}
|
||||
|
||||
// Start the mgr daemon sidecar
|
||||
func runMgrSidecar(cmd *cobra.Command, args []string) error {
|
||||
rook.SetLogLevel()
|
||||
|
||||
context := createContext()
|
||||
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
|
||||
rook.LogStartupInfo(mgrSidecarCmd.Flags())
|
||||
|
||||
ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID)
|
||||
clusterInfo.OwnerRef = ownerRef
|
||||
|
||||
if err := client.WriteCephConfig(context, &clusterInfo); err != nil {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
|
||||
interval, err := time.ParseDuration(updateMgrServicesInterval)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
|
||||
m := mgr.New(context, &clusterInfo, clusterSpec, "")
|
||||
for {
|
||||
err := m.ReconcileMultipleServices(daemonName)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to reconcile services. %v", err)
|
||||
} else {
|
||||
logger.Infof("successfully reconciled services. checking again in %ds", (int)(interval.Seconds()))
|
||||
}
|
||||
time.Sleep(interval)
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,7 @@ var (
|
||||
osdDataDeviceFilter string
|
||||
osdDataDevicePathFilter string
|
||||
ownerRefID string
|
||||
clusterName string
|
||||
osdID int
|
||||
osdStoreType string
|
||||
osdStringID string
|
||||
@@ -106,7 +107,8 @@ func addOSDFlags(command *cobra.Command) {
|
||||
}
|
||||
|
||||
func addOSDConfigFlags(command *cobra.Command) {
|
||||
command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CRD that owns this cluster")
|
||||
command.Flags().StringVar(&ownerRefID, "cluster-id", "", "the UID of the cluster CR that owns this cluster")
|
||||
command.Flags().StringVar(&clusterName, "cluster-name", "", "the name of the cluster CR that owns this cluster")
|
||||
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
|
||||
command.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node")
|
||||
|
||||
@@ -218,7 +220,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
|
||||
logger.Infof("crush location of osd: %s", crushLocation)
|
||||
|
||||
forceFormat := false
|
||||
ownerRef := opcontroller.ClusterOwnerRef(clusterInfo.Namespace, ownerRefID)
|
||||
ownerRef := opcontroller.ClusterOwnerRef(clusterName, ownerRefID)
|
||||
clusterInfo.OwnerRef = ownerRef
|
||||
kv := k8sutil.NewConfigMapKVStore(clusterInfo.Namespace, context.Clientset, ownerRef)
|
||||
agent := osddaemon.NewAgent(context, dataDevices, cfg.metadataDevice, forceFormat,
|
||||
|
||||
@@ -20,9 +20,11 @@ package client
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -270,3 +272,33 @@ func PopulateMonHostMembers(monitors map[string]*MonInfo) ([]string, []string) {
|
||||
|
||||
return monMembers, monHosts
|
||||
}
|
||||
|
||||
// WriteCephConfig writes the ceph config so ceph commands can be executed
|
||||
func WriteCephConfig(context *clusterd.Context, clusterInfo *ClusterInfo) error {
|
||||
// create the ceph.conf with the default settings
|
||||
cephConfig, err := CreateDefaultCephConfig(context, clusterInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create default ceph config")
|
||||
}
|
||||
|
||||
// write the latest config to the config dir
|
||||
confFilePath, err := GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to write connection config")
|
||||
}
|
||||
src, err := ioutil.ReadFile(filepath.Clean(confFilePath))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config")
|
||||
}
|
||||
err = ioutil.WriteFile(DefaultConfigFilePath(), src, 0444)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", DefaultConfigFilePath())
|
||||
}
|
||||
dst, err := ioutil.ReadFile(DefaultConfigFilePath())
|
||||
if err == nil {
|
||||
logger.Debugf("config file @ %s: %s", DefaultConfigFilePath(), dst)
|
||||
} else {
|
||||
logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", DefaultConfigFilePath(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -81,6 +81,7 @@ func AdminClusterInfo(namespace string) *ClusterInfo {
|
||||
CephCred: CephCred{
|
||||
Username: AdminUsername,
|
||||
},
|
||||
name: "testing",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,22 +30,22 @@ var (
|
||||
moduleEnableWaitTime = 5 * time.Second
|
||||
)
|
||||
|
||||
func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (MgrMap, error) {
|
||||
func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (*MgrMap, error) {
|
||||
args := []string{"mgr", "dump"}
|
||||
buf, err := NewCephCommand(context, clusterInfo, args).Run()
|
||||
if err != nil {
|
||||
if len(buf) > 0 {
|
||||
return MgrMap{}, errors.Wrapf(err, "failed to get status. %s", string(buf))
|
||||
return nil, errors.Wrapf(err, "failed to get mgr dump. %s", string(buf))
|
||||
}
|
||||
return MgrMap{}, errors.Wrap(err, "failed to get ceph status")
|
||||
return nil, errors.Wrap(err, "failed to get mgr dump")
|
||||
}
|
||||
|
||||
var mgrMap MgrMap
|
||||
if err := json.Unmarshal([]byte(buf), &mgrMap); err != nil {
|
||||
return MgrMap{}, errors.Wrap(err, "failed to unmarshal status response")
|
||||
return nil, errors.Wrap(err, "failed to unmarshal mgr dump")
|
||||
}
|
||||
|
||||
return mgrMap, nil
|
||||
return &mgrMap, nil
|
||||
}
|
||||
|
||||
// MgrEnableModule enables a mgr module
|
||||
|
||||
@@ -18,7 +18,6 @@ package osd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
@@ -29,7 +28,7 @@ import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
oposd "github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
"github.com/rook/rook/pkg/util/sys"
|
||||
)
|
||||
@@ -163,37 +162,6 @@ func configRawDevice(name string, context *clusterd.Context) (*sys.LocalDisk, er
|
||||
return rawDevice, nil
|
||||
}
|
||||
|
||||
// writeCephConfig writes the ceph config so ceph commands can be executed
|
||||
func writeCephConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
|
||||
|
||||
// create the ceph.conf with the default settings
|
||||
cephConfig, err := cephclient.CreateDefaultCephConfig(context, clusterInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create default ceph config")
|
||||
}
|
||||
|
||||
// write the latest config to the config dir
|
||||
confFilePath, err := cephclient.GenerateConnectionConfigWithSettings(context, clusterInfo, cephConfig)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to write connection config")
|
||||
}
|
||||
src, err := ioutil.ReadFile(filepath.Clean(confFilePath))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to copy connection config to /etc/ceph. failed to read the connection config")
|
||||
}
|
||||
err = ioutil.WriteFile(cephclient.DefaultConfigFilePath(), src, 0444)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to copy connection config to /etc/ceph. failed to write %q", cephclient.DefaultConfigFilePath())
|
||||
}
|
||||
dst, err := ioutil.ReadFile(cephclient.DefaultConfigFilePath())
|
||||
if err == nil {
|
||||
logger.Debugf("config file @ %s: %s", cephclient.DefaultConfigFilePath(), dst)
|
||||
} else {
|
||||
logger.Warningf("wrote and copied config file but failed to read it back from %s for logging. %v", cephclient.DefaultConfigFilePath(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Provision provisions an OSD
|
||||
func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topologyAffinity string) error {
|
||||
if agent.pvcBacked {
|
||||
@@ -214,7 +182,7 @@ func Provision(context *clusterd.Context, agent *OsdAgent, crushLocation, topolo
|
||||
status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusOrchestrating}
|
||||
oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status)
|
||||
|
||||
if err := writeCephConfig(context, agent.clusterInfo); err != nil {
|
||||
if err := client.WriteCephConfig(context, agent.clusterInfo); err != nil {
|
||||
return errors.Wrap(err, "failed to generate ceph config")
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
func RemoveOSDs(context *clusterd.Context, clusterInfo *client.ClusterInfo, osdsToRemove []string) error {
|
||||
|
||||
// Generate the ceph config for running ceph commands similar to the operator
|
||||
if err := writeCephConfig(context, clusterInfo); err != nil {
|
||||
if err := client.WriteCephConfig(context, clusterInfo); err != nil {
|
||||
return errors.Wrap(err, "failed to write the ceph config")
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
v1 "k8s.io/api/apps/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -67,13 +68,11 @@ const (
|
||||
|
||||
// Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs.
|
||||
type Cluster struct {
|
||||
context *clusterd.Context
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
Replicas int
|
||||
rookVersion string
|
||||
exitCode func(err error) (int, bool)
|
||||
appliedHttpBind bool
|
||||
spec cephv1.ClusterSpec
|
||||
context *clusterd.Context
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
rookVersion string
|
||||
exitCode func(err error) (int, bool)
|
||||
spec cephv1.ClusterSpec
|
||||
}
|
||||
|
||||
// New creates an instance of the mgr
|
||||
@@ -83,16 +82,16 @@ func New(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, spec ce
|
||||
clusterInfo: clusterInfo,
|
||||
spec: spec,
|
||||
rookVersion: rookVersion,
|
||||
Replicas: spec.Mgr.Count,
|
||||
exitCode: exec.ExitStatus,
|
||||
}
|
||||
}
|
||||
|
||||
var waitForDeploymentToStart = k8sutil.WaitForDeploymentToStart
|
||||
var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait
|
||||
|
||||
// for backward compatibility, default to 1 mgr
|
||||
func (c *Cluster) getReplicas() int {
|
||||
replicas := c.Replicas
|
||||
replicas := c.spec.Mgr.Count
|
||||
if replicas == 0 {
|
||||
replicas = 1
|
||||
}
|
||||
@@ -118,6 +117,8 @@ func (c *Cluster) Start() error {
|
||||
|
||||
logger.Infof("start running mgr")
|
||||
daemonIDs := c.getDaemonIDs()
|
||||
var deploymentsToWaitFor []*v1.Deployment
|
||||
|
||||
for _, daemonID := range daemonIDs {
|
||||
// Check whether we need to cancel the orchestration
|
||||
if err := controller.CheckForCancelledOrchestration(c.context); err != nil {
|
||||
@@ -150,7 +151,7 @@ func (c *Cluster) Start() error {
|
||||
return errors.Wrapf(err, "failed to set annotation for deployment %q", d.Name)
|
||||
}
|
||||
|
||||
_, err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{})
|
||||
newDeployment, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Create(ctx, d, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if !kerrors.IsAlreadyExists(err) {
|
||||
return errors.Wrapf(err, "failed to create mgr deployment %s", resourceName)
|
||||
@@ -160,11 +161,26 @@ func (c *Cluster) Start() error {
|
||||
if err := updateDeploymentAndWait(c.context, c.clusterInfo, d, config.MgrType, mgrConfig.DaemonID, c.spec.SkipUpgradeChecks, false); err != nil {
|
||||
logger.Errorf("failed to update mgr deployment %q. %v", resourceName, err)
|
||||
}
|
||||
} else {
|
||||
// wait for the new deployment
|
||||
deploymentsToWaitFor = append(deploymentsToWaitFor, newDeployment)
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.reconcileServices(); err != nil {
|
||||
return errors.Wrap(err, "failed to enable mgr services")
|
||||
// If the mgr is newly created, wait for it to start before continuing with the service and
|
||||
// module configuration
|
||||
for _, d := range deploymentsToWaitFor {
|
||||
if err := waitForDeploymentToStart(c.context, d); err != nil {
|
||||
return errors.Wrapf(err, "failed to wait for mgr %q to start", d.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if len(daemonIDs) == 1 {
|
||||
// Only reconcile the mgr services in the operator if there is a single mgr.
|
||||
// If there are multiple mgrs they will be managed by the mgr sidecar.
|
||||
if err := c.reconcileService(daemonIDs[0]); err != nil {
|
||||
return errors.Wrap(err, "failed to enable mgr services")
|
||||
}
|
||||
}
|
||||
|
||||
// configure the mgr modules
|
||||
@@ -188,17 +204,40 @@ func (c *Cluster) Start() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) reconcileServices() error {
|
||||
// ReconcileMultipleServices reconciles the services if the active mgr is the one running
|
||||
// in the sidecar
|
||||
func (c *Cluster) ReconcileMultipleServices(daemonNameToUpdate string) error {
|
||||
// If the services are already set to this daemon, no need to attempt to update
|
||||
svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), AppName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Errorf("failed to check current mgr service, proceeding to update. %v", err)
|
||||
} else {
|
||||
currentDaemon := svc.Spec.Selector[controller.DaemonIDLabel]
|
||||
if currentDaemon == daemonNameToUpdate {
|
||||
logger.Infof("mgr services already set to daemon %q, no need to update", daemonNameToUpdate)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("mgr service currently set to %q, checking if need to update to %q", currentDaemon, daemonNameToUpdate)
|
||||
}
|
||||
|
||||
mgrMap, err := cephclient.CephMgrMap(c.context, c.clusterInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get ceph status for the active mgr")
|
||||
}
|
||||
activeDaemon := mgrMap.ActiveName
|
||||
if activeDaemon == "" {
|
||||
return errors.Errorf("active mgr not found")
|
||||
if mgrMap.ActiveName == "" {
|
||||
return errors.New("active mgr not found")
|
||||
}
|
||||
logger.Infof("Active mgr is %q", activeDaemon)
|
||||
if daemonNameToUpdate != mgrMap.ActiveName {
|
||||
logger.Infof("no need for the mgr update since the active mgr is %q, rather than the local mgr %q", mgrMap.ActiveName, daemonNameToUpdate)
|
||||
return nil
|
||||
}
|
||||
|
||||
return c.reconcileService(mgrMap.ActiveName)
|
||||
}
|
||||
|
||||
// reconcile the services, if the active mgr is not detected, use the default mgr
|
||||
func (c *Cluster) reconcileService(activeDaemon string) error {
|
||||
logger.Infof("setting services to point to mgr %q", activeDaemon)
|
||||
|
||||
if err := c.configureDashboardService(activeDaemon); err != nil {
|
||||
return errors.Wrap(err, "failed to configure dashboard svc")
|
||||
@@ -222,7 +261,6 @@ func (c *Cluster) reconcileServices() error {
|
||||
|
||||
func (c *Cluster) configureModules(daemonIDs []string) {
|
||||
// Configure the modules asynchronously so we can complete all the configuration much sooner.
|
||||
startModuleConfiguration("http bind settings", c.clearHTTPBindFix)
|
||||
startModuleConfiguration("prometheus", c.enablePrometheusModule)
|
||||
startModuleConfiguration("dashboard", c.configureDashboardModules)
|
||||
// "crash" is part of the "always_on_modules" list as of Octopus
|
||||
|
||||
@@ -35,12 +35,13 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/tevino/abool"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/apps/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestStartMgr(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
var deploymentsUpdated *[]*apps.Deployment
|
||||
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
||||
|
||||
@@ -49,11 +50,15 @@ func TestStartMgr(t *testing.T) {
|
||||
return "{\"key\":\"mysecurekey\"}", nil
|
||||
},
|
||||
}
|
||||
waitForDeploymentToStart = func(clusterdContext *clusterd.Context, deployment *v1.Deployment) error {
|
||||
logger.Infof("simulated mgr deployment starting")
|
||||
return nil
|
||||
}
|
||||
|
||||
clientset := testop.New(t, 3)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
context := &clusterd.Context{
|
||||
ctx := &clusterd.Context{
|
||||
Executor: executor,
|
||||
ConfigDir: configDir,
|
||||
Clientset: clientset,
|
||||
@@ -68,13 +73,13 @@ func TestStartMgr(t *testing.T) {
|
||||
PriorityClassNames: map[rookv1.KeyType]string{cephv1.KeyMgr: "my-priority-class"},
|
||||
DataDirHostPath: "/var/lib/rook/",
|
||||
}
|
||||
c := New(context, clusterInfo, clusterSpec, "myversion")
|
||||
c := New(ctx, clusterInfo, clusterSpec, "myversion")
|
||||
defer os.RemoveAll(c.spec.DataDirHostPath)
|
||||
|
||||
// start a basic service
|
||||
err := c.Start()
|
||||
assert.Nil(t, err)
|
||||
validateStart(ctx, t, c)
|
||||
validateStart(t, c)
|
||||
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
||||
|
||||
@@ -82,40 +87,64 @@ func TestStartMgr(t *testing.T) {
|
||||
c.spec.Dashboard.Port = 12345
|
||||
err = c.Start()
|
||||
assert.Nil(t, err)
|
||||
validateStart(ctx, t, c)
|
||||
validateStart(t, c)
|
||||
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
||||
|
||||
// starting again with more replicas
|
||||
c.Replicas = 3
|
||||
// starting with more replicas
|
||||
c.spec.Mgr.Count = 2
|
||||
c.spec.Dashboard.Enabled = false
|
||||
// delete the previous mgr since the mocked test won't update the existing one
|
||||
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{})
|
||||
assert.Nil(t, err)
|
||||
err = c.Start()
|
||||
assert.Nil(t, err)
|
||||
validateStart(ctx, t, c)
|
||||
assert.ElementsMatch(t, []string{"rook-ceph-mgr-a"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
||||
// trigger the sidecar reconcile since the operator didn't do it so we can perform the full validation
|
||||
err = c.reconcileService("a")
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, c)
|
||||
|
||||
// the dashboard service is only deleted by the operator reconcile if the replicas are 1,
|
||||
// otherwise the sidecar has the responsibility
|
||||
c.spec.Mgr.Count = 1
|
||||
c.spec.Dashboard.Enabled = false
|
||||
// clean the previous deployments
|
||||
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-a", metav1.DeleteOptions{})
|
||||
assert.Nil(t, err)
|
||||
err = c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr-b", metav1.DeleteOptions{})
|
||||
assert.Nil(t, err)
|
||||
err = c.Start()
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, c)
|
||||
}
|
||||
|
||||
func validateStart(ctx context.Context, t *testing.T, c *Cluster) {
|
||||
func validateStart(t *testing.T, c *Cluster) {
|
||||
mgrNames := []string{"a", "b"}
|
||||
for i := 0; i < c.Replicas; i++ {
|
||||
if i == 2 {
|
||||
logger.Warning("cannot have more than 2 mgrs")
|
||||
break
|
||||
}
|
||||
for i := 0; i < c.spec.Mgr.Count; i++ {
|
||||
logger.Infof("Looking for cephmgr replica %d", i)
|
||||
daemonName := mgrNames[i]
|
||||
d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(ctx, fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
|
||||
d, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Get(context.TODO(), fmt.Sprintf("rook-ceph-mgr-%s", daemonName), metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, map[string]string{"my": "annotation"}, d.Spec.Template.Annotations)
|
||||
assert.Contains(t, d.Spec.Template.Labels, "my-label-key")
|
||||
assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName)
|
||||
if c.spec.Mgr.Count == 1 {
|
||||
assert.Equal(t, 1, len(d.Spec.Template.Spec.Containers))
|
||||
} else {
|
||||
// The sidecar container is only there when multiple mgrs are enabled
|
||||
assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers))
|
||||
assert.Equal(t, "watch-active", d.Spec.Template.Spec.Containers[1].Name)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr", metav1.GetOptions{})
|
||||
validateServices(t, c)
|
||||
}
|
||||
|
||||
func validateServices(t *testing.T, c *Cluster) {
|
||||
_, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(ctx, "rook-ceph-mgr-dashboard", metav1.GetOptions{})
|
||||
ds, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr-dashboard", metav1.GetOptions{})
|
||||
if c.spec.Dashboard.Enabled {
|
||||
assert.NoError(t, err)
|
||||
if c.spec.Dashboard.Port == 0 {
|
||||
@@ -130,6 +159,65 @@ func validateStart(ctx context.Context, t *testing.T, c *Cluster) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgrSidecarReconcile(t *testing.T) {
|
||||
activeMgr := "a"
|
||||
spec := cephv1.ClusterSpec{
|
||||
Mgr: cephv1.MgrSpec{Count: 1},
|
||||
Dashboard: cephv1.DashboardSpec{
|
||||
Enabled: true,
|
||||
Port: 7000,
|
||||
},
|
||||
}
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutputFile: func(command, outFile string, args ...string) (string, error) {
|
||||
return fmt.Sprintf(`{"active_name":"%s"}`, activeMgr), nil
|
||||
},
|
||||
}
|
||||
clientset := testop.New(t, 3)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
ctx := &clusterd.Context{
|
||||
Executor: executor,
|
||||
ConfigDir: configDir,
|
||||
Clientset: clientset,
|
||||
}
|
||||
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
|
||||
clusterInfo.SetName("test")
|
||||
c := &Cluster{spec: spec, context: ctx, clusterInfo: clusterInfo}
|
||||
|
||||
// Update services according to the active mgr
|
||||
err := c.ReconcileMultipleServices(activeMgr)
|
||||
assert.NoError(t, err)
|
||||
validateServices(t, c)
|
||||
validateServiceMatches(t, c, "a")
|
||||
|
||||
// nothing is created or updated when the requested mgr is not the active mgr
|
||||
err = c.ReconcileMultipleServices("b")
|
||||
assert.NoError(t, err)
|
||||
_, err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
|
||||
assert.True(t, kerrors.IsNotFound(err))
|
||||
|
||||
// nothing is updated when the requested mgr is not the active mgr
|
||||
activeMgr = "b"
|
||||
err = c.ReconcileMultipleServices("b")
|
||||
assert.NoError(t, err)
|
||||
validateServices(t, c)
|
||||
validateServiceMatches(t, c, "b")
|
||||
}
|
||||
|
||||
func validateServiceMatches(t *testing.T, c *Cluster, expectedActive string) {
|
||||
// The service labels should match the active mgr
|
||||
svc, err := c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Get(context.TODO(), "rook-ceph-mgr", metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
matchDaemon, ok := svc.Spec.Selector["ceph_daemon_id"]
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, expectedActive, matchDaemon)
|
||||
|
||||
// clean up the service for the next test
|
||||
err = c.context.Clientset.CoreV1().Services(c.clusterInfo.Namespace).Delete(context.TODO(), "rook-ceph-mgr", metav1.DeleteOptions{})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestConfigureModules(t *testing.T) {
|
||||
modulesEnabled := 0
|
||||
modulesDisabled := 0
|
||||
@@ -205,8 +293,16 @@ func TestConfigureModules(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMgrDaemons(t *testing.T) {
|
||||
c := &Cluster{Replicas: 2}
|
||||
spec := cephv1.ClusterSpec{
|
||||
Mgr: cephv1.MgrSpec{Count: 1},
|
||||
}
|
||||
c := &Cluster{spec: spec}
|
||||
daemons := c.getDaemonIDs()
|
||||
require.Equal(t, 1, len(daemons))
|
||||
assert.Equal(t, "a", daemons[0])
|
||||
|
||||
c.spec.Mgr.Count = 2
|
||||
daemons = c.getDaemonIDs()
|
||||
require.Equal(t, 2, len(daemons))
|
||||
assert.Equal(t, "a", daemons[0])
|
||||
assert.Equal(t, "b", daemons[1])
|
||||
|
||||
@@ -21,11 +21,10 @@ import (
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -63,6 +62,9 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
|
||||
PriorityClassName: cephv1.GetMgrPriorityClassName(c.spec.PriorityClassNames),
|
||||
},
|
||||
}
|
||||
if c.spec.Mgr.Count > 1 {
|
||||
podSpec.Spec.Containers = append(podSpec.Spec.Containers, c.makeMgrSidecarContainer(mgrConfig))
|
||||
}
|
||||
|
||||
// If the log collector is enabled we add the side-car container
|
||||
if c.spec.LogCollector.Enabled {
|
||||
@@ -113,42 +115,6 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// if we do not need the http bind fix, then we need to be careful. if we are
|
||||
// upgrading from a cluster that had the fix applied, then the fix is no longer
|
||||
// needed, and furthermore, needs to be removed so that there is not a lingering
|
||||
// ceph configuration option that contains an old ip. by clearing the option,
|
||||
// we let ceph bind to its default ANYADDR address. However, since we don't
|
||||
// know which version of Ceph we are may be upgrading _from_ we need to (a)
|
||||
// always do this and (b) make sure that all forms of the configuration option
|
||||
// are removed (see the init container factory method). Once the minimum
|
||||
// supported version of Rook contains the fix, all of this can be removed.
|
||||
func (c *Cluster) clearHTTPBindFix() error {
|
||||
// We only need to apply these changes once. No harm in once each time the operator restarts.
|
||||
if c.appliedHttpBind {
|
||||
return nil
|
||||
}
|
||||
for _, daemonID := range c.getDaemonIDs() {
|
||||
for _, module := range []string{"dashboard", "prometheus"} {
|
||||
// there are two forms of the configuration key that might exist which
|
||||
// depends not on the current version, but on the version that may be
|
||||
// the version being upgraded from.
|
||||
if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID,
|
||||
fmt.Sprintf("mgr/%s/server_addr", module), "", false); err != nil {
|
||||
return errors.Wrap(err, "failed to set config for an mgr daemon using v2 format")
|
||||
}
|
||||
|
||||
// this is for the format used in v1.0
|
||||
// https://github.com/rook/rook/commit/11d318fb2f77a6ac9a8f2b9be42c826d3b4a93c3
|
||||
if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID,
|
||||
fmt.Sprintf("mgr/%s/%s/server_addr", module, daemonID), "", false); err != nil {
|
||||
return errors.Wrap(err, "failed to set config for an mgr daemon using v1 format")
|
||||
}
|
||||
}
|
||||
}
|
||||
c.appliedHttpBind = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
return controller.ChownCephDataDirsInitContainer(
|
||||
*mgrConfig.DataPathMap,
|
||||
@@ -217,6 +183,39 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
return container
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
|
||||
envVars := []v1.EnvVar{
|
||||
{Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)},
|
||||
{Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)},
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.SecretEnvVar(),
|
||||
opmon.CephUsernameEnvVar(),
|
||||
opmon.CephSecretEnvVar(),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
{Name: "ROOK_FSID", ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{Name: "rook-ceph-mon"},
|
||||
Key: "fsid",
|
||||
},
|
||||
}},
|
||||
{Name: "ROOK_DASHBOARD_ENABLED", Value: strconv.FormatBool(c.spec.Dashboard.Enabled)},
|
||||
{Name: "ROOK_MONITORING_ENABLED", Value: strconv.FormatBool(c.spec.Monitoring.Enabled)},
|
||||
{Name: "ROOK_UPDATE_INTERVAL", Value: "15s"},
|
||||
{Name: "ROOK_DAEMON_NAME", Value: mgrConfig.DaemonID},
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Args: []string{"ceph", "mgr", "watch-active"},
|
||||
Name: "watch-active",
|
||||
Image: c.rookVersion,
|
||||
Env: envVars,
|
||||
//Resources: **TODO**,
|
||||
}
|
||||
}
|
||||
|
||||
func getDefaultMgrLivenessProbe() *v1.Probe {
|
||||
return &v1.Probe{
|
||||
Handler: v1.Handler{
|
||||
|
||||
@@ -362,6 +362,6 @@ func UpdateCephDeploymentAndWait(context *clusterd.Context, clusterInfo *client.
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback)
|
||||
err := k8sutil.UpdateDeploymentAndWait(context, deployment, clusterInfo.Namespace, callback)
|
||||
return err
|
||||
}
|
||||
@@ -57,6 +57,7 @@ func (c *Cluster) getConfigEnvVars(osdProps osdProperties, dataDir string) []v1.
|
||||
envVars := []v1.EnvVar{
|
||||
nodeNameEnvVar(osdProps.crushHostname),
|
||||
{Name: "ROOK_CLUSTER_ID", Value: string(c.clusterInfo.OwnerRef.UID)},
|
||||
{Name: "ROOK_CLUSTER_NAME", Value: string(c.clusterInfo.NamespacedName().Name)},
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
opmon.PodNamespaceEnvVar(c.clusterInfo.Namespace),
|
||||
|
||||
@@ -224,6 +224,7 @@ func testOSDsOnPVC(t *testing.T) {
|
||||
Namespace: namespace,
|
||||
CephVersion: cephver.Nautilus,
|
||||
}
|
||||
clusterInfo.SetName("mycluster")
|
||||
executor := osdPVCTestExecutor(t, clientset, namespace)
|
||||
|
||||
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: executor, RequestCancelOrchestration: abool.New()}
|
||||
|
||||
@@ -295,6 +295,7 @@ func TestAddNodeFailure(t *testing.T) {
|
||||
Namespace: "ns-add-remove",
|
||||
CephVersion: cephver.Nautilus,
|
||||
}
|
||||
clusterInfo.SetName("testcluster")
|
||||
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}, RequestCancelOrchestration: abool.New()}
|
||||
spec := cephv1.ClusterSpec{
|
||||
DataDirHostPath: context.ConfigDir,
|
||||
@@ -381,6 +382,7 @@ func TestGetPVCHostName(t *testing.T) {
|
||||
|
||||
func TestGetOSDInfo(t *testing.T) {
|
||||
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
|
||||
clusterInfo.SetName("test")
|
||||
context := &clusterd.Context{}
|
||||
spec := cephv1.ClusterSpec{DataDirHostPath: "/rook"}
|
||||
c := New(context, clusterInfo, spec, "myversion")
|
||||
@@ -517,6 +519,7 @@ func TestDetectCrushLocation(t *testing.T) {
|
||||
|
||||
func TestGetOSDInfoWithCustomRoot(t *testing.T) {
|
||||
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
|
||||
clusterInfo.SetName("test")
|
||||
context := &clusterd.Context{}
|
||||
spec := cephv1.ClusterSpec{
|
||||
DataDirHostPath: "/rook",
|
||||
|
||||
@@ -488,7 +488,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
|
||||
v1.Container{
|
||||
Args: []string{"ceph", "osd", "init"},
|
||||
Name: controller.ConfigInitContainerName,
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Image: c.rookVersion,
|
||||
VolumeMounts: configVolumeMounts,
|
||||
Env: configEnvVars,
|
||||
SecurityContext: securityContext,
|
||||
@@ -708,7 +708,7 @@ func (c *Cluster) getCopyBinariesContainer() (v1.Volume, *v1.Container) {
|
||||
"copy-binaries",
|
||||
"--copy-to-dir", rookBinariesMountPath},
|
||||
Name: "copy-bins",
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Image: c.rookVersion,
|
||||
VolumeMounts: []v1.VolumeMount{mount},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
Namespace: "ns",
|
||||
CephVersion: cephver.Nautilus,
|
||||
}
|
||||
clusterInfo.SetName("test")
|
||||
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
|
||||
spec := cephv1.ClusterSpec{
|
||||
CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"},
|
||||
@@ -430,6 +431,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) {
|
||||
Namespace: "ns",
|
||||
CephVersion: cephver.Octopus,
|
||||
}
|
||||
clusterInfo.SetName("test")
|
||||
c := New(context, clusterInfo, spec, "rook/rook:myversion")
|
||||
deployment, err = c.makeDeployment(osdProp, osd, dataPathMap)
|
||||
assert.NoError(t, err)
|
||||
@@ -464,6 +466,7 @@ func TestStorageSpecConfig(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
CephVersion: cephver.Nautilus,
|
||||
}
|
||||
clusterInfo.SetName("testing")
|
||||
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
|
||||
spec := cephv1.ClusterSpec{
|
||||
DataDirHostPath: context.ConfigDir,
|
||||
@@ -545,6 +548,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
CephVersion: cephver.Nautilus,
|
||||
}
|
||||
clusterInfo.SetName("test")
|
||||
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
|
||||
spec := cephv1.ClusterSpec{
|
||||
Storage: storageSpec,
|
||||
@@ -583,6 +587,7 @@ func TestOsdPrepareResources(t *testing.T) {
|
||||
|
||||
context := &clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}
|
||||
clusterInfo := &cephclient.ClusterInfo{Namespace: "ns"}
|
||||
clusterInfo.SetName("test")
|
||||
spec := cephv1.ClusterSpec{
|
||||
Resources: map[string]v1.ResourceRequirements{"prepareosd": {
|
||||
Limits: v1.ResourceList{
|
||||
|
||||
@@ -61,11 +61,11 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c
|
||||
// 2. verify that we can continue the update procedure
|
||||
// Basically, we go one resource by one and check if we can stop and then if the resource has been successfully updated
|
||||
// we check if we can go ahead and move to the next one.
|
||||
func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) (*v1.Deployment, error) {
|
||||
func UpdateDeploymentAndWait(clusterContext *clusterd.Context, modifiedDeployment *apps.Deployment, namespace string, verifyCallback func(action string) error) error {
|
||||
ctx := context.TODO()
|
||||
currentDeployment, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{})
|
||||
currentDeployment, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err)
|
||||
return fmt.Errorf("failed to get deployment %s. %+v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
// Check whether the current deployment and newly generated one are identical
|
||||
@@ -78,67 +78,66 @@ func UpdateDeploymentAndWait(clusterdContext *clusterd.Context, modifiedDeployme
|
||||
patchChanged = true
|
||||
}
|
||||
|
||||
// If deployments are different, let's update!
|
||||
if patchChanged {
|
||||
logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name)
|
||||
|
||||
// Let's verify the deployment can be stopped
|
||||
err = verifyCallback("stop")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
// Set hash annotation to the newly generated deployment
|
||||
err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
if _, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
// wait for the deployment to be restarted
|
||||
sleepTime := 2
|
||||
attempts := 30
|
||||
if currentDeployment.Spec.ProgressDeadlineSeconds != nil {
|
||||
// make the attempts double the progress deadline since the pod is both stopping and starting
|
||||
attempts = 2 * (int(*currentDeployment.Spec.ProgressDeadlineSeconds) / sleepTime)
|
||||
}
|
||||
for i := 0; i < attempts; i++ {
|
||||
// check for the status of the deployment
|
||||
d, err := clusterdContext.Clientset.AppsV1().Deployments(namespace).Get(ctx, modifiedDeployment.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get deployment %q. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
if d.Status.ObservedGeneration != currentDeployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 {
|
||||
logger.Infof("finished waiting for updated deployment %q", d.Name)
|
||||
|
||||
// Now we check if we can go to the next daemon
|
||||
err = verifyCallback("continue")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// If ProgressDeadlineExceeded is reached let's fail earlier
|
||||
// This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state
|
||||
for _, condition := range d.Status.Conditions {
|
||||
if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" {
|
||||
return nil, fmt.Errorf("gave up waiting for deployment %q to update because %q", modifiedDeployment.Name, condition.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("deployment %q status=%+v", d.Name, d.Status)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
return nil, fmt.Errorf("gave up waiting for deployment %q to update", modifiedDeployment.Name)
|
||||
if !patchChanged {
|
||||
logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("deployment %q did not change, nothing to update", currentDeployment.Name)
|
||||
return nil, nil
|
||||
// If deployments are different, let's update!
|
||||
logger.Infof("updating deployment %q after verifying it is safe to stop", modifiedDeployment.Name)
|
||||
|
||||
// Let's verify the deployment can be stopped
|
||||
if err := verifyCallback("stop"); err != nil {
|
||||
return fmt.Errorf("failed to check if deployment %q can be updated. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
// Set hash annotation to the newly generated deployment
|
||||
if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(modifiedDeployment); err != nil {
|
||||
return fmt.Errorf("failed to set hash annotation on deployment %q. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
if _, err := clusterContext.Clientset.AppsV1().Deployments(namespace).Update(ctx, modifiedDeployment, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("failed to update deployment %q. %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
|
||||
if err := WaitForDeploymentToStart(clusterContext, currentDeployment); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Now we check if we can go to the next daemon
|
||||
if err := verifyCallback("continue"); err != nil {
|
||||
return fmt.Errorf("failed to check if deployment %q can continue: %v", modifiedDeployment.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WaitForDeploymentToStart(clusterdContext *clusterd.Context, deployment *v1.Deployment) error {
|
||||
// wait for the deployment to be restarted up to 300s
|
||||
sleepTime := 3
|
||||
attempts := 100
|
||||
for i := 0; i < attempts; i++ {
|
||||
// check for the status of the deployment
|
||||
d, err := clusterdContext.Clientset.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get deployment %q. %v", deployment.Name, err)
|
||||
}
|
||||
if d.Status.ObservedGeneration != deployment.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 {
|
||||
logger.Infof("finished waiting for updated deployment %q", d.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// If ProgressDeadlineExceeded is reached let's fail earlier
|
||||
// This can happen if one of the deployment cannot be scheduled on a node and stays in "pending" state
|
||||
for _, condition := range d.Status.Conditions {
|
||||
if condition.Type == v1.DeploymentProgressing && condition.Reason == "ProgressDeadlineExceeded" {
|
||||
return fmt.Errorf("gave up waiting for deployment %q to update because %q", deployment.Name, condition.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debugf("deployment %q status=%+v", d.Name, d.Status)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
return fmt.Errorf("gave up waiting for deployment %q to update", deployment.Name)
|
||||
}
|
||||
|
||||
// GetDeployments returns a list of deployment names labels matching a given selector
|
||||
|
||||
@@ -51,7 +51,6 @@ const (
|
||||
ConfigOverrideName = "rook-config-override"
|
||||
// ConfigOverrideVal config override value
|
||||
ConfigOverrideVal = "config"
|
||||
defaultVersion = "rook/rook:latest"
|
||||
configMountDir = "/etc/rook/config"
|
||||
overrideFilename = "override.conf"
|
||||
)
|
||||
@@ -203,15 +202,6 @@ func GetMatchingContainer(containers []v1.Container, name string) (v1.Container,
|
||||
return *result, nil
|
||||
}
|
||||
|
||||
// MakeRookImage formats the container name
|
||||
func MakeRookImage(version string) string {
|
||||
if version == "" {
|
||||
return defaultVersion
|
||||
}
|
||||
|
||||
return version
|
||||
}
|
||||
|
||||
// PodsRunningWithLabel returns the number of running pods with the given label
|
||||
func PodsRunningWithLabel(clientset kubernetes.Interface, namespace, label string) (int, error) {
|
||||
running, _, err := podStatusWithLabel(clientset, namespace, label)
|
||||
|
||||
@@ -26,11 +26,6 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestMakeRookImage(t *testing.T) {
|
||||
assert.Equal(t, "rook/rook:v1", MakeRookImage("rook/rook:v1"))
|
||||
assert.Equal(t, defaultVersion, MakeRookImage(""))
|
||||
}
|
||||
|
||||
func TestGetContainerInPod(t *testing.T) {
|
||||
expectedName := "mycontainer"
|
||||
imageName := "myimage"
|
||||
|
||||
@@ -33,6 +33,7 @@ func CreateOrUpdateService(
|
||||
ctx := context.TODO()
|
||||
name := serviceDefinition.Name
|
||||
logger.Debugf("creating service %s", name)
|
||||
|
||||
s, err := clientset.CoreV1().Services(namespace).Create(ctx, serviceDefinition, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
@@ -42,6 +43,8 @@ func CreateOrUpdateService(
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to update service %s. %+v", name, err)
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("created service %s", s.Name)
|
||||
}
|
||||
return s, err
|
||||
}
|
||||
|
||||
@@ -1061,6 +1061,8 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- batch
|
||||
|
||||
Reference in new issue
Block a user