From 83047a17d1bc52a891286c7aeeec3c8b15c4d7da Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 7 Nov 2025 12:42:59 -0700 Subject: [PATCH] operator: fix concurrency issues in the cluster controller Use a sync map for the clusterMap in the cluster controller and fix other issues to allow for concurrent cluster reconcile. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/cephstatus.go | 12 +- pkg/operator/ceph/cluster/cluster.go | 14 +-- pkg/operator/ceph/cluster/cluster_external.go | 20 ++-- pkg/operator/ceph/cluster/controller.go | 107 ++++++++++-------- pkg/operator/ceph/cluster/monitoring.go | 4 +- pkg/operator/ceph/cluster/version.go | 2 +- tests/framework/installer/ceph_settings.go | 4 + tests/integration/ceph_multi_cluster_test.go | 21 ++-- 8 files changed, 99 insertions(+), 85 deletions(-) diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index 2b7ea1527..c7d8314f4 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -236,28 +236,28 @@ func formatTime(t time.Time) string { return t.Format(time.RFC3339) } -func (c *ClusterController) updateClusterCephVersion(image string, cephVersion cephver.CephVersion) { - logger.Infof("cluster %q: version %q detected for image %q", c.namespacedName.Namespace, cephVersion.String(), image) +func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersion cephver.CephVersion) { + logger.Infof("cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image) - cephCluster, err := c.context.RookClientset.CephV1().CephClusters(c.namespacedName.Namespace).Get(c.OpManagerCtx, c.namespacedName.Name, metav1.GetOptions{}) + cephCluster, err := c.context.RookClientset.CephV1().CephClusters(cluster.namespacedName.Namespace).Get(c.OpManagerCtx, cluster.namespacedName.Name, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.") return } - logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", c.namespacedName.Name, cephVersion, err) + logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err) return } cephClusterVersion := &cephv1.ClusterVersion{ - Image: image, + Image: cluster.Spec.CephVersion.Image, Version: opcontroller.GetCephVersionLabel(cephVersion), // DO NOT CHANGE FORMAT FROM "Maj.Min.Ext-Bld" } // update the Ceph version on the retrieved cluster object // do not overwrite the ceph status that is updated in a separate goroutine cephCluster.Status.CephVersion = cephClusterVersion if err := reporting.UpdateStatus(c.client, cephCluster); err != nil { - logger.Errorf("failed to update cluster %q version. %v", c.namespacedName.Name, err) + logger.Errorf("failed to update cluster %q version. %v", cluster.namespacedName.Name, err) return } } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 0340ab2c0..fcd3105c8 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -195,7 +195,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { if cluster.Spec.External.Enable { err := c.configureExternalCephCluster(cluster) if err != nil { - controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error()) + controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error()) return errors.Wrap(err, "failed to configure external ceph cluster") } } else { @@ -211,7 +211,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { } } else { clusterInfo.OwnerInfo = cluster.ownerInfo - clusterInfo.SetName(c.namespacedName.Name) + clusterInfo.SetName(cluster.namespacedName.Name) cluster.ClusterInfo = clusterInfo if cluster.mons.ClusterInfo == nil { // ClusterInfo stored in cluster.mons can be lost from the object stored in the @@ -239,7 +239,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { err = c.configureLocalCephCluster(cluster) if err != nil { - controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error()) + controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error()) return errors.Wrap(err, "failed to configure local ceph cluster") } @@ -254,7 +254,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { // Populate ClusterInfo with the last value cluster.mons.ClusterInfo = cluster.ClusterInfo - cluster.mons.ClusterInfo.SetName(c.namespacedName.Name) + cluster.mons.ClusterInfo.SetName(cluster.namespacedName.Name) // Start the monitoring if not already started c.configureCephMonitoring(cluster, cluster.ClusterInfo) @@ -271,7 +271,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error { } // Run image validation job - controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Detecting Ceph version") + controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Detecting Ceph version") cephVersion, isUpgrade, err := c.detectAndValidateCephVersion(cluster) if err != nil { return errors.Wrap(err, "failed the ceph version check") @@ -286,7 +286,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error { } } - controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring the Ceph cluster") + controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring the Ceph cluster") // Run the orchestration err = cluster.reconcileCephDaemons(c.rookImage, *cephVersion) if err != nil { @@ -294,7 +294,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error { } // Set the condition to the cluster object - controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, cluster.observedGeneration, cephv1.ConditionReady, v1.ConditionTrue, cephv1.ClusterCreatedReason, "Cluster created successfully") + controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, cluster.observedGeneration, cephv1.ConditionReady, v1.ConditionTrue, cephv1.ClusterCreatedReason, "Cluster created successfully") return nil } diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 8f32a5f21..5baeac457 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -46,15 +46,15 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error return errors.Wrap(err, "failed to validate external cluster specs") } - opcontroller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnecting, v1.ConditionTrue, cephv1.ClusterConnectingReason, "Attempting to connect to an external Ceph cluster") + opcontroller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnecting, v1.ConditionTrue, cephv1.ClusterConnectingReason, "Attempting to connect to an external Ceph cluster") // loop until we find the secret necessary to connect to the external cluster // then populate clusterInfo - cluster.ClusterInfo, err = opcontroller.PopulateExternalClusterInfo(cluster.Spec, c.context, c.OpManagerCtx, c.namespacedName.Namespace, cluster.ownerInfo) + cluster.ClusterInfo, err = opcontroller.PopulateExternalClusterInfo(cluster.Spec, c.context, c.OpManagerCtx, cluster.namespacedName.Namespace, cluster.ownerInfo) if err != nil { return errors.Wrap(err, "failed to populate external cluster info") } - cluster.ClusterInfo.SetName(c.namespacedName.Name) + cluster.ClusterInfo.SetName(cluster.namespacedName.Name) cluster.ClusterInfo.Context = c.OpManagerCtx if !client.IsKeyringBase64Encoded(cluster.ClusterInfo.CephCred.Secret) { @@ -82,13 +82,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // // Only do this when doing a bit of management... logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName) - err = populateConfigOverrideConfigMap(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata) + err = populateConfigOverrideConfigMap(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata) if err != nil { return errors.Wrap(err, "failed to populate config override config map") } logger.Infof("creating %q secret", config.StoreName) - err = config.GetStore(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo) + err = config.GetStore(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo) if err != nil { return errors.Wrap(err, "failed to update the global config") } @@ -102,7 +102,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error // Create CSI Secrets only if the user has provided the admin key if cluster.ClusterInfo.CephCred.Username == client.AdminUsername { - err = csi.CreateCSISecrets(c.context, cluster.ClusterInfo, c.namespacedName) + err = csi.CreateCSISecrets(c.context, cluster.ClusterInfo, cluster.namespacedName) if err != nil { return errors.Wrap(err, "failed to create csi kubernetes secrets") } @@ -131,8 +131,8 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error }, } - clusterId := c.namespacedName.Namespace // cluster id is same as cluster namespace for CephClusters - err = csi.SaveClusterConfig(c.context.Clientset, clusterId, c.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry) + clusterId := cluster.namespacedName.Namespace // cluster id is same as cluster namespace for CephClusters + err = csi.SaveClusterConfig(c.context.Clientset, clusterId, cluster.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry) if err != nil { return errors.Wrap(err, "failed to update csi cluster config") } @@ -168,7 +168,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error cluster.ClusterInfo.CephVersion = *externalVersion // Populate ceph version - c.updateClusterCephVersion("", *externalVersion) + c.updateClusterCephVersion(cluster, *externalVersion) err = c.configureExternalClusterMonitoring(c.context, cluster) if err != nil { @@ -188,7 +188,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error } } - opcontroller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnected, v1.ConditionTrue, cephv1.ClusterConnectedReason, "Cluster connected successfully") + opcontroller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnected, v1.ConditionTrue, cephv1.ClusterConnectedReason, "Cluster connected successfully") return nil } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index a0749fc68..cfc3685ca 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "strconv" + "sync" csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" "github.com/coreos/pkg/capnslog" @@ -86,14 +87,12 @@ var ControllerTypeMeta = metav1.TypeMeta{ // ClusterController controls an instance of a Rook cluster type ClusterController struct { - context *clusterd.Context - rookImage string - clusterMap map[string]*cluster - osdChecker *osd.OSDHealthMonitor - client client.Client - namespacedName types.NamespacedName - recorder record.EventRecorder - OpManagerCtx context.Context + context *clusterd.Context + rookImage string + clusterMap sync.Map // map[string]*cluster + client client.Client + recorder record.EventRecorder + OpManagerCtx context.Context } // ReconcileCephCluster reconciles a CephCluster object @@ -280,9 +279,6 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R // Used by functions not part of the ClusterController struct but are given the context to execute actions r.clusterController.context.Client = r.client - // Pass object name and namespace - r.clusterController.namespacedName = request.NamespacedName - // Fetch the cephCluster instance cephCluster := &cephv1.CephCluster{} err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster) @@ -333,7 +329,10 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R } func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) (reconcile.Result, cephv1.CephCluster, error) { - nsName := r.clusterController.namespacedName + nsName := types.NamespacedName{ + Namespace: cephCluster.Namespace, + Name: cephCluster.Name, + } var err error // Set the deleting status @@ -394,9 +393,8 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) // NewClusterController create controller for watching cluster custom resources created func NewClusterController(context *clusterd.Context, rookImage string) *ClusterController { return &ClusterController{ - context: context, - rookImage: rookImage, - clusterMap: make(map[string]*cluster), + context: context, + rookImage: rookImage, } } @@ -405,49 +403,57 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster, logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace) return nil } - - cluster, ok := c.clusterMap[clusterObj.Namespace] - if !ok { + var clustr *cluster + if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok { + clustr = rawCluster.(*cluster) + } else { // It's a new cluster so let's populate the struct - cluster = newCluster(c.OpManagerCtx, clusterObj, c.context, ownerInfo, c.rookImage) + clustr = newCluster(c.OpManagerCtx, clusterObj, c.context, ownerInfo, c.rookImage) + } + clustr.namespacedName = types.NamespacedName{ + Namespace: clusterObj.Namespace, + Name: clusterObj.Name, } - cluster.namespacedName = c.namespacedName // updating observedGeneration in cluster if it's not the first reconcile - cluster.observedGeneration = clusterObj.ObjectMeta.Generation + clustr.observedGeneration = clusterObj.ObjectMeta.Generation // Pass down the client to interact with Kubernetes objects // This will be used later down by spec code to create objects like deployment, services etc - cluster.context.Client = c.client + clustr.context.Client = c.client // Set the spec - cluster.Spec = &clusterObj.Spec + clustr.Spec = &clusterObj.Spec - c.clusterMap[cluster.Namespace] = cluster - logger.Infof("reconciling ceph cluster in namespace %q", cluster.Namespace) + c.clusterMap.Store(clustr.Namespace, clustr) + logger.Infof("reconciling ceph cluster in namespace %q", clustr.namespacedName.Namespace) // Start the main ceph cluster orchestration - return c.initializeCluster(cluster) + return c.initializeCluster(clustr) } -func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (reconcile.Result, error) { - nsName := fmt.Sprintf("%s/%s", cluster.Namespace, cluster.Name) +func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) (reconcile.Result, error) { + nsName := fmt.Sprintf("%s/%s", clusterObj.Namespace, clusterObj.Name) - if existing, ok := c.clusterMap[cluster.Namespace]; ok && existing.namespacedName.Name != cluster.Name { - logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.", - nsName, existing.namespacedName.Name) - return reconcile.Result{}, nil // do not requeue the delete + var existingCluster *cluster + if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok { + existingCluster = rawCluster.(*cluster) + if existingCluster.namespacedName.Name != clusterObj.Name { + logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.", + nsName, existingCluster.namespacedName.Name) + return reconcile.Result{}, nil // do not requeue the delete + } } _, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(c.OpManagerCtx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{}) if err == nil { logger.Info("removing networkFence if matching cephCluster UID found") networkFenceList := &addonsv1alpha1.NetworkFenceList{} labelSelector := labels.SelectorFromSet(map[string]string{ - networkFenceLabel: string(cluster.GetUID()), + networkFenceLabel: string(clusterObj.GetUID()), }) opts := []client.DeleteAllOfOption{ client.MatchingLabels{ - networkFenceLabel: string(cluster.GetUID()), + networkFenceLabel: string(clusterObj.GetUID()), }, client.GracePeriodSeconds(0), } @@ -471,45 +477,45 @@ func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (r } logger.Infof("cleaning up CephCluster %q", nsName) - if cluster, ok := c.clusterMap[cluster.Namespace]; ok { + if existingCluster != nil { // We used to stop the bucket controller here but when we get a DELETE event for the CephCluster // we will reload the CRD manager anyway so the bucket controller go routine will be stopped // since the op manager context is cancelled. // close the goroutines watching the health of the cluster (mons, osds, ceph status) for _, daemon := range monitorDaemonList { // if the context hasn't been cancelled - if monitoring, ok := cluster.monitoringRoutines.Load(daemon); ok && monitoring.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil { + if monitoring, ok := existingCluster.monitoringRoutines.Load(daemon); ok && monitoring.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil { // Stop the monitoring routine monitoring.(*opcontroller.ClusterHealth).InternalCancel() // Remove the monitoring routine from the map - cluster.monitoringRoutines.Delete(daemon) + existingCluster.monitoringRoutines.Delete(daemon) } } } - if cluster.Spec.CleanupPolicy.AllowUninstallWithVolumes { + if clusterObj.Spec.CleanupPolicy.AllowUninstallWithVolumes { logger.Info("skipping check for existing PVs as allowUninstallWithVolumes is set to true") } else { - err := c.checkIfVolumesExist(cluster) + err := c.checkIfVolumesExist(clusterObj) if err != nil { - return opcontroller.WaitForRequeueIfFinalizerBlocked, errors.Wrapf(err, "failed to check if volumes exist for CephCluster in namespace %q", cluster.Namespace) + return opcontroller.WaitForRequeueIfFinalizerBlocked, errors.Wrapf(err, "failed to check if volumes exist for CephCluster in namespace %q", clusterObj.Namespace) } } - if cluster.Spec.External.Enable { - purgeExternalCluster(c.context.Clientset, cluster.Namespace) - } else if cluster.Spec.Storage.IsOnPVCEncrypted() && cluster.Spec.Security.KeyManagementService.IsEnabled() { + if clusterObj.Spec.External.Enable { + purgeExternalCluster(c.context.Clientset, clusterObj.Namespace) + } else if clusterObj.Spec.Storage.IsOnPVCEncrypted() && clusterObj.Spec.Security.KeyManagementService.IsEnabled() { // If the StorageClass retain policy of an encrypted cluster with KMS is Delete we also delete the keys // Delete keys from KMS - err := c.deleteOSDEncryptionKeyFromKMS(cluster) + err := c.deleteOSDEncryptionKeyFromKMS(clusterObj) if err != nil { logger.Errorf("failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err) } } - if cluster, ok := c.clusterMap[cluster.Namespace]; ok { - delete(c.clusterMap, cluster.Namespace) + if _, ok := c.clusterMap.Load(clusterObj.Namespace); ok { + c.clusterMap.Delete(clusterObj.Namespace) } return reconcile.Result{}, nil @@ -620,8 +626,11 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1 // So do NOT use another context ctx := context.TODO() // If the operator was stopped and we enter this code, the map is empty - if _, ok := c.clusterMap[currentCluster.Namespace]; !ok { - c.clusterMap[currentCluster.Namespace] = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}} + var clusterObj *cluster + if rawCluster, ok := c.clusterMap.Load(currentCluster.Namespace); ok { + clusterObj = rawCluster.(*cluster) + } else { + clusterObj = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}} } // Fetch PVCs @@ -631,7 +640,7 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1 } // Initialize the KMS code - kmsConfig := kms.NewConfig(c.context, ¤tCluster.Spec, c.clusterMap[currentCluster.Namespace].ClusterInfo) + kmsConfig := kms.NewConfig(c.context, ¤tCluster.Spec, clusterObj.ClusterInfo) kmsConfig.ClusterInfo.Context = ctx // If token auth is used by the KMS we set it as an env variable diff --git a/pkg/operator/ceph/cluster/monitoring.go b/pkg/operator/ceph/cluster/monitoring.go index caaf05573..ef4116fd1 100644 --- a/pkg/operator/ceph/cluster/monitoring.go +++ b/pkg/operator/ceph/cluster/monitoring.go @@ -84,9 +84,9 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo * case "osd": if !cluster.Spec.External.Enable { - c.osdChecker = osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck) + osdChecker := osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck) logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) - go c.osdChecker.Start(&cluster.monitoringRoutines, daemon) + go osdChecker.Start(&cluster.monitoringRoutines, daemon) } case "status": diff --git a/pkg/operator/ceph/cluster/version.go b/pkg/operator/ceph/cluster/version.go index 6d3406f8d..bef0ab249 100644 --- a/pkg/operator/ceph/cluster/version.go +++ b/pkg/operator/ceph/cluster/version.go @@ -46,7 +46,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep } // Update ceph version field in cluster object status - c.updateClusterCephVersion(cluster.Spec.CephVersion.Image, *version) + c.updateClusterCephVersion(cluster, *version) return version, cluster.isUpgrade, nil } diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 10982e301..4554eb05c 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -53,6 +53,7 @@ type TestCephSettings struct { CephVersion cephv1.CephVersionSpec KubernetesVersion string EnableCsiOperator bool + ClusterConcurrency int } func (s *TestCephSettings) ApplyEnvVars() { @@ -98,6 +99,9 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { if !s.EnableCsiOperator { manifest = strings.ReplaceAll(manifest, `ROOK_USE_CSI_OPERATOR: "true"`, `ROOK_USE_CSI_OPERATOR: "false"`) } + if s.ClusterConcurrency > 1 { + manifest = strings.ReplaceAll(manifest, `ROOK_RECONCILE_CONCURRENT_CLUSTERS: "1"`, fmt.Sprintf(`ROOK_RECONCILE_CONCURRENT_CLUSTERS: "%d"`, s.ClusterConcurrency)) + } return manifest } diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index 794457fac..2a63aafaa 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -73,16 +73,17 @@ func (s *MultiClusterDeploySuite) SetupSuite() { s.poolName = "multi-cluster-pool1" coreNamespace := "multi-core" s.settings = &installer.TestCephSettings{ - ClusterName: "multi-cluster", - Namespace: coreNamespace, - OperatorNamespace: installer.SystemNamespace(coreNamespace), - StorageClassName: "manual", - UsePVC: installer.UsePVC(), - Mons: 1, - MultipleMgrs: true, - RookVersion: installer.LocalBuildTag, - CephVersion: installer.SquidVersion, - RequireMsgr2: true, + ClusterName: "multi-cluster", + Namespace: coreNamespace, + OperatorNamespace: installer.SystemNamespace(coreNamespace), + StorageClassName: "manual", + UsePVC: installer.UsePVC(), + Mons: 1, + MultipleMgrs: true, + RookVersion: installer.LocalBuildTag, + CephVersion: installer.SquidVersion, + RequireMsgr2: true, + ClusterConcurrency: 2, } s.settings.ApplyEnvVars() externalSettings := &installer.TestCephSettings{