operator: fix concurrency issues in the cluster controller
Use a sync map for the clusterMap in the cluster controller and fix other issues to allow for concurrent cluster reconcile. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
8 files changed
+99
-85
No files matched your search
@@ -236,28 +236,28 @@ func formatTime(t time.Time) string {
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (c *ClusterController) updateClusterCephVersion(image string, cephVersion cephver.CephVersion) {
|
||||
logger.Infof("cluster %q: version %q detected for image %q", c.namespacedName.Namespace, cephVersion.String(), image)
|
||||
func (c *ClusterController) updateClusterCephVersion(cluster *cluster, cephVersion cephver.CephVersion) {
|
||||
logger.Infof("cluster %q: version %q detected for image %q", cluster.namespacedName.Namespace, cephVersion.String(), cluster.Spec.CephVersion.Image)
|
||||
|
||||
cephCluster, err := c.context.RookClientset.CephV1().CephClusters(c.namespacedName.Namespace).Get(c.OpManagerCtx, c.namespacedName.Name, metav1.GetOptions{})
|
||||
cephCluster, err := c.context.RookClientset.CephV1().CephClusters(cluster.namespacedName.Namespace).Get(c.OpManagerCtx, cluster.namespacedName.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephCluster resource not found. Ignoring since object must be deleted.")
|
||||
return
|
||||
}
|
||||
logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", c.namespacedName.Name, cephVersion, err)
|
||||
logger.Errorf("failed to retrieve ceph cluster %q to update ceph version to %+v. %v", cluster.namespacedName.Name, cephVersion, err)
|
||||
return
|
||||
}
|
||||
|
||||
cephClusterVersion := &cephv1.ClusterVersion{
|
||||
Image: image,
|
||||
Image: cluster.Spec.CephVersion.Image,
|
||||
Version: opcontroller.GetCephVersionLabel(cephVersion), // DO NOT CHANGE FORMAT FROM "Maj.Min.Ext-Bld"
|
||||
}
|
||||
// update the Ceph version on the retrieved cluster object
|
||||
// do not overwrite the ceph status that is updated in a separate goroutine
|
||||
cephCluster.Status.CephVersion = cephClusterVersion
|
||||
if err := reporting.UpdateStatus(c.client, cephCluster); err != nil {
|
||||
logger.Errorf("failed to update cluster %q version. %v", c.namespacedName.Name, err)
|
||||
logger.Errorf("failed to update cluster %q version. %v", cluster.namespacedName.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,7 +195,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
if cluster.Spec.External.Enable {
|
||||
err := c.configureExternalCephCluster(cluster)
|
||||
if err != nil {
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error())
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error())
|
||||
return errors.Wrap(err, "failed to configure external ceph cluster")
|
||||
}
|
||||
} else {
|
||||
@@ -211,7 +211,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
}
|
||||
} else {
|
||||
clusterInfo.OwnerInfo = cluster.ownerInfo
|
||||
clusterInfo.SetName(c.namespacedName.Name)
|
||||
clusterInfo.SetName(cluster.namespacedName.Name)
|
||||
cluster.ClusterInfo = clusterInfo
|
||||
if cluster.mons.ClusterInfo == nil {
|
||||
// ClusterInfo stored in cluster.mons can be lost from the object stored in the
|
||||
@@ -239,7 +239,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
|
||||
err = c.configureLocalCephCluster(cluster)
|
||||
if err != nil {
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error())
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionFalse, cephv1.ClusterProgressingReason, err.Error())
|
||||
return errors.Wrap(err, "failed to configure local ceph cluster")
|
||||
}
|
||||
|
||||
@@ -254,7 +254,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
|
||||
|
||||
// Populate ClusterInfo with the last value
|
||||
cluster.mons.ClusterInfo = cluster.ClusterInfo
|
||||
cluster.mons.ClusterInfo.SetName(c.namespacedName.Name)
|
||||
cluster.mons.ClusterInfo.SetName(cluster.namespacedName.Name)
|
||||
|
||||
// Start the monitoring if not already started
|
||||
c.configureCephMonitoring(cluster, cluster.ClusterInfo)
|
||||
@@ -271,7 +271,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
|
||||
}
|
||||
|
||||
// Run image validation job
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Detecting Ceph version")
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Detecting Ceph version")
|
||||
cephVersion, isUpgrade, err := c.detectAndValidateCephVersion(cluster)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed the ceph version check")
|
||||
@@ -286,7 +286,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
|
||||
}
|
||||
}
|
||||
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring the Ceph cluster")
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionProgressing, v1.ConditionTrue, cephv1.ClusterProgressingReason, "Configuring the Ceph cluster")
|
||||
// Run the orchestration
|
||||
err = cluster.reconcileCephDaemons(c.rookImage, *cephVersion)
|
||||
if err != nil {
|
||||
@@ -294,7 +294,7 @@ func (c *ClusterController) configureLocalCephCluster(cluster *cluster) error {
|
||||
}
|
||||
|
||||
// Set the condition to the cluster object
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, cluster.observedGeneration, cephv1.ConditionReady, v1.ConditionTrue, cephv1.ClusterCreatedReason, "Cluster created successfully")
|
||||
controller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, cluster.observedGeneration, cephv1.ConditionReady, v1.ConditionTrue, cephv1.ClusterCreatedReason, "Cluster created successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -46,15 +46,15 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
return errors.Wrap(err, "failed to validate external cluster specs")
|
||||
}
|
||||
|
||||
opcontroller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnecting, v1.ConditionTrue, cephv1.ClusterConnectingReason, "Attempting to connect to an external Ceph cluster")
|
||||
opcontroller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnecting, v1.ConditionTrue, cephv1.ClusterConnectingReason, "Attempting to connect to an external Ceph cluster")
|
||||
|
||||
// loop until we find the secret necessary to connect to the external cluster
|
||||
// then populate clusterInfo
|
||||
cluster.ClusterInfo, err = opcontroller.PopulateExternalClusterInfo(cluster.Spec, c.context, c.OpManagerCtx, c.namespacedName.Namespace, cluster.ownerInfo)
|
||||
cluster.ClusterInfo, err = opcontroller.PopulateExternalClusterInfo(cluster.Spec, c.context, c.OpManagerCtx, cluster.namespacedName.Namespace, cluster.ownerInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to populate external cluster info")
|
||||
}
|
||||
cluster.ClusterInfo.SetName(c.namespacedName.Name)
|
||||
cluster.ClusterInfo.SetName(cluster.namespacedName.Name)
|
||||
cluster.ClusterInfo.Context = c.OpManagerCtx
|
||||
|
||||
if !client.IsKeyringBase64Encoded(cluster.ClusterInfo.CephCred.Secret) {
|
||||
@@ -82,13 +82,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
//
|
||||
// Only do this when doing a bit of management...
|
||||
logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName)
|
||||
err = populateConfigOverrideConfigMap(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata)
|
||||
err = populateConfigOverrideConfigMap(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo, cluster.clusterMetadata)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to populate config override config map")
|
||||
}
|
||||
|
||||
logger.Infof("creating %q secret", config.StoreName)
|
||||
err = config.GetStore(c.context, c.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo)
|
||||
err = config.GetStore(c.context, cluster.namespacedName.Namespace, cluster.ClusterInfo.OwnerInfo).CreateOrUpdate(cluster.ClusterInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update the global config")
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
|
||||
// Create CSI Secrets only if the user has provided the admin key
|
||||
if cluster.ClusterInfo.CephCred.Username == client.AdminUsername {
|
||||
err = csi.CreateCSISecrets(c.context, cluster.ClusterInfo, c.namespacedName)
|
||||
err = csi.CreateCSISecrets(c.context, cluster.ClusterInfo, cluster.namespacedName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create csi kubernetes secrets")
|
||||
}
|
||||
@@ -131,8 +131,8 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
},
|
||||
}
|
||||
|
||||
clusterId := c.namespacedName.Namespace // cluster id is same as cluster namespace for CephClusters
|
||||
err = csi.SaveClusterConfig(c.context.Clientset, clusterId, c.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry)
|
||||
clusterId := cluster.namespacedName.Namespace // cluster id is same as cluster namespace for CephClusters
|
||||
err = csi.SaveClusterConfig(c.context.Clientset, clusterId, cluster.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update csi cluster config")
|
||||
}
|
||||
@@ -168,7 +168,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
cluster.ClusterInfo.CephVersion = *externalVersion
|
||||
|
||||
// Populate ceph version
|
||||
c.updateClusterCephVersion("", *externalVersion)
|
||||
c.updateClusterCephVersion(cluster, *externalVersion)
|
||||
|
||||
err = c.configureExternalClusterMonitoring(c.context, cluster)
|
||||
if err != nil {
|
||||
@@ -188,7 +188,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
}
|
||||
}
|
||||
|
||||
opcontroller.UpdateCondition(c.OpManagerCtx, c.context, c.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnected, v1.ConditionTrue, cephv1.ClusterConnectedReason, "Cluster connected successfully")
|
||||
opcontroller.UpdateCondition(c.OpManagerCtx, c.context, cluster.namespacedName, k8sutil.ObservedGenerationNotAvailable, cephv1.ConditionConnected, v1.ConditionTrue, cephv1.ClusterConnectedReason, "Cluster connected successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
csiopv1 "github.com/ceph/ceph-csi-operator/api/v1"
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
@@ -86,14 +87,12 @@ var ControllerTypeMeta = metav1.TypeMeta{
|
||||
|
||||
// ClusterController controls an instance of a Rook cluster
|
||||
type ClusterController struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
clusterMap map[string]*cluster
|
||||
osdChecker *osd.OSDHealthMonitor
|
||||
client client.Client
|
||||
namespacedName types.NamespacedName
|
||||
recorder record.EventRecorder
|
||||
OpManagerCtx context.Context
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
clusterMap sync.Map // map[string]*cluster
|
||||
client client.Client
|
||||
recorder record.EventRecorder
|
||||
OpManagerCtx context.Context
|
||||
}
|
||||
|
||||
// ReconcileCephCluster reconciles a CephCluster object
|
||||
@@ -280,9 +279,6 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
// Used by functions not part of the ClusterController struct but are given the context to execute actions
|
||||
r.clusterController.context.Client = r.client
|
||||
|
||||
// Pass object name and namespace
|
||||
r.clusterController.namespacedName = request.NamespacedName
|
||||
|
||||
// Fetch the cephCluster instance
|
||||
cephCluster := &cephv1.CephCluster{}
|
||||
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCluster)
|
||||
@@ -333,7 +329,10 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
}
|
||||
|
||||
func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster) (reconcile.Result, cephv1.CephCluster, error) {
|
||||
nsName := r.clusterController.namespacedName
|
||||
nsName := types.NamespacedName{
|
||||
Namespace: cephCluster.Namespace,
|
||||
Name: cephCluster.Name,
|
||||
}
|
||||
var err error
|
||||
|
||||
// Set the deleting status
|
||||
@@ -394,9 +393,8 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
|
||||
// NewClusterController create controller for watching cluster custom resources created
|
||||
func NewClusterController(context *clusterd.Context, rookImage string) *ClusterController {
|
||||
return &ClusterController{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
clusterMap: make(map[string]*cluster),
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,49 +403,57 @@ func (c *ClusterController) reconcileCephCluster(clusterObj *cephv1.CephCluster,
|
||||
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
cluster, ok := c.clusterMap[clusterObj.Namespace]
|
||||
if !ok {
|
||||
var clustr *cluster
|
||||
if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok {
|
||||
clustr = rawCluster.(*cluster)
|
||||
} else {
|
||||
// It's a new cluster so let's populate the struct
|
||||
cluster = newCluster(c.OpManagerCtx, clusterObj, c.context, ownerInfo, c.rookImage)
|
||||
clustr = newCluster(c.OpManagerCtx, clusterObj, c.context, ownerInfo, c.rookImage)
|
||||
}
|
||||
clustr.namespacedName = types.NamespacedName{
|
||||
Namespace: clusterObj.Namespace,
|
||||
Name: clusterObj.Name,
|
||||
}
|
||||
cluster.namespacedName = c.namespacedName
|
||||
// updating observedGeneration in cluster if it's not the first reconcile
|
||||
cluster.observedGeneration = clusterObj.ObjectMeta.Generation
|
||||
clustr.observedGeneration = clusterObj.ObjectMeta.Generation
|
||||
|
||||
// Pass down the client to interact with Kubernetes objects
|
||||
// This will be used later down by spec code to create objects like deployment, services etc
|
||||
cluster.context.Client = c.client
|
||||
clustr.context.Client = c.client
|
||||
|
||||
// Set the spec
|
||||
cluster.Spec = &clusterObj.Spec
|
||||
clustr.Spec = &clusterObj.Spec
|
||||
|
||||
c.clusterMap[cluster.Namespace] = cluster
|
||||
logger.Infof("reconciling ceph cluster in namespace %q", cluster.Namespace)
|
||||
c.clusterMap.Store(clustr.Namespace, clustr)
|
||||
logger.Infof("reconciling ceph cluster in namespace %q", clustr.namespacedName.Namespace)
|
||||
|
||||
// Start the main ceph cluster orchestration
|
||||
return c.initializeCluster(cluster)
|
||||
return c.initializeCluster(clustr)
|
||||
}
|
||||
|
||||
func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (reconcile.Result, error) {
|
||||
nsName := fmt.Sprintf("%s/%s", cluster.Namespace, cluster.Name)
|
||||
func (c *ClusterController) requestClusterDelete(clusterObj *cephv1.CephCluster) (reconcile.Result, error) {
|
||||
nsName := fmt.Sprintf("%s/%s", clusterObj.Namespace, clusterObj.Name)
|
||||
|
||||
if existing, ok := c.clusterMap[cluster.Namespace]; ok && existing.namespacedName.Name != cluster.Name {
|
||||
logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.",
|
||||
nsName, existing.namespacedName.Name)
|
||||
return reconcile.Result{}, nil // do not requeue the delete
|
||||
var existingCluster *cluster
|
||||
if rawCluster, ok := c.clusterMap.Load(clusterObj.Namespace); ok {
|
||||
existingCluster = rawCluster.(*cluster)
|
||||
if existingCluster.namespacedName.Name != clusterObj.Name {
|
||||
logger.Errorf("skipping deletion of CephCluster %q. CephCluster CR %q already exists in this namespace. only one cluster cr per namespace is supported.",
|
||||
nsName, existingCluster.namespacedName.Name)
|
||||
return reconcile.Result{}, nil // do not requeue the delete
|
||||
}
|
||||
}
|
||||
_, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(c.OpManagerCtx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Info("removing networkFence if matching cephCluster UID found")
|
||||
networkFenceList := &addonsv1alpha1.NetworkFenceList{}
|
||||
labelSelector := labels.SelectorFromSet(map[string]string{
|
||||
networkFenceLabel: string(cluster.GetUID()),
|
||||
networkFenceLabel: string(clusterObj.GetUID()),
|
||||
})
|
||||
|
||||
opts := []client.DeleteAllOfOption{
|
||||
client.MatchingLabels{
|
||||
networkFenceLabel: string(cluster.GetUID()),
|
||||
networkFenceLabel: string(clusterObj.GetUID()),
|
||||
},
|
||||
client.GracePeriodSeconds(0),
|
||||
}
|
||||
@@ -471,45 +477,45 @@ func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (r
|
||||
}
|
||||
|
||||
logger.Infof("cleaning up CephCluster %q", nsName)
|
||||
if cluster, ok := c.clusterMap[cluster.Namespace]; ok {
|
||||
if existingCluster != nil {
|
||||
// We used to stop the bucket controller here but when we get a DELETE event for the CephCluster
|
||||
// we will reload the CRD manager anyway so the bucket controller go routine will be stopped
|
||||
// since the op manager context is cancelled.
|
||||
// close the goroutines watching the health of the cluster (mons, osds, ceph status)
|
||||
for _, daemon := range monitorDaemonList {
|
||||
// if the context hasn't been cancelled
|
||||
if monitoring, ok := cluster.monitoringRoutines.Load(daemon); ok && monitoring.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil {
|
||||
if monitoring, ok := existingCluster.monitoringRoutines.Load(daemon); ok && monitoring.(*opcontroller.ClusterHealth).InternalCtx.Err() == nil {
|
||||
// Stop the monitoring routine
|
||||
monitoring.(*opcontroller.ClusterHealth).InternalCancel()
|
||||
|
||||
// Remove the monitoring routine from the map
|
||||
cluster.monitoringRoutines.Delete(daemon)
|
||||
existingCluster.monitoringRoutines.Delete(daemon)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cluster.Spec.CleanupPolicy.AllowUninstallWithVolumes {
|
||||
if clusterObj.Spec.CleanupPolicy.AllowUninstallWithVolumes {
|
||||
logger.Info("skipping check for existing PVs as allowUninstallWithVolumes is set to true")
|
||||
} else {
|
||||
err := c.checkIfVolumesExist(cluster)
|
||||
err := c.checkIfVolumesExist(clusterObj)
|
||||
if err != nil {
|
||||
return opcontroller.WaitForRequeueIfFinalizerBlocked, errors.Wrapf(err, "failed to check if volumes exist for CephCluster in namespace %q", cluster.Namespace)
|
||||
return opcontroller.WaitForRequeueIfFinalizerBlocked, errors.Wrapf(err, "failed to check if volumes exist for CephCluster in namespace %q", clusterObj.Namespace)
|
||||
}
|
||||
}
|
||||
|
||||
if cluster.Spec.External.Enable {
|
||||
purgeExternalCluster(c.context.Clientset, cluster.Namespace)
|
||||
} else if cluster.Spec.Storage.IsOnPVCEncrypted() && cluster.Spec.Security.KeyManagementService.IsEnabled() {
|
||||
if clusterObj.Spec.External.Enable {
|
||||
purgeExternalCluster(c.context.Clientset, clusterObj.Namespace)
|
||||
} else if clusterObj.Spec.Storage.IsOnPVCEncrypted() && clusterObj.Spec.Security.KeyManagementService.IsEnabled() {
|
||||
// If the StorageClass retain policy of an encrypted cluster with KMS is Delete we also delete the keys
|
||||
// Delete keys from KMS
|
||||
err := c.deleteOSDEncryptionKeyFromKMS(cluster)
|
||||
err := c.deleteOSDEncryptionKeyFromKMS(clusterObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete osd encryption keys for CephCluster %q from kms; deletion will continue. %v", nsName, err)
|
||||
}
|
||||
}
|
||||
|
||||
if cluster, ok := c.clusterMap[cluster.Namespace]; ok {
|
||||
delete(c.clusterMap, cluster.Namespace)
|
||||
if _, ok := c.clusterMap.Load(clusterObj.Namespace); ok {
|
||||
c.clusterMap.Delete(clusterObj.Namespace)
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
@@ -620,8 +626,11 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1
|
||||
// So do NOT use another context
|
||||
ctx := context.TODO()
|
||||
// If the operator was stopped and we enter this code, the map is empty
|
||||
if _, ok := c.clusterMap[currentCluster.Namespace]; !ok {
|
||||
c.clusterMap[currentCluster.Namespace] = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}}
|
||||
var clusterObj *cluster
|
||||
if rawCluster, ok := c.clusterMap.Load(currentCluster.Namespace); ok {
|
||||
clusterObj = rawCluster.(*cluster)
|
||||
} else {
|
||||
clusterObj = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}}
|
||||
}
|
||||
|
||||
// Fetch PVCs
|
||||
@@ -631,7 +640,7 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1
|
||||
}
|
||||
|
||||
// Initialize the KMS code
|
||||
kmsConfig := kms.NewConfig(c.context, ¤tCluster.Spec, c.clusterMap[currentCluster.Namespace].ClusterInfo)
|
||||
kmsConfig := kms.NewConfig(c.context, ¤tCluster.Spec, clusterObj.ClusterInfo)
|
||||
kmsConfig.ClusterInfo.Context = ctx
|
||||
|
||||
// If token auth is used by the KMS we set it as an env variable
|
||||
|
||||
@@ -84,9 +84,9 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *
|
||||
|
||||
case "osd":
|
||||
if !cluster.Spec.External.Enable {
|
||||
c.osdChecker = osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
|
||||
osdChecker := osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
|
||||
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
|
||||
go c.osdChecker.Start(&cluster.monitoringRoutines, daemon)
|
||||
go osdChecker.Start(&cluster.monitoringRoutines, daemon)
|
||||
}
|
||||
|
||||
case "status":
|
||||
|
||||
@@ -46,7 +46,7 @@ func (c *ClusterController) detectAndValidateCephVersion(cluster *cluster) (*cep
|
||||
}
|
||||
|
||||
// Update ceph version field in cluster object status
|
||||
c.updateClusterCephVersion(cluster.Spec.CephVersion.Image, *version)
|
||||
c.updateClusterCephVersion(cluster, *version)
|
||||
|
||||
return version, cluster.isUpgrade, nil
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ type TestCephSettings struct {
|
||||
CephVersion cephv1.CephVersionSpec
|
||||
KubernetesVersion string
|
||||
EnableCsiOperator bool
|
||||
ClusterConcurrency int
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) ApplyEnvVars() {
|
||||
@@ -98,6 +99,9 @@ func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
|
||||
if !s.EnableCsiOperator {
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_USE_CSI_OPERATOR: "true"`, `ROOK_USE_CSI_OPERATOR: "false"`)
|
||||
}
|
||||
if s.ClusterConcurrency > 1 {
|
||||
manifest = strings.ReplaceAll(manifest, `ROOK_RECONCILE_CONCURRENT_CLUSTERS: "1"`, fmt.Sprintf(`ROOK_RECONCILE_CONCURRENT_CLUSTERS: "%d"`, s.ClusterConcurrency))
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
|
||||
@@ -73,16 +73,17 @@ func (s *MultiClusterDeploySuite) SetupSuite() {
|
||||
s.poolName = "multi-cluster-pool1"
|
||||
coreNamespace := "multi-core"
|
||||
s.settings = &installer.TestCephSettings{
|
||||
ClusterName: "multi-cluster",
|
||||
Namespace: coreNamespace,
|
||||
OperatorNamespace: installer.SystemNamespace(coreNamespace),
|
||||
StorageClassName: "manual",
|
||||
UsePVC: installer.UsePVC(),
|
||||
Mons: 1,
|
||||
MultipleMgrs: true,
|
||||
RookVersion: installer.LocalBuildTag,
|
||||
CephVersion: installer.SquidVersion,
|
||||
RequireMsgr2: true,
|
||||
ClusterName: "multi-cluster",
|
||||
Namespace: coreNamespace,
|
||||
OperatorNamespace: installer.SystemNamespace(coreNamespace),
|
||||
StorageClassName: "manual",
|
||||
UsePVC: installer.UsePVC(),
|
||||
Mons: 1,
|
||||
MultipleMgrs: true,
|
||||
RookVersion: installer.LocalBuildTag,
|
||||
CephVersion: installer.SquidVersion,
|
||||
RequireMsgr2: true,
|
||||
ClusterConcurrency: 2,
|
||||
}
|
||||
s.settings.ApplyEnvVars()
|
||||
externalSettings := &installer.TestCephSettings{
|
||||
|
||||
Reference in new issue
Block a user