core: update loadclusterInfo with multus check

if Multus is enabled the clusterinfo should be updated with
network as multus as to run the ceph cmds in remote
executor

Signed-off-by: parth-gr <paarora@redhat.com>
This commit is contained in:
parth-gr
2022-09-22 14:46:51 +05:30
parent ddea97ebde
commit 26584fc6e5
26 changed files with 48 additions and 39 deletions
@@ -1424,7 +1424,7 @@ jobs:
- name: set up tmate session for debugging
if: failure() && env.USE_TMATE
uses: mxschmitt/action-tmate@v3
timeout-minutes: 600
timeout-minutes: 60
csi-hostnetwork-disabled:
runs-on: ubuntu-20.04
+2 -2
View File
@@ -154,7 +154,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
}
// Make sure a CephCluster is present otherwise do nothing
_, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
if !isReadyToReconcile {
// This handles the case where the Ceph Cluster is gone and we want to delete that CR
// We skip the deletePool() function since everything is gone already
@@ -176,7 +176,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, &cephCluster.Spec)
if err != nil {
return reconcile.Result{}, *cephClient, errors.Wrap(err, "failed to populate cluster info")
}
+2 -2
View File
@@ -248,8 +248,8 @@ func (c *ClusterController) getCephHosts(namespace string) ([]string, error) {
return hostNameList, nil
}
func (c *ClusterController) getCleanUpDetails(namespace string) (string, string, error) {
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, namespace)
func (c *ClusterController) getCleanUpDetails(cephClusterSpec *cephv1.ClusterSpec, namespace string) (string, string, error) {
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, namespace, cephClusterSpec)
if err != nil {
return "", "", errors.Wrap(err, "failed to get cluster info")
}
+1 -1
View File
@@ -187,7 +187,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
return errors.Wrap(err, "failed to configure external ceph cluster")
}
} else {
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace)
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.OpManagerCtx, cluster.Namespace, cluster.Spec)
if err != nil {
if errors.Is(err, controller.ClusterInfoNoClusterNoSecret) {
logger.Info("clusterInfo not yet found, must be a new cluster.")
@@ -48,8 +48,7 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
// loop until we find the secret necessary to connect to the external cluster
// then populate clusterInfo
cluster.ClusterInfo, err = opcontroller.PopulateExternalClusterInfo(c.context, c.OpManagerCtx, c.namespacedName.Namespace, cluster.ownerInfo)
cluster.ClusterInfo, err = opcontroller.PopulateExternalClusterInfo(cluster.Spec, c.context, c.OpManagerCtx, c.namespacedName.Namespace, cluster.ownerInfo)
if err != nil {
return errors.Wrap(err, "failed to populate external cluster info")
}
+1 -1
View File
@@ -299,7 +299,7 @@ func (r *ReconcileCephCluster) reconcileDelete(cephCluster *cephv1.CephCluster)
// Start cluster clean up only if cleanupPolicy is applied to the ceph cluster
internalCtx := context.Context(r.opManagerContext)
if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.Spec.External.Enable {
monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(cephCluster.Namespace)
monSecret, clusterFSID, err := r.clusterController.getCleanUpDetails(&cephCluster.Spec, cephCluster.Namespace)
if err != nil {
logger.Warningf("failed to get mon secret. skip cluster cleanup. remove finalizer. %v", err)
doCleanup = false
+1 -1
View File
@@ -480,7 +480,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
context := c.ClusterInfo.Context
// get the cluster info from secret
c.ClusterInfo, c.maxMonID, c.mapping, err = controller.CreateOrLoadClusterInfo(c.context, context, c.Namespace, c.ownerInfo)
c.ClusterInfo, c.maxMonID, c.mapping, err = controller.CreateOrLoadClusterInfo(c.context, context, c.Namespace, c.ownerInfo, &c.spec)
if err != nil {
return errors.Wrap(err, "failed to get cluster info")
}
+1 -1
View File
@@ -187,7 +187,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile
// Populate clusterInfo
// Always populate it during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
if err != nil {
return opcontroller.ImmediateRetryResult, *cephRBDMirror, errors.Wrap(err, "failed to populate cluster info")
}
+1 -1
View File
@@ -143,7 +143,7 @@ func (c *cluster) validateCephVersion(version *cephver.CephVersion) error {
// state we are in and if we should upgrade or not
// Try to load clusterInfo so we can compare the running version with the one from the spec image
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.ClusterInfo.Context, c.Namespace)
clusterInfo, _, _, err := controller.LoadClusterInfo(c.context, c.ClusterInfo.Context, c.Namespace, c.Spec)
if err == nil {
clusterInfo.Context = c.ClusterInfo.Context
// Write connection info (ceph config file and keyring) for ceph commands
+10 -5
View File
@@ -86,12 +86,12 @@ type MonScheduleInfo struct {
}
// LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID
func LoadClusterInfo(ctx *clusterd.Context, context context.Context, namespace string) (*cephclient.ClusterInfo, int, *Mapping, error) {
return CreateOrLoadClusterInfo(ctx, context, namespace, nil)
func LoadClusterInfo(ctx *clusterd.Context, context context.Context, namespace string, cephClusterSpec *cephv1.ClusterSpec) (*cephclient.ClusterInfo, int, *Mapping, error) {
return CreateOrLoadClusterInfo(ctx, context, namespace, nil, cephClusterSpec)
}
// CreateOrLoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID
func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, context context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, int, *Mapping, error) {
func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, context context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo, cephClusterSpec *cephv1.ClusterSpec) (*cephclient.ClusterInfo, int, *Mapping, error) {
var clusterInfo *cephclient.ClusterInfo
maxMonID := -1
monMapping := &Mapping{
@@ -148,6 +148,11 @@ func CreateOrLoadClusterInfo(clusterdContext *clusterd.Context, context context.
return nil, maxMonID, monMapping, errors.Wrap(err, "failed to get mon config")
}
// update clusterInfo with cephCLusterSpec Network info
// it will also update if Multus is enabled, so ceph cmds
// can run on remote executor
clusterInfo.NetworkSpec = cephClusterSpec.Network
// If an admin key was provided we don't need to load the other resources
// Some people might want to give the admin key
// The necessary users/keys/secrets will be created by Rook
@@ -357,13 +362,13 @@ func ParseMonEndpoints(input string) map[string]*cephclient.MonInfo {
// PopulateExternalClusterInfo Add validation in the code to fail if the external cluster has no
// OSDs keep waiting
func PopulateExternalClusterInfo(context *clusterd.Context, ctx context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, error) {
func PopulateExternalClusterInfo(cephClusterSpec *cephv1.ClusterSpec, context *clusterd.Context, ctx context.Context, namespace string, ownerInfo *k8sutil.OwnerInfo) (*cephclient.ClusterInfo, error) {
for {
// Checking for the context makes sure we don't loop forever with a canceled context
if ctx.Err() != nil {
return nil, ctx.Err()
}
clusterInfo, _, _, err := CreateOrLoadClusterInfo(context, ctx, namespace, nil)
clusterInfo, _, _, err := CreateOrLoadClusterInfo(context, ctx, namespace, nil, cephClusterSpec)
if err != nil {
logger.Warningf("waiting for connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
logger.Debugf("%v", err)
@@ -23,6 +23,7 @@ import (
"os"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/test"
@@ -54,9 +55,14 @@ func TestCreateClusterSecrets(t *testing.T) {
Clientset: clientset,
Executor: executor,
}
provider := "Multus"
cephClusterSpec := &cephv1.ClusterSpec{
Network: cephv1.NetworkSpec{
Provider: provider},
}
namespace := "ns"
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
info, maxID, mapping, err := CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
info, maxID, mapping, err := CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
assert.NoError(t, err)
assert.Equal(t, -1, maxID)
require.NotNil(t, info)
@@ -79,7 +85,7 @@ func TestCreateClusterSecrets(t *testing.T) {
assert.NoError(t, err)
// Check that the cluster info can now be loaded
info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
assert.NoError(t, err)
assert.Equal(t, "client.admin", info.CephCred.Username)
assert.Equal(t, adminSecret, info.CephCred.Secret)
@@ -88,7 +94,7 @@ func TestCreateClusterSecrets(t *testing.T) {
secret.Data[adminSecretNameKey] = []byte(adminSecretNameKey)
_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
assert.NoError(t, err)
_, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
_, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
assert.Error(t, err)
// Load the external cluster with the legacy external creds
@@ -99,7 +105,7 @@ func TestCreateClusterSecrets(t *testing.T) {
}
_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
assert.NoError(t, err)
info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo)
info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
assert.NoError(t, err)
assert.Equal(t, "testid", info.CephCred.Username)
assert.Equal(t, "testkey", info.CephCred.Secret)
+1 -1
View File
@@ -193,7 +193,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
// If so deploy the plugin holder with the fsid attached
if holderEnabled {
// Load cluster info for later use in updating the ceph-csi configmap
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace)
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cluster.Spec)
if err != nil {
// This avoids a requeue with exponential backoff and allows the controller to reconcile
// more quickly when the cluster is ready.
+1 -1
View File
@@ -237,7 +237,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil
// Populate clusterInfo
// Always populate it during each reconcile
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
if err != nil {
return reconcile.Result{}, *cephFilesystem, errors.Wrap(err, "failed to populate cluster info")
}
+1 -1
View File
@@ -177,7 +177,7 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc
r.cephClusterSpec = &cephCluster.Spec
// Populate clusterInfo
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
if err != nil {
return opcontroller.ImmediateRetryResult, *filesystemMirror, errors.Wrap(err, "failed to populate cluster info")
}
@@ -172,7 +172,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, &cephCluster.Spec)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
}
+1 -1
View File
@@ -203,7 +203,7 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul
// Populate clusterInfo
// Always populate it during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
if err != nil {
return reconcile.Result{}, *cephNFS, errors.Wrap(err, "failed to populate cluster info")
}
@@ -147,7 +147,7 @@ func (r *ReconcileBucket) reconcile(request reconcile.Request) (reconcile.Result
}
// Populate clusterInfo during each reconcile
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace)
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace, &cephCluster.Spec)
if err != nil {
// This avoids a requeue with exponential backoff and allows the controller to reconcile
// more quickly when the cluster is ready.
+1 -1
View File
@@ -233,7 +233,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.clusterSpec)
if err != nil {
return reconcile.Result{}, *cephObjectStore, errors.Wrap(err, "failed to populate cluster info")
}
@@ -245,7 +245,7 @@ func getReadyCluster(client client.Client, opManagerContext context.Context, con
logger.Debug("Ceph cluster not yet present.")
return nil, nil, nil
}
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(&context, opManagerContext, cephCluster.Namespace)
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(&context, opManagerContext, cephCluster.Namespace, &cephCluster.Spec)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to populate cluster info")
}
+2 -2
View File
@@ -148,7 +148,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R
}
// Make sure a CephCluster is present otherwise do nothing
_, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
if !isReadyToReconcile {
// This handles the case where the Ceph Cluster is gone and we want to delete that CR
if !cephObjectRealm.GetDeletionTimestamp().IsZero() && !cephClusterExists {
@@ -168,7 +168,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, &cephCluster.Spec)
if err != nil {
return reconcile.Result{}, *cephObjectRealm, errors.Wrap(err, "failed to populate cluster info")
}
+1 -1
View File
@@ -147,7 +147,7 @@ func (r *ReconcileBucketTopic) reconcile(request reconcile.Request) (reconcile.R
r.clusterSpec = &cephCluster.Spec
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace, r.clusterSpec)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
}
+1 -1
View File
@@ -187,7 +187,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
r.cephClusterSpec = &cephCluster.Spec
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
if err != nil {
return reconcile.Result{}, *cephObjectStoreUser, errors.Wrap(err, "failed to populate cluster info")
}
+1 -1
View File
@@ -173,7 +173,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
r.clusterSpec = &cephCluster.Spec
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.clusterSpec)
if err != nil {
return reconcile.Result{}, *cephObjectZone, errors.Wrap(err, "failed to populate cluster info")
}
@@ -142,7 +142,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci
}
// Make sure a CephCluster is present otherwise do nothing
_, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
if !isReadyToReconcile {
// This handles the case where the Ceph Cluster is gone and we want to delete that CR
if !cephObjectZoneGroup.GetDeletionTimestamp().IsZero() && !cephClusterExists {
@@ -161,7 +161,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, &cephCluster.Spec)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
}
+1 -2
View File
@@ -202,12 +202,11 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
}
// Populate clusterInfo during each reconcile
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, &cephCluster.Spec)
if err != nil {
return opcontroller.ImmediateRetryResult, *cephBlockPool, errors.Wrap(err, "failed to populate cluster info")
}
r.clusterInfo = clusterInfo
r.clusterInfo.NetworkSpec = cephCluster.Spec.Network
// Initialize the channel for this pool
// This allows us to track multiple CephBlockPool in the same namespace
@@ -168,7 +168,7 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace)
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, &cephCluster.Spec)
if err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
}