diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 4bccf2592..19dd3b838 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -259,22 +259,22 @@ func LeastUptodateDaemonVersion(context *clusterd.Context, clusterInfo *ClusterI // Always invoke ceph version before an upgrade so we are sure to be up-to-date versions, err := GetAllCephDaemonVersions(context, clusterInfo) if err != nil { - logger.Warningf("failed to get ceph daemons versions, this likely means there is no cluster yet. %v", err) - } else { - r, err = daemonMapEntry(versions, daemonType) + return vv, errors.Wrap(err, "failed to get ceph daemons versions") + } + + r, err = daemonMapEntry(versions, daemonType) + if err != nil { + return vv, errors.Wrap(err, "failed to find daemon map entry") + } + for v := range r { + version, err := cephver.ExtractCephVersion(v) if err != nil { - return vv, errors.Wrap(err, "failed to find daemon map entry") - } - for v := range r { - version, err := cephver.ExtractCephVersion(v) - if err != nil { - return vv, errors.Wrap(err, "failed to extract ceph version") - } - vv = *version - // break right after the first iteration - // the first one is always the least up-to-date - break + return vv, errors.Wrap(err, "failed to extract ceph version") } + vv = *version + // break right after the first iteration + // the first one is always the least up-to-date + break } return vv, nil diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index 9d3d4f222..eb74c2e18 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -22,6 +22,7 @@ import ( "fmt" "reflect" "regexp" + "strings" "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" @@ -209,6 +210,10 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re // Create or Update client err = r.createOrUpdateClient(cephClient) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) return reconcile.Result{}, errors.Wrapf(err, "failed to create or update client %q", cephClient.Name) } diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index 8773a9e89..3ecf67ae4 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -20,6 +20,7 @@ package cluster import ( "context" "os" + "strings" "time" "github.com/pkg/errors" @@ -111,6 +112,10 @@ func (c *cephStatusChecker) checkStatus() { // Check ceph's status status, err = cephclient.StatusWithUser(c.context, c.clusterInfo) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping ceph status since operator is still initializing") + return + } logger.Errorf("failed to get ceph status. %v", err) condition, reason, message := c.conditionMessageReason(cephv1.ConditionFailure) if err := c.updateCephStatus(cephStatusOnError(err.Error()), condition, reason, message); err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 986dbd159..c0f9c7cd5 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -208,6 +209,10 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile daemon := string(opconfig.MonType) currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon) } r.clusterInfo.CephVersion = currentCephVersion diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index d15d8261a..d03e6cc21 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -33,8 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator -const OperatorSettingConfigMapName string = "rook-ceph-operator-config" +const ( + // OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator + OperatorSettingConfigMapName string = "rook-ceph-operator-config" + + // UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file + // This typically is raised when the operator has not finished initializing + UninitializedCephConfigError = "error calling conf_read_file" +) var ( // ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object. @@ -46,6 +52,9 @@ var ( // WaitForRequeueIfFinalizerBlocked waits for resources to be cleaned up before the finalizer can be removed WaitForRequeueIfFinalizerBlocked = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} + // WaitForRequeueIfOperatorNotInitialized waits for resources to be cleaned up before the finalizer can be removed + WaitForRequeueIfOperatorNotInitialized = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} + // OperatorCephBaseImageVersion is the ceph version in the operator image OperatorCephBaseImageVersion string ) diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd.go b/pkg/operator/ceph/disruption/clusterdisruption/osd.go index abf5a3465..3f6d1ea19 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd.go @@ -220,7 +220,7 @@ func (r *ReconcileClusterDisruption) reconcilePDBsForOSDs( if err != nil { // If the error contains that message, this means the cluster is not up and running // No monitors are present and thus no ceph configuration has been created - if strings.Contains(err.Error(), "error calling conf_read_file") { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { logger.Debugf("Ceph %q cluster not ready, cannot check Ceph status yet.", request.Namespace) return nil } diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index c07d18bd3..825c325e6 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -214,6 +215,10 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil // Populate CephVersion currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) } r.clusterInfo.CephVersion = currentCephVersion @@ -238,6 +243,10 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil // validate the filesystem settings if err := validateFilesystem(r.context, r.clusterInfo, r.cephClusterSpec, cephFilesystem); err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "invalid object filesystem %q arguments", cephFilesystem.Name) } diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index 62e0e9ea9..9130578ba 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -194,6 +195,10 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul // Populate CephVersion currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) } r.clusterInfo.CephVersion = currentCephVersion diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 7f759d829..0117ab0b3 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -240,6 +240,10 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci // Populate CephVersion currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) } r.clusterInfo.CephVersion = currentCephVersion