From 8d033efb5a2fa17ab6ff4906db1a52d364c9b350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Fri, 22 Jan 2021 16:00:06 +0100 Subject: [PATCH] ceph: silence harmless errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the ceph cli outputs an error with "error calling conf_read_file" this means that the operator has not written its ceph configuration file. Thus ceph cli commands will fail, so we can just ignore that since the operator will soon write this file in its initialization sequence. Signed-off-by: Sébastien Han --- pkg/daemon/ceph/client/upgrade.go | 28 +++++++++---------- pkg/operator/ceph/client/controller.go | 5 ++++ pkg/operator/ceph/cluster/cephstatus.go | 5 ++++ pkg/operator/ceph/cluster/rbd/controller.go | 5 ++++ .../ceph/controller/controller_utils.go | 13 +++++++-- .../ceph/disruption/clusterdisruption/osd.go | 2 +- pkg/operator/ceph/file/controller.go | 9 ++++++ pkg/operator/ceph/nfs/controller.go | 5 ++++ pkg/operator/ceph/object/controller.go | 4 +++ 9 files changed, 59 insertions(+), 17 deletions(-) diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 4bccf2592..19dd3b838 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -259,22 +259,22 @@ func LeastUptodateDaemonVersion(context *clusterd.Context, clusterInfo *ClusterI // Always invoke ceph version before an upgrade so we are sure to be up-to-date versions, err := GetAllCephDaemonVersions(context, clusterInfo) if err != nil { - logger.Warningf("failed to get ceph daemons versions, this likely means there is no cluster yet. %v", err) - } else { - r, err = daemonMapEntry(versions, daemonType) + return vv, errors.Wrap(err, "failed to get ceph daemons versions") + } + + r, err = daemonMapEntry(versions, daemonType) + if err != nil { + return vv, errors.Wrap(err, "failed to find daemon map entry") + } + for v := range r { + version, err := cephver.ExtractCephVersion(v) if err != nil { - return vv, errors.Wrap(err, "failed to find daemon map entry") - } - for v := range r { - version, err := cephver.ExtractCephVersion(v) - if err != nil { - return vv, errors.Wrap(err, "failed to extract ceph version") - } - vv = *version - // break right after the first iteration - // the first one is always the least up-to-date - break + return vv, errors.Wrap(err, "failed to extract ceph version") } + vv = *version + // break right after the first iteration + // the first one is always the least up-to-date + break } return vv, nil diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index 9d3d4f222..eb74c2e18 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -22,6 +22,7 @@ import ( "fmt" "reflect" "regexp" + "strings" "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" @@ -209,6 +210,10 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re // Create or Update client err = r.createOrUpdateClient(cephClient) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } updateStatus(r.client, request.NamespacedName, cephv1.ConditionFailure) return reconcile.Result{}, errors.Wrapf(err, "failed to create or update client %q", cephClient.Name) } diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index 8773a9e89..3ecf67ae4 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -20,6 +20,7 @@ package cluster import ( "context" "os" + "strings" "time" "github.com/pkg/errors" @@ -111,6 +112,10 @@ func (c *cephStatusChecker) checkStatus() { // Check ceph's status status, err = cephclient.StatusWithUser(c.context, c.clusterInfo) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping ceph status since operator is still initializing") + return + } logger.Errorf("failed to get ceph status. %v", err) condition, reason, message := c.conditionMessageReason(cephv1.ConditionFailure) if err := c.updateCephStatus(cephStatusOnError(err.Error()), condition, reason, message); err != nil { diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 986dbd159..c0f9c7cd5 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -208,6 +209,10 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile daemon := string(opconfig.MonType) currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, daemon) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to retrieve current ceph %q version", daemon) } r.clusterInfo.CephVersion = currentCephVersion diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index d15d8261a..d03e6cc21 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -33,8 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator -const OperatorSettingConfigMapName string = "rook-ceph-operator-config" +const ( + // OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator + OperatorSettingConfigMapName string = "rook-ceph-operator-config" + + // UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file + // This typically is raised when the operator has not finished initializing + UninitializedCephConfigError = "error calling conf_read_file" +) var ( // ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object. @@ -46,6 +52,9 @@ var ( // WaitForRequeueIfFinalizerBlocked waits for resources to be cleaned up before the finalizer can be removed WaitForRequeueIfFinalizerBlocked = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} + // WaitForRequeueIfOperatorNotInitialized waits for resources to be cleaned up before the finalizer can be removed + WaitForRequeueIfOperatorNotInitialized = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} + // OperatorCephBaseImageVersion is the ceph version in the operator image OperatorCephBaseImageVersion string ) diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd.go b/pkg/operator/ceph/disruption/clusterdisruption/osd.go index abf5a3465..3f6d1ea19 100644 --- a/pkg/operator/ceph/disruption/clusterdisruption/osd.go +++ b/pkg/operator/ceph/disruption/clusterdisruption/osd.go @@ -220,7 +220,7 @@ func (r *ReconcileClusterDisruption) reconcilePDBsForOSDs( if err != nil { // If the error contains that message, this means the cluster is not up and running // No monitors are present and thus no ceph configuration has been created - if strings.Contains(err.Error(), "error calling conf_read_file") { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { logger.Debugf("Ceph %q cluster not ready, cannot check Ceph status yet.", request.Namespace) return nil } diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index c07d18bd3..825c325e6 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -214,6 +215,10 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil // Populate CephVersion currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) } r.clusterInfo.CephVersion = currentCephVersion @@ -238,6 +243,10 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil // validate the filesystem settings if err := validateFilesystem(r.context, r.clusterInfo, r.cephClusterSpec, cephFilesystem); err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "invalid object filesystem %q arguments", cephFilesystem.Name) } diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index 62e0e9ea9..9130578ba 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" @@ -194,6 +195,10 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul // Populate CephVersion currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) } r.clusterInfo.CephVersion = currentCephVersion diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 7f759d829..0117ab0b3 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -240,6 +240,10 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci // Populate CephVersion currentCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, opconfig.MonType) if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info("skipping reconcile since operator is still initializing") + return opcontroller.WaitForRequeueIfOperatorNotInitialized, nil + } return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve current ceph %q version", opconfig.MonType) } r.clusterInfo.CephVersion = currentCephVersion