diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index f746d1920..145dacb16 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -48,6 +48,25 @@ var ( OperatorCephBaseImageVersion string ) +// canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely. +func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool { + // Get a list of all the keys causing the HEALTH_ERR status. + var healthErrKeys = make([]string, 0) + for key, health := range cephCluster.Status.CephStatus.Details { + if health.Severity == "HEALTH_ERR" { + healthErrKeys = append(healthErrKeys, key) + } + } + + // If there is only one cause for HEALTH_ERR and it's on the allowed list of errors, ignore it. + var allowedErrStatus = []string{"MDS_ALL_DOWN"} + var ignoreHealthErr = len(healthErrKeys) == 1 && contains(allowedErrStatus, healthErrKeys[0]) + if ignoreHealthErr { + logger.Debugf("%q: ignoring ceph status %q because only cause is %q (full status is %q)", controllerName, cephCluster.Status.CephStatus.Health, healthErrKeys[0], cephCluster.Status.CephStatus) + } + return ignoreHealthErr +} + // IsReadyToReconcile determines if a controller is ready to reconcile or not func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) { cephClusterExists := false @@ -72,10 +91,13 @@ func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, names // read the CR status of the cluster if cephCluster.Status.CephStatus != nil { - if cephCluster.Status.CephStatus.Health == "HEALTH_OK" || cephCluster.Status.CephStatus.Health == "HEALTH_WARN" { + var operatorDeploymentOk = cephCluster.Status.CephStatus.Health == "HEALTH_OK" || cephCluster.Status.CephStatus.Health == "HEALTH_WARN" + + if operatorDeploymentOk || canIgnoreHealthErrStatusInReconcile(cephCluster, controllerName) { logger.Debugf("%q: ceph status is %q, operator is ready to run ceph command, reconciling", controllerName, cephCluster.Status.CephStatus.Health) return cephCluster, true, cephClusterExists, WaitForRequeueIfCephClusterNotReady } + logger.Infof("%s: CephCluster %q found but skipping reconcile since ceph health is %q", controllerName, cephCluster.Name, cephCluster.Status.CephStatus) } diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go new file mode 100644 index 000000000..0fbdb1649 --- /dev/null +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/stretchr/testify/assert" +) + +func CreateTestClusterFromStatusDetails(details map[string]cephv1.CephHealthMessage) cephv1.CephCluster { + return cephv1.CephCluster{ + Status: cephv1.ClusterStatus{ + CephStatus: &cephv1.CephStatus{ + Details: details, + }, + }, + } +} + +func TestCanIgnoreHealthErrStatusInReconcile(t *testing.T) { + var cluster = CreateTestClusterFromStatusDetails(map[string]cephv1.CephHealthMessage{ + "MDS_ALL_DOWN": { + Severity: "HEALTH_ERR", + Message: "MDS_ALL_DOWN", + }, + "TEST_OTHER": { + Severity: "HEALTH_WARN", + Message: "TEST_OTHER", + }, + "TEST_ANOTHER": { + Severity: "HEALTH_OK", + Message: "TEST_ANOTHER", + }, + }) + assert.True(t, canIgnoreHealthErrStatusInReconcile(cluster, "controller")) + + cluster = CreateTestClusterFromStatusDetails(map[string]cephv1.CephHealthMessage{ + "MDS_ALL_DOWN": { + Severity: "HEALTH_ERR", + Message: "MDS_ALL_DOWN", + }, + "TEST_UNIGNORABLE": { + Severity: "HEALTH_ERR", + Message: "TEST_UNIGNORABLE", + }, + }) + assert.False(t, canIgnoreHealthErrStatusInReconcile(cluster, "controller")) + + cluster = CreateTestClusterFromStatusDetails(map[string]cephv1.CephHealthMessage{ + "TEST_UNIGNORABLE": { + Severity: "HEALTH_ERR", + Message: "TEST_UNIGNORABLE", + }, + }) + assert.False(t, canIgnoreHealthErrStatusInReconcile(cluster, "controller")) +}