ceph: let the healthcheck set the status phase

Instead of setting the Phase of the CR once the reconcile is done, let's
actually set in from the healthcheck so it is more accurate.

Closes: https://github.com/rook/rook/issues/5249
Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2020-07-22 17:25:12 +02:00
1 parent 89f815bf9b
commit 8729a206f3
10 files changed
+107 -22

No files matched your search

-1
View File
@@ -230,7 +230,6 @@ const (
ConditionFailure ConditionType = "Failure"
ConditionUpgrading ConditionType = "Upgrading"
ConditionDeleting ConditionType = "Deleting"
ConditionHealthy ConditionType = "Healthy"
// DefaultFailureDomain for PoolSpec
DefaultFailureDomain = "host"
)
+40 -6
View File
@@ -26,8 +26,10 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/config"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -43,21 +45,23 @@ type cephStatusChecker struct {
clusterInfo *cephclient.ClusterInfo
interval time.Duration
client client.Client
isExternal bool
}
// newCephStatusChecker creates a new HealthChecker object
func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, healthCheck cephv1.CephClusterHealthCheckSpec) *cephStatusChecker {
func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, clusterSpec *cephv1.ClusterSpec) *cephStatusChecker {
c := &cephStatusChecker{
context: context,
clusterInfo: clusterInfo,
interval: defaultStatusCheckInterval,
client: context.Client,
isExternal: clusterSpec.External.Enable,
}
// allow overriding the check interval with an env var on the operator
// Keep the existing behavior
var checkInterval string
checkIntervalCRSetting := healthCheck.DaemonHealth.Status.Interval
checkIntervalCRSetting := clusterSpec.HealthCheck.DaemonHealth.Status.Interval
checkIntervalEnv := os.Getenv("ROOK_CEPH_STATUS_CHECK_INTERVAL")
if checkIntervalEnv != "" {
checkInterval = checkIntervalEnv
@@ -106,20 +110,22 @@ func (c *cephStatusChecker) checkStatus() {
status, err = cephclient.StatusWithUser(c.context, c.clusterInfo)
if err != nil {
logger.Errorf("failed to get ceph status. %v", err)
if err := c.updateCephStatus(cephStatusOnError(err.Error())); err != nil {
condition, reason, message := c.conditionMessageReason(cephv1.ConditionFailure)
if err := c.updateCephStatus(cephStatusOnError(err.Error()), condition, reason, message); err != nil {
logger.Errorf("failed to query cluster status in namespace %q. %v", c.clusterInfo.Namespace, err)
}
return
}
logger.Debugf("cluster status: %+v", status)
if err := c.updateCephStatus(&status); err != nil {
condition, reason, message := c.conditionMessageReason(cephv1.ConditionReady)
if err := c.updateCephStatus(&status, condition, reason, message); err != nil {
logger.Errorf("failed to query cluster status in namespace %q. %v", c.clusterInfo.Namespace, err)
}
}
// updateStatus updates an object with a given status
func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) error {
func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus, condition cephv1.ConditionType, reason, message string) error {
clusterName := c.clusterInfo.NamespacedName()
cephCluster := &cephv1.CephCluster{}
err := c.client.Get(context.TODO(), clusterName, cephCluster)
@@ -131,12 +137,17 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) erro
return errors.Wrapf(err, "failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status)
}
// Update with Ceph Status
cephCluster.Status.CephStatus = toCustomResourceStatus(cephCluster.Status, status)
cephCluster.Status.Phase = condition
if err := opcontroller.UpdateStatus(c.client, cephCluster); err != nil {
return errors.Wrapf(err, "failed to update cluster %q status", clusterName.Namespace)
}
logger.Debugf("ceph cluster %q status updated to %+v", clusterName.Namespace, status)
// Update condition
config.ConditionExport(c.context, c.clusterInfo.NamespacedName(), condition, v1.ConditionTrue, reason, message)
logger.Debugf("ceph cluster %q status and condition updated to %+v, %v, %s, %s", clusterName.Namespace, status, v1.ConditionTrue, reason, message)
return nil
}
@@ -211,3 +222,26 @@ func cephStatusOnError(errorMessage string) *cephclient.CephStatus {
},
}
}
func (c *cephStatusChecker) conditionMessageReason(condition cephv1.ConditionType) (cephv1.ConditionType, string, string) {
var reason, message string
switch condition {
case cephv1.ConditionFailure:
reason = "ClusterFailure"
message = "Failed to configure ceph cluster"
if c.isExternal {
message = "Failed to configure external ceph cluster"
}
case cephv1.ConditionReady:
reason = "ClusterCreated"
message = "Cluster created successfully"
if c.isExternal {
condition = cephv1.ConditionConnected
reason = "ClusterConnected"
message = "Cluster connected successfully"
}
}
return condition, reason, message
}
+56 -4
View File
@@ -27,6 +27,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/stretchr/testify/assert"
controllerclient "sigs.k8s.io/controller-runtime/pkg/client"
)
func TestCephStatus(t *testing.T) {
@@ -100,21 +101,72 @@ func TestNewCephStatusChecker(t *testing.T) {
type args struct {
context *clusterd.Context
healthCheck cephv1.CephClusterHealthCheckSpec
clusterInfo *cephclient.ClusterInfo
clusterSpec *cephv1.ClusterSpec
}
tests := []struct {
name string
args args
want *cephStatusChecker
}{
{"default-interval", args{c, cephv1.CephClusterHealthCheckSpec{}}, &cephStatusChecker{c, clusterInfo, defaultStatusCheckInterval, c.Client}},
{"default-interval", args{c, cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client}},
{"default-interval", args{c, clusterInfo, &cephv1.ClusterSpec{}}, &cephStatusChecker{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}},
{"10s-interval", args{c, clusterInfo, &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client, false}},
{"10s-interval-external", args{c, clusterInfo, &cephv1.ClusterSpec{External: cephv1.ExternalSpec{Enable: true}, HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client, true}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := newCephStatusChecker(tt.args.context, clusterInfo, tt.args.healthCheck); !reflect.DeepEqual(got, tt.want) {
if got := newCephStatusChecker(tt.args.context, tt.args.clusterInfo, tt.args.clusterSpec); !reflect.DeepEqual(got, tt.want) {
t.Errorf("newCephStatusChecker() = %v, want %v", got, tt.want)
}
})
}
}
func Test_cephStatusChecker_conditionMessageReason(t *testing.T) {
c := &clusterd.Context{}
clusterInfo := client.AdminClusterInfo("ns")
type fields struct {
context *clusterd.Context
clusterInfo *cephclient.ClusterInfo
interval time.Duration
client controllerclient.Client
isExternal bool
}
type args struct {
condition cephv1.ConditionType
}
tests := []struct {
name string
fields fields
args args
want cephv1.ConditionType
want1 string
want2 string
}{
{"failure-converged", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}, args{cephv1.ConditionFailure}, cephv1.ConditionFailure, "ClusterFailure", "Failed to configure ceph cluster"},
{"failure-external", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, true}, args{cephv1.ConditionFailure}, cephv1.ConditionFailure, "ClusterFailure", "Failed to configure external ceph cluster"},
{"success-converged", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}, args{cephv1.ConditionReady}, cephv1.ConditionReady, "ClusterCreated", "Cluster created successfully"},
{"success-external", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, true}, args{cephv1.ConditionReady}, cephv1.ConditionConnected, "ClusterConnected", "Cluster connected successfully"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &cephStatusChecker{
context: tt.fields.context,
clusterInfo: tt.fields.clusterInfo,
interval: tt.fields.interval,
client: tt.fields.client,
isExternal: tt.fields.isExternal,
}
got, got1, got2 := c.conditionMessageReason(tt.args.condition)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("cephStatusChecker.conditionMessageReason() got = %v, want %v", got, tt.want)
}
if got1 != tt.want1 {
t.Errorf("cephStatusChecker.conditionMessageReason() got1 = %v, want %v", got1, tt.want1)
}
if got2 != tt.want2 {
t.Errorf("cephStatusChecker.conditionMessageReason() got2 = %v, want %v", got2, tt.want2)
}
})
}
}
@@ -139,9 +139,6 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
}
}
// Everything went well so let's update the CR's status to "connected"
config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnected, v1.ConditionTrue, "ClusterConnected", "Cluster connected successfully")
// Mark initialization has done
cluster.initCompleted = true
+1 -1
View File
@@ -120,7 +120,7 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *
}
case "status":
cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec.HealthCheck)
cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec)
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan)
}
+3 -3
View File
@@ -279,8 +279,8 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
return r.setFailedStatus(request.NamespacedName, "failed to create object store deployments", err)
}
// Set Ready status, we are done reconciling
updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore))
// Set Progressing status, we are done reconciling, the health check go routine will update the status
updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore))
// Return and do not requeue
logger.Debug("done reconciling")
@@ -479,7 +479,7 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort))
}
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck)
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.cephClusterSpec.External.Enable)
logger.Info("starting rgw healthcheck")
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
}
+1 -1
View File
@@ -328,7 +328,7 @@ func TestCephObjectStoreController(t *testing.T) {
assert.False(t, res.Requeue)
err = r.client.Get(context.TODO(), req.NamespacedName, objectStore)
assert.NoError(t, err)
assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore)
assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore)
assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore)
assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph:80", objectStore.Status.Info["endpoint"], objectStore)
logger.Info("PHASE 3 DONE")
+4 -2
View File
@@ -48,10 +48,11 @@ type bucketChecker struct {
client client.Client
namespacedName types.NamespacedName
healthCheckSpec *cephv1.BucketHealthCheckSpec
isExternal bool
}
// newbucketChecker creates a new HealthChecker object
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker {
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec, isExternal bool) *bucketChecker {
c := &bucketChecker{
context: context,
objContext: objContext,
@@ -61,6 +62,7 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP,
namespacedName: namespacedName,
client: client,
healthCheckSpec: healthCheckSpec,
isExternal: isExternal,
}
// allow overriding the check interval
@@ -169,7 +171,7 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
logger.Debug("successfully checked object store endpoint for object store %q", c.namespacedName.Name)
// Update the EndpointStatus in the CR to reflect the healthyness
updateStatusBucket(c.client, c.namespacedName, cephv1.ConditionHealthy, "")
updateStatusBucket(c.client, c.namespacedName, cephv1.ConditionConnected, "")
return nil
}
+1
View File
@@ -71,6 +71,7 @@ func updateStatusBucket(client client.Client, name types.NamespacedName, phase c
}
objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, phase)
objectStore.Status.Phase = phase
if err := opcontroller.UpdateStatus(client, objectStore); err != nil {
logger.Errorf("failed to set object store %q status to %v. %v", name, phase, err)
return
+1 -1
View File
@@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
// Check object store status
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(storeName, metav1.GetOptions{})
assert.Nil(s.T(), err)
assert.Equal(s.T(), cephv1.ConditionHealthy, objectStore.Status.BucketStatus.Health)
assert.Equal(s.T(), cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health)
// Info field has the endpoint in it
assert.NotEmpty(s.T(), objectStore.Status.Info)
assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"])