ceph: let the healthcheck set the status phase
Instead of setting the Phase of the CR once the reconcile is done, let's actually set in from the healthcheck so it is more accurate. Closes: https://github.com/rook/rook/issues/5249 Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
1 parent
89f815bf9b
commit
8729a206f3
10 files changed
+107
-22
No files matched your search
@@ -230,7 +230,6 @@ const (
|
||||
ConditionFailure ConditionType = "Failure"
|
||||
ConditionUpgrading ConditionType = "Upgrading"
|
||||
ConditionDeleting ConditionType = "Deleting"
|
||||
ConditionHealthy ConditionType = "Healthy"
|
||||
// DefaultFailureDomain for PoolSpec
|
||||
DefaultFailureDomain = "host"
|
||||
)
|
||||
|
||||
@@ -26,8 +26,10 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
@@ -43,21 +45,23 @@ type cephStatusChecker struct {
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
interval time.Duration
|
||||
client client.Client
|
||||
isExternal bool
|
||||
}
|
||||
|
||||
// newCephStatusChecker creates a new HealthChecker object
|
||||
func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, healthCheck cephv1.CephClusterHealthCheckSpec) *cephStatusChecker {
|
||||
func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, clusterSpec *cephv1.ClusterSpec) *cephStatusChecker {
|
||||
c := &cephStatusChecker{
|
||||
context: context,
|
||||
clusterInfo: clusterInfo,
|
||||
interval: defaultStatusCheckInterval,
|
||||
client: context.Client,
|
||||
isExternal: clusterSpec.External.Enable,
|
||||
}
|
||||
|
||||
// allow overriding the check interval with an env var on the operator
|
||||
// Keep the existing behavior
|
||||
var checkInterval string
|
||||
checkIntervalCRSetting := healthCheck.DaemonHealth.Status.Interval
|
||||
checkIntervalCRSetting := clusterSpec.HealthCheck.DaemonHealth.Status.Interval
|
||||
checkIntervalEnv := os.Getenv("ROOK_CEPH_STATUS_CHECK_INTERVAL")
|
||||
if checkIntervalEnv != "" {
|
||||
checkInterval = checkIntervalEnv
|
||||
@@ -106,20 +110,22 @@ func (c *cephStatusChecker) checkStatus() {
|
||||
status, err = cephclient.StatusWithUser(c.context, c.clusterInfo)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get ceph status. %v", err)
|
||||
if err := c.updateCephStatus(cephStatusOnError(err.Error())); err != nil {
|
||||
condition, reason, message := c.conditionMessageReason(cephv1.ConditionFailure)
|
||||
if err := c.updateCephStatus(cephStatusOnError(err.Error()), condition, reason, message); err != nil {
|
||||
logger.Errorf("failed to query cluster status in namespace %q. %v", c.clusterInfo.Namespace, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debugf("cluster status: %+v", status)
|
||||
if err := c.updateCephStatus(&status); err != nil {
|
||||
condition, reason, message := c.conditionMessageReason(cephv1.ConditionReady)
|
||||
if err := c.updateCephStatus(&status, condition, reason, message); err != nil {
|
||||
logger.Errorf("failed to query cluster status in namespace %q. %v", c.clusterInfo.Namespace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// updateStatus updates an object with a given status
|
||||
func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) error {
|
||||
func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus, condition cephv1.ConditionType, reason, message string) error {
|
||||
clusterName := c.clusterInfo.NamespacedName()
|
||||
cephCluster := &cephv1.CephCluster{}
|
||||
err := c.client.Get(context.TODO(), clusterName, cephCluster)
|
||||
@@ -131,12 +137,17 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) erro
|
||||
return errors.Wrapf(err, "failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status)
|
||||
}
|
||||
|
||||
// Update with Ceph Status
|
||||
cephCluster.Status.CephStatus = toCustomResourceStatus(cephCluster.Status, status)
|
||||
cephCluster.Status.Phase = condition
|
||||
if err := opcontroller.UpdateStatus(c.client, cephCluster); err != nil {
|
||||
return errors.Wrapf(err, "failed to update cluster %q status", clusterName.Namespace)
|
||||
}
|
||||
|
||||
logger.Debugf("ceph cluster %q status updated to %+v", clusterName.Namespace, status)
|
||||
// Update condition
|
||||
config.ConditionExport(c.context, c.clusterInfo.NamespacedName(), condition, v1.ConditionTrue, reason, message)
|
||||
|
||||
logger.Debugf("ceph cluster %q status and condition updated to %+v, %v, %s, %s", clusterName.Namespace, status, v1.ConditionTrue, reason, message)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -211,3 +222,26 @@ func cephStatusOnError(errorMessage string) *cephclient.CephStatus {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cephStatusChecker) conditionMessageReason(condition cephv1.ConditionType) (cephv1.ConditionType, string, string) {
|
||||
var reason, message string
|
||||
|
||||
switch condition {
|
||||
case cephv1.ConditionFailure:
|
||||
reason = "ClusterFailure"
|
||||
message = "Failed to configure ceph cluster"
|
||||
if c.isExternal {
|
||||
message = "Failed to configure external ceph cluster"
|
||||
}
|
||||
case cephv1.ConditionReady:
|
||||
reason = "ClusterCreated"
|
||||
message = "Cluster created successfully"
|
||||
if c.isExternal {
|
||||
condition = cephv1.ConditionConnected
|
||||
reason = "ClusterConnected"
|
||||
message = "Cluster connected successfully"
|
||||
}
|
||||
}
|
||||
|
||||
return condition, reason, message
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
controllerclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
func TestCephStatus(t *testing.T) {
|
||||
@@ -100,21 +101,72 @@ func TestNewCephStatusChecker(t *testing.T) {
|
||||
|
||||
type args struct {
|
||||
context *clusterd.Context
|
||||
healthCheck cephv1.CephClusterHealthCheckSpec
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want *cephStatusChecker
|
||||
}{
|
||||
{"default-interval", args{c, cephv1.CephClusterHealthCheckSpec{}}, &cephStatusChecker{c, clusterInfo, defaultStatusCheckInterval, c.Client}},
|
||||
{"default-interval", args{c, cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client}},
|
||||
{"default-interval", args{c, clusterInfo, &cephv1.ClusterSpec{}}, &cephStatusChecker{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}},
|
||||
{"10s-interval", args{c, clusterInfo, &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client, false}},
|
||||
{"10s-interval-external", args{c, clusterInfo, &cephv1.ClusterSpec{External: cephv1.ExternalSpec{Enable: true}, HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client, true}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := newCephStatusChecker(tt.args.context, clusterInfo, tt.args.healthCheck); !reflect.DeepEqual(got, tt.want) {
|
||||
if got := newCephStatusChecker(tt.args.context, tt.args.clusterInfo, tt.args.clusterSpec); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("newCephStatusChecker() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_cephStatusChecker_conditionMessageReason(t *testing.T) {
|
||||
c := &clusterd.Context{}
|
||||
clusterInfo := client.AdminClusterInfo("ns")
|
||||
type fields struct {
|
||||
context *clusterd.Context
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
interval time.Duration
|
||||
client controllerclient.Client
|
||||
isExternal bool
|
||||
}
|
||||
type args struct {
|
||||
condition cephv1.ConditionType
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
args args
|
||||
want cephv1.ConditionType
|
||||
want1 string
|
||||
want2 string
|
||||
}{
|
||||
{"failure-converged", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}, args{cephv1.ConditionFailure}, cephv1.ConditionFailure, "ClusterFailure", "Failed to configure ceph cluster"},
|
||||
{"failure-external", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, true}, args{cephv1.ConditionFailure}, cephv1.ConditionFailure, "ClusterFailure", "Failed to configure external ceph cluster"},
|
||||
{"success-converged", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}, args{cephv1.ConditionReady}, cephv1.ConditionReady, "ClusterCreated", "Cluster created successfully"},
|
||||
{"success-external", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, true}, args{cephv1.ConditionReady}, cephv1.ConditionConnected, "ClusterConnected", "Cluster connected successfully"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &cephStatusChecker{
|
||||
context: tt.fields.context,
|
||||
clusterInfo: tt.fields.clusterInfo,
|
||||
interval: tt.fields.interval,
|
||||
client: tt.fields.client,
|
||||
isExternal: tt.fields.isExternal,
|
||||
}
|
||||
got, got1, got2 := c.conditionMessageReason(tt.args.condition)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("cephStatusChecker.conditionMessageReason() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
if got1 != tt.want1 {
|
||||
t.Errorf("cephStatusChecker.conditionMessageReason() got1 = %v, want %v", got1, tt.want1)
|
||||
}
|
||||
if got2 != tt.want2 {
|
||||
t.Errorf("cephStatusChecker.conditionMessageReason() got2 = %v, want %v", got2, tt.want2)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -139,9 +139,6 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
}
|
||||
}
|
||||
|
||||
// Everything went well so let's update the CR's status to "connected"
|
||||
config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnected, v1.ConditionTrue, "ClusterConnected", "Cluster connected successfully")
|
||||
|
||||
// Mark initialization has done
|
||||
cluster.initCompleted = true
|
||||
|
||||
|
||||
@@ -120,7 +120,7 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *
|
||||
}
|
||||
|
||||
case "status":
|
||||
cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec.HealthCheck)
|
||||
cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec)
|
||||
logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
|
||||
go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan)
|
||||
}
|
||||
|
||||
@@ -279,8 +279,8 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
return r.setFailedStatus(request.NamespacedName, "failed to create object store deployments", err)
|
||||
}
|
||||
|
||||
// Set Ready status, we are done reconciling
|
||||
updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore))
|
||||
// Set Progressing status, we are done reconciling, the health check go routine will update the status
|
||||
updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore))
|
||||
|
||||
// Return and do not requeue
|
||||
logger.Debug("done reconciling")
|
||||
@@ -479,7 +479,7 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
|
||||
port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort))
|
||||
}
|
||||
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck)
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.cephClusterSpec.External.Enable)
|
||||
logger.Info("starting rgw healthcheck")
|
||||
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
|
||||
}
|
||||
|
||||
@@ -328,7 +328,7 @@ func TestCephObjectStoreController(t *testing.T) {
|
||||
assert.False(t, res.Requeue)
|
||||
err = r.client.Get(context.TODO(), req.NamespacedName, objectStore)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore)
|
||||
assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore)
|
||||
assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore)
|
||||
assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph:80", objectStore.Status.Info["endpoint"], objectStore)
|
||||
logger.Info("PHASE 3 DONE")
|
||||
|
||||
@@ -48,10 +48,11 @@ type bucketChecker struct {
|
||||
client client.Client
|
||||
namespacedName types.NamespacedName
|
||||
healthCheckSpec *cephv1.BucketHealthCheckSpec
|
||||
isExternal bool
|
||||
}
|
||||
|
||||
// newbucketChecker creates a new HealthChecker object
|
||||
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker {
|
||||
func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec, isExternal bool) *bucketChecker {
|
||||
c := &bucketChecker{
|
||||
context: context,
|
||||
objContext: objContext,
|
||||
@@ -61,6 +62,7 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP,
|
||||
namespacedName: namespacedName,
|
||||
client: client,
|
||||
healthCheckSpec: healthCheckSpec,
|
||||
isExternal: isExternal,
|
||||
}
|
||||
|
||||
// allow overriding the check interval
|
||||
@@ -169,7 +171,7 @@ func (c *bucketChecker) checkObjectStoreHealth() error {
|
||||
logger.Debug("successfully checked object store endpoint for object store %q", c.namespacedName.Name)
|
||||
|
||||
// Update the EndpointStatus in the CR to reflect the healthyness
|
||||
updateStatusBucket(c.client, c.namespacedName, cephv1.ConditionHealthy, "")
|
||||
updateStatusBucket(c.client, c.namespacedName, cephv1.ConditionConnected, "")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -71,6 +71,7 @@ func updateStatusBucket(client client.Client, name types.NamespacedName, phase c
|
||||
}
|
||||
|
||||
objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, phase)
|
||||
objectStore.Status.Phase = phase
|
||||
if err := opcontroller.UpdateStatus(client, objectStore); err != nil {
|
||||
logger.Errorf("failed to set object store %q status to %v. %v", name, phase, err)
|
||||
return
|
||||
|
||||
@@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite
|
||||
// Check object store status
|
||||
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(storeName, metav1.GetOptions{})
|
||||
assert.Nil(s.T(), err)
|
||||
assert.Equal(s.T(), cephv1.ConditionHealthy, objectStore.Status.BucketStatus.Health)
|
||||
assert.Equal(s.T(), cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health)
|
||||
// Info field has the endpoint in it
|
||||
assert.NotEmpty(s.T(), objectStore.Status.Info)
|
||||
assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"])
|
||||
|
||||
Reference in new issue
Block a user