diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 3ace76e1b..c808e7ca9 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -230,7 +230,6 @@ const ( ConditionFailure ConditionType = "Failure" ConditionUpgrading ConditionType = "Upgrading" ConditionDeleting ConditionType = "Deleting" - ConditionHealthy ConditionType = "Healthy" // DefaultFailureDomain for PoolSpec DefaultFailureDomain = "host" ) diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index 584ccb9a5..16e33766b 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -26,8 +26,10 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" cephver "github.com/rook/rook/pkg/operator/ceph/version" + v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -43,21 +45,23 @@ type cephStatusChecker struct { clusterInfo *cephclient.ClusterInfo interval time.Duration client client.Client + isExternal bool } // newCephStatusChecker creates a new HealthChecker object -func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, healthCheck cephv1.CephClusterHealthCheckSpec) *cephStatusChecker { +func newCephStatusChecker(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, clusterSpec *cephv1.ClusterSpec) *cephStatusChecker { c := &cephStatusChecker{ context: context, clusterInfo: clusterInfo, interval: defaultStatusCheckInterval, client: context.Client, + isExternal: clusterSpec.External.Enable, } // allow overriding the check interval with an env var on the operator // Keep the existing behavior var checkInterval string - checkIntervalCRSetting := healthCheck.DaemonHealth.Status.Interval + checkIntervalCRSetting := clusterSpec.HealthCheck.DaemonHealth.Status.Interval checkIntervalEnv := os.Getenv("ROOK_CEPH_STATUS_CHECK_INTERVAL") if checkIntervalEnv != "" { checkInterval = checkIntervalEnv @@ -106,20 +110,22 @@ func (c *cephStatusChecker) checkStatus() { status, err = cephclient.StatusWithUser(c.context, c.clusterInfo) if err != nil { logger.Errorf("failed to get ceph status. %v", err) - if err := c.updateCephStatus(cephStatusOnError(err.Error())); err != nil { + condition, reason, message := c.conditionMessageReason(cephv1.ConditionFailure) + if err := c.updateCephStatus(cephStatusOnError(err.Error()), condition, reason, message); err != nil { logger.Errorf("failed to query cluster status in namespace %q. %v", c.clusterInfo.Namespace, err) } return } logger.Debugf("cluster status: %+v", status) - if err := c.updateCephStatus(&status); err != nil { + condition, reason, message := c.conditionMessageReason(cephv1.ConditionReady) + if err := c.updateCephStatus(&status, condition, reason, message); err != nil { logger.Errorf("failed to query cluster status in namespace %q. %v", c.clusterInfo.Namespace, err) } } // updateStatus updates an object with a given status -func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) error { +func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus, condition cephv1.ConditionType, reason, message string) error { clusterName := c.clusterInfo.NamespacedName() cephCluster := &cephv1.CephCluster{} err := c.client.Get(context.TODO(), clusterName, cephCluster) @@ -131,12 +137,17 @@ func (c *cephStatusChecker) updateCephStatus(status *cephclient.CephStatus) erro return errors.Wrapf(err, "failed to retrieve ceph cluster %q in namespace %q to update status to %+v", clusterName.Name, clusterName.Namespace, status) } + // Update with Ceph Status cephCluster.Status.CephStatus = toCustomResourceStatus(cephCluster.Status, status) + cephCluster.Status.Phase = condition if err := opcontroller.UpdateStatus(c.client, cephCluster); err != nil { return errors.Wrapf(err, "failed to update cluster %q status", clusterName.Namespace) } - logger.Debugf("ceph cluster %q status updated to %+v", clusterName.Namespace, status) + // Update condition + config.ConditionExport(c.context, c.clusterInfo.NamespacedName(), condition, v1.ConditionTrue, reason, message) + + logger.Debugf("ceph cluster %q status and condition updated to %+v, %v, %s, %s", clusterName.Namespace, status, v1.ConditionTrue, reason, message) return nil } @@ -211,3 +222,26 @@ func cephStatusOnError(errorMessage string) *cephclient.CephStatus { }, } } + +func (c *cephStatusChecker) conditionMessageReason(condition cephv1.ConditionType) (cephv1.ConditionType, string, string) { + var reason, message string + + switch condition { + case cephv1.ConditionFailure: + reason = "ClusterFailure" + message = "Failed to configure ceph cluster" + if c.isExternal { + message = "Failed to configure external ceph cluster" + } + case cephv1.ConditionReady: + reason = "ClusterCreated" + message = "Cluster created successfully" + if c.isExternal { + condition = cephv1.ConditionConnected + reason = "ClusterConnected" + message = "Cluster connected successfully" + } + } + + return condition, reason, message +} diff --git a/pkg/operator/ceph/cluster/cephstatus_test.go b/pkg/operator/ceph/cluster/cephstatus_test.go index f42e7c64d..ff3d72873 100644 --- a/pkg/operator/ceph/cluster/cephstatus_test.go +++ b/pkg/operator/ceph/cluster/cephstatus_test.go @@ -27,6 +27,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/stretchr/testify/assert" + controllerclient "sigs.k8s.io/controller-runtime/pkg/client" ) func TestCephStatus(t *testing.T) { @@ -100,21 +101,72 @@ func TestNewCephStatusChecker(t *testing.T) { type args struct { context *clusterd.Context - healthCheck cephv1.CephClusterHealthCheckSpec + clusterInfo *cephclient.ClusterInfo + clusterSpec *cephv1.ClusterSpec } tests := []struct { name string args args want *cephStatusChecker }{ - {"default-interval", args{c, cephv1.CephClusterHealthCheckSpec{}}, &cephStatusChecker{c, clusterInfo, defaultStatusCheckInterval, c.Client}}, - {"default-interval", args{c, cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client}}, + {"default-interval", args{c, clusterInfo, &cephv1.ClusterSpec{}}, &cephStatusChecker{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}}, + {"10s-interval", args{c, clusterInfo, &cephv1.ClusterSpec{HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client, false}}, + {"10s-interval-external", args{c, clusterInfo, &cephv1.ClusterSpec{External: cephv1.ExternalSpec{Enable: true}, HealthCheck: cephv1.CephClusterHealthCheckSpec{DaemonHealth: cephv1.DaemonHealthSpec{Status: cephv1.HealthCheckSpec{Interval: "10s"}}}}}, &cephStatusChecker{c, clusterInfo, time10s, c.Client, true}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := newCephStatusChecker(tt.args.context, clusterInfo, tt.args.healthCheck); !reflect.DeepEqual(got, tt.want) { + if got := newCephStatusChecker(tt.args.context, tt.args.clusterInfo, tt.args.clusterSpec); !reflect.DeepEqual(got, tt.want) { t.Errorf("newCephStatusChecker() = %v, want %v", got, tt.want) } }) } } + +func Test_cephStatusChecker_conditionMessageReason(t *testing.T) { + c := &clusterd.Context{} + clusterInfo := client.AdminClusterInfo("ns") + type fields struct { + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + interval time.Duration + client controllerclient.Client + isExternal bool + } + type args struct { + condition cephv1.ConditionType + } + tests := []struct { + name string + fields fields + args args + want cephv1.ConditionType + want1 string + want2 string + }{ + {"failure-converged", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}, args{cephv1.ConditionFailure}, cephv1.ConditionFailure, "ClusterFailure", "Failed to configure ceph cluster"}, + {"failure-external", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, true}, args{cephv1.ConditionFailure}, cephv1.ConditionFailure, "ClusterFailure", "Failed to configure external ceph cluster"}, + {"success-converged", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, false}, args{cephv1.ConditionReady}, cephv1.ConditionReady, "ClusterCreated", "Cluster created successfully"}, + {"success-external", fields{c, clusterInfo, defaultStatusCheckInterval, c.Client, true}, args{cephv1.ConditionReady}, cephv1.ConditionConnected, "ClusterConnected", "Cluster connected successfully"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &cephStatusChecker{ + context: tt.fields.context, + clusterInfo: tt.fields.clusterInfo, + interval: tt.fields.interval, + client: tt.fields.client, + isExternal: tt.fields.isExternal, + } + got, got1, got2 := c.conditionMessageReason(tt.args.condition) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("cephStatusChecker.conditionMessageReason() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("cephStatusChecker.conditionMessageReason() got1 = %v, want %v", got1, tt.want1) + } + if got2 != tt.want2 { + t.Errorf("cephStatusChecker.conditionMessageReason() got2 = %v, want %v", got2, tt.want2) + } + }) + } +} diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 31bfea95e..0313f5c56 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -139,9 +139,6 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error } } - // Everything went well so let's update the CR's status to "connected" - config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnected, v1.ConditionTrue, "ClusterConnected", "Cluster connected successfully") - // Mark initialization has done cluster.initCompleted = true diff --git a/pkg/operator/ceph/cluster/monitoring.go b/pkg/operator/ceph/cluster/monitoring.go index 106238b60..1dbe6da3a 100644 --- a/pkg/operator/ceph/cluster/monitoring.go +++ b/pkg/operator/ceph/cluster/monitoring.go @@ -120,7 +120,7 @@ func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo * } case "status": - cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec.HealthCheck) + cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec) logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace) go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan) } diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index da47ca1b3..8c21991b4 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -279,8 +279,8 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci return r.setFailedStatus(request.NamespacedName, "failed to create object store deployments", err) } - // Set Ready status, we are done reconciling - updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore)) + // Set Progressing status, we are done reconciling, the health check go routine will update the status + updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore)) // Return and do not requeue logger.Debug("done reconciling") @@ -479,7 +479,7 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec port = strconv.Itoa(int(objectstore.Spec.Gateway.SecurePort)) } - rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck) + rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.cephClusterSpec.External.Enable) logger.Info("starting rgw healthcheck") go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan) } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index 0c2a3bba5..bc50a46ad 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -328,7 +328,7 @@ func TestCephObjectStoreController(t *testing.T) { assert.False(t, res.Requeue) err = r.client.Get(context.TODO(), req.NamespacedName, objectStore) assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore) + assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore) assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore) assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph:80", objectStore.Status.Info["endpoint"], objectStore) logger.Info("PHASE 3 DONE") diff --git a/pkg/operator/ceph/object/health.go b/pkg/operator/ceph/object/health.go index 921d5013a..16fcb9401 100644 --- a/pkg/operator/ceph/object/health.go +++ b/pkg/operator/ceph/object/health.go @@ -48,10 +48,11 @@ type bucketChecker struct { client client.Client namespacedName types.NamespacedName healthCheckSpec *cephv1.BucketHealthCheckSpec + isExternal bool } // newbucketChecker creates a new HealthChecker object -func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec) *bucketChecker { +func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, port string, client client.Client, namespacedName types.NamespacedName, healthCheckSpec *cephv1.BucketHealthCheckSpec, isExternal bool) *bucketChecker { c := &bucketChecker{ context: context, objContext: objContext, @@ -61,6 +62,7 @@ func newBucketChecker(context *clusterd.Context, objContext *Context, serviceIP, namespacedName: namespacedName, client: client, healthCheckSpec: healthCheckSpec, + isExternal: isExternal, } // allow overriding the check interval @@ -169,7 +171,7 @@ func (c *bucketChecker) checkObjectStoreHealth() error { logger.Debug("successfully checked object store endpoint for object store %q", c.namespacedName.Name) // Update the EndpointStatus in the CR to reflect the healthyness - updateStatusBucket(c.client, c.namespacedName, cephv1.ConditionHealthy, "") + updateStatusBucket(c.client, c.namespacedName, cephv1.ConditionConnected, "") return nil } diff --git a/pkg/operator/ceph/object/status.go b/pkg/operator/ceph/object/status.go index 97656e67b..9d77e6f5a 100644 --- a/pkg/operator/ceph/object/status.go +++ b/pkg/operator/ceph/object/status.go @@ -71,6 +71,7 @@ func updateStatusBucket(client client.Client, name types.NamespacedName, phase c } objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, phase) + objectStore.Status.Phase = phase if err := opcontroller.UpdateStatus(client, objectStore); err != nil { logger.Errorf("failed to set object store %q status to %v. %v", name, phase, err) return diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 3a1ed75c6..6fe84e74e 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -86,7 +86,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite // Check object store status objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(storeName, metav1.GetOptions{}) assert.Nil(s.T(), err) - assert.Equal(s.T(), cephv1.ConditionHealthy, objectStore.Status.BucketStatus.Health) + assert.Equal(s.T(), cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health) // Info field has the endpoint in it assert.NotEmpty(s.T(), objectStore.Status.Info) assert.NotEmpty(s.T(), objectStore.Status.Info["endpoint"])