From a7c0c7ee93743ec794a68502a78c746606f28a78 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Fri, 7 Oct 2022 09:20:37 -0600 Subject: [PATCH] object: remove health checker Remove the health checker for CephObjectStore. The liveness and readiness probes go through the same code paths in RGW as creating buckets without as much affect on the storage backend. Full discussion: https://github.com/rook/rook/issues/11031 Signed-off-by: Blaine Gardner --- .../Object-Storage/ceph-object-store-crd.md | 23 +- .../Object-Storage-RGW/object-storage.md | 14 +- deploy/charts/rook-ceph-cluster/values.yaml | 3 - .../charts/rook-ceph/templates/resources.yaml | 26 +- deploy/examples/crds.yaml | 26 +- deploy/examples/object-ec.yaml | 5 +- deploy/examples/object-openshift.yaml | 15 +- deploy/examples/object.yaml | 5 +- pkg/apis/ceph.rook.io/v1/types.go | 24 +- .../ceph.rook.io/v1/zz_generated.deepcopy.go | 84 ++--- pkg/operator/ceph/object/controller.go | 158 ++++------ pkg/operator/ceph/object/controller_test.go | 81 ++--- pkg/operator/ceph/object/dependents.go | 9 +- pkg/operator/ceph/object/health.go | 292 ------------------ pkg/operator/ceph/object/spec.go | 2 +- pkg/operator/ceph/object/status.go | 36 --- tests/framework/installer/ceph_manifests.go | 8 - tests/integration/ceph_base_object_test.go | 31 +- 18 files changed, 198 insertions(+), 644 deletions(-) delete mode 100644 pkg/operator/ceph/object/health.go diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 04e1c382d..163ec5e0d 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -162,9 +162,6 @@ Rook will be default monitor the state of the object store endpoints. The following CRD settings are available: * `healthCheck`: main object store health monitoring section - * `bucket`: Rook checks that the object store is usable regularly. This is explained in more - detail below. Use this config to disable or change the interval at which Rook verifies the - object store connectivity. * `startupProbe`: Disable, or override timing and threshold values of the object gateway startup probe. * `livenessProbe`: Disable, or override timing and threshold values of the object gateway liveness probe. * `readinessProbe`: Disable, or override timing and threshold values of the object gateway readiness probe. @@ -173,31 +170,21 @@ Here is a complete example: ```yaml healthCheck: - bucket: - disabled: false - interval: 60s startupProbe: disabled: false livenessProbe: disabled: false - periodSeconds: 5 - failureThreshold: 4 + periodSeconds: 30 + failureThreshold: 13 readinessProbe: disabled: false periodSeconds: 5 failureThreshold: 2 ``` -The endpoint health check procedure is the following: - -1. Create an S3 user -2. Create a bucket with that user -3. PUT the file in the object store -4. GET the file from the object store -5. Verify object consistency -6. Update CR health status check - -The Rook Ceph operator always keeps the bucket and the user for the health check, it just does a PUT and GET of an s3 object since creating a bucket is an expensive operation. +You can monitor the health of a CephObjectStore by monitoring the gateway deployments it creates. +The primary deployment created is named `rook-ceph-rgw--a` where `store-name` is the +name of the CephObjectStore (don't forget the `-a` at the end). ## Security settings diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index 55f491099..03d31cc61 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -48,10 +48,6 @@ spec: port: 80 # securePort: 443 instances: 1 - healthCheck: - bucket: - disabled: false - interval: 60s ``` After the `CephObjectStore` is created, the Rook operator will then create all the pools and other resources necessary to start the service. This may take a minute to complete. @@ -84,10 +80,6 @@ spec: port: 8080 externalRgwEndpoints: - ip: 192.168.39.182 - healthCheck: - bucket: - enabled: true - interval: 60s ``` You can use the existing `object-external.yaml` file. @@ -249,6 +241,12 @@ s5cmd --endpoint-url http://$AWS_HOST:$PORT cp s3://$BUCKET_NAME/rookObj /tmp/ro cat /tmp/rookObj-download ``` +## Monitoring health + +Rook configures health probes on the deployment created for CephObjectStore gateways. Refer to +[the CRD document](../../CRDs/Object-Storage/ceph-object-store-crd.md#health-settings) for +information about configuring the probes and monitoring the deployment status. + ## Access External to the Cluster Rook sets up the object storage so pods will have access internal to the cluster. If your applications are running outside the cluster, diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index fc8f8ce76..19b9f0082 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -528,9 +528,6 @@ cephObjectStores: # sslCertificateRef: instances: 1 priorityClassName: system-cluster-critical - healthCheck: - bucket: - interval: 60s storageClass: enabled: true name: ceph-bucket diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 9fd067c4b..73fff8679 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -12147,20 +12147,9 @@ spec: type: string type: object healthCheck: - description: The rgw Bucket healthchecks and liveness probe + description: The RGW health probes nullable: true properties: - bucket: - description: HealthCheckSpec represents the health check of an object store bucket - properties: - disabled: - type: boolean - interval: - description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds - type: string - timeout: - type: string - type: object livenessProbe: description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon properties: @@ -12704,19 +12693,6 @@ spec: status: description: ObjectStoreStatus represents the status of a Ceph Object Store resource properties: - bucketStatus: - description: BucketStatus represents the status of a bucket - properties: - details: - type: string - health: - description: ConditionType represent a resource's status - type: string - lastChanged: - type: string - lastChecked: - type: string - type: object conditions: items: description: Condition represents a status condition on any Rook-Ceph Custom Resource. diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 4353a4860..c9ea8f8cb 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -12139,20 +12139,9 @@ spec: type: string type: object healthCheck: - description: The rgw Bucket healthchecks and liveness probe + description: The RGW health probes nullable: true properties: - bucket: - description: HealthCheckSpec represents the health check of an object store bucket - properties: - disabled: - type: boolean - interval: - description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds - type: string - timeout: - type: string - type: object livenessProbe: description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon properties: @@ -12696,19 +12685,6 @@ spec: status: description: ObjectStoreStatus represents the status of a Ceph Object Store resource properties: - bucketStatus: - description: BucketStatus represents the status of a bucket - properties: - details: - type: string - health: - description: ConditionType represent a resource's status - type: string - lastChanged: - type: string - lastChecked: - type: string - type: object conditions: items: description: Condition represents a status condition on any Rook-Ceph Custom Resource. diff --git a/deploy/examples/object-ec.yaml b/deploy/examples/object-ec.yaml index 8dabb8fde..66743943f 100644 --- a/deploy/examples/object-ec.yaml +++ b/deploy/examples/object-ec.yaml @@ -81,12 +81,9 @@ spec: # memory: "1024Mi" # priorityClassName: my-priority-class #zone: - #name: zone-a + #name: zone-a # service endpoint healthcheck healthCheck: - bucket: - disabled: false - interval: 60s # Configure the pod probes for the rgw daemon livenessProbe: disabled: false diff --git a/deploy/examples/object-openshift.yaml b/deploy/examples/object-openshift.yaml index 893cdd1bb..41377c6a8 100644 --- a/deploy/examples/object-openshift.yaml +++ b/deploy/examples/object-openshift.yaml @@ -98,9 +98,6 @@ spec: # service.beta.openshift.io/serving-cert-secret-name: # service endpoint healthcheck healthCheck: - bucket: - disabled: false - interval: 60s # Configure the pod probes for the rgw daemon livenessProbe: disabled: false @@ -149,12 +146,12 @@ metadata: spec: port: targetPort: http -# if TLS is enabled, remove the above http port and uncomment the https port and tls settings -# port: -# targetPort: https -# tls: -# insecureEdgeTerminationPolicy: Redirect -# termination: reencrypt + # if TLS is enabled, remove the above http port and uncomment the https port and tls settings + # port: + # targetPort: https + # tls: + # insecureEdgeTerminationPolicy: Redirect + # termination: reencrypt to: kind: Service name: rook-ceph-rgw-my-store # The name of the RGW service is in the form 'rook-ceph-rgw-' diff --git a/deploy/examples/object.yaml b/deploy/examples/object.yaml index 2d4fa47e7..3311ab6ec 100644 --- a/deploy/examples/object.yaml +++ b/deploy/examples/object.yaml @@ -101,12 +101,9 @@ spec: # memory: "1024Mi" priorityClassName: system-cluster-critical #zone: - #name: zone-a + #name: zone-a # service endpoint healthcheck healthCheck: - bucket: - disabled: false - interval: 60s # Configure the pod probes for the rgw daemon startupProbe: disabled: false diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index c60c7adb0..3c818e79c 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1335,10 +1335,10 @@ type ObjectStoreSpec struct { // +nullable Zone ZoneSpec `json:"zone,omitempty"` - // The rgw Bucket healthchecks and liveness probe + // The RGW health probes // +optional // +nullable - HealthCheck BucketHealthCheckSpec `json:"healthCheck,omitempty"` + HealthCheck ObjectHealthCheckSpec `json:"healthCheck,omitempty"` // Security represents security settings // +optional @@ -1346,10 +1346,8 @@ type ObjectStoreSpec struct { Security *ObjectStoreSecuritySpec `json:"security,omitempty"` } -// BucketHealthCheckSpec represents the health check of an object store -type BucketHealthCheckSpec struct { - // +optional - Bucket HealthCheckSpec `json:"bucket,omitempty"` +// ObjectHealthCheckSpec represents the health check of an object store +type ObjectHealthCheckSpec struct { // +optional LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` // +optional @@ -1455,8 +1453,6 @@ type ObjectStoreStatus struct { // +optional Message string `json:"message,omitempty"` // +optional - BucketStatus *BucketStatus `json:"bucketStatus,omitempty"` - // +optional // +nullable Info map[string]string `json:"info,omitempty"` Conditions []Condition `json:"conditions,omitempty"` @@ -1465,18 +1461,6 @@ type ObjectStoreStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` } -// BucketStatus represents the status of a bucket -type BucketStatus struct { - // +optional - Health ConditionType `json:"health,omitempty"` - // +optional - Details string `json:"details,omitempty"` - // +optional - LastChecked string `json:"lastChecked,omitempty"` - // +optional - LastChanged string `json:"lastChanged,omitempty"` -} - // CephObjectStoreUser represents a Ceph Object Store Gateway User // +genclient // +genclient:noStatus diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index bfd059916..979fcd3f3 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -97,38 +97,6 @@ func (in AnnotationsSpec) DeepCopy() AnnotationsSpec { return *out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) { - *out = *in - in.Bucket.DeepCopyInto(&out.Bucket) - if in.LivenessProbe != nil { - in, out := &in.LivenessProbe, &out.LivenessProbe - *out = new(ProbeSpec) - (*in).DeepCopyInto(*out) - } - if in.ReadinessProbe != nil { - in, out := &in.ReadinessProbe, &out.ReadinessProbe - *out = new(ProbeSpec) - (*in).DeepCopyInto(*out) - } - if in.StartupProbe != nil { - in, out := &in.StartupProbe, &out.StartupProbe - *out = new(ProbeSpec) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketHealthCheckSpec. -func (in *BucketHealthCheckSpec) DeepCopy() *BucketHealthCheckSpec { - if in == nil { - return nil - } - out := new(BucketHealthCheckSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BucketNotificationSpec) DeepCopyInto(out *BucketNotificationSpec) { *out = *in @@ -155,22 +123,6 @@ func (in *BucketNotificationSpec) DeepCopy() *BucketNotificationSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BucketStatus) DeepCopyInto(out *BucketStatus) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketStatus. -func (in *BucketStatus) DeepCopy() *BucketStatus { - if in == nil { - return nil - } - out := new(BucketStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BucketTopicSpec) DeepCopyInto(out *BucketTopicSpec) { *out = *in @@ -3059,6 +3011,37 @@ func (in *NotificationKeyFilterRule) DeepCopy() *NotificationKeyFilterRule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectHealthCheckSpec) DeepCopyInto(out *ObjectHealthCheckSpec) { + *out = *in + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + if in.StartupProbe != nil { + in, out := &in.StartupProbe, &out.StartupProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectHealthCheckSpec. +func (in *ObjectHealthCheckSpec) DeepCopy() *ObjectHealthCheckSpec { + if in == nil { + return nil + } + out := new(ObjectHealthCheckSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectRealmSpec) DeepCopyInto(out *ObjectRealmSpec) { *out = *in @@ -3123,11 +3106,6 @@ func (in *ObjectStoreSpec) DeepCopy() *ObjectStoreSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStoreStatus) DeepCopyInto(out *ObjectStoreStatus) { *out = *in - if in.BucketStatus != nil { - in, out := &in.BucketStatus, &out.BucketStatus - *out = new(BucketStatus) - **out = **in - } if in.Info != nil { in, out := &in.Info, &out.Info *out = make(map[string]string, len(*in)) diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index e8bc4dd29..c76cc7fc4 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -24,6 +24,7 @@ import ( "syscall" "time" + "github.com/ceph/go-ceph/rgw/admin" "github.com/coreos/pkg/capnslog" bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned" "github.com/pkg/errors" @@ -80,22 +81,15 @@ var cephObjectStoreDependents = CephObjectStoreDependents // ReconcileCephObjectStore reconciles a cephObjectStore object type ReconcileCephObjectStore struct { - client client.Client - bktclient bktclient.Interface - scheme *runtime.Scheme - context *clusterd.Context - clusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo - objectStoreContexts map[string]*objectStoreHealth - recorder record.EventRecorder - opManagerContext context.Context - opConfig opcontroller.OperatorConfig -} - -type objectStoreHealth struct { - internalCtx context.Context - internalCancel context.CancelFunc - started bool + client client.Client + bktclient bktclient.Interface + scheme *runtime.Scheme + context *clusterd.Context + clusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + recorder record.EventRecorder + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Add creates a new cephObjectStore Controller and adds it to the Manager. The Manager will set fields on the Controller @@ -108,14 +102,13 @@ func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext contex func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { context.Client = mgr.GetClient() return &ReconcileCephObjectStore{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, - bktclient: bktclient.NewForConfigOrDie(context.KubeConfig), - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: mgr.GetEventRecorderFor("rook-" + controllerName), - opManagerContext: opManagerContext, - opConfig: opConfig, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + bktclient: bktclient.NewForConfigOrDie(context.KubeConfig), + recorder: mgr.GetEventRecorderFor("rook-" + controllerName), + opManagerContext: opManagerContext, + opConfig: opConfig, } } @@ -166,12 +159,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephObjectStore resource not found. Ignoring since object must be deleted.") - // If there was a previous error or if a user removed this resource's finalizer, it's - // possible Rook didn't clean up the monitoring routine for this resource. Ensure the - // routine is stopped when we see the resource is gone. - cephObjectStore.Name = request.Name - cephObjectStore.Namespace = request.Namespace - r.stopMonitoring(cephObjectStore) return reconcile.Result{}, *cephObjectStore, nil } // Error reading the object - requeue the request. @@ -204,9 +191,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci // If not, we should wait for it to be ready // This handles the case where the operator is not ready to accept Ceph command but the cluster exists if !cephObjectStore.GetDeletionTimestamp().IsZero() && !cephClusterExists { - // don't leak the health checker routine if we are force deleting - r.stopMonitoring(cephObjectStore) - // Remove finalizer err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephObjectStore) if err != nil { @@ -221,17 +205,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } r.clusterSpec = &cephCluster.Spec - // Initialize the channel for this object store - // This allows us to track multiple ObjectStores in the same namespace - _, ok := r.objectStoreContexts[monitoringChannelKey(cephObjectStore)] - if !ok { - internalCtx, internalCancel := context.WithCancel(r.opManagerContext) - r.objectStoreContexts[monitoringChannelKey(cephObjectStore)] = &objectStoreHealth{ - internalCtx: internalCtx, - internalCancel: internalCancel, - } - } - // Populate clusterInfo during each reconcile r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.clusterSpec) if err != nil { @@ -257,12 +230,18 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) if err != nil { - return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get object context") + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context") } opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) if err != nil { - return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get admin ops API context") + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context") } + + // ignore any errors coming from this for the deletion case since we could get into a + // partially-deleted case where RGWs aren't responding. Deletion will be blocked later if + // the bucket fails to be deleted and the RGWs are running. + _ = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore) + deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx) if err != nil { return reconcile.Result{}, *cephObjectStore, err @@ -273,9 +252,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } reporting.ReportDeletionNotBlockedDueToDependents(r.opManagerContext, logger, r.client, r.recorder, cephObjectStore) - // Cancel the context to stop monitoring the health of the object store - r.stopMonitoring(cephObjectStore) - cfg := clusterConfig{ context: r.context, store: cephObjectStore, @@ -349,9 +325,22 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci return result, *cephObjectStore, err } + objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context") + } + opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context") + } + err = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrap(err, "updated object store but failed to remove deprecated health check bucket") + } + // update ObservedGeneration in status at the end of reconcile // Set Progressing status, we are done reconciling, the health check go routine will update the status - updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore)) + updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore)) // Return and do not requeue logger.Debug("done reconciling") @@ -455,14 +444,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c } } - // Start monitoring - if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled { - err = r.startMonitoring(cephObjectStore, objContext, namespacedName) - if err != nil { - return reconcile.Result{}, err - } - } - return reconcile.Result{}, nil } @@ -525,43 +506,40 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil } -func monitoringChannelKey(o *cephv1.CephObjectStore) string { - return types.NamespacedName{Namespace: o.Namespace, Name: o.Name}.String() -} +// handle upgrade from v1.10; we need to remove the health checker bucket from the store, or +// deleting the store will be stuck on dependents because an unknown bucket exists +// TODO: remove this for Rook v1.12 release +var removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + healthCheckBucket := genHealthCheckerBucketName(string(cos.UID)) + doPurge := true // purge all content of the bucket + bucket := admin.Bucket{Bucket: healthCheckBucket, PurgeObject: &doPurge} + remErr := opsCtx.AdminOpsClient.RemoveBucket(ctx, bucket) -func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) error { - channelKey := monitoringChannelKey(objectstore) - - // Start monitoring object store - if r.objectStoreContexts[channelKey].started { - logger.Info("external rgw endpoint monitoring go routine already running!") + if remErr == nil { + logger.Info("successfully deleted deprecated health checker bucket") return nil } - rgwChecker, err := newBucketChecker(r.context, objContext, r.client, namespacedName, &objectstore.Spec) - if err != nil { - return errors.Wrapf(err, "failed to start rgw health checker for CephObjectStore %q, will re-reconcile", namespacedName.String()) + if errors.Is(remErr, admin.ErrNoSuchBucket) { + logger.Debug("deprecated health checker bucket already does not exist: NoSuchBucket") + return nil } - logger.Infof("starting rgw health checker for CephObjectStore %q", namespacedName.String()) - go rgwChecker.checkObjectStore(r.objectStoreContexts, channelKey) - - // Set the monitoring flag so we don't start more than one go routine - r.objectStoreContexts[channelKey].started = true - - return nil -} - -// cancel monitoring. This is a noop if monitoring is not running. -func (r *ReconcileCephObjectStore) stopMonitoring(objectstore *cephv1.CephObjectStore) { - channelKey := monitoringChannelKey(objectstore) - - _, monitoringContextExists := r.objectStoreContexts[channelKey] - if monitoringContextExists { - // stop the monitoring routine - r.objectStoreContexts[channelKey].internalCancel() - - // remove the monitoring routine from the map - delete(r.objectStoreContexts, channelKey) + if errors.Is(remErr, admin.ErrNoSuchKey) { + // ceph might return NoSuchKey than NoSuchBucket when the target bucket does not exist. + // then we can use GetBucketInfo() to judge the existence of the bucket. + // see: https://github.com/ceph/ceph/pull/44413 + _, getErr := opsCtx.AdminOpsClient.GetBucketInfo(ctx, bucket) + if getErr != nil { + if errors.Is(getErr, admin.ErrNoSuchBucket) { + logger.Debug("deprecated health checker bucket already does not exist: NoSuchKey") + return nil + } + // both commands errored; something is up; be sure to return both errors + return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q. failed to get bucket info. %v", healthCheckBucket, getErr) + } + // remove bucket failed, and bucket exists; something else went wrong with remove bucket } + + return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q", healthCheckBucket) } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index ab489c0d1..2c16df541 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -19,7 +19,7 @@ package object import ( "context" - "net/http" + "fmt" "os" "reflect" "testing" @@ -383,12 +383,11 @@ func TestCephObjectStoreController(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() // Create a ReconcileCephObjectStore object with the scheme and fake client. r := &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: record.NewFakeRecorder(5), - opManagerContext: context.TODO(), + client: cl, + scheme: s, + context: c, + recorder: record.NewFakeRecorder(5), + opManagerContext: context.TODO(), } return r @@ -526,29 +525,13 @@ func TestCephObjectStoreController(t *testing.T) { return r } - t.Run("error - failed to start health checker", func(t *testing.T) { - r := setupEnvironmentWithReadyCephCluster() - - // cause a failure when creating the admin ops api for the health check - origHTTPClientFunc := genObjectStoreHTTPClientFunc - genObjectStoreHTTPClientFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (client *http.Client, tlsCert []byte, err error) { - return nil, []byte{}, errors.New("induced error creating admin ops API connection") - } - defer func() { genObjectStoreHTTPClientFunc = origHTTPClientFunc }() - - _, err := r.Reconcile(ctx, req) - assert.Error(t, err) - // we don't actually care if Requeue is true if there is an error assert.True(t, res.Requeue) - assert.Contains(t, err.Error(), "failed to start rgw health checker") - assert.Contains(t, err.Error(), "induced error creating admin ops API connection") - - // health checker should start up after committing config changes - assert.True(t, calledCommitConfigChanges) - }) - t.Run("success - object store is running", func(t *testing.T) { r := setupEnvironmentWithReadyCephCluster() + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return nil + } + res, err := r.Reconcile(ctx, req) assert.NoError(t, err) assert.False(t, res.Requeue) @@ -556,12 +539,24 @@ func TestCephObjectStoreController(t *testing.T) { objectStore := &cephv1.CephObjectStore{} err = r.client.Get(context.TODO(), req.NamespacedName, objectStore) assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore) + assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore) assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore) assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore) assert.True(t, calledCommitConfigChanges) assert.Equal(t, 16, r.clusterInfo.CephVersion.Major) }) + + t.Run("failed to remove deprecated health check bucket", func(t *testing.T) { + r := setupEnvironmentWithReadyCephCluster() + + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return fmt.Errorf("induced error") + } + + _, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.Contains(t, err.Error(), "deprecated health check bucket") + }) } func TestCephObjectStoreControllerMultisite(t *testing.T) { @@ -735,12 +730,11 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() r := &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: record.NewFakeRecorder(5), - opManagerContext: ctx, + client: cl, + scheme: s, + context: c, + recorder: record.NewFakeRecorder(5), + opManagerContext: ctx, } _, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) @@ -757,6 +751,10 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { return &cephver.Pacific, &cephver.Pacific, nil } + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return nil + } + t.Run("create an object store", func(t *testing.T) { res, err := r.Reconcile(ctx, req) assert.NoError(t, err) @@ -894,12 +892,11 @@ func TestCephObjectExternalStoreController(t *testing.T) { } r := &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: record.NewFakeRecorder(5), - opManagerContext: ctx, + client: cl, + scheme: s, + context: c, + recorder: record.NewFakeRecorder(5), + opManagerContext: ctx, } _, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) @@ -931,6 +928,10 @@ func TestCephObjectExternalStoreController(t *testing.T) { r := getReconciler(objects) + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return nil + } + t.Run("create an external object store", func(t *testing.T) { res, err := r.Reconcile(ctx, req) assert.NoError(t, err) diff --git a/pkg/operator/ceph/object/dependents.go b/pkg/operator/ceph/object/dependents.go index bba074993..4493cd80e 100644 --- a/pkg/operator/ceph/object/dependents.go +++ b/pkg/operator/ceph/object/dependents.go @@ -30,6 +30,7 @@ import ( const ( bucketDependentType = "buckets in the object store (could be from ObjectBucketClaims or COSI Buckets)" zoneIsMasterWithPeersDependentType = "zone is master and has peers" + s3HealthCheckBucketName = "rook-ceph-bucket-checker" ) // CephObjectStoreDependents returns the buckets which exist in the object store that should block @@ -127,11 +128,7 @@ func getBucketDependents( if err != nil { return errors.Wrapf(err, "failed to list buckets in CephObjectStore %q", nsName) } - healthCheckBucket := genHealthCheckerBucketName(string(store.UID)) for _, b := range buckets { - if b == healthCheckBucket { - continue // don't include the health checker bucket as a blocking dependent - } deps.Add(bucketDependentType, b) } @@ -165,3 +162,7 @@ func getMasterZoneDependents( } return nil } + +func genHealthCheckerBucketName(uuid string) string { + return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid) +} diff --git a/pkg/operator/ceph/object/health.go b/pkg/operator/ceph/object/health.go deleted file mode 100644 index fe2ca1760..000000000 --- a/pkg/operator/ceph/object/health.go +++ /dev/null @@ -1,292 +0,0 @@ -/* -Copyright 2020 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package object - -import ( - "fmt" - "time" - - "github.com/ceph/go-ceph/rgw/admin" - "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - s3UserHealthCheckName = "rook-ceph-internal-s3-user-checker" - s3HealthCheckBucketName = "rook-ceph-bucket-checker" - s3HealthCheckObjectBody = "Test Rook Object Data" - s3HealthCheckObjectKey = "rookHealthCheckTestObject" - contentType = "plain/text" -) - -var ( - defaultHealthCheckInterval = 1 * time.Minute -) - -// bucketChecker aggregates the mon/cluster info needed to check the health of the monitors -type bucketChecker struct { - context *clusterd.Context - objContext *AdminOpsContext - interval *time.Duration - port int32 - client client.Client - namespacedName types.NamespacedName - objectStoreSpec *cephv1.ObjectStoreSpec -} - -// newbucketChecker creates a new HealthChecker object -func newBucketChecker( - ctx *clusterd.Context, objContext *Context, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec, -) (*bucketChecker, error) { - port, err := objectStoreSpec.GetPort() - if err != nil { - return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String()) - } - - opsCtx, err := NewMultisiteAdminOpsContext(objContext, objectStoreSpec) - if err != nil { - return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String()) - } - - c := &bucketChecker{ - context: ctx, - objContext: opsCtx, - interval: &defaultHealthCheckInterval, - port: port, - namespacedName: namespacedName, - client: client, - objectStoreSpec: objectStoreSpec, - } - - // allow overriding the check interval - checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval - if checkInterval != nil { - logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String()) - c.interval = &checkInterval.Duration - } - - return c, nil -} - -// checkObjectStore periodically checks the health of the cluster -func (c *bucketChecker) checkObjectStore(objectStoreContexts map[string]*objectStoreHealth, channelKey string) { - // check the object store health immediately before starting the loop - err := c.checkObjectStoreHealth() - if err != nil { - updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error()) - logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err) - } - - for { - // We must perform this check otherwise the case will check an index that does not exist anymore and - // we will get an invalid pointer error and the go routine will panic - if _, ok := objectStoreContexts[channelKey]; !ok { - logger.Infof("object store %q has been deleted. stopping monitoring of rgw endpoints", c.namespacedName.Name) - return - } - select { - case <-objectStoreContexts[channelKey].internalCtx.Done(): - // purge bucket and s3 user - // Needed for external mode where in converged everything goes away with the CR deletion - c.cleanupHealthCheck() - delete(objectStoreContexts, channelKey) - logger.Infof("stopping monitoring of rgw endpoints for object store %q", c.namespacedName.Name) - return - - case <-time.After(*c.interval): - logger.Debugf("checking rgw health of object store %q", c.namespacedName.Name) - err := c.checkObjectStoreHealth() - if err != nil { - updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error()) - logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err) - } - } - } -} - -func (c *bucketChecker) checkObjectStoreHealth() error { - /* - 0. purge the s3 object by default - 1. create an S3 user - 2. always use the same user - 3. if already exists just re-hydrate the s3 credentials - 4. create a bucket with that user or use the existing one (always use the same bucket) - 5. create a check file - 6. get the hash of the file - 7. PUT the file - 8. GET the file - 9. compare hashes - 10. delete object on bucket - 11. update CR health status check - - Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive - */ - - // Keep admin ops context up-to date if there are config changes - if err := UpdateEndpoint(&c.objContext.Context, c.objectStoreSpec); err != nil { - return errors.Wrapf(err, "failed to parse updated CephObjectStore spec") - } - - // Generate unique user and bucket name - bucketName := genHealthCheckerBucketName(c.objContext.UID) - userConfig := genUserCheckerConfig(c.objContext.UID) - - // Create checker user - logger.Debugf("creating s3 user object %q for object store %q health check", userConfig.ID, c.namespacedName.Name) - var user admin.User - user, err := c.objContext.AdminOpsClient.GetUser(c.objContext.clusterInfo.Context, userConfig) - if err != nil { - if errors.Is(err, admin.ErrNoSuchUser) { - user, err = c.objContext.AdminOpsClient.CreateUser(c.objContext.clusterInfo.Context, userConfig) - if err != nil { - return errors.Wrapf(err, "failed to create from ceph object user %v", userConfig.ID) - } - } else { - return errors.Wrapf(err, "failed to get details from ceph object user %q", userConfig.ID) - } - } - - // Set access and secret key - s3endpoint := c.objContext.Endpoint - s3AccessKey := user.Keys[0].AccessKey - s3SecretKey := user.Keys[0].SecretKey - - // Initiate s3 agent - logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name) - s3client, err := NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false) - if err != nil { - return errors.Wrap(err, "failed to initialize s3 connection") - } - - // Force purge the s3 object before starting anything - cleanupObjectHealthCheck(s3client, c.objContext.UID) - - // Bucket health test - err = c.testBucketHealth(s3client, bucketName) - if err != nil { - return errors.Wrapf(err, "failed to run bucket health checks for object store %q", c.namespacedName.String()) - } - - logger.Debugf("successfully checked object store endpoint for object store %q", c.namespacedName.String()) - - // Update the EndpointStatus in the CR to reflect the healthiness - updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionConnected, "") - - return nil -} - -func cleanupObjectHealthCheck(s3client *S3Agent, objectStoreUID string) { - bucketToDelete := genHealthCheckerBucketName(objectStoreUID) - logger.Debugf("deleting object %q from bucket %q", s3HealthCheckObjectKey, bucketToDelete) - _, err := s3client.DeleteObjectInBucket(bucketToDelete, s3HealthCheckObjectKey) - if err != nil { - logger.Errorf("failed to delete object in bucket. %v", err) - } -} - -func (c *bucketChecker) cleanupHealthCheck() { - bucketToDelete := genHealthCheckerBucketName(c.objContext.UID) - logger.Infof("deleting object %q from bucket %q in object store %q", s3HealthCheckObjectKey, bucketToDelete, c.namespacedName.Name) - - thePurge := true - err := c.objContext.AdminOpsClient.RemoveBucket(c.objContext.clusterInfo.Context, admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge}) - if err != nil { - if errors.Is(err, admin.ErrNoSuchBucket) { - // opinion: "not found" is not an error - logger.Debugf("bucket %q does not exist", bucketToDelete) - } else { - logger.Errorf("failed to delete bucket %q for object store %q. %v", bucketToDelete, c.namespacedName.Name, err) - } - } - - userToDelete := genUserCheckerConfig(c.objContext.UID) - err = c.objContext.AdminOpsClient.RemoveUser(c.objContext.clusterInfo.Context, userToDelete) - if err != nil && !errors.Is(err, admin.ErrNoSuchUser) { - logger.Errorf("failed to delete object user %q for object store %q. %v", userToDelete.ID, c.namespacedName.Name, err) - } - - logger.Debugf("successfully deleted object user %q for object store %q", userToDelete.ID, c.namespacedName.Name) -} - -func toCustomResourceStatus(currentStatus *cephv1.BucketStatus, details string, health cephv1.ConditionType) *cephv1.BucketStatus { - s := &cephv1.BucketStatus{ - Health: health, - LastChecked: time.Now().UTC().Format(time.RFC3339), - Details: details, - } - - if currentStatus != nil { - s.LastChanged = currentStatus.LastChanged - if currentStatus.Details != s.Details { - s.LastChanged = s.LastChecked - } - } - return s -} - -func genHealthCheckerBucketName(uuid string) string { - return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid) -} - -func genUserCheckerConfig(cephObjectStoreUID string) admin.User { - userName := fmt.Sprintf("%s-%s", s3UserHealthCheckName, cephObjectStoreUID) - - return admin.User{ - ID: userName, - DisplayName: userName, - } -} - -func (c *bucketChecker) testBucketHealth(s3client *S3Agent, bucket string) error { - // Purge on exit - defer cleanupObjectHealthCheck(s3client, c.objContext.UID) - - // Create S3 bucket - logger.Debugf("creating bucket %q", bucket) - err := s3client.CreateBucketNoInfoLogging(bucket) - if err != nil { - return errors.Wrapf(err, "failed to create bucket %q for object store %q", bucket, c.namespacedName.Name) - } - - // Put an object into the bucket - logger.Debugf("putting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - _, err = s3client.PutObjectInBucket(bucket, string(s3HealthCheckObjectBody), s3HealthCheckObjectKey, contentType) - if err != nil { - return errors.Wrapf(err, "failed to put object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - } - - // Get the object from the bucket - logger.Debugf("getting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - read, err := s3client.GetObjectInBucket(bucket, s3HealthCheckObjectKey) - if err != nil { - return errors.Wrapf(err, "failed to get object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - } - - // Compare the old and the existing object - logger.Debugf("comparing objects hash for object store %q", c.namespacedName.Name) - oldHash := k8sutil.Hash(s3HealthCheckObjectBody) - currentHash := k8sutil.Hash(read) - if currentHash != oldHash { - return errors.Wrapf(err, "wrong file content, old file hash is %q and new one is %q for object store %q", oldHash, currentHash, c.namespacedName.Name) - } - - return nil -} diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 11bb15600..709351553 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -391,7 +391,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, } // configureReadinessProbe returns the desired readiness probe for a given daemon -func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) { +func configureReadinessProbe(container *v1.Container, healthCheck cephv1.ObjectHealthCheckSpec) { if ok := healthCheck.ReadinessProbe; ok != nil { if !healthCheck.ReadinessProbe.Disabled { probe := healthCheck.ReadinessProbe.Probe diff --git a/pkg/operator/ceph/object/status.go b/pkg/operator/ceph/object/status.go index ee34182c9..cfcd9178b 100644 --- a/pkg/operator/ceph/object/status.go +++ b/pkg/operator/ceph/object/status.go @@ -75,42 +75,6 @@ func updateStatus(ctx context.Context, observedGeneration int64, client client.C logger.Debugf("object store %q status updated to %q", namespacedName.String(), status) } -// updateStatusBucket updates an object with a given status -func updateStatusBucket(ctx context.Context, client client.Client, name types.NamespacedName, status cephv1.ConditionType, details string) { - // Updating the status is important to users, but we can still keep operating if there is a - // failure. Retry a few times to give it our best effort attempt. - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - objectStore := &cephv1.CephObjectStore{} - if err := client.Get(ctx, name, objectStore); err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("CephObjectStore resource not found. Ignoring since object must be deleted.") - return nil - } - return errors.Wrapf(err, "failed to retrieve object store %q to update status to %v", name.String(), status) - } - if objectStore.Status == nil { - objectStore.Status = &cephv1.ObjectStoreStatus{} - } - objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, status) - - // do not transition to other statuses once deletion begins - if objectStore.Status.Phase != cephv1.ConditionDeleting { - objectStore.Status.Phase = status - } - - // but we still need to update the health checker status - if err := reporting.UpdateStatus(client, objectStore); err != nil { - return errors.Wrapf(err, "failed to set object store %q status to %v", name.String(), status) - } - return nil - }) - if err != nil { - logger.Error(err) - } - - logger.Debugf("object store %q status updated to %v", name.String(), status) -} - func buildStatusInfo(cephObjectStore *cephv1.CephObjectStore) map[string]string { m := make(map[string]string) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 2022e87b4..ccc58c86f 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -452,10 +452,6 @@ spec: securePort: ` + strconv.Itoa(port) + ` instances: ` + strconv.Itoa(replicaCount) + ` sslCertificateRef: ` + name + ` - healthCheck: - bucket: - disabled: false - interval: 10s ` } return `apiVersion: ceph.rook.io/v1 @@ -477,10 +473,6 @@ spec: resources: null port: ` + strconv.Itoa(port) + ` instances: ` + strconv.Itoa(replicaCount) + ` - healthCheck: - bucket: - disabled: false - interval: 5s ` } diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index deecbe5f1..37216bbfa 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -36,6 +36,7 @@ import ( "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -116,18 +117,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils for i = 0; i < retryCount; i++ { objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{}) assert.Nil(t, err) - if objectStore.Status == nil || objectStore.Status.BucketStatus == nil { + // TODO: check that object store status is good, and also check that status of + // deployment is good based on health checks + + if objectStore.Status == nil { logger.Infof("(%d) object status check sleeping for 5 seconds ...%+v", i, objectStore.Status) time.Sleep(5 * time.Second) continue } logger.Info("objectstore status is", objectStore.Status) - if objectStore.Status.BucketStatus.Health == cephv1.ConditionFailure { - logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.BucketStatus) + // ConditionConnected supports Rook v1.10 clusters that still had health check + // TODO: remove that half of check after Rook v1.12 release + if objectStore.Status.Phase != cephv1.ConditionReady && objectStore.Status.Phase != cephv1.ConditionConnected { + logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.Phase) time.Sleep(5 * time.Second) continue } - assert.Equal(t, cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health) + // Info field has the endpoint in it assert.NotEmpty(t, objectStore.Status.Info) assert.NotEmpty(t, objectStore.Status.Info["endpoint"]) @@ -138,6 +144,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils } }) + t.Run("verify RGW liveness probes show healthy", func(t *testing.T) { + err := wait.PollImmediate(2*time.Second, 90*time.Second, func() (done bool, err error) { + deployName := "rook-ceph-rgw-" + storeName + "-a" + d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, deployName, metav1.GetOptions{}) + if err != nil { + logger.Infof("waiting for rgw deployment %q to be ready; failed to get deployment: %v", deployName, err) + return false, nil + } + if d.Status.UnavailableReplicas != 0 { + logger.Infof("waiting rgw deployment %q to be ready; %d replicas are unavailable", deployName, d.Status.UnavailableReplicas) + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + }) + t.Run("verify RGW service is up", func(t *testing.T) { assert.True(t, k8sh.IsServiceUp("rook-ceph-rgw-"+storeName, namespace)) })