diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 04e1c382d..163ec5e0d 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -162,9 +162,6 @@ Rook will be default monitor the state of the object store endpoints. The following CRD settings are available: * `healthCheck`: main object store health monitoring section - * `bucket`: Rook checks that the object store is usable regularly. This is explained in more - detail below. Use this config to disable or change the interval at which Rook verifies the - object store connectivity. * `startupProbe`: Disable, or override timing and threshold values of the object gateway startup probe. * `livenessProbe`: Disable, or override timing and threshold values of the object gateway liveness probe. * `readinessProbe`: Disable, or override timing and threshold values of the object gateway readiness probe. @@ -173,31 +170,21 @@ Here is a complete example: ```yaml healthCheck: - bucket: - disabled: false - interval: 60s startupProbe: disabled: false livenessProbe: disabled: false - periodSeconds: 5 - failureThreshold: 4 + periodSeconds: 30 + failureThreshold: 13 readinessProbe: disabled: false periodSeconds: 5 failureThreshold: 2 ``` -The endpoint health check procedure is the following: - -1. Create an S3 user -2. Create a bucket with that user -3. PUT the file in the object store -4. GET the file from the object store -5. Verify object consistency -6. Update CR health status check - -The Rook Ceph operator always keeps the bucket and the user for the health check, it just does a PUT and GET of an s3 object since creating a bucket is an expensive operation. +You can monitor the health of a CephObjectStore by monitoring the gateway deployments it creates. +The primary deployment created is named `rook-ceph-rgw--a` where `store-name` is the +name of the CephObjectStore (don't forget the `-a` at the end). ## Security settings diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index 55f491099..03d31cc61 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -48,10 +48,6 @@ spec: port: 80 # securePort: 443 instances: 1 - healthCheck: - bucket: - disabled: false - interval: 60s ``` After the `CephObjectStore` is created, the Rook operator will then create all the pools and other resources necessary to start the service. This may take a minute to complete. @@ -84,10 +80,6 @@ spec: port: 8080 externalRgwEndpoints: - ip: 192.168.39.182 - healthCheck: - bucket: - enabled: true - interval: 60s ``` You can use the existing `object-external.yaml` file. @@ -249,6 +241,12 @@ s5cmd --endpoint-url http://$AWS_HOST:$PORT cp s3://$BUCKET_NAME/rookObj /tmp/ro cat /tmp/rookObj-download ``` +## Monitoring health + +Rook configures health probes on the deployment created for CephObjectStore gateways. Refer to +[the CRD document](../../CRDs/Object-Storage/ceph-object-store-crd.md#health-settings) for +information about configuring the probes and monitoring the deployment status. + ## Access External to the Cluster Rook sets up the object storage so pods will have access internal to the cluster. If your applications are running outside the cluster, diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index 232239c60..0702d8c7b 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -532,9 +532,6 @@ cephObjectStores: # sslCertificateRef: instances: 1 priorityClassName: system-cluster-critical - healthCheck: - bucket: - interval: 60s storageClass: enabled: true name: ceph-bucket diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 87374aa1b..e51e54c93 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -12231,20 +12231,9 @@ spec: type: string type: object healthCheck: - description: The rgw Bucket healthchecks and liveness probe + description: The RGW health probes nullable: true properties: - bucket: - description: HealthCheckSpec represents the health check of an object store bucket - properties: - disabled: - type: boolean - interval: - description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds - type: string - timeout: - type: string - type: object livenessProbe: description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon properties: @@ -12788,19 +12777,6 @@ spec: status: description: ObjectStoreStatus represents the status of a Ceph Object Store resource properties: - bucketStatus: - description: BucketStatus represents the status of a bucket - properties: - details: - type: string - health: - description: ConditionType represent a resource's status - type: string - lastChanged: - type: string - lastChecked: - type: string - type: object conditions: items: description: Condition represents a status condition on any Rook-Ceph Custom Resource. diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index f76bba1cf..aa1aec165 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -12223,20 +12223,9 @@ spec: type: string type: object healthCheck: - description: The rgw Bucket healthchecks and liveness probe + description: The RGW health probes nullable: true properties: - bucket: - description: HealthCheckSpec represents the health check of an object store bucket - properties: - disabled: - type: boolean - interval: - description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds - type: string - timeout: - type: string - type: object livenessProbe: description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon properties: @@ -12780,19 +12769,6 @@ spec: status: description: ObjectStoreStatus represents the status of a Ceph Object Store resource properties: - bucketStatus: - description: BucketStatus represents the status of a bucket - properties: - details: - type: string - health: - description: ConditionType represent a resource's status - type: string - lastChanged: - type: string - lastChecked: - type: string - type: object conditions: items: description: Condition represents a status condition on any Rook-Ceph Custom Resource. diff --git a/deploy/examples/object-ec.yaml b/deploy/examples/object-ec.yaml index 8dabb8fde..66743943f 100644 --- a/deploy/examples/object-ec.yaml +++ b/deploy/examples/object-ec.yaml @@ -81,12 +81,9 @@ spec: # memory: "1024Mi" # priorityClassName: my-priority-class #zone: - #name: zone-a + #name: zone-a # service endpoint healthcheck healthCheck: - bucket: - disabled: false - interval: 60s # Configure the pod probes for the rgw daemon livenessProbe: disabled: false diff --git a/deploy/examples/object-openshift.yaml b/deploy/examples/object-openshift.yaml index 893cdd1bb..41377c6a8 100644 --- a/deploy/examples/object-openshift.yaml +++ b/deploy/examples/object-openshift.yaml @@ -98,9 +98,6 @@ spec: # service.beta.openshift.io/serving-cert-secret-name: # service endpoint healthcheck healthCheck: - bucket: - disabled: false - interval: 60s # Configure the pod probes for the rgw daemon livenessProbe: disabled: false @@ -149,12 +146,12 @@ metadata: spec: port: targetPort: http -# if TLS is enabled, remove the above http port and uncomment the https port and tls settings -# port: -# targetPort: https -# tls: -# insecureEdgeTerminationPolicy: Redirect -# termination: reencrypt + # if TLS is enabled, remove the above http port and uncomment the https port and tls settings + # port: + # targetPort: https + # tls: + # insecureEdgeTerminationPolicy: Redirect + # termination: reencrypt to: kind: Service name: rook-ceph-rgw-my-store # The name of the RGW service is in the form 'rook-ceph-rgw-' diff --git a/deploy/examples/object.yaml b/deploy/examples/object.yaml index 2d4fa47e7..3311ab6ec 100644 --- a/deploy/examples/object.yaml +++ b/deploy/examples/object.yaml @@ -101,12 +101,9 @@ spec: # memory: "1024Mi" priorityClassName: system-cluster-critical #zone: - #name: zone-a + #name: zone-a # service endpoint healthcheck healthCheck: - bucket: - disabled: false - interval: 60s # Configure the pod probes for the rgw daemon startupProbe: disabled: false diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index c60c7adb0..3c818e79c 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1335,10 +1335,10 @@ type ObjectStoreSpec struct { // +nullable Zone ZoneSpec `json:"zone,omitempty"` - // The rgw Bucket healthchecks and liveness probe + // The RGW health probes // +optional // +nullable - HealthCheck BucketHealthCheckSpec `json:"healthCheck,omitempty"` + HealthCheck ObjectHealthCheckSpec `json:"healthCheck,omitempty"` // Security represents security settings // +optional @@ -1346,10 +1346,8 @@ type ObjectStoreSpec struct { Security *ObjectStoreSecuritySpec `json:"security,omitempty"` } -// BucketHealthCheckSpec represents the health check of an object store -type BucketHealthCheckSpec struct { - // +optional - Bucket HealthCheckSpec `json:"bucket,omitempty"` +// ObjectHealthCheckSpec represents the health check of an object store +type ObjectHealthCheckSpec struct { // +optional LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` // +optional @@ -1455,8 +1453,6 @@ type ObjectStoreStatus struct { // +optional Message string `json:"message,omitempty"` // +optional - BucketStatus *BucketStatus `json:"bucketStatus,omitempty"` - // +optional // +nullable Info map[string]string `json:"info,omitempty"` Conditions []Condition `json:"conditions,omitempty"` @@ -1465,18 +1461,6 @@ type ObjectStoreStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` } -// BucketStatus represents the status of a bucket -type BucketStatus struct { - // +optional - Health ConditionType `json:"health,omitempty"` - // +optional - Details string `json:"details,omitempty"` - // +optional - LastChecked string `json:"lastChecked,omitempty"` - // +optional - LastChanged string `json:"lastChanged,omitempty"` -} - // CephObjectStoreUser represents a Ceph Object Store Gateway User // +genclient // +genclient:noStatus diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index bfd059916..979fcd3f3 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -97,38 +97,6 @@ func (in AnnotationsSpec) DeepCopy() AnnotationsSpec { return *out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) { - *out = *in - in.Bucket.DeepCopyInto(&out.Bucket) - if in.LivenessProbe != nil { - in, out := &in.LivenessProbe, &out.LivenessProbe - *out = new(ProbeSpec) - (*in).DeepCopyInto(*out) - } - if in.ReadinessProbe != nil { - in, out := &in.ReadinessProbe, &out.ReadinessProbe - *out = new(ProbeSpec) - (*in).DeepCopyInto(*out) - } - if in.StartupProbe != nil { - in, out := &in.StartupProbe, &out.StartupProbe - *out = new(ProbeSpec) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketHealthCheckSpec. -func (in *BucketHealthCheckSpec) DeepCopy() *BucketHealthCheckSpec { - if in == nil { - return nil - } - out := new(BucketHealthCheckSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BucketNotificationSpec) DeepCopyInto(out *BucketNotificationSpec) { *out = *in @@ -155,22 +123,6 @@ func (in *BucketNotificationSpec) DeepCopy() *BucketNotificationSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BucketStatus) DeepCopyInto(out *BucketStatus) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketStatus. -func (in *BucketStatus) DeepCopy() *BucketStatus { - if in == nil { - return nil - } - out := new(BucketStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BucketTopicSpec) DeepCopyInto(out *BucketTopicSpec) { *out = *in @@ -3059,6 +3011,37 @@ func (in *NotificationKeyFilterRule) DeepCopy() *NotificationKeyFilterRule { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectHealthCheckSpec) DeepCopyInto(out *ObjectHealthCheckSpec) { + *out = *in + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + if in.StartupProbe != nil { + in, out := &in.StartupProbe, &out.StartupProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectHealthCheckSpec. +func (in *ObjectHealthCheckSpec) DeepCopy() *ObjectHealthCheckSpec { + if in == nil { + return nil + } + out := new(ObjectHealthCheckSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectRealmSpec) DeepCopyInto(out *ObjectRealmSpec) { *out = *in @@ -3123,11 +3106,6 @@ func (in *ObjectStoreSpec) DeepCopy() *ObjectStoreSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStoreStatus) DeepCopyInto(out *ObjectStoreStatus) { *out = *in - if in.BucketStatus != nil { - in, out := &in.BucketStatus, &out.BucketStatus - *out = new(BucketStatus) - **out = **in - } if in.Info != nil { in, out := &in.Info, &out.Info *out = make(map[string]string, len(*in)) diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index e8bc4dd29..c76cc7fc4 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -24,6 +24,7 @@ import ( "syscall" "time" + "github.com/ceph/go-ceph/rgw/admin" "github.com/coreos/pkg/capnslog" bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned" "github.com/pkg/errors" @@ -80,22 +81,15 @@ var cephObjectStoreDependents = CephObjectStoreDependents // ReconcileCephObjectStore reconciles a cephObjectStore object type ReconcileCephObjectStore struct { - client client.Client - bktclient bktclient.Interface - scheme *runtime.Scheme - context *clusterd.Context - clusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo - objectStoreContexts map[string]*objectStoreHealth - recorder record.EventRecorder - opManagerContext context.Context - opConfig opcontroller.OperatorConfig -} - -type objectStoreHealth struct { - internalCtx context.Context - internalCancel context.CancelFunc - started bool + client client.Client + bktclient bktclient.Interface + scheme *runtime.Scheme + context *clusterd.Context + clusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + recorder record.EventRecorder + opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Add creates a new cephObjectStore Controller and adds it to the Manager. The Manager will set fields on the Controller @@ -108,14 +102,13 @@ func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext contex func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { context.Client = mgr.GetClient() return &ReconcileCephObjectStore{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, - bktclient: bktclient.NewForConfigOrDie(context.KubeConfig), - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: mgr.GetEventRecorderFor("rook-" + controllerName), - opManagerContext: opManagerContext, - opConfig: opConfig, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + bktclient: bktclient.NewForConfigOrDie(context.KubeConfig), + recorder: mgr.GetEventRecorderFor("rook-" + controllerName), + opManagerContext: opManagerContext, + opConfig: opConfig, } } @@ -166,12 +159,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci if err != nil { if kerrors.IsNotFound(err) { logger.Debug("cephObjectStore resource not found. Ignoring since object must be deleted.") - // If there was a previous error or if a user removed this resource's finalizer, it's - // possible Rook didn't clean up the monitoring routine for this resource. Ensure the - // routine is stopped when we see the resource is gone. - cephObjectStore.Name = request.Name - cephObjectStore.Namespace = request.Namespace - r.stopMonitoring(cephObjectStore) return reconcile.Result{}, *cephObjectStore, nil } // Error reading the object - requeue the request. @@ -204,9 +191,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci // If not, we should wait for it to be ready // This handles the case where the operator is not ready to accept Ceph command but the cluster exists if !cephObjectStore.GetDeletionTimestamp().IsZero() && !cephClusterExists { - // don't leak the health checker routine if we are force deleting - r.stopMonitoring(cephObjectStore) - // Remove finalizer err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephObjectStore) if err != nil { @@ -221,17 +205,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } r.clusterSpec = &cephCluster.Spec - // Initialize the channel for this object store - // This allows us to track multiple ObjectStores in the same namespace - _, ok := r.objectStoreContexts[monitoringChannelKey(cephObjectStore)] - if !ok { - internalCtx, internalCancel := context.WithCancel(r.opManagerContext) - r.objectStoreContexts[monitoringChannelKey(cephObjectStore)] = &objectStoreHealth{ - internalCtx: internalCtx, - internalCancel: internalCancel, - } - } - // Populate clusterInfo during each reconcile r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.clusterSpec) if err != nil { @@ -257,12 +230,18 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) if err != nil { - return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get object context") + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context") } opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) if err != nil { - return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get admin ops API context") + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context") } + + // ignore any errors coming from this for the deletion case since we could get into a + // partially-deleted case where RGWs aren't responding. Deletion will be blocked later if + // the bucket fails to be deleted and the RGWs are running. + _ = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore) + deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx) if err != nil { return reconcile.Result{}, *cephObjectStore, err @@ -273,9 +252,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } reporting.ReportDeletionNotBlockedDueToDependents(r.opManagerContext, logger, r.client, r.recorder, cephObjectStore) - // Cancel the context to stop monitoring the health of the object store - r.stopMonitoring(cephObjectStore) - cfg := clusterConfig{ context: r.context, store: cephObjectStore, @@ -349,9 +325,22 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci return result, *cephObjectStore, err } + objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context") + } + opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context") + } + err = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore) + if err != nil { + return reconcile.Result{}, *cephObjectStore, errors.Wrap(err, "updated object store but failed to remove deprecated health check bucket") + } + // update ObservedGeneration in status at the end of reconcile // Set Progressing status, we are done reconciling, the health check go routine will update the status - updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore)) + updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore)) // Return and do not requeue logger.Debug("done reconciling") @@ -455,14 +444,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c } } - // Start monitoring - if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled { - err = r.startMonitoring(cephObjectStore, objContext, namespacedName) - if err != nil { - return reconcile.Result{}, err - } - } - return reconcile.Result{}, nil } @@ -525,43 +506,40 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil } -func monitoringChannelKey(o *cephv1.CephObjectStore) string { - return types.NamespacedName{Namespace: o.Namespace, Name: o.Name}.String() -} +// handle upgrade from v1.10; we need to remove the health checker bucket from the store, or +// deleting the store will be stuck on dependents because an unknown bucket exists +// TODO: remove this for Rook v1.12 release +var removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + healthCheckBucket := genHealthCheckerBucketName(string(cos.UID)) + doPurge := true // purge all content of the bucket + bucket := admin.Bucket{Bucket: healthCheckBucket, PurgeObject: &doPurge} + remErr := opsCtx.AdminOpsClient.RemoveBucket(ctx, bucket) -func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) error { - channelKey := monitoringChannelKey(objectstore) - - // Start monitoring object store - if r.objectStoreContexts[channelKey].started { - logger.Info("external rgw endpoint monitoring go routine already running!") + if remErr == nil { + logger.Info("successfully deleted deprecated health checker bucket") return nil } - rgwChecker, err := newBucketChecker(r.context, objContext, r.client, namespacedName, &objectstore.Spec) - if err != nil { - return errors.Wrapf(err, "failed to start rgw health checker for CephObjectStore %q, will re-reconcile", namespacedName.String()) + if errors.Is(remErr, admin.ErrNoSuchBucket) { + logger.Debug("deprecated health checker bucket already does not exist: NoSuchBucket") + return nil } - logger.Infof("starting rgw health checker for CephObjectStore %q", namespacedName.String()) - go rgwChecker.checkObjectStore(r.objectStoreContexts, channelKey) - - // Set the monitoring flag so we don't start more than one go routine - r.objectStoreContexts[channelKey].started = true - - return nil -} - -// cancel monitoring. This is a noop if monitoring is not running. -func (r *ReconcileCephObjectStore) stopMonitoring(objectstore *cephv1.CephObjectStore) { - channelKey := monitoringChannelKey(objectstore) - - _, monitoringContextExists := r.objectStoreContexts[channelKey] - if monitoringContextExists { - // stop the monitoring routine - r.objectStoreContexts[channelKey].internalCancel() - - // remove the monitoring routine from the map - delete(r.objectStoreContexts, channelKey) + if errors.Is(remErr, admin.ErrNoSuchKey) { + // ceph might return NoSuchKey than NoSuchBucket when the target bucket does not exist. + // then we can use GetBucketInfo() to judge the existence of the bucket. + // see: https://github.com/ceph/ceph/pull/44413 + _, getErr := opsCtx.AdminOpsClient.GetBucketInfo(ctx, bucket) + if getErr != nil { + if errors.Is(getErr, admin.ErrNoSuchBucket) { + logger.Debug("deprecated health checker bucket already does not exist: NoSuchKey") + return nil + } + // both commands errored; something is up; be sure to return both errors + return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q. failed to get bucket info. %v", healthCheckBucket, getErr) + } + // remove bucket failed, and bucket exists; something else went wrong with remove bucket } + + return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q", healthCheckBucket) } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index ab489c0d1..2c16df541 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -19,7 +19,7 @@ package object import ( "context" - "net/http" + "fmt" "os" "reflect" "testing" @@ -383,12 +383,11 @@ func TestCephObjectStoreController(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build() // Create a ReconcileCephObjectStore object with the scheme and fake client. r := &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: record.NewFakeRecorder(5), - opManagerContext: context.TODO(), + client: cl, + scheme: s, + context: c, + recorder: record.NewFakeRecorder(5), + opManagerContext: context.TODO(), } return r @@ -526,29 +525,13 @@ func TestCephObjectStoreController(t *testing.T) { return r } - t.Run("error - failed to start health checker", func(t *testing.T) { - r := setupEnvironmentWithReadyCephCluster() - - // cause a failure when creating the admin ops api for the health check - origHTTPClientFunc := genObjectStoreHTTPClientFunc - genObjectStoreHTTPClientFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (client *http.Client, tlsCert []byte, err error) { - return nil, []byte{}, errors.New("induced error creating admin ops API connection") - } - defer func() { genObjectStoreHTTPClientFunc = origHTTPClientFunc }() - - _, err := r.Reconcile(ctx, req) - assert.Error(t, err) - // we don't actually care if Requeue is true if there is an error assert.True(t, res.Requeue) - assert.Contains(t, err.Error(), "failed to start rgw health checker") - assert.Contains(t, err.Error(), "induced error creating admin ops API connection") - - // health checker should start up after committing config changes - assert.True(t, calledCommitConfigChanges) - }) - t.Run("success - object store is running", func(t *testing.T) { r := setupEnvironmentWithReadyCephCluster() + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return nil + } + res, err := r.Reconcile(ctx, req) assert.NoError(t, err) assert.False(t, res.Requeue) @@ -556,12 +539,24 @@ func TestCephObjectStoreController(t *testing.T) { objectStore := &cephv1.CephObjectStore{} err = r.client.Get(context.TODO(), req.NamespacedName, objectStore) assert.NoError(t, err) - assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore) + assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore) assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore) assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore) assert.True(t, calledCommitConfigChanges) assert.Equal(t, 16, r.clusterInfo.CephVersion.Major) }) + + t.Run("failed to remove deprecated health check bucket", func(t *testing.T) { + r := setupEnvironmentWithReadyCephCluster() + + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return fmt.Errorf("induced error") + } + + _, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.Contains(t, err.Error(), "deprecated health check bucket") + }) } func TestCephObjectStoreControllerMultisite(t *testing.T) { @@ -735,12 +730,11 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() r := &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: record.NewFakeRecorder(5), - opManagerContext: ctx, + client: cl, + scheme: s, + context: c, + recorder: record.NewFakeRecorder(5), + opManagerContext: ctx, } _, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) @@ -757,6 +751,10 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { return &cephver.Pacific, &cephver.Pacific, nil } + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return nil + } + t.Run("create an object store", func(t *testing.T) { res, err := r.Reconcile(ctx, req) assert.NoError(t, err) @@ -894,12 +892,11 @@ func TestCephObjectExternalStoreController(t *testing.T) { } r := &ReconcileCephObjectStore{ - client: cl, - scheme: s, - context: c, - objectStoreContexts: make(map[string]*objectStoreHealth), - recorder: record.NewFakeRecorder(5), - opManagerContext: ctx, + client: cl, + scheme: s, + context: c, + recorder: record.NewFakeRecorder(5), + opManagerContext: ctx, } _, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) @@ -931,6 +928,10 @@ func TestCephObjectExternalStoreController(t *testing.T) { r := getReconciler(objects) + removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error { + return nil + } + t.Run("create an external object store", func(t *testing.T) { res, err := r.Reconcile(ctx, req) assert.NoError(t, err) diff --git a/pkg/operator/ceph/object/dependents.go b/pkg/operator/ceph/object/dependents.go index bba074993..4493cd80e 100644 --- a/pkg/operator/ceph/object/dependents.go +++ b/pkg/operator/ceph/object/dependents.go @@ -30,6 +30,7 @@ import ( const ( bucketDependentType = "buckets in the object store (could be from ObjectBucketClaims or COSI Buckets)" zoneIsMasterWithPeersDependentType = "zone is master and has peers" + s3HealthCheckBucketName = "rook-ceph-bucket-checker" ) // CephObjectStoreDependents returns the buckets which exist in the object store that should block @@ -127,11 +128,7 @@ func getBucketDependents( if err != nil { return errors.Wrapf(err, "failed to list buckets in CephObjectStore %q", nsName) } - healthCheckBucket := genHealthCheckerBucketName(string(store.UID)) for _, b := range buckets { - if b == healthCheckBucket { - continue // don't include the health checker bucket as a blocking dependent - } deps.Add(bucketDependentType, b) } @@ -165,3 +162,7 @@ func getMasterZoneDependents( } return nil } + +func genHealthCheckerBucketName(uuid string) string { + return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid) +} diff --git a/pkg/operator/ceph/object/health.go b/pkg/operator/ceph/object/health.go deleted file mode 100644 index fe2ca1760..000000000 --- a/pkg/operator/ceph/object/health.go +++ /dev/null @@ -1,292 +0,0 @@ -/* -Copyright 2020 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package object - -import ( - "fmt" - "time" - - "github.com/ceph/go-ceph/rgw/admin" - "github.com/pkg/errors" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - s3UserHealthCheckName = "rook-ceph-internal-s3-user-checker" - s3HealthCheckBucketName = "rook-ceph-bucket-checker" - s3HealthCheckObjectBody = "Test Rook Object Data" - s3HealthCheckObjectKey = "rookHealthCheckTestObject" - contentType = "plain/text" -) - -var ( - defaultHealthCheckInterval = 1 * time.Minute -) - -// bucketChecker aggregates the mon/cluster info needed to check the health of the monitors -type bucketChecker struct { - context *clusterd.Context - objContext *AdminOpsContext - interval *time.Duration - port int32 - client client.Client - namespacedName types.NamespacedName - objectStoreSpec *cephv1.ObjectStoreSpec -} - -// newbucketChecker creates a new HealthChecker object -func newBucketChecker( - ctx *clusterd.Context, objContext *Context, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec, -) (*bucketChecker, error) { - port, err := objectStoreSpec.GetPort() - if err != nil { - return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String()) - } - - opsCtx, err := NewMultisiteAdminOpsContext(objContext, objectStoreSpec) - if err != nil { - return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String()) - } - - c := &bucketChecker{ - context: ctx, - objContext: opsCtx, - interval: &defaultHealthCheckInterval, - port: port, - namespacedName: namespacedName, - client: client, - objectStoreSpec: objectStoreSpec, - } - - // allow overriding the check interval - checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval - if checkInterval != nil { - logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String()) - c.interval = &checkInterval.Duration - } - - return c, nil -} - -// checkObjectStore periodically checks the health of the cluster -func (c *bucketChecker) checkObjectStore(objectStoreContexts map[string]*objectStoreHealth, channelKey string) { - // check the object store health immediately before starting the loop - err := c.checkObjectStoreHealth() - if err != nil { - updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error()) - logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err) - } - - for { - // We must perform this check otherwise the case will check an index that does not exist anymore and - // we will get an invalid pointer error and the go routine will panic - if _, ok := objectStoreContexts[channelKey]; !ok { - logger.Infof("object store %q has been deleted. stopping monitoring of rgw endpoints", c.namespacedName.Name) - return - } - select { - case <-objectStoreContexts[channelKey].internalCtx.Done(): - // purge bucket and s3 user - // Needed for external mode where in converged everything goes away with the CR deletion - c.cleanupHealthCheck() - delete(objectStoreContexts, channelKey) - logger.Infof("stopping monitoring of rgw endpoints for object store %q", c.namespacedName.Name) - return - - case <-time.After(*c.interval): - logger.Debugf("checking rgw health of object store %q", c.namespacedName.Name) - err := c.checkObjectStoreHealth() - if err != nil { - updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error()) - logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err) - } - } - } -} - -func (c *bucketChecker) checkObjectStoreHealth() error { - /* - 0. purge the s3 object by default - 1. create an S3 user - 2. always use the same user - 3. if already exists just re-hydrate the s3 credentials - 4. create a bucket with that user or use the existing one (always use the same bucket) - 5. create a check file - 6. get the hash of the file - 7. PUT the file - 8. GET the file - 9. compare hashes - 10. delete object on bucket - 11. update CR health status check - - Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive - */ - - // Keep admin ops context up-to date if there are config changes - if err := UpdateEndpoint(&c.objContext.Context, c.objectStoreSpec); err != nil { - return errors.Wrapf(err, "failed to parse updated CephObjectStore spec") - } - - // Generate unique user and bucket name - bucketName := genHealthCheckerBucketName(c.objContext.UID) - userConfig := genUserCheckerConfig(c.objContext.UID) - - // Create checker user - logger.Debugf("creating s3 user object %q for object store %q health check", userConfig.ID, c.namespacedName.Name) - var user admin.User - user, err := c.objContext.AdminOpsClient.GetUser(c.objContext.clusterInfo.Context, userConfig) - if err != nil { - if errors.Is(err, admin.ErrNoSuchUser) { - user, err = c.objContext.AdminOpsClient.CreateUser(c.objContext.clusterInfo.Context, userConfig) - if err != nil { - return errors.Wrapf(err, "failed to create from ceph object user %v", userConfig.ID) - } - } else { - return errors.Wrapf(err, "failed to get details from ceph object user %q", userConfig.ID) - } - } - - // Set access and secret key - s3endpoint := c.objContext.Endpoint - s3AccessKey := user.Keys[0].AccessKey - s3SecretKey := user.Keys[0].SecretKey - - // Initiate s3 agent - logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name) - s3client, err := NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false) - if err != nil { - return errors.Wrap(err, "failed to initialize s3 connection") - } - - // Force purge the s3 object before starting anything - cleanupObjectHealthCheck(s3client, c.objContext.UID) - - // Bucket health test - err = c.testBucketHealth(s3client, bucketName) - if err != nil { - return errors.Wrapf(err, "failed to run bucket health checks for object store %q", c.namespacedName.String()) - } - - logger.Debugf("successfully checked object store endpoint for object store %q", c.namespacedName.String()) - - // Update the EndpointStatus in the CR to reflect the healthiness - updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionConnected, "") - - return nil -} - -func cleanupObjectHealthCheck(s3client *S3Agent, objectStoreUID string) { - bucketToDelete := genHealthCheckerBucketName(objectStoreUID) - logger.Debugf("deleting object %q from bucket %q", s3HealthCheckObjectKey, bucketToDelete) - _, err := s3client.DeleteObjectInBucket(bucketToDelete, s3HealthCheckObjectKey) - if err != nil { - logger.Errorf("failed to delete object in bucket. %v", err) - } -} - -func (c *bucketChecker) cleanupHealthCheck() { - bucketToDelete := genHealthCheckerBucketName(c.objContext.UID) - logger.Infof("deleting object %q from bucket %q in object store %q", s3HealthCheckObjectKey, bucketToDelete, c.namespacedName.Name) - - thePurge := true - err := c.objContext.AdminOpsClient.RemoveBucket(c.objContext.clusterInfo.Context, admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge}) - if err != nil { - if errors.Is(err, admin.ErrNoSuchBucket) { - // opinion: "not found" is not an error - logger.Debugf("bucket %q does not exist", bucketToDelete) - } else { - logger.Errorf("failed to delete bucket %q for object store %q. %v", bucketToDelete, c.namespacedName.Name, err) - } - } - - userToDelete := genUserCheckerConfig(c.objContext.UID) - err = c.objContext.AdminOpsClient.RemoveUser(c.objContext.clusterInfo.Context, userToDelete) - if err != nil && !errors.Is(err, admin.ErrNoSuchUser) { - logger.Errorf("failed to delete object user %q for object store %q. %v", userToDelete.ID, c.namespacedName.Name, err) - } - - logger.Debugf("successfully deleted object user %q for object store %q", userToDelete.ID, c.namespacedName.Name) -} - -func toCustomResourceStatus(currentStatus *cephv1.BucketStatus, details string, health cephv1.ConditionType) *cephv1.BucketStatus { - s := &cephv1.BucketStatus{ - Health: health, - LastChecked: time.Now().UTC().Format(time.RFC3339), - Details: details, - } - - if currentStatus != nil { - s.LastChanged = currentStatus.LastChanged - if currentStatus.Details != s.Details { - s.LastChanged = s.LastChecked - } - } - return s -} - -func genHealthCheckerBucketName(uuid string) string { - return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid) -} - -func genUserCheckerConfig(cephObjectStoreUID string) admin.User { - userName := fmt.Sprintf("%s-%s", s3UserHealthCheckName, cephObjectStoreUID) - - return admin.User{ - ID: userName, - DisplayName: userName, - } -} - -func (c *bucketChecker) testBucketHealth(s3client *S3Agent, bucket string) error { - // Purge on exit - defer cleanupObjectHealthCheck(s3client, c.objContext.UID) - - // Create S3 bucket - logger.Debugf("creating bucket %q", bucket) - err := s3client.CreateBucketNoInfoLogging(bucket) - if err != nil { - return errors.Wrapf(err, "failed to create bucket %q for object store %q", bucket, c.namespacedName.Name) - } - - // Put an object into the bucket - logger.Debugf("putting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - _, err = s3client.PutObjectInBucket(bucket, string(s3HealthCheckObjectBody), s3HealthCheckObjectKey, contentType) - if err != nil { - return errors.Wrapf(err, "failed to put object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - } - - // Get the object from the bucket - logger.Debugf("getting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - read, err := s3client.GetObjectInBucket(bucket, s3HealthCheckObjectKey) - if err != nil { - return errors.Wrapf(err, "failed to get object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name) - } - - // Compare the old and the existing object - logger.Debugf("comparing objects hash for object store %q", c.namespacedName.Name) - oldHash := k8sutil.Hash(s3HealthCheckObjectBody) - currentHash := k8sutil.Hash(read) - if currentHash != oldHash { - return errors.Wrapf(err, "wrong file content, old file hash is %q and new one is %q for object store %q", oldHash, currentHash, c.namespacedName.Name) - } - - return nil -} diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 11bb15600..709351553 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -391,7 +391,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, } // configureReadinessProbe returns the desired readiness probe for a given daemon -func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) { +func configureReadinessProbe(container *v1.Container, healthCheck cephv1.ObjectHealthCheckSpec) { if ok := healthCheck.ReadinessProbe; ok != nil { if !healthCheck.ReadinessProbe.Disabled { probe := healthCheck.ReadinessProbe.Probe diff --git a/pkg/operator/ceph/object/status.go b/pkg/operator/ceph/object/status.go index ee34182c9..cfcd9178b 100644 --- a/pkg/operator/ceph/object/status.go +++ b/pkg/operator/ceph/object/status.go @@ -75,42 +75,6 @@ func updateStatus(ctx context.Context, observedGeneration int64, client client.C logger.Debugf("object store %q status updated to %q", namespacedName.String(), status) } -// updateStatusBucket updates an object with a given status -func updateStatusBucket(ctx context.Context, client client.Client, name types.NamespacedName, status cephv1.ConditionType, details string) { - // Updating the status is important to users, but we can still keep operating if there is a - // failure. Retry a few times to give it our best effort attempt. - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - objectStore := &cephv1.CephObjectStore{} - if err := client.Get(ctx, name, objectStore); err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("CephObjectStore resource not found. Ignoring since object must be deleted.") - return nil - } - return errors.Wrapf(err, "failed to retrieve object store %q to update status to %v", name.String(), status) - } - if objectStore.Status == nil { - objectStore.Status = &cephv1.ObjectStoreStatus{} - } - objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, status) - - // do not transition to other statuses once deletion begins - if objectStore.Status.Phase != cephv1.ConditionDeleting { - objectStore.Status.Phase = status - } - - // but we still need to update the health checker status - if err := reporting.UpdateStatus(client, objectStore); err != nil { - return errors.Wrapf(err, "failed to set object store %q status to %v", name.String(), status) - } - return nil - }) - if err != nil { - logger.Error(err) - } - - logger.Debugf("object store %q status updated to %v", name.String(), status) -} - func buildStatusInfo(cephObjectStore *cephv1.CephObjectStore) map[string]string { m := make(map[string]string) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index cf69074e6..a1dd441d2 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -473,10 +473,6 @@ spec: securePort: ` + strconv.Itoa(port) + ` instances: ` + strconv.Itoa(replicaCount) + ` sslCertificateRef: ` + name + ` - healthCheck: - bucket: - disabled: false - interval: 10s ` } return `apiVersion: ceph.rook.io/v1 @@ -498,10 +494,6 @@ spec: resources: null port: ` + strconv.Itoa(port) + ` instances: ` + strconv.Itoa(replicaCount) + ` - healthCheck: - bucket: - disabled: false - interval: 5s ` } diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index deecbe5f1..37216bbfa 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -36,6 +36,7 @@ import ( "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -116,18 +117,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils for i = 0; i < retryCount; i++ { objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{}) assert.Nil(t, err) - if objectStore.Status == nil || objectStore.Status.BucketStatus == nil { + // TODO: check that object store status is good, and also check that status of + // deployment is good based on health checks + + if objectStore.Status == nil { logger.Infof("(%d) object status check sleeping for 5 seconds ...%+v", i, objectStore.Status) time.Sleep(5 * time.Second) continue } logger.Info("objectstore status is", objectStore.Status) - if objectStore.Status.BucketStatus.Health == cephv1.ConditionFailure { - logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.BucketStatus) + // ConditionConnected supports Rook v1.10 clusters that still had health check + // TODO: remove that half of check after Rook v1.12 release + if objectStore.Status.Phase != cephv1.ConditionReady && objectStore.Status.Phase != cephv1.ConditionConnected { + logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.Phase) time.Sleep(5 * time.Second) continue } - assert.Equal(t, cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health) + // Info field has the endpoint in it assert.NotEmpty(t, objectStore.Status.Info) assert.NotEmpty(t, objectStore.Status.Info["endpoint"]) @@ -138,6 +144,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils } }) + t.Run("verify RGW liveness probes show healthy", func(t *testing.T) { + err := wait.PollImmediate(2*time.Second, 90*time.Second, func() (done bool, err error) { + deployName := "rook-ceph-rgw-" + storeName + "-a" + d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, deployName, metav1.GetOptions{}) + if err != nil { + logger.Infof("waiting for rgw deployment %q to be ready; failed to get deployment: %v", deployName, err) + return false, nil + } + if d.Status.UnavailableReplicas != 0 { + logger.Infof("waiting rgw deployment %q to be ready; %d replicas are unavailable", deployName, d.Status.UnavailableReplicas) + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + }) + t.Run("verify RGW service is up", func(t *testing.T) { assert.True(t, k8sh.IsServiceUp("rook-ceph-rgw-"+storeName, namespace)) })