Merge pull request #11124 from BlaineEXE/object-revise-health-check
object: remove health checker
This commit is contained in:
18 files changed
+198
-644
No files matched your search
@@ -162,9 +162,6 @@ Rook will be default monitor the state of the object store endpoints.
|
||||
The following CRD settings are available:
|
||||
|
||||
* `healthCheck`: main object store health monitoring section
|
||||
* `bucket`: Rook checks that the object store is usable regularly. This is explained in more
|
||||
detail below. Use this config to disable or change the interval at which Rook verifies the
|
||||
object store connectivity.
|
||||
* `startupProbe`: Disable, or override timing and threshold values of the object gateway startup probe.
|
||||
* `livenessProbe`: Disable, or override timing and threshold values of the object gateway liveness probe.
|
||||
* `readinessProbe`: Disable, or override timing and threshold values of the object gateway readiness probe.
|
||||
@@ -173,31 +170,21 @@ Here is a complete example:
|
||||
|
||||
```yaml
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 60s
|
||||
startupProbe:
|
||||
disabled: false
|
||||
livenessProbe:
|
||||
disabled: false
|
||||
periodSeconds: 5
|
||||
failureThreshold: 4
|
||||
periodSeconds: 30
|
||||
failureThreshold: 13
|
||||
readinessProbe:
|
||||
disabled: false
|
||||
periodSeconds: 5
|
||||
failureThreshold: 2
|
||||
```
|
||||
|
||||
The endpoint health check procedure is the following:
|
||||
|
||||
1. Create an S3 user
|
||||
2. Create a bucket with that user
|
||||
3. PUT the file in the object store
|
||||
4. GET the file from the object store
|
||||
5. Verify object consistency
|
||||
6. Update CR health status check
|
||||
|
||||
The Rook Ceph operator always keeps the bucket and the user for the health check, it just does a PUT and GET of an s3 object since creating a bucket is an expensive operation.
|
||||
You can monitor the health of a CephObjectStore by monitoring the gateway deployments it creates.
|
||||
The primary deployment created is named `rook-ceph-rgw-<store-name>-a` where `store-name` is the
|
||||
name of the CephObjectStore (don't forget the `-a` at the end).
|
||||
|
||||
## Security settings
|
||||
|
||||
|
||||
@@ -48,10 +48,6 @@ spec:
|
||||
port: 80
|
||||
# securePort: 443
|
||||
instances: 1
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 60s
|
||||
```
|
||||
|
||||
After the `CephObjectStore` is created, the Rook operator will then create all the pools and other resources necessary to start the service. This may take a minute to complete.
|
||||
@@ -84,10 +80,6 @@ spec:
|
||||
port: 8080
|
||||
externalRgwEndpoints:
|
||||
- ip: 192.168.39.182
|
||||
healthCheck:
|
||||
bucket:
|
||||
enabled: true
|
||||
interval: 60s
|
||||
```
|
||||
|
||||
You can use the existing `object-external.yaml` file.
|
||||
@@ -249,6 +241,12 @@ s5cmd --endpoint-url http://$AWS_HOST:$PORT cp s3://$BUCKET_NAME/rookObj /tmp/ro
|
||||
cat /tmp/rookObj-download
|
||||
```
|
||||
|
||||
## Monitoring health
|
||||
|
||||
Rook configures health probes on the deployment created for CephObjectStore gateways. Refer to
|
||||
[the CRD document](../../CRDs/Object-Storage/ceph-object-store-crd.md#health-settings) for
|
||||
information about configuring the probes and monitoring the deployment status.
|
||||
|
||||
## Access External to the Cluster
|
||||
|
||||
Rook sets up the object storage so pods will have access internal to the cluster. If your applications are running outside the cluster,
|
||||
|
||||
@@ -532,9 +532,6 @@ cephObjectStores:
|
||||
# sslCertificateRef:
|
||||
instances: 1
|
||||
priorityClassName: system-cluster-critical
|
||||
healthCheck:
|
||||
bucket:
|
||||
interval: 60s
|
||||
storageClass:
|
||||
enabled: true
|
||||
name: ceph-bucket
|
||||
|
||||
@@ -12231,20 +12231,9 @@ spec:
|
||||
type: string
|
||||
type: object
|
||||
healthCheck:
|
||||
description: The rgw Bucket healthchecks and liveness probe
|
||||
description: The RGW health probes
|
||||
nullable: true
|
||||
properties:
|
||||
bucket:
|
||||
description: HealthCheckSpec represents the health check of an object store bucket
|
||||
properties:
|
||||
disabled:
|
||||
type: boolean
|
||||
interval:
|
||||
description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds
|
||||
type: string
|
||||
timeout:
|
||||
type: string
|
||||
type: object
|
||||
livenessProbe:
|
||||
description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
|
||||
properties:
|
||||
@@ -12788,19 +12777,6 @@ spec:
|
||||
status:
|
||||
description: ObjectStoreStatus represents the status of a Ceph Object Store resource
|
||||
properties:
|
||||
bucketStatus:
|
||||
description: BucketStatus represents the status of a bucket
|
||||
properties:
|
||||
details:
|
||||
type: string
|
||||
health:
|
||||
description: ConditionType represent a resource's status
|
||||
type: string
|
||||
lastChanged:
|
||||
type: string
|
||||
lastChecked:
|
||||
type: string
|
||||
type: object
|
||||
conditions:
|
||||
items:
|
||||
description: Condition represents a status condition on any Rook-Ceph Custom Resource.
|
||||
|
||||
@@ -12223,20 +12223,9 @@ spec:
|
||||
type: string
|
||||
type: object
|
||||
healthCheck:
|
||||
description: The rgw Bucket healthchecks and liveness probe
|
||||
description: The RGW health probes
|
||||
nullable: true
|
||||
properties:
|
||||
bucket:
|
||||
description: HealthCheckSpec represents the health check of an object store bucket
|
||||
properties:
|
||||
disabled:
|
||||
type: boolean
|
||||
interval:
|
||||
description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds
|
||||
type: string
|
||||
timeout:
|
||||
type: string
|
||||
type: object
|
||||
livenessProbe:
|
||||
description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
|
||||
properties:
|
||||
@@ -12780,19 +12769,6 @@ spec:
|
||||
status:
|
||||
description: ObjectStoreStatus represents the status of a Ceph Object Store resource
|
||||
properties:
|
||||
bucketStatus:
|
||||
description: BucketStatus represents the status of a bucket
|
||||
properties:
|
||||
details:
|
||||
type: string
|
||||
health:
|
||||
description: ConditionType represent a resource's status
|
||||
type: string
|
||||
lastChanged:
|
||||
type: string
|
||||
lastChecked:
|
||||
type: string
|
||||
type: object
|
||||
conditions:
|
||||
items:
|
||||
description: Condition represents a status condition on any Rook-Ceph Custom Resource.
|
||||
|
||||
@@ -81,12 +81,9 @@ spec:
|
||||
# memory: "1024Mi"
|
||||
# priorityClassName: my-priority-class
|
||||
#zone:
|
||||
#name: zone-a
|
||||
#name: zone-a
|
||||
# service endpoint healthcheck
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 60s
|
||||
# Configure the pod probes for the rgw daemon
|
||||
livenessProbe:
|
||||
disabled: false
|
||||
|
||||
@@ -98,9 +98,6 @@ spec:
|
||||
# service.beta.openshift.io/serving-cert-secret-name: <name of TLS secret automatic generation>
|
||||
# service endpoint healthcheck
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 60s
|
||||
# Configure the pod probes for the rgw daemon
|
||||
livenessProbe:
|
||||
disabled: false
|
||||
@@ -149,12 +146,12 @@ metadata:
|
||||
spec:
|
||||
port:
|
||||
targetPort: http
|
||||
# if TLS is enabled, remove the above http port and uncomment the https port and tls settings
|
||||
# port:
|
||||
# targetPort: https
|
||||
# tls:
|
||||
# insecureEdgeTerminationPolicy: Redirect
|
||||
# termination: reencrypt
|
||||
# if TLS is enabled, remove the above http port and uncomment the https port and tls settings
|
||||
# port:
|
||||
# targetPort: https
|
||||
# tls:
|
||||
# insecureEdgeTerminationPolicy: Redirect
|
||||
# termination: reencrypt
|
||||
to:
|
||||
kind: Service
|
||||
name: rook-ceph-rgw-my-store # The name of the RGW service is in the form 'rook-ceph-rgw-<objectstore-name>'
|
||||
@@ -101,12 +101,9 @@ spec:
|
||||
# memory: "1024Mi"
|
||||
priorityClassName: system-cluster-critical
|
||||
#zone:
|
||||
#name: zone-a
|
||||
#name: zone-a
|
||||
# service endpoint healthcheck
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 60s
|
||||
# Configure the pod probes for the rgw daemon
|
||||
startupProbe:
|
||||
disabled: false
|
||||
|
||||
@@ -1335,10 +1335,10 @@ type ObjectStoreSpec struct {
|
||||
// +nullable
|
||||
Zone ZoneSpec `json:"zone,omitempty"`
|
||||
|
||||
// The rgw Bucket healthchecks and liveness probe
|
||||
// The RGW health probes
|
||||
// +optional
|
||||
// +nullable
|
||||
HealthCheck BucketHealthCheckSpec `json:"healthCheck,omitempty"`
|
||||
HealthCheck ObjectHealthCheckSpec `json:"healthCheck,omitempty"`
|
||||
|
||||
// Security represents security settings
|
||||
// +optional
|
||||
@@ -1346,10 +1346,8 @@ type ObjectStoreSpec struct {
|
||||
Security *ObjectStoreSecuritySpec `json:"security,omitempty"`
|
||||
}
|
||||
|
||||
// BucketHealthCheckSpec represents the health check of an object store
|
||||
type BucketHealthCheckSpec struct {
|
||||
// +optional
|
||||
Bucket HealthCheckSpec `json:"bucket,omitempty"`
|
||||
// ObjectHealthCheckSpec represents the health check of an object store
|
||||
type ObjectHealthCheckSpec struct {
|
||||
// +optional
|
||||
LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
|
||||
// +optional
|
||||
@@ -1455,8 +1453,6 @@ type ObjectStoreStatus struct {
|
||||
// +optional
|
||||
Message string `json:"message,omitempty"`
|
||||
// +optional
|
||||
BucketStatus *BucketStatus `json:"bucketStatus,omitempty"`
|
||||
// +optional
|
||||
// +nullable
|
||||
Info map[string]string `json:"info,omitempty"`
|
||||
Conditions []Condition `json:"conditions,omitempty"`
|
||||
@@ -1465,18 +1461,6 @@ type ObjectStoreStatus struct {
|
||||
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
|
||||
}
|
||||
|
||||
// BucketStatus represents the status of a bucket
|
||||
type BucketStatus struct {
|
||||
// +optional
|
||||
Health ConditionType `json:"health,omitempty"`
|
||||
// +optional
|
||||
Details string `json:"details,omitempty"`
|
||||
// +optional
|
||||
LastChecked string `json:"lastChecked,omitempty"`
|
||||
// +optional
|
||||
LastChanged string `json:"lastChanged,omitempty"`
|
||||
}
|
||||
|
||||
// CephObjectStoreUser represents a Ceph Object Store Gateway User
|
||||
// +genclient
|
||||
// +genclient:noStatus
|
||||
|
||||
@@ -97,38 +97,6 @@ func (in AnnotationsSpec) DeepCopy() AnnotationsSpec {
|
||||
return *out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) {
|
||||
*out = *in
|
||||
in.Bucket.DeepCopyInto(&out.Bucket)
|
||||
if in.LivenessProbe != nil {
|
||||
in, out := &in.LivenessProbe, &out.LivenessProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
if in.ReadinessProbe != nil {
|
||||
in, out := &in.ReadinessProbe, &out.ReadinessProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
if in.StartupProbe != nil {
|
||||
in, out := &in.StartupProbe, &out.StartupProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketHealthCheckSpec.
|
||||
func (in *BucketHealthCheckSpec) DeepCopy() *BucketHealthCheckSpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(BucketHealthCheckSpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *BucketNotificationSpec) DeepCopyInto(out *BucketNotificationSpec) {
|
||||
*out = *in
|
||||
@@ -155,22 +123,6 @@ func (in *BucketNotificationSpec) DeepCopy() *BucketNotificationSpec {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *BucketStatus) DeepCopyInto(out *BucketStatus) {
|
||||
*out = *in
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketStatus.
|
||||
func (in *BucketStatus) DeepCopy() *BucketStatus {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(BucketStatus)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *BucketTopicSpec) DeepCopyInto(out *BucketTopicSpec) {
|
||||
*out = *in
|
||||
@@ -3059,6 +3011,37 @@ func (in *NotificationKeyFilterRule) DeepCopy() *NotificationKeyFilterRule {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ObjectHealthCheckSpec) DeepCopyInto(out *ObjectHealthCheckSpec) {
|
||||
*out = *in
|
||||
if in.LivenessProbe != nil {
|
||||
in, out := &in.LivenessProbe, &out.LivenessProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
if in.ReadinessProbe != nil {
|
||||
in, out := &in.ReadinessProbe, &out.ReadinessProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
if in.StartupProbe != nil {
|
||||
in, out := &in.StartupProbe, &out.StartupProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectHealthCheckSpec.
|
||||
func (in *ObjectHealthCheckSpec) DeepCopy() *ObjectHealthCheckSpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(ObjectHealthCheckSpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ObjectRealmSpec) DeepCopyInto(out *ObjectRealmSpec) {
|
||||
*out = *in
|
||||
@@ -3123,11 +3106,6 @@ func (in *ObjectStoreSpec) DeepCopy() *ObjectStoreSpec {
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ObjectStoreStatus) DeepCopyInto(out *ObjectStoreStatus) {
|
||||
*out = *in
|
||||
if in.BucketStatus != nil {
|
||||
in, out := &in.BucketStatus, &out.BucketStatus
|
||||
*out = new(BucketStatus)
|
||||
**out = **in
|
||||
}
|
||||
if in.Info != nil {
|
||||
in, out := &in.Info, &out.Info
|
||||
*out = make(map[string]string, len(*in))
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ceph/go-ceph/rgw/admin"
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned"
|
||||
"github.com/pkg/errors"
|
||||
@@ -80,22 +81,15 @@ var cephObjectStoreDependents = CephObjectStoreDependents
|
||||
|
||||
// ReconcileCephObjectStore reconciles a cephObjectStore object
|
||||
type ReconcileCephObjectStore struct {
|
||||
client client.Client
|
||||
bktclient bktclient.Interface
|
||||
scheme *runtime.Scheme
|
||||
context *clusterd.Context
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
objectStoreContexts map[string]*objectStoreHealth
|
||||
recorder record.EventRecorder
|
||||
opManagerContext context.Context
|
||||
opConfig opcontroller.OperatorConfig
|
||||
}
|
||||
|
||||
type objectStoreHealth struct {
|
||||
internalCtx context.Context
|
||||
internalCancel context.CancelFunc
|
||||
started bool
|
||||
client client.Client
|
||||
bktclient bktclient.Interface
|
||||
scheme *runtime.Scheme
|
||||
context *clusterd.Context
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
recorder record.EventRecorder
|
||||
opManagerContext context.Context
|
||||
opConfig opcontroller.OperatorConfig
|
||||
}
|
||||
|
||||
// Add creates a new cephObjectStore Controller and adds it to the Manager. The Manager will set fields on the Controller
|
||||
@@ -108,14 +102,13 @@ func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext contex
|
||||
func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
|
||||
context.Client = mgr.GetClient()
|
||||
return &ReconcileCephObjectStore{
|
||||
client: mgr.GetClient(),
|
||||
scheme: mgr.GetScheme(),
|
||||
context: context,
|
||||
bktclient: bktclient.NewForConfigOrDie(context.KubeConfig),
|
||||
objectStoreContexts: make(map[string]*objectStoreHealth),
|
||||
recorder: mgr.GetEventRecorderFor("rook-" + controllerName),
|
||||
opManagerContext: opManagerContext,
|
||||
opConfig: opConfig,
|
||||
client: mgr.GetClient(),
|
||||
scheme: mgr.GetScheme(),
|
||||
context: context,
|
||||
bktclient: bktclient.NewForConfigOrDie(context.KubeConfig),
|
||||
recorder: mgr.GetEventRecorderFor("rook-" + controllerName),
|
||||
opManagerContext: opManagerContext,
|
||||
opConfig: opConfig,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,12 +159,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("cephObjectStore resource not found. Ignoring since object must be deleted.")
|
||||
// If there was a previous error or if a user removed this resource's finalizer, it's
|
||||
// possible Rook didn't clean up the monitoring routine for this resource. Ensure the
|
||||
// routine is stopped when we see the resource is gone.
|
||||
cephObjectStore.Name = request.Name
|
||||
cephObjectStore.Namespace = request.Namespace
|
||||
r.stopMonitoring(cephObjectStore)
|
||||
return reconcile.Result{}, *cephObjectStore, nil
|
||||
}
|
||||
// Error reading the object - requeue the request.
|
||||
@@ -204,9 +191,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
// If not, we should wait for it to be ready
|
||||
// This handles the case where the operator is not ready to accept Ceph command but the cluster exists
|
||||
if !cephObjectStore.GetDeletionTimestamp().IsZero() && !cephClusterExists {
|
||||
// don't leak the health checker routine if we are force deleting
|
||||
r.stopMonitoring(cephObjectStore)
|
||||
|
||||
// Remove finalizer
|
||||
err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephObjectStore)
|
||||
if err != nil {
|
||||
@@ -221,17 +205,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
}
|
||||
r.clusterSpec = &cephCluster.Spec
|
||||
|
||||
// Initialize the channel for this object store
|
||||
// This allows us to track multiple ObjectStores in the same namespace
|
||||
_, ok := r.objectStoreContexts[monitoringChannelKey(cephObjectStore)]
|
||||
if !ok {
|
||||
internalCtx, internalCancel := context.WithCancel(r.opManagerContext)
|
||||
r.objectStoreContexts[monitoringChannelKey(cephObjectStore)] = &objectStoreHealth{
|
||||
internalCtx: internalCtx,
|
||||
internalCancel: internalCancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Populate clusterInfo during each reconcile
|
||||
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.clusterSpec)
|
||||
if err != nil {
|
||||
@@ -257,12 +230,18 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
}
|
||||
objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get object context")
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context")
|
||||
}
|
||||
opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get admin ops API context")
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context")
|
||||
}
|
||||
|
||||
// ignore any errors coming from this for the deletion case since we could get into a
|
||||
// partially-deleted case where RGWs aren't responding. Deletion will be blocked later if
|
||||
// the bucket fails to be deleted and the RGWs are running.
|
||||
_ = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore)
|
||||
|
||||
deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephObjectStore, err
|
||||
@@ -273,9 +252,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
}
|
||||
reporting.ReportDeletionNotBlockedDueToDependents(r.opManagerContext, logger, r.client, r.recorder, cephObjectStore)
|
||||
|
||||
// Cancel the context to stop monitoring the health of the object store
|
||||
r.stopMonitoring(cephObjectStore)
|
||||
|
||||
cfg := clusterConfig{
|
||||
context: r.context,
|
||||
store: cephObjectStore,
|
||||
@@ -349,9 +325,22 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
return result, *cephObjectStore, err
|
||||
}
|
||||
|
||||
objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context")
|
||||
}
|
||||
opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context")
|
||||
}
|
||||
err = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephObjectStore, errors.Wrap(err, "updated object store but failed to remove deprecated health check bucket")
|
||||
}
|
||||
|
||||
// update ObservedGeneration in status at the end of reconcile
|
||||
// Set Progressing status, we are done reconciling, the health check go routine will update the status
|
||||
updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore))
|
||||
updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore))
|
||||
|
||||
// Return and do not requeue
|
||||
logger.Debug("done reconciling")
|
||||
@@ -455,14 +444,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
}
|
||||
}
|
||||
|
||||
// Start monitoring
|
||||
if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled {
|
||||
err = r.startMonitoring(cephObjectStore, objContext, namespacedName)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
@@ -525,43 +506,40 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
|
||||
return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func monitoringChannelKey(o *cephv1.CephObjectStore) string {
|
||||
return types.NamespacedName{Namespace: o.Namespace, Name: o.Name}.String()
|
||||
}
|
||||
// handle upgrade from v1.10; we need to remove the health checker bucket from the store, or
|
||||
// deleting the store will be stuck on dependents because an unknown bucket exists
|
||||
// TODO: remove this for Rook v1.12 release
|
||||
var removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
|
||||
healthCheckBucket := genHealthCheckerBucketName(string(cos.UID))
|
||||
doPurge := true // purge all content of the bucket
|
||||
bucket := admin.Bucket{Bucket: healthCheckBucket, PurgeObject: &doPurge}
|
||||
remErr := opsCtx.AdminOpsClient.RemoveBucket(ctx, bucket)
|
||||
|
||||
func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) error {
|
||||
channelKey := monitoringChannelKey(objectstore)
|
||||
|
||||
// Start monitoring object store
|
||||
if r.objectStoreContexts[channelKey].started {
|
||||
logger.Info("external rgw endpoint monitoring go routine already running!")
|
||||
if remErr == nil {
|
||||
logger.Info("successfully deleted deprecated health checker bucket")
|
||||
return nil
|
||||
}
|
||||
|
||||
rgwChecker, err := newBucketChecker(r.context, objContext, r.client, namespacedName, &objectstore.Spec)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to start rgw health checker for CephObjectStore %q, will re-reconcile", namespacedName.String())
|
||||
if errors.Is(remErr, admin.ErrNoSuchBucket) {
|
||||
logger.Debug("deprecated health checker bucket already does not exist: NoSuchBucket")
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("starting rgw health checker for CephObjectStore %q", namespacedName.String())
|
||||
go rgwChecker.checkObjectStore(r.objectStoreContexts, channelKey)
|
||||
|
||||
// Set the monitoring flag so we don't start more than one go routine
|
||||
r.objectStoreContexts[channelKey].started = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancel monitoring. This is a noop if monitoring is not running.
|
||||
func (r *ReconcileCephObjectStore) stopMonitoring(objectstore *cephv1.CephObjectStore) {
|
||||
channelKey := monitoringChannelKey(objectstore)
|
||||
|
||||
_, monitoringContextExists := r.objectStoreContexts[channelKey]
|
||||
if monitoringContextExists {
|
||||
// stop the monitoring routine
|
||||
r.objectStoreContexts[channelKey].internalCancel()
|
||||
|
||||
// remove the monitoring routine from the map
|
||||
delete(r.objectStoreContexts, channelKey)
|
||||
if errors.Is(remErr, admin.ErrNoSuchKey) {
|
||||
// ceph might return NoSuchKey than NoSuchBucket when the target bucket does not exist.
|
||||
// then we can use GetBucketInfo() to judge the existence of the bucket.
|
||||
// see: https://github.com/ceph/ceph/pull/44413
|
||||
_, getErr := opsCtx.AdminOpsClient.GetBucketInfo(ctx, bucket)
|
||||
if getErr != nil {
|
||||
if errors.Is(getErr, admin.ErrNoSuchBucket) {
|
||||
logger.Debug("deprecated health checker bucket already does not exist: NoSuchKey")
|
||||
return nil
|
||||
}
|
||||
// both commands errored; something is up; be sure to return both errors
|
||||
return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q. failed to get bucket info. %v", healthCheckBucket, getErr)
|
||||
}
|
||||
// remove bucket failed, and bucket exists; something else went wrong with remove bucket
|
||||
}
|
||||
|
||||
return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q", healthCheckBucket)
|
||||
}
|
||||
@@ -19,7 +19,7 @@ package object
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
@@ -383,12 +383,11 @@ func TestCephObjectStoreController(t *testing.T) {
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
|
||||
// Create a ReconcileCephObjectStore object with the scheme and fake client.
|
||||
r := &ReconcileCephObjectStore{
|
||||
client: cl,
|
||||
scheme: s,
|
||||
context: c,
|
||||
objectStoreContexts: make(map[string]*objectStoreHealth),
|
||||
recorder: record.NewFakeRecorder(5),
|
||||
opManagerContext: context.TODO(),
|
||||
client: cl,
|
||||
scheme: s,
|
||||
context: c,
|
||||
recorder: record.NewFakeRecorder(5),
|
||||
opManagerContext: context.TODO(),
|
||||
}
|
||||
|
||||
return r
|
||||
@@ -526,29 +525,13 @@ func TestCephObjectStoreController(t *testing.T) {
|
||||
return r
|
||||
}
|
||||
|
||||
t.Run("error - failed to start health checker", func(t *testing.T) {
|
||||
r := setupEnvironmentWithReadyCephCluster()
|
||||
|
||||
// cause a failure when creating the admin ops api for the health check
|
||||
origHTTPClientFunc := genObjectStoreHTTPClientFunc
|
||||
genObjectStoreHTTPClientFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (client *http.Client, tlsCert []byte, err error) {
|
||||
return nil, []byte{}, errors.New("induced error creating admin ops API connection")
|
||||
}
|
||||
defer func() { genObjectStoreHTTPClientFunc = origHTTPClientFunc }()
|
||||
|
||||
_, err := r.Reconcile(ctx, req)
|
||||
assert.Error(t, err)
|
||||
// we don't actually care if Requeue is true if there is an error assert.True(t, res.Requeue)
|
||||
assert.Contains(t, err.Error(), "failed to start rgw health checker")
|
||||
assert.Contains(t, err.Error(), "induced error creating admin ops API connection")
|
||||
|
||||
// health checker should start up after committing config changes
|
||||
assert.True(t, calledCommitConfigChanges)
|
||||
})
|
||||
|
||||
t.Run("success - object store is running", func(t *testing.T) {
|
||||
r := setupEnvironmentWithReadyCephCluster()
|
||||
|
||||
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, res.Requeue)
|
||||
@@ -556,12 +539,24 @@ func TestCephObjectStoreController(t *testing.T) {
|
||||
objectStore := &cephv1.CephObjectStore{}
|
||||
err = r.client.Get(context.TODO(), req.NamespacedName, objectStore)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore)
|
||||
assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore)
|
||||
assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore)
|
||||
assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore)
|
||||
assert.True(t, calledCommitConfigChanges)
|
||||
assert.Equal(t, 16, r.clusterInfo.CephVersion.Major)
|
||||
})
|
||||
|
||||
t.Run("failed to remove deprecated health check bucket", func(t *testing.T) {
|
||||
r := setupEnvironmentWithReadyCephCluster()
|
||||
|
||||
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
|
||||
return fmt.Errorf("induced error")
|
||||
}
|
||||
|
||||
_, err := r.Reconcile(ctx, req)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "deprecated health check bucket")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCephObjectStoreControllerMultisite(t *testing.T) {
|
||||
@@ -735,12 +730,11 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
|
||||
r := &ReconcileCephObjectStore{
|
||||
client: cl,
|
||||
scheme: s,
|
||||
context: c,
|
||||
objectStoreContexts: make(map[string]*objectStoreHealth),
|
||||
recorder: record.NewFakeRecorder(5),
|
||||
opManagerContext: ctx,
|
||||
client: cl,
|
||||
scheme: s,
|
||||
context: c,
|
||||
recorder: record.NewFakeRecorder(5),
|
||||
opManagerContext: ctx,
|
||||
}
|
||||
|
||||
_, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
||||
@@ -757,6 +751,10 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
|
||||
return &cephver.Pacific, &cephver.Pacific, nil
|
||||
}
|
||||
|
||||
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
t.Run("create an object store", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
@@ -894,12 +892,11 @@ func TestCephObjectExternalStoreController(t *testing.T) {
|
||||
}
|
||||
|
||||
r := &ReconcileCephObjectStore{
|
||||
client: cl,
|
||||
scheme: s,
|
||||
context: c,
|
||||
objectStoreContexts: make(map[string]*objectStoreHealth),
|
||||
recorder: record.NewFakeRecorder(5),
|
||||
opManagerContext: ctx,
|
||||
client: cl,
|
||||
scheme: s,
|
||||
context: c,
|
||||
recorder: record.NewFakeRecorder(5),
|
||||
opManagerContext: ctx,
|
||||
}
|
||||
|
||||
_, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
||||
@@ -931,6 +928,10 @@ func TestCephObjectExternalStoreController(t *testing.T) {
|
||||
|
||||
r := getReconciler(objects)
|
||||
|
||||
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
t.Run("create an external object store", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
const (
|
||||
bucketDependentType = "buckets in the object store (could be from ObjectBucketClaims or COSI Buckets)"
|
||||
zoneIsMasterWithPeersDependentType = "zone is master and has peers"
|
||||
s3HealthCheckBucketName = "rook-ceph-bucket-checker"
|
||||
)
|
||||
|
||||
// CephObjectStoreDependents returns the buckets which exist in the object store that should block
|
||||
@@ -127,11 +128,7 @@ func getBucketDependents(
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to list buckets in CephObjectStore %q", nsName)
|
||||
}
|
||||
healthCheckBucket := genHealthCheckerBucketName(string(store.UID))
|
||||
for _, b := range buckets {
|
||||
if b == healthCheckBucket {
|
||||
continue // don't include the health checker bucket as a blocking dependent
|
||||
}
|
||||
deps.Add(bucketDependentType, b)
|
||||
}
|
||||
|
||||
@@ -165,3 +162,7 @@ func getMasterZoneDependents(
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func genHealthCheckerBucketName(uuid string) string {
|
||||
return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid)
|
||||
}
|
||||
@@ -1,292 +0,0 @@
|
||||
/*
|
||||
Copyright 2020 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package object
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ceph/go-ceph/rgw/admin"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
const (
|
||||
s3UserHealthCheckName = "rook-ceph-internal-s3-user-checker"
|
||||
s3HealthCheckBucketName = "rook-ceph-bucket-checker"
|
||||
s3HealthCheckObjectBody = "Test Rook Object Data"
|
||||
s3HealthCheckObjectKey = "rookHealthCheckTestObject"
|
||||
contentType = "plain/text"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultHealthCheckInterval = 1 * time.Minute
|
||||
)
|
||||
|
||||
// bucketChecker aggregates the mon/cluster info needed to check the health of the monitors
|
||||
type bucketChecker struct {
|
||||
context *clusterd.Context
|
||||
objContext *AdminOpsContext
|
||||
interval *time.Duration
|
||||
port int32
|
||||
client client.Client
|
||||
namespacedName types.NamespacedName
|
||||
objectStoreSpec *cephv1.ObjectStoreSpec
|
||||
}
|
||||
|
||||
// newbucketChecker creates a new HealthChecker object
|
||||
func newBucketChecker(
|
||||
ctx *clusterd.Context, objContext *Context, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec,
|
||||
) (*bucketChecker, error) {
|
||||
port, err := objectStoreSpec.GetPort()
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String())
|
||||
}
|
||||
|
||||
opsCtx, err := NewMultisiteAdminOpsContext(objContext, objectStoreSpec)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String())
|
||||
}
|
||||
|
||||
c := &bucketChecker{
|
||||
context: ctx,
|
||||
objContext: opsCtx,
|
||||
interval: &defaultHealthCheckInterval,
|
||||
port: port,
|
||||
namespacedName: namespacedName,
|
||||
client: client,
|
||||
objectStoreSpec: objectStoreSpec,
|
||||
}
|
||||
|
||||
// allow overriding the check interval
|
||||
checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval
|
||||
if checkInterval != nil {
|
||||
logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String())
|
||||
c.interval = &checkInterval.Duration
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// checkObjectStore periodically checks the health of the cluster
|
||||
func (c *bucketChecker) checkObjectStore(objectStoreContexts map[string]*objectStoreHealth, channelKey string) {
|
||||
// check the object store health immediately before starting the loop
|
||||
err := c.checkObjectStoreHealth()
|
||||
if err != nil {
|
||||
updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error())
|
||||
logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err)
|
||||
}
|
||||
|
||||
for {
|
||||
// We must perform this check otherwise the case will check an index that does not exist anymore and
|
||||
// we will get an invalid pointer error and the go routine will panic
|
||||
if _, ok := objectStoreContexts[channelKey]; !ok {
|
||||
logger.Infof("object store %q has been deleted. stopping monitoring of rgw endpoints", c.namespacedName.Name)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-objectStoreContexts[channelKey].internalCtx.Done():
|
||||
// purge bucket and s3 user
|
||||
// Needed for external mode where in converged everything goes away with the CR deletion
|
||||
c.cleanupHealthCheck()
|
||||
delete(objectStoreContexts, channelKey)
|
||||
logger.Infof("stopping monitoring of rgw endpoints for object store %q", c.namespacedName.Name)
|
||||
return
|
||||
|
||||
case <-time.After(*c.interval):
|
||||
logger.Debugf("checking rgw health of object store %q", c.namespacedName.Name)
|
||||
err := c.checkObjectStoreHealth()
|
||||
if err != nil {
|
||||
updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error())
|
||||
logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *bucketChecker) checkObjectStoreHealth() error {
|
||||
/*
|
||||
0. purge the s3 object by default
|
||||
1. create an S3 user
|
||||
2. always use the same user
|
||||
3. if already exists just re-hydrate the s3 credentials
|
||||
4. create a bucket with that user or use the existing one (always use the same bucket)
|
||||
5. create a check file
|
||||
6. get the hash of the file
|
||||
7. PUT the file
|
||||
8. GET the file
|
||||
9. compare hashes
|
||||
10. delete object on bucket
|
||||
11. update CR health status check
|
||||
|
||||
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
|
||||
*/
|
||||
|
||||
// Keep admin ops context up-to date if there are config changes
|
||||
if err := UpdateEndpoint(&c.objContext.Context, c.objectStoreSpec); err != nil {
|
||||
return errors.Wrapf(err, "failed to parse updated CephObjectStore spec")
|
||||
}
|
||||
|
||||
// Generate unique user and bucket name
|
||||
bucketName := genHealthCheckerBucketName(c.objContext.UID)
|
||||
userConfig := genUserCheckerConfig(c.objContext.UID)
|
||||
|
||||
// Create checker user
|
||||
logger.Debugf("creating s3 user object %q for object store %q health check", userConfig.ID, c.namespacedName.Name)
|
||||
var user admin.User
|
||||
user, err := c.objContext.AdminOpsClient.GetUser(c.objContext.clusterInfo.Context, userConfig)
|
||||
if err != nil {
|
||||
if errors.Is(err, admin.ErrNoSuchUser) {
|
||||
user, err = c.objContext.AdminOpsClient.CreateUser(c.objContext.clusterInfo.Context, userConfig)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create from ceph object user %v", userConfig.ID)
|
||||
}
|
||||
} else {
|
||||
return errors.Wrapf(err, "failed to get details from ceph object user %q", userConfig.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Set access and secret key
|
||||
s3endpoint := c.objContext.Endpoint
|
||||
s3AccessKey := user.Keys[0].AccessKey
|
||||
s3SecretKey := user.Keys[0].SecretKey
|
||||
|
||||
// Initiate s3 agent
|
||||
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
|
||||
s3client, err := NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to initialize s3 connection")
|
||||
}
|
||||
|
||||
// Force purge the s3 object before starting anything
|
||||
cleanupObjectHealthCheck(s3client, c.objContext.UID)
|
||||
|
||||
// Bucket health test
|
||||
err = c.testBucketHealth(s3client, bucketName)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to run bucket health checks for object store %q", c.namespacedName.String())
|
||||
}
|
||||
|
||||
logger.Debugf("successfully checked object store endpoint for object store %q", c.namespacedName.String())
|
||||
|
||||
// Update the EndpointStatus in the CR to reflect the healthiness
|
||||
updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionConnected, "")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupObjectHealthCheck(s3client *S3Agent, objectStoreUID string) {
|
||||
bucketToDelete := genHealthCheckerBucketName(objectStoreUID)
|
||||
logger.Debugf("deleting object %q from bucket %q", s3HealthCheckObjectKey, bucketToDelete)
|
||||
_, err := s3client.DeleteObjectInBucket(bucketToDelete, s3HealthCheckObjectKey)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete object in bucket. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *bucketChecker) cleanupHealthCheck() {
|
||||
bucketToDelete := genHealthCheckerBucketName(c.objContext.UID)
|
||||
logger.Infof("deleting object %q from bucket %q in object store %q", s3HealthCheckObjectKey, bucketToDelete, c.namespacedName.Name)
|
||||
|
||||
thePurge := true
|
||||
err := c.objContext.AdminOpsClient.RemoveBucket(c.objContext.clusterInfo.Context, admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge})
|
||||
if err != nil {
|
||||
if errors.Is(err, admin.ErrNoSuchBucket) {
|
||||
// opinion: "not found" is not an error
|
||||
logger.Debugf("bucket %q does not exist", bucketToDelete)
|
||||
} else {
|
||||
logger.Errorf("failed to delete bucket %q for object store %q. %v", bucketToDelete, c.namespacedName.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
userToDelete := genUserCheckerConfig(c.objContext.UID)
|
||||
err = c.objContext.AdminOpsClient.RemoveUser(c.objContext.clusterInfo.Context, userToDelete)
|
||||
if err != nil && !errors.Is(err, admin.ErrNoSuchUser) {
|
||||
logger.Errorf("failed to delete object user %q for object store %q. %v", userToDelete.ID, c.namespacedName.Name, err)
|
||||
}
|
||||
|
||||
logger.Debugf("successfully deleted object user %q for object store %q", userToDelete.ID, c.namespacedName.Name)
|
||||
}
|
||||
|
||||
func toCustomResourceStatus(currentStatus *cephv1.BucketStatus, details string, health cephv1.ConditionType) *cephv1.BucketStatus {
|
||||
s := &cephv1.BucketStatus{
|
||||
Health: health,
|
||||
LastChecked: time.Now().UTC().Format(time.RFC3339),
|
||||
Details: details,
|
||||
}
|
||||
|
||||
if currentStatus != nil {
|
||||
s.LastChanged = currentStatus.LastChanged
|
||||
if currentStatus.Details != s.Details {
|
||||
s.LastChanged = s.LastChecked
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func genHealthCheckerBucketName(uuid string) string {
|
||||
return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid)
|
||||
}
|
||||
|
||||
func genUserCheckerConfig(cephObjectStoreUID string) admin.User {
|
||||
userName := fmt.Sprintf("%s-%s", s3UserHealthCheckName, cephObjectStoreUID)
|
||||
|
||||
return admin.User{
|
||||
ID: userName,
|
||||
DisplayName: userName,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *bucketChecker) testBucketHealth(s3client *S3Agent, bucket string) error {
|
||||
// Purge on exit
|
||||
defer cleanupObjectHealthCheck(s3client, c.objContext.UID)
|
||||
|
||||
// Create S3 bucket
|
||||
logger.Debugf("creating bucket %q", bucket)
|
||||
err := s3client.CreateBucketNoInfoLogging(bucket)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create bucket %q for object store %q", bucket, c.namespacedName.Name)
|
||||
}
|
||||
|
||||
// Put an object into the bucket
|
||||
logger.Debugf("putting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
|
||||
_, err = s3client.PutObjectInBucket(bucket, string(s3HealthCheckObjectBody), s3HealthCheckObjectKey, contentType)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to put object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
|
||||
}
|
||||
|
||||
// Get the object from the bucket
|
||||
logger.Debugf("getting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
|
||||
read, err := s3client.GetObjectInBucket(bucket, s3HealthCheckObjectKey)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to get object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
|
||||
}
|
||||
|
||||
// Compare the old and the existing object
|
||||
logger.Debugf("comparing objects hash for object store %q", c.namespacedName.Name)
|
||||
oldHash := k8sutil.Hash(s3HealthCheckObjectBody)
|
||||
currentHash := k8sutil.Hash(read)
|
||||
if currentHash != oldHash {
|
||||
return errors.Wrapf(err, "wrong file content, old file hash is %q and new one is %q for object store %q", oldHash, currentHash, c.namespacedName.Name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -391,7 +391,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
|
||||
}
|
||||
|
||||
// configureReadinessProbe returns the desired readiness probe for a given daemon
|
||||
func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) {
|
||||
func configureReadinessProbe(container *v1.Container, healthCheck cephv1.ObjectHealthCheckSpec) {
|
||||
if ok := healthCheck.ReadinessProbe; ok != nil {
|
||||
if !healthCheck.ReadinessProbe.Disabled {
|
||||
probe := healthCheck.ReadinessProbe.Probe
|
||||
|
||||
@@ -75,42 +75,6 @@ func updateStatus(ctx context.Context, observedGeneration int64, client client.C
|
||||
logger.Debugf("object store %q status updated to %q", namespacedName.String(), status)
|
||||
}
|
||||
|
||||
// updateStatusBucket updates an object with a given status
|
||||
func updateStatusBucket(ctx context.Context, client client.Client, name types.NamespacedName, status cephv1.ConditionType, details string) {
|
||||
// Updating the status is important to users, but we can still keep operating if there is a
|
||||
// failure. Retry a few times to give it our best effort attempt.
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
objectStore := &cephv1.CephObjectStore{}
|
||||
if err := client.Get(ctx, name, objectStore); err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
logger.Debug("CephObjectStore resource not found. Ignoring since object must be deleted.")
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "failed to retrieve object store %q to update status to %v", name.String(), status)
|
||||
}
|
||||
if objectStore.Status == nil {
|
||||
objectStore.Status = &cephv1.ObjectStoreStatus{}
|
||||
}
|
||||
objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, status)
|
||||
|
||||
// do not transition to other statuses once deletion begins
|
||||
if objectStore.Status.Phase != cephv1.ConditionDeleting {
|
||||
objectStore.Status.Phase = status
|
||||
}
|
||||
|
||||
// but we still need to update the health checker status
|
||||
if err := reporting.UpdateStatus(client, objectStore); err != nil {
|
||||
return errors.Wrapf(err, "failed to set object store %q status to %v", name.String(), status)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
}
|
||||
|
||||
logger.Debugf("object store %q status updated to %v", name.String(), status)
|
||||
}
|
||||
|
||||
func buildStatusInfo(cephObjectStore *cephv1.CephObjectStore) map[string]string {
|
||||
m := make(map[string]string)
|
||||
|
||||
|
||||
@@ -473,10 +473,6 @@ spec:
|
||||
securePort: ` + strconv.Itoa(port) + `
|
||||
instances: ` + strconv.Itoa(replicaCount) + `
|
||||
sslCertificateRef: ` + name + `
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 10s
|
||||
`
|
||||
}
|
||||
return `apiVersion: ceph.rook.io/v1
|
||||
@@ -498,10 +494,6 @@ spec:
|
||||
resources: null
|
||||
port: ` + strconv.Itoa(port) + `
|
||||
instances: ` + strconv.Itoa(replicaCount) + `
|
||||
healthCheck:
|
||||
bucket:
|
||||
disabled: false
|
||||
interval: 5s
|
||||
`
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -116,18 +117,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils
|
||||
for i = 0; i < retryCount; i++ {
|
||||
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
if objectStore.Status == nil || objectStore.Status.BucketStatus == nil {
|
||||
// TODO: check that object store status is good, and also check that status of
|
||||
// deployment is good based on health checks
|
||||
|
||||
if objectStore.Status == nil {
|
||||
logger.Infof("(%d) object status check sleeping for 5 seconds ...%+v", i, objectStore.Status)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
logger.Info("objectstore status is", objectStore.Status)
|
||||
if objectStore.Status.BucketStatus.Health == cephv1.ConditionFailure {
|
||||
logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.BucketStatus)
|
||||
// ConditionConnected supports Rook v1.10 clusters that still had health check
|
||||
// TODO: remove that half of check after Rook v1.12 release
|
||||
if objectStore.Status.Phase != cephv1.ConditionReady && objectStore.Status.Phase != cephv1.ConditionConnected {
|
||||
logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.Phase)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
assert.Equal(t, cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health)
|
||||
|
||||
// Info field has the endpoint in it
|
||||
assert.NotEmpty(t, objectStore.Status.Info)
|
||||
assert.NotEmpty(t, objectStore.Status.Info["endpoint"])
|
||||
@@ -138,6 +144,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("verify RGW liveness probes show healthy", func(t *testing.T) {
|
||||
err := wait.PollImmediate(2*time.Second, 90*time.Second, func() (done bool, err error) {
|
||||
deployName := "rook-ceph-rgw-" + storeName + "-a"
|
||||
d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, deployName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Infof("waiting for rgw deployment %q to be ready; failed to get deployment: %v", deployName, err)
|
||||
return false, nil
|
||||
}
|
||||
if d.Status.UnavailableReplicas != 0 {
|
||||
logger.Infof("waiting rgw deployment %q to be ready; %d replicas are unavailable", deployName, d.Status.UnavailableReplicas)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("verify RGW service is up", func(t *testing.T) {
|
||||
assert.True(t, k8sh.IsServiceUp("rook-ceph-rgw-"+storeName, namespace))
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user