Merge pull request #11124 from BlaineEXE/object-revise-health-check

object: remove health checker
This commit is contained in:
Blaine Gardner authored and GitHub committed 2022-10-24 11:31:23 -06:00
commit 0937eaee38
18 files changed
+198 -644

No files matched your search

@@ -162,9 +162,6 @@ Rook will be default monitor the state of the object store endpoints.
The following CRD settings are available:
* `healthCheck`: main object store health monitoring section
* `bucket`: Rook checks that the object store is usable regularly. This is explained in more
detail below. Use this config to disable or change the interval at which Rook verifies the
object store connectivity.
* `startupProbe`: Disable, or override timing and threshold values of the object gateway startup probe.
* `livenessProbe`: Disable, or override timing and threshold values of the object gateway liveness probe.
* `readinessProbe`: Disable, or override timing and threshold values of the object gateway readiness probe.
@@ -173,31 +170,21 @@ Here is a complete example:
```yaml
healthCheck:
bucket:
disabled: false
interval: 60s
startupProbe:
disabled: false
livenessProbe:
disabled: false
periodSeconds: 5
failureThreshold: 4
periodSeconds: 30
failureThreshold: 13
readinessProbe:
disabled: false
periodSeconds: 5
failureThreshold: 2
```
The endpoint health check procedure is the following:
1. Create an S3 user
2. Create a bucket with that user
3. PUT the file in the object store
4. GET the file from the object store
5. Verify object consistency
6. Update CR health status check
The Rook Ceph operator always keeps the bucket and the user for the health check, it just does a PUT and GET of an s3 object since creating a bucket is an expensive operation.
You can monitor the health of a CephObjectStore by monitoring the gateway deployments it creates.
The primary deployment created is named `rook-ceph-rgw-<store-name>-a` where `store-name` is the
name of the CephObjectStore (don't forget the `-a` at the end).
## Security settings
@@ -48,10 +48,6 @@ spec:
port: 80
# securePort: 443
instances: 1
healthCheck:
bucket:
disabled: false
interval: 60s
```
After the `CephObjectStore` is created, the Rook operator will then create all the pools and other resources necessary to start the service. This may take a minute to complete.
@@ -84,10 +80,6 @@ spec:
port: 8080
externalRgwEndpoints:
- ip: 192.168.39.182
healthCheck:
bucket:
enabled: true
interval: 60s
```
You can use the existing `object-external.yaml` file.
@@ -249,6 +241,12 @@ s5cmd --endpoint-url http://$AWS_HOST:$PORT cp s3://$BUCKET_NAME/rookObj /tmp/ro
cat /tmp/rookObj-download
```
## Monitoring health
Rook configures health probes on the deployment created for CephObjectStore gateways. Refer to
[the CRD document](../../CRDs/Object-Storage/ceph-object-store-crd.md#health-settings) for
information about configuring the probes and monitoring the deployment status.
## Access External to the Cluster
Rook sets up the object storage so pods will have access internal to the cluster. If your applications are running outside the cluster,
@@ -532,9 +532,6 @@ cephObjectStores:
# sslCertificateRef:
instances: 1
priorityClassName: system-cluster-critical
healthCheck:
bucket:
interval: 60s
storageClass:
enabled: true
name: ceph-bucket
@@ -12231,20 +12231,9 @@ spec:
type: string
type: object
healthCheck:
description: The rgw Bucket healthchecks and liveness probe
description: The RGW health probes
nullable: true
properties:
bucket:
description: HealthCheckSpec represents the health check of an object store bucket
properties:
disabled:
type: boolean
interval:
description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds
type: string
timeout:
type: string
type: object
livenessProbe:
description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
properties:
@@ -12788,19 +12777,6 @@ spec:
status:
description: ObjectStoreStatus represents the status of a Ceph Object Store resource
properties:
bucketStatus:
description: BucketStatus represents the status of a bucket
properties:
details:
type: string
health:
description: ConditionType represent a resource's status
type: string
lastChanged:
type: string
lastChecked:
type: string
type: object
conditions:
items:
description: Condition represents a status condition on any Rook-Ceph Custom Resource.
+1 -25
View File
@@ -12223,20 +12223,9 @@ spec:
type: string
type: object
healthCheck:
description: The rgw Bucket healthchecks and liveness probe
description: The RGW health probes
nullable: true
properties:
bucket:
description: HealthCheckSpec represents the health check of an object store bucket
properties:
disabled:
type: boolean
interval:
description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds
type: string
timeout:
type: string
type: object
livenessProbe:
description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
properties:
@@ -12780,19 +12769,6 @@ spec:
status:
description: ObjectStoreStatus represents the status of a Ceph Object Store resource
properties:
bucketStatus:
description: BucketStatus represents the status of a bucket
properties:
details:
type: string
health:
description: ConditionType represent a resource's status
type: string
lastChanged:
type: string
lastChecked:
type: string
type: object
conditions:
items:
description: Condition represents a status condition on any Rook-Ceph Custom Resource.
+1 -4
View File
@@ -81,12 +81,9 @@ spec:
# memory: "1024Mi"
# priorityClassName: my-priority-class
#zone:
#name: zone-a
#name: zone-a
# service endpoint healthcheck
healthCheck:
bucket:
disabled: false
interval: 60s
# Configure the pod probes for the rgw daemon
livenessProbe:
disabled: false
+6 -9
View File
@@ -98,9 +98,6 @@ spec:
# service.beta.openshift.io/serving-cert-secret-name: <name of TLS secret automatic generation>
# service endpoint healthcheck
healthCheck:
bucket:
disabled: false
interval: 60s
# Configure the pod probes for the rgw daemon
livenessProbe:
disabled: false
@@ -149,12 +146,12 @@ metadata:
spec:
port:
targetPort: http
# if TLS is enabled, remove the above http port and uncomment the https port and tls settings
# port:
# targetPort: https
# tls:
# insecureEdgeTerminationPolicy: Redirect
# termination: reencrypt
# if TLS is enabled, remove the above http port and uncomment the https port and tls settings
# port:
# targetPort: https
# tls:
# insecureEdgeTerminationPolicy: Redirect
# termination: reencrypt
to:
kind: Service
name: rook-ceph-rgw-my-store # The name of the RGW service is in the form 'rook-ceph-rgw-<objectstore-name>'
+1 -4
View File
@@ -101,12 +101,9 @@ spec:
# memory: "1024Mi"
priorityClassName: system-cluster-critical
#zone:
#name: zone-a
#name: zone-a
# service endpoint healthcheck
healthCheck:
bucket:
disabled: false
interval: 60s
# Configure the pod probes for the rgw daemon
startupProbe:
disabled: false
+4 -20
View File
@@ -1335,10 +1335,10 @@ type ObjectStoreSpec struct {
// +nullable
Zone ZoneSpec `json:"zone,omitempty"`
// The rgw Bucket healthchecks and liveness probe
// The RGW health probes
// +optional
// +nullable
HealthCheck BucketHealthCheckSpec `json:"healthCheck,omitempty"`
HealthCheck ObjectHealthCheckSpec `json:"healthCheck,omitempty"`
// Security represents security settings
// +optional
@@ -1346,10 +1346,8 @@ type ObjectStoreSpec struct {
Security *ObjectStoreSecuritySpec `json:"security,omitempty"`
}
// BucketHealthCheckSpec represents the health check of an object store
type BucketHealthCheckSpec struct {
// +optional
Bucket HealthCheckSpec `json:"bucket,omitempty"`
// ObjectHealthCheckSpec represents the health check of an object store
type ObjectHealthCheckSpec struct {
// +optional
LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
// +optional
@@ -1455,8 +1453,6 @@ type ObjectStoreStatus struct {
// +optional
Message string `json:"message,omitempty"`
// +optional
BucketStatus *BucketStatus `json:"bucketStatus,omitempty"`
// +optional
// +nullable
Info map[string]string `json:"info,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
@@ -1465,18 +1461,6 @@ type ObjectStoreStatus struct {
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}
// BucketStatus represents the status of a bucket
type BucketStatus struct {
// +optional
Health ConditionType `json:"health,omitempty"`
// +optional
Details string `json:"details,omitempty"`
// +optional
LastChecked string `json:"lastChecked,omitempty"`
// +optional
LastChanged string `json:"lastChanged,omitempty"`
}
// CephObjectStoreUser represents a Ceph Object Store Gateway User
// +genclient
// +genclient:noStatus
@@ -97,38 +97,6 @@ func (in AnnotationsSpec) DeepCopy() AnnotationsSpec {
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketHealthCheckSpec) DeepCopyInto(out *BucketHealthCheckSpec) {
*out = *in
in.Bucket.DeepCopyInto(&out.Bucket)
if in.LivenessProbe != nil {
in, out := &in.LivenessProbe, &out.LivenessProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.ReadinessProbe != nil {
in, out := &in.ReadinessProbe, &out.ReadinessProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.StartupProbe != nil {
in, out := &in.StartupProbe, &out.StartupProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketHealthCheckSpec.
func (in *BucketHealthCheckSpec) DeepCopy() *BucketHealthCheckSpec {
if in == nil {
return nil
}
out := new(BucketHealthCheckSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketNotificationSpec) DeepCopyInto(out *BucketNotificationSpec) {
*out = *in
@@ -155,22 +123,6 @@ func (in *BucketNotificationSpec) DeepCopy() *BucketNotificationSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketStatus) DeepCopyInto(out *BucketStatus) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketStatus.
func (in *BucketStatus) DeepCopy() *BucketStatus {
if in == nil {
return nil
}
out := new(BucketStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BucketTopicSpec) DeepCopyInto(out *BucketTopicSpec) {
*out = *in
@@ -3059,6 +3011,37 @@ func (in *NotificationKeyFilterRule) DeepCopy() *NotificationKeyFilterRule {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectHealthCheckSpec) DeepCopyInto(out *ObjectHealthCheckSpec) {
*out = *in
if in.LivenessProbe != nil {
in, out := &in.LivenessProbe, &out.LivenessProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.ReadinessProbe != nil {
in, out := &in.ReadinessProbe, &out.ReadinessProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.StartupProbe != nil {
in, out := &in.StartupProbe, &out.StartupProbe
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectHealthCheckSpec.
func (in *ObjectHealthCheckSpec) DeepCopy() *ObjectHealthCheckSpec {
if in == nil {
return nil
}
out := new(ObjectHealthCheckSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectRealmSpec) DeepCopyInto(out *ObjectRealmSpec) {
*out = *in
@@ -3123,11 +3106,6 @@ func (in *ObjectStoreSpec) DeepCopy() *ObjectStoreSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ObjectStoreStatus) DeepCopyInto(out *ObjectStoreStatus) {
*out = *in
if in.BucketStatus != nil {
in, out := &in.BucketStatus, &out.BucketStatus
*out = new(BucketStatus)
**out = **in
}
if in.Info != nil {
in, out := &in.Info, &out.Info
*out = make(map[string]string, len(*in))
+68 -90
View File
@@ -24,6 +24,7 @@ import (
"syscall"
"time"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/coreos/pkg/capnslog"
bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned"
"github.com/pkg/errors"
@@ -80,22 +81,15 @@ var cephObjectStoreDependents = CephObjectStoreDependents
// ReconcileCephObjectStore reconciles a cephObjectStore object
type ReconcileCephObjectStore struct {
client client.Client
bktclient bktclient.Interface
scheme *runtime.Scheme
context *clusterd.Context
clusterSpec *cephv1.ClusterSpec
clusterInfo *cephclient.ClusterInfo
objectStoreContexts map[string]*objectStoreHealth
recorder record.EventRecorder
opManagerContext context.Context
opConfig opcontroller.OperatorConfig
}
type objectStoreHealth struct {
internalCtx context.Context
internalCancel context.CancelFunc
started bool
client client.Client
bktclient bktclient.Interface
scheme *runtime.Scheme
context *clusterd.Context
clusterSpec *cephv1.ClusterSpec
clusterInfo *cephclient.ClusterInfo
recorder record.EventRecorder
opManagerContext context.Context
opConfig opcontroller.OperatorConfig
}
// Add creates a new cephObjectStore Controller and adds it to the Manager. The Manager will set fields on the Controller
@@ -108,14 +102,13 @@ func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext contex
func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
context.Client = mgr.GetClient()
return &ReconcileCephObjectStore{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
context: context,
bktclient: bktclient.NewForConfigOrDie(context.KubeConfig),
objectStoreContexts: make(map[string]*objectStoreHealth),
recorder: mgr.GetEventRecorderFor("rook-" + controllerName),
opManagerContext: opManagerContext,
opConfig: opConfig,
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
context: context,
bktclient: bktclient.NewForConfigOrDie(context.KubeConfig),
recorder: mgr.GetEventRecorderFor("rook-" + controllerName),
opManagerContext: opManagerContext,
opConfig: opConfig,
}
}
@@ -166,12 +159,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
if err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("cephObjectStore resource not found. Ignoring since object must be deleted.")
// If there was a previous error or if a user removed this resource's finalizer, it's
// possible Rook didn't clean up the monitoring routine for this resource. Ensure the
// routine is stopped when we see the resource is gone.
cephObjectStore.Name = request.Name
cephObjectStore.Namespace = request.Namespace
r.stopMonitoring(cephObjectStore)
return reconcile.Result{}, *cephObjectStore, nil
}
// Error reading the object - requeue the request.
@@ -204,9 +191,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
// If not, we should wait for it to be ready
// This handles the case where the operator is not ready to accept Ceph command but the cluster exists
if !cephObjectStore.GetDeletionTimestamp().IsZero() && !cephClusterExists {
// don't leak the health checker routine if we are force deleting
r.stopMonitoring(cephObjectStore)
// Remove finalizer
err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephObjectStore)
if err != nil {
@@ -221,17 +205,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
}
r.clusterSpec = &cephCluster.Spec
// Initialize the channel for this object store
// This allows us to track multiple ObjectStores in the same namespace
_, ok := r.objectStoreContexts[monitoringChannelKey(cephObjectStore)]
if !ok {
internalCtx, internalCancel := context.WithCancel(r.opManagerContext)
r.objectStoreContexts[monitoringChannelKey(cephObjectStore)] = &objectStoreHealth{
internalCtx: internalCtx,
internalCancel: internalCancel,
}
}
// Populate clusterInfo during each reconcile
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.clusterSpec)
if err != nil {
@@ -257,12 +230,18 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
}
objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore)
if err != nil {
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get object context")
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context")
}
opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec)
if err != nil {
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to check for object buckets. failed to get admin ops API context")
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context")
}
// ignore any errors coming from this for the deletion case since we could get into a
// partially-deleted case where RGWs aren't responding. Deletion will be blocked later if
// the bucket fails to be deleted and the RGWs are running.
_ = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore)
deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx)
if err != nil {
return reconcile.Result{}, *cephObjectStore, err
@@ -273,9 +252,6 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
}
reporting.ReportDeletionNotBlockedDueToDependents(r.opManagerContext, logger, r.client, r.recorder, cephObjectStore)
// Cancel the context to stop monitoring the health of the object store
r.stopMonitoring(cephObjectStore)
cfg := clusterConfig{
context: r.context,
store: cephObjectStore,
@@ -349,9 +325,22 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
return result, *cephObjectStore, err
}
objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore)
if err != nil {
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context")
}
opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec)
if err != nil {
return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context")
}
err = removeDeprecatedHealthCheckBucket(r.clusterInfo.Context, opsCtx, cephObjectStore)
if err != nil {
return reconcile.Result{}, *cephObjectStore, errors.Wrap(err, "updated object store but failed to remove deprecated health check bucket")
}
// update ObservedGeneration in status at the end of reconcile
// Set Progressing status, we are done reconciling, the health check go routine will update the status
updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionProgressing, buildStatusInfo(cephObjectStore))
updateStatus(r.opManagerContext, observedGeneration, r.client, request.NamespacedName, cephv1.ConditionReady, buildStatusInfo(cephObjectStore))
// Return and do not requeue
logger.Debug("done reconciling")
@@ -455,14 +444,6 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
}
}
// Start monitoring
if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled {
err = r.startMonitoring(cephObjectStore, objContext, namespacedName)
if err != nil {
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
}
@@ -525,43 +506,40 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil
}
func monitoringChannelKey(o *cephv1.CephObjectStore) string {
return types.NamespacedName{Namespace: o.Namespace, Name: o.Name}.String()
}
// handle upgrade from v1.10; we need to remove the health checker bucket from the store, or
// deleting the store will be stuck on dependents because an unknown bucket exists
// TODO: remove this for Rook v1.12 release
var removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
healthCheckBucket := genHealthCheckerBucketName(string(cos.UID))
doPurge := true // purge all content of the bucket
bucket := admin.Bucket{Bucket: healthCheckBucket, PurgeObject: &doPurge}
remErr := opsCtx.AdminOpsClient.RemoveBucket(ctx, bucket)
func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) error {
channelKey := monitoringChannelKey(objectstore)
// Start monitoring object store
if r.objectStoreContexts[channelKey].started {
logger.Info("external rgw endpoint monitoring go routine already running!")
if remErr == nil {
logger.Info("successfully deleted deprecated health checker bucket")
return nil
}
rgwChecker, err := newBucketChecker(r.context, objContext, r.client, namespacedName, &objectstore.Spec)
if err != nil {
return errors.Wrapf(err, "failed to start rgw health checker for CephObjectStore %q, will re-reconcile", namespacedName.String())
if errors.Is(remErr, admin.ErrNoSuchBucket) {
logger.Debug("deprecated health checker bucket already does not exist: NoSuchBucket")
return nil
}
logger.Infof("starting rgw health checker for CephObjectStore %q", namespacedName.String())
go rgwChecker.checkObjectStore(r.objectStoreContexts, channelKey)
// Set the monitoring flag so we don't start more than one go routine
r.objectStoreContexts[channelKey].started = true
return nil
}
// cancel monitoring. This is a noop if monitoring is not running.
func (r *ReconcileCephObjectStore) stopMonitoring(objectstore *cephv1.CephObjectStore) {
channelKey := monitoringChannelKey(objectstore)
_, monitoringContextExists := r.objectStoreContexts[channelKey]
if monitoringContextExists {
// stop the monitoring routine
r.objectStoreContexts[channelKey].internalCancel()
// remove the monitoring routine from the map
delete(r.objectStoreContexts, channelKey)
if errors.Is(remErr, admin.ErrNoSuchKey) {
// ceph might return NoSuchKey than NoSuchBucket when the target bucket does not exist.
// then we can use GetBucketInfo() to judge the existence of the bucket.
// see: https://github.com/ceph/ceph/pull/44413
_, getErr := opsCtx.AdminOpsClient.GetBucketInfo(ctx, bucket)
if getErr != nil {
if errors.Is(getErr, admin.ErrNoSuchBucket) {
logger.Debug("deprecated health checker bucket already does not exist: NoSuchKey")
return nil
}
// both commands errored; something is up; be sure to return both errors
return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q. failed to get bucket info. %v", healthCheckBucket, getErr)
}
// remove bucket failed, and bucket exists; something else went wrong with remove bucket
}
return errors.Wrapf(remErr, "failed to delete deprecated health checker bucket %q", healthCheckBucket)
}
+41 -40
View File
@@ -19,7 +19,7 @@ package object
import (
"context"
"net/http"
"fmt"
"os"
"reflect"
"testing"
@@ -383,12 +383,11 @@ func TestCephObjectStoreController(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
// Create a ReconcileCephObjectStore object with the scheme and fake client.
r := &ReconcileCephObjectStore{
client: cl,
scheme: s,
context: c,
objectStoreContexts: make(map[string]*objectStoreHealth),
recorder: record.NewFakeRecorder(5),
opManagerContext: context.TODO(),
client: cl,
scheme: s,
context: c,
recorder: record.NewFakeRecorder(5),
opManagerContext: context.TODO(),
}
return r
@@ -526,29 +525,13 @@ func TestCephObjectStoreController(t *testing.T) {
return r
}
t.Run("error - failed to start health checker", func(t *testing.T) {
r := setupEnvironmentWithReadyCephCluster()
// cause a failure when creating the admin ops api for the health check
origHTTPClientFunc := genObjectStoreHTTPClientFunc
genObjectStoreHTTPClientFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (client *http.Client, tlsCert []byte, err error) {
return nil, []byte{}, errors.New("induced error creating admin ops API connection")
}
defer func() { genObjectStoreHTTPClientFunc = origHTTPClientFunc }()
_, err := r.Reconcile(ctx, req)
assert.Error(t, err)
// we don't actually care if Requeue is true if there is an error assert.True(t, res.Requeue)
assert.Contains(t, err.Error(), "failed to start rgw health checker")
assert.Contains(t, err.Error(), "induced error creating admin ops API connection")
// health checker should start up after committing config changes
assert.True(t, calledCommitConfigChanges)
})
t.Run("success - object store is running", func(t *testing.T) {
r := setupEnvironmentWithReadyCephCluster()
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
return nil
}
res, err := r.Reconcile(ctx, req)
assert.NoError(t, err)
assert.False(t, res.Requeue)
@@ -556,12 +539,24 @@ func TestCephObjectStoreController(t *testing.T) {
objectStore := &cephv1.CephObjectStore{}
err = r.client.Get(context.TODO(), req.NamespacedName, objectStore)
assert.NoError(t, err)
assert.Equal(t, cephv1.ConditionProgressing, objectStore.Status.Phase, objectStore)
assert.Equal(t, cephv1.ConditionReady, objectStore.Status.Phase, objectStore)
assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore)
assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore)
assert.True(t, calledCommitConfigChanges)
assert.Equal(t, 16, r.clusterInfo.CephVersion.Major)
})
t.Run("failed to remove deprecated health check bucket", func(t *testing.T) {
r := setupEnvironmentWithReadyCephCluster()
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
return fmt.Errorf("induced error")
}
_, err := r.Reconcile(ctx, req)
assert.Error(t, err)
assert.Contains(t, err.Error(), "deprecated health check bucket")
})
}
func TestCephObjectStoreControllerMultisite(t *testing.T) {
@@ -735,12 +730,11 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
r := &ReconcileCephObjectStore{
client: cl,
scheme: s,
context: c,
objectStoreContexts: make(map[string]*objectStoreHealth),
recorder: record.NewFakeRecorder(5),
opManagerContext: ctx,
client: cl,
scheme: s,
context: c,
recorder: record.NewFakeRecorder(5),
opManagerContext: ctx,
}
_, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
@@ -757,6 +751,10 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
return &cephver.Pacific, &cephver.Pacific, nil
}
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
return nil
}
t.Run("create an object store", func(t *testing.T) {
res, err := r.Reconcile(ctx, req)
assert.NoError(t, err)
@@ -894,12 +892,11 @@ func TestCephObjectExternalStoreController(t *testing.T) {
}
r := &ReconcileCephObjectStore{
client: cl,
scheme: s,
context: c,
objectStoreContexts: make(map[string]*objectStoreHealth),
recorder: record.NewFakeRecorder(5),
opManagerContext: ctx,
client: cl,
scheme: s,
context: c,
recorder: record.NewFakeRecorder(5),
opManagerContext: ctx,
}
_, err := r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
@@ -931,6 +928,10 @@ func TestCephObjectExternalStoreController(t *testing.T) {
r := getReconciler(objects)
removeDeprecatedHealthCheckBucket = func(ctx context.Context, opsCtx *AdminOpsContext, cos *cephv1.CephObjectStore) error {
return nil
}
t.Run("create an external object store", func(t *testing.T) {
res, err := r.Reconcile(ctx, req)
assert.NoError(t, err)
+5 -4
View File
@@ -30,6 +30,7 @@ import (
const (
bucketDependentType = "buckets in the object store (could be from ObjectBucketClaims or COSI Buckets)"
zoneIsMasterWithPeersDependentType = "zone is master and has peers"
s3HealthCheckBucketName = "rook-ceph-bucket-checker"
)
// CephObjectStoreDependents returns the buckets which exist in the object store that should block
@@ -127,11 +128,7 @@ func getBucketDependents(
if err != nil {
return errors.Wrapf(err, "failed to list buckets in CephObjectStore %q", nsName)
}
healthCheckBucket := genHealthCheckerBucketName(string(store.UID))
for _, b := range buckets {
if b == healthCheckBucket {
continue // don't include the health checker bucket as a blocking dependent
}
deps.Add(bucketDependentType, b)
}
@@ -165,3 +162,7 @@ func getMasterZoneDependents(
}
return nil
}
func genHealthCheckerBucketName(uuid string) string {
return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid)
}
-292
View File
@@ -1,292 +0,0 @@
/*
Copyright 2020 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package object
import (
"fmt"
"time"
"github.com/ceph/go-ceph/rgw/admin"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
s3UserHealthCheckName = "rook-ceph-internal-s3-user-checker"
s3HealthCheckBucketName = "rook-ceph-bucket-checker"
s3HealthCheckObjectBody = "Test Rook Object Data"
s3HealthCheckObjectKey = "rookHealthCheckTestObject"
contentType = "plain/text"
)
var (
defaultHealthCheckInterval = 1 * time.Minute
)
// bucketChecker aggregates the mon/cluster info needed to check the health of the monitors
type bucketChecker struct {
context *clusterd.Context
objContext *AdminOpsContext
interval *time.Duration
port int32
client client.Client
namespacedName types.NamespacedName
objectStoreSpec *cephv1.ObjectStoreSpec
}
// newbucketChecker creates a new HealthChecker object
func newBucketChecker(
ctx *clusterd.Context, objContext *Context, client client.Client, namespacedName types.NamespacedName, objectStoreSpec *cephv1.ObjectStoreSpec,
) (*bucketChecker, error) {
port, err := objectStoreSpec.GetPort()
if err != nil {
return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String())
}
opsCtx, err := NewMultisiteAdminOpsContext(objContext, objectStoreSpec)
if err != nil {
return nil, errors.Wrapf(err, "failed to create bucket checker for CephObjectStore %q", namespacedName.String())
}
c := &bucketChecker{
context: ctx,
objContext: opsCtx,
interval: &defaultHealthCheckInterval,
port: port,
namespacedName: namespacedName,
client: client,
objectStoreSpec: objectStoreSpec,
}
// allow overriding the check interval
checkInterval := objectStoreSpec.HealthCheck.Bucket.Interval
if checkInterval != nil {
logger.Infof("ceph rgw status check interval for object store %q is %q", namespacedName.Name, checkInterval.Duration.String())
c.interval = &checkInterval.Duration
}
return c, nil
}
// checkObjectStore periodically checks the health of the cluster
func (c *bucketChecker) checkObjectStore(objectStoreContexts map[string]*objectStoreHealth, channelKey string) {
// check the object store health immediately before starting the loop
err := c.checkObjectStoreHealth()
if err != nil {
updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error())
logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err)
}
for {
// We must perform this check otherwise the case will check an index that does not exist anymore and
// we will get an invalid pointer error and the go routine will panic
if _, ok := objectStoreContexts[channelKey]; !ok {
logger.Infof("object store %q has been deleted. stopping monitoring of rgw endpoints", c.namespacedName.Name)
return
}
select {
case <-objectStoreContexts[channelKey].internalCtx.Done():
// purge bucket and s3 user
// Needed for external mode where in converged everything goes away with the CR deletion
c.cleanupHealthCheck()
delete(objectStoreContexts, channelKey)
logger.Infof("stopping monitoring of rgw endpoints for object store %q", c.namespacedName.Name)
return
case <-time.After(*c.interval):
logger.Debugf("checking rgw health of object store %q", c.namespacedName.Name)
err := c.checkObjectStoreHealth()
if err != nil {
updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionFailure, err.Error())
logger.Debugf("failed to check rgw health for object store %q. %v", c.namespacedName.Name, err)
}
}
}
}
func (c *bucketChecker) checkObjectStoreHealth() error {
/*
0. purge the s3 object by default
1. create an S3 user
2. always use the same user
3. if already exists just re-hydrate the s3 credentials
4. create a bucket with that user or use the existing one (always use the same bucket)
5. create a check file
6. get the hash of the file
7. PUT the file
8. GET the file
9. compare hashes
10. delete object on bucket
11. update CR health status check
Always keep the bucket and the user for the health check, just do PUT and GET because bucket creation is expensive
*/
// Keep admin ops context up-to date if there are config changes
if err := UpdateEndpoint(&c.objContext.Context, c.objectStoreSpec); err != nil {
return errors.Wrapf(err, "failed to parse updated CephObjectStore spec")
}
// Generate unique user and bucket name
bucketName := genHealthCheckerBucketName(c.objContext.UID)
userConfig := genUserCheckerConfig(c.objContext.UID)
// Create checker user
logger.Debugf("creating s3 user object %q for object store %q health check", userConfig.ID, c.namespacedName.Name)
var user admin.User
user, err := c.objContext.AdminOpsClient.GetUser(c.objContext.clusterInfo.Context, userConfig)
if err != nil {
if errors.Is(err, admin.ErrNoSuchUser) {
user, err = c.objContext.AdminOpsClient.CreateUser(c.objContext.clusterInfo.Context, userConfig)
if err != nil {
return errors.Wrapf(err, "failed to create from ceph object user %v", userConfig.ID)
}
} else {
return errors.Wrapf(err, "failed to get details from ceph object user %q", userConfig.ID)
}
}
// Set access and secret key
s3endpoint := c.objContext.Endpoint
s3AccessKey := user.Keys[0].AccessKey
s3SecretKey := user.Keys[0].SecretKey
// Initiate s3 agent
logger.Debugf("initializing s3 connection for object store %q", c.namespacedName.Name)
s3client, err := NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, false)
if err != nil {
return errors.Wrap(err, "failed to initialize s3 connection")
}
// Force purge the s3 object before starting anything
cleanupObjectHealthCheck(s3client, c.objContext.UID)
// Bucket health test
err = c.testBucketHealth(s3client, bucketName)
if err != nil {
return errors.Wrapf(err, "failed to run bucket health checks for object store %q", c.namespacedName.String())
}
logger.Debugf("successfully checked object store endpoint for object store %q", c.namespacedName.String())
// Update the EndpointStatus in the CR to reflect the healthiness
updateStatusBucket(c.objContext.clusterInfo.Context, c.client, c.namespacedName, cephv1.ConditionConnected, "")
return nil
}
func cleanupObjectHealthCheck(s3client *S3Agent, objectStoreUID string) {
bucketToDelete := genHealthCheckerBucketName(objectStoreUID)
logger.Debugf("deleting object %q from bucket %q", s3HealthCheckObjectKey, bucketToDelete)
_, err := s3client.DeleteObjectInBucket(bucketToDelete, s3HealthCheckObjectKey)
if err != nil {
logger.Errorf("failed to delete object in bucket. %v", err)
}
}
func (c *bucketChecker) cleanupHealthCheck() {
bucketToDelete := genHealthCheckerBucketName(c.objContext.UID)
logger.Infof("deleting object %q from bucket %q in object store %q", s3HealthCheckObjectKey, bucketToDelete, c.namespacedName.Name)
thePurge := true
err := c.objContext.AdminOpsClient.RemoveBucket(c.objContext.clusterInfo.Context, admin.Bucket{Bucket: bucketToDelete, PurgeObject: &thePurge})
if err != nil {
if errors.Is(err, admin.ErrNoSuchBucket) {
// opinion: "not found" is not an error
logger.Debugf("bucket %q does not exist", bucketToDelete)
} else {
logger.Errorf("failed to delete bucket %q for object store %q. %v", bucketToDelete, c.namespacedName.Name, err)
}
}
userToDelete := genUserCheckerConfig(c.objContext.UID)
err = c.objContext.AdminOpsClient.RemoveUser(c.objContext.clusterInfo.Context, userToDelete)
if err != nil && !errors.Is(err, admin.ErrNoSuchUser) {
logger.Errorf("failed to delete object user %q for object store %q. %v", userToDelete.ID, c.namespacedName.Name, err)
}
logger.Debugf("successfully deleted object user %q for object store %q", userToDelete.ID, c.namespacedName.Name)
}
func toCustomResourceStatus(currentStatus *cephv1.BucketStatus, details string, health cephv1.ConditionType) *cephv1.BucketStatus {
s := &cephv1.BucketStatus{
Health: health,
LastChecked: time.Now().UTC().Format(time.RFC3339),
Details: details,
}
if currentStatus != nil {
s.LastChanged = currentStatus.LastChanged
if currentStatus.Details != s.Details {
s.LastChanged = s.LastChecked
}
}
return s
}
func genHealthCheckerBucketName(uuid string) string {
return fmt.Sprintf("%s-%s", s3HealthCheckBucketName, uuid)
}
func genUserCheckerConfig(cephObjectStoreUID string) admin.User {
userName := fmt.Sprintf("%s-%s", s3UserHealthCheckName, cephObjectStoreUID)
return admin.User{
ID: userName,
DisplayName: userName,
}
}
func (c *bucketChecker) testBucketHealth(s3client *S3Agent, bucket string) error {
// Purge on exit
defer cleanupObjectHealthCheck(s3client, c.objContext.UID)
// Create S3 bucket
logger.Debugf("creating bucket %q", bucket)
err := s3client.CreateBucketNoInfoLogging(bucket)
if err != nil {
return errors.Wrapf(err, "failed to create bucket %q for object store %q", bucket, c.namespacedName.Name)
}
// Put an object into the bucket
logger.Debugf("putting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
_, err = s3client.PutObjectInBucket(bucket, string(s3HealthCheckObjectBody), s3HealthCheckObjectKey, contentType)
if err != nil {
return errors.Wrapf(err, "failed to put object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
}
// Get the object from the bucket
logger.Debugf("getting object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
read, err := s3client.GetObjectInBucket(bucket, s3HealthCheckObjectKey)
if err != nil {
return errors.Wrapf(err, "failed to get object %q in bucket %q for object store %q", s3HealthCheckObjectKey, bucket, c.namespacedName.Name)
}
// Compare the old and the existing object
logger.Debugf("comparing objects hash for object store %q", c.namespacedName.Name)
oldHash := k8sutil.Hash(s3HealthCheckObjectBody)
currentHash := k8sutil.Hash(read)
if currentHash != oldHash {
return errors.Wrapf(err, "wrong file content, old file hash is %q and new one is %q for object store %q", oldHash, currentHash, c.namespacedName.Name)
}
return nil
}
+1 -1
View File
@@ -391,7 +391,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
}
// configureReadinessProbe returns the desired readiness probe for a given daemon
func configureReadinessProbe(container *v1.Container, healthCheck cephv1.BucketHealthCheckSpec) {
func configureReadinessProbe(container *v1.Container, healthCheck cephv1.ObjectHealthCheckSpec) {
if ok := healthCheck.ReadinessProbe; ok != nil {
if !healthCheck.ReadinessProbe.Disabled {
probe := healthCheck.ReadinessProbe.Probe
-36
View File
@@ -75,42 +75,6 @@ func updateStatus(ctx context.Context, observedGeneration int64, client client.C
logger.Debugf("object store %q status updated to %q", namespacedName.String(), status)
}
// updateStatusBucket updates an object with a given status
func updateStatusBucket(ctx context.Context, client client.Client, name types.NamespacedName, status cephv1.ConditionType, details string) {
// Updating the status is important to users, but we can still keep operating if there is a
// failure. Retry a few times to give it our best effort attempt.
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
objectStore := &cephv1.CephObjectStore{}
if err := client.Get(ctx, name, objectStore); err != nil {
if kerrors.IsNotFound(err) {
logger.Debug("CephObjectStore resource not found. Ignoring since object must be deleted.")
return nil
}
return errors.Wrapf(err, "failed to retrieve object store %q to update status to %v", name.String(), status)
}
if objectStore.Status == nil {
objectStore.Status = &cephv1.ObjectStoreStatus{}
}
objectStore.Status.BucketStatus = toCustomResourceStatus(objectStore.Status.BucketStatus, details, status)
// do not transition to other statuses once deletion begins
if objectStore.Status.Phase != cephv1.ConditionDeleting {
objectStore.Status.Phase = status
}
// but we still need to update the health checker status
if err := reporting.UpdateStatus(client, objectStore); err != nil {
return errors.Wrapf(err, "failed to set object store %q status to %v", name.String(), status)
}
return nil
})
if err != nil {
logger.Error(err)
}
logger.Debugf("object store %q status updated to %v", name.String(), status)
}
func buildStatusInfo(cephObjectStore *cephv1.CephObjectStore) map[string]string {
m := make(map[string]string)
@@ -473,10 +473,6 @@ spec:
securePort: ` + strconv.Itoa(port) + `
instances: ` + strconv.Itoa(replicaCount) + `
sslCertificateRef: ` + name + `
healthCheck:
bucket:
disabled: false
interval: 10s
`
}
return `apiVersion: ceph.rook.io/v1
@@ -498,10 +494,6 @@ spec:
resources: null
port: ` + strconv.Itoa(port) + `
instances: ` + strconv.Itoa(replicaCount) + `
healthCheck:
bucket:
disabled: false
interval: 5s
`
}
+27 -4
View File
@@ -36,6 +36,7 @@ import (
"github.com/stretchr/testify/suite"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
)
const (
@@ -116,18 +117,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils
for i = 0; i < retryCount; i++ {
objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{})
assert.Nil(t, err)
if objectStore.Status == nil || objectStore.Status.BucketStatus == nil {
// TODO: check that object store status is good, and also check that status of
// deployment is good based on health checks
if objectStore.Status == nil {
logger.Infof("(%d) object status check sleeping for 5 seconds ...%+v", i, objectStore.Status)
time.Sleep(5 * time.Second)
continue
}
logger.Info("objectstore status is", objectStore.Status)
if objectStore.Status.BucketStatus.Health == cephv1.ConditionFailure {
logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.BucketStatus)
// ConditionConnected supports Rook v1.10 clusters that still had health check
// TODO: remove that half of check after Rook v1.12 release
if objectStore.Status.Phase != cephv1.ConditionReady && objectStore.Status.Phase != cephv1.ConditionConnected {
logger.Infof("(%d) bucket status check sleeping for 5 seconds ...%+v", i, objectStore.Status.Phase)
time.Sleep(5 * time.Second)
continue
}
assert.Equal(t, cephv1.ConditionConnected, objectStore.Status.BucketStatus.Health)
// Info field has the endpoint in it
assert.NotEmpty(t, objectStore.Status.Info)
assert.NotEmpty(t, objectStore.Status.Info["endpoint"])
@@ -138,6 +144,23 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils
}
})
t.Run("verify RGW liveness probes show healthy", func(t *testing.T) {
err := wait.PollImmediate(2*time.Second, 90*time.Second, func() (done bool, err error) {
deployName := "rook-ceph-rgw-" + storeName + "-a"
d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, deployName, metav1.GetOptions{})
if err != nil {
logger.Infof("waiting for rgw deployment %q to be ready; failed to get deployment: %v", deployName, err)
return false, nil
}
if d.Status.UnavailableReplicas != 0 {
logger.Infof("waiting rgw deployment %q to be ready; %d replicas are unavailable", deployName, d.Status.UnavailableReplicas)
return false, nil
}
return true, nil
})
assert.NoError(t, err)
})
t.Run("verify RGW service is up", func(t *testing.T) {
assert.True(t, k8sh.IsServiceUp("rook-ceph-rgw-"+storeName, namespace))
})