my-rook-config/pkg/operator/ceph/object/notification/obc_label_controller.go

/*
Copyright 2021 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package notification to manage a rook bucket notifications.
package notification

import (
	"context"
	"reflect"
	"strings"

	bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
	"github.com/rook/rook/pkg/operator/ceph/object/bucket"
	"github.com/rook/rook/pkg/operator/ceph/object/topic"
	"github.com/rook/rook/pkg/operator/ceph/reporting"
	"github.com/rook/rook/pkg/util/log"
	kapiv1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/event"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/predicate"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/controller-runtime/pkg/source"
)

const (
	notificationLabelPrefix   = "bucket-notification-"
	bucketProvisionerLabelKey = "bucket-provisioner"
	bucketProvisionerLabelVal = "ceph.rook.io-bucket"
)

// ReconcileOBCLabels reconciles a ObjectBucketClaim labels
type ReconcileOBCLabels struct {
	client           client.Client
	context          *clusterd.Context
	opManagerContext context.Context
	recorder         record.EventRecorder
}

func obcPredicate[T *bktv1alpha1.ObjectBucketClaim]() predicate.TypedFuncs[T] {
	return predicate.TypedFuncs[T]{
		CreateFunc: func(e event.TypedCreateEvent[T]) bool {
			obj := (*bktv1alpha1.ObjectBucketClaim)(e.Object)
			nsName := opcontroller.NsName(obj.GetNamespace(), obj.GetName())
			log.NamedDebug(nsName, logger, "create event from a CR")
			return true
		},
		DeleteFunc: func(e event.TypedDeleteEvent[T]) bool {
			obj := (*bktv1alpha1.ObjectBucketClaim)(e.Object)
			nsName := opcontroller.NsName(obj.GetNamespace(), obj.GetName())
			log.NamedDebug(nsName, logger, "delete event from a CR")
			return true
		},
		UpdateFunc: func(e event.TypedUpdateEvent[T]) bool {
			// break generic-ness in order to access .Spec.ObjectBucketName
			objOld := (*bktv1alpha1.ObjectBucketClaim)(e.ObjectOld)
			objNew := (*bktv1alpha1.ObjectBucketClaim)(e.ObjectNew)
			nsName := opcontroller.NsName(objNew.GetNamespace(), objNew.GetName())

			log.NamedDebug(nsName, logger, "update event on ObjectBucketClaim CR")
			// If the labels "do_not_reconcile" is set on the object, let's not reconcile that request
			if opcontroller.IsDoNotReconcile(objNew.GetLabels()) {
				log.NamedDebug(nsName, logger, "object %q matched on update but %q label is set, doing nothing", objNew.GetName(), opcontroller.DoNotReconcileLabelName)
				return false
			}
			if !reflect.DeepEqual(objOld.GetLabels(), objNew.GetLabels()) {
				log.NamedInfo(nsName, logger, "CR labels has changed")
				return true
			} else if objOld.Spec.ObjectBucketName != objNew.Spec.ObjectBucketName {
				log.NamedInfo(nsName, logger, "CR bucket name changed from %q to %q", objOld.Spec.ObjectBucketName, objNew.Spec.ObjectBucketName)
				return true
			}
			log.NamedDebug(nsName, logger, "no change in CR %q", objNew.GetName())
			return false
		},
		GenericFunc: func(e event.TypedGenericEvent[T]) bool {
			return false
		},
	}
}

func addOBCLabelReconciler(mgr manager.Manager, r reconcile.Reconciler) error {
	// Create a new controller
	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
	if err != nil {
		return err
	}
	logger.Info("successfully started")

	// Watch for changes on the OBC CRD object
	err = c.Watch(
		source.Kind(
			mgr.GetCache(),
			&bktv1alpha1.ObjectBucketClaim{},
			&handler.TypedEnqueueRequestForObject[*bktv1alpha1.ObjectBucketClaim]{},
			obcPredicate(),
		),
	)
	if err != nil {
		return err
	}

	return nil
}

// Reconcile reads that state of the cluster for a ObjectBucketClaim object and makes changes based on the state read
// and the ObjectBucketClaim labels
// The Controller will requeue the Request to be processed again if the returned error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
func (r *ReconcileOBCLabels) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
	defer opcontroller.RecoverAndLogException()
	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
	reconcileResponse, err := r.reconcile(request)
	if err != nil {
		log.NamedError(request.NamespacedName, logger, "failed to reconcile %v", err)
	}

	return reconcileResponse, err
}

func (r *ReconcileOBCLabels) reconcile(request reconcile.Request) (reconcile.Result, error) {
	log.NamedDebug(request.NamespacedName, logger, "reconciling ObjectBucketClaim labels for bucket notifications")
	// Fetch the ObjectBucketClaim instance
	obc := bktv1alpha1.ObjectBucketClaim{}
	err := r.client.Get(r.opManagerContext, request.NamespacedName, &obc)
	if err != nil {
		if kerrors.IsNotFound(err) {
			log.NamedDebug(request.NamespacedName, logger, "ObjectBucketClaim resource not found. Ignoring since resource must be deleted.")
			return reconcile.Result{}, nil
		}
		// Error reading the object - requeue the request.
		return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve ObjectBucketClaim %q", request.NamespacedName)
	}

	// DELETE: the CR was deleted
	if !obc.GetDeletionTimestamp().IsZero() {
		log.NamedDebug(request.NamespacedName, logger, "ObjectBucketClaim was deleted")
		// Return and do not requeue. Successful deletion.
		return reconcile.Result{}, nil
	}

	// reschedule if ObjectBucket was not created yet
	if obc.Spec.ObjectBucketName == "" {
		log.NamedDebug(request.NamespacedName, logger, "ObjectBucketClaim resource did not create the bucket yet. will retry")
		return waitForRequeueIfObjectBucketNotReady, nil
	}

	// get the ObjectBucket
	ob := bktv1alpha1.ObjectBucket{}
	bucketName := types.NamespacedName{Namespace: obc.Namespace, Name: obc.Spec.ObjectBucketName}
	if err := r.client.Get(r.opManagerContext, bucketName, &ob); err != nil {
		return reconcile.Result{}, errors.Wrapf(err, "failed to retrieve ObjectBucket %q", bucketName)
	}

	// validate if the bucket is provisioned by the ceph provisioner
	if !strings.Contains(ob.Labels[bucketProvisionerLabelKey], bucketProvisionerLabelVal) {
		log.NamedDebug(request.NamespacedName, logger, "ObjectBucket %q was not provisioned by the ceph object store provisioner and tagged with provisioner %q. ignoring",
			bucketName, ob.Labels[bucketProvisionerLabelKey])
		return reconcile.Result{}, nil
	}
	// validate object store name
	objectStoreName, err := getCephObjectStoreName(ob)
	if err != nil {
		return reconcile.Result{}, errors.Wrapf(err, "failed to get object store from ObjectBucket %q", bucketName)
	}
	// Populate clusterInfo during each reconcile
	clusterInfo, clusterSpec, err := getReadyCluster(r.client, r.opManagerContext, *r.context, objectStoreName.Namespace)
	if err != nil {
		return opcontroller.WaitForRequeueIfCephClusterNotReady, errors.Wrapf(err, "cluster is not ready")
	}
	if clusterInfo == nil || clusterSpec == nil {
		return opcontroller.WaitForRequeueIfCephClusterNotReady, nil
	}

	// get all existing notifications
	p := provisioner{
		context:          r.context,
		clusterInfo:      clusterInfo,
		clusterSpec:      clusterSpec,
		opManagerContext: r.opManagerContext,
		owner:            ob.Spec.AdditionalState[bucket.CephUser],
		objectStoreName:  objectStoreName,
	}
	bnList, err := getAllNotificationsFunc(p, &ob)
	if err != nil {
		return reconcile.Result{}, errors.Wrapf(err, "failed to list bucket notifications in ObjectbucketClaim %q", bucketName)
	}

	labelList := make([]string, 0)
	deleteList := make([]string, 0)
	// looking for notifications in the labels
	for labelKey, labelValue := range obc.Labels {
		notifyLabels := strings.SplitAfterN(labelKey, notificationLabelPrefix, 2)
		if len(notifyLabels) > 1 && notifyLabels[1] != "" {
			if labelValue != notifyLabels[1] {
				log.NamedWarning(request.NamespacedName, logger, "bucket notification label mismatch. ignoring key %q value %q", labelKey, labelValue)
				continue
			}
			labelList = append(labelList, labelValue)
			log.NamedDebug(request.NamespacedName, logger, "bucket notification label %q found on ObjectbucketClaim %q", labelValue, bucketName)
		}
	}

	// remove notifications which are no longer specified in the OBC labels
	for _, oldValue := range bnList {
		if !sets.NewString(labelList...).Has(oldValue) {
			deleteList = append(deleteList, oldValue)
		}
	}
	retry := false
	for _, notificationId := range deleteList {
		err = deleteNotificationFunc(p, &ob, notificationId)
		if err != nil {
			log.NamedError(request.NamespacedName, logger, "notification %q failed remove from %q, returned error %v", notificationId, ob.Spec.Endpoint.BucketName, err)
			retry = true
		}
	}
	if retry {
		return waitForRequeueIfNotificationNotDeleted, nil
	}
	// add new notifications to the list
	for _, label := range labelList {
		reconcileResponse, notification, err := r.addNewNotification(p, ob, label, objectStoreName, obc.Namespace)
		_, _ = reporting.ReportReconcileResult(logger, r.recorder, request, &notification, reconcileResponse, err)
		if err != nil {
			return reconcileResponse, err
		}
	}
	return reconcile.Result{}, nil
}

func (r *ReconcileOBCLabels) addNewNotification(p provisioner, ob bktv1alpha1.ObjectBucket, label string, objectStoreName types.NamespacedName, namespace string) (reconcile.Result, cephv1.CephBucketNotification, error) {
	// for each notification label fetch the bucket notification CRD
	notification := &cephv1.CephBucketNotification{ObjectMeta: metav1.ObjectMeta{Name: label, Namespace: namespace}}
	bnName := types.NamespacedName{Namespace: namespace, Name: label}
	bucketName := types.NamespacedName{Name: ob.Spec.ClaimRef.Name, Namespace: namespace}
	r.recorder.Eventf(notification, kapiv1.EventTypeNormal, string(cephv1.ReconcileStarted), "Started reconciling CephBucketNotification %q for ObjectBucketClaim %q", bnName, bucketName)
	if err := r.client.Get(r.opManagerContext, bnName, notification); err != nil {
		if kerrors.IsNotFound(err) {
			return waitForRequeueIfNotificationNotReady, *notification, errors.Wrapf(err, "CephBucketNotification %q not provisioned yet", bnName)
		}
		return reconcile.Result{}, *notification, errors.Wrapf(err, "failed to retrieve CephBucketNotification %q", bnName)
	}

	// get the topic associated with the notification, and make sure it is provisioned
	topicName := types.NamespacedName{Namespace: notification.Namespace, Name: notification.Spec.Topic}
	bucketTopic, err := topic.GetProvisioned(r.client, r.opManagerContext, topicName)
	if err != nil {
		return waitForRequeueIfTopicNotReady, *notification, errors.Wrapf(err, "topic %q not provisioned yet", topicName)
	}

	if err = validateObjectStoreName(bucketTopic, objectStoreName); err != nil {
		return reconcile.Result{}, *notification, err
	}

	// provision the notification
	err = createNotificationFunc(p, &ob, *bucketTopic.Status.ARN, notification)
	if err != nil {
		return reconcile.Result{}, *notification, errors.Wrapf(err, "failed to provision notification for ObjectBucketClaims %q", bucketName)
	}
	logger.Infof("provisioned CephBucketNotification %q for ObjectBucketClaims %q", bnName, bucketName)

	return reconcile.Result{}, *notification, nil
}