my-rook-config/pkg/operator/ceph/object/cosi/controller.go

/*
Copyright 2023 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cosi implements the controller for the Ceph COSI Driver.
package cosi

import (
	"context"
	"os"
	"reflect"
	"time"

	"github.com/coreos/pkg/capnslog"
	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
	"github.com/rook/rook/pkg/operator/ceph/reporting"
	"github.com/rook/rook/pkg/operator/k8sutil"
	appsv1 "k8s.io/api/apps/v1"
	kapiv1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/controller-runtime/pkg/source"
)

const (
	packageName               = "ceph-cosi"
	controllerName            = "ceph-cosi-controller"
	CephCOSIDriverName        = "ceph-cosi-driver"
	COSISideCarName           = "objectstorage-provisioner-sidecar"
	cosiSocketMountPath       = "/var/lib/cosi"
	DefaultServiceAccountName = "objectstorage-provisioner"
	cosiSocketVolumeName      = "socket"
	CephCOSIDriverPrefix      = "rook-ceph"
)

var (
	logger                              = capnslog.NewPackageLogger("github.com/rook/rook", controllerName)
	waitForRequeueObjectStoreNotPresent = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
)

// ReconcileCephCOSIDriver reconciles the Ceph COSI Driver
type ReconcileCephCOSIDriver struct {
	client           client.Client
	context          *clusterd.Context
	scheme           *runtime.Scheme
	opManagerContext context.Context
	recorder         record.EventRecorder
}

// Add creates a new CephCOSIDriver Controller and adds it to the Manager. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
	return add(mgr, newReconciler(mgr, context, opManagerContext))
}

// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler {
	return &ReconcileCephCOSIDriver{
		client:           mgr.GetClient(),
		context:          context,
		scheme:           mgr.GetScheme(),
		opManagerContext: opManagerContext,
		recorder:         mgr.GetEventRecorderFor(controllerName),
	}
}

func add(mgr manager.Manager, r reconcile.Reconciler) error {
	// Create a new controller
	controller, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
	if err != nil {
		return errors.Wrapf(err, "failed to create %s controller", controllerName)
	}

	logger.Info("successfully started")
	// Watch for changes to CephCOSIDriver
	err = controller.Watch(
		source.Kind(
			mgr.GetCache(),
			&cephv1.CephCOSIDriver{},
			&handler.TypedEnqueueRequestForObject[*cephv1.CephCOSIDriver]{},
			opcontroller.WatchControllerPredicate[*cephv1.CephCOSIDriver](mgr.GetScheme()),
		),
	)
	if err != nil {
		return errors.Wrap(err, "failed to watch for CephCOSIDriver object changes")
	}

	// Watch for changes to CephObjectStore as arbitrary resource and predicate functions
	err = controller.Watch(
		source.Kind(
			mgr.GetCache(),
			&cephv1.CephObjectStore{},
			&handler.TypedEnqueueRequestForObject[*cephv1.CephObjectStore]{},
			opcontroller.WatchControllerPredicate[*cephv1.CephObjectStore](mgr.GetScheme()),
		),
	)
	if err != nil {
		return errors.Wrap(err, "failed to watch for CephObjectStore object changes")
	}

	return nil
}

// Reconcile reads that state of the cluster for a CephCOSIDriver object and makes changes based on the state read
// and what is in the CephCOSIDriver.Spec
func (r *ReconcileCephCOSIDriver) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
	reconcileResponse, cephCOSIDriver, err := r.reconcile(request)

	return reporting.ReportReconcileResult(logger, r.recorder, request, &cephCOSIDriver, reconcileResponse, err)
}

func (r *ReconcileCephCOSIDriver) reconcile(request reconcile.Request) (reconcile.Result, cephv1.CephCOSIDriver, error) {
	cephCOSIDriver := &cephv1.CephCOSIDriver{}

	// Check No of instances of CephCOSIDriver
	cephCOSIDriverList := &cephv1.CephCOSIDriverList{}
	err := r.client.List(r.opManagerContext, cephCOSIDriverList)
	if err != nil && client.IgnoreNotFound(err) != nil {
		return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to list CephCOSIDriver")
	}
	if len(cephCOSIDriverList.Items) > 1 {
		return reconcile.Result{}, *cephCOSIDriver, errors.New("more than one instance of CephCOSIDriver found")
	} else if len(cephCOSIDriverList.Items) == 1 {
		cephCOSIDriver = &cephCOSIDriverList.Items[0]
		logger.Debugf("CephCOSIDriver: %+v", cephCOSIDriver)
	}

	// While in experimental mode, the COSI driver is not enabled by default
	cosiDeploymentStrategy := cephv1.COSIDeploymentStrategyNever

	// Get the setting from the CephCOSIDriver CR if exists
	if !reflect.DeepEqual(cephCOSIDriver.Spec, cephv1.CephCOSIDriverSpec{}) && cephCOSIDriver.Spec.DeploymentStrategy != "" {
		cosiDeploymentStrategy = cephCOSIDriver.Spec.DeploymentStrategy
	}

	if cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyNever {
		logger.Debug("Ceph COSI Driver is disabled, delete if exists")
		cephCOSIDriverDeployment := &appsv1.Deployment{}
		err = r.client.Get(r.opManagerContext, request.NamespacedName, cephCOSIDriverDeployment)
		if kerrors.IsNotFound(err) {
			// nothing to delete
			return reconcile.Result{}, *cephCOSIDriver, nil
		}
		if err != nil && client.IgnoreNotFound(err) != nil {
			return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to get Ceph COSI Driver Deployment")
		}
		err = r.client.Delete(r.opManagerContext, cephCOSIDriverDeployment)
		if err != nil && client.IgnoreNotFound(err) != nil {
			return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to delete Ceph COSI Driver Deployment")
		}
		return reconcile.Result{}, *cephCOSIDriver, nil
	}

	r.recorder.Eventf(cephCOSIDriver, kapiv1.EventTypeNormal, "Reconciling", "Reconciling Ceph COSI Driver %q", cephCOSIDriver.Name)
	// Set the default CephCOSIDriver name if not already set
	if cephCOSIDriver.Name == "" {
		cephCOSIDriver.Name = CephCOSIDriverName
	}

	// The ceph-cosi-driver CRD should be same namespace as the operator
	if cephCOSIDriver.Namespace != os.Getenv(k8sutil.PodNamespaceEnvVar) {
		return reconcile.Result{}, *cephCOSIDriver, errors.New("Ceph COSI Driver namespace must be same as operator")
	}

	// Check whether object store is running
	list := &cephv1.CephObjectStoreList{}
	err = r.client.List(r.opManagerContext, list)
	if err != nil {
		return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to list CephObjectStore")
	}

	if len(list.Items) == 0 && cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyAuto {
		logger.Info("no object stores found, skipping cosi driver config")
		return waitForRequeueObjectStoreNotPresent, *cephCOSIDriver, nil
	}

	// Start the Ceph COSI Driver
	err = r.startCephCOSIDriver(cephCOSIDriver)
	if err != nil {
		return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to start Ceph COSI Driver")
	}

	return reconcile.Result{}, *cephCOSIDriver, nil
}

// Start the Ceph COSI Driver
func (r *ReconcileCephCOSIDriver) startCephCOSIDriver(cephCOSIDriver *cephv1.CephCOSIDriver) error {
	// Create the Ceph COSI Driver Deployment
	cephcosidriverDeployment, err := createCephCOSIDriverDeployment(cephCOSIDriver)
	if err != nil {
		return errors.Wrap(err, "failed to create Ceph COSI Driver Deployment CRD")
	}

	logger.Infof("starting Ceph COSI Driver deployment %q in namespace %q", cephcosidriverDeployment.Name, cephcosidriverDeployment.Namespace)

	err = r.client.Create(r.opManagerContext, cephcosidriverDeployment)
	if err != nil {
		if !kerrors.IsAlreadyExists(err) {
			return errors.Wrap(err, "failed to create Ceph COSI Driver deployment")
		}
		logger.Info("Ceph COSI Driver deployment already exists, updating")
		err = r.client.Update(r.opManagerContext, cephcosidriverDeployment)
		if err != nil {
			return errors.Wrap(err, "failed to update Ceph COSI Driver deployment")
		}
	} else {
		logger.Info("Ceph COSI Driver deployment started")
	}

	return nil
}