From 990cb40043620cb0aa397c5b3d3e8abab15b96b8 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 18 Jul 2023 14:29:05 -0600 Subject: [PATCH 1/3] cosi: change the default deployment strategy to never While experimental, the cosi driver should not be enabled by default. Signed-off-by: travisn --- pkg/operator/ceph/object/cosi/controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/object/cosi/controller.go b/pkg/operator/ceph/object/cosi/controller.go index 6876ea711..c9ba73db8 100644 --- a/pkg/operator/ceph/object/cosi/controller.go +++ b/pkg/operator/ceph/object/cosi/controller.go @@ -125,10 +125,14 @@ func (r *ReconcileCephCOSIDriver) reconcile(request reconcile.Request) (reconcil return reconcile.Result{}, *cephCOSIDriver, errors.Wrapf(err, "failed to get Ceph COSI Driver %s", request.NamespacedName) } - cosiDeploymentStrategy := cephv1.COSIDeploymentStrategyAuto + // While in experimental mode, the COSI driver is not enabled by default + cosiDeploymentStrategy := cephv1.COSIDeploymentStrategyNever + + // Get the setting from the CephCOSIDriver CR if exists if !reflect.DeepEqual(cephCOSIDriver.Spec, cephv1.CephCOSIDriverSpec{}) && cephCOSIDriver.Spec.DeploymentStrategy != "" { cosiDeploymentStrategy = cephCOSIDriver.Spec.DeploymentStrategy } + if cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyNever { logger.Info("Ceph COSI Driver is disabled, delete if exists") cephCOSIDriverDeployment := &appsv1.Deployment{} From 4145ecec37e9ff4b5de02dd8dfebb5e037fb1ded Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 18 Jul 2023 14:34:23 -0600 Subject: [PATCH 2/3] cosi: document enabling the cosi driver Add steps for enabling the cosi driver that is disabled by default. Also add an introduction to the COSI driver and a section for prerequisites. Signed-off-by: travisn --- .../Object-Storage-RGW/cosi.md | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md index aa2b30775..059a059af 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md @@ -2,24 +2,52 @@ title: Container Object Storage Interface (COSI) --- +The Ceph COSI driver provisions buckets for object storage. This document instructs on enabling the driver and consuming a bucket from a sample application. -## Ceph COSI Driver +!!! note + The Ceph COSI driver is currently in experimental mode. -The Ceph COSI driver will be started automatically with default settings when first CephObjectStore gets created. The driver will be deleted when Rook operator is uninstalled. The driver will be deployed in the same namespace as Rook operator. The [COSI controller](https://github.com/kubernetes-sigs/container-object-storage-interface-controller#readme) must be running as a prerequisite. The COSI controller can be deployed by following the commands below: +## Prerequisites + +COSI requires: +1. A running Rook [object store](object-storage.md) +2. [COSI controller](https://github.com/kubernetes-sigs/container-object-storage-interface-controller#readme) + +Deploy the COSI controller with these commands: ```bash kubectl apply -k github.com/kubernetes-sigs/container-object-storage-interface-api kubectl apply -k github.com/kubernetes-sigs/container-object-storage-interface-controller ``` +## Ceph COSI Driver + +The Ceph COSI driver will be started when the CephCOSIDriver CR is created and when the first CephObjectStore is created. + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephCOSIDriver +metadata: + name: ceph-cosi-driver + namespace: rook-ceph +spec: + deploymentStrategy: "Auto" +``` + +```console +cd deploy/examples/cosi +kubectl create -f cephcosidriver.yaml +``` + +The driver is created in the same namespace as Rook operator. + ## Admin Operations ### Create a Ceph Object Store User -First admin need to create CephObjectStoreUser use the following command, this is required for BucketClass and BucketAccessClass: +Create a CephObjectStoreUser to be used by the COSI driver for provisioning buckets. -```bash -kubectl -n rook-ceph create -f - < Date: Tue, 18 Jul 2023 14:35:32 -0600 Subject: [PATCH 3/3] cosi: handle deletion of the cosi driver The controller was attempting to delete the cosi driver again and again, always failing the reconcile when the driver did not exist. Since the desired state is for the driver not to exist when disabled, return a successful reconcile. Signed-off-by: travisn --- pkg/operator/ceph/object/cosi/controller.go | 6 +++++- pkg/operator/ceph/object/cosi/controller_test.go | 13 ++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/operator/ceph/object/cosi/controller.go b/pkg/operator/ceph/object/cosi/controller.go index c9ba73db8..a11838bf7 100644 --- a/pkg/operator/ceph/object/cosi/controller.go +++ b/pkg/operator/ceph/object/cosi/controller.go @@ -134,9 +134,13 @@ func (r *ReconcileCephCOSIDriver) reconcile(request reconcile.Request) (reconcil } if cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyNever { - logger.Info("Ceph COSI Driver is disabled, delete if exists") + logger.Debug("Ceph COSI Driver is disabled, delete if exists") cephCOSIDriverDeployment := &appsv1.Deployment{} err = r.client.Get(r.opManagerContext, request.NamespacedName, cephCOSIDriverDeployment) + if kerrors.IsNotFound(err) { + // nothing to delete + return reconcile.Result{}, *cephCOSIDriver, nil + } if err != nil && client.IgnoreNotFound(err) != nil { return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to get Ceph COSI Driver Deployment") } diff --git a/pkg/operator/ceph/object/cosi/controller_test.go b/pkg/operator/ceph/object/cosi/controller_test.go index de8b977a2..ec6761e3c 100644 --- a/pkg/operator/ceph/object/cosi/controller_test.go +++ b/pkg/operator/ceph/object/cosi/controller_test.go @@ -14,6 +14,7 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -78,11 +79,11 @@ func TestCephCOSIDriverController(t *testing.T) { Namespace: namespace, }, } - t.Run("requeue no object store exists", func(t *testing.T) { + t.Run("no requeue no object store or cosi driver exists", func(t *testing.T) { r := setupNewEnvironment() res, err := r.Reconcile(ctx, req) assert.NoError(t, err) - assert.Equal(t, true, res.Requeue) + assert.Equal(t, false, res.Requeue) }) t.Run("object store exists", func(t *testing.T) { @@ -98,7 +99,7 @@ func TestCephCOSIDriverController(t *testing.T) { assert.Equal(t, false, res.Requeue) cephCOSIDriverDeployment := &appsv1.Deployment{} err = r.client.Get(ctx, types.NamespacedName{Name: CephCOSIDriverName, Namespace: namespace}, cephCOSIDriverDeployment) - assert.NoError(t, err) + assert.True(t, kerrors.IsNotFound(err)) }) t.Run("ceph cosi driver CRD with disabled mode without any object stores", func(t *testing.T) { @@ -215,7 +216,8 @@ func TestCephCOSIDriverController(t *testing.T) { Namespace: namespace, }, Spec: cephv1.CephCOSIDriverSpec{ - Image: "quay.io/ceph/cosi:custom", + Image: "quay.io/ceph/cosi:custom", + DeploymentStrategy: cephv1.COSIDeploymentStrategyAuto, }, } r := setupNewEnvironment(cephCOSIDriver) @@ -234,7 +236,8 @@ func TestCephCOSIDriverController(t *testing.T) { Namespace: namespace, }, Spec: cephv1.CephCOSIDriverSpec{ - Image: "quay.io/ceph/cosi:custom", + Image: "quay.io/ceph/cosi:custom", + DeploymentStrategy: cephv1.COSIDeploymentStrategyAuto, }, } objectStore := &cephv1.CephObjectStore{