Merge pull request #12548 from travisn/cosi-disable-default

cosi: The COSI driver should be disabled by default

Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md

@@ -2,24 +2,52 @@
 title: Container Object Storage Interface (COSI)
 ---
 
+The Ceph COSI driver provisions buckets for object storage. This document instructs on enabling the driver and consuming a bucket from a sample application.
 
-## Ceph COSI Driver
+!!! note
+    The Ceph COSI driver is currently in experimental mode.
 
-The Ceph COSI driver will be started automatically with default settings when first CephObjectStore gets created. The driver will be deleted when Rook operator is uninstalled. The driver will be deployed in the same namespace as Rook operator. The [COSI controller](https://github.com/kubernetes-sigs/container-object-storage-interface-controller#readme) must be running as a prerequisite. The COSI controller can be deployed by following the commands below:
+## Prerequisites
+
+COSI requires:
+1. A running Rook [object store](object-storage.md)
+2. [COSI controller](https://github.com/kubernetes-sigs/container-object-storage-interface-controller#readme)
+
+Deploy the COSI controller with these commands:
 
 ```bash
 kubectl apply -k github.com/kubernetes-sigs/container-object-storage-interface-api
 kubectl apply -k github.com/kubernetes-sigs/container-object-storage-interface-controller
 ```
 
+## Ceph COSI Driver
+
+The Ceph COSI driver will be started when the CephCOSIDriver CR is created and when the first CephObjectStore is created.
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephCOSIDriver
+metadata:
+  name: ceph-cosi-driver
+  namespace: rook-ceph
+spec:
+  deploymentStrategy: "Auto"
+```
+
+```console
+cd deploy/examples/cosi
+kubectl create -f cephcosidriver.yaml
+```
+
+The driver is created in the same namespace as Rook operator.
+
 ## Admin Operations
 
 ### Create a Ceph Object Store User
 
-First admin need to create CephObjectStoreUser use the following command, this is required for BucketClass and BucketAccessClass:
+Create a CephObjectStoreUser to be used by the COSI driver for provisioning buckets.
 
-```bash
-kubectl -n rook-ceph create -f - <<EOF
+```yaml
 apiVersion: ceph.rook.io/v1
 kind: CephObjectStoreUser
 metadata:
@@ -31,7 +59,10 @@ spec:
   capabilities:
     bucket: "*"
     user: "*"
-EOF
+```
+
+```console
+kubectl create -f cosi-user.yaml
 ```
 
 Above step will be automated in future by the Rook operator.
@@ -62,8 +93,7 @@ parameters:
   objectStoreUserSecretNamespace: rook-ceph
 ```
 
-```command
-cd deploy/examples/cosi
+```console
 kubectl create -f bucketclass.yaml -f bucketaccessclass.yaml
 ```
 

pkg/operator/ceph/object/cosi/controller.go

@@ -125,14 +125,22 @@ func (r *ReconcileCephCOSIDriver) reconcile(request reconcile.Request) (reconcil
 		return reconcile.Result{}, *cephCOSIDriver, errors.Wrapf(err, "failed to get Ceph COSI Driver %s", request.NamespacedName)
 	}
 
-	cosiDeploymentStrategy := cephv1.COSIDeploymentStrategyAuto
+	// While in experimental mode, the COSI driver is not enabled by default
+	cosiDeploymentStrategy := cephv1.COSIDeploymentStrategyNever
+
+	// Get the setting from the CephCOSIDriver CR if exists
 	if !reflect.DeepEqual(cephCOSIDriver.Spec, cephv1.CephCOSIDriverSpec{}) && cephCOSIDriver.Spec.DeploymentStrategy != "" {
 		cosiDeploymentStrategy = cephCOSIDriver.Spec.DeploymentStrategy
 	}
+
 	if cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyNever {
-		logger.Info("Ceph COSI Driver is disabled, delete if exists")
+		logger.Debug("Ceph COSI Driver is disabled, delete if exists")
 		cephCOSIDriverDeployment := &appsv1.Deployment{}
 		err = r.client.Get(r.opManagerContext, request.NamespacedName, cephCOSIDriverDeployment)
+		if kerrors.IsNotFound(err) {
+			// nothing to delete
+			return reconcile.Result{}, *cephCOSIDriver, nil
+		}
 		if err != nil && client.IgnoreNotFound(err) != nil {
 			return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to get Ceph COSI Driver Deployment")
 		}

pkg/operator/ceph/object/cosi/controller_test.go

@@ -14,6 +14,7 @@ import (
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -78,11 +79,11 @@ func TestCephCOSIDriverController(t *testing.T) {
 			Namespace: namespace,
 		},
 	}
-	t.Run("requeue no object store exists", func(t *testing.T) {
+	t.Run("no requeue no object store or cosi driver exists", func(t *testing.T) {
 		r := setupNewEnvironment()
 		res, err := r.Reconcile(ctx, req)
 		assert.NoError(t, err)
-		assert.Equal(t, true, res.Requeue)
+		assert.Equal(t, false, res.Requeue)
 	})
 
 	t.Run("object store exists", func(t *testing.T) {
@@ -98,7 +99,7 @@ func TestCephCOSIDriverController(t *testing.T) {
 		assert.Equal(t, false, res.Requeue)
 		cephCOSIDriverDeployment := &appsv1.Deployment{}
 		err = r.client.Get(ctx, types.NamespacedName{Name: CephCOSIDriverName, Namespace: namespace}, cephCOSIDriverDeployment)
-		assert.NoError(t, err)
+		assert.True(t, kerrors.IsNotFound(err))
 	})
 
 	t.Run("ceph cosi driver CRD with disabled mode without any object stores", func(t *testing.T) {
@@ -215,7 +216,8 @@ func TestCephCOSIDriverController(t *testing.T) {
 				Namespace: namespace,
 			},
 			Spec: cephv1.CephCOSIDriverSpec{
-				Image: "quay.io/ceph/cosi:custom",
+				Image:              "quay.io/ceph/cosi:custom",
+				DeploymentStrategy: cephv1.COSIDeploymentStrategyAuto,
 			},
 		}
 		r := setupNewEnvironment(cephCOSIDriver)
@@ -234,7 +236,8 @@ func TestCephCOSIDriverController(t *testing.T) {
 				Namespace: namespace,
 			},
 			Spec: cephv1.CephCOSIDriverSpec{
-				Image: "quay.io/ceph/cosi:custom",
+				Image:              "quay.io/ceph/cosi:custom",
+				DeploymentStrategy: cephv1.COSIDeploymentStrategyAuto,
 			},
 		}
 		objectStore := &cephv1.CephObjectStore{