ceph: convert the CephCluster controller to the controller-runtime

This is the final conversion to controller-runtime conversion. This time the
CephCluster CRD has been converted to use the controller-runtime
library.
The controller incorporates all the previous watchers too, so the Node
and hot-plug configmap are been watched too.
Only the operator setting configmap is not being watcher since it's not
related to the CephCluster CRD.
Not only the patch converts to controller-runtime but also tries to
re-organize the tree of the repo to actually make the code more
readable and have better functions/methods/tests separations.

Closes: https://github.com/rook/rook/issues/4939
Signed-off-by: Sébastien Han <seb@redhat.com>

pkg/operator/ceph/object/controller.go

@@ -101,6 +101,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 	if err != nil {
 		return err
 	}
+	logger.Info("successfully started")
 
 	// Watch for changes on the cephObjectStore CRD object
 	err = c.Watch(&source.Kind{Type: &cephv1.CephObjectStore{TypeMeta: controllerTypeMeta}}, &handler.EnqueueRequestForObject{}, opcontroller.WatchControllerPredicate())

pkg/operator/ceph/object/rgw.go

@@ -156,19 +156,12 @@ func (c *clusterConfig) startRGWPods() error {
 				return errors.Wrapf(createErr, "failed to create rgw deployment")
 			}
 			logger.Infof("object store %q deployment %q already exists. updating if needed", c.store.Name, deployment.Name)
-			_, err = c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Get(deployment.Name, metav1.GetOptions{})
-			if err != nil {
-				return errors.Wrapf(err, "failed to get existing rgw deployment %q for update", deployment.Name)
-			}
-		}
-
-		// Generate the mime.types file after the rep. controller as well for the same reason as keyring
-		if createErr != nil && kerrors.IsAlreadyExists(createErr) {
 			if err := updateDeploymentAndWait(c.context, deployment, c.store.Namespace, config.RgwType, daemonLetterID, c.skipUpgradeChecks, c.clusterSpec.ContinueUpgradeAfterChecksEvenIfNotHealthy); err != nil {
 				return errors.Wrapf(err, "failed to update object store %q deployment %q", c.store.Name, deployment.Name)
 			}
 		}
 
+		// Generate the mime.types file after the rep. controller as well for the same reason as keyring
 		if err := c.generateMimeTypes(); err != nil {
 			return errors.Wrap(err, "failed to generate the rgw mime.types config")
 		}

pkg/operator/ceph/object/user/controller.go

@@ -95,6 +95,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 	if err != nil {
 		return err
 	}
+	logger.Info("successfully started")
 
 	// Watch for changes on the CephObjectStoreUser CRD object
 	err = c.Watch(&source.Kind{Type: &cephv1.CephObjectStoreUser{TypeMeta: controllerTypeMeta}}, &handler.EnqueueRequestForObject{}, opcontroller.WatchControllerPredicate())