From 08219a449264cefb7b9dfd77d827a2c7e65f0707 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 22 Jun 2019 12:12:15 -0400
Subject: [PATCH] ceph: update a cross-cluster csi configmap when mons change

Instead of requring csi instances for every ceph cluster that rook
manages, maintain one config map for all ceph clusters. This allows
for one set of csi drivers per rook operator.
The config is intialized as empty when Ceph CSI components are first
created and then updated every time a cluster's mons change.
As before clusters are identified by the k8s namespace the ceph cluster
is running in.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 pkg/operator/ceph/cluster/mon/mon.go | 47 +++++++++++++++++++++++++---
 pkg/operator/ceph/csi/spec.go        | 30 ++++++++++++++++++
 2 files changed, 73 insertions(+), 4 deletions(-)

diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go
index 71e2868d5..b3f0b3fc0 100644
--- a/pkg/operator/ceph/cluster/mon/mon.go
+++ b/pkg/operator/ceph/cluster/mon/mon.go
@@ -22,6 +22,7 @@ package mon
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +36,7 @@ import (
 	cephutil "github.com/rook/rook/pkg/daemon/ceph/util"
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
+	"github.com/rook/rook/pkg/operator/ceph/csi"
 	opspec "github.com/rook/rook/pkg/operator/ceph/spec"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -63,9 +65,6 @@ const (
 	adminSecretName   = "admin-secret"
 	clusterSecretName = "cluster-name"
 
-	// configuration map for csi
-	csiConfigKey = "csi-cluster-config-json"
-
 	// DefaultMonCount Default mon count for a cluster
 	DefaultMonCount = 3
 	// MaxMonCount Maximum allowed mon count for a cluster
@@ -594,7 +593,7 @@ func (c *Cluster) saveMonConfig() error {
 		EndpointDataKey: FlattenMonEndpoints(c.clusterInfo.Monitors),
 		MaxMonIDKey:     strconv.Itoa(c.maxMonID),
 		MappingKey:      string(monMapping),
-		csiConfigKey:    csiConfigValue,
+		csi.ConfigKey:   csiConfigValue,
 	}
 
 	if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil {
@@ -619,6 +618,46 @@ func (c *Cluster) saveMonConfig() error {
 		return fmt.Errorf("failed to write connection config for new mons. %+v", err)
 	}
 
+	if err := c.saveCsiClusterConfig(); err != nil {
+		return fmt.Errorf("failed to update csi cluster config: %+v", err)
+	}
+
+	return nil
+}
+
+func (c *Cluster) saveCsiClusterConfig() error {
+	if !csi.CSIEnabled() {
+		return nil
+	}
+	c.csiConfigMutex.Lock()
+	defer c.csiConfigMutex.Unlock()
+	// csi is deployed into the same namespace as the operator
+	csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
+	logger.Debug("Using %+v for CSI ConfigMap Namespace", csiNamespace)
+
+	// fetch current ConfigMap contents
+	configMap, err := c.context.Clientset.CoreV1().ConfigMaps(csiNamespace).Get(
+		csi.ConfigName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to fetch current csi config map: %+v", err)
+	}
+
+	// update ConfigMap contents for current cluster
+	currData := configMap.Data[csi.ConfigKey]
+	if currData == "" {
+		currData = "[]"
+	}
+	newData, err := UpdateCsiClusterConfig(
+		currData, c.Namespace, c.clusterInfo.Monitors)
+	if err != nil {
+		return fmt.Errorf("failed to update csi config map data: %+v", err)
+	}
+	configMap.Data[csi.ConfigKey] = newData
+
+	// update ConfigMap with new contents
+	if _, err := c.context.Clientset.CoreV1().ConfigMaps(csiNamespace).Update(configMap); err != nil {
+		return fmt.Errorf("failed to update csi config map: %+v", err)
+	}
 	return nil
 }
 
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index d1cc05e9e..83f61d4e6 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -23,6 +23,9 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 
 	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -48,6 +51,10 @@ var (
 
 	CephFSPluginTemplatePath      string
 	CephFSProvisionerTemplatePath string
+
+	// configuration map for csi
+	ConfigName = "rook-ceph-csi-config"
+	ConfigKey  = "csi-cluster-config-json"
 )
 
 const (
@@ -118,6 +125,10 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 		rbdProvisioner, cephfsProvisioner *apps.StatefulSet
 	)
 
+	// create an empty config map. config map will be filled with data
+	// later when clusters have mons
+	CreateCsiConfigMap(namespace, clientset)
+
 	if EnableRBD {
 		rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath)
 		if err != nil {
@@ -168,3 +179,22 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 	}
 	return nil
 }
+
+func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface) error {
+	configMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ConfigName,
+			Namespace: namespace,
+		},
+	}
+	configMap.Data = map[string]string{
+		ConfigKey: "[]",
+	}
+
+	if _, err := clientset.CoreV1().ConfigMaps(namespace).Create(configMap); err != nil {
+		if !k8serrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+	return nil
+}