ceph: update a cross-cluster csi configmap when mons change

Instead of requring csi instances for every ceph cluster that rook
manages, maintain one config map for all ceph clusters. This allows
for one set of csi drivers per rook operator.
The config is intialized as empty when Ceph CSI components are first
created and then updated every time a cluster's mons change.
As before clusters are identified by the k8s namespace the ceph cluster
is running in.

Signed-off-by: John Mulligan <jmulligan@redhat.com>

pkg/operator/ceph/cluster/mon/mon.go

@@ -22,6 +22,7 @@ package mon
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +36,7 @@ import (
 	cephutil "github.com/rook/rook/pkg/daemon/ceph/util"
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
+	"github.com/rook/rook/pkg/operator/ceph/csi"
 	opspec "github.com/rook/rook/pkg/operator/ceph/spec"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -63,9 +65,6 @@ const (
 	adminSecretName   = "admin-secret"
 	clusterSecretName = "cluster-name"
 
-	// configuration map for csi
-	csiConfigKey = "csi-cluster-config-json"
-
 	// DefaultMonCount Default mon count for a cluster
 	DefaultMonCount = 3
 	// MaxMonCount Maximum allowed mon count for a cluster
@@ -594,7 +593,7 @@ func (c *Cluster) saveMonConfig() error {
 		EndpointDataKey: FlattenMonEndpoints(c.clusterInfo.Monitors),
 		MaxMonIDKey:     strconv.Itoa(c.maxMonID),
 		MappingKey:      string(monMapping),
-		csiConfigKey:    csiConfigValue,
+		csi.ConfigKey:   csiConfigValue,
 	}
 
 	if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil {
@@ -619,6 +618,46 @@ func (c *Cluster) saveMonConfig() error {
 		return fmt.Errorf("failed to write connection config for new mons. %+v", err)
 	}
 
+	if err := c.saveCsiClusterConfig(); err != nil {
+		return fmt.Errorf("failed to update csi cluster config: %+v", err)
+	}
+
+	return nil
+}
+
+func (c *Cluster) saveCsiClusterConfig() error {
+	if !csi.CSIEnabled() {
+		return nil
+	}
+	c.csiConfigMutex.Lock()
+	defer c.csiConfigMutex.Unlock()
+	// csi is deployed into the same namespace as the operator
+	csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
+	logger.Debug("Using %+v for CSI ConfigMap Namespace", csiNamespace)
+
+	// fetch current ConfigMap contents
+	configMap, err := c.context.Clientset.CoreV1().ConfigMaps(csiNamespace).Get(
+		csi.ConfigName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to fetch current csi config map: %+v", err)
+	}
+
+	// update ConfigMap contents for current cluster
+	currData := configMap.Data[csi.ConfigKey]
+	if currData == "" {
+		currData = "[]"
+	}
+	newData, err := UpdateCsiClusterConfig(
+		currData, c.Namespace, c.clusterInfo.Monitors)
+	if err != nil {
+		return fmt.Errorf("failed to update csi config map data: %+v", err)
+	}
+	configMap.Data[csi.ConfigKey] = newData
+
+	// update ConfigMap with new contents
+	if _, err := c.context.Clientset.CoreV1().ConfigMaps(csiNamespace).Update(configMap); err != nil {
+		return fmt.Errorf("failed to update csi config map: %+v", err)
+	}
 	return nil
 }
 

pkg/operator/ceph/csi/spec.go

@@ -23,6 +23,9 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 
 	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -48,6 +51,10 @@ var (
 
 	CephFSPluginTemplatePath      string
 	CephFSProvisionerTemplatePath string
+
+	// configuration map for csi
+	ConfigName = "rook-ceph-csi-config"
+	ConfigKey  = "csi-cluster-config-json"
 )
 
 const (
@@ -118,6 +125,10 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 		rbdProvisioner, cephfsProvisioner *apps.StatefulSet
 	)
 
+	// create an empty config map. config map will be filled with data
+	// later when clusters have mons
+	CreateCsiConfigMap(namespace, clientset)
+
 	if EnableRBD {
 		rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath)
 		if err != nil {
@@ -168,3 +179,22 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 	}
 	return nil
 }
+
+func CreateCsiConfigMap(namespace string, clientset kubernetes.Interface) error {
+	configMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ConfigName,
+			Namespace: namespace,
+		},
+	}
+	configMap.Data = map[string]string{
+		ConfigKey: "[]",
+	}
+
+	if _, err := clientset.CoreV1().ConfigMaps(namespace).Create(configMap); err != nil {
+		if !k8serrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+	return nil
+}