ceph: mds use ceph mon config store

The Ceph Mimic release introduced the monitor config store which allows
us to set configuration flag per daemon in a centralized fashion. Let's
use it for Mimic and newer and keep the CLI flag of daemon startup for
older versions.
The main benefit is that the config of a daemon can now be overridden.

Also, this commit refactors some code and add methods that are unit
tested.

Closes: https://github.com/rook/rook/issues/4307
Signed-off-by: Sébastien Han <seb@redhat.com>

pkg/operator/ceph/file/mds/config.go

@@ -18,9 +18,11 @@ package mds
 
 import (
 	"fmt"
+	"strconv"
 
 	apps "k8s.io/api/apps/v1"
 
+	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -66,3 +68,30 @@ func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) e
 	s := keyring.GetSecretStoreForDeployment(c.context, d)
 	return s.CreateOrUpdate(d.GetName(), existingKeyring)
 }
+
+func (c *Cluster) setDefaultFlagsMonConfigStore(mdsID string) error {
+	monStore := config.GetMonStore(c.context, c.fs.Namespace)
+	who := fmt.Sprintf("mds.%s", mdsID)
+	configOptions := make(map[string]string)
+
+	// Set mds cache memory limit to the best appropriate value
+	if !c.fs.Spec.MetadataServer.Resources.Limits.Memory().IsZero() {
+		mdsCacheMemoryLimit := float64(c.fs.Spec.MetadataServer.Resources.Limits.Memory().Value()) * mdsCacheMemoryLimitFactor
+		configOptions["mds_cache_memory_limit"] = strconv.Itoa(int(mdsCacheMemoryLimit))
+	}
+
+	// These flags are obsoleted as of Nautilus
+	if !c.clusterInfo.CephVersion.IsAtLeastNautilus() {
+		configOptions["mds_standby_for_fscid"] = c.fsID
+		configOptions["mds_standby_replay"] = strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby)
+	}
+
+	for flag, val := range configOptions {
+		err := monStore.Set(who, flag, val)
+		if err != nil {
+			return fmt.Errorf("failed to set %q to %q on %q. %+v", flag, val, who, err)
+		}
+	}
+
+	return nil
+}

pkg/operator/ceph/file/mds/mds.go

@@ -141,6 +141,20 @@ func (c *Cluster) Start() error {
 			return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err)
 		}
 
+		// Check for existing deployment and set the daemon config flags
+		_, err = c.context.Clientset.AppsV1().Deployments(c.fs.Namespace).Get(mdsConfig.ResourceName, metav1.GetOptions{})
+		// We don't need to handle any error here
+		if err != nil {
+			// Apply the flag only when the deployment is not found
+			if errors.IsNotFound(err) {
+				logger.Info("setting mds config flags")
+				err = c.setDefaultFlagsMonConfigStore(mdsConfig.DaemonID)
+				if err != nil {
+					return fmt.Errorf("failed to set default mds config options. %+v", err)
+				}
+			}
+		}
+
 		// start the deployment
 		d := c.makeDeployment(mdsConfig)
 		logger.Debugf("starting mds: %+v", d)

pkg/operator/ceph/file/mds/spec.go

@@ -18,11 +18,9 @@ package mds
 
 import (
 	"fmt"
-	"strconv"
 
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
-	"github.com/rook/rook/pkg/operator/ceph/config"
 	opspec "github.com/rook/rook/pkg/operator/ceph/spec"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	apps "k8s.io/api/apps/v1"
@@ -104,20 +102,6 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
 		"--foreground",
 	)
 
-	// These flags are obsoleted as of Nautilus
-	if !c.clusterInfo.CephVersion.IsAtLeastNautilus() {
-		args = append(
-			args,
-			config.NewFlag("mds-standby-for-fscid", c.fsID),
-			config.NewFlag("mds-standby-replay", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby)))
-	}
-
-	// Set mds cache memory limit to the best appropriate value
-	if !c.fs.Spec.MetadataServer.Resources.Limits.Memory().IsZero() {
-		mdsCacheMemoryLimit := float64(c.fs.Spec.MetadataServer.Resources.Limits.Memory().Value()) * mdsCacheMemoryLimitFactor
-		args = append(args, config.NewFlag("mds-cache-memory-limit", strconv.Itoa(int(mdsCacheMemoryLimit))))
-	}
-
 	container := v1.Container{
 		Name: "mds",
 		Command: []string{