ceph: merge rook-config-override to the default global config file

Some parameter like bluestore_min_alloc_size, bluestore_min_alloc_size_hdd need to be set during the bootstrap of the
ceph cluster in order to have bluestore created accordingly.
To do so rook users rely on the rook-config-override configmap.
Sadly the rook-ceph-osd-prepare job does not take in account this configmap to generate its ceph config.
Changing the behaviour to merge the config from rook-config-override configmap to the default ceph config

Signed-off-by: n.fraison <n.fraison@criteo.com>

pkg/daemon/ceph/client/config.go

@@ -19,6 +19,9 @@ package client
 
 import (
 	"fmt"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"net"
 	"os"
 	"path"
@@ -118,6 +121,10 @@ func generateConfigFile(context *clusterd.Context, clusterInfo *ClusterInfo, pat
 		return "", errors.Wrap(err, "failed to create global config section")
 	}
 
+	if err := mergeDefaultConfigWithRookConfigOverride(context, clusterInfo, configFile); err != nil {
+		return "", errors.Wrapf(err, "failed to merge global config with %q", k8sutil.ConfigOverrideName)
+	}
+
 	qualifiedUser := getQualifiedUser(clusterInfo.CephCred.Username)
 	if err := addClientConfigFileSection(configFile, qualifiedUser, keyringPath, clientSettings); err != nil {
 		return "", errors.Wrap(err, "failed to add admin client config section")
@@ -133,6 +140,28 @@ func generateConfigFile(context *clusterd.Context, clusterInfo *ClusterInfo, pat
 	return filePath, nil
 }
 
+func mergeDefaultConfigWithRookConfigOverride(context *clusterd.Context, clusterInfo *ClusterInfo, configFile *ini.File) error {
+	cm, err := context.Clientset.CoreV1().ConfigMaps(clusterInfo.Namespace).Get(k8sutil.ConfigOverrideName, metav1.GetOptions{})
+	if err != nil {
+		if !kerrors.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to read configmap %q", k8sutil.ConfigOverrideName)
+		}
+		return nil
+	}
+
+	config, ok := cm.Data["config"]
+	if !ok || config == "" {
+		logger.Debugf("No ceph configuration override to merge as %q configmap is empty", k8sutil.ConfigOverrideName)
+		return nil
+	}
+
+	if err := configFile.Append([]byte(config)); err != nil {
+		return errors.Wrapf(err, "failed to load config data from %q", k8sutil.ConfigOverrideName)
+	}
+
+	return nil
+}
+
 // prepends "client." if a user namespace is not already specified
 func getQualifiedUser(user string) string {
 	if strings.Index(user, ".") == -1 {

pkg/daemon/ceph/client/config_test.go

@@ -17,7 +17,11 @@ limitations under the License.
 package client
 
 import (
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/operator/test"
 	"io/ioutil"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"os"
 	"path/filepath"
 	"strings"
@@ -84,13 +88,29 @@ func TestGenerateConfigFile(t *testing.T) {
 	defer os.RemoveAll(configDir)
 
 	// create mocked cluster context and info
+	clientset := test.New(t, 3)
+
 	context := &clusterd.Context{
 		ConfigDir: configDir,
+		Clientset: clientset,
 	}
+
+	ns := "foo-cluster"
+	data := make(map[string]string, 1)
+	data["config"] = "[global]\n    bluestore_min_alloc_size_hdd = 4096"
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      k8sutil.ConfigOverrideName,
+			Namespace: ns,
+		},
+		Data: data,
+	}
+	clientset.CoreV1().ConfigMaps(ns).Create(cm)
+
 	clusterInfo := &ClusterInfo{
 		FSID:          "myfsid",
 		MonitorSecret: "monsecret",
-		Namespace:     "foo-cluster",
+		Namespace:     ns,
 		Monitors: map[string]*MonInfo{
 			"node0": {Name: "mon0", Endpoint: "10.0.0.1:6789"},
 		},
@@ -110,6 +130,7 @@ func TestGenerateConfigFile(t *testing.T) {
 	actualConf, err := ini.Load(configFilePath)
 	assert.Nil(t, err)
 	verifyConfigValue(t, actualConf, "global", "fsid", clusterInfo.FSID)
+	verifyConfigValue(t, actualConf, "global", "bluestore_min_alloc_size_hdd", "4096")
 }
 
 func verifyConfig(t *testing.T, cephConfig *CephConfig, cluster *ClusterInfo, loggingLevel int) {