osd: enable node specific ceph conf overrides

Some ceph.conf settings need to be unique per node
depending on the hardware. The ceph.conf overrides
can now be customized per-node by creating a node-specific
configmap that will be loaded for all OSDs and OSD prepare
jobs on that node, instead of the default ceph.conf
that is loaded from the rook-config-override configmap.

The node-specific configmaps must have the label
node.config.rook.io/osd, and be named as:
rook-config-override-<hostname>

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>

pkg/operator/ceph/cluster/osd/osd.go

@@ -88,6 +88,7 @@ type Cluster struct {
 	deviceSets     []deviceSet
 	migrateOSD     *OSDInfo
 	deprecatedOSDs map[string][]int
+	nodeConfigmaps map[string]struct{}
 }
 
 // New creates an instance of the OSD manager
@@ -206,6 +207,9 @@ func (c *Cluster) Start() error {
 	if err := c.validateOSDSettings(); err != nil {
 		return err
 	}
+	if err := c.initializeNodeConfigmaps(); err != nil {
+		return err
+	}
 	logger.Infof("start running osds in namespace %q", namespace)
 
 	if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 && len(c.spec.Storage.StorageClassDeviceSets) == 0 {
@@ -1045,3 +1049,24 @@ func getOSDLocationFromArgs(args []string) (string, bool) {
 
 	return "", false
 }
+
+func (c *Cluster) initializeNodeConfigmaps() error {
+	// Find existing configmaps matching the label
+	label := "node.config.rook.io/osd"
+	options := metav1.ListOptions{LabelSelector: label}
+	configmaps, err := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace).List(c.clusterInfo.Context, options)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list configmaps in namespace %q with label %q", c.clusterInfo.Namespace, label)
+	}
+
+	c.nodeConfigmaps = map[string]struct{}{}
+	prefix := k8sutil.ConfigOverrideName + "-"
+	for _, configmap := range configmaps.Items {
+		if strings.HasPrefix(configmap.Name, prefix) {
+			nodeName := strings.TrimPrefix(configmap.Name, prefix)
+			logger.Infof("found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName)
+			c.nodeConfigmaps[nodeName] = struct{}{}
+		}
+	}
+	return nil
+}

pkg/operator/ceph/cluster/osd/provision_spec.go

@@ -104,6 +104,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
 	// ceph-volume is currently set up to use /etc/ceph/ceph.conf; this means no user config
 	// overrides will apply to ceph-volume, but this is unnecessary anyway
 	volumes := append(controller.PodVolumes(provisionConfig.DataPathMap, c.spec.DataDirHostPath, c.spec.DataDirHostPath, true), copyBinariesVolume)
+	volumes = c.updateCephConfigVolume(volumes, osdProps.crushHostname)
 
 	// create a volume on /dev so the pod can access devices on the host
 	devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}

pkg/operator/ceph/cluster/osd/spec.go

@@ -325,6 +325,24 @@ func deploymentName(osdID int) string {
 	return fmt.Sprintf(osdAppNameFmt, osdID)
 }
 
+func (c *Cluster) updateCephConfigVolume(volumes []v1.Volume, nodeName string) []v1.Volume {
+	if _, ok := c.nodeConfigmaps[nodeName]; !ok {
+		logger.Debugf("no configmap to override for node %q", nodeName)
+		return volumes
+	}
+
+	updatedVolumes := []v1.Volume{}
+	for _, volume := range volumes {
+		if volume.Name == k8sutil.ConfigOverrideName {
+			logger.Debugf("found configmap volume to override for node %q", nodeName)
+			name := fmt.Sprintf("%s-%s", k8sutil.ConfigOverrideName, nodeName)
+			volume.VolumeSource.Projected.Sources[0].ConfigMap.LocalObjectReference.Name = name
+		}
+		updatedVolumes = append(updatedVolumes, volume)
+	}
+	return updatedVolumes
+}
+
 func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) {
 	deploymentName := deploymentName(osd.ID)
 	replicaCount := int32(1)
@@ -336,6 +354,8 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
 		dataDirHostPath = ""
 	}
 	volumes := controller.PodVolumes(provisionConfig.DataPathMap, dataDirHostPath, c.spec.DataDirHostPath, false)
+	volumes = c.updateCephConfigVolume(volumes, osdProps.crushHostname)
+
 	failureDomainValue := osdProps.crushHostname
 	doConfigInit := true     // initialize ceph.conf in init container?
 	doBinaryCopyInit := true // copy rook binary in an init container?