From b0aefed8e56c2bc36bb33fa1f06fd2a9f095ac7d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 17 Jan 2025 15:57:48 -0700 Subject: [PATCH] osd: enable node specific ceph conf overrides Some ceph.conf settings need to be unique per node depending on the hardware. The ceph.conf overrides can now be customized per-node by creating a node-specific configmap that will be loaded for all OSDs and OSD prepare jobs on that node, instead of the default ceph.conf that is loaded from the rook-config-override configmap. The node-specific configmaps must have the label node.config.rook.io/osd, and be named as: rook-config-override- Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/osd/osd.go | 25 +++++++++++++++++++ .../ceph/cluster/osd/provision_spec.go | 1 + pkg/operator/ceph/cluster/osd/spec.go | 20 +++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index e1bf09cb3..434d1c41b 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -88,6 +88,7 @@ type Cluster struct { deviceSets []deviceSet migrateOSD *OSDInfo deprecatedOSDs map[string][]int + nodeConfigmaps map[string]struct{} } // New creates an instance of the OSD manager @@ -206,6 +207,9 @@ func (c *Cluster) Start() error { if err := c.validateOSDSettings(); err != nil { return err } + if err := c.initializeNodeConfigmaps(); err != nil { + return err + } logger.Infof("start running osds in namespace %q", namespace) if !c.spec.Storage.UseAllNodes && len(c.spec.Storage.Nodes) == 0 && len(c.spec.Storage.StorageClassDeviceSets) == 0 { @@ -1045,3 +1049,24 @@ func getOSDLocationFromArgs(args []string) (string, bool) { return "", false } + +func (c *Cluster) initializeNodeConfigmaps() error { + // Find existing configmaps matching the label + label := "node.config.rook.io/osd" + options := metav1.ListOptions{LabelSelector: label} + configmaps, err := c.context.Clientset.CoreV1().ConfigMaps(c.clusterInfo.Namespace).List(c.clusterInfo.Context, options) + if err != nil { + return errors.Wrapf(err, "failed to list configmaps in namespace %q with label %q", c.clusterInfo.Namespace, label) + } + + c.nodeConfigmaps = map[string]struct{}{} + prefix := k8sutil.ConfigOverrideName + "-" + for _, configmap := range configmaps.Items { + if strings.HasPrefix(configmap.Name, prefix) { + nodeName := strings.TrimPrefix(configmap.Name, prefix) + logger.Infof("found node %q configmap, will mount it for alternate ceph conf overrides for osds on this node", nodeName) + c.nodeConfigmaps[nodeName] = struct{}{} + } + } + return nil +} diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 61f31664a..aa2c39437 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -104,6 +104,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re // ceph-volume is currently set up to use /etc/ceph/ceph.conf; this means no user config // overrides will apply to ceph-volume, but this is unnecessary anyway volumes := append(controller.PodVolumes(provisionConfig.DataPathMap, c.spec.DataDirHostPath, c.spec.DataDirHostPath, true), copyBinariesVolume) + volumes = c.updateCephConfigVolume(volumes, osdProps.crushHostname) // create a volume on /dev so the pod can access devices on the host devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 79c066bf2..7f38fc549 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -325,6 +325,24 @@ func deploymentName(osdID int) string { return fmt.Sprintf(osdAppNameFmt, osdID) } +func (c *Cluster) updateCephConfigVolume(volumes []v1.Volume, nodeName string) []v1.Volume { + if _, ok := c.nodeConfigmaps[nodeName]; !ok { + logger.Debugf("no configmap to override for node %q", nodeName) + return volumes + } + + updatedVolumes := []v1.Volume{} + for _, volume := range volumes { + if volume.Name == k8sutil.ConfigOverrideName { + logger.Debugf("found configmap volume to override for node %q", nodeName) + name := fmt.Sprintf("%s-%s", k8sutil.ConfigOverrideName, nodeName) + volume.VolumeSource.Projected.Sources[0].ConfigMap.LocalObjectReference.Name = name + } + updatedVolumes = append(updatedVolumes, volume) + } + return updatedVolumes +} + func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provisionConfig *provisionConfig) (*apps.Deployment, error) { deploymentName := deploymentName(osd.ID) replicaCount := int32(1) @@ -336,6 +354,8 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision dataDirHostPath = "" } volumes := controller.PodVolumes(provisionConfig.DataPathMap, dataDirHostPath, c.spec.DataDirHostPath, false) + volumes = c.updateCephConfigVolume(volumes, osdProps.crushHostname) + failureDomainValue := osdProps.crushHostname doConfigInit := true // initialize ceph.conf in init container? doBinaryCopyInit := true // copy rook binary in an init container?