Files
my-rook-config/pkg/operator/ceph/cluster/osd/topology.go
T
Travis Nielsen 291bf9e061 ceph: enforce portable osds in same topology as osd prepare job
The osd prepare job detects the failure domain for the OSD by querying
the node topology labels. This topology is then assigned to the OSD
daemon in the CRUSH map. Previously, the affinity was required to be
set in the cluster CR, but it was very difficult to get right. Now the
operator will enforce the correct topology label on the OSD daemon
nodeAffinity by setting the label of the lowest topology in the hierarcy.

For example, if there are region, zone, and rack labels, the rack label
would be used to set the node affinity for the OSD daemon. If an
OSD prepare job is run in rack1, the corresponding OSD daemon will
have node affinity to rack1 to ensure the same topology. Previously,
the OSD could have ended up in rack2 unless the storageClassDeviceSet
placement was very carefully crafted.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
2021-02-18 12:49:37 -07:00

119 lines
4.1 KiB
Go

/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package config provides methods for generating the Ceph config for a Ceph cluster and for
// producing a "ceph.conf" compatible file from the config as well as Ceph command line-compatible
// flags.
package osd
import (
"fmt"
"github.com/rook/rook/pkg/daemon/ceph/client"
corev1 "k8s.io/api/core/v1"
)
var (
// The labels that can be specified with the K8s labels such as topology.kubernetes.io/zone
// These are all at the top layers of the CRUSH map.
KubernetesTopologyLabels = []string{"zone", "region"}
// The node labels that are supported with the topology.rook.io prefix such as topology.rook.io/rack
// The labels are in order from lowest to highest in the CRUSH hierarchy
CRUSHTopologyLabels = []string{"chassis", "rack", "row", "pdu", "pod", "room", "datacenter"}
// The list of supported failure domains in the CRUSH map, ordered from lowest to highest
CRUSHMapLevelsOrdered = append([]string{"host"}, append(CRUSHTopologyLabels, KubernetesTopologyLabels...)...)
)
const (
topologyLabelPrefix = "topology.rook.io/"
)
// ExtractTopologyFromLabels extracts rook topology from labels and returns a map from topology type to value
func ExtractOSDTopologyFromLabels(labels map[string]string) (map[string]string, string) {
topology, topologyAffinity := extractTopologyFromLabels(labels)
// Ensure the topology names are normalized for CRUSH
for name, value := range topology {
topology[name] = client.NormalizeCrushName(value)
}
return topology, topologyAffinity
}
// ExtractTopologyFromLabels extracts rook topology from labels and returns a map from topology type to value
func extractTopologyFromLabels(labels map[string]string) (map[string]string, string) {
topology := make(map[string]string)
// The topology affinity for the osd is the lowest topology label found in the hierarchy,
// not including the host name
var topologyAffinity string
// check for the region k8s topology label that was deprecated in 1.17
const regionLabel = "region"
region, ok := labels[corev1.LabelZoneRegion]
if ok {
topology[regionLabel] = region
topologyAffinity = formatTopologyAffinity(corev1.LabelZoneRegion, region)
}
// check for the region k8s topology label that is GA in 1.17.
region, ok = labels[corev1.LabelZoneRegionStable]
if ok {
topology[regionLabel] = region
topologyAffinity = formatTopologyAffinity(corev1.LabelZoneRegionStable, region)
}
// check for the zone k8s topology label that was deprecated in 1.17
const zoneLabel = "zone"
zone, ok := labels[corev1.LabelZoneFailureDomain]
if ok {
topology[zoneLabel] = zone
topologyAffinity = formatTopologyAffinity(corev1.LabelZoneFailureDomain, zone)
}
// check for the zone k8s topology label that is GA in 1.17.
zone, ok = labels[corev1.LabelZoneFailureDomainStable]
if ok {
topology[zoneLabel] = zone
topologyAffinity = formatTopologyAffinity(corev1.LabelZoneFailureDomainStable, zone)
}
// get host
host, ok := labels[corev1.LabelHostname]
if ok {
topology["host"] = host
}
// get the labels for the CRUSH map hierarchy
// iterate in reverse order so that the last topology found will be the lowest level in the hierarchy
// for the topology affinity
for i := len(CRUSHTopologyLabels) - 1; i >= 0; i-- {
topologyID := CRUSHTopologyLabels[i]
label := topologyLabelPrefix + topologyID
if value, ok := labels[label]; ok {
topology[topologyID] = value
topologyAffinity = formatTopologyAffinity(label, value)
}
}
return topology, topologyAffinity
}
func formatTopologyAffinity(label, value string) string {
return fmt.Sprintf("%s=%s", label, value)
}