ceph: remove topologyAware CRD option
We removed the topologyAware CRD option since it was redundant with the use of an OSD being backed by a PVC. So now, if an OSD is backed by a PVC we assume the topology aware decision and will discover zone and region labels on that host. Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
1 parent
1f788cf637
commit
ef6815ebfa
12 files changed
+24
-53
No files matched your search
@@ -125,8 +125,6 @@ For more details on the mons and when to choose a number other than `3`, see the
|
||||
* `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
|
||||
* `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values.
|
||||
If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`.
|
||||
* `topologyAware`: `true` or `false`, indicating whether Rook will look for and use topology/failure domain labels on Kubernetes nodes (e.g. "region" or "zone") as part of the CRUSH location for OSDs.
|
||||
Node labels must follow the formatting of the failure domain [well-known labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/).
|
||||
* `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below.
|
||||
`useAllNodes` must be set to `false` to use specific nodes and their config.
|
||||
See [node settings](#node-settings) below.
|
||||
|
||||
@@ -143,8 +143,6 @@ spec:
|
||||
path:
|
||||
type: string
|
||||
config: {}
|
||||
topologyAware:
|
||||
type: boolean
|
||||
storageClassDeviceSets: {}
|
||||
monitoring:
|
||||
properties:
|
||||
|
||||
@@ -162,8 +162,6 @@ spec:
|
||||
path:
|
||||
type: string
|
||||
config: {}
|
||||
topologyAware:
|
||||
type: boolean
|
||||
storageClassDeviceSets: {}
|
||||
monitoring:
|
||||
properties:
|
||||
|
||||
@@ -145,8 +145,6 @@ spec:
|
||||
path:
|
||||
type: string
|
||||
config: {}
|
||||
topologyAware:
|
||||
type: boolean
|
||||
storageClassDeviceSets: {}
|
||||
monitoring:
|
||||
properties:
|
||||
|
||||
@@ -51,7 +51,6 @@ type config struct {
|
||||
networkInfo clusterd.NetworkInfo
|
||||
monEndpoints string
|
||||
nodeName string
|
||||
topologyAware bool
|
||||
pvcBacked bool
|
||||
}
|
||||
|
||||
|
||||
+12
-20
@@ -87,7 +87,6 @@ func addOSDFlags(command *cobra.Command) {
|
||||
provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
|
||||
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
|
||||
provisionCmd.Flags().BoolVar(&cfg.pvcBacked, "pvc-backed-osd", false, "true to specify a block mode pvc is backing the OSD")
|
||||
provisionCmd.Flags().BoolVar(&cfg.topologyAware, "topology-aware", false, "true to specify crush location should be set based on the node's zone/region labels")
|
||||
// flags for generating the osd config
|
||||
osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
|
||||
osdConfigCmd.Flags().BoolVar(&osdIsDevice, "is-device", false, "whether the osd is a device")
|
||||
@@ -150,8 +149,7 @@ func startOSD(cmd *cobra.Command, args []string) error {
|
||||
|
||||
context := createContext()
|
||||
|
||||
// Get crush location
|
||||
crushLocation, err := getLocation(context.Clientset, cfg.topologyAware)
|
||||
crushLocation, err := getLocation(context.Clientset)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
@@ -209,7 +207,7 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error {
|
||||
|
||||
context := createContext()
|
||||
commonOSDInit(osdConfigCmd)
|
||||
crushLocation, err := getLocation(context.Clientset, cfg.topologyAware)
|
||||
crushLocation, err := getLocation(context.Clientset)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
@@ -251,7 +249,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
|
||||
|
||||
context := createContext()
|
||||
commonOSDInit(provisionCmd)
|
||||
crushLocation, err := getLocation(context.Clientset, cfg.topologyAware)
|
||||
crushLocation, err := getLocation(context.Clientset)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
@@ -286,27 +284,21 @@ func commonOSDInit(cmd *cobra.Command) {
|
||||
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
|
||||
}
|
||||
|
||||
func getLocation(clientset kubernetes.Interface, topologyAware bool) (string, error) {
|
||||
func getLocation(clientset kubernetes.Interface) (string, error) {
|
||||
// Format the basic location properties, then detect the node labels below
|
||||
initialLocation := cfg.location
|
||||
if cfg.topologyAware {
|
||||
// If topology aware, we will detect the node labels rather than allowing them to be specified in the cluster CR
|
||||
initialLocation = ""
|
||||
}
|
||||
locArgs, err := client.FormatLocation(initialLocation, cfg.nodeName)
|
||||
locArgs, err := client.FormatLocation("", cfg.nodeName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid location. %+v", err)
|
||||
}
|
||||
|
||||
// use zone/region/hostname labels in the crushmap
|
||||
if cfg.topologyAware {
|
||||
nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
|
||||
node, err := getNode(clientset, nodeName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("topologyAware is set, but could not get the node: %+v", err)
|
||||
}
|
||||
nodeLabels := node.GetLabels()
|
||||
updateLocationWithNodeLabels(&locArgs, nodeLabels)
|
||||
nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
|
||||
node, err := getNode(clientset, nodeName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("topologyAware is set, but could not get the node: %+v", err)
|
||||
}
|
||||
nodeLabels := node.GetLabels()
|
||||
updateLocationWithNodeLabels(&locArgs, nodeLabels)
|
||||
|
||||
loc := strings.Join(locArgs, " ")
|
||||
logger.Infof("CRUSH location=%s", loc)
|
||||
|
||||
@@ -210,7 +210,7 @@ type CephBlockPoolList struct {
|
||||
|
||||
// PoolSpec represents the spec of ceph pool
|
||||
type PoolSpec struct {
|
||||
// The failure domain: osd/host/(region or zone if topologyAware) - technically also any type in the crush map
|
||||
// The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map
|
||||
FailureDomain string `json:"failureDomain"`
|
||||
|
||||
// The root of the crush hierarchy utilized by the pool
|
||||
|
||||
@@ -27,12 +27,11 @@ import (
|
||||
// ************************************************************************************
|
||||
|
||||
type StorageScopeSpec struct {
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
UseAllNodes bool `json:"useAllNodes,omitempty"`
|
||||
TopologyAware bool `json:"topologyAware,omitempty"`
|
||||
NodeCount int `json:"nodeCount,omitempty"`
|
||||
Location string `json:"location,omitempty"`
|
||||
Config map[string]string `json:"config"`
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
UseAllNodes bool `json:"useAllNodes,omitempty"`
|
||||
NodeCount int `json:"nodeCount,omitempty"`
|
||||
Location string `json:"location,omitempty"`
|
||||
Config map[string]string `json:"config"`
|
||||
Selection
|
||||
VolumeSources []VolumeSource `json:"volumeSources,omitempty"`
|
||||
StorageClassDeviceSets []StorageClassDeviceSet `json:"storageClassDeviceSets"`
|
||||
|
||||
@@ -52,7 +52,6 @@ const (
|
||||
osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE"
|
||||
pvcBackedOSDVarName = "ROOK_PVC_BACKED_OSD"
|
||||
lvPathVarName = "ROOK_LV_PATH"
|
||||
topologyAwareEnvVarName = "ROOK_TOPOLOGY_AWARE"
|
||||
rookBinariesMountPath = "/rook"
|
||||
rookBinariesVolumeName = "rook-binaries"
|
||||
blockPVCMapperInitContainer = "blkdevmapper"
|
||||
@@ -580,11 +579,6 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, node
|
||||
{Name: "CEPH_VOLUME_DEBUG", Value: "1"},
|
||||
k8sutil.NodeEnvVar(),
|
||||
}
|
||||
// pass on the topologyAware flag to the provion pod so that portable OSDs can reconcile zone/region
|
||||
if c.DesiredStorage.TopologyAware {
|
||||
envVars = append(envVars, topologyAwareEnvVar("true"))
|
||||
}
|
||||
|
||||
if storeConfig.StoreType != "" {
|
||||
envVars = append(envVars, v1.EnvVar{Name: osdStoreEnvVarName, Value: storeConfig.StoreType})
|
||||
}
|
||||
@@ -794,10 +788,6 @@ func lvPathEnvVariable(lvPath string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: lvPathVarName, Value: lvPath}
|
||||
}
|
||||
|
||||
func topologyAwareEnvVar(topologyAware string) v1.EnvVar {
|
||||
return v1.EnvVar{Name: topologyAwareEnvVarName, Value: topologyAware}
|
||||
}
|
||||
|
||||
func getDirectoriesFromContainer(osdContainer v1.Container) []rookalpha.Directory {
|
||||
var dirsArg string
|
||||
for _, envVar := range osdContainer.Env {
|
||||
|
||||
@@ -268,10 +268,8 @@ func RookNodesMatchingKubernetesNodes(rookStorage rookalpha.StorageScopeSpec, ku
|
||||
if rookNodeMatchesKubernetesNode(rn, kn) {
|
||||
rn.Name = normalizeHostname(kn)
|
||||
|
||||
// modify the node Location if topology awareness enabled
|
||||
if rookStorage.TopologyAware == true {
|
||||
rn.Location = nodeTopologyLocation(kn, rn.Location)
|
||||
}
|
||||
// modify the OSD node location
|
||||
rn.Location = nodeTopologyLocation(kn, rn.Location)
|
||||
|
||||
nodes = append(nodes, rn)
|
||||
}
|
||||
|
||||
@@ -305,7 +305,10 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) {
|
||||
{Name: "node0"},
|
||||
{Name: "node1"},
|
||||
{Name: "node2"}}
|
||||
rookStorage.TopologyAware = true
|
||||
|
||||
rookStorage.StorageClassDeviceSets = []rookalpha.StorageClassDeviceSet{
|
||||
{Name: "foobar"},
|
||||
}
|
||||
|
||||
k8sNodes[0].ObjectMeta.Labels = map[string]string{
|
||||
"failure-domain.beta.kubernetes.io/region": "region1",
|
||||
|
||||
@@ -216,8 +216,6 @@ spec:
|
||||
path:
|
||||
type: string
|
||||
config: {}
|
||||
topologyAware:
|
||||
type: boolean
|
||||
storageClassDeviceSets: {}
|
||||
monitoring:
|
||||
properties:
|
||||
|
||||
Reference in new issue
Block a user