ceph: remove topologyAware CRD option

We removed the topologyAware CRD option since it was redundant with the
use of an OSD being backed by a PVC.
So now, if an OSD is backed by a PVC we assume the topology aware
decision and will discover zone and region labels on that host.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han authored and Travis Nielsen committed 2019-11-04 09:53:14 -07:00
1 parent 1f788cf637
commit ef6815ebfa
12 files changed
+24 -53

No files matched your search

-2
View File
@@ -125,8 +125,6 @@ For more details on the mons and when to choose a number other than `3`, see the
* `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
* `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values.
If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`.
* `topologyAware`: `true` or `false`, indicating whether Rook will look for and use topology/failure domain labels on Kubernetes nodes (e.g. "region" or "zone") as part of the CRUSH location for OSDs.
Node labels must follow the formatting of the failure domain [well-known labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/).
* `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below.
`useAllNodes` must be set to `false` to use specific nodes and their config.
See [node settings](#node-settings) below.
@@ -143,8 +143,6 @@ spec:
path:
type: string
config: {}
topologyAware:
type: boolean
storageClassDeviceSets: {}
monitoring:
properties:
@@ -162,8 +162,6 @@ spec:
path:
type: string
config: {}
topologyAware:
type: boolean
storageClassDeviceSets: {}
monitoring:
properties:
@@ -145,8 +145,6 @@ spec:
path:
type: string
config: {}
topologyAware:
type: boolean
storageClassDeviceSets: {}
monitoring:
properties:
-1
View File
@@ -51,7 +51,6 @@ type config struct {
networkInfo clusterd.NetworkInfo
monEndpoints string
nodeName string
topologyAware bool
pvcBacked bool
}
+12 -20
View File
@@ -87,7 +87,6 @@ func addOSDFlags(command *cobra.Command) {
provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
provisionCmd.Flags().BoolVar(&cfg.pvcBacked, "pvc-backed-osd", false, "true to specify a block mode pvc is backing the OSD")
provisionCmd.Flags().BoolVar(&cfg.topologyAware, "topology-aware", false, "true to specify crush location should be set based on the node's zone/region labels")
// flags for generating the osd config
osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config")
osdConfigCmd.Flags().BoolVar(&osdIsDevice, "is-device", false, "whether the osd is a device")
@@ -150,8 +149,7 @@ func startOSD(cmd *cobra.Command, args []string) error {
context := createContext()
// Get crush location
crushLocation, err := getLocation(context.Clientset, cfg.topologyAware)
crushLocation, err := getLocation(context.Clientset)
if err != nil {
rook.TerminateFatal(err)
}
@@ -209,7 +207,7 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error {
context := createContext()
commonOSDInit(osdConfigCmd)
crushLocation, err := getLocation(context.Clientset, cfg.topologyAware)
crushLocation, err := getLocation(context.Clientset)
if err != nil {
rook.TerminateFatal(err)
}
@@ -251,7 +249,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error {
context := createContext()
commonOSDInit(provisionCmd)
crushLocation, err := getLocation(context.Clientset, cfg.topologyAware)
crushLocation, err := getLocation(context.Clientset)
if err != nil {
rook.TerminateFatal(err)
}
@@ -286,27 +284,21 @@ func commonOSDInit(cmd *cobra.Command) {
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
}
func getLocation(clientset kubernetes.Interface, topologyAware bool) (string, error) {
func getLocation(clientset kubernetes.Interface) (string, error) {
// Format the basic location properties, then detect the node labels below
initialLocation := cfg.location
if cfg.topologyAware {
// If topology aware, we will detect the node labels rather than allowing them to be specified in the cluster CR
initialLocation = ""
}
locArgs, err := client.FormatLocation(initialLocation, cfg.nodeName)
locArgs, err := client.FormatLocation("", cfg.nodeName)
if err != nil {
return "", fmt.Errorf("invalid location. %+v", err)
}
// use zone/region/hostname labels in the crushmap
if cfg.topologyAware {
nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
node, err := getNode(clientset, nodeName)
if err != nil {
return "", fmt.Errorf("topologyAware is set, but could not get the node: %+v", err)
}
nodeLabels := node.GetLabels()
updateLocationWithNodeLabels(&locArgs, nodeLabels)
nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
node, err := getNode(clientset, nodeName)
if err != nil {
return "", fmt.Errorf("topologyAware is set, but could not get the node: %+v", err)
}
nodeLabels := node.GetLabels()
updateLocationWithNodeLabels(&locArgs, nodeLabels)
loc := strings.Join(locArgs, " ")
logger.Infof("CRUSH location=%s", loc)
+1 -1
View File
@@ -210,7 +210,7 @@ type CephBlockPoolList struct {
// PoolSpec represents the spec of ceph pool
type PoolSpec struct {
// The failure domain: osd/host/(region or zone if topologyAware) - technically also any type in the crush map
// The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map
FailureDomain string `json:"failureDomain"`
// The root of the crush hierarchy utilized by the pool
+5 -6
View File
@@ -27,12 +27,11 @@ import (
// ************************************************************************************
type StorageScopeSpec struct {
Nodes []Node `json:"nodes,omitempty"`
UseAllNodes bool `json:"useAllNodes,omitempty"`
TopologyAware bool `json:"topologyAware,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
Location string `json:"location,omitempty"`
Config map[string]string `json:"config"`
Nodes []Node `json:"nodes,omitempty"`
UseAllNodes bool `json:"useAllNodes,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
Location string `json:"location,omitempty"`
Config map[string]string `json:"config"`
Selection
VolumeSources []VolumeSource `json:"volumeSources,omitempty"`
StorageClassDeviceSets []StorageClassDeviceSet `json:"storageClassDeviceSets"`
-10
View File
@@ -52,7 +52,6 @@ const (
osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE"
pvcBackedOSDVarName = "ROOK_PVC_BACKED_OSD"
lvPathVarName = "ROOK_LV_PATH"
topologyAwareEnvVarName = "ROOK_TOPOLOGY_AWARE"
rookBinariesMountPath = "/rook"
rookBinariesVolumeName = "rook-binaries"
blockPVCMapperInitContainer = "blkdevmapper"
@@ -580,11 +579,6 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, node
{Name: "CEPH_VOLUME_DEBUG", Value: "1"},
k8sutil.NodeEnvVar(),
}
// pass on the topologyAware flag to the provion pod so that portable OSDs can reconcile zone/region
if c.DesiredStorage.TopologyAware {
envVars = append(envVars, topologyAwareEnvVar("true"))
}
if storeConfig.StoreType != "" {
envVars = append(envVars, v1.EnvVar{Name: osdStoreEnvVarName, Value: storeConfig.StoreType})
}
@@ -794,10 +788,6 @@ func lvPathEnvVariable(lvPath string) v1.EnvVar {
return v1.EnvVar{Name: lvPathVarName, Value: lvPath}
}
func topologyAwareEnvVar(topologyAware string) v1.EnvVar {
return v1.EnvVar{Name: topologyAwareEnvVarName, Value: topologyAware}
}
func getDirectoriesFromContainer(osdContainer v1.Container) []rookalpha.Directory {
var dirsArg string
for _, envVar := range osdContainer.Env {
+2 -4
View File
@@ -268,10 +268,8 @@ func RookNodesMatchingKubernetesNodes(rookStorage rookalpha.StorageScopeSpec, ku
if rookNodeMatchesKubernetesNode(rn, kn) {
rn.Name = normalizeHostname(kn)
// modify the node Location if topology awareness enabled
if rookStorage.TopologyAware == true {
rn.Location = nodeTopologyLocation(kn, rn.Location)
}
// modify the OSD node location
rn.Location = nodeTopologyLocation(kn, rn.Location)
nodes = append(nodes, rn)
}
+4 -1
View File
@@ -305,7 +305,10 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) {
{Name: "node0"},
{Name: "node1"},
{Name: "node2"}}
rookStorage.TopologyAware = true
rookStorage.StorageClassDeviceSets = []rookalpha.StorageClassDeviceSet{
{Name: "foobar"},
}
k8sNodes[0].ObjectMeta.Labels = map[string]string{
"failure-domain.beta.kubernetes.io/region": "region1",
@@ -216,8 +216,6 @@ spec:
path:
type: string
config: {}
topologyAware:
type: boolean
storageClassDeviceSets: {}
monitoring:
properties: