diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index f937a501e..cf23c11ef 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -125,8 +125,6 @@ For more details on the mons and when to choose a number other than `3`, see the * `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes. * `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values. If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`. - * `topologyAware`: `true` or `false`, indicating whether Rook will look for and use topology/failure domain labels on Kubernetes nodes (e.g. "region" or "zone") as part of the CRUSH location for OSDs. - Node labels must follow the formatting of the failure domain [well-known labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/). * `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below. `useAllNodes` must be set to `false` to use specific nodes and their config. See [node settings](#node-settings) below. diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 41fa15eb4..2e848b1a9 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -143,8 +143,6 @@ spec: path: type: string config: {} - topologyAware: - type: boolean storageClassDeviceSets: {} monitoring: properties: diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 9bd09a213..8eb214b81 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -162,8 +162,6 @@ spec: path: type: string config: {} - topologyAware: - type: boolean storageClassDeviceSets: {} monitoring: properties: diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.0-crds.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.0-crds.yaml index 3ec11055d..afbc44959 100644 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.0-crds.yaml +++ b/cluster/examples/kubernetes/ceph/upgrade-from-v1.0-crds.yaml @@ -145,8 +145,6 @@ spec: path: type: string config: {} - topologyAware: - type: boolean storageClassDeviceSets: {} monitoring: properties: diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index a48137b5d..ec8d49826 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -51,7 +51,6 @@ type config struct { networkInfo clusterd.NetworkInfo monEndpoints string nodeName string - topologyAware bool pvcBacked bool } diff --git a/cmd/rook/ceph/osd.go b/cmd/rook/ceph/osd.go index 1b6ab4a38..be811c70f 100644 --- a/cmd/rook/ceph/osd.go +++ b/cmd/rook/ceph/osd.go @@ -87,7 +87,6 @@ func addOSDFlags(command *cobra.Command) { provisionCmd.Flags().BoolVar(&cfg.forceFormat, "force-format", false, "true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!") provisionCmd.Flags().BoolVar(&cfg.pvcBacked, "pvc-backed-osd", false, "true to specify a block mode pvc is backing the OSD") - provisionCmd.Flags().BoolVar(&cfg.topologyAware, "topology-aware", false, "true to specify crush location should be set based on the node's zone/region labels") // flags for generating the osd config osdConfigCmd.Flags().IntVar(&osdID, "osd-id", -1, "osd id for which to generate config") osdConfigCmd.Flags().BoolVar(&osdIsDevice, "is-device", false, "whether the osd is a device") @@ -150,8 +149,7 @@ func startOSD(cmd *cobra.Command, args []string) error { context := createContext() - // Get crush location - crushLocation, err := getLocation(context.Clientset, cfg.topologyAware) + crushLocation, err := getLocation(context.Clientset) if err != nil { rook.TerminateFatal(err) } @@ -209,7 +207,7 @@ func writeOSDConfig(cmd *cobra.Command, args []string) error { context := createContext() commonOSDInit(osdConfigCmd) - crushLocation, err := getLocation(context.Clientset, cfg.topologyAware) + crushLocation, err := getLocation(context.Clientset) if err != nil { rook.TerminateFatal(err) } @@ -251,7 +249,7 @@ func prepareOSD(cmd *cobra.Command, args []string) error { context := createContext() commonOSDInit(provisionCmd) - crushLocation, err := getLocation(context.Clientset, cfg.topologyAware) + crushLocation, err := getLocation(context.Clientset) if err != nil { rook.TerminateFatal(err) } @@ -286,27 +284,21 @@ func commonOSDInit(cmd *cobra.Command) { clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) } -func getLocation(clientset kubernetes.Interface, topologyAware bool) (string, error) { +func getLocation(clientset kubernetes.Interface) (string, error) { // Format the basic location properties, then detect the node labels below - initialLocation := cfg.location - if cfg.topologyAware { - // If topology aware, we will detect the node labels rather than allowing them to be specified in the cluster CR - initialLocation = "" - } - locArgs, err := client.FormatLocation(initialLocation, cfg.nodeName) + locArgs, err := client.FormatLocation("", cfg.nodeName) if err != nil { return "", fmt.Errorf("invalid location. %+v", err) } + // use zone/region/hostname labels in the crushmap - if cfg.topologyAware { - nodeName := os.Getenv(k8sutil.NodeNameEnvVar) - node, err := getNode(clientset, nodeName) - if err != nil { - return "", fmt.Errorf("topologyAware is set, but could not get the node: %+v", err) - } - nodeLabels := node.GetLabels() - updateLocationWithNodeLabels(&locArgs, nodeLabels) + nodeName := os.Getenv(k8sutil.NodeNameEnvVar) + node, err := getNode(clientset, nodeName) + if err != nil { + return "", fmt.Errorf("topologyAware is set, but could not get the node: %+v", err) } + nodeLabels := node.GetLabels() + updateLocationWithNodeLabels(&locArgs, nodeLabels) loc := strings.Join(locArgs, " ") logger.Infof("CRUSH location=%s", loc) diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index eed5064f6..2206e74f2 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -210,7 +210,7 @@ type CephBlockPoolList struct { // PoolSpec represents the spec of ceph pool type PoolSpec struct { - // The failure domain: osd/host/(region or zone if topologyAware) - technically also any type in the crush map + // The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map FailureDomain string `json:"failureDomain"` // The root of the crush hierarchy utilized by the pool diff --git a/pkg/apis/rook.io/v1alpha2/types.go b/pkg/apis/rook.io/v1alpha2/types.go index 59bf4fc54..9e9279f06 100644 --- a/pkg/apis/rook.io/v1alpha2/types.go +++ b/pkg/apis/rook.io/v1alpha2/types.go @@ -27,12 +27,11 @@ import ( // ************************************************************************************ type StorageScopeSpec struct { - Nodes []Node `json:"nodes,omitempty"` - UseAllNodes bool `json:"useAllNodes,omitempty"` - TopologyAware bool `json:"topologyAware,omitempty"` - NodeCount int `json:"nodeCount,omitempty"` - Location string `json:"location,omitempty"` - Config map[string]string `json:"config"` + Nodes []Node `json:"nodes,omitempty"` + UseAllNodes bool `json:"useAllNodes,omitempty"` + NodeCount int `json:"nodeCount,omitempty"` + Location string `json:"location,omitempty"` + Config map[string]string `json:"config"` Selection VolumeSources []VolumeSource `json:"volumeSources,omitempty"` StorageClassDeviceSets []StorageClassDeviceSet `json:"storageClassDeviceSets"` diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 98beed920..1ca61b609 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -52,7 +52,6 @@ const ( osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" pvcBackedOSDVarName = "ROOK_PVC_BACKED_OSD" lvPathVarName = "ROOK_LV_PATH" - topologyAwareEnvVarName = "ROOK_TOPOLOGY_AWARE" rookBinariesMountPath = "/rook" rookBinariesVolumeName = "rook-binaries" blockPVCMapperInitContainer = "blkdevmapper" @@ -580,11 +579,6 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, node {Name: "CEPH_VOLUME_DEBUG", Value: "1"}, k8sutil.NodeEnvVar(), } - // pass on the topologyAware flag to the provion pod so that portable OSDs can reconcile zone/region - if c.DesiredStorage.TopologyAware { - envVars = append(envVars, topologyAwareEnvVar("true")) - } - if storeConfig.StoreType != "" { envVars = append(envVars, v1.EnvVar{Name: osdStoreEnvVarName, Value: storeConfig.StoreType}) } @@ -794,10 +788,6 @@ func lvPathEnvVariable(lvPath string) v1.EnvVar { return v1.EnvVar{Name: lvPathVarName, Value: lvPath} } -func topologyAwareEnvVar(topologyAware string) v1.EnvVar { - return v1.EnvVar{Name: topologyAwareEnvVarName, Value: topologyAware} -} - func getDirectoriesFromContainer(osdContainer v1.Container) []rookalpha.Directory { var dirsArg string for _, envVar := range osdContainer.Env { diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index 55c5efa24..78fe50cd0 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -268,10 +268,8 @@ func RookNodesMatchingKubernetesNodes(rookStorage rookalpha.StorageScopeSpec, ku if rookNodeMatchesKubernetesNode(rn, kn) { rn.Name = normalizeHostname(kn) - // modify the node Location if topology awareness enabled - if rookStorage.TopologyAware == true { - rn.Location = nodeTopologyLocation(kn, rn.Location) - } + // modify the OSD node location + rn.Location = nodeTopologyLocation(kn, rn.Location) nodes = append(nodes, rn) } diff --git a/pkg/operator/k8sutil/node_test.go b/pkg/operator/k8sutil/node_test.go index 2d235147e..c151c38ff 100644 --- a/pkg/operator/k8sutil/node_test.go +++ b/pkg/operator/k8sutil/node_test.go @@ -305,7 +305,10 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) { {Name: "node0"}, {Name: "node1"}, {Name: "node2"}} - rookStorage.TopologyAware = true + + rookStorage.StorageClassDeviceSets = []rookalpha.StorageClassDeviceSet{ + {Name: "foobar"}, + } k8sNodes[0].ObjectMeta.Labels = map[string]string{ "failure-domain.beta.kubernetes.io/region": "region1", diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index af587a9d7..0716d93ef 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -216,8 +216,6 @@ spec: path: type: string config: {} - topologyAware: - type: boolean storageClassDeviceSets: {} monitoring: properties: