diff --git a/Documentation/cluster-tpr.md b/Documentation/cluster-tpr.md index 1e139b3d5..b5b6c01af 100644 --- a/Documentation/cluster-tpr.md +++ b/Documentation/cluster-tpr.md @@ -13,6 +13,7 @@ Settings can be specified at the global level to apply to the cluster as a whole - `versionTag`: The version (tag) of the `quay.io/rook/rookd` container that will be deployed. Upgrades are not yet supported if this setting is updated for an existing cluster, but upgrades will be coming. - `dataDirHostPath`: The host path where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Therefore, for test scenarios, the path must be deleted if you are going to delete a cluster and start a new cluster on the same hosts. More details can be found in the Kubernetes [host path docs](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath). If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir). +- `placement`: [placement configuration settings](#placement-configuration-settings) - `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes. - `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values. If individual nodes are specified under the `nodes` field below, then `useAllNodes` must be set to `false`. @@ -51,5 +52,12 @@ Below are the settings available, both at the cluster and individual node level, - `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). - `journalSizeMB`: The size in MB of a filestore journal. +### Placement Configuration Settings +Placement configuration for the cluster services. It includes the following keys: `api`, `mds`, `mon`, `osd`, `rgw` and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes). + +A Placement configuration is specified (according to the kubernetes [PodSpec](https://kubernetes.io/docs/api-reference/v1.6/#podspec-v1-core)) as: +- `nodeAffinity`: kubernetes [NodeAffinity](https://kubernetes.io/docs/api-reference/v1.6/#nodeaffinity-v1-core) +- `tolerations`: list of kubernetes [Toleration](https://kubernetes.io/docs/api-reference/v1.6/#toleration-v1-core) + ## Sample A sample cluster TPR can be found and used in the [rook-cluster.yaml](../demo/kubernetes/rook-cluster.yaml) file in the Kubernetes demo directory. diff --git a/demo/kubernetes/rook-cluster.yaml b/demo/kubernetes/rook-cluster.yaml index 64eb39fd2..ec7b664cf 100644 --- a/demo/kubernetes/rook-cluster.yaml +++ b/demo/kubernetes/rook-cluster.yaml @@ -6,6 +6,37 @@ metadata: spec: versionTag: master-latest dataDirHostPath: +# To control where various services will be scheduled by kubernetes, use the placement configuration sections below. +# The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage' and +# tolerate taints with a key of 'storage-node'. +# placement: +# all: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: role +# operator: In +# values: +# - storage-node +# tolerations: +# - key: storage-node +# operator: Exists +# api: +# nodeAffinity: +# tolerations: +# mds: +# nodeAffinity: +# tolerations: +# mon: +# nodeAffinity: +# tolerations: +# osd: +# nodeAffinity: +# tolerations: +# rgw: +# nodeAffinity: +# tolerations: storage: # cluster level storage configuration and selection useAllNodes: true useAllDevices: false diff --git a/pkg/api/k8s/cluster_handler.go b/pkg/api/k8s/cluster_handler.go index 81079a572..f04ac7a3f 100644 --- a/pkg/api/k8s/cluster_handler.go +++ b/pkg/api/k8s/cluster_handler.go @@ -46,7 +46,9 @@ func (s *clusterHandler) GetClusterInfo() (*mon.ClusterInfo, error) { func (s *clusterHandler) EnableObjectStore() error { logger.Infof("Starting the Object store") - r := k8srgw.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag) + // Passing an empty Placement{} as the api doesn't know about placement + // information. This should be resolved with the transition to CRD (TPR). + r := k8srgw.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{}) err := r.Start(s.clusterInfo) if err != nil { return fmt.Errorf("failed to start rgw. %+v", err) @@ -76,7 +78,9 @@ func (s *clusterHandler) GetObjectStoreConnectionInfo() (*model.ObjectStoreConne func (s *clusterHandler) StartFileSystem(fs *model.FilesystemRequest) error { logger.Infof("Starting the MDS") - c := k8smds.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag) + // Passing an empty Placement{} as the api doesn't know about placement + // information. This should be resolved with the transition to CRD (TPR). + c := k8smds.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{}) return c.Start(s.k8sContext.Clientset, s.clusterInfo) } diff --git a/pkg/operator/api/api.go b/pkg/operator/api/api.go index a015e9687..0e2c3f2f3 100644 --- a/pkg/operator/api/api.go +++ b/pkg/operator/api/api.go @@ -53,15 +53,17 @@ var clusterAccessRules = []v1beta1.PolicyRule{ type Cluster struct { context *k8sutil.Context + placement k8sutil.Placement Name string Namespace string Version string Replicas int32 } -func New(context *k8sutil.Context, name, namespace, version string) *Cluster { +func New(context *k8sutil.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster { return &Cluster{ context: context, + placement: placement, Name: name, Namespace: namespace, Version: version, @@ -141,23 +143,26 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { deployment.Name = DeploymentName deployment.Namespace = c.Namespace - podSpec := v1.PodTemplateSpec{ + podSpec := v1.PodSpec{ + ServiceAccountName: DeploymentName, + Containers: []v1.Container{c.apiContainer()}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + }, + } + c.placement.ApplyToPodSpec(&podSpec) + + podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: DeploymentName, Labels: c.getLabels(), Annotations: map[string]string{}, }, - Spec: v1.PodSpec{ - ServiceAccountName: DeploymentName, - Containers: []v1.Container{c.apiContainer()}, - RestartPolicy: v1.RestartPolicyAlways, - Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, - }, - }, + Spec: podSpec, } - deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas} + deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas} return deployment } diff --git a/pkg/operator/api/api_test.go b/pkg/operator/api/api_test.go index 26b953af8..b1926cbc8 100644 --- a/pkg/operator/api/api_test.go +++ b/pkg/operator/api/api_test.go @@ -32,7 +32,7 @@ import ( func TestStartAPI(t *testing.T) { clientset := testop.New(3) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", k8sutil.Placement{}) // start a basic cluster err := c.Start() @@ -60,7 +60,7 @@ func validateStart(t *testing.T, c *Cluster) { func TestPodSpecs(t *testing.T) { clientset := testop.New(1) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", k8sutil.Placement{}) d := c.makeDeployment() assert.NotNil(t, d) @@ -89,7 +89,7 @@ func TestPodSpecs(t *testing.T) { func TestClusterRole(t *testing.T) { clientset := testop.New(1) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", k8sutil.Placement{}) // the role is create err := c.makeClusterRole() diff --git a/pkg/operator/cluster/cluster.go b/pkg/operator/cluster/cluster.go index c83058474..1f2297eed 100644 --- a/pkg/operator/cluster/cluster.go +++ b/pkg/operator/cluster/cluster.go @@ -86,20 +86,20 @@ func (c *Cluster) CreateInstance() error { } // Start the mon pods - c.mons = mon.New(c.context, c.Name, c.Namespace, c.Spec.DataDirHostPath, c.Spec.VersionTag) + c.mons = mon.New(c.context, c.Name, c.Namespace, c.Spec.DataDirHostPath, c.Spec.VersionTag, c.Spec.Placement.GetMON()) clusterInfo, err := c.mons.Start() if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) } - c.apis = api.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag) + c.apis = api.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Placement.GetAPI()) err = c.apis.Start() if err != nil { return fmt.Errorf("failed to start the REST api. %+v", err) } // Start the OSDs - c.osds = osd.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Storage, c.Spec.DataDirHostPath) + c.osds = osd.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.Placement.GetOSD()) err = c.osds.Start() if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) diff --git a/pkg/operator/cluster/spec.go b/pkg/operator/cluster/spec.go index 1a312f2c5..c59d50616 100644 --- a/pkg/operator/cluster/spec.go +++ b/pkg/operator/cluster/spec.go @@ -18,7 +18,10 @@ which also has the apache 2.0 license. */ package cluster -import "github.com/rook/rook/pkg/operator/osd" +import ( + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/osd" +) type Spec struct { // VersionTag is the expected version of the rook container to run in the cluster. @@ -32,6 +35,9 @@ type Spec struct { // The path on the host where config and data can be persisted. DataDirHostPath string `json:"dataDirHostPath"` + // The placement-related configuration to pass to kubernetes (affinity, node selector, tolerations). + Placement PlacementSpec `json:"placement,omitempty"` + // A spec for available storage in the cluster and how it should be used Storage osd.StorageSpec `json:"storage"` } @@ -56,3 +62,19 @@ type ErasureCodeSpec struct { // Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) DataChunks uint `json:"dataChunks"` } + +// PlacementSpec is a set of Placement configurations for the rook cluster. +type PlacementSpec struct { + All k8sutil.Placement `json:"all,omitempty"` + API k8sutil.Placement `json:"api,omitempty"` + MDS k8sutil.Placement `json:"mds,omitempty"` + MON k8sutil.Placement `json:"mon,omitempty"` + OSD k8sutil.Placement `json:"osd,omitempty"` + RGW k8sutil.Placement `json:"rgw,omitempty"` +} + +func (p PlacementSpec) GetAPI() k8sutil.Placement { return p.All.Merge(p.API) } +func (p PlacementSpec) GetMDS() k8sutil.Placement { return p.All.Merge(p.MDS) } +func (p PlacementSpec) GetMON() k8sutil.Placement { return p.All.Merge(p.MON) } +func (p PlacementSpec) GetOSD() k8sutil.Placement { return p.All.Merge(p.OSD) } +func (p PlacementSpec) GetRGW() k8sutil.Placement { return p.All.Merge(p.RGW) } diff --git a/pkg/operator/k8sutil/placement.go b/pkg/operator/k8sutil/placement.go new file mode 100644 index 000000000..9b6b4c9ff --- /dev/null +++ b/pkg/operator/k8sutil/placement.go @@ -0,0 +1,53 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package k8sutil + +import ( + "k8s.io/client-go/pkg/api/v1" +) + +// Placement encapsulates the various kubernetes options that control where +// pods are scheduled and executed. +type Placement struct { + NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitemtpy"` +} + +func (p Placement) ApplyToPodSpec(t *v1.PodSpec) { + if p.NodeAffinity != nil { + if t.Affinity == nil { + t.Affinity = &v1.Affinity{} + } + t.Affinity.NodeAffinity = p.NodeAffinity + } + if p.Tolerations != nil { + t.Tolerations = p.Tolerations + } +} + +// Merge returns a Placement which results from merging the attributes of the +// original Placement with the attributes of the supplied one. The supplied +// Placement's attributes will override the original ones if defined. +func (p Placement) Merge(with Placement) Placement { + ret := p + if with.NodeAffinity != nil { + ret.NodeAffinity = with.NodeAffinity + } + if with.Tolerations != nil { + ret.Tolerations = with.Tolerations + } + return ret +} diff --git a/pkg/operator/k8sutil/placement_test.go b/pkg/operator/k8sutil/placement_test.go new file mode 100644 index 000000000..19a2c0174 --- /dev/null +++ b/pkg/operator/k8sutil/placement_test.go @@ -0,0 +1,178 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package k8sutil + +import ( + "encoding/json" + "testing" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/pkg/api/v1" +) + +func TestPlacement_spec(t *testing.T) { + specYaml := []byte(` +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: foo + operator: In + values: + - bar +tolerations: + - key: foo + operator: Exists`) + + // convert the raw spec yaml into JSON + rawJson, err := yaml.YAMLToJSON(specYaml) + assert.Nil(t, err) + + // unmarshal the JSON into a strongly typed placement spec object + var placement Placement + err = json.Unmarshal(rawJson, &placement) + assert.Nil(t, err) + + // the unmarshalled placement spec should equal the expected spec below + expected := Placement{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + v1.NodeSelectorRequirement{ + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + Tolerations: []v1.Toleration{ + v1.Toleration{ + Key: "foo", + Operator: v1.TolerationOpExists, + }, + }, + } + assert.Equal(t, expected, placement) +} + +func TestPlacement_ApplyToPodSpec(t *testing.T) { + to := placementTestGetTolerations("foo", "bar") + na := placementTestGetNodeAffinity() + expected := &v1.PodSpec{Affinity: &v1.Affinity{NodeAffinity: na}, Tolerations: to} + + var p Placement + var ps *v1.PodSpec + + p = Placement{NodeAffinity: na, Tolerations: to} + ps = &v1.PodSpec{} + p.ApplyToPodSpec(ps) + assert.Equal(t, expected, ps) + + // partial update + p = Placement{NodeAffinity: na} + ps = &v1.PodSpec{Tolerations: to} + p.ApplyToPodSpec(ps) + assert.Equal(t, expected, ps) + + // overridden attributes + p = Placement{NodeAffinity: na, Tolerations: to} + ps = &v1.PodSpec{Tolerations: placementTestGetTolerations("bar", "baz")} + p.ApplyToPodSpec(ps) + assert.Equal(t, expected, ps) + + p = Placement{NodeAffinity: na} + nap := placementTestGetNodeAffinity() + nap.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 5 + ps = &v1.PodSpec{Affinity: &v1.Affinity{NodeAffinity: nap}, Tolerations: to} + p.ApplyToPodSpec(ps) + assert.Equal(t, expected, ps) +} + +func TestPlacement_Merge(t *testing.T) { + to := placementTestGetTolerations("foo", "bar") + na := placementTestGetNodeAffinity() + + var original, with, expected, merged Placement + + original = Placement{} + with = Placement{Tolerations: to} + expected = Placement{Tolerations: to} + merged = original.Merge(with) + assert.Equal(t, expected, merged) + + original = Placement{NodeAffinity: na} + with = Placement{Tolerations: to} + expected = Placement{NodeAffinity: na, Tolerations: to} + merged = original.Merge(with) + assert.Equal(t, expected, merged) + + original = Placement{Tolerations: placementTestGetTolerations("bar", "baz")} + with = Placement{NodeAffinity: na, Tolerations: to} + expected = Placement{NodeAffinity: na, Tolerations: to} + merged = original.Merge(with) + assert.Equal(t, expected, merged) +} + +func placementTestGetTolerations(key, value string) []v1.Toleration { + var ts int64 = 10 + return []v1.Toleration{ + v1.Toleration{ + Key: key, + Operator: v1.TolerationOpExists, + Value: value, + Effect: v1.TaintEffectNoSchedule, + TolerationSeconds: &ts, + }, + } +} + +func placementTestGetNodeAffinity() *v1.NodeAffinity { + return &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + v1.NodeSelectorRequirement{ + Key: "foo", + Operator: v1.NodeSelectorOpExists, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + v1.PreferredSchedulingTerm{ + Weight: 10, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + v1.NodeSelectorRequirement{ + Key: "foo", + Operator: v1.NodeSelectorOpExists, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/operator/mds/mds.go b/pkg/operator/mds/mds.go index 7cbd5d649..00b232ed9 100644 --- a/pkg/operator/mds/mds.go +++ b/pkg/operator/mds/mds.go @@ -44,12 +44,14 @@ type Cluster struct { Version string Replicas int32 context *k8sutil.Context + placement k8sutil.Placement dataDir string } -func New(context *k8sutil.Context, name, namespace, version string) *Cluster { +func New(context *k8sutil.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster { return &Cluster{ context: context, + placement: placement, Name: name, Namespace: namespace, Version: version, @@ -131,23 +133,26 @@ func (c *Cluster) makeDeployment(id string) *extensions.Deployment { deployment.Name = appName deployment.Namespace = c.Namespace - podSpec := v1.PodTemplateSpec{ + podSpec := v1.PodSpec{ + Containers: []v1.Container{c.mdsContainer(id)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + k8sutil.ConfigOverrideVolume(), + }, + } + c.placement.ApplyToPodSpec(&podSpec) + + podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: appName, Labels: c.getLabels(), Annotations: map[string]string{}, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{c.mdsContainer(id)}, - RestartPolicy: v1.RestartPolicyAlways, - Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, - k8sutil.ConfigOverrideVolume(), - }, - }, + Spec: podSpec, } - deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas} + deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas} return deployment } diff --git a/pkg/operator/mds/mds_test.go b/pkg/operator/mds/mds_test.go index c60bcf54c..fbd50e1d7 100644 --- a/pkg/operator/mds/mds_test.go +++ b/pkg/operator/mds/mds_test.go @@ -40,7 +40,7 @@ func TestStartMDS(t *testing.T) { return []byte(response), "", nil } - c := New(&k8sutil.Context{Factory: factory}, "myname", "ns", "myversion") + c := New(&k8sutil.Context{Factory: factory}, "myname", "ns", "myversion", k8sutil.Placement{}) c.dataDir = "/tmp/mdstest" defer os.RemoveAll(c.dataDir) @@ -67,7 +67,7 @@ func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) { } func TestPodSpecs(t *testing.T) { - c := New(nil, "myname", "ns", "myversion") + c := New(nil, "myname", "ns", "myversion", k8sutil.Placement{}) mdsID := "mds1" d := c.makeDeployment(mdsID) diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index 3bb27fb90..e09ead223 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/util" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/pkg/api/v1" ) @@ -53,10 +54,10 @@ type Cluster struct { MasterHost string Size int Paused bool - AntiAffinity bool Port int32 context *k8sutil.Context clusterInfo *mon.ClusterInfo + placement k8sutil.Placement maxMonID int configDir string waitForStart bool @@ -68,15 +69,15 @@ type MonConfig struct { Port int32 } -func New(context *k8sutil.Context, name, namespace, dataDirHostPath, version string) *Cluster { +func New(context *k8sutil.Context, name, namespace, dataDirHostPath, version string, placement k8sutil.Placement) *Cluster { return &Cluster{ context: context, + placement: placement, dataDirHostPath: dataDirHostPath, Name: name, Namespace: namespace, Version: version, Size: 3, - AntiAffinity: true, maxMonID: -1, configDir: k8sutil.DataDir, waitForStart: true, @@ -422,7 +423,7 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, error) { // choose nodes for the new mons that don't have mons currently availableNodes := []v1.Node{} for _, node := range nodes.Items { - if !nodesInUse.Contains(node.Name) && validNode(node) { + if !nodesInUse.Contains(node.Name) && validNode(node, c.placement) { availableNodes = append(availableNodes, node) } } @@ -432,7 +433,7 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, error) { if len(availableNodes) == 0 { logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items)) for _, node := range nodes.Items { - if validNode(node) { + if validNode(node, c.placement) { availableNodes = append(availableNodes, node) } } @@ -444,15 +445,43 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, error) { return availableNodes, nil } -func validNode(node v1.Node) bool { +func validNode(node v1.Node, placement k8sutil.Placement) bool { // a node cannot be disabled if node.Spec.Unschedulable { return false } - // a node cannot be tainted - for _, t := range node.Spec.Taints { - if t.Effect == v1.TaintEffectNoSchedule || t.Effect == v1.TaintEffectPreferNoSchedule { + // a node matches the NodeAffinity configuration + // ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they + // should not be used to judge a node unusable + if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + nodeMatches := false + for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + nodeSelector, err := v1.NodeSelectorRequirementsAsSelector(req.MatchExpressions) + if err != nil { + logger.Infof("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions) + return false + } + if nodeSelector.Matches(labels.Set(node.Labels)) { + nodeMatches = true + break + } + } + if !nodeMatches { + return false + } + } + + // a node is tainted and cannot be tolerated + for _, taint := range node.Spec.Taints { + isTolerated := false + for _, toleration := range placement.Tolerations { + if toleration.ToleratesTaint(&taint) { + isTolerated = true + break + } + } + if !isTolerated { return false } } diff --git a/pkg/operator/mon/mon_test.go b/pkg/operator/mon/mon_test.go index 4650488c9..f803bc220 100644 --- a/pkg/operator/mon/mon_test.go +++ b/pkg/operator/mon/mon_test.go @@ -33,7 +33,7 @@ import ( func TestStartMonPods(t *testing.T) { clientset := test.New(3) factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"} - c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, MaxRetries: 1}, "myname", "ns", "", "myversion") + c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, MaxRetries: 1}, "myname", "ns", "", "myversion", k8sutil.Placement{}) // start a basic cluster // an error is expected since mocking always creates pods that are not running @@ -68,7 +68,7 @@ func validateStart(t *testing.T, c *Cluster) { func TestSaveMonEndpoints(t *testing.T) { clientset := test.New(1) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{}) c.clusterInfo = test.CreateClusterInfo(1) // create the initial config map @@ -92,7 +92,7 @@ func TestSaveMonEndpoints(t *testing.T) { func TestCheckHealth(t *testing.T) { clientset := test.New(1) factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"} - c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, RetryDelay: 1, MaxRetries: 1}, "myname", "ns", "", "myversion") + c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, RetryDelay: 1, MaxRetries: 1}, "myname", "ns", "", "myversion", k8sutil.Placement{}) c.clusterInfo = test.CreateClusterInfo(1) c.configDir = "/tmp/healthtest" c.waitForStart = false @@ -152,7 +152,7 @@ func TestMonID(t *testing.T) { func TestAvailableMonNodes(t *testing.T) { clientset := test.New(1) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{}) c.clusterInfo = test.CreateClusterInfo(0) nodes, err := c.getAvailableMonNodes() assert.Nil(t, err) @@ -169,7 +169,7 @@ func TestAvailableMonNodes(t *testing.T) { func TestAvailableNodesInUse(t *testing.T) { clientset := test.New(3) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{}) c.clusterInfo = test.CreateClusterInfo(0) // all three nodes are available by default @@ -200,7 +200,8 @@ func TestAvailableNodesInUse(t *testing.T) { func TestTaintedNodes(t *testing.T) { clientset := test.New(3) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion") + placement := k8sutil.Placement{Tolerations: []v1.Toleration{{Effect: v1.TaintEffectNoExecute}}} + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", placement) c.clusterInfo = test.CreateClusterInfo(0) nodes, err := c.getAvailableMonNodes() @@ -228,9 +229,48 @@ func TestTaintedNodes(t *testing.T) { clientset.CoreV1().Nodes().Update(&nodes[0]) clientset.CoreV1().Nodes().Update(&nodes[1]) clientset.CoreV1().Nodes().Update(&nodes[2]) - assert.Nil(t, err) cleanNodes, err := c.getAvailableMonNodes() assert.Nil(t, err) assert.Equal(t, 1, len(cleanNodes)) assert.Equal(t, nodes[2].Name, cleanNodes[0].Name) } + +func TestNodeAffinity(t *testing.T) { + clientset := test.New(3) + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{}) + c.clusterInfo = test.CreateClusterInfo(0) + + nodes, err := c.getAvailableMonNodes() + assert.Nil(t, err) + assert.Equal(t, 3, len(nodes)) + + c.placement.NodeAffinity = &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + v1.NodeSelectorRequirement{ + Key: "label", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "baz"}, + }, + }, + }, + }, + }, + } + + // label nodes so node0 will not be schedulable for new pods + nodes[0].Labels = map[string]string{"label": "foo"} + nodes[1].Labels = map[string]string{"label": "bar"} + nodes[2].Labels = map[string]string{"label": "baz"} + clientset.CoreV1().Nodes().Update(&nodes[0]) + clientset.CoreV1().Nodes().Update(&nodes[1]) + clientset.CoreV1().Nodes().Update(&nodes[2]) + cleanNodes, err := c.getAvailableMonNodes() + + assert.Nil(t, err) + assert.Equal(t, 2, len(cleanNodes)) + assert.Equal(t, nodes[1].Name, cleanNodes[0].Name) + assert.Equal(t, nodes[2].Name, cleanNodes[1].Name) +} diff --git a/pkg/operator/mon/spec.go b/pkg/operator/mon/spec.go index 676c53de4..577347f93 100644 --- a/pkg/operator/mon/spec.go +++ b/pkg/operator/mon/spec.go @@ -77,6 +77,16 @@ func (c *Cluster) makeMonPod(config *MonConfig, nodeName string) *v1.Pod { } container := c.monContainer(config, c.clusterInfo.FSID) + podSpec := v1.PodSpec{ + Containers: []v1.Container{container}, + RestartPolicy: v1.RestartPolicyAlways, + NodeSelector: map[string]string{metav1.LabelHostname: nodeName}, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, + k8sutil.ConfigOverrideVolume(), + }, + } + c.placement.ApplyToPodSpec(&podSpec) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -85,15 +95,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, nodeName string) *v1.Pod { Labels: c.getLabels(config.Name), Annotations: map[string]string{}, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{container}, - RestartPolicy: v1.RestartPolicyAlways, - NodeSelector: map[string]string{metav1.LabelHostname: nodeName}, - Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, - k8sutil.ConfigOverrideVolume(), - }, - }, + Spec: podSpec, } k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version) diff --git a/pkg/operator/mon/spec_test.go b/pkg/operator/mon/spec_test.go index c1b6cebb5..43f3f7e67 100644 --- a/pkg/operator/mon/spec_test.go +++ b/pkg/operator/mon/spec_test.go @@ -34,7 +34,7 @@ func TestPodSpecs(t *testing.T) { func testPodSpec(t *testing.T, dataDir string) { clientset := testop.New(1) - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", dataDir, "myversion") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", dataDir, "myversion", k8sutil.Placement{}) c.clusterInfo = testop.CreateClusterInfo(0) config := &MonConfig{Name: "mon0", Port: 6790} diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index f893cc177..1552e7ed5 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -37,6 +37,7 @@ const ( type Cluster struct { context *k8sutil.Context + placement k8sutil.Placement Name string Namespace string Keyring string @@ -45,9 +46,10 @@ type Cluster struct { dataDirHostPath string } -func New(context *k8sutil.Context, name, namespace, version string, storageSpec StorageSpec, dataDirHostPath string) *Cluster { +func New(context *k8sutil.Context, name, namespace, version string, storageSpec StorageSpec, dataDirHostPath string, placement k8sutil.Placement) *Cluster { return &Cluster{ context: context, + placement: placement, Name: name, Namespace: namespace, Version: version, @@ -148,6 +150,13 @@ func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, sel volumes = append(volumes, dirVolume) } + podSpec := v1.PodSpec{ + Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + } + c.placement.ApplyToPodSpec(&podSpec) + return v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: appName, @@ -157,11 +166,7 @@ func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, sel }, Annotations: map[string]string{}, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)}, - RestartPolicy: v1.RestartPolicyAlways, - Volumes: volumes, - }, + Spec: podSpec, } } diff --git a/pkg/operator/osd/osd_test.go b/pkg/operator/osd/osd_test.go index 88877e8e4..f93c2c57f 100644 --- a/pkg/operator/osd/osd_test.go +++ b/pkg/operator/osd/osd_test.go @@ -32,7 +32,7 @@ import ( func TestStartDaemonset(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", StorageSpec{}, "") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", StorageSpec{}, "", k8sutil.Placement{}) // Start the first time err := c.Start() @@ -69,7 +69,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) } clientset := fake.NewSimpleClientset() - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, dataDir) + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, dataDir, k8sutil.Placement{}) n := c.Storage.resolveNode(storageSpec.Nodes[0].Name) replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) @@ -144,7 +144,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "", k8sutil.Placement{}) n := c.Storage.resolveNode(storageSpec.Nodes[0].Name) replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) @@ -186,7 +186,7 @@ func TestStorageSpecConfig(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "") + c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "", k8sutil.Placement{}) n := c.Storage.resolveNode(storageSpec.Nodes[0].Name) replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config) diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index d4f5bcdee..5bc199681 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -37,6 +37,7 @@ const ( type Cluster struct { context *k8sutil.Context + placement k8sutil.Placement Name string Namespace string Version string @@ -44,9 +45,10 @@ type Cluster struct { dataDir string } -func New(context *k8sutil.Context, name, namespace, version string) *Cluster { +func New(context *k8sutil.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster { return &Cluster{ context: context, + placement: placement, Name: name, Namespace: namespace, Version: version, @@ -135,23 +137,26 @@ func (c *Cluster) makeDeployment() *extensions.Deployment { deployment.Name = appName deployment.Namespace = c.Namespace - podSpec := v1.PodTemplateSpec{ + podSpec := v1.PodSpec{ + Containers: []v1.Container{c.rgwContainer()}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + k8sutil.ConfigOverrideVolume(), + }, + } + c.placement.ApplyToPodSpec(&podSpec) + + podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "rook-rgw", Labels: c.getLabels(), Annotations: map[string]string{}, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{c.rgwContainer()}, - RestartPolicy: v1.RestartPolicyAlways, - Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, - k8sutil.ConfigOverrideVolume(), - }, - }, + Spec: podSpec, } - deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas} + deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas} return deployment } diff --git a/pkg/operator/rgw/rgw_test.go b/pkg/operator/rgw/rgw_test.go index 861d6571f..bc47d7911 100644 --- a/pkg/operator/rgw/rgw_test.go +++ b/pkg/operator/rgw/rgw_test.go @@ -42,7 +42,7 @@ func TestStartRGW(t *testing.T) { return []byte(response), "", nil } - c := New(&k8sutil.Context{Clientset: clientset, Factory: factory}, "myname", "ns", "version") + c := New(&k8sutil.Context{Clientset: clientset, Factory: factory}, "myname", "ns", "version", k8sutil.Placement{}) c.dataDir = "/tmp/rgwtest" defer os.RemoveAll(c.dataDir) @@ -77,7 +77,7 @@ func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) { } func TestPodSpecs(t *testing.T) { - c := New(nil, "myname", "ns", "myversion") + c := New(nil, "myname", "ns", "myversion", k8sutil.Placement{}) d := c.makeDeployment() assert.NotNil(t, d)