Merge pull request #715 from utilitywarehouse/placement

Placement information for kubernetes deployments
This commit is contained in:
Travis Nielsen authored and GitHub committed 2017-06-14 08:10:16 -07:00
commit 056845897c
19 files changed
+469 -82

No files matched your search

+8
View File
@@ -13,6 +13,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
- `versionTag`: The version (tag) of the `quay.io/rook/rookd` container that will be deployed. Upgrades are not yet supported if this setting is updated for an existing cluster, but upgrades will be coming.
- `dataDirHostPath`: The host path where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Therefore, for test scenarios, the path must be deleted if you are going to delete a cluster and start a new cluster on the same hosts. More details can be found in the Kubernetes [host path docs](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath).
If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir).
- `placement`: [placement configuration settings](#placement-configuration-settings)
- `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
- `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values.
If individual nodes are specified under the `nodes` field below, then `useAllNodes` must be set to `false`.
@@ -51,5 +52,12 @@ Below are the settings available, both at the cluster and individual node level,
- `walSizeMB`: The size in MB of a bluestore write ahead log (WAL).
- `journalSizeMB`: The size in MB of a filestore journal.
### Placement Configuration Settings
Placement configuration for the cluster services. It includes the following keys: `api`, `mds`, `mon`, `osd`, `rgw` and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes).
A Placement configuration is specified (according to the kubernetes [PodSpec](https://kubernetes.io/docs/api-reference/v1.6/#podspec-v1-core)) as:
- `nodeAffinity`: kubernetes [NodeAffinity](https://kubernetes.io/docs/api-reference/v1.6/#nodeaffinity-v1-core)
- `tolerations`: list of kubernetes [Toleration](https://kubernetes.io/docs/api-reference/v1.6/#toleration-v1-core)
## Sample
A sample cluster TPR can be found and used in the [rook-cluster.yaml](../demo/kubernetes/rook-cluster.yaml) file in the Kubernetes demo directory.
+31
View File
@@ -6,6 +6,37 @@ metadata:
spec:
versionTag: master-latest
dataDirHostPath:
# To control where various services will be scheduled by kubernetes, use the placement configuration sections below.
# The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage' and
# tolerate taints with a key of 'storage-node'.
# placement:
# all:
# nodeAffinity:
# requiredDuringSchedulingIgnoredDuringExecution:
# nodeSelectorTerms:
# - matchExpressions:
# - key: role
# operator: In
# values:
# - storage-node
# tolerations:
# - key: storage-node
# operator: Exists
# api:
# nodeAffinity:
# tolerations:
# mds:
# nodeAffinity:
# tolerations:
# mon:
# nodeAffinity:
# tolerations:
# osd:
# nodeAffinity:
# tolerations:
# rgw:
# nodeAffinity:
# tolerations:
storage: # cluster level storage configuration and selection
useAllNodes: true
useAllDevices: false
+6 -2
View File
@@ -46,7 +46,9 @@ func (s *clusterHandler) GetClusterInfo() (*mon.ClusterInfo, error) {
func (s *clusterHandler) EnableObjectStore() error {
logger.Infof("Starting the Object store")
r := k8srgw.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag)
// Passing an empty Placement{} as the api doesn't know about placement
// information. This should be resolved with the transition to CRD (TPR).
r := k8srgw.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{})
err := r.Start(s.clusterInfo)
if err != nil {
return fmt.Errorf("failed to start rgw. %+v", err)
@@ -76,7 +78,9 @@ func (s *clusterHandler) GetObjectStoreConnectionInfo() (*model.ObjectStoreConne
func (s *clusterHandler) StartFileSystem(fs *model.FilesystemRequest) error {
logger.Infof("Starting the MDS")
c := k8smds.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag)
// Passing an empty Placement{} as the api doesn't know about placement
// information. This should be resolved with the transition to CRD (TPR).
c := k8smds.New(s.k8sContext, s.clusterInfo.Name, s.namespace, s.versionTag, k8sutil.Placement{})
return c.Start(s.k8sContext.Clientset, s.clusterInfo)
}
+16 -11
View File
@@ -53,15 +53,17 @@ var clusterAccessRules = []v1beta1.PolicyRule{
type Cluster struct {
context *k8sutil.Context
placement k8sutil.Placement
Name string
Namespace string
Version string
Replicas int32
}
func New(context *k8sutil.Context, name, namespace, version string) *Cluster {
func New(context *k8sutil.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster {
return &Cluster{
context: context,
placement: placement,
Name: name,
Namespace: namespace,
Version: version,
@@ -141,23 +143,26 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
deployment.Name = DeploymentName
deployment.Namespace = c.Namespace
podSpec := v1.PodTemplateSpec{
podSpec := v1.PodSpec{
ServiceAccountName: DeploymentName,
Containers: []v1.Container{c.apiContainer()},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
},
}
c.placement.ApplyToPodSpec(&podSpec)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: DeploymentName,
Labels: c.getLabels(),
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
ServiceAccountName: DeploymentName,
Containers: []v1.Container{c.apiContainer()},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
},
},
Spec: podSpec,
}
deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas}
deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas}
return deployment
}
+3 -3
View File
@@ -32,7 +32,7 @@ import (
func TestStartAPI(t *testing.T) {
clientset := testop.New(3)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", k8sutil.Placement{})
// start a basic cluster
err := c.Start()
@@ -60,7 +60,7 @@ func validateStart(t *testing.T, c *Cluster) {
func TestPodSpecs(t *testing.T) {
clientset := testop.New(1)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", k8sutil.Placement{})
d := c.makeDeployment()
assert.NotNil(t, d)
@@ -89,7 +89,7 @@ func TestPodSpecs(t *testing.T) {
func TestClusterRole(t *testing.T) {
clientset := testop.New(1)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", k8sutil.Placement{})
// the role is create
err := c.makeClusterRole()
+3 -3
View File
@@ -86,20 +86,20 @@ func (c *Cluster) CreateInstance() error {
}
// Start the mon pods
c.mons = mon.New(c.context, c.Name, c.Namespace, c.Spec.DataDirHostPath, c.Spec.VersionTag)
c.mons = mon.New(c.context, c.Name, c.Namespace, c.Spec.DataDirHostPath, c.Spec.VersionTag, c.Spec.Placement.GetMON())
clusterInfo, err := c.mons.Start()
if err != nil {
return fmt.Errorf("failed to start the mons. %+v", err)
}
c.apis = api.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag)
c.apis = api.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Placement.GetAPI())
err = c.apis.Start()
if err != nil {
return fmt.Errorf("failed to start the REST api. %+v", err)
}
// Start the OSDs
c.osds = osd.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Storage, c.Spec.DataDirHostPath)
c.osds = osd.New(c.context, c.Name, c.Namespace, c.Spec.VersionTag, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.Placement.GetOSD())
err = c.osds.Start()
if err != nil {
return fmt.Errorf("failed to start the osds. %+v", err)
+23 -1
View File
@@ -18,7 +18,10 @@ which also has the apache 2.0 license.
*/
package cluster
import "github.com/rook/rook/pkg/operator/osd"
import (
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/osd"
)
type Spec struct {
// VersionTag is the expected version of the rook container to run in the cluster.
@@ -32,6 +35,9 @@ type Spec struct {
// The path on the host where config and data can be persisted.
DataDirHostPath string `json:"dataDirHostPath"`
// The placement-related configuration to pass to kubernetes (affinity, node selector, tolerations).
Placement PlacementSpec `json:"placement,omitempty"`
// A spec for available storage in the cluster and how it should be used
Storage osd.StorageSpec `json:"storage"`
}
@@ -56,3 +62,19 @@ type ErasureCodeSpec struct {
// Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type)
DataChunks uint `json:"dataChunks"`
}
// PlacementSpec is a set of Placement configurations for the rook cluster.
type PlacementSpec struct {
All k8sutil.Placement `json:"all,omitempty"`
API k8sutil.Placement `json:"api,omitempty"`
MDS k8sutil.Placement `json:"mds,omitempty"`
MON k8sutil.Placement `json:"mon,omitempty"`
OSD k8sutil.Placement `json:"osd,omitempty"`
RGW k8sutil.Placement `json:"rgw,omitempty"`
}
func (p PlacementSpec) GetAPI() k8sutil.Placement { return p.All.Merge(p.API) }
func (p PlacementSpec) GetMDS() k8sutil.Placement { return p.All.Merge(p.MDS) }
func (p PlacementSpec) GetMON() k8sutil.Placement { return p.All.Merge(p.MON) }
func (p PlacementSpec) GetOSD() k8sutil.Placement { return p.All.Merge(p.OSD) }
func (p PlacementSpec) GetRGW() k8sutil.Placement { return p.All.Merge(p.RGW) }
+53
View File
@@ -0,0 +1,53 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8sutil
import (
"k8s.io/client-go/pkg/api/v1"
)
// Placement encapsulates the various kubernetes options that control where
// pods are scheduled and executed.
type Placement struct {
NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"`
Tolerations []v1.Toleration `json:"tolerations,omitemtpy"`
}
func (p Placement) ApplyToPodSpec(t *v1.PodSpec) {
if p.NodeAffinity != nil {
if t.Affinity == nil {
t.Affinity = &v1.Affinity{}
}
t.Affinity.NodeAffinity = p.NodeAffinity
}
if p.Tolerations != nil {
t.Tolerations = p.Tolerations
}
}
// Merge returns a Placement which results from merging the attributes of the
// original Placement with the attributes of the supplied one. The supplied
// Placement's attributes will override the original ones if defined.
func (p Placement) Merge(with Placement) Placement {
ret := p
if with.NodeAffinity != nil {
ret.NodeAffinity = with.NodeAffinity
}
if with.Tolerations != nil {
ret.Tolerations = with.Tolerations
}
return ret
}
+178
View File
@@ -0,0 +1,178 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8sutil
import (
"encoding/json"
"testing"
"github.com/ghodss/yaml"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/pkg/api/v1"
)
func TestPlacement_spec(t *testing.T) {
specYaml := []byte(`
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: foo
operator: In
values:
- bar
tolerations:
- key: foo
operator: Exists`)
// convert the raw spec yaml into JSON
rawJson, err := yaml.YAMLToJSON(specYaml)
assert.Nil(t, err)
// unmarshal the JSON into a strongly typed placement spec object
var placement Placement
err = json.Unmarshal(rawJson, &placement)
assert.Nil(t, err)
// the unmarshalled placement spec should equal the expected spec below
expected := Placement{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
v1.NodeSelectorRequirement{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
},
},
},
},
Tolerations: []v1.Toleration{
v1.Toleration{
Key: "foo",
Operator: v1.TolerationOpExists,
},
},
}
assert.Equal(t, expected, placement)
}
func TestPlacement_ApplyToPodSpec(t *testing.T) {
to := placementTestGetTolerations("foo", "bar")
na := placementTestGetNodeAffinity()
expected := &v1.PodSpec{Affinity: &v1.Affinity{NodeAffinity: na}, Tolerations: to}
var p Placement
var ps *v1.PodSpec
p = Placement{NodeAffinity: na, Tolerations: to}
ps = &v1.PodSpec{}
p.ApplyToPodSpec(ps)
assert.Equal(t, expected, ps)
// partial update
p = Placement{NodeAffinity: na}
ps = &v1.PodSpec{Tolerations: to}
p.ApplyToPodSpec(ps)
assert.Equal(t, expected, ps)
// overridden attributes
p = Placement{NodeAffinity: na, Tolerations: to}
ps = &v1.PodSpec{Tolerations: placementTestGetTolerations("bar", "baz")}
p.ApplyToPodSpec(ps)
assert.Equal(t, expected, ps)
p = Placement{NodeAffinity: na}
nap := placementTestGetNodeAffinity()
nap.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight = 5
ps = &v1.PodSpec{Affinity: &v1.Affinity{NodeAffinity: nap}, Tolerations: to}
p.ApplyToPodSpec(ps)
assert.Equal(t, expected, ps)
}
func TestPlacement_Merge(t *testing.T) {
to := placementTestGetTolerations("foo", "bar")
na := placementTestGetNodeAffinity()
var original, with, expected, merged Placement
original = Placement{}
with = Placement{Tolerations: to}
expected = Placement{Tolerations: to}
merged = original.Merge(with)
assert.Equal(t, expected, merged)
original = Placement{NodeAffinity: na}
with = Placement{Tolerations: to}
expected = Placement{NodeAffinity: na, Tolerations: to}
merged = original.Merge(with)
assert.Equal(t, expected, merged)
original = Placement{Tolerations: placementTestGetTolerations("bar", "baz")}
with = Placement{NodeAffinity: na, Tolerations: to}
expected = Placement{NodeAffinity: na, Tolerations: to}
merged = original.Merge(with)
assert.Equal(t, expected, merged)
}
func placementTestGetTolerations(key, value string) []v1.Toleration {
var ts int64 = 10
return []v1.Toleration{
v1.Toleration{
Key: key,
Operator: v1.TolerationOpExists,
Value: value,
Effect: v1.TaintEffectNoSchedule,
TolerationSeconds: &ts,
},
}
}
func placementTestGetNodeAffinity() *v1.NodeAffinity {
return &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
v1.NodeSelectorRequirement{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
Values: []string{"bar"},
},
},
},
},
},
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
v1.PreferredSchedulingTerm{
Weight: 10,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
v1.NodeSelectorRequirement{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
Values: []string{"bar"},
},
},
},
},
},
}
}
+16 -11
View File
@@ -44,12 +44,14 @@ type Cluster struct {
Version string
Replicas int32
context *k8sutil.Context
placement k8sutil.Placement
dataDir string
}
func New(context *k8sutil.Context, name, namespace, version string) *Cluster {
func New(context *k8sutil.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster {
return &Cluster{
context: context,
placement: placement,
Name: name,
Namespace: namespace,
Version: version,
@@ -131,23 +133,26 @@ func (c *Cluster) makeDeployment(id string) *extensions.Deployment {
deployment.Name = appName
deployment.Namespace = c.Namespace
podSpec := v1.PodTemplateSpec{
podSpec := v1.PodSpec{
Containers: []v1.Container{c.mdsContainer(id)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
k8sutil.ConfigOverrideVolume(),
},
}
c.placement.ApplyToPodSpec(&podSpec)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: appName,
Labels: c.getLabels(),
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.mdsContainer(id)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
k8sutil.ConfigOverrideVolume(),
},
},
Spec: podSpec,
}
deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas}
deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas}
return deployment
}
+2 -2
View File
@@ -40,7 +40,7 @@ func TestStartMDS(t *testing.T) {
return []byte(response), "", nil
}
c := New(&k8sutil.Context{Factory: factory}, "myname", "ns", "myversion")
c := New(&k8sutil.Context{Factory: factory}, "myname", "ns", "myversion", k8sutil.Placement{})
c.dataDir = "/tmp/mdstest"
defer os.RemoveAll(c.dataDir)
@@ -67,7 +67,7 @@ func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) {
}
func TestPodSpecs(t *testing.T) {
c := New(nil, "myname", "ns", "myversion")
c := New(nil, "myname", "ns", "myversion", k8sutil.Placement{})
mdsID := "mds1"
d := c.makeDeployment(mdsID)
+38 -9
View File
@@ -28,6 +28,7 @@ import (
"github.com/rook/rook/pkg/util"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/pkg/api/v1"
)
@@ -53,10 +54,10 @@ type Cluster struct {
MasterHost string
Size int
Paused bool
AntiAffinity bool
Port int32
context *k8sutil.Context
clusterInfo *mon.ClusterInfo
placement k8sutil.Placement
maxMonID int
configDir string
waitForStart bool
@@ -68,15 +69,15 @@ type MonConfig struct {
Port int32
}
func New(context *k8sutil.Context, name, namespace, dataDirHostPath, version string) *Cluster {
func New(context *k8sutil.Context, name, namespace, dataDirHostPath, version string, placement k8sutil.Placement) *Cluster {
return &Cluster{
context: context,
placement: placement,
dataDirHostPath: dataDirHostPath,
Name: name,
Namespace: namespace,
Version: version,
Size: 3,
AntiAffinity: true,
maxMonID: -1,
configDir: k8sutil.DataDir,
waitForStart: true,
@@ -422,7 +423,7 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, error) {
// choose nodes for the new mons that don't have mons currently
availableNodes := []v1.Node{}
for _, node := range nodes.Items {
if !nodesInUse.Contains(node.Name) && validNode(node) {
if !nodesInUse.Contains(node.Name) && validNode(node, c.placement) {
availableNodes = append(availableNodes, node)
}
}
@@ -432,7 +433,7 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, error) {
if len(availableNodes) == 0 {
logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
for _, node := range nodes.Items {
if validNode(node) {
if validNode(node, c.placement) {
availableNodes = append(availableNodes, node)
}
}
@@ -444,15 +445,43 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, error) {
return availableNodes, nil
}
func validNode(node v1.Node) bool {
func validNode(node v1.Node, placement k8sutil.Placement) bool {
// a node cannot be disabled
if node.Spec.Unschedulable {
return false
}
// a node cannot be tainted
for _, t := range node.Spec.Taints {
if t.Effect == v1.TaintEffectNoSchedule || t.Effect == v1.TaintEffectPreferNoSchedule {
// a node matches the NodeAffinity configuration
// ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they
// should not be used to judge a node unusable
if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
nodeMatches := false
for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
nodeSelector, err := v1.NodeSelectorRequirementsAsSelector(req.MatchExpressions)
if err != nil {
logger.Infof("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions)
return false
}
if nodeSelector.Matches(labels.Set(node.Labels)) {
nodeMatches = true
break
}
}
if !nodeMatches {
return false
}
}
// a node is tainted and cannot be tolerated
for _, taint := range node.Spec.Taints {
isTolerated := false
for _, toleration := range placement.Tolerations {
if toleration.ToleratesTaint(&taint) {
isTolerated = true
break
}
}
if !isTolerated {
return false
}
}
+47 -7
View File
@@ -33,7 +33,7 @@ import (
func TestStartMonPods(t *testing.T) {
clientset := test.New(3)
factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"}
c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, MaxRetries: 1}, "myname", "ns", "", "myversion")
c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, MaxRetries: 1}, "myname", "ns", "", "myversion", k8sutil.Placement{})
// start a basic cluster
// an error is expected since mocking always creates pods that are not running
@@ -68,7 +68,7 @@ func validateStart(t *testing.T, c *Cluster) {
func TestSaveMonEndpoints(t *testing.T) {
clientset := test.New(1)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{})
c.clusterInfo = test.CreateClusterInfo(1)
// create the initial config map
@@ -92,7 +92,7 @@ func TestSaveMonEndpoints(t *testing.T) {
func TestCheckHealth(t *testing.T) {
clientset := test.New(1)
factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"}
c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, RetryDelay: 1, MaxRetries: 1}, "myname", "ns", "", "myversion")
c := New(&k8sutil.Context{Clientset: clientset, Factory: factory, RetryDelay: 1, MaxRetries: 1}, "myname", "ns", "", "myversion", k8sutil.Placement{})
c.clusterInfo = test.CreateClusterInfo(1)
c.configDir = "/tmp/healthtest"
c.waitForStart = false
@@ -152,7 +152,7 @@ func TestMonID(t *testing.T) {
func TestAvailableMonNodes(t *testing.T) {
clientset := test.New(1)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{})
c.clusterInfo = test.CreateClusterInfo(0)
nodes, err := c.getAvailableMonNodes()
assert.Nil(t, err)
@@ -169,7 +169,7 @@ func TestAvailableMonNodes(t *testing.T) {
func TestAvailableNodesInUse(t *testing.T) {
clientset := test.New(3)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{})
c.clusterInfo = test.CreateClusterInfo(0)
// all three nodes are available by default
@@ -200,7 +200,8 @@ func TestAvailableNodesInUse(t *testing.T) {
func TestTaintedNodes(t *testing.T) {
clientset := test.New(3)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion")
placement := k8sutil.Placement{Tolerations: []v1.Toleration{{Effect: v1.TaintEffectNoExecute}}}
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", placement)
c.clusterInfo = test.CreateClusterInfo(0)
nodes, err := c.getAvailableMonNodes()
@@ -228,9 +229,48 @@ func TestTaintedNodes(t *testing.T) {
clientset.CoreV1().Nodes().Update(&nodes[0])
clientset.CoreV1().Nodes().Update(&nodes[1])
clientset.CoreV1().Nodes().Update(&nodes[2])
assert.Nil(t, err)
cleanNodes, err := c.getAvailableMonNodes()
assert.Nil(t, err)
assert.Equal(t, 1, len(cleanNodes))
assert.Equal(t, nodes[2].Name, cleanNodes[0].Name)
}
func TestNodeAffinity(t *testing.T) {
clientset := test.New(3)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "", "myversion", k8sutil.Placement{})
c.clusterInfo = test.CreateClusterInfo(0)
nodes, err := c.getAvailableMonNodes()
assert.Nil(t, err)
assert.Equal(t, 3, len(nodes))
c.placement.NodeAffinity = &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
v1.NodeSelectorRequirement{
Key: "label",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar", "baz"},
},
},
},
},
},
}
// label nodes so node0 will not be schedulable for new pods
nodes[0].Labels = map[string]string{"label": "foo"}
nodes[1].Labels = map[string]string{"label": "bar"}
nodes[2].Labels = map[string]string{"label": "baz"}
clientset.CoreV1().Nodes().Update(&nodes[0])
clientset.CoreV1().Nodes().Update(&nodes[1])
clientset.CoreV1().Nodes().Update(&nodes[2])
cleanNodes, err := c.getAvailableMonNodes()
assert.Nil(t, err)
assert.Equal(t, 2, len(cleanNodes))
assert.Equal(t, nodes[1].Name, cleanNodes[0].Name)
assert.Equal(t, nodes[2].Name, cleanNodes[1].Name)
}
+11 -9
View File
@@ -77,6 +77,16 @@ func (c *Cluster) makeMonPod(config *MonConfig, nodeName string) *v1.Pod {
}
container := c.monContainer(config, c.clusterInfo.FSID)
podSpec := v1.PodSpec{
Containers: []v1.Container{container},
RestartPolicy: v1.RestartPolicyAlways,
NodeSelector: map[string]string{metav1.LabelHostname: nodeName},
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
k8sutil.ConfigOverrideVolume(),
},
}
c.placement.ApplyToPodSpec(&podSpec)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
@@ -85,15 +95,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, nodeName string) *v1.Pod {
Labels: c.getLabels(config.Name),
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{container},
RestartPolicy: v1.RestartPolicyAlways,
NodeSelector: map[string]string{metav1.LabelHostname: nodeName},
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
k8sutil.ConfigOverrideVolume(),
},
},
Spec: podSpec,
}
k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version)
+1 -1
View File
@@ -34,7 +34,7 @@ func TestPodSpecs(t *testing.T) {
func testPodSpec(t *testing.T, dataDir string) {
clientset := testop.New(1)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", dataDir, "myversion")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", dataDir, "myversion", k8sutil.Placement{})
c.clusterInfo = testop.CreateClusterInfo(0)
config := &MonConfig{Name: "mon0", Port: 6790}
+11 -6
View File
@@ -37,6 +37,7 @@ const (
type Cluster struct {
context *k8sutil.Context
placement k8sutil.Placement
Name string
Namespace string
Keyring string
@@ -45,9 +46,10 @@ type Cluster struct {
dataDirHostPath string
}
func New(context *k8sutil.Context, name, namespace, version string, storageSpec StorageSpec, dataDirHostPath string) *Cluster {
func New(context *k8sutil.Context, name, namespace, version string, storageSpec StorageSpec, dataDirHostPath string, placement k8sutil.Placement) *Cluster {
return &Cluster{
context: context,
placement: placement,
Name: name,
Namespace: namespace,
Version: version,
@@ -148,6 +150,13 @@ func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, sel
volumes = append(volumes, dirVolume)
}
podSpec := v1.PodSpec{
Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
}
c.placement.ApplyToPodSpec(&podSpec)
return v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: appName,
@@ -157,11 +166,7 @@ func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, sel
},
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.osdContainer(devices, directories, selection, config)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
},
Spec: podSpec,
}
}
+4 -4
View File
@@ -32,7 +32,7 @@ import (
func TestStartDaemonset(t *testing.T) {
clientset := fake.NewSimpleClientset()
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", StorageSpec{}, "")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", StorageSpec{}, "", k8sutil.Placement{})
// Start the first time
err := c.Start()
@@ -69,7 +69,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool)
}
clientset := fake.NewSimpleClientset()
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, dataDir)
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, dataDir, k8sutil.Placement{})
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
@@ -144,7 +144,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
}
clientset := fake.NewSimpleClientset()
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "", k8sutil.Placement{})
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
@@ -186,7 +186,7 @@ func TestStorageSpecConfig(t *testing.T) {
}
clientset := fake.NewSimpleClientset()
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "")
c := New(&k8sutil.Context{Clientset: clientset}, "myname", "ns", "myversion", storageSpec, "", k8sutil.Placement{})
n := c.Storage.resolveNode(storageSpec.Nodes[0].Name)
replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Directories, n.Selection, n.Config)
+16 -11
View File
@@ -37,6 +37,7 @@ const (
type Cluster struct {
context *k8sutil.Context
placement k8sutil.Placement
Name string
Namespace string
Version string
@@ -44,9 +45,10 @@ type Cluster struct {
dataDir string
}
func New(context *k8sutil.Context, name, namespace, version string) *Cluster {
func New(context *k8sutil.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster {
return &Cluster{
context: context,
placement: placement,
Name: name,
Namespace: namespace,
Version: version,
@@ -135,23 +137,26 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
deployment.Name = appName
deployment.Namespace = c.Namespace
podSpec := v1.PodTemplateSpec{
podSpec := v1.PodSpec{
Containers: []v1.Container{c.rgwContainer()},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
k8sutil.ConfigOverrideVolume(),
},
}
c.placement.ApplyToPodSpec(&podSpec)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: "rook-rgw",
Labels: c.getLabels(),
Annotations: map[string]string{},
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.rgwContainer()},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
k8sutil.ConfigOverrideVolume(),
},
},
Spec: podSpec,
}
deployment.Spec = extensions.DeploymentSpec{Template: podSpec, Replicas: &c.Replicas}
deployment.Spec = extensions.DeploymentSpec{Template: podTemplateSpec, Replicas: &c.Replicas}
return deployment
}
+2 -2
View File
@@ -42,7 +42,7 @@ func TestStartRGW(t *testing.T) {
return []byte(response), "", nil
}
c := New(&k8sutil.Context{Clientset: clientset, Factory: factory}, "myname", "ns", "version")
c := New(&k8sutil.Context{Clientset: clientset, Factory: factory}, "myname", "ns", "version", k8sutil.Placement{})
c.dataDir = "/tmp/rgwtest"
defer os.RemoveAll(c.dataDir)
@@ -77,7 +77,7 @@ func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) {
}
func TestPodSpecs(t *testing.T) {
c := New(nil, "myname", "ns", "myversion")
c := New(nil, "myname", "ns", "myversion", k8sutil.Placement{})
d := c.makeDeployment()
assert.NotNil(t, d)