From 1b2fb9657462734644b165e81418d0bfa105d828 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 10 Mar 2017 13:46:55 -0800 Subject: [PATCH] allow specifying a host path for the data-dir --- pkg/operator/cluster.go | 7 +++++-- pkg/operator/mon/mon.go | 38 +++++++++++++++++++----------------- pkg/operator/mon/mon_test.go | 2 +- pkg/operator/mon/pod.go | 6 +++++- pkg/operator/mon/pod_test.go | 14 ++++++++++++- pkg/operator/osd/osd.go | 29 ++++++++++++++++----------- pkg/operator/osd/osd_test.go | 17 +++++++++++----- 7 files changed, 74 insertions(+), 39 deletions(-) diff --git a/pkg/operator/cluster.go b/pkg/operator/cluster.go index 60a513099..b5827ea00 100644 --- a/pkg/operator/cluster.go +++ b/pkg/operator/cluster.go @@ -59,6 +59,9 @@ type ClusterSpec struct { // A regular expression to allow more fine-grained selection of devices on nodes across the cluster DeviceFilter string `json:"deviceFilter"` + + // The path on the host where config and data can be persisted. + DataDirHostPath string `json:"dataDirHostPath"` } func newCluster(spec ClusterSpec, factory client.ConnectionFactory, clientset kubernetes.Interface) *Cluster { @@ -82,7 +85,7 @@ func (c *Cluster) CreateInstance() error { } // Start the mon pods - m := mon.New(c.Spec.Namespace, c.factory, c.Spec.Version) + m := mon.New(c.Spec.Namespace, c.factory, c.Spec.DataDirHostPath, c.Spec.Version) cluster, err := m.Start(c.clientset) if err != nil { return fmt.Errorf("failed to start the mons. %+v", err) @@ -95,7 +98,7 @@ func (c *Cluster) CreateInstance() error { } // Start the OSDs - osds := osd.New(c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.UseAllDevices) + osds := osd.New(c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices) err = osds.Start(c.clientset, cluster) if err != nil { return fmt.Errorf("failed to start the osds. %+v", err) diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index 27b223fb4..20ffa1e1a 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -39,17 +39,18 @@ const ( ) type Cluster struct { - Namespace string - Keyring string - ClusterName string - Version string - MasterHost string - Size int - Paused bool - AntiAffinity bool - Port int32 - factory client.ConnectionFactory - retryDelay int + Namespace string + Keyring string + ClusterName string + Version string + MasterHost string + Size int + Paused bool + AntiAffinity bool + Port int32 + factory client.ConnectionFactory + retryDelay int + dataDirHostPath string } type MonConfig struct { @@ -57,14 +58,15 @@ type MonConfig struct { Port int32 } -func New(namespace string, factory client.ConnectionFactory, version string) *Cluster { +func New(namespace string, factory client.ConnectionFactory, dataDirHostPath, version string) *Cluster { return &Cluster{ - Namespace: namespace, - Version: version, - Size: 3, - factory: factory, - AntiAffinity: true, - retryDelay: 6, + Namespace: namespace, + Version: version, + Size: 3, + factory: factory, + dataDirHostPath: dataDirHostPath, + AntiAffinity: true, + retryDelay: 6, } } diff --git a/pkg/operator/mon/mon_test.go b/pkg/operator/mon/mon_test.go index 1f9a2db6e..2fd13a318 100644 --- a/pkg/operator/mon/mon_test.go +++ b/pkg/operator/mon/mon_test.go @@ -29,7 +29,7 @@ import ( func TestStartMonPods(t *testing.T) { factory := &testclient.MockConnectionFactory{Fsid: "fsid", SecretKey: "mysecret"} - c := New("ns", factory, "myversion") + c := New("ns", factory, "", "myversion") c.retryDelay = 0 clientset := test.New(3) diff --git a/pkg/operator/mon/pod.go b/pkg/operator/mon/pod.go index 073519984..d0baf8750 100644 --- a/pkg/operator/mon/pod.go +++ b/pkg/operator/mon/pod.go @@ -43,6 +43,10 @@ func getLabels(clusterName string) map[string]string { func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, antiAffinity bool) *v1.Pod { container := c.monContainer(config, clusterInfo) + dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} + if c.dataDirHostPath != "" { + dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} + } pod := &v1.Pod{ ObjectMeta: v1.ObjectMeta{ @@ -55,7 +59,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an Containers: []v1.Container{container}, RestartPolicy: v1.RestartPolicyAlways, Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, }, }, } diff --git a/pkg/operator/mon/pod_test.go b/pkg/operator/mon/pod_test.go index 0f3dc323e..de384c8d5 100644 --- a/pkg/operator/mon/pod_test.go +++ b/pkg/operator/mon/pod_test.go @@ -27,7 +27,12 @@ import ( ) func TestPodSpecs(t *testing.T) { - c := New("ns", nil, "myversion") + testPodSpec(t, "") + testPodSpec(t, "/var/lib/mydatadir") +} + +func testPodSpec(t *testing.T, dataDir string) { + c := New("ns", nil, dataDir, "myversion") config := &MonConfig{Name: "mon0", Port: 6790} info := testop.CreateClusterInfo(0) @@ -37,6 +42,13 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, v1.RestartPolicyAlways, pod.Spec.RestartPolicy) assert.Equal(t, 1, len(pod.Spec.Volumes)) assert.Equal(t, "rook-data", pod.Spec.Volumes[0].Name) + if dataDir == "" { + assert.NotNil(t, pod.Spec.Volumes[0].EmptyDir) + assert.Nil(t, pod.Spec.Volumes[0].HostPath) + } else { + assert.Nil(t, pod.Spec.Volumes[0].EmptyDir) + assert.Equal(t, dataDir, pod.Spec.Volumes[0].HostPath.Path) + } assert.Equal(t, "mon0", pod.ObjectMeta.Name) assert.Equal(t, "mon", pod.ObjectMeta.Labels["app"]) diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index d8adaa954..99d6d4951 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -32,19 +32,21 @@ const ( ) type Cluster struct { - Namespace string - Keyring string - Version string - useAllDevices bool - deviceFilter string + Namespace string + Keyring string + Version string + dataDirHostPath string + deviceFilter string + useAllDevices bool } -func New(namespace, version, deviceFilter string, useAllDevices bool) *Cluster { +func New(namespace, version, deviceFilter, dataDirHostPath string, useAllDevices bool) *Cluster { return &Cluster{ - Namespace: namespace, - Version: version, - deviceFilter: deviceFilter, - useAllDevices: useAllDevices, + Namespace: namespace, + Version: version, + deviceFilter: deviceFilter, + dataDirHostPath: dataDirHostPath, + useAllDevices: useAllDevices, } } @@ -74,6 +76,11 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet ds.Name = appName ds.Namespace = c.Namespace + dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} + if c.dataDirHostPath != "" { + dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}} + } + podSpec := v1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ Name: appName, @@ -87,7 +94,7 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet Containers: []v1.Container{c.osdContainer(cluster)}, RestartPolicy: v1.RestartPolicyAlways, Volumes: []v1.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + {Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource}, {Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}, }, }, diff --git a/pkg/operator/osd/osd_test.go b/pkg/operator/osd/osd_test.go index f7677927f..2a6e219dd 100644 --- a/pkg/operator/osd/osd_test.go +++ b/pkg/operator/osd/osd_test.go @@ -27,7 +27,7 @@ import ( ) func TestStartDaemonset(t *testing.T) { - c := New("ns", "myversion", "", false) + c := New("ns", "myversion", "", "", false) clientset := fake.NewSimpleClientset() @@ -47,12 +47,12 @@ func TestStartDaemonset(t *testing.T) { } func TestDaemonset(t *testing.T) { - testPodDevices(t, true) - testPodDevices(t, false) + testPodDevices(t, "", true) + testPodDevices(t, "/var/lib/mydatadir", false) } -func testPodDevices(t *testing.T, useDevices bool) { - c := New("ns", "myversion", "", useDevices) +func testPodDevices(t *testing.T, dataDir string, useDevices bool) { + c := New("ns", "myversion", "", dataDir, useDevices) info := testop.CreateClusterInfo(1) daemonSet, err := c.makeDaemonSet(info) assert.Nil(t, err) @@ -63,6 +63,13 @@ func testPodDevices(t *testing.T, useDevices bool) { assert.Equal(t, 2, len(daemonSet.Spec.Template.Spec.Volumes)) assert.Equal(t, "rook-data", daemonSet.Spec.Template.Spec.Volumes[0].Name) assert.Equal(t, "devices", daemonSet.Spec.Template.Spec.Volumes[1].Name) + if dataDir == "" { + assert.NotNil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir) + assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].HostPath) + } else { + assert.Nil(t, daemonSet.Spec.Template.Spec.Volumes[0].EmptyDir) + assert.Equal(t, dataDir, daemonSet.Spec.Template.Spec.Volumes[0].HostPath.Path) + } assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Name) assert.Equal(t, "osd", daemonSet.Spec.Template.ObjectMeta.Labels["app"])