services: start daemons with tini at pid 1

This commit is contained in:
Travis Nielsen committed 2017-07-21 14:35:56 -07:00
1 parent 15b1f00d37
commit 07cbf9f596
20 files changed
+107 -83

No files matched your search

+2 -1
View File
@@ -69,6 +69,7 @@ type config struct {
storeConfig osd.StoreConfig
networkInfo clusterd.NetworkInfo
monEndpoints string
nodeName string
}
func main() {
@@ -155,7 +156,7 @@ func joinCluster() error {
services := []*clusterd.ClusterService{
ceph.NewCephService(cfg.devices, cfg.metadataDevice, cfg.directories,
cfg.forceFormat, cfg.location, clusterInfo.AdminSecret, cfg.storeConfig),
cfg.forceFormat, cfg.location, clusterInfo.AdminSecret, cfg.storeConfig, cfg.nodeName),
}
cfg.nodeID, err = util.LoadPersistedNodeID(cfg.dataDir)
+3 -2
View File
@@ -43,6 +43,7 @@ func addOSDFlags(command *cobra.Command) {
command.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
command.Flags().BoolVar(&cfg.forceFormat, "force-format", false,
"true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!")
command.Flags().StringVar(&cfg.nodeName, "node-name", os.Getenv("HOSTNAME"), "the host name of the node")
// OSD store config flags
command.Flags().IntVar(&cfg.storeConfig.WalSizeMB, "osd-wal-size", osd.WalDefaultSizeMB, "default size (MB) for OSD write ahead log (WAL) (bluestore)")
@@ -60,7 +61,7 @@ func init() {
}
func startOSD(cmd *cobra.Command, args []string) error {
if err := flags.VerifyRequiredFlags(osdCmd, []string{"cluster-name", "mon-endpoints", "mon-secret", "admin-secret"}); err != nil {
if err := flags.VerifyRequiredFlags(osdCmd, []string{"cluster-name", "mon-endpoints", "mon-secret", "admin-secret", "node-name"}); err != nil {
return err
}
@@ -82,7 +83,7 @@ func startOSD(cmd *cobra.Command, args []string) error {
forceFormat := false
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
agent := osd.NewAgent(dataDevices, usingDeviceFilter, cfg.metadataDevice, cfg.directories, forceFormat,
cfg.location, cfg.storeConfig, &clusterInfo)
cfg.location, cfg.storeConfig, &clusterInfo, cfg.nodeName)
err := osd.Run(createContext(), agent)
if err != nil {
+2 -2
View File
@@ -30,7 +30,7 @@ const (
// create a new ceph service
func NewCephService(devices, metadataDevice, directories string, forceFormat bool,
location, adminSecret string, storeConfig osd.StoreConfig) *clusterd.ClusterService {
location, adminSecret string, storeConfig osd.StoreConfig, nodeName string) *clusterd.ClusterService {
return &clusterd.ClusterService{
Name: cephName,
@@ -38,7 +38,7 @@ func NewCephService(devices, metadataDevice, directories string, forceFormat boo
Agents: []clusterd.ServiceAgent{
mon.NewAgent(),
mgr.NewAgent(),
osd.NewAgent(devices, false, metadataDevice, directories, forceFormat, location, storeConfig, nil),
osd.NewAgent(devices, false, metadataDevice, directories, forceFormat, location, storeConfig, nil, nodeName),
mds.NewAgent(),
rgw.NewAgent(),
},
+1 -1
View File
@@ -140,7 +140,7 @@ func TestRefreshKeys(t *testing.T) {
func TestNewCephService(t *testing.T) {
service := NewCephService("a,b,c", "", "", true, "root=default", "", osd.StoreConfig{})
service := NewCephService("a,b,c", "", "", true, "root=default", "", osd.StoreConfig{}, "mynode")
assert.NotNil(t, service)
assert.Equal(t, "/rook/services/ceph/osd/desired", service.Leader.RefreshKeys()[0].Path)
assert.Equal(t, 5, len(service.Agents))
+5 -2
View File
@@ -52,6 +52,7 @@ const (
type OsdAgent struct {
cluster *mon.ClusterInfo
nodeName string
forceFormat bool
location string
osdProc map[int]*proc.MonitoredProc
@@ -67,10 +68,12 @@ type OsdAgent struct {
}
func NewAgent(devices string, usingDeviceFilter bool, metadataDevice, directories string, forceFormat bool,
location string, storeConfig StoreConfig, cluster *mon.ClusterInfo) *OsdAgent {
location string, storeConfig StoreConfig, cluster *mon.ClusterInfo, nodeName string) *OsdAgent {
return &OsdAgent{devices: devices, usingDeviceFilter: usingDeviceFilter, metadataDevice: metadataDevice,
directories: directories, forceFormat: forceFormat, location: location, storeConfig: storeConfig, cluster: cluster}
directories: directories, forceFormat: forceFormat, location: location, storeConfig: storeConfig, cluster: cluster,
nodeName: nodeName,
}
}
func (a *OsdAgent) Name() string {
+1 -1
View File
@@ -349,7 +349,7 @@ func createTestAgent(t *testing.T, nodeID, devices, configDir string, storeConfi
storeConfig = &StoreConfig{StoreType: Bluestore}
}
etcdClient := util.NewMockEtcdClient()
agent := NewAgent(devices, false, "", "", forceFormat, location, *storeConfig, nil)
agent := NewAgent(devices, false, "", "", forceFormat, location, *storeConfig, nil, "myhost")
agent.cluster = &mon.ClusterInfo{Name: "myclust"}
agent.Initialize(&clusterd.Context{
DirectContext: clusterd.DirectContext{EtcdClient: etcdClient, NodeID: nodeID},
+20
View File
@@ -38,6 +38,11 @@ const (
)
func Run(context *clusterd.Context, agent *OsdAgent) error {
if err := setNodeName(context, agent.nodeName); err != nil {
// It's best effort so will not block creation of osd if there is a failure.
logger.Warningf("failed to set hostname: %+v", err)
}
// write the latest config to the config dir
if err := mon.GenerateAdminConnectionConfig(context, agent.cluster); err != nil {
return fmt.Errorf("failed to write connection config. %+v", err)
@@ -86,6 +91,21 @@ func Run(context *clusterd.Context, agent *OsdAgent) error {
return nil
}
// Set the name of the node. We don't want the name of the pod,
// which would change if the pod is re-created.
func setNodeName(context *clusterd.Context, nodeName string) error {
nodeName = strings.Replace(nodeName, ".", "-", -1)
if nodeName == "" {
return fmt.Errorf("node name is not set")
}
err := context.ProcMan.Run("", "hostname", nodeName)
if err != nil {
return fmt.Errorf("hostname failed to set: %+v", err)
}
return nil
}
func getAvailableDevices(context *clusterd.Context, devices []*inventory.LocalDisk, desiredDevices string,
metadataDevice string, usingDeviceFilter bool) (*DeviceOsdMapping, error) {
+12
View File
@@ -30,6 +30,18 @@ import (
"github.com/stretchr/testify/assert"
)
func TestSetNodeName(t *testing.T) {
executor := &exectest.MockExecutor{}
executor.MockExecuteCommand = func(name string, command string, args ...string) error {
assert.Equal(t, "hostname", command)
assert.Equal(t, args[0], "myhost")
return nil
}
context := &clusterd.Context{ProcMan: proc.New(executor), Executor: executor}
setNodeName(context, "myhost")
}
func TestStoreOSDDirMap(t *testing.T) {
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
+7 -6
View File
@@ -171,13 +171,14 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
func (c *Cluster) apiContainer() v1.Container {
command := fmt.Sprintf("/usr/local/bin/rook api --config-dir=%s --port=%d ", k8sutil.DataDir, model.Port)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: DeploymentName,
Image: k8sutil.MakeRookImage(c.Version),
Args: []string{
"api",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
fmt.Sprintf("--port=%d", model.Port),
},
Name: DeploymentName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
},
+3 -5
View File
@@ -16,12 +16,10 @@ limitations under the License.
package api
import (
"fmt"
"strings"
"testing"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/model"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/kit"
testop "github.com/rook/rook/pkg/operator/test"
@@ -84,9 +82,9 @@ func TestPodSpecs(t *testing.T) {
assert.True(t, strings.HasPrefix(v.Name, "ROOKD_"))
}
expectedCommand := fmt.Sprintf("/usr/local/bin/rook api --config-dir=/var/lib/rook --port=%d", model.Port)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
assert.Equal(t, "api", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
assert.Equal(t, "--port=8124", cont.Args[2])
}
func TestClusterRole(t *testing.T) {
+7 -7
View File
@@ -147,14 +147,14 @@ func (c *Cluster) makeDeployment(id string) *extensions.Deployment {
func (c *Cluster) mdsContainer(id string) v1.Container {
command := fmt.Sprintf("/usr/local/bin/rook mds --config-dir=%s --mds-id=%s ",
k8sutil.DataDir, id)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
Args: []string{
"mds",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
fmt.Sprintf("--mds-id=%s", id),
},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
+3 -6
View File
@@ -16,10 +16,8 @@ limitations under the License.
package mds
import (
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/rook/rook/pkg/clusterd"
@@ -87,8 +85,7 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 6, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/local/bin/rook mds --config-dir=/var/lib/rook --mds-id=%s ",
mdsID)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
assert.Equal(t, "mds", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
assert.Equal(t, "--mds-id=mds1", cont.Args[2])
}
+6 -5
View File
@@ -108,12 +108,13 @@ func (c *Cluster) makeDeployment(name string) *extensions.Deployment {
func (c *Cluster) mgrContainer(name string) v1.Container {
command := fmt.Sprintf("/usr/local/bin/rook mgr --config-dir=%s", k8sutil.DataDir)
return v1.Container{
// FIX: sleep 5 for a flaky network setup
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: name,
Image: k8sutil.MakeRookImage(c.Version),
Args: []string{
"mgr",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
},
Name: name,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
+2 -4
View File
@@ -19,7 +19,6 @@ import (
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/rook/rook/pkg/clusterd"
@@ -88,7 +87,6 @@ func TestPodSpec(t *testing.T) {
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 7, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/local/bin/rook mgr --config-dir=/var/lib/rook")
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
assert.Equal(t, "mgr", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
}
+9 -7
View File
@@ -104,15 +104,17 @@ func (c *Cluster) makeMonPod(config *MonConfig, nodeName string) *v1.Pod {
}
func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container {
command := fmt.Sprintf("/usr/local/bin/rook mon --config-dir=%s --name=%s --port=%d --fsid=%s",
k8sutil.DataDir, config.Name, config.Port, fsid)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
Args: []string{
"mon",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
fmt.Sprintf("--name=%s", config.Name),
fmt.Sprintf("--port=%d", config.Port),
fmt.Sprintf("--fsid=%s", fsid),
},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
Ports: []v1.ContainerPort{
{
Name: "client",
+6 -5
View File
@@ -17,7 +17,6 @@ package mon
import (
"fmt"
"strings"
"testing"
"github.com/rook/rook/pkg/clusterd"
@@ -66,8 +65,10 @@ func testPodSpec(t *testing.T, dataDir string) {
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 6, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/local/bin/rook mon --config-dir=/var/lib/rook --name=%s --port=%d --fsid=%s",
config.Name, config.Port, c.clusterInfo.FSID)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
logger.Infof("Command : %+v", cont.Command)
assert.Equal(t, "mon", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
assert.Equal(t, "--name=mon0", cont.Args[2])
assert.Equal(t, "--port=6790", cont.Args[3])
assert.Equal(t, fmt.Sprintf("--fsid=%s", c.clusterInfo.FSID), cont.Args[4])
}
+4 -11
View File
@@ -176,7 +176,7 @@ func (c *Cluster) podTemplateSpec(devices []Device, directories []Directory, sel
func (c *Cluster) osdContainer(devices []Device, directories []Directory, selection Selection, config Config) v1.Container {
envVars := []v1.EnvVar{
hostnameEnvVar(),
nodeNameEnvVar(),
opmon.ClusterNameEnvVar(c.Namespace),
opmon.MonEndpointEnvVar(),
opmon.MonSecretEnvVar(),
@@ -240,17 +240,10 @@ func (c *Cluster) osdContainer(devices []Device, directories []Directory, select
envVars = append(envVars, locationEnvVar(config.Location))
}
// set the hostname to the host's name from the downstream api.
// the crush map doesn't like hostnames with periods, so we replace them with underscores.
hostnameUpdate := `echo $(HOSTNAME) | sed "s/\./_/g" > /etc/hostname; hostname -F /etc/hostname`
command := "/usr/local/bin/rook osd"
privileged := true
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
// Set the hostname so we have the pod's host in the crush map rather than the pod container name
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s; %s", hostnameUpdate, command)},
Args: []string{"osd"},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: volumeMounts,
@@ -259,8 +252,8 @@ func (c *Cluster) osdContainer(devices []Device, directories []Directory, select
}
}
func hostnameEnvVar() v1.EnvVar {
return v1.EnvVar{Name: "HOSTNAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}
func nodeNameEnvVar() v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_NODE_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}
}
func dataDevicesEnvVar(dataDevices string) v1.EnvVar {
+2 -6
View File
@@ -17,7 +17,6 @@ package osd
import (
"strconv"
"strings"
"testing"
cephosd "github.com/rook/rook/pkg/ceph/osd"
@@ -51,8 +50,7 @@ func TestPodContainer(t *testing.T) {
assert.Equal(t, 1, len(c.Spec.Containers))
container := c.Spec.Containers[0]
assert.Equal(t, 7, len(container.Env))
assert.True(t, strings.Contains(container.Command[2], `echo $(HOSTNAME) | sed "s/\./_/g" > /etc/hostname; hostname -F /etc/hostname`))
assert.True(t, strings.Contains(container.Command[2], "/usr/local/bin/rook osd"))
assert.Equal(t, "osd", container.Args[0])
}
func TestDaemonset(t *testing.T) {
@@ -98,9 +96,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool)
cont := replicaSet.Spec.Template.Spec.Containers[0]
assert.Equal(t, "rook/rook:myversion", cont.Image)
assert.Equal(t, 3, len(cont.VolumeMounts))
expectedCommand := "/usr/local/bin/rook osd"
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
assert.Equal(t, "osd", cont.Args[0])
// verify the config dir env var
verifyEnvVar(t, cont.Env, "ROOKD_CONFIG_DIR", "/var/lib/rook", true)
+8 -7
View File
@@ -149,14 +149,15 @@ func (c *Cluster) makeDeployment() *extensions.Deployment {
func (c *Cluster) rgwContainer() v1.Container {
command := fmt.Sprintf("/usr/local/bin/rook rgw --config-dir=%s --rgw-port=%d --rgw-host=%s",
k8sutil.DataDir, cephrgw.RGWPort, cephrgw.DNSName)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
Args: []string{
"rgw",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort),
fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName),
},
Name: appName,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
+4 -5
View File
@@ -19,7 +19,6 @@ import (
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
cephrgw "github.com/rook/rook/pkg/ceph/rgw"
@@ -97,8 +96,8 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, 6, len(cont.Env))
expectedCommand := fmt.Sprintf("/usr/local/bin/rook rgw --config-dir=/var/lib/rook --rgw-port=%d --rgw-host=%s",
cephrgw.RGWPort, cephrgw.DNSName)
assert.NotEqual(t, -1, strings.Index(cont.Command[2], expectedCommand), cont.Command[2])
assert.Equal(t, "rgw", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
assert.Equal(t, fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort), cont.Args[2])
assert.Equal(t, fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName), cont.Args[3])
}