save the mon endpoints in a configmap and consume from pods
This commit is contained in:
1 parent
02fe77d907
commit
fef56b31cc
8 files changed
+154
-75
No files matched your search
+22
-21
@@ -18,10 +18,9 @@ package api
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/model"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
k8smon "github.com/rook/rook/pkg/operator/mon"
|
||||
opmon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
@@ -49,21 +48,17 @@ func New(clientset kubernetes.Interface, namespace, version string) *Cluster {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) Start(cluster *mon.ClusterInfo) error {
|
||||
func (c *Cluster) Start() error {
|
||||
logger.Infof("starting the Rook api")
|
||||
|
||||
if cluster == nil || len(cluster.Monitors) == 0 {
|
||||
return fmt.Errorf("missing mons to start")
|
||||
}
|
||||
|
||||
// start the service
|
||||
err := c.startService(cluster)
|
||||
err := c.startService()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start api service. %+v", err)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(cluster)
|
||||
deployment := c.makeDeployment()
|
||||
_, err = c.clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
@@ -77,7 +72,7 @@ func (c *Cluster) Start(cluster *mon.ClusterInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) *extensions.Deployment {
|
||||
func (c *Cluster) makeDeployment() *extensions.Deployment {
|
||||
deployment := &extensions.Deployment{}
|
||||
deployment.Name = deploymentName
|
||||
deployment.Namespace = c.Namespace
|
||||
@@ -85,11 +80,11 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) *extensions.Deploymen
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: deploymentName,
|
||||
Labels: getLabels(cluster.Name),
|
||||
Labels: c.getLabels(),
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.apiContainer(cluster)},
|
||||
Containers: []v1.Container{c.apiContainer()},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
|
||||
@@ -102,15 +97,19 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) *extensions.Deploymen
|
||||
return deployment
|
||||
}
|
||||
|
||||
func (c *Cluster) apiContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
func (c *Cluster) apiContainer() v1.Container {
|
||||
// need a different prefix on the env vars
|
||||
monSecretVar := k8smon.MonSecretEnvVar()
|
||||
adminSecretVar := k8smon.AdminSecretEnvVar()
|
||||
monSecretVar := opmon.MonSecretEnvVar()
|
||||
adminSecretVar := opmon.AdminSecretEnvVar()
|
||||
endpointVar := opmon.MonEndpointEnvVar()
|
||||
clusterNameVar := opmon.ClusterNameEnvVar()
|
||||
monSecretVar.Name = "ROOK_OPERATOR_MON_SECRET"
|
||||
adminSecretVar.Name = "ROOK_OPERATOR_ADMIN_SECRET"
|
||||
endpointVar.Name = "ROOK_OPERATOR_MON_ENDPOINTS"
|
||||
clusterNameVar.Name = "ROOK_OPERATOR_CLUSTER_NAME"
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=%s --mon-endpoints=%s --cluster-name=%s --api-port=%d --container-version=%s",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, model.Port, c.Version)
|
||||
command := fmt.Sprintf("/usr/bin/rook-operator api --data-dir=%s --api-port=%d --container-version=%s",
|
||||
k8sutil.DataDir, model.Port, c.Version)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
@@ -122,14 +121,16 @@ func (c *Cluster) apiContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
k8sutil.NamespaceEnvVar(),
|
||||
clusterNameVar,
|
||||
endpointVar,
|
||||
monSecretVar,
|
||||
adminSecretVar,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) startService(clusterInfo *mon.ClusterInfo) error {
|
||||
labels := getLabels(clusterInfo.Name)
|
||||
func (c *Cluster) startService() error {
|
||||
labels := c.getLabels()
|
||||
s := &v1.Service{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: deploymentName,
|
||||
@@ -162,9 +163,9 @@ func (c *Cluster) startService(clusterInfo *mon.ClusterInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLabels(clusterName string) map[string]string {
|
||||
func (c *Cluster) getLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: deploymentName,
|
||||
k8sutil.ClusterAttr: clusterName,
|
||||
k8sutil.ClusterAttr: c.Namespace,
|
||||
}
|
||||
}
|
||||
@@ -92,14 +92,14 @@ func (c *Cluster) CreateInstance() error {
|
||||
}
|
||||
|
||||
a := api.New(c.clientset, c.Spec.Namespace, c.Spec.Version)
|
||||
err = a.Start(cluster)
|
||||
err = a.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the REST api. %+v", err)
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
osds := osd.New(c.clientset, c.Spec.Namespace, c.Spec.Version, c.Spec.DeviceFilter, c.Spec.DataDirHostPath, c.Spec.UseAllDevices)
|
||||
err = osds.Start(cluster)
|
||||
err = osds.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the osds. %+v", err)
|
||||
}
|
||||
|
||||
+14
-12
@@ -23,7 +23,7 @@ import (
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
k8smon "github.com/rook/rook/pkg/operator/mon"
|
||||
opmon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
@@ -76,7 +76,7 @@ func (c *Cluster) Start(clientset kubernetes.Interface, cluster *mon.ClusterInfo
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(cluster, id)
|
||||
deployment := c.makeDeployment(id)
|
||||
_, err = clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
@@ -123,7 +123,7 @@ func (c *Cluster) createKeyring(clientset kubernetes.Interface, context *cluster
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, id string) *extensions.Deployment {
|
||||
func (c *Cluster) makeDeployment(id string) *extensions.Deployment {
|
||||
deployment := &extensions.Deployment{}
|
||||
deployment.Name = appName
|
||||
deployment.Namespace = c.Namespace
|
||||
@@ -131,11 +131,11 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, id string) *extension
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: appName,
|
||||
Labels: getLabels(cluster.Name),
|
||||
Labels: c.getLabels(),
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.mdsContainer(cluster, id)},
|
||||
Containers: []v1.Container{c.mdsContainer(id)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
|
||||
@@ -148,10 +148,10 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, id string) *extension
|
||||
return deployment
|
||||
}
|
||||
|
||||
func (c *Cluster) mdsContainer(cluster *mon.ClusterInfo, id string) v1.Container {
|
||||
func (c *Cluster) mdsContainer(id string) v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mds-id=%s ",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, id)
|
||||
command := fmt.Sprintf("/usr/bin/rookd mds --data-dir=%s --mds-id=%s ",
|
||||
k8sutil.DataDir, id)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
@@ -163,15 +163,17 @@ func (c *Cluster) mdsContainer(cluster *mon.ClusterInfo, id string) v1.Container
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: "ROOKD_MDS_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: keyringName}}},
|
||||
k8smon.MonSecretEnvVar(),
|
||||
k8smon.AdminSecretEnvVar(),
|
||||
opmon.ClusterNameEnvVar(),
|
||||
opmon.MonEndpointEnvVar(),
|
||||
opmon.MonSecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getLabels(clusterName string) map[string]string {
|
||||
func (c *Cluster) getLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: clusterName,
|
||||
k8sutil.ClusterAttr: c.Namespace,
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,8 @@ const (
|
||||
monSecretName = "mon-secret"
|
||||
adminSecretName = "admin-secret"
|
||||
clusterSecretName = "cluster-name"
|
||||
monConfigMapName = "mon-config"
|
||||
monEndpointKey = "endpoints"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
@@ -179,6 +181,11 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
|
||||
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
|
||||
}
|
||||
|
||||
err = c.saveMonEndpoints(clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save endpoints for %d running mons. %+v", len(clusterInfo.Monitors), err)
|
||||
}
|
||||
|
||||
if len(running) == c.Size {
|
||||
logger.Infof("pods are already running")
|
||||
return nil
|
||||
@@ -205,6 +212,11 @@ func (c *Cluster) startPods(clusterInfo *mon.ClusterInfo, mons []*MonConfig) err
|
||||
return fmt.Errorf("failed to start pod %s. %+v", name, err)
|
||||
}
|
||||
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, podIP)
|
||||
|
||||
err = c.saveMonEndpoints(clusterInfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save endpoints after starting mon %s. %+v", m.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("started %d/%d mons (%d already running)", (started + alreadyRunning), c.Size, alreadyRunning)
|
||||
@@ -235,6 +247,34 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) {
|
||||
return "", fmt.Errorf("timed out waiting for pod %s to start", name)
|
||||
}
|
||||
|
||||
func (c *Cluster) saveMonEndpoints(clusterInfo *mon.ClusterInfo) error {
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: monConfigMapName,
|
||||
Namespace: c.Namespace,
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
}
|
||||
configMap.Data = map[string]string{
|
||||
monEndpointKey: mon.FlattenMonEndpoints(clusterInfo.Monitors),
|
||||
}
|
||||
|
||||
_, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create mon endpoint config map. %+v", err)
|
||||
}
|
||||
|
||||
logger.Debugf("updating config map %s that already exists", configMap.Name)
|
||||
if _, err = c.clientset.CoreV1().ConfigMaps(c.Namespace).Update(configMap); err != nil {
|
||||
return fmt.Errorf("failed to update mon endpoint config map. %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("saved mon endpoints to config map %s", configMap.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// detect whether we have a big enough cluster to run services on different nodes.
|
||||
// the anti-affinity will prevent pods of the same type of running on the same node.
|
||||
func (c *Cluster) getAntiAffinity() (bool, error) {
|
||||
|
||||
@@ -72,3 +72,27 @@ func validateStart(t *testing.T, c *Cluster) {
|
||||
assert.Equal(t, 0, len(running))
|
||||
assert.Equal(t, 0, len(pending))
|
||||
}
|
||||
|
||||
func TestSaveMonEndpoints(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
c := New(clientset, nil, "ns", "myversion")
|
||||
info := test.CreateClusterInfo(1)
|
||||
|
||||
// create the initial config map
|
||||
err := c.saveMonEndpoints(info)
|
||||
assert.Nil(t, err)
|
||||
|
||||
cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "mon1=1.2.3.1:6790", cm.Data["endpoints"])
|
||||
|
||||
// update the config map
|
||||
info.Monitors["mon1"].Endpoint = "2.3.4.5:6790"
|
||||
err = c.saveMonEndpoints(info)
|
||||
assert.Nil(t, err)
|
||||
|
||||
cm, err = c.clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config")
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "mon1=2.3.4.5:6790", cm.Data["endpoints"])
|
||||
|
||||
}
|
||||
+23
-9
@@ -24,24 +24,36 @@ import (
|
||||
"k8s.io/client-go/pkg/labels"
|
||||
)
|
||||
|
||||
func ClusterNameEnvVar() v1.EnvVar {
|
||||
ref := &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}
|
||||
return v1.EnvVar{Name: "ROOKD_CLUSTER_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: ref}}
|
||||
}
|
||||
|
||||
func MonEndpointEnvVar() v1.EnvVar {
|
||||
ref := &v1.ConfigMapKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monConfigMapName}, Key: monEndpointKey}
|
||||
return v1.EnvVar{Name: "ROOKD_MON_ENDPOINTS", ValueFrom: &v1.EnvVarSource{ConfigMapKeyRef: ref}}
|
||||
}
|
||||
|
||||
func MonSecretEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_MON_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: monSecretName}}}
|
||||
ref := &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: monSecretName}
|
||||
return v1.EnvVar{Name: "ROOKD_MON_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: ref}}
|
||||
}
|
||||
|
||||
func AdminSecretEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_ADMIN_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: adminSecretName}}}
|
||||
ref := &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: adminSecretName}
|
||||
return v1.EnvVar{Name: "ROOKD_ADMIN_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: ref}}
|
||||
}
|
||||
|
||||
func getLabels(clusterName string) map[string]string {
|
||||
func (c *Cluster) getLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
monClusterAttr: clusterName,
|
||||
monClusterAttr: c.Namespace,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, antiAffinity bool) *v1.Pod {
|
||||
|
||||
container := c.monContainer(config, clusterInfo)
|
||||
container := c.monContainer(config, clusterInfo.FSID)
|
||||
dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}
|
||||
if c.dataDirHostPath != "" {
|
||||
dataDirSource = v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: c.dataDirHostPath}}
|
||||
@@ -51,7 +63,7 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: config.Name,
|
||||
Namespace: c.Namespace,
|
||||
Labels: getLabels(clusterInfo.Name),
|
||||
Labels: c.getLabels(),
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
@@ -71,9 +83,9 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an
|
||||
return pod
|
||||
}
|
||||
|
||||
func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo) v1.Container {
|
||||
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --cluster-name=%s",
|
||||
k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, clusterInfo.FSID, clusterInfo.Name)
|
||||
func (c *Cluster) monContainer(config *MonConfig, fsid string) v1.Container {
|
||||
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --port=%d --fsid=%s",
|
||||
k8sutil.DataDir, config.Name, config.Port, fsid)
|
||||
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
@@ -93,6 +105,8 @@ func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo)
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: k8sutil.PodIPEnvVar, ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}}},
|
||||
ClusterNameEnvVar(),
|
||||
MonEndpointEnvVar(),
|
||||
MonSecretEnvVar(),
|
||||
AdminSecretEnvVar(),
|
||||
},
|
||||
|
||||
+12
-16
@@ -18,9 +18,8 @@ package osd
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
k8smon "github.com/rook/rook/pkg/operator/mon"
|
||||
opmon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
@@ -52,14 +51,10 @@ func New(clientset kubernetes.Interface, namespace, version, deviceFilter, dataD
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) Start(cluster *mon.ClusterInfo) error {
|
||||
func (c *Cluster) Start() error {
|
||||
logger.Infof("start running osds")
|
||||
|
||||
if cluster == nil || len(cluster.Monitors) == 0 {
|
||||
return fmt.Errorf("missing mons to start osds")
|
||||
}
|
||||
|
||||
ds, err := c.makeDaemonSet(cluster)
|
||||
ds, err := c.makeDaemonSet()
|
||||
_, err = c.clientset.Extensions().DaemonSets(c.Namespace).Create(ds)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
@@ -73,7 +68,7 @@ func (c *Cluster) Start(cluster *mon.ClusterInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet, error) {
|
||||
func (c *Cluster) makeDaemonSet() (*extensions.DaemonSet, error) {
|
||||
ds := &extensions.DaemonSet{}
|
||||
ds.Name = appName
|
||||
ds.Namespace = c.Namespace
|
||||
@@ -88,12 +83,12 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
|
||||
Name: appName,
|
||||
Labels: map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: cluster.Name,
|
||||
k8sutil.ClusterAttr: c.Namespace,
|
||||
},
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.osdContainer(cluster)},
|
||||
Containers: []v1.Container{c.osdContainer()},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: dataDirSource},
|
||||
@@ -107,10 +102,9 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
func (c *Cluster) osdContainer() v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name)
|
||||
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s ", k8sutil.DataDir)
|
||||
if c.deviceFilter != "" {
|
||||
command += fmt.Sprintf("--data-devices=%s ", c.deviceFilter)
|
||||
} else if c.useAllDevices {
|
||||
@@ -129,8 +123,10 @@ func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
{Name: "devices", MountPath: "/dev"},
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
k8smon.MonSecretEnvVar(),
|
||||
k8smon.AdminSecretEnvVar(),
|
||||
opmon.ClusterNameEnvVar(),
|
||||
opmon.MonEndpointEnvVar(),
|
||||
opmon.MonSecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
},
|
||||
SecurityContext: &v1.SecurityContext{Privileged: &privileged},
|
||||
}
|
||||
|
||||
+17
-15
@@ -23,7 +23,7 @@ import (
|
||||
cephrgw "github.com/rook/rook/pkg/cephmgr/rgw"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
k8smon "github.com/rook/rook/pkg/operator/mon"
|
||||
opmon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/pkg/api/errors"
|
||||
"k8s.io/client-go/pkg/api/v1"
|
||||
@@ -67,13 +67,13 @@ func (c *Cluster) Start(clientset kubernetes.Interface, cluster *mon.ClusterInfo
|
||||
}
|
||||
|
||||
// start the service
|
||||
err = c.startService(clientset, cluster)
|
||||
err = c.startService(clientset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rgw service. %+v", err)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(cluster)
|
||||
deployment := c.makeDeployment()
|
||||
_, err = clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
@@ -129,7 +129,7 @@ func (c *Cluster) createKeyring(clientset kubernetes.Interface, cluster *mon.Clu
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) *extensions.Deployment {
|
||||
func (c *Cluster) makeDeployment() *extensions.Deployment {
|
||||
deployment := &extensions.Deployment{}
|
||||
deployment.Name = appName
|
||||
deployment.Namespace = c.Namespace
|
||||
@@ -137,11 +137,11 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) *extensions.Deploymen
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: "rook-rgw",
|
||||
Labels: getLabels(cluster.Name),
|
||||
Labels: c.getLabels(),
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.rgwContainer(cluster)},
|
||||
Containers: []v1.Container{c.rgwContainer()},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
|
||||
@@ -154,10 +154,10 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) *extensions.Deploymen
|
||||
return deployment
|
||||
}
|
||||
|
||||
func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
func (c *Cluster) rgwContainer() v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --rgw-port=%d --rgw-host=%s",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cephrgw.RGWPort, cephrgw.DNSName)
|
||||
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --rgw-port=%d --rgw-host=%s",
|
||||
k8sutil.DataDir, cephrgw.RGWPort, cephrgw.DNSName)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
@@ -169,14 +169,16 @@ func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: "ROOKD_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: keyringName}}},
|
||||
k8smon.MonSecretEnvVar(),
|
||||
k8smon.AdminSecretEnvVar(),
|
||||
opmon.ClusterNameEnvVar(),
|
||||
opmon.MonEndpointEnvVar(),
|
||||
opmon.MonSecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) startService(clientset kubernetes.Interface, clusterInfo *mon.ClusterInfo) error {
|
||||
labels := getLabels(clusterInfo.Name)
|
||||
func (c *Cluster) startService(clientset kubernetes.Interface) error {
|
||||
labels := c.getLabels()
|
||||
s := &v1.Service{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: appName,
|
||||
@@ -209,9 +211,9 @@ func (c *Cluster) startService(clientset kubernetes.Interface, clusterInfo *mon.
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLabels(clusterName string) map[string]string {
|
||||
func (c *Cluster) getLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: clusterName,
|
||||
k8sutil.ClusterAttr: c.Namespace,
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user