store mon and rgw secrets

This commit is contained in:
Travis Nielsen committed 2017-02-06 17:47:57 -08:00
1 parent 252db979bc
commit a8440dff44
4 files changed
+136 -53

No files matched your search

+57 -14
View File
@@ -28,10 +28,15 @@ import (
)
const (
monApp = "cephmon"
monNodeAttr = "mon_node"
monClusterAttr = "mon_cluster"
tprName = "mon.rook.io"
monApp = "cephmon"
monNodeAttr = "mon_node"
monClusterAttr = "mon_cluster"
tprName = "mon.rook.io"
monSecretsName = "mon-secrets"
fsidSecretName = "fsid"
monSecretName = "mon-secret"
adminSecretName = "admin-secret"
clusterSecretName = "cluster-name"
)
type Cluster struct {
@@ -50,7 +55,6 @@ type Cluster struct {
type MonConfig struct {
Name string
Port int32
Info *mon.ClusterInfo
}
func New(namespace string, factory client.ConnectionFactory, version string) *Cluster {
@@ -66,13 +70,13 @@ func New(namespace string, factory client.ConnectionFactory, version string) *Cl
func (c *Cluster) Start(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) {
logger.Infof("start running mons")
clusterInfo, err := c.initClusterInfo()
clusterInfo, err := c.initClusterInfo(clientset)
if err != nil {
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
}
mons := []*MonConfig{}
for i := 0; i < c.Size; i++ {
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Info: clusterInfo, Port: int32(mon.Port)})
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)})
}
err = c.startPods(clientset, clusterInfo, mons)
@@ -85,8 +89,47 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset) (*mon.ClusterInfo, erro
// Retrieve the ceph cluster info if it already exists.
// If a new cluster create new keys.
func (c *Cluster) initClusterInfo() (*mon.ClusterInfo, error) {
return mon.CreateClusterInfo(c.factory, "")
func (c *Cluster) initClusterInfo(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) {
secrets, err := clientset.Secrets(c.Namespace).Get(monSecretsName)
if err != nil {
if !k8sutil.IsKubernetesResourceNotFoundError(err) {
return nil, fmt.Errorf("failed to get mon secrets. %+v", err)
}
return c.createMonSecretsAndSave(clientset)
}
info := &mon.ClusterInfo{
Name: string(secrets.Data[clusterSecretName]),
FSID: string(secrets.Data[fsidSecretName]),
MonitorSecret: string(secrets.Data[monSecretName]),
AdminSecret: string(secrets.Data[adminSecretName]),
Monitors: map[string]*mon.CephMonitorConfig{},
}
logger.Infof("found existing monitor secrets for cluster %s with fsid %s", info.Name, info.FSID)
return info, nil
}
func (c *Cluster) createMonSecretsAndSave(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) {
logger.Infof("creating mon secrets for a new cluster")
info, err := mon.CreateClusterInfo(c.factory, "")
if err != nil {
return nil, fmt.Errorf("failed to create mon secrets. %+v", err)
}
// store the secrets
secrets := map[string]string{
clusterSecretName: info.Name,
fsidSecretName: info.FSID,
monSecretName: info.MonitorSecret,
adminSecretName: info.AdminSecret,
}
_, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: monSecretsName}, StringData: secrets})
if err != nil {
return nil, fmt.Errorf("failed to save mon secrets. %+v", err)
}
return info, nil
}
func (c *Cluster) startPods(clientset *kubernetes.Clientset, clusterInfo *mon.ClusterInfo, mons []*MonConfig) error {
@@ -102,16 +145,16 @@ func (c *Cluster) startPods(clientset *kubernetes.Clientset, clusterInfo *mon.Cl
}
logger.Infof("%d running, %d pending pods", len(running), len(pending))
if len(running) == c.Size {
logger.Infof("pods are already running")
return nil
}
clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
for _, m := range running {
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
}
if len(running) == c.Size {
logger.Infof("pods are already running")
return nil
}
started := 0
alreadyRunning := 0
for _, m := range mons {
+13 -3
View File
@@ -26,6 +26,14 @@ import (
"k8s.io/client-go/1.5/pkg/labels"
)
func MonSecretEnvVar() v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_MON_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monSecretsName}, Key: monSecretName}}}
}
func AdminSecretEnvVar() v1.EnvVar {
return v1.EnvVar{Name: "ROOKD_ADMIN_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monSecretsName}, Key: adminSecretName}}}
}
func getLabels(clusterName string) map[string]string {
return map[string]string{
k8sutil.AppAttr: monApp,
@@ -55,14 +63,14 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an
k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version)
if antiAffinity {
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, config.Info.Name)
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, clusterInfo.Name)
}
return pod
}
func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --mon-secret=%s --admin-secret=%s --cluster-name=%s",
k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, config.Info.FSID, config.Info.MonitorSecret, config.Info.AdminSecret, config.Info.Name)
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --cluster-name=%s",
k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, clusterInfo.FSID, clusterInfo.Name)
return v1.Container{
// TODO: fix "sleep 5".
@@ -82,6 +90,8 @@ func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo)
},
Env: []v1.EnvVar{
{Name: k8sutil.PodIPEnvVar, ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}}},
MonSecretEnvVar(),
AdminSecretEnvVar(),
},
}
}
+20 -15
View File
@@ -20,8 +20,9 @@ import (
"github.com/rook/rook/pkg/cephmgr/mon"
"github.com/rook/rook/pkg/operator/k8sutil"
k8smon "github.com/rook/rook/pkg/operator/mon"
"k8s.io/client-go/1.5/kubernetes"
api "k8s.io/client-go/1.5/pkg/api/v1"
"k8s.io/client-go/1.5/pkg/api/v1"
extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
)
@@ -69,8 +70,8 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
ds.Name = daemonSetName
ds.Namespace = c.Namespace
podSpec := api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
podSpec := v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Name: "rookosd",
Labels: map[string]string{
k8sutil.AppAttr: osdApp,
@@ -78,12 +79,12 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
},
Annotations: map[string]string{},
},
Spec: api.PodSpec{
Containers: []api.Container{c.osdContainer(cluster)},
RestartPolicy: api.RestartPolicyAlways,
Volumes: []api.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
{Name: "devices", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev"}}},
Spec: v1.PodSpec{
Containers: []v1.Container{c.osdContainer(cluster)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}},
},
},
}
@@ -93,21 +94,25 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
return ds, nil
}
func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) api.Container {
func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mon-secret=%s --admin-secret=%s ",
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret)
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ",
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name)
privileged := true
return api.Container{
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: "cephosd",
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []api.VolumeMount{
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
{Name: "devices", MountPath: "/dev"},
},
SecurityContext: &api.SecurityContext{Privileged: &privileged},
Env: []v1.EnvVar{
k8smon.MonSecretEnvVar(),
k8smon.AdminSecretEnvVar(),
},
SecurityContext: &v1.SecurityContext{Privileged: &privileged},
}
}
+46 -21
View File
@@ -23,9 +23,9 @@ import (
cephrgw "github.com/rook/rook/pkg/cephmgr/rgw"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
k8smon "github.com/rook/rook/pkg/operator/mon"
"k8s.io/client-go/1.5/kubernetes"
"k8s.io/client-go/1.5/pkg/api/v1"
api "k8s.io/client-go/1.5/pkg/api/v1"
extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
"k8s.io/client-go/1.5/pkg/util/intstr"
)
@@ -33,6 +33,8 @@ import (
const (
rgwApp = "cephrgw"
deploymentName = "rgw"
rgwSecretsName = "rgw-secrets"
rgwKeyringName = "keyring"
)
type Cluster struct {
@@ -58,7 +60,7 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf
return fmt.Errorf("missing mons to start rgw")
}
keyring, err := c.createKeyring(cluster)
err := c.createKeyring(clientset, cluster)
if err != nil {
return fmt.Errorf("failed to create rgw keyring. %+v", err)
}
@@ -70,7 +72,7 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf
}
// start the deployment
deployment, err := c.makeDeployment(cluster, keyring)
deployment, err := c.makeDeployment(cluster)
_, err = clientset.Deployments(c.Namespace).Create(deployment)
if err != nil {
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
@@ -84,41 +86,59 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf
return nil
}
func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) (string, error) {
context := &clusterd.Context{ConfigDir: "/var/lib/rook"}
func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, cluster *mon.ClusterInfo) error {
_, err := clientset.Secrets(c.Namespace).Get(rgwSecretsName)
if err == nil {
logger.Infof("the rgw keyring was already generated")
return nil
}
if !k8sutil.IsKubernetesResourceNotFoundError(err) {
return fmt.Errorf("failed to get rgw secrets. %+v", err)
}
// connect to the ceph cluster
logger.Infof("generating rgw keyring")
context := &clusterd.Context{ConfigDir: "/var/lib/rook"}
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, cluster)
if err != nil {
return "", fmt.Errorf("failed to connect to cluster. %+v", err)
return fmt.Errorf("failed to connect to cluster. %+v", err)
}
defer conn.Shutdown()
// create the keyring
keyring, err := cephrgw.CreateKeyring(conn)
if err != nil {
return "", fmt.Errorf("failed to create keyring. %+v", err)
return fmt.Errorf("failed to create keyring. %+v", err)
}
return keyring, nil
// store the secrets
secrets := map[string]string{
rgwKeyringName: keyring,
}
_, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: rgwSecretsName}, StringData: secrets})
if err != nil {
return fmt.Errorf("failed to save rgw secrets. %+v", err)
}
return nil
}
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*extensions.Deployment, error) {
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) (*extensions.Deployment, error) {
deployment := &extensions.Deployment{}
deployment.Name = deploymentName
deployment.Namespace = c.Namespace
podSpec := api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
podSpec := v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Name: "rook-rgw",
Labels: getLabels(cluster.Name),
Annotations: map[string]string{},
},
Spec: api.PodSpec{
Containers: []api.Container{c.rgwContainer(cluster, keyring)},
RestartPolicy: api.RestartPolicyAlways,
Volumes: []api.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
Spec: v1.PodSpec{
Containers: []v1.Container{c.rgwContainer(cluster)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: []v1.Volume{
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
},
},
}
@@ -128,19 +148,24 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*ext
return deployment, nil
}
func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo, keyring string) api.Container {
func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo) v1.Container {
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mon-secret=%s --admin-secret=%s --rgw-port=%d --rgw-host=%s --rgw-keyring=%s",
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret, cephrgw.RGWPort, cephrgw.DNSName, keyring)
return api.Container{
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --rgw-port=%d --rgw-host=%s",
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cephrgw.RGWPort, cephrgw.DNSName)
return v1.Container{
// TODO: fix "sleep 5".
// Without waiting some time, there is highly probable flakes in network setup.
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
Name: rgwApp,
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []api.VolumeMount{
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
},
Env: []v1.EnvVar{
{Name: "ROOKD_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: rgwSecretsName}, Key: rgwKeyringName}}},
k8smon.MonSecretEnvVar(),
k8smon.AdminSecretEnvVar(),
},
}
}