From a8440dff448cc415d0988a4fafc84947129ee097 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Mon, 6 Feb 2017 15:17:48 -0800 Subject: [PATCH] store mon and rgw secrets --- pkg/operator/mon/mon.go | 71 +++++++++++++++++++++++++++++++++-------- pkg/operator/mon/pod.go | 16 ++++++++-- pkg/operator/osd/osd.go | 35 +++++++++++--------- pkg/operator/rgw/rgw.go | 67 ++++++++++++++++++++++++++------------ 4 files changed, 136 insertions(+), 53 deletions(-) diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index 771f87013..5412d4952 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -28,10 +28,15 @@ import ( ) const ( - monApp = "cephmon" - monNodeAttr = "mon_node" - monClusterAttr = "mon_cluster" - tprName = "mon.rook.io" + monApp = "cephmon" + monNodeAttr = "mon_node" + monClusterAttr = "mon_cluster" + tprName = "mon.rook.io" + monSecretsName = "mon-secrets" + fsidSecretName = "fsid" + monSecretName = "mon-secret" + adminSecretName = "admin-secret" + clusterSecretName = "cluster-name" ) type Cluster struct { @@ -50,7 +55,6 @@ type Cluster struct { type MonConfig struct { Name string Port int32 - Info *mon.ClusterInfo } func New(namespace string, factory client.ConnectionFactory, version string) *Cluster { @@ -66,13 +70,13 @@ func New(namespace string, factory client.ConnectionFactory, version string) *Cl func (c *Cluster) Start(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) { logger.Infof("start running mons") - clusterInfo, err := c.initClusterInfo() + clusterInfo, err := c.initClusterInfo(clientset) if err != nil { return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } mons := []*MonConfig{} for i := 0; i < c.Size; i++ { - mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Info: clusterInfo, Port: int32(mon.Port)}) + mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)}) } err = c.startPods(clientset, clusterInfo, mons) @@ -85,8 +89,47 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset) (*mon.ClusterInfo, erro // Retrieve the ceph cluster info if it already exists. // If a new cluster create new keys. -func (c *Cluster) initClusterInfo() (*mon.ClusterInfo, error) { - return mon.CreateClusterInfo(c.factory, "") +func (c *Cluster) initClusterInfo(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) { + secrets, err := clientset.Secrets(c.Namespace).Get(monSecretsName) + if err != nil { + if !k8sutil.IsKubernetesResourceNotFoundError(err) { + return nil, fmt.Errorf("failed to get mon secrets. %+v", err) + } + + return c.createMonSecretsAndSave(clientset) + } + + info := &mon.ClusterInfo{ + Name: string(secrets.Data[clusterSecretName]), + FSID: string(secrets.Data[fsidSecretName]), + MonitorSecret: string(secrets.Data[monSecretName]), + AdminSecret: string(secrets.Data[adminSecretName]), + Monitors: map[string]*mon.CephMonitorConfig{}, + } + logger.Infof("found existing monitor secrets for cluster %s with fsid %s", info.Name, info.FSID) + return info, nil +} + +func (c *Cluster) createMonSecretsAndSave(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) { + logger.Infof("creating mon secrets for a new cluster") + info, err := mon.CreateClusterInfo(c.factory, "") + if err != nil { + return nil, fmt.Errorf("failed to create mon secrets. %+v", err) + } + + // store the secrets + secrets := map[string]string{ + clusterSecretName: info.Name, + fsidSecretName: info.FSID, + monSecretName: info.MonitorSecret, + adminSecretName: info.AdminSecret, + } + _, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: monSecretsName}, StringData: secrets}) + if err != nil { + return nil, fmt.Errorf("failed to save mon secrets. %+v", err) + } + + return info, nil } func (c *Cluster) startPods(clientset *kubernetes.Clientset, clusterInfo *mon.ClusterInfo, mons []*MonConfig) error { @@ -102,16 +145,16 @@ func (c *Cluster) startPods(clientset *kubernetes.Clientset, clusterInfo *mon.Cl } logger.Infof("%d running, %d pending pods", len(running), len(pending)) - if len(running) == c.Size { - logger.Infof("pods are already running") - return nil - } - clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} for _, m := range running { clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP) } + if len(running) == c.Size { + logger.Infof("pods are already running") + return nil + } + started := 0 alreadyRunning := 0 for _, m := range mons { diff --git a/pkg/operator/mon/pod.go b/pkg/operator/mon/pod.go index 736b0a628..d03b14a77 100644 --- a/pkg/operator/mon/pod.go +++ b/pkg/operator/mon/pod.go @@ -26,6 +26,14 @@ import ( "k8s.io/client-go/1.5/pkg/labels" ) +func MonSecretEnvVar() v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_MON_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monSecretsName}, Key: monSecretName}}} +} + +func AdminSecretEnvVar() v1.EnvVar { + return v1.EnvVar{Name: "ROOKD_ADMIN_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monSecretsName}, Key: adminSecretName}}} +} + func getLabels(clusterName string) map[string]string { return map[string]string{ k8sutil.AppAttr: monApp, @@ -55,14 +63,14 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version) if antiAffinity { - k8sutil.PodWithAntiAffinity(pod, monClusterAttr, config.Info.Name) + k8sutil.PodWithAntiAffinity(pod, monClusterAttr, clusterInfo.Name) } return pod } func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo) v1.Container { - command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --mon-secret=%s --admin-secret=%s --cluster-name=%s", - k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, config.Info.FSID, config.Info.MonitorSecret, config.Info.AdminSecret, config.Info.Name) + command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --cluster-name=%s", + k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, clusterInfo.FSID, clusterInfo.Name) return v1.Container{ // TODO: fix "sleep 5". @@ -82,6 +90,8 @@ func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo) }, Env: []v1.EnvVar{ {Name: k8sutil.PodIPEnvVar, ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}}}, + MonSecretEnvVar(), + AdminSecretEnvVar(), }, } } diff --git a/pkg/operator/osd/osd.go b/pkg/operator/osd/osd.go index 71ba49010..1afc52503 100644 --- a/pkg/operator/osd/osd.go +++ b/pkg/operator/osd/osd.go @@ -20,8 +20,9 @@ import ( "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/operator/k8sutil" + k8smon "github.com/rook/rook/pkg/operator/mon" "k8s.io/client-go/1.5/kubernetes" - api "k8s.io/client-go/1.5/pkg/api/v1" + "k8s.io/client-go/1.5/pkg/api/v1" extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1" ) @@ -69,8 +70,8 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet ds.Name = daemonSetName ds.Namespace = c.Namespace - podSpec := api.PodTemplateSpec{ - ObjectMeta: api.ObjectMeta{ + podSpec := v1.PodTemplateSpec{ + ObjectMeta: v1.ObjectMeta{ Name: "rookosd", Labels: map[string]string{ k8sutil.AppAttr: osdApp, @@ -78,12 +79,12 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet }, Annotations: map[string]string{}, }, - Spec: api.PodSpec{ - Containers: []api.Container{c.osdContainer(cluster)}, - RestartPolicy: api.RestartPolicyAlways, - Volumes: []api.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}, - {Name: "devices", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev"}}}, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.osdContainer(cluster)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, + {Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}, }, }, } @@ -93,21 +94,25 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet return ds, nil } -func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) api.Container { +func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container { - command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mon-secret=%s --admin-secret=%s ", - k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret) + command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ", + k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name) privileged := true - return api.Container{ + return v1.Container{ // TODO: fix "sleep 5". // Without waiting some time, there is highly probable flakes in network setup. Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)}, Name: "cephosd", Image: k8sutil.MakeRookImage(c.Version), - VolumeMounts: []api.VolumeMount{ + VolumeMounts: []v1.VolumeMount{ {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, {Name: "devices", MountPath: "/dev"}, }, - SecurityContext: &api.SecurityContext{Privileged: &privileged}, + Env: []v1.EnvVar{ + k8smon.MonSecretEnvVar(), + k8smon.AdminSecretEnvVar(), + }, + SecurityContext: &v1.SecurityContext{Privileged: &privileged}, } } diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index c5c231fe5..eed778207 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -23,9 +23,9 @@ import ( cephrgw "github.com/rook/rook/pkg/cephmgr/rgw" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" + k8smon "github.com/rook/rook/pkg/operator/mon" "k8s.io/client-go/1.5/kubernetes" "k8s.io/client-go/1.5/pkg/api/v1" - api "k8s.io/client-go/1.5/pkg/api/v1" extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1" "k8s.io/client-go/1.5/pkg/util/intstr" ) @@ -33,6 +33,8 @@ import ( const ( rgwApp = "cephrgw" deploymentName = "rgw" + rgwSecretsName = "rgw-secrets" + rgwKeyringName = "keyring" ) type Cluster struct { @@ -58,7 +60,7 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf return fmt.Errorf("missing mons to start rgw") } - keyring, err := c.createKeyring(cluster) + err := c.createKeyring(clientset, cluster) if err != nil { return fmt.Errorf("failed to create rgw keyring. %+v", err) } @@ -70,7 +72,7 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf } // start the deployment - deployment, err := c.makeDeployment(cluster, keyring) + deployment, err := c.makeDeployment(cluster) _, err = clientset.Deployments(c.Namespace).Create(deployment) if err != nil { if !k8sutil.IsKubernetesResourceAlreadyExistError(err) { @@ -84,41 +86,59 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf return nil } -func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) (string, error) { - context := &clusterd.Context{ConfigDir: "/var/lib/rook"} +func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, cluster *mon.ClusterInfo) error { + _, err := clientset.Secrets(c.Namespace).Get(rgwSecretsName) + if err == nil { + logger.Infof("the rgw keyring was already generated") + return nil + } + if !k8sutil.IsKubernetesResourceNotFoundError(err) { + return fmt.Errorf("failed to get rgw secrets. %+v", err) + } // connect to the ceph cluster + logger.Infof("generating rgw keyring") + context := &clusterd.Context{ConfigDir: "/var/lib/rook"} conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, cluster) if err != nil { - return "", fmt.Errorf("failed to connect to cluster. %+v", err) + return fmt.Errorf("failed to connect to cluster. %+v", err) } defer conn.Shutdown() // create the keyring keyring, err := cephrgw.CreateKeyring(conn) if err != nil { - return "", fmt.Errorf("failed to create keyring. %+v", err) + return fmt.Errorf("failed to create keyring. %+v", err) } - return keyring, nil + // store the secrets + secrets := map[string]string{ + rgwKeyringName: keyring, + } + _, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: rgwSecretsName}, StringData: secrets}) + if err != nil { + return fmt.Errorf("failed to save rgw secrets. %+v", err) + } + + return nil } -func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*extensions.Deployment, error) { +func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) (*extensions.Deployment, error) { deployment := &extensions.Deployment{} deployment.Name = deploymentName deployment.Namespace = c.Namespace - podSpec := api.PodTemplateSpec{ - ObjectMeta: api.ObjectMeta{ + podSpec := v1.PodTemplateSpec{ + ObjectMeta: v1.ObjectMeta{ Name: "rook-rgw", Labels: getLabels(cluster.Name), Annotations: map[string]string{}, }, - Spec: api.PodSpec{ - Containers: []api.Container{c.rgwContainer(cluster, keyring)}, - RestartPolicy: api.RestartPolicyAlways, - Volumes: []api.Volume{ - {Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.rgwContainer(cluster)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + {Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, }, }, } @@ -128,19 +148,24 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*ext return deployment, nil } -func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo, keyring string) api.Container { +func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo) v1.Container { - command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mon-secret=%s --admin-secret=%s --rgw-port=%d --rgw-host=%s --rgw-keyring=%s", - k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret, cephrgw.RGWPort, cephrgw.DNSName, keyring) - return api.Container{ + command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --rgw-port=%d --rgw-host=%s", + k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cephrgw.RGWPort, cephrgw.DNSName) + return v1.Container{ // TODO: fix "sleep 5". // Without waiting some time, there is highly probable flakes in network setup. Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)}, Name: rgwApp, Image: k8sutil.MakeRookImage(c.Version), - VolumeMounts: []api.VolumeMount{ + VolumeMounts: []v1.VolumeMount{ {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, }, + Env: []v1.EnvVar{ + {Name: "ROOKD_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: rgwSecretsName}, Key: rgwKeyringName}}}, + k8smon.MonSecretEnvVar(), + k8smon.AdminSecretEnvVar(), + }, } }