store mon and rgw secrets
This commit is contained in:
1 parent
252db979bc
commit
a8440dff44
4 files changed
+136
-53
No files matched your search
+57
-14
@@ -28,10 +28,15 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
monApp = "cephmon"
|
||||
monNodeAttr = "mon_node"
|
||||
monClusterAttr = "mon_cluster"
|
||||
tprName = "mon.rook.io"
|
||||
monApp = "cephmon"
|
||||
monNodeAttr = "mon_node"
|
||||
monClusterAttr = "mon_cluster"
|
||||
tprName = "mon.rook.io"
|
||||
monSecretsName = "mon-secrets"
|
||||
fsidSecretName = "fsid"
|
||||
monSecretName = "mon-secret"
|
||||
adminSecretName = "admin-secret"
|
||||
clusterSecretName = "cluster-name"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
@@ -50,7 +55,6 @@ type Cluster struct {
|
||||
type MonConfig struct {
|
||||
Name string
|
||||
Port int32
|
||||
Info *mon.ClusterInfo
|
||||
}
|
||||
|
||||
func New(namespace string, factory client.ConnectionFactory, version string) *Cluster {
|
||||
@@ -66,13 +70,13 @@ func New(namespace string, factory client.ConnectionFactory, version string) *Cl
|
||||
func (c *Cluster) Start(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) {
|
||||
logger.Infof("start running mons")
|
||||
|
||||
clusterInfo, err := c.initClusterInfo()
|
||||
clusterInfo, err := c.initClusterInfo(clientset)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
|
||||
}
|
||||
mons := []*MonConfig{}
|
||||
for i := 0; i < c.Size; i++ {
|
||||
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Info: clusterInfo, Port: int32(mon.Port)})
|
||||
mons = append(mons, &MonConfig{Name: fmt.Sprintf("mon%d", i), Port: int32(mon.Port)})
|
||||
}
|
||||
|
||||
err = c.startPods(clientset, clusterInfo, mons)
|
||||
@@ -85,8 +89,47 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset) (*mon.ClusterInfo, erro
|
||||
|
||||
// Retrieve the ceph cluster info if it already exists.
|
||||
// If a new cluster create new keys.
|
||||
func (c *Cluster) initClusterInfo() (*mon.ClusterInfo, error) {
|
||||
return mon.CreateClusterInfo(c.factory, "")
|
||||
func (c *Cluster) initClusterInfo(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) {
|
||||
secrets, err := clientset.Secrets(c.Namespace).Get(monSecretsName)
|
||||
if err != nil {
|
||||
if !k8sutil.IsKubernetesResourceNotFoundError(err) {
|
||||
return nil, fmt.Errorf("failed to get mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
return c.createMonSecretsAndSave(clientset)
|
||||
}
|
||||
|
||||
info := &mon.ClusterInfo{
|
||||
Name: string(secrets.Data[clusterSecretName]),
|
||||
FSID: string(secrets.Data[fsidSecretName]),
|
||||
MonitorSecret: string(secrets.Data[monSecretName]),
|
||||
AdminSecret: string(secrets.Data[adminSecretName]),
|
||||
Monitors: map[string]*mon.CephMonitorConfig{},
|
||||
}
|
||||
logger.Infof("found existing monitor secrets for cluster %s with fsid %s", info.Name, info.FSID)
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createMonSecretsAndSave(clientset *kubernetes.Clientset) (*mon.ClusterInfo, error) {
|
||||
logger.Infof("creating mon secrets for a new cluster")
|
||||
info, err := mon.CreateClusterInfo(c.factory, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
// store the secrets
|
||||
secrets := map[string]string{
|
||||
clusterSecretName: info.Name,
|
||||
fsidSecretName: info.FSID,
|
||||
monSecretName: info.MonitorSecret,
|
||||
adminSecretName: info.AdminSecret,
|
||||
}
|
||||
_, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: monSecretsName}, StringData: secrets})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to save mon secrets. %+v", err)
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startPods(clientset *kubernetes.Clientset, clusterInfo *mon.ClusterInfo, mons []*MonConfig) error {
|
||||
@@ -102,16 +145,16 @@ func (c *Cluster) startPods(clientset *kubernetes.Clientset, clusterInfo *mon.Cl
|
||||
}
|
||||
logger.Infof("%d running, %d pending pods", len(running), len(pending))
|
||||
|
||||
if len(running) == c.Size {
|
||||
logger.Infof("pods are already running")
|
||||
return nil
|
||||
}
|
||||
|
||||
clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{}
|
||||
for _, m := range running {
|
||||
clusterInfo.Monitors[m.Name] = mon.ToCephMon(m.Name, m.Status.PodIP)
|
||||
}
|
||||
|
||||
if len(running) == c.Size {
|
||||
logger.Infof("pods are already running")
|
||||
return nil
|
||||
}
|
||||
|
||||
started := 0
|
||||
alreadyRunning := 0
|
||||
for _, m := range mons {
|
||||
|
||||
+13
-3
@@ -26,6 +26,14 @@ import (
|
||||
"k8s.io/client-go/1.5/pkg/labels"
|
||||
)
|
||||
|
||||
func MonSecretEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_MON_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monSecretsName}, Key: monSecretName}}}
|
||||
}
|
||||
|
||||
func AdminSecretEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOKD_ADMIN_SECRET", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monSecretsName}, Key: adminSecretName}}}
|
||||
}
|
||||
|
||||
func getLabels(clusterName string) map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: monApp,
|
||||
@@ -55,14 +63,14 @@ func (c *Cluster) makeMonPod(config *MonConfig, clusterInfo *mon.ClusterInfo, an
|
||||
k8sutil.SetPodVersion(pod, k8sutil.VersionAttr, c.Version)
|
||||
|
||||
if antiAffinity {
|
||||
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, config.Info.Name)
|
||||
k8sutil.PodWithAntiAffinity(pod, monClusterAttr, clusterInfo.Name)
|
||||
}
|
||||
return pod
|
||||
}
|
||||
|
||||
func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo) v1.Container {
|
||||
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --mon-secret=%s --admin-secret=%s --cluster-name=%s",
|
||||
k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, config.Info.FSID, config.Info.MonitorSecret, config.Info.AdminSecret, config.Info.Name)
|
||||
command := fmt.Sprintf("/usr/bin/rookd mon --data-dir=%s --name=%s --mon-endpoints=%s --port=%d --fsid=%s --cluster-name=%s",
|
||||
k8sutil.DataDir, config.Name, mon.FlattenMonEndpoints(clusterInfo.Monitors), config.Port, clusterInfo.FSID, clusterInfo.Name)
|
||||
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
@@ -82,6 +90,8 @@ func (c *Cluster) monContainer(config *MonConfig, clusterInfo *mon.ClusterInfo)
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: k8sutil.PodIPEnvVar, ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}}},
|
||||
MonSecretEnvVar(),
|
||||
AdminSecretEnvVar(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
+20
-15
@@ -20,8 +20,9 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/cephmgr/mon"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
k8smon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/1.5/kubernetes"
|
||||
api "k8s.io/client-go/1.5/pkg/api/v1"
|
||||
"k8s.io/client-go/1.5/pkg/api/v1"
|
||||
extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
|
||||
)
|
||||
|
||||
@@ -69,8 +70,8 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
|
||||
ds.Name = daemonSetName
|
||||
ds.Namespace = c.Namespace
|
||||
|
||||
podSpec := api.PodTemplateSpec{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: "rookosd",
|
||||
Labels: map[string]string{
|
||||
k8sutil.AppAttr: osdApp,
|
||||
@@ -78,12 +79,12 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
|
||||
},
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Containers: []api.Container{c.osdContainer(cluster)},
|
||||
RestartPolicy: api.RestartPolicyAlways,
|
||||
Volumes: []api.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
|
||||
{Name: "devices", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev"}}},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.osdContainer(cluster)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
|
||||
{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -93,21 +94,25 @@ func (c *Cluster) makeDaemonSet(cluster *mon.ClusterInfo) (*extensions.DaemonSet
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) api.Container {
|
||||
func (c *Cluster) osdContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mon-secret=%s --admin-secret=%s ",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret)
|
||||
command := fmt.Sprintf("/usr/bin/rookd osd --data-dir=%s --mon-endpoints=%s --cluster-name=%s ",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name)
|
||||
privileged := true
|
||||
return api.Container{
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
|
||||
Name: "cephosd",
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
VolumeMounts: []api.VolumeMount{
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
{Name: "devices", MountPath: "/dev"},
|
||||
},
|
||||
SecurityContext: &api.SecurityContext{Privileged: &privileged},
|
||||
Env: []v1.EnvVar{
|
||||
k8smon.MonSecretEnvVar(),
|
||||
k8smon.AdminSecretEnvVar(),
|
||||
},
|
||||
SecurityContext: &v1.SecurityContext{Privileged: &privileged},
|
||||
}
|
||||
}
|
||||
+46
-21
@@ -23,9 +23,9 @@ import (
|
||||
cephrgw "github.com/rook/rook/pkg/cephmgr/rgw"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
k8smon "github.com/rook/rook/pkg/operator/mon"
|
||||
"k8s.io/client-go/1.5/kubernetes"
|
||||
"k8s.io/client-go/1.5/pkg/api/v1"
|
||||
api "k8s.io/client-go/1.5/pkg/api/v1"
|
||||
extensions "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
|
||||
"k8s.io/client-go/1.5/pkg/util/intstr"
|
||||
)
|
||||
@@ -33,6 +33,8 @@ import (
|
||||
const (
|
||||
rgwApp = "cephrgw"
|
||||
deploymentName = "rgw"
|
||||
rgwSecretsName = "rgw-secrets"
|
||||
rgwKeyringName = "keyring"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
@@ -58,7 +60,7 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf
|
||||
return fmt.Errorf("missing mons to start rgw")
|
||||
}
|
||||
|
||||
keyring, err := c.createKeyring(cluster)
|
||||
err := c.createKeyring(clientset, cluster)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rgw keyring. %+v", err)
|
||||
}
|
||||
@@ -70,7 +72,7 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment, err := c.makeDeployment(cluster, keyring)
|
||||
deployment, err := c.makeDeployment(cluster)
|
||||
_, err = clientset.Deployments(c.Namespace).Create(deployment)
|
||||
if err != nil {
|
||||
if !k8sutil.IsKubernetesResourceAlreadyExistError(err) {
|
||||
@@ -84,41 +86,59 @@ func (c *Cluster) Start(clientset *kubernetes.Clientset, cluster *mon.ClusterInf
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) createKeyring(cluster *mon.ClusterInfo) (string, error) {
|
||||
context := &clusterd.Context{ConfigDir: "/var/lib/rook"}
|
||||
func (c *Cluster) createKeyring(clientset *kubernetes.Clientset, cluster *mon.ClusterInfo) error {
|
||||
_, err := clientset.Secrets(c.Namespace).Get(rgwSecretsName)
|
||||
if err == nil {
|
||||
logger.Infof("the rgw keyring was already generated")
|
||||
return nil
|
||||
}
|
||||
if !k8sutil.IsKubernetesResourceNotFoundError(err) {
|
||||
return fmt.Errorf("failed to get rgw secrets. %+v", err)
|
||||
}
|
||||
|
||||
// connect to the ceph cluster
|
||||
logger.Infof("generating rgw keyring")
|
||||
context := &clusterd.Context{ConfigDir: "/var/lib/rook"}
|
||||
conn, err := mon.ConnectToClusterAsAdmin(context, c.factory, cluster)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to connect to cluster. %+v", err)
|
||||
return fmt.Errorf("failed to connect to cluster. %+v", err)
|
||||
}
|
||||
defer conn.Shutdown()
|
||||
|
||||
// create the keyring
|
||||
keyring, err := cephrgw.CreateKeyring(conn)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create keyring. %+v", err)
|
||||
return fmt.Errorf("failed to create keyring. %+v", err)
|
||||
}
|
||||
|
||||
return keyring, nil
|
||||
// store the secrets
|
||||
secrets := map[string]string{
|
||||
rgwKeyringName: keyring,
|
||||
}
|
||||
_, err = clientset.Secrets(c.Namespace).Create(&v1.Secret{ObjectMeta: v1.ObjectMeta{Name: rgwSecretsName}, StringData: secrets})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save rgw secrets. %+v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*extensions.Deployment, error) {
|
||||
func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo) (*extensions.Deployment, error) {
|
||||
deployment := &extensions.Deployment{}
|
||||
deployment.Name = deploymentName
|
||||
deployment.Namespace = c.Namespace
|
||||
|
||||
podSpec := api.PodTemplateSpec{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: "rook-rgw",
|
||||
Labels: getLabels(cluster.Name),
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Containers: []api.Container{c.rgwContainer(cluster, keyring)},
|
||||
RestartPolicy: api.RestartPolicyAlways,
|
||||
Volumes: []api.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.rgwContainer(cluster)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
{Name: k8sutil.DataDirVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -128,19 +148,24 @@ func (c *Cluster) makeDeployment(cluster *mon.ClusterInfo, keyring string) (*ext
|
||||
return deployment, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo, keyring string) api.Container {
|
||||
func (c *Cluster) rgwContainer(cluster *mon.ClusterInfo) v1.Container {
|
||||
|
||||
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --mon-secret=%s --admin-secret=%s --rgw-port=%d --rgw-host=%s --rgw-keyring=%s",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cluster.MonitorSecret, cluster.AdminSecret, cephrgw.RGWPort, cephrgw.DNSName, keyring)
|
||||
return api.Container{
|
||||
command := fmt.Sprintf("/usr/bin/rookd rgw --data-dir=%s --mon-endpoints=%s --cluster-name=%s --rgw-port=%d --rgw-host=%s",
|
||||
k8sutil.DataDir, mon.FlattenMonEndpoints(cluster.Monitors), cluster.Name, cephrgw.RGWPort, cephrgw.DNSName)
|
||||
return v1.Container{
|
||||
// TODO: fix "sleep 5".
|
||||
// Without waiting some time, there is highly probable flakes in network setup.
|
||||
Command: []string{"/bin/sh", "-c", fmt.Sprintf("sleep 5; %s", command)},
|
||||
Name: rgwApp,
|
||||
Image: k8sutil.MakeRookImage(c.Version),
|
||||
VolumeMounts: []api.VolumeMount{
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
|
||||
},
|
||||
Env: []v1.EnvVar{
|
||||
{Name: "ROOKD_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: rgwSecretsName}, Key: rgwKeyringName}}},
|
||||
k8smon.MonSecretEnvVar(),
|
||||
k8smon.AdminSecretEnvVar(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user