From bb7aac4b6217d057cef1387fb13df44296458698 Mon Sep 17 00:00:00 2001 From: egafford Date: Thu, 31 Oct 2019 12:38:39 -0400 Subject: [PATCH] ceph: Standardize association of keyrings with deployments This change standardizes the ownerRefs of Ceph keyrings to k8s Deployment resources across all daemons and refactors a bit of duplicate code on keyring association for ease of maintenance. Fixes: #4089 Signed-off-by: egafford --- pkg/operator/ceph/cluster/mgr/config.go | 13 +++++++++---- pkg/operator/ceph/cluster/mgr/mgr.go | 15 ++++++++++++--- pkg/operator/ceph/cluster/osd/config.go | 14 ++------------ pkg/operator/ceph/cluster/osd/osd.go | 2 +- pkg/operator/ceph/cluster/rbd/config.go | 12 +++++++++--- pkg/operator/ceph/cluster/rbd/mirror.go | 14 +++++++++++--- pkg/operator/ceph/config/keyring/store.go | 16 ++++++++++++++++ pkg/operator/ceph/file/mds/config.go | 13 ++----------- 8 files changed, 62 insertions(+), 37 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index 211b30969..db8bf2516 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -21,6 +21,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" + apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -55,16 +56,15 @@ func (c *Cluster) dashboardPort() int { return c.dashboard.Port } -func (c *Cluster) generateKeyring(m *mgrConfig) error { +func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) { user := fmt.Sprintf("mgr.%s", m.DaemonID) /* TODO: the access string here does not match the access from the keyring template. should they match? */ access := []string{"mon", "allow *", "mds", "allow *", "osd", "allow *"} - /* TODO: can we change this ownerref to be the deployment or service? */ s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef) key, err := s.GenerateKey(user, access) if err != nil { - return err + return "", err } // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x @@ -78,5 +78,10 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error { } keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key) - return s.CreateOrUpdate(m.ResourceName, keyring) + return keyring, s.CreateOrUpdate(m.ResourceName, keyring) +} + +func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error { + s := keyring.GetSecretStoreForDeployment(c.context, d) + return s.CreateOrUpdate(d.GetName(), existingKeyring) } diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 34789d58a..1e9fe2509 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -154,19 +154,21 @@ func (c *Cluster) Start() error { } // generate keyring specific to this mgr daemon saved to k8s secret - if err := c.generateKeyring(mgrConfig); err != nil { - return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err) + keyring, err := c.generateKeyring(mgrConfig) + if err != nil { + return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err) } // start the deployment d := c.makeDeployment(mgrConfig) logger.Debugf("starting mgr deployment: %+v", d) - _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) + _, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) if err != nil { if !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err) } logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName) + // Always invoke ceph version before an upgrade so we are sure to be up-to-date daemon := string(config.MgrType) var cephVersionToUse cephver.CephVersion @@ -188,6 +190,13 @@ func (c *Cluster) Start() error { return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err) } } + if existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil { + logger.Warningf("failed to find mgr deployment %s for keyring association: %+v", resourceName, err) + } else { + if err = c.associateKeyring(keyring, existingDeployment); err != nil { + logger.Warningf("failed to associate keyring with mgr deployment %s: %+v", resourceName, err) + } + } } if err := c.configureDashboardService(); err != nil { diff --git a/pkg/operator/ceph/cluster/osd/config.go b/pkg/operator/ceph/cluster/osd/config.go index 8b108b940..8d4d5ccad 100644 --- a/pkg/operator/ceph/cluster/osd/config.go +++ b/pkg/operator/ceph/cluster/osd/config.go @@ -22,7 +22,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" apps "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -51,15 +50,6 @@ func (c *Cluster) generateKeyring(osdID int) (string, error) { } func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error { - resourceName := d.GetName() - - ownerRef := &metav1.OwnerReference{ - UID: d.UID, - APIVersion: "v1", - Kind: "deployment", - Name: resourceName, - } - s := keyring.GetSecretStore(c.context, c.Namespace, ownerRef) - - return s.CreateOrUpdate(resourceName, existingKeyring) + s := keyring.GetSecretStoreForDeployment(c.context, d) + return s.CreateOrUpdate(d.GetName(), existingKeyring) } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 36805ac29..0594f0681 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -443,7 +443,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig, logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) createdDeployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(dp.Name, metav1.GetOptions{}) if err != nil { - logger.Warningf("failed to get existing OSD deployment %s for update: %+v", dp.Name, err) + logger.Warningf("failed to get existing OSD deployment %q for update: %+v", dp.Name, err) continue } } diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index 75c12341a..780a8bc22 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -21,6 +21,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" + apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -41,14 +42,14 @@ type daemonConfig struct { DataPathMap *config.DataPathMap // location to store data in container } -func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error { +func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) (string, error) { user := fullDaemonName(daemonConfig.DaemonID) access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"} s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef) key, err := s.GenerateKey(user, access) if err != nil { - return err + return "", err } // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x @@ -62,7 +63,12 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error { } keyring := fmt.Sprintf(keyringTemplate, daemonConfig.DaemonID, key) - return s.CreateOrUpdate(daemonConfig.ResourceName, keyring) + return keyring, s.CreateOrUpdate(daemonConfig.ResourceName, keyring) +} + +func (m *Mirroring) associateKeyring(existingKeyring string, d *apps.Deployment) error { + s := keyring.GetSecretStoreForDeployment(m.context, d) + return s.CreateOrUpdate(d.GetName(), existingKeyring) } func fullDaemonName(daemonID string) string { diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index 3f29c5956..7fffccb5e 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -117,7 +117,8 @@ func (m *Mirroring) Start() error { DataPathMap: config.NewDatalessDaemonDataPathMap(m.Namespace, m.dataDirHostPath), } - if err := m.generateKeyring(daemonConf); err != nil { + keyring, err := m.generateKeyring(daemonConf) + if err != nil { return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err) } @@ -158,9 +159,16 @@ func (m *Mirroring) Start() error { return fmt.Errorf("failed to recreate rbd-mirror deployment %s during del-and-recreate update attempt. %+v", resourceName, err) } } - } else { - logger.Infof("%s deployment started", resourceName) } + + if existingDeployment, err := m.context.Clientset.AppsV1().Deployments(m.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil { + logger.Warningf("failed to find rbd-mirror deployment %s for keyring association: %+v", resourceName, err) + } else { + if err = m.associateKeyring(keyring, existingDeployment); err != nil { + logger.Warningf("failed to associate keyring with rbd-mirror deployment %s: %+v", resourceName, err) + } + } + logger.Infof("%s deployment started", resourceName) } // Remove extra rbd-mirror deployments if necessary diff --git a/pkg/operator/ceph/config/keyring/store.go b/pkg/operator/ceph/config/keyring/store.go index 737f55d7e..dc64b0a44 100644 --- a/pkg/operator/ceph/config/keyring/store.go +++ b/pkg/operator/ceph/config/keyring/store.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" + apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -54,6 +55,21 @@ func GetSecretStore(context *clusterd.Context, namespace string, ownerRef *metav } } +// GetSecretStoreForDeployment returns a new SecretStore struct owned by the provided Deployment. +func GetSecretStoreForDeployment(context *clusterd.Context, d *apps.Deployment) *SecretStore { + ownerRef := &metav1.OwnerReference{ + UID: d.UID, + APIVersion: "v1", + Kind: "deployment", + Name: d.GetName(), + } + return &SecretStore{ + context: context, + namespace: d.GetNamespace(), + ownerRef: ownerRef, + } +} + func keySecretName(resourceName string) string { return resourceName + "-key" // all keys named by suffixing key to the resource name } diff --git a/pkg/operator/ceph/file/mds/config.go b/pkg/operator/ceph/file/mds/config.go index 5532945b9..a4df37bde 100644 --- a/pkg/operator/ceph/file/mds/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -63,15 +63,6 @@ func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) { } func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error { - resourceName := d.GetName() - - ownerRef := &metav1.OwnerReference{ - UID: d.UID, - APIVersion: "v1", - Kind: "deployment", - Name: resourceName, - } - s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef) - - return s.CreateOrUpdate(resourceName, existingKeyring) + s := keyring.GetSecretStoreForDeployment(c.context, d) + return s.CreateOrUpdate(d.GetName(), existingKeyring) }