ceph: Standardize association of keyrings with deployments

This change standardizes the ownerRefs of Ceph keyrings to
k8s Deployment resources across all daemons and refactors a
bit of duplicate code on keyring association for ease of
maintenance.

Fixes: #4089

Signed-off-by: egafford <egafford@redhat.com>
This commit is contained in:
egafford committed 2019-11-08 09:50:37 -05:00
1 parent 4c3dc2828d
commit bb7aac4b62
8 files changed
+62 -37

No files matched your search

+9 -4
View File
@@ -21,6 +21,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
apps "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -55,16 +56,15 @@ func (c *Cluster) dashboardPort() int {
return c.dashboard.Port
}
func (c *Cluster) generateKeyring(m *mgrConfig) error {
func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
user := fmt.Sprintf("mgr.%s", m.DaemonID)
/* TODO: the access string here does not match the access from the keyring template. should they match? */
access := []string{"mon", "allow *", "mds", "allow *", "osd", "allow *"}
/* TODO: can we change this ownerref to be the deployment or service? */
s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
return "", err
}
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
@@ -78,5 +78,10 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error {
}
keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key)
return s.CreateOrUpdate(m.ResourceName, keyring)
return keyring, s.CreateOrUpdate(m.ResourceName, keyring)
}
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
s := keyring.GetSecretStoreForDeployment(c.context, d)
return s.CreateOrUpdate(d.GetName(), existingKeyring)
}
+12 -3
View File
@@ -154,19 +154,21 @@ func (c *Cluster) Start() error {
}
// generate keyring specific to this mgr daemon saved to k8s secret
if err := c.generateKeyring(mgrConfig); err != nil {
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
keyring, err := c.generateKeyring(mgrConfig)
if err != nil {
return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err)
}
// start the deployment
d := c.makeDeployment(mgrConfig)
logger.Debugf("starting mgr deployment: %+v", d)
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d)
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err)
}
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
// Always invoke ceph version before an upgrade so we are sure to be up-to-date
daemon := string(config.MgrType)
var cephVersionToUse cephver.CephVersion
@@ -188,6 +190,13 @@ func (c *Cluster) Start() error {
return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err)
}
}
if existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil {
logger.Warningf("failed to find mgr deployment %s for keyring association: %+v", resourceName, err)
} else {
if err = c.associateKeyring(keyring, existingDeployment); err != nil {
logger.Warningf("failed to associate keyring with mgr deployment %s: %+v", resourceName, err)
}
}
}
if err := c.configureDashboardService(); err != nil {
+2 -12
View File
@@ -22,7 +22,6 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
apps "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
@@ -51,15 +50,6 @@ func (c *Cluster) generateKeyring(osdID int) (string, error) {
}
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
resourceName := d.GetName()
ownerRef := &metav1.OwnerReference{
UID: d.UID,
APIVersion: "v1",
Kind: "deployment",
Name: resourceName,
}
s := keyring.GetSecretStore(c.context, c.Namespace, ownerRef)
return s.CreateOrUpdate(resourceName, existingKeyring)
s := keyring.GetSecretStoreForDeployment(c.context, d)
return s.CreateOrUpdate(d.GetName(), existingKeyring)
}
+1 -1
View File
@@ -443,7 +443,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig,
logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID)
createdDeployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(dp.Name, metav1.GetOptions{})
if err != nil {
logger.Warningf("failed to get existing OSD deployment %s for update: %+v", dp.Name, err)
logger.Warningf("failed to get existing OSD deployment %q for update: %+v", dp.Name, err)
continue
}
}
+9 -3
View File
@@ -21,6 +21,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
apps "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -41,14 +42,14 @@ type daemonConfig struct {
DataPathMap *config.DataPathMap // location to store data in container
}
func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) (string, error) {
user := fullDaemonName(daemonConfig.DaemonID)
access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"}
s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
return "", err
}
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
@@ -62,7 +63,12 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
}
keyring := fmt.Sprintf(keyringTemplate, daemonConfig.DaemonID, key)
return s.CreateOrUpdate(daemonConfig.ResourceName, keyring)
return keyring, s.CreateOrUpdate(daemonConfig.ResourceName, keyring)
}
func (m *Mirroring) associateKeyring(existingKeyring string, d *apps.Deployment) error {
s := keyring.GetSecretStoreForDeployment(m.context, d)
return s.CreateOrUpdate(d.GetName(), existingKeyring)
}
func fullDaemonName(daemonID string) string {
+11 -3
View File
@@ -117,7 +117,8 @@ func (m *Mirroring) Start() error {
DataPathMap: config.NewDatalessDaemonDataPathMap(m.Namespace, m.dataDirHostPath),
}
if err := m.generateKeyring(daemonConf); err != nil {
keyring, err := m.generateKeyring(daemonConf)
if err != nil {
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
}
@@ -158,9 +159,16 @@ func (m *Mirroring) Start() error {
return fmt.Errorf("failed to recreate rbd-mirror deployment %s during del-and-recreate update attempt. %+v", resourceName, err)
}
}
} else {
logger.Infof("%s deployment started", resourceName)
}
if existingDeployment, err := m.context.Clientset.AppsV1().Deployments(m.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil {
logger.Warningf("failed to find rbd-mirror deployment %s for keyring association: %+v", resourceName, err)
} else {
if err = m.associateKeyring(keyring, existingDeployment); err != nil {
logger.Warningf("failed to associate keyring with rbd-mirror deployment %s: %+v", resourceName, err)
}
}
logger.Infof("%s deployment started", resourceName)
}
// Remove extra rbd-mirror deployments if necessary
+16
View File
@@ -26,6 +26,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,6 +55,21 @@ func GetSecretStore(context *clusterd.Context, namespace string, ownerRef *metav
}
}
// GetSecretStoreForDeployment returns a new SecretStore struct owned by the provided Deployment.
func GetSecretStoreForDeployment(context *clusterd.Context, d *apps.Deployment) *SecretStore {
ownerRef := &metav1.OwnerReference{
UID: d.UID,
APIVersion: "v1",
Kind: "deployment",
Name: d.GetName(),
}
return &SecretStore{
context: context,
namespace: d.GetNamespace(),
ownerRef: ownerRef,
}
}
func keySecretName(resourceName string) string {
return resourceName + "-key" // all keys named by suffixing key to the resource name
}
+2 -11
View File
@@ -63,15 +63,6 @@ func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) {
}
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
resourceName := d.GetName()
ownerRef := &metav1.OwnerReference{
UID: d.UID,
APIVersion: "v1",
Kind: "deployment",
Name: resourceName,
}
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
return s.CreateOrUpdate(resourceName, existingKeyring)
s := keyring.GetSecretStoreForDeployment(c.context, d)
return s.CreateOrUpdate(d.GetName(), existingKeyring)
}