ceph: Standardize association of keyrings with deployments
This change standardizes the ownerRefs of Ceph keyrings to k8s Deployment resources across all daemons and refactors a bit of duplicate code on keyring association for ease of maintenance. Fixes: #4089 Signed-off-by: egafford <egafford@redhat.com>
This commit is contained in:
1 parent
4c3dc2828d
commit
bb7aac4b62
8 files changed
+62
-37
No files matched your search
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -55,16 +56,15 @@ func (c *Cluster) dashboardPort() int {
|
||||
return c.dashboard.Port
|
||||
}
|
||||
|
||||
func (c *Cluster) generateKeyring(m *mgrConfig) error {
|
||||
func (c *Cluster) generateKeyring(m *mgrConfig) (string, error) {
|
||||
user := fmt.Sprintf("mgr.%s", m.DaemonID)
|
||||
/* TODO: the access string here does not match the access from the keyring template. should they match? */
|
||||
access := []string{"mon", "allow *", "mds", "allow *", "osd", "allow *"}
|
||||
/* TODO: can we change this ownerref to be the deployment or service? */
|
||||
s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
|
||||
@@ -78,5 +78,10 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error {
|
||||
}
|
||||
|
||||
keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key)
|
||||
return s.CreateOrUpdate(m.ResourceName, keyring)
|
||||
return keyring, s.CreateOrUpdate(m.ResourceName, keyring)
|
||||
}
|
||||
|
||||
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
|
||||
s := keyring.GetSecretStoreForDeployment(c.context, d)
|
||||
return s.CreateOrUpdate(d.GetName(), existingKeyring)
|
||||
}
|
||||
@@ -154,19 +154,21 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
|
||||
// generate keyring specific to this mgr daemon saved to k8s secret
|
||||
if err := c.generateKeyring(mgrConfig); err != nil {
|
||||
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
|
||||
keyring, err := c.generateKeyring(mgrConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
d := c.makeDeployment(mgrConfig)
|
||||
logger.Debugf("starting mgr deployment: %+v", d)
|
||||
_, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d)
|
||||
_, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err)
|
||||
}
|
||||
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
|
||||
|
||||
// Always invoke ceph version before an upgrade so we are sure to be up-to-date
|
||||
daemon := string(config.MgrType)
|
||||
var cephVersionToUse cephver.CephVersion
|
||||
@@ -188,6 +190,13 @@ func (c *Cluster) Start() error {
|
||||
return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err)
|
||||
}
|
||||
}
|
||||
if existingDeployment, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil {
|
||||
logger.Warningf("failed to find mgr deployment %s for keyring association: %+v", resourceName, err)
|
||||
} else {
|
||||
if err = c.associateKeyring(keyring, existingDeployment); err != nil {
|
||||
logger.Warningf("failed to associate keyring with mgr deployment %s: %+v", resourceName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.configureDashboardService(); err != nil {
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -51,15 +50,6 @@ func (c *Cluster) generateKeyring(osdID int) (string, error) {
|
||||
}
|
||||
|
||||
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
|
||||
resourceName := d.GetName()
|
||||
|
||||
ownerRef := &metav1.OwnerReference{
|
||||
UID: d.UID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: resourceName,
|
||||
}
|
||||
s := keyring.GetSecretStore(c.context, c.Namespace, ownerRef)
|
||||
|
||||
return s.CreateOrUpdate(resourceName, existingKeyring)
|
||||
s := keyring.GetSecretStoreForDeployment(c.context, d)
|
||||
return s.CreateOrUpdate(d.GetName(), existingKeyring)
|
||||
}
|
||||
@@ -443,7 +443,7 @@ func (c *Cluster) startOSDDaemonsOnPVC(pvcName string, config *provisionConfig,
|
||||
logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID)
|
||||
createdDeployment, err = c.context.Clientset.AppsV1().Deployments(c.Namespace).Get(dp.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get existing OSD deployment %s for update: %+v", dp.Name, err)
|
||||
logger.Warningf("failed to get existing OSD deployment %q for update: %+v", dp.Name, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -41,14 +42,14 @@ type daemonConfig struct {
|
||||
DataPathMap *config.DataPathMap // location to store data in container
|
||||
}
|
||||
|
||||
func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
|
||||
func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) (string, error) {
|
||||
user := fullDaemonName(daemonConfig.DaemonID)
|
||||
access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"}
|
||||
s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
|
||||
@@ -62,7 +63,12 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
|
||||
}
|
||||
|
||||
keyring := fmt.Sprintf(keyringTemplate, daemonConfig.DaemonID, key)
|
||||
return s.CreateOrUpdate(daemonConfig.ResourceName, keyring)
|
||||
return keyring, s.CreateOrUpdate(daemonConfig.ResourceName, keyring)
|
||||
}
|
||||
|
||||
func (m *Mirroring) associateKeyring(existingKeyring string, d *apps.Deployment) error {
|
||||
s := keyring.GetSecretStoreForDeployment(m.context, d)
|
||||
return s.CreateOrUpdate(d.GetName(), existingKeyring)
|
||||
}
|
||||
|
||||
func fullDaemonName(daemonID string) string {
|
||||
|
||||
@@ -117,7 +117,8 @@ func (m *Mirroring) Start() error {
|
||||
DataPathMap: config.NewDatalessDaemonDataPathMap(m.Namespace, m.dataDirHostPath),
|
||||
}
|
||||
|
||||
if err := m.generateKeyring(daemonConf); err != nil {
|
||||
keyring, err := m.generateKeyring(daemonConf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
|
||||
}
|
||||
|
||||
@@ -158,9 +159,16 @@ func (m *Mirroring) Start() error {
|
||||
return fmt.Errorf("failed to recreate rbd-mirror deployment %s during del-and-recreate update attempt. %+v", resourceName, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Infof("%s deployment started", resourceName)
|
||||
}
|
||||
|
||||
if existingDeployment, err := m.context.Clientset.AppsV1().Deployments(m.Namespace).Get(d.GetName(), metav1.GetOptions{}); err != nil {
|
||||
logger.Warningf("failed to find rbd-mirror deployment %s for keyring association: %+v", resourceName, err)
|
||||
} else {
|
||||
if err = m.associateKeyring(keyring, existingDeployment); err != nil {
|
||||
logger.Warningf("failed to associate keyring with rbd-mirror deployment %s: %+v", resourceName, err)
|
||||
}
|
||||
}
|
||||
logger.Infof("%s deployment started", resourceName)
|
||||
}
|
||||
|
||||
// Remove extra rbd-mirror deployments if necessary
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -54,6 +55,21 @@ func GetSecretStore(context *clusterd.Context, namespace string, ownerRef *metav
|
||||
}
|
||||
}
|
||||
|
||||
// GetSecretStoreForDeployment returns a new SecretStore struct owned by the provided Deployment.
|
||||
func GetSecretStoreForDeployment(context *clusterd.Context, d *apps.Deployment) *SecretStore {
|
||||
ownerRef := &metav1.OwnerReference{
|
||||
UID: d.UID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: d.GetName(),
|
||||
}
|
||||
return &SecretStore{
|
||||
context: context,
|
||||
namespace: d.GetNamespace(),
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
}
|
||||
|
||||
func keySecretName(resourceName string) string {
|
||||
return resourceName + "-key" // all keys named by suffixing key to the resource name
|
||||
}
|
||||
|
||||
@@ -63,15 +63,6 @@ func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) {
|
||||
}
|
||||
|
||||
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
|
||||
resourceName := d.GetName()
|
||||
|
||||
ownerRef := &metav1.OwnerReference{
|
||||
UID: d.UID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: resourceName,
|
||||
}
|
||||
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
|
||||
|
||||
return s.CreateOrUpdate(resourceName, existingKeyring)
|
||||
s := keyring.GetSecretStoreForDeployment(c.context, d)
|
||||
return s.CreateOrUpdate(d.GetName(), existingKeyring)
|
||||
}
|
||||
Reference in new issue
Block a user