From ecd9ff1672de8312fe2fe65b6b5c58d9c7e9e935 Mon Sep 17 00:00:00 2001 From: egafford Date: Wed, 30 Oct 2019 17:02:41 -0400 Subject: [PATCH] ceph: Generate MDS keyrings before creating dependent deployments Failure to create a keyring prior to creating a deployment dependent on that keyring created a race conditition resulting in an intermittent and unnecessary pod failure. This change creates a keyring prior to deployment creation for MDS daemons. Partially fixes: rook#4089 Signed-off-by: egafford --- pkg/operator/ceph/file/controller.go | 12 ++++----- pkg/operator/ceph/file/filesystem.go | 2 +- pkg/operator/ceph/file/filesystem_test.go | 10 +++---- pkg/operator/ceph/file/mds/config.go | 33 +++++++++++++++-------- pkg/operator/ceph/file/mds/mds.go | 19 ++++++++----- pkg/operator/ceph/file/mds/spec.go | 2 +- pkg/operator/ceph/file/mds/spec_test.go | 2 +- 7 files changed, 49 insertions(+), 31 deletions(-) diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index ed16a527b..90cab10b6 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -122,7 +122,7 @@ func (c *FilesystemController) onAdd(obj interface{}) { } } - err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.clusterSpec, c.filesystemOwners(filesystem), c.clusterSpec.DataDirHostPath, c.isUpgrade) + err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.clusterSpec, c.filesystemOwner(filesystem), c.clusterSpec.DataDirHostPath, c.isUpgrade) if err != nil { logger.Errorf("failed to create filesystem %s: %+v", filesystem.Name, err) } @@ -155,7 +155,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) { // if the filesystem is modified, allow the filesystem to be created if it wasn't already logger.Infof("updating filesystem %s", newFS.Name) - err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.clusterSpec, c.filesystemOwners(newFS), c.clusterSpec.DataDirHostPath, c.isUpgrade) + err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.clusterSpec, c.filesystemOwner(newFS), c.clusterSpec.DataDirHostPath, c.isUpgrade) if err != nil { logger.Errorf("failed to create (modify) filesystem %s: %+v", newFS.Name, err) } @@ -183,7 +183,7 @@ func (c *FilesystemController) ParentClusterChanged(cluster cephv1.ClusterSpec, } for _, fs := range filesystems.Items { logger.Infof("updating the ceph version for filesystem %s to %s", fs.Name, c.clusterSpec.CephVersion.Image) - err = createFilesystem(c.clusterInfo, c.context, fs, c.rookVersion, c.clusterSpec, c.filesystemOwners(&fs), c.clusterSpec.DataDirHostPath, c.isUpgrade) + err = createFilesystem(c.clusterInfo, c.context, fs, c.rookVersion, c.clusterSpec, c.filesystemOwner(&fs), c.clusterSpec.DataDirHostPath, c.isUpgrade) if err != nil { logger.Errorf("failed to update filesystem %s. %+v", fs.Name, err) } else { @@ -213,14 +213,14 @@ func (c *FilesystemController) onDelete(obj interface{}) { } } -func (c *FilesystemController) filesystemOwners(fs *cephv1.CephFilesystem) []metav1.OwnerReference { +func (c *FilesystemController) filesystemOwner(fs *cephv1.CephFilesystem) metav1.OwnerReference { // Set the filesystem CR as the owner - return []metav1.OwnerReference{{ + return metav1.OwnerReference{ APIVersion: fmt.Sprintf("%s/%s", FilesystemResource.Group, FilesystemResource.Version), Kind: FilesystemResource.Kind, Name: fs.Name, UID: fs.UID, - }} + } } func filesystemChanged(oldFS, newFS cephv1.FilesystemSpec) bool { diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 6f3f47064..af6e82cdd 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -51,7 +51,7 @@ func createFilesystem( fs cephv1.CephFilesystem, rookVersion string, clusterSpec *cephv1.ClusterSpec, - ownerRefs []metav1.OwnerReference, + ownerRefs metav1.OwnerReference, dataDirHostPath string, isUpgrade bool, ) error { diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index b07d6cc46..a9f65e092 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -114,14 +114,14 @@ func TestCreateFilesystem(t *testing.T) { clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} // start a basic cluster - err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false) + err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false) assert.Nil(t, err) validateStart(t, context, fs) assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) // starting again should be a no-op - err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false) + err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false) assert.Nil(t, err) validateStart(t, context, fs) assert.ElementsMatch(t, []string{"rook-ceph-mds-myfs-a", "rook-ceph-mds-myfs-b"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) @@ -142,7 +142,7 @@ func TestCreateFilesystem(t *testing.T) { Clientset: testop.New(3)} //Create another filesystem which should fail - err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false) + err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false) assert.Equal(t, "failed to create filesystem myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error()) } @@ -179,12 +179,12 @@ func TestCreateNopoolFilesystem(t *testing.T) { clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} // start a basic cluster - err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false) + err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false) assert.Nil(t, err) validateStart(t, context, fs) // starting again should be a no-op - err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false) + err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false) assert.Nil(t, err) validateStart(t, context, fs) diff --git a/pkg/operator/ceph/file/mds/config.go b/pkg/operator/ceph/file/mds/config.go index cf90c826c..5532945b9 100644 --- a/pkg/operator/ceph/file/mds/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -19,10 +19,11 @@ package mds import ( "fmt" + apps "k8s.io/api/apps/v1" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) const ( @@ -35,20 +36,16 @@ caps mds = "allow" ` ) -func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error { +func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) { user := fmt.Sprintf("mds.%s", m.DaemonID) access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} - ownerRef := &metav1.OwnerReference{ - UID: deploymentUID, - APIVersion: "v1", - Kind: "deployment", - Name: m.ResourceName, - } - s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef) + + // At present + s := keyring.GetSecretStore(c.context, c.fs.Namespace, &c.ownerRef) key, err := s.GenerateKey(user, access) if err != nil { - return err + return "", err } // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x @@ -62,5 +59,19 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error { } keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key) - return s.CreateOrUpdate(m.ResourceName, keyring) + return keyring, s.CreateOrUpdate(m.ResourceName, keyring) +} + +func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error { + resourceName := d.GetName() + + ownerRef := &metav1.OwnerReference{ + UID: d.UID, + APIVersion: "v1", + Kind: "deployment", + Name: resourceName, + } + s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef) + + return s.CreateOrUpdate(resourceName, existingKeyring) } diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index 2014af3f0..012a00827 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -58,7 +58,7 @@ type Cluster struct { clusterSpec *cephv1.ClusterSpec fs cephv1.CephFilesystem fsID string - ownerRefs []metav1.OwnerReference + ownerRef metav1.OwnerReference dataDirHostPath string isUpgrade bool } @@ -77,7 +77,7 @@ func NewCluster( clusterSpec *cephv1.ClusterSpec, fs cephv1.CephFilesystem, fsdetails *client.CephFilesystemDetails, - ownerRefs []metav1.OwnerReference, + ownerRef metav1.OwnerReference, dataDirHostPath string, isUpgrade bool, ) *Cluster { @@ -88,7 +88,7 @@ func NewCluster( clusterSpec: clusterSpec, fs: fs, fsID: strconv.Itoa(fsdetails.ID), - ownerRefs: ownerRefs, + ownerRef: ownerRef, dataDirHostPath: dataDirHostPath, isUpgrade: isUpgrade, } @@ -135,6 +135,12 @@ func (c *Cluster) Start() error { DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, daemonName, c.fs.Namespace, c.dataDirHostPath), } + // create unique key for each mds saved to k8s secret + keyring, err := c.generateKeyring(mdsConfig) + if err != nil { + return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err) + } + // start the deployment d := c.makeDeployment(mdsConfig) logger.Debugf("starting mds: %+v", d) @@ -149,10 +155,11 @@ func (c *Cluster) Start() error { return fmt.Errorf("failed to get existing mds deployment %s for update: %+v", d.Name, err) } } - // create unique key for each mds saved to k8s secret - if err := c.generateKeyring(mdsConfig, createdDeployment.UID); err != nil { - return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err) + + if err := c.associateKeyring(keyring, createdDeployment); err != nil { + logger.Warningf("failed to associate keyring with deployment for %q. %+v", resourceName, err) } + // keyring must be generated before update-and-wait since no keyring will prevent the // deployment from reaching ready state if createErr != nil && errors.IsAlreadyExists(createErr) { diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 202afe669..c05fd517c 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -83,7 +83,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { k8sutil.AddRookVersionLabelToDeployment(d) c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&d.ObjectMeta) opspec.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d) - k8sutil.SetOwnerRefs(&d.ObjectMeta, c.ownerRefs) + k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef) return d } diff --git a/pkg/operator/ceph/file/mds/spec_test.go b/pkg/operator/ceph/file/mds/spec_test.go index da8f4847a..c211e2bac 100644 --- a/pkg/operator/ceph/file/mds/spec_test.go +++ b/pkg/operator/ceph/file/mds/spec_test.go @@ -71,7 +71,7 @@ func testDeploymentObject(network cephv1.NetworkSpec) *apps.Deployment { }, fs, &client.CephFilesystemDetails{ID: 15}, - []metav1.OwnerReference{{}}, + metav1.OwnerReference{}, "/var/lib/rook/", false, )