ceph: Generate MDS keyrings before creating dependent deployments

Failure to create a keyring prior to creating a deployment
dependent on that keyring created a race conditition resulting
in an intermittent and unnecessary pod failure. This change
creates a keyring prior to deployment creation for MDS
daemons.

Partially fixes: rook#4089

Signed-off-by: egafford <egafford@redhat.com>
This commit is contained in:
egafford committed 2019-11-05 15:02:15 -05:00
1 parent 1f788cf637
commit ecd9ff1672
7 files changed
+49 -31

No files matched your search

+6 -6
View File
@@ -122,7 +122,7 @@ func (c *FilesystemController) onAdd(obj interface{}) {
}
}
err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.clusterSpec, c.filesystemOwners(filesystem), c.clusterSpec.DataDirHostPath, c.isUpgrade)
err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.clusterSpec, c.filesystemOwner(filesystem), c.clusterSpec.DataDirHostPath, c.isUpgrade)
if err != nil {
logger.Errorf("failed to create filesystem %s: %+v", filesystem.Name, err)
}
@@ -155,7 +155,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) {
// if the filesystem is modified, allow the filesystem to be created if it wasn't already
logger.Infof("updating filesystem %s", newFS.Name)
err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.clusterSpec, c.filesystemOwners(newFS), c.clusterSpec.DataDirHostPath, c.isUpgrade)
err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.clusterSpec, c.filesystemOwner(newFS), c.clusterSpec.DataDirHostPath, c.isUpgrade)
if err != nil {
logger.Errorf("failed to create (modify) filesystem %s: %+v", newFS.Name, err)
}
@@ -183,7 +183,7 @@ func (c *FilesystemController) ParentClusterChanged(cluster cephv1.ClusterSpec,
}
for _, fs := range filesystems.Items {
logger.Infof("updating the ceph version for filesystem %s to %s", fs.Name, c.clusterSpec.CephVersion.Image)
err = createFilesystem(c.clusterInfo, c.context, fs, c.rookVersion, c.clusterSpec, c.filesystemOwners(&fs), c.clusterSpec.DataDirHostPath, c.isUpgrade)
err = createFilesystem(c.clusterInfo, c.context, fs, c.rookVersion, c.clusterSpec, c.filesystemOwner(&fs), c.clusterSpec.DataDirHostPath, c.isUpgrade)
if err != nil {
logger.Errorf("failed to update filesystem %s. %+v", fs.Name, err)
} else {
@@ -213,14 +213,14 @@ func (c *FilesystemController) onDelete(obj interface{}) {
}
}
func (c *FilesystemController) filesystemOwners(fs *cephv1.CephFilesystem) []metav1.OwnerReference {
func (c *FilesystemController) filesystemOwner(fs *cephv1.CephFilesystem) metav1.OwnerReference {
// Set the filesystem CR as the owner
return []metav1.OwnerReference{{
return metav1.OwnerReference{
APIVersion: fmt.Sprintf("%s/%s", FilesystemResource.Group, FilesystemResource.Version),
Kind: FilesystemResource.Kind,
Name: fs.Name,
UID: fs.UID,
}}
}
}
func filesystemChanged(oldFS, newFS cephv1.FilesystemSpec) bool {
+1 -1
View File
@@ -51,7 +51,7 @@ func createFilesystem(
fs cephv1.CephFilesystem,
rookVersion string,
clusterSpec *cephv1.ClusterSpec,
ownerRefs []metav1.OwnerReference,
ownerRefs metav1.OwnerReference,
dataDirHostPath string,
isUpgrade bool,
) error {
+5 -5
View File
@@ -114,14 +114,14 @@ func TestCreateFilesystem(t *testing.T) {
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
// start a basic cluster
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
assert.Nil(t, err)
validateStart(t, context, fs)
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
// starting again should be a no-op
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
assert.Nil(t, err)
validateStart(t, context, fs)
assert.ElementsMatch(t, []string{"rook-ceph-mds-myfs-a", "rook-ceph-mds-myfs-b"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
@@ -142,7 +142,7 @@ func TestCreateFilesystem(t *testing.T) {
Clientset: testop.New(3)}
//Create another filesystem which should fail
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
assert.Equal(t, "failed to create filesystem myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error())
}
@@ -179,12 +179,12 @@ func TestCreateNopoolFilesystem(t *testing.T) {
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
// start a basic cluster
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
assert.Nil(t, err)
validateStart(t, context, fs)
// starting again should be a no-op
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
assert.Nil(t, err)
validateStart(t, context, fs)
+22 -11
View File
@@ -19,10 +19,11 @@ package mds
import (
"fmt"
apps "k8s.io/api/apps/v1"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
const (
@@ -35,20 +36,16 @@ caps mds = "allow"
`
)
func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) {
user := fmt.Sprintf("mds.%s", m.DaemonID)
access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"}
ownerRef := &metav1.OwnerReference{
UID: deploymentUID,
APIVersion: "v1",
Kind: "deployment",
Name: m.ResourceName,
}
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
// At present
s := keyring.GetSecretStore(c.context, c.fs.Namespace, &c.ownerRef)
key, err := s.GenerateKey(user, access)
if err != nil {
return err
return "", err
}
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
@@ -62,5 +59,19 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
}
keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key)
return s.CreateOrUpdate(m.ResourceName, keyring)
return keyring, s.CreateOrUpdate(m.ResourceName, keyring)
}
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
resourceName := d.GetName()
ownerRef := &metav1.OwnerReference{
UID: d.UID,
APIVersion: "v1",
Kind: "deployment",
Name: resourceName,
}
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
return s.CreateOrUpdate(resourceName, existingKeyring)
}
+13 -6
View File
@@ -58,7 +58,7 @@ type Cluster struct {
clusterSpec *cephv1.ClusterSpec
fs cephv1.CephFilesystem
fsID string
ownerRefs []metav1.OwnerReference
ownerRef metav1.OwnerReference
dataDirHostPath string
isUpgrade bool
}
@@ -77,7 +77,7 @@ func NewCluster(
clusterSpec *cephv1.ClusterSpec,
fs cephv1.CephFilesystem,
fsdetails *client.CephFilesystemDetails,
ownerRefs []metav1.OwnerReference,
ownerRef metav1.OwnerReference,
dataDirHostPath string,
isUpgrade bool,
) *Cluster {
@@ -88,7 +88,7 @@ func NewCluster(
clusterSpec: clusterSpec,
fs: fs,
fsID: strconv.Itoa(fsdetails.ID),
ownerRefs: ownerRefs,
ownerRef: ownerRef,
dataDirHostPath: dataDirHostPath,
isUpgrade: isUpgrade,
}
@@ -135,6 +135,12 @@ func (c *Cluster) Start() error {
DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, daemonName, c.fs.Namespace, c.dataDirHostPath),
}
// create unique key for each mds saved to k8s secret
keyring, err := c.generateKeyring(mdsConfig)
if err != nil {
return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err)
}
// start the deployment
d := c.makeDeployment(mdsConfig)
logger.Debugf("starting mds: %+v", d)
@@ -149,10 +155,11 @@ func (c *Cluster) Start() error {
return fmt.Errorf("failed to get existing mds deployment %s for update: %+v", d.Name, err)
}
}
// create unique key for each mds saved to k8s secret
if err := c.generateKeyring(mdsConfig, createdDeployment.UID); err != nil {
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
if err := c.associateKeyring(keyring, createdDeployment); err != nil {
logger.Warningf("failed to associate keyring with deployment for %q. %+v", resourceName, err)
}
// keyring must be generated before update-and-wait since no keyring will prevent the
// deployment from reaching ready state
if createErr != nil && errors.IsAlreadyExists(createErr) {
+1 -1
View File
@@ -83,7 +83,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
k8sutil.AddRookVersionLabelToDeployment(d)
c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&d.ObjectMeta)
opspec.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d)
k8sutil.SetOwnerRefs(&d.ObjectMeta, c.ownerRefs)
k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef)
return d
}
+1 -1
View File
@@ -71,7 +71,7 @@ func testDeploymentObject(network cephv1.NetworkSpec) *apps.Deployment {
},
fs,
&client.CephFilesystemDetails{ID: 15},
[]metav1.OwnerReference{{}},
metav1.OwnerReference{},
"/var/lib/rook/",
false,
)