ceph: Generate MDS keyrings before creating dependent deployments
Failure to create a keyring prior to creating a deployment dependent on that keyring created a race conditition resulting in an intermittent and unnecessary pod failure. This change creates a keyring prior to deployment creation for MDS daemons. Partially fixes: rook#4089 Signed-off-by: egafford <egafford@redhat.com>
This commit is contained in:
1 parent
1f788cf637
commit
ecd9ff1672
7 files changed
+49
-31
No files matched your search
@@ -122,7 +122,7 @@ func (c *FilesystemController) onAdd(obj interface{}) {
|
||||
}
|
||||
}
|
||||
|
||||
err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.clusterSpec, c.filesystemOwners(filesystem), c.clusterSpec.DataDirHostPath, c.isUpgrade)
|
||||
err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.clusterSpec, c.filesystemOwner(filesystem), c.clusterSpec.DataDirHostPath, c.isUpgrade)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create filesystem %s: %+v", filesystem.Name, err)
|
||||
}
|
||||
@@ -155,7 +155,7 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) {
|
||||
|
||||
// if the filesystem is modified, allow the filesystem to be created if it wasn't already
|
||||
logger.Infof("updating filesystem %s", newFS.Name)
|
||||
err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.clusterSpec, c.filesystemOwners(newFS), c.clusterSpec.DataDirHostPath, c.isUpgrade)
|
||||
err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.clusterSpec, c.filesystemOwner(newFS), c.clusterSpec.DataDirHostPath, c.isUpgrade)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create (modify) filesystem %s: %+v", newFS.Name, err)
|
||||
}
|
||||
@@ -183,7 +183,7 @@ func (c *FilesystemController) ParentClusterChanged(cluster cephv1.ClusterSpec,
|
||||
}
|
||||
for _, fs := range filesystems.Items {
|
||||
logger.Infof("updating the ceph version for filesystem %s to %s", fs.Name, c.clusterSpec.CephVersion.Image)
|
||||
err = createFilesystem(c.clusterInfo, c.context, fs, c.rookVersion, c.clusterSpec, c.filesystemOwners(&fs), c.clusterSpec.DataDirHostPath, c.isUpgrade)
|
||||
err = createFilesystem(c.clusterInfo, c.context, fs, c.rookVersion, c.clusterSpec, c.filesystemOwner(&fs), c.clusterSpec.DataDirHostPath, c.isUpgrade)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to update filesystem %s. %+v", fs.Name, err)
|
||||
} else {
|
||||
@@ -213,14 +213,14 @@ func (c *FilesystemController) onDelete(obj interface{}) {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *FilesystemController) filesystemOwners(fs *cephv1.CephFilesystem) []metav1.OwnerReference {
|
||||
func (c *FilesystemController) filesystemOwner(fs *cephv1.CephFilesystem) metav1.OwnerReference {
|
||||
// Set the filesystem CR as the owner
|
||||
return []metav1.OwnerReference{{
|
||||
return metav1.OwnerReference{
|
||||
APIVersion: fmt.Sprintf("%s/%s", FilesystemResource.Group, FilesystemResource.Version),
|
||||
Kind: FilesystemResource.Kind,
|
||||
Name: fs.Name,
|
||||
UID: fs.UID,
|
||||
}}
|
||||
}
|
||||
}
|
||||
|
||||
func filesystemChanged(oldFS, newFS cephv1.FilesystemSpec) bool {
|
||||
|
||||
@@ -51,7 +51,7 @@ func createFilesystem(
|
||||
fs cephv1.CephFilesystem,
|
||||
rookVersion string,
|
||||
clusterSpec *cephv1.ClusterSpec,
|
||||
ownerRefs []metav1.OwnerReference,
|
||||
ownerRefs metav1.OwnerReference,
|
||||
dataDirHostPath string,
|
||||
isUpgrade bool,
|
||||
) error {
|
||||
|
||||
@@ -114,14 +114,14 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
|
||||
// start a basic cluster
|
||||
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
||||
|
||||
// starting again should be a no-op
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
assert.ElementsMatch(t, []string{"rook-ceph-mds-myfs-a", "rook-ceph-mds-myfs-b"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
@@ -142,7 +142,7 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
Clientset: testop.New(3)}
|
||||
|
||||
//Create another filesystem which should fail
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
assert.Equal(t, "failed to create filesystem myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error())
|
||||
}
|
||||
|
||||
@@ -179,12 +179,12 @@ func TestCreateNopoolFilesystem(t *testing.T) {
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
|
||||
// start a basic cluster
|
||||
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
err := createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
|
||||
// starting again should be a no-op
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, []metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", &cephv1.ClusterSpec{}, metav1.OwnerReference{}, "/var/lib/rook/", false)
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
|
||||
|
||||
@@ -19,10 +19,11 @@ package mds
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
apps "k8s.io/api/apps/v1"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -35,20 +36,16 @@ caps mds = "allow"
|
||||
`
|
||||
)
|
||||
|
||||
func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
|
||||
func (c *Cluster) generateKeyring(m *mdsConfig) (string, error) {
|
||||
user := fmt.Sprintf("mds.%s", m.DaemonID)
|
||||
access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"}
|
||||
ownerRef := &metav1.OwnerReference{
|
||||
UID: deploymentUID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: m.ResourceName,
|
||||
}
|
||||
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
|
||||
|
||||
// At present
|
||||
s := keyring.GetSecretStore(c.context, c.fs.Namespace, &c.ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
|
||||
@@ -62,5 +59,19 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
|
||||
}
|
||||
|
||||
keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key)
|
||||
return s.CreateOrUpdate(m.ResourceName, keyring)
|
||||
return keyring, s.CreateOrUpdate(m.ResourceName, keyring)
|
||||
}
|
||||
|
||||
func (c *Cluster) associateKeyring(existingKeyring string, d *apps.Deployment) error {
|
||||
resourceName := d.GetName()
|
||||
|
||||
ownerRef := &metav1.OwnerReference{
|
||||
UID: d.UID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: resourceName,
|
||||
}
|
||||
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
|
||||
|
||||
return s.CreateOrUpdate(resourceName, existingKeyring)
|
||||
}
|
||||
@@ -58,7 +58,7 @@ type Cluster struct {
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
fs cephv1.CephFilesystem
|
||||
fsID string
|
||||
ownerRefs []metav1.OwnerReference
|
||||
ownerRef metav1.OwnerReference
|
||||
dataDirHostPath string
|
||||
isUpgrade bool
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func NewCluster(
|
||||
clusterSpec *cephv1.ClusterSpec,
|
||||
fs cephv1.CephFilesystem,
|
||||
fsdetails *client.CephFilesystemDetails,
|
||||
ownerRefs []metav1.OwnerReference,
|
||||
ownerRef metav1.OwnerReference,
|
||||
dataDirHostPath string,
|
||||
isUpgrade bool,
|
||||
) *Cluster {
|
||||
@@ -88,7 +88,7 @@ func NewCluster(
|
||||
clusterSpec: clusterSpec,
|
||||
fs: fs,
|
||||
fsID: strconv.Itoa(fsdetails.ID),
|
||||
ownerRefs: ownerRefs,
|
||||
ownerRef: ownerRef,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
isUpgrade: isUpgrade,
|
||||
}
|
||||
@@ -135,6 +135,12 @@ func (c *Cluster) Start() error {
|
||||
DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, daemonName, c.fs.Namespace, c.dataDirHostPath),
|
||||
}
|
||||
|
||||
// create unique key for each mds saved to k8s secret
|
||||
keyring, err := c.generateKeyring(mdsConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate keyring for %q. %+v", resourceName, err)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
d := c.makeDeployment(mdsConfig)
|
||||
logger.Debugf("starting mds: %+v", d)
|
||||
@@ -149,10 +155,11 @@ func (c *Cluster) Start() error {
|
||||
return fmt.Errorf("failed to get existing mds deployment %s for update: %+v", d.Name, err)
|
||||
}
|
||||
}
|
||||
// create unique key for each mds saved to k8s secret
|
||||
if err := c.generateKeyring(mdsConfig, createdDeployment.UID); err != nil {
|
||||
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
|
||||
|
||||
if err := c.associateKeyring(keyring, createdDeployment); err != nil {
|
||||
logger.Warningf("failed to associate keyring with deployment for %q. %+v", resourceName, err)
|
||||
}
|
||||
|
||||
// keyring must be generated before update-and-wait since no keyring will prevent the
|
||||
// deployment from reaching ready state
|
||||
if createErr != nil && errors.IsAlreadyExists(createErr) {
|
||||
|
||||
@@ -83,7 +83,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
|
||||
k8sutil.AddRookVersionLabelToDeployment(d)
|
||||
c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&d.ObjectMeta)
|
||||
opspec.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d)
|
||||
k8sutil.SetOwnerRefs(&d.ObjectMeta, c.ownerRefs)
|
||||
k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef)
|
||||
return d
|
||||
}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ func testDeploymentObject(network cephv1.NetworkSpec) *apps.Deployment {
|
||||
},
|
||||
fs,
|
||||
&client.CephFilesystemDetails{ID: 15},
|
||||
[]metav1.OwnerReference{{}},
|
||||
metav1.OwnerReference{},
|
||||
"/var/lib/rook/",
|
||||
false,
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user