From 2c7a5f156b85aed8381bcd4990d5d5efeeaa7264 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 11 Jun 2025 11:52:04 -0600 Subject: [PATCH] osd: load latest cephx key during osd pod init Create a new OSD pod init container that loads the latest CephX key for the OSD. The init container must be run after OSD activation, due to keyfile reset behavior of ceph-bluestore-tool, used during activation. The primary purpose of this work is to allow OSDs to start and run correctly after key rotation, whether done manually by users or automatically by future Rook work. This work makes redundant prior code changes done in #14826. The CI test added in the prior work is kept to ensure no regression. Signed-off-by: Blaine Gardner --- pkg/operator/ceph/cluster/osd/spec.go | 126 +++++++++++++++------ pkg/operator/ceph/cluster/osd/spec_test.go | 67 ++++++----- 2 files changed, 133 insertions(+), 60 deletions(-) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 0fdd7f040..e484a9df6 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -111,39 +111,6 @@ KEYRING_FILE="$OSD_DATA_DIR"/keyring CV_MODE=%s DEVICE="$%s" -# In rare cases keyring file created with prepare-osd but did not -# being stored in ceph auth system therefore we need to import it -# from keyring file instead of creating new one -if ! ceph -n client.admin auth get osd."$OSD_ID" -k /etc/ceph/admin-keyring-store/keyring; then - if [ -f "$KEYRING_FILE" ]; then - # import keyring from existing file - TMP_DIR=$(mktemp -d) - - python3 -c " -import configparser - -config = configparser.ConfigParser() -config.read('$KEYRING_FILE') - -if not config.has_section('osd.$OSD_ID'): - exit() - -config['osd.$OSD_ID'] = {'key': config['osd.$OSD_ID']['key'], 'caps mon': '\"allow profile osd\"', 'caps mgr': '\"allow profile osd\"', 'caps osd': '\"allow *\"'} - -with open('$TMP_DIR/keyring', 'w') as configfile: - config.write(configfile) -" - - cat "$TMP_DIR"/keyring - ceph -n client.admin auth import -i "$TMP_DIR"/keyring -k /etc/ceph/admin-keyring-store/keyring - - rm --recursive --force "$TMP_DIR" - else - # create new keyring if no keyring file found - ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring - fi -fi - # active the osd with ceph-volume if [[ "$CV_MODE" == "lvm" ]]; then TMP_DIR=$(mktemp -d) @@ -651,6 +618,13 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd)) } + // key update init container must go after activate + adminKeyVol, cephxContainer := c.getCephxKeyUpdateInitContainer(osdID, osdProps) + if !volumeExistsWithName(volumes, adminKeyVol.Name) { + volumes = append(volumes, adminKeyVol) + } + initContainers = append(initContainers, cephxContainer) + // Doing a chown in a post start lifecycle hook does not reliably complete before the OSD // process starts, which can cause the pod to fail without the lifecycle hook's chown command // completing. It can take an arbitrarily long time for a pod restart to successfully chown the @@ -1430,6 +1404,83 @@ func (c *Cluster) getEncryptedStatusPVCInitContainer(mountPath string, osdProps } } +// init container to get (or create, if needed) the most up-to-date keyring for the given OSD and +// update the OSD config directory keyring file. this allows OSD cephx keys to be rotated without +// causing OSD pod crash loops. +// +// this must come after OSD activate init container because 'ceph-bluestore-tool' (and consequently, +// 'ceph-volume activate') pulls the keyring from the OSD disk's bluestore data and overwrites the +// config dir's keyring file, even when the on-disk key is not the most up-to-date. +func (c *Cluster) getCephxKeyUpdateInitContainer(osdID string, osdProps osdProperties) (v1.Volume, v1.Container) { + volMounts := []v1.VolumeMount{ + {Name: k8sutil.ConfigOverrideName, ReadOnly: true, MountPath: opconfig.EtcCephDir}, + } + + adminKeyringVol, adminKeyringVolMount := cephkey.Volume().Admin(), cephkey.VolumeMount().Admin() + volMounts = append(volMounts, adminKeyringVolMount) + + // OSD config dir is at /var/lib/ceph/osd/ceph- + osdConfigMountPath := activateOSDMountPath + osdID + + if osdProps.onPVC() { + // assumption: bridge vol will already be added to pod spec for PVC OSDs + volMounts = append(volMounts, getPvcOSDBridgeMountActivate(osdConfigMountPath, osdProps.pvc.ClaimName)) + } else { + // assumption: activate OSD volume will already be added to pod spec for non-PVC OSDs at the + // correct location for mounting to /var/lib/ceph/osd/ceph- + volMounts = append(volMounts, + v1.VolumeMount{Name: activateOSDVolumeName, MountPath: osdConfigMountPath}, + ) + } + + keyUpdateScript := ` +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +OSD_ID="` + osdID + `" +KEYRING_FILE=/var/lib/ceph/osd/ceph-"${OSD_ID}"/keyring + +ceph --name client.admin auth get-or-create osd."${OSD_ID}" \ + mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' \ + --keyring /etc/ceph/admin-keyring-store/keyring > "$KEYRING_FILE" +` + + envVars := []v1.EnvVar{ + { + Name: "ROOK_CEPH_MON_HOST", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "rook-ceph-config", + }, + Key: "mon_host", + }, + }, + }, + {Name: "CEPH_ARGS", Value: "--mon-host=$(ROOK_CEPH_MON_HOST)"}, + } + + container := v1.Container{ + Command: []string{ + "/bin/bash", + "-c", + keyUpdateScript, + }, + Name: "cephx-keyring-update", + Image: c.spec.CephVersion.Image, + ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), + VolumeMounts: volMounts, + SecurityContext: controller.PrivilegedContext(true), + Env: envVars, + EnvFrom: getEnvFromSources(), + Resources: osdProps.resources, + } + + return adminKeyringVol, container +} + func (c *Cluster) getOSDContainerPorts() []v1.ContainerPort { var ports []v1.ContainerPort if c.spec.RequireMsgr2() { @@ -1495,3 +1546,12 @@ func getOSDCmd(cmd []string, interval int) []string { } return cmd } + +func volumeExistsWithName(vols []v1.Volume, name string) bool { + for _, v := range vols { + if v.Name == name { + return true + } + } + return false +} diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index f9504af4c..2979220c2 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -168,7 +168,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, c.clusterInfo.Namespace, deployment.Spec.Template.ObjectMeta.Labels["rook_cluster"]) assert.Equal(t, 1, len(deployment.Spec.Template.ObjectMeta.Annotations)) - assert.Equal(t, 3, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 4, len(deployment.Spec.Template.Spec.InitContainers)) initCont := deployment.Spec.Template.Spec.InitContainers[0] assert.Equal(t, "quay.io/ceph/ceph:v15", initCont.Image) assert.Equal(t, "activate", initCont.Name) @@ -179,6 +179,10 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 2, len(initCont.VolumeMounts)) initCont = deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, "quay.io/ceph/ceph:v15", initCont.Image) + assert.Equal(t, "cephx-keyring-update", initCont.Name) + assert.Equal(t, 3, len(initCont.VolumeMounts)) + initCont = deployment.Spec.Template.Spec.InitContainers[3] + assert.Equal(t, "quay.io/ceph/ceph:v15", initCont.Image) assert.Equal(t, "chown-container-data-dir", initCont.Name) assert.Equal(t, 8, len(initCont.VolumeMounts)) @@ -206,11 +210,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 4, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "config-init", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "copy-bins", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[2].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) initCont = deployment.Spec.Template.Spec.InitContainers[0] assert.Equal(t, 5, len(initCont.VolumeMounts), initCont.VolumeMounts) @@ -227,11 +232,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 4, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[2].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) @@ -241,7 +247,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 8, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, 9, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper-encryption", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -251,12 +257,13 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "dev-mapper", deployment.Spec.Template.Spec.InitContainers[4].VolumeMounts[1].Name, deployment.Spec.Template.Spec.InitContainers[4].VolumeMounts) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[6].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[7].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[7].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[8].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) osdProp.encrypted = false - assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test OSD on PVC with RAW and metadata device osd = &OSDInfo{ @@ -267,12 +274,13 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 6, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[3].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[4].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) @@ -280,7 +288,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont := deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) - assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test encrypted OSD on PVC with RAW and metadata device osd = &OSDInfo{ @@ -292,7 +300,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 11, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 12, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -303,7 +311,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[7].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[8].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[9].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[10].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[10].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[11].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) @@ -312,7 +321,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[8] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) osdProp.encrypted = false - assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 13, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test OSD on PVC with RAW / metadata and wal device osd = &OSDInfo{ @@ -324,13 +333,14 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 6, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 7, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper-wal", deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[3].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[4].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[5].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[5].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[6].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) @@ -338,7 +348,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) - assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 13, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test encrypted OSD on PVC with RAW / metadata and wal device osd = &OSDInfo{ @@ -351,7 +361,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 14, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 15, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper-wal", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -365,7 +375,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[10].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[11].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[12].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[13].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[13].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[14].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) @@ -373,7 +384,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[11] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) - assert.Equal(t, 14, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 15, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // Test with encrypted OSD on PVC with RAW with KMS osdProp.encrypted = true @@ -384,7 +395,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 9, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) + assert.Equal(t, 10, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "encryption-kms-get-kek", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -393,11 +404,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[6].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[7].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[9].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) - assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container + assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container // Test with encrypted OSD on PVC with RAW with KMS osdProp.encrypted = true @@ -408,7 +420,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 9, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) + assert.Equal(t, 10, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "encryption-kms-get-kek", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -417,11 +429,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[6].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[7].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[9].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) - assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // Test with encrypted OSD on PVC with RAW with KMS with TLS osdProp.encrypted = true @@ -432,7 +445,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container + assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container assert.Equal(t, 3, len(deployment.Spec.Template.Spec.Volumes[8].VolumeSource.Projected.Sources), deployment.Spec.Template.Spec.Volumes[0]) // 3 more since we have the tls secrets osdProp.encrypted = false