diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 0fdd7f040..e484a9df6 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -111,39 +111,6 @@ KEYRING_FILE="$OSD_DATA_DIR"/keyring CV_MODE=%s DEVICE="$%s" -# In rare cases keyring file created with prepare-osd but did not -# being stored in ceph auth system therefore we need to import it -# from keyring file instead of creating new one -if ! ceph -n client.admin auth get osd."$OSD_ID" -k /etc/ceph/admin-keyring-store/keyring; then - if [ -f "$KEYRING_FILE" ]; then - # import keyring from existing file - TMP_DIR=$(mktemp -d) - - python3 -c " -import configparser - -config = configparser.ConfigParser() -config.read('$KEYRING_FILE') - -if not config.has_section('osd.$OSD_ID'): - exit() - -config['osd.$OSD_ID'] = {'key': config['osd.$OSD_ID']['key'], 'caps mon': '\"allow profile osd\"', 'caps mgr': '\"allow profile osd\"', 'caps osd': '\"allow *\"'} - -with open('$TMP_DIR/keyring', 'w') as configfile: - config.write(configfile) -" - - cat "$TMP_DIR"/keyring - ceph -n client.admin auth import -i "$TMP_DIR"/keyring -k /etc/ceph/admin-keyring-store/keyring - - rm --recursive --force "$TMP_DIR" - else - # create new keyring if no keyring file found - ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring - fi -fi - # active the osd with ceph-volume if [[ "$CV_MODE" == "lvm" ]]; then TMP_DIR=$(mktemp -d) @@ -651,6 +618,13 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd)) } + // key update init container must go after activate + adminKeyVol, cephxContainer := c.getCephxKeyUpdateInitContainer(osdID, osdProps) + if !volumeExistsWithName(volumes, adminKeyVol.Name) { + volumes = append(volumes, adminKeyVol) + } + initContainers = append(initContainers, cephxContainer) + // Doing a chown in a post start lifecycle hook does not reliably complete before the OSD // process starts, which can cause the pod to fail without the lifecycle hook's chown command // completing. It can take an arbitrarily long time for a pod restart to successfully chown the @@ -1430,6 +1404,83 @@ func (c *Cluster) getEncryptedStatusPVCInitContainer(mountPath string, osdProps } } +// init container to get (or create, if needed) the most up-to-date keyring for the given OSD and +// update the OSD config directory keyring file. this allows OSD cephx keys to be rotated without +// causing OSD pod crash loops. +// +// this must come after OSD activate init container because 'ceph-bluestore-tool' (and consequently, +// 'ceph-volume activate') pulls the keyring from the OSD disk's bluestore data and overwrites the +// config dir's keyring file, even when the on-disk key is not the most up-to-date. +func (c *Cluster) getCephxKeyUpdateInitContainer(osdID string, osdProps osdProperties) (v1.Volume, v1.Container) { + volMounts := []v1.VolumeMount{ + {Name: k8sutil.ConfigOverrideName, ReadOnly: true, MountPath: opconfig.EtcCephDir}, + } + + adminKeyringVol, adminKeyringVolMount := cephkey.Volume().Admin(), cephkey.VolumeMount().Admin() + volMounts = append(volMounts, adminKeyringVolMount) + + // OSD config dir is at /var/lib/ceph/osd/ceph- + osdConfigMountPath := activateOSDMountPath + osdID + + if osdProps.onPVC() { + // assumption: bridge vol will already be added to pod spec for PVC OSDs + volMounts = append(volMounts, getPvcOSDBridgeMountActivate(osdConfigMountPath, osdProps.pvc.ClaimName)) + } else { + // assumption: activate OSD volume will already be added to pod spec for non-PVC OSDs at the + // correct location for mounting to /var/lib/ceph/osd/ceph- + volMounts = append(volMounts, + v1.VolumeMount{Name: activateOSDVolumeName, MountPath: osdConfigMountPath}, + ) + } + + keyUpdateScript := ` +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +OSD_ID="` + osdID + `" +KEYRING_FILE=/var/lib/ceph/osd/ceph-"${OSD_ID}"/keyring + +ceph --name client.admin auth get-or-create osd."${OSD_ID}" \ + mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' \ + --keyring /etc/ceph/admin-keyring-store/keyring > "$KEYRING_FILE" +` + + envVars := []v1.EnvVar{ + { + Name: "ROOK_CEPH_MON_HOST", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "rook-ceph-config", + }, + Key: "mon_host", + }, + }, + }, + {Name: "CEPH_ARGS", Value: "--mon-host=$(ROOK_CEPH_MON_HOST)"}, + } + + container := v1.Container{ + Command: []string{ + "/bin/bash", + "-c", + keyUpdateScript, + }, + Name: "cephx-keyring-update", + Image: c.spec.CephVersion.Image, + ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), + VolumeMounts: volMounts, + SecurityContext: controller.PrivilegedContext(true), + Env: envVars, + EnvFrom: getEnvFromSources(), + Resources: osdProps.resources, + } + + return adminKeyringVol, container +} + func (c *Cluster) getOSDContainerPorts() []v1.ContainerPort { var ports []v1.ContainerPort if c.spec.RequireMsgr2() { @@ -1495,3 +1546,12 @@ func getOSDCmd(cmd []string, interval int) []string { } return cmd } + +func volumeExistsWithName(vols []v1.Volume, name string) bool { + for _, v := range vols { + if v.Name == name { + return true + } + } + return false +} diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index f9504af4c..2979220c2 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -168,7 +168,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, c.clusterInfo.Namespace, deployment.Spec.Template.ObjectMeta.Labels["rook_cluster"]) assert.Equal(t, 1, len(deployment.Spec.Template.ObjectMeta.Annotations)) - assert.Equal(t, 3, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 4, len(deployment.Spec.Template.Spec.InitContainers)) initCont := deployment.Spec.Template.Spec.InitContainers[0] assert.Equal(t, "quay.io/ceph/ceph:v15", initCont.Image) assert.Equal(t, "activate", initCont.Name) @@ -179,6 +179,10 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 2, len(initCont.VolumeMounts)) initCont = deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, "quay.io/ceph/ceph:v15", initCont.Image) + assert.Equal(t, "cephx-keyring-update", initCont.Name) + assert.Equal(t, 3, len(initCont.VolumeMounts)) + initCont = deployment.Spec.Template.Spec.InitContainers[3] + assert.Equal(t, "quay.io/ceph/ceph:v15", initCont.Image) assert.Equal(t, "chown-container-data-dir", initCont.Name) assert.Equal(t, 8, len(initCont.VolumeMounts)) @@ -206,11 +210,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 4, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "config-init", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "copy-bins", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[2].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) initCont = deployment.Spec.Template.Spec.InitContainers[0] assert.Equal(t, 5, len(initCont.VolumeMounts), initCont.VolumeMounts) @@ -227,11 +232,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 4, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[2].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[3].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) @@ -241,7 +247,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 8, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) + assert.Equal(t, 9, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper-encryption", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -251,12 +257,13 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "dev-mapper", deployment.Spec.Template.Spec.InitContainers[4].VolumeMounts[1].Name, deployment.Spec.Template.Spec.InitContainers[4].VolumeMounts) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[6].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[7].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[7].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[8].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) osdProp.encrypted = false - assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test OSD on PVC with RAW and metadata device osd = &OSDInfo{ @@ -267,12 +274,13 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 5, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 6, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[3].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[4].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[4].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) @@ -280,7 +288,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont := deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) - assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test encrypted OSD on PVC with RAW and metadata device osd = &OSDInfo{ @@ -292,7 +300,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 11, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 12, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -303,7 +311,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[7].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[8].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[9].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[10].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[10].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[11].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) @@ -312,7 +321,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[8] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) osdProp.encrypted = false - assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 13, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test OSD on PVC with RAW / metadata and wal device osd = &OSDInfo{ @@ -324,13 +333,14 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 6, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 7, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper-wal", deployment.Spec.Template.Spec.InitContainers[2].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[3].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[4].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[5].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[5].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[6].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 7, len(cont.VolumeMounts), cont.VolumeMounts) @@ -338,7 +348,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[2] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) - assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 13, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // // Test encrypted OSD on PVC with RAW / metadata and wal device osd = &OSDInfo{ @@ -351,7 +361,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 14, len(deployment.Spec.Template.Spec.InitContainers)) + assert.Equal(t, 15, len(deployment.Spec.Template.Spec.InitContainers)) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "blkdevmapper-metadata", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "blkdevmapper-wal", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -365,7 +375,8 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[10].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[11].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[12].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[13].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[13].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[14].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) @@ -373,7 +384,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, 1, len(blkInitCont.VolumeDevices)) blkMetaInitCont = deployment.Spec.Template.Spec.InitContainers[11] assert.Equal(t, 1, len(blkMetaInitCont.VolumeDevices)) - assert.Equal(t, 14, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 15, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // Test with encrypted OSD on PVC with RAW with KMS osdProp.encrypted = true @@ -384,7 +395,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 9, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) + assert.Equal(t, 10, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "encryption-kms-get-kek", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -393,11 +404,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[6].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[7].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[9].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) - assert.Equal(t, 10, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container + assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container // Test with encrypted OSD on PVC with RAW with KMS osdProp.encrypted = true @@ -408,7 +420,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 9, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) + assert.Equal(t, 10, len(deployment.Spec.Template.Spec.InitContainers), deployment.Spec.Template.Spec.InitContainers) assert.Equal(t, "blkdevmapper", deployment.Spec.Template.Spec.InitContainers[0].Name) assert.Equal(t, "encryption-kms-get-kek", deployment.Spec.Template.Spec.InitContainers[1].Name) assert.Equal(t, "encryption-open", deployment.Spec.Template.Spec.InitContainers[2].Name) @@ -417,11 +429,12 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { assert.Equal(t, "expand-encrypted-bluefs", deployment.Spec.Template.Spec.InitContainers[5].Name) assert.Equal(t, "activate", deployment.Spec.Template.Spec.InitContainers[6].Name) assert.Equal(t, "expand-bluefs", deployment.Spec.Template.Spec.InitContainers[7].Name) - assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "cephx-keyring-update", deployment.Spec.Template.Spec.InitContainers[8].Name) + assert.Equal(t, "chown-container-data-dir", deployment.Spec.Template.Spec.InitContainers[9].Name) assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) cont = deployment.Spec.Template.Spec.Containers[0] assert.Equal(t, 8, len(cont.VolumeMounts), cont.VolumeMounts) - assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) + assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // Test with encrypted OSD on PVC with RAW with KMS with TLS osdProp.encrypted = true @@ -432,7 +445,7 @@ func testPodDevices(t *testing.T, dataDir, deviceName string, allDevices bool) { deployment, err = c.makeDeployment(osdProp, osd, dataPathMap) assert.Nil(t, err) assert.NotNil(t, deployment) - assert.Equal(t, 11, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container + assert.Equal(t, 12, len(deployment.Spec.Template.Spec.Volumes), deployment.Spec.Template.Spec.Volumes) // One more than the encryption with k8s for the kek get init container assert.Equal(t, 3, len(deployment.Spec.Template.Spec.Volumes[8].VolumeSource.Projected.Sources), deployment.Spec.Template.Spec.Volumes[0]) // 3 more since we have the tls secrets osdProp.encrypted = false