From a2176e89d712a58d8c32fd53debc4f0e1fd2604e Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 14 Oct 2025 10:19:21 -0600 Subject: [PATCH] osd: allow OSD init keyring update to be best-effort When OSDs start, init containers attempt to get the latest CephX key for the OSD to allow OSDs to potentially continue working after key rotation. However, in Ceph corner cases, mons could be unavailable while OSDs could still be viable with existing on-disk keys. Since key rotation is a rare event, it's likely that the on-disk keys are still good. The worst case scenario is that key updates were needed but didn't succeed in init containers. In this case, instead of the init container failing, the OSD startup will fail instead. Thus, this should introduce no extra risk but merely shift the failure to a different container. Signed-off-by: Blaine Gardner (cherry picked from commit d9d31fe52704e3a5175700a48749f884c9834eb2) --- pkg/operator/ceph/cluster/osd/spec.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index c30a47614..2a9cee3e4 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -118,9 +118,12 @@ if [ "$ENCRYPTED" == "true" ] ; then LOCKBOX_KEYRING_FILE="$OSD_DATA_DIR"/lockbox.keyring LOCKBOX_USER=client.osd-lockbox."$OSD_UUID" - ceph --name client.admin auth get-or-create "$LOCKBOX_USER" \ - mon 'allow command "config-key get" with key="dm-crypt/osd/'$OSD_UUID'/luks"' \ - --keyring /etc/ceph/admin-keyring-store/keyring > "$LOCKBOX_KEYRING_FILE" + if ! ceph --name client.admin auth get-or-create "$LOCKBOX_USER" \ + mon 'allow command "config-key get" with key="dm-crypt/osd/'$OSD_UUID'/luks"' \ + --keyring /etc/ceph/admin-keyring-store/keyring > "$LOCKBOX_KEYRING_FILE"; then + echo "failed to get latest cephx lockbox key for OSD. continuing OSD startup using on-disk key" >/dev/stderr + # allowing OSD to attempt to start could avoid full OSD outage due to mon/system issues + fi fi # active the osd with ceph-volume @@ -1471,10 +1474,15 @@ set -o xtrace OSD_ID="` + osdID + `" KEYRING_FILE=/var/lib/ceph/osd/ceph-"${OSD_ID}"/keyring -ceph --name client.admin auth get-or-create osd."${OSD_ID}" \ - mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' \ - --keyring /etc/ceph/admin-keyring-store/keyring > "$KEYRING_FILE" +if ! ceph --name client.admin auth get-or-create osd."${OSD_ID}" \ + mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' \ + --keyring /etc/ceph/admin-keyring-store/keyring > "$KEYRING_FILE"; then + echo "failed to get latest cephx key for OSD. continuing OSD startup using on-disk key" >/dev/stderr +fi ` + // ^ (above) Continue on failure here. Getting latest key can fail due to system issues that + // blocking here could make worse. Key rotation is rare, so on-disk key is likely good. If not, + // allow main OSD process to fail with auth issue. envVars := []v1.EnvVar{ {