From a467f57350cc81d83b149a0c78b506ae5f6bc866 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Mon, 23 Sep 2024 11:13:45 +0530 Subject: [PATCH 1/3] security: key rotation support for vault kms Signed-off-by: Praveen M (cherry picked from commit d8c0242c29eb0d8bb7f8ff6c5f36525a830e05b3) --- pkg/daemon/ceph/osd/kms/kms.go | 19 +++++++++++++++++++ pkg/operator/ceph/cluster/osd/key_rotation.go | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go index de555e425..24763f002 100644 --- a/pkg/daemon/ceph/osd/kms/kms.go +++ b/pkg/daemon/ceph/osd/kms/kms.go @@ -238,6 +238,25 @@ func (c *Config) UpdateSecret(secretName, secretValue string) error { return nil } + if c.IsVault() { + // Store the secret in Vault + v, err := InitVault(c.ClusterInfo.Context, c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + if err != nil { + return errors.Wrap(err, "failed to init vault kms") + } + k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + // Build Secret + secretName = GenerateOSDEncryptionSecretName(secretName) + data := make(map[string]interface{}) + data[secretName] = secretValue + + _, err = v.PutSecret(secretName, data, k) + if err != nil { + return errors.Wrapf(err, "failed to put secret %q in vault kms", secretName) + } + + return nil + } return errors.Errorf("update secret is not supported for the %q KMS", c.Provider) } diff --git a/pkg/operator/ceph/cluster/osd/key_rotation.go b/pkg/operator/ceph/cluster/osd/key_rotation.go index 6401a8eb4..ae10ca7b6 100644 --- a/pkg/operator/ceph/cluster/osd/key_rotation.go +++ b/pkg/operator/ceph/cluster/osd/key_rotation.go @@ -137,6 +137,12 @@ func (c *Cluster) getKeyRotationPodTemplateSpec(osdProps osdProperties, osd OSDI devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreWalName)) } + if c.spec.Security.KeyManagementService.IsVaultKMS() { + volumeTLS, volumeMountTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") + volumes = append(volumes, volumeTLS) + volumeMounts = append(volumeMounts, volumeMountTLS) + } + keyRotationContainer, err := c.getKeyRotationContainer(osdProps, volumeMounts, devices) if err != nil { return nil, errors.Wrap(err, "failed to generate key rotation container") From ca9e893bcbd963e0bb3bdacad9b94a6f8a6a7baa Mon Sep 17 00:00:00 2001 From: Praveen M Date: Tue, 8 Oct 2024 16:44:58 +0530 Subject: [PATCH 2/3] docs: update docs for vault KMS key rotation support Signed-off-by: Praveen M (cherry picked from commit 01ccb5e20a01fe0ff818ab169deaa2217df418e8) --- .../Storage-Configuration/Advanced/key-management-system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Storage-Configuration/Advanced/key-management-system.md b/Documentation/Storage-Configuration/Advanced/key-management-system.md index 841beb39b..098ebcc28 100644 --- a/Documentation/Storage-Configuration/Advanced/key-management-system.md +++ b/Documentation/Storage-Configuration/Advanced/key-management-system.md @@ -21,7 +21,7 @@ The `security` section contains settings related to encryption of the cluster. * `schedule`: the schedule, written in [cron format](https://en.wikipedia.org/wiki/Cron), with which key rotation [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) is created, default value is `"@weekly"`. !!! note - Currently key rotation is only supported for the default type, where the Key Encryption Keys are stored in a Kubernetes Secret. + Currently key rotation is supported when the Key Encryption Keys are stored in a Kubernetes Secret or Vault KMS. Supported KMS providers: From 794b5655edab78decdf8e0c109f95404c1a19a84 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Thu, 10 Oct 2024 10:43:06 +0530 Subject: [PATCH 3/3] ci: add test for vault key rotation Signed-off-by: Praveen M (cherry picked from commit c9bc3a2685822a342bf390a5f490e0ee35661d60) --- .github/workflows/canary-integration-test.yml | 8 +++++ tests/scripts/deploy-validate-vault.sh | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index e8e5965d1..49f4902c9 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1085,6 +1085,7 @@ jobs: yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 kubectl create -f tests/manifests/test-object.yaml @@ -1105,6 +1106,9 @@ jobs: tests/scripts/deploy-validate-vault.sh validate_osd sudo lsblk + - name: wait and verify key rotation + run: tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1 + - name: validate rgw vault kv run: | tests/scripts/deploy-validate-vault.sh validate_rgw @@ -1170,6 +1174,7 @@ jobs: yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - name: wait for prepare pod @@ -1186,6 +1191,9 @@ jobs: tests/scripts/deploy-validate-vault.sh validate_osd sudo lsblk + - name: wait and verify key rotation + run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1 + - name: collect common logs if: always() uses: ./.github/workflows/collect-logs diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index df793a07e..b0003c0e8 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -276,6 +276,32 @@ function validate_osd_secret { fi } +function validate_key_rotation() { + local backend_path=$1 + pvc_name=$(kubectl get pvc -n rook-ceph -l ceph.rook.io/setIndex=0 -o jsonpath='{.items[0].metadata.name}') + key_name="rook-ceph-osd-encryption-key-$pvc_name" + cmd="vault kv get -format=json $backend_path/$key_name" + old_key=$(kubectl exec vault-0 -- sh -c "$cmd" | jq -r ".data.\"$key_name\"") + local new_key + runtime=180 + endtime=$((SECONDS + runtime)) + while [ $SECONDS -le $endtime ]; do + echo "Time Now: $(date +%H:%M:%S)" + new_key=$(kubectl exec vault-0 -- sh -c "$cmd" | jq -r ".data.\"$key_name\"") + + if [ "$old_key" != "$new_key" ]; then + echo "encryption passphrase is successfully rotated" + exit 0 + fi + + echo "encryption passphrase is not rotated, sleeping for 10 seconds" + sleep 10 + done + + echo "encryption passphrase is not rotated" + exit 1 +} + ######## # MAIN # ######## @@ -290,6 +316,9 @@ validate_osd) validate_rgw) validate_rgw_deployment ;; +validate_key_rotation) + validate_key_rotation "$2" + ;; *) echo "invalid action $ACTION" >&2 exit 1