diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index e8e5965d1..49f4902c9 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1085,6 +1085,7 @@ jobs: yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 kubectl create -f tests/manifests/test-object.yaml @@ -1105,6 +1106,9 @@ jobs: tests/scripts/deploy-validate-vault.sh validate_osd sudo lsblk + - name: wait and verify key rotation + run: tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1 + - name: validate rgw vault kv run: | tests/scripts/deploy-validate-vault.sh validate_rgw @@ -1170,6 +1174,7 @@ jobs: yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml - name: wait for prepare pod @@ -1186,6 +1191,9 @@ jobs: tests/scripts/deploy-validate-vault.sh validate_osd sudo lsblk + - name: wait and verify key rotation + run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh validate_key_rotation rook/ver1 + - name: collect common logs if: always() uses: ./.github/workflows/collect-logs diff --git a/Documentation/Storage-Configuration/Advanced/key-management-system.md b/Documentation/Storage-Configuration/Advanced/key-management-system.md index 841beb39b..098ebcc28 100644 --- a/Documentation/Storage-Configuration/Advanced/key-management-system.md +++ b/Documentation/Storage-Configuration/Advanced/key-management-system.md @@ -21,7 +21,7 @@ The `security` section contains settings related to encryption of the cluster. * `schedule`: the schedule, written in [cron format](https://en.wikipedia.org/wiki/Cron), with which key rotation [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) is created, default value is `"@weekly"`. !!! note - Currently key rotation is only supported for the default type, where the Key Encryption Keys are stored in a Kubernetes Secret. + Currently key rotation is supported when the Key Encryption Keys are stored in a Kubernetes Secret or Vault KMS. Supported KMS providers: diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go index de555e425..24763f002 100644 --- a/pkg/daemon/ceph/osd/kms/kms.go +++ b/pkg/daemon/ceph/osd/kms/kms.go @@ -238,6 +238,25 @@ func (c *Config) UpdateSecret(secretName, secretValue string) error { return nil } + if c.IsVault() { + // Store the secret in Vault + v, err := InitVault(c.ClusterInfo.Context, c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + if err != nil { + return errors.Wrap(err, "failed to init vault kms") + } + k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + // Build Secret + secretName = GenerateOSDEncryptionSecretName(secretName) + data := make(map[string]interface{}) + data[secretName] = secretValue + + _, err = v.PutSecret(secretName, data, k) + if err != nil { + return errors.Wrapf(err, "failed to put secret %q in vault kms", secretName) + } + + return nil + } return errors.Errorf("update secret is not supported for the %q KMS", c.Provider) } diff --git a/pkg/operator/ceph/cluster/osd/key_rotation.go b/pkg/operator/ceph/cluster/osd/key_rotation.go index 6401a8eb4..ae10ca7b6 100644 --- a/pkg/operator/ceph/cluster/osd/key_rotation.go +++ b/pkg/operator/ceph/cluster/osd/key_rotation.go @@ -137,6 +137,12 @@ func (c *Cluster) getKeyRotationPodTemplateSpec(osdProps osdProperties, osd OSDI devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreWalName)) } + if c.spec.Security.KeyManagementService.IsVaultKMS() { + volumeTLS, volumeMountTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") + volumes = append(volumes, volumeTLS) + volumeMounts = append(volumeMounts, volumeMountTLS) + } + keyRotationContainer, err := c.getKeyRotationContainer(osdProps, volumeMounts, devices) if err != nil { return nil, errors.Wrap(err, "failed to generate key rotation container") diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index df793a07e..b0003c0e8 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -276,6 +276,32 @@ function validate_osd_secret { fi } +function validate_key_rotation() { + local backend_path=$1 + pvc_name=$(kubectl get pvc -n rook-ceph -l ceph.rook.io/setIndex=0 -o jsonpath='{.items[0].metadata.name}') + key_name="rook-ceph-osd-encryption-key-$pvc_name" + cmd="vault kv get -format=json $backend_path/$key_name" + old_key=$(kubectl exec vault-0 -- sh -c "$cmd" | jq -r ".data.\"$key_name\"") + local new_key + runtime=180 + endtime=$((SECONDS + runtime)) + while [ $SECONDS -le $endtime ]; do + echo "Time Now: $(date +%H:%M:%S)" + new_key=$(kubectl exec vault-0 -- sh -c "$cmd" | jq -r ".data.\"$key_name\"") + + if [ "$old_key" != "$new_key" ]; then + echo "encryption passphrase is successfully rotated" + exit 0 + fi + + echo "encryption passphrase is not rotated, sleeping for 10 seconds" + sleep 10 + done + + echo "encryption passphrase is not rotated" + exit 1 +} + ######## # MAIN # ######## @@ -290,6 +316,9 @@ validate_osd) validate_rgw) validate_rgw_deployment ;; +validate_key_rotation) + validate_key_rotation "$2" + ;; *) echo "invalid action $ACTION" >&2 exit 1