From 99e00dea1ed6dcf185246ef6ceac4e6a9b21ee84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Mon, 5 Jul 2021 18:15:09 +0200 Subject: [PATCH] ceph: auto detect vault k/v version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rook will now auto detect the kv version of the vault server. This allows users not having to pass the VAULT_BACKEND configuration in the CephCluster CR. Signed-off-by: Sébastien Han --- .github/workflows/canary-integration-test.yml | 1 - pkg/daemon/ceph/osd/kms/kms.go | 58 ++++++--- pkg/daemon/ceph/osd/kms/kms_test.go | 27 ++-- pkg/daemon/ceph/osd/kms/vault_api.go | 120 ++++++++++++++++++ pkg/operator/ceph/object/spec.go | 15 ++- pkg/operator/ceph/object/spec_test.go | 1 + tests/manifests/test-kms-vault-spec.yaml | 1 - tests/scripts/deploy-validate-vault.sh | 60 ++++----- 8 files changed, 217 insertions(+), 66 deletions(-) create mode 100644 pkg/daemon/ceph/osd/kms/vault_api.go diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index d63006fed..d2d89fcad 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -580,7 +580,6 @@ jobs: kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec.yaml sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml - sed -i 's/VAULT_BACKEND: v1/VAULT_BACKEND: v2/g' tests/manifests/test-object.yaml kubectl create -f tests/manifests/test-object.yaml kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go index 8730383c9..2531ba637 100644 --- a/pkg/daemon/ceph/osd/kms/kms.go +++ b/pkg/daemon/ceph/osd/kms/kms.go @@ -24,6 +24,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/hashicorp/vault/api" "github.com/libopenstorage/secrets" + "github.com/libopenstorage/secrets/vault" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" @@ -155,21 +156,8 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec c return errors.New("failed to validate kms configuration (missing token in spec)") } - // Lookup mandatory connection details - for _, config := range kmsMandatoryConnectionDetails { - if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" { - return errors.Errorf("failed to validate kms config %q. cannot be empty", config) - } - } - - // Validate KMS provider connection details - switch GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider) { - case "vault": - err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails) - if err != nil { - return errors.Wrap(err, "failed to validate vault connection details") - } - } + // KMS provider must be specified + provider := GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider) // Validate potential token Secret presence if securitySpec.KeyManagementService.IsTokenAuthEnabled() { @@ -183,6 +171,46 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec c if !ok || len(token) == 0 { return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName) } + + switch provider { + case "vault": + // Set the env variable + err = os.Setenv(api.EnvVaultToken, string(token)) + if err != nil { + return errors.Wrap(err, "failed to set vault kms token to an env var") + } + } + } + + // Lookup mandatory connection details + for _, config := range kmsMandatoryConnectionDetails { + if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" { + return errors.Errorf("failed to validate kms config %q. cannot be empty", config) + } + } + + // Validate KMS provider connection details + switch provider { + case "vault": + err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails) + if err != nil { + return errors.Wrap(err, "failed to validate vault connection details") + } + + secretEngine := securitySpec.KeyManagementService.ConnectionDetails[VaultSecretEngineKey] + switch secretEngine { + case VaultKVSecretEngineKey: + // Append Backend Version if not already present + if GetParam(securitySpec.KeyManagementService.ConnectionDetails, vault.VaultBackendKey) == "" { + backendVersion, err := BackendVersion(securitySpec.KeyManagementService.ConnectionDetails) + if err != nil { + return errors.Wrap(err, "failed to get backend version") + } + securitySpec.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = backendVersion + } + } + default: + return errors.Errorf("failed to validate kms provider connection details (provider %q not supported)", provider) } return nil diff --git a/pkg/daemon/ceph/osd/kms/kms_test.go b/pkg/daemon/ceph/osd/kms/kms_test.go index 20202b7cd..36cbd1f94 100644 --- a/pkg/daemon/ceph/osd/kms/kms_test.go +++ b/pkg/daemon/ceph/osd/kms/kms_test.go @@ -43,20 +43,6 @@ func TestValidateConnectionDetails(t *testing.T) { securitySpec.KeyManagementService.TokenSecretName = "vault-token" - // Error: Data is present but no provider - securitySpec.KeyManagementService.ConnectionDetails = map[string]string{"foo": "bar"} - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty") - - // Error: Data has a KMS_PROVIDER but missing details - securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault" - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"") - - // Error: connection details are correct but the token secret does not exist - securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200" err = ValidateConnectionDetails(context, securitySpec, ns) assert.Error(t, err, "") assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found") @@ -87,7 +73,18 @@ func TestValidateConnectionDetails(t *testing.T) { _, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{}) assert.NoError(t, err) err = ValidateConnectionDetails(context, securitySpec, ns) - assert.NoError(t, err, "") + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty") + securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault" + + // Error: Data has a KMS_PROVIDER but missing details + err = ValidateConnectionDetails(context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"") + + // Error: connection details are correct but the token secret does not exist + securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200" + securitySpec.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v1" // Error: TLS is configured but secrets do not exist securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" diff --git a/pkg/daemon/ceph/osd/kms/vault_api.go b/pkg/daemon/ceph/osd/kms/vault_api.go new file mode 100644 index 000000000..469c6e9f5 --- /dev/null +++ b/pkg/daemon/ceph/osd/kms/vault_api.go @@ -0,0 +1,120 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kms + +import ( + "os" + "strings" + + "github.com/libopenstorage/secrets/vault" + "github.com/libopenstorage/secrets/vault/utils" + "github.com/pkg/errors" + + "github.com/hashicorp/vault/api" +) + +const ( + kvVersionKey = "version" + kvVersion1 = "kv" + kvVersion2 = "kv-v2" +) + +// newVaultClient returns a vault client, there is no need for any secretConfig validation +// Since this is called after an already validated call InitVault() +func newVaultClient(secretConfig map[string]string) (*api.Client, error) { + // DefaultConfig uses the environment variables if present. + config := api.DefaultConfig() + + // Convert map string to map interface + c := make(map[string]interface{}) + for k, v := range secretConfig { + c[k] = v + } + + // Configure TLS + if err := utils.ConfigureTLS(config, c); err != nil { + return nil, err + } + + // Initialize the vault client + client, err := api.NewClient(config) + if err != nil { + return nil, err + } + + // Set the token if provided, token should be set by ValidateConnectionDetails() if applicable + // api.NewClient() already looks up the token from the environment but we need to set it here and remove potential malformed tokens + client.SetToken(strings.TrimSuffix(os.Getenv(api.EnvVaultToken), "\n")) + + // Set Vault address, was validated by ValidateConnectionDetails() + err = client.SetAddress(strings.TrimSuffix(secretConfig[api.EnvVaultAddress], "\n")) + if err != nil { + return nil, err + } + + return client, nil +} + +func BackendVersion(secretConfig map[string]string) (string, error) { + v1 := "v1" + v2 := "v2" + + backendPath := GetParam(secretConfig, vault.VaultBackendPathKey) + if backendPath == "" { + backendPath = vault.DefaultBackendPath + } + + backend := GetParam(secretConfig, vault.VaultBackendKey) + switch backend { + case kvVersion1, v1: + logger.Info("vault kv secret engine version set to v1") + return v1, nil + case kvVersion2, v2: + logger.Info("vault kv secret engine version set to v2") + return v2, nil + default: + // Initialize Vault client + vaultClient, err := newVaultClient(secretConfig) + if err != nil { + return "", errors.Wrap(err, "failed to initialize vault client") + } + + mounts, err := vaultClient.Sys().ListMounts() + if err != nil { + return "", errors.Wrap(err, "failed to list vault system mounts") + } + + for path, mount := range mounts { + // path is represented as 'path/' + if trimSlash(path) == trimSlash(backendPath) { + version := mount.Options[kvVersionKey] + if version == "2" { + logger.Info("vault kv secret engine version auto-detected to v2") + return v2, nil + } + logger.Info("vault kv secret engine version auto-detected to v1") + return v1, nil + } + } + } + + return "", errors.Errorf("secrets engine with mount path %q not found", backendPath) +} + +func trimSlash(in string) string { + return strings.Trim(in, "/") +} diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index f1f1ffa04..9e37b9150 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -244,7 +244,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { } kmsEnabled, err := c.CheckRGWKMS() if err != nil { - logger.Errorf("enabling KMS failed %v", err) + logger.Errorf("failed to enable KMS. %v", err) return v1.Container{} } if kmsEnabled { @@ -444,13 +444,19 @@ func (c *clusterConfig) CheckRGWKMS() (bool, error) { return false, err } secretEngine := c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey] - kvVers := c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] // currently RGW supports kv(version 2) and transit secret engines in vault switch secretEngine { case kms.VaultKVSecretEngineKey: - if kvVers != "v2" { - return false, errors.New("failed to validate vault kv version, only v2 is supported") + kvVers := c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] + if kvVers != "" { + if kvVers != "v2" { + return false, errors.New("failed to validate vault kv version, only v2 is supported") + } + } else { + // If VAUL_BACKEND is not specified let's assume it's v2 + logger.Warningf("%s is not set, assuming the only supported version 2", vault.VaultBackendKey) + c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = "v2" } return true, nil case kms.VaultTransitSecretEngineKey: @@ -460,6 +466,7 @@ func (c *clusterConfig) CheckRGWKMS() (bool, error) { } } + return false, nil } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 479186937..ac4bb76d7 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -344,6 +344,7 @@ func TestCheckRGWKMS(t *testing.T) { // kv engine version v1, will fail c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "kv" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v1" b, err = c.CheckRGWKMS() assert.False(t, b) assert.Error(t, err) diff --git a/tests/manifests/test-kms-vault-spec.yaml b/tests/manifests/test-kms-vault-spec.yaml index 1fde77554..d9541f960 100644 --- a/tests/manifests/test-kms-vault-spec.yaml +++ b/tests/manifests/test-kms-vault-spec.yaml @@ -6,6 +6,5 @@ spec: VAULT_ADDR: https://vault.default.svc.cluster.local:8200 VAULT_BACKEND_PATH: rook/ver1 VAULT_SECRET_ENGINE: kv - VAULT_BACKEND: v1 VAULT_SKIP_VERIFY: "true" tokenSecretName: rook-vault-token diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 4c769c189..fc3809652 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -26,13 +26,13 @@ function install_helm { } if [[ "$(uname)" == "Linux" ]]; then - sudo apt-get install jq -y - install_helm + sudo apt-get install jq -y + install_helm fi function generate_vault_tls_config { openssl genrsa -out "${TMPDIR}"/vault.key 2048 - + cat <"${TMPDIR}"/csr.conf [req] req_extensions = v3_req @@ -50,11 +50,11 @@ DNS.3 = ${SERVICE}.${NAMESPACE}.svc DNS.4 = ${SERVICE}.${NAMESPACE}.svc.cluster.local IP.1 = 127.0.0.1 EOF - + openssl req -new -key "${TMPDIR}"/vault.key -subj "/CN=${SERVICE}.${NAMESPACE}.svc" -out "${TMPDIR}"/server.csr -config "${TMPDIR}"/csr.conf - + export CSR_NAME=vault-csr - + cat <"${TMPDIR}"/csr.yaml apiVersion: certificates.k8s.io/v1beta1 kind: CertificateSigningRequest @@ -69,20 +69,20 @@ spec: - key encipherment - server auth EOF - + kubectl create -f "${TMPDIR}/"csr.yaml - + kubectl certificate approve ${CSR_NAME} - + serverCert=$(kubectl get csr ${CSR_NAME} -o jsonpath='{.status.certificate}') echo "${serverCert}" | openssl base64 -d -A -out "${TMPDIR}"/vault.crt kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "${TMPDIR}"/vault.ca kubectl create secret generic ${SECRET_NAME} \ - --namespace ${NAMESPACE} \ - --from-file=vault.key="${TMPDIR}"/vault.key \ - --from-file=vault.crt="${TMPDIR}"/vault.crt \ - --from-file=vault.ca="${TMPDIR}"/vault.ca - + --namespace ${NAMESPACE} \ + --from-file=vault.key="${TMPDIR}"/vault.key \ + --from-file=vault.crt="${TMPDIR}"/vault.crt \ + --from-file=vault.ca="${TMPDIR}"/vault.ca + # for rook kubectl create secret generic vault-ca-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.ca kubectl create secret generic vault-client-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.crt @@ -90,7 +90,7 @@ EOF } function vault_helm_tls { - + cat <"${TMPDIR}/"custom-values.yaml global: enabled: true @@ -119,30 +119,30 @@ server: path = "/vault/data" } EOF - + } function deploy_vault { # TLS config generate_vault_tls_config vault_helm_tls - + # Install Vault with Helm helm repo add hashicorp https://helm.releases.hashicorp.com helm install vault hashicorp/vault --values "${TMPDIR}/"custom-values.yaml timeout 120 sh -c 'until kubectl get pods -l app.kubernetes.io/name=vault --field-selector=status.phase=Running|grep vault-0; do sleep 5; done' - + # Unseal Vault VAULT_INIT_TEMP_DIR=$(mktemp) kubectl exec -ti vault-0 -- vault operator init -format "json" -ca-cert /vault/userconfig/vault-server-tls/vault.crt | tee -a "$VAULT_INIT_TEMP_DIR" for i in $(seq 0 2); do - kubectl exec -ti vault-0 -- vault operator unseal -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$(jq -r ".unseal_keys_b64[$i]" "$VAULT_INIT_TEMP_DIR")" + kubectl exec -ti vault-0 -- vault operator unseal -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$(jq -r ".unseal_keys_b64[$i]" "$VAULT_INIT_TEMP_DIR")" done kubectl get pods -l app.kubernetes.io/name=vault - + # Wait for vault to be ready once unsealed while [[ $(kubectl get pods -l app.kubernetes.io/name=vault -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting vault to be ready" && sleep 1; done - + # Configure Vault ROOT_TOKEN=$(jq -r '.root_token' "$VAULT_INIT_TEMP_DIR") kubectl exec -it vault-0 -- vault login -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$ROOT_TOKEN" @@ -151,7 +151,7 @@ function deploy_vault { kubectl exec -ti vault-0 -- vault secrets enable -ca-cert /vault/userconfig/vault-server-tls/vault.crt -path=rook/ver2 kv-v2 kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1 || true # failure is expected kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver2 || true # failure is expected - + # Configure Vault Policy for Rook echo ' path "rook/*" { @@ -160,12 +160,12 @@ function deploy_vault { path "sys/mounts" { capabilities = ["read"] }'| kubectl exec -i vault-0 -- vault policy write -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook - - + # Create a token for Rook ROOK_TOKEN=$(kubectl exec vault-0 -- vault token create -policy=rook -format json -ca-cert /vault/userconfig/vault-server-tls/vault.crt|jq -r '.auth.client_token'|base64) - + # Configure cluster - sed -i "s|ROOK_TOKEN|$ROOK_TOKEN|" tests/manifests/test-kms-vault.yaml + sed -i "s|ROOK_TOKEN|${ROOK_TOKEN//[$'\t\r\n']}|" tests/manifests/test-kms-vault.yaml } function validate_rgw_token { @@ -176,7 +176,7 @@ function validate_rgw_token { RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) - + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then @@ -196,7 +196,7 @@ function validate_rgw_deployment { function validate_osd_secret { NB_OSD_PVC=$(kubectl -n rook-ceph get pvc|grep -c set1) NB_VAULT_SECRET=$(kubectl -n default exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1|grep -c set1) - + if [ "$NB_OSD_PVC" -ne "$NB_VAULT_SECRET" ]; then echo "number of osd pvc is $NB_OSD_PVC and number of vault secret is $NB_VAULT_SECRET, mismatch" exit 1 @@ -210,14 +210,14 @@ function validate_osd_secret { case "$ACTION" in deploy) deploy_vault - ;; + ;; validate_osd) validate_osd_deployment - ;; + ;; validate_rgw) validate_rgw_deployment ;; *) echo "invalid action $ACTION" >&2 exit 1 - esac +esac