ceph: auto detect vault k/v version

Rook will now auto detect the kv version of the vault server. This
allows users not having to pass the VAULT_BACKEND configuration in the
CephCluster CR.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2021-07-29 10:16:31 +02:00
1 parent c2a551123f
commit 99e00dea1e
8 files changed
+217 -66

No files matched your search

@@ -580,7 +580,6 @@ jobs:
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec.yaml
sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml
sed -i 's/VAULT_BACKEND: v1/VAULT_BACKEND: v2/g' tests/manifests/test-object.yaml
kubectl create -f tests/manifests/test-object.yaml
kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
+43 -15
View File
@@ -24,6 +24,7 @@ import (
"github.com/coreos/pkg/capnslog"
"github.com/hashicorp/vault/api"
"github.com/libopenstorage/secrets"
"github.com/libopenstorage/secrets/vault"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
@@ -155,21 +156,8 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec c
return errors.New("failed to validate kms configuration (missing token in spec)")
}
// Lookup mandatory connection details
for _, config := range kmsMandatoryConnectionDetails {
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
}
}
// Validate KMS provider connection details
switch GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider) {
case "vault":
err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to validate vault connection details")
}
}
// KMS provider must be specified
provider := GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider)
// Validate potential token Secret presence
if securitySpec.KeyManagementService.IsTokenAuthEnabled() {
@@ -183,6 +171,46 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec c
if !ok || len(token) == 0 {
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName)
}
switch provider {
case "vault":
// Set the env variable
err = os.Setenv(api.EnvVaultToken, string(token))
if err != nil {
return errors.Wrap(err, "failed to set vault kms token to an env var")
}
}
}
// Lookup mandatory connection details
for _, config := range kmsMandatoryConnectionDetails {
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
}
}
// Validate KMS provider connection details
switch provider {
case "vault":
err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to validate vault connection details")
}
secretEngine := securitySpec.KeyManagementService.ConnectionDetails[VaultSecretEngineKey]
switch secretEngine {
case VaultKVSecretEngineKey:
// Append Backend Version if not already present
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, vault.VaultBackendKey) == "" {
backendVersion, err := BackendVersion(securitySpec.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to get backend version")
}
securitySpec.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = backendVersion
}
}
default:
return errors.Errorf("failed to validate kms provider connection details (provider %q not supported)", provider)
}
return nil
+12 -15
View File
@@ -43,20 +43,6 @@ func TestValidateConnectionDetails(t *testing.T) {
securitySpec.KeyManagementService.TokenSecretName = "vault-token"
// Error: Data is present but no provider
securitySpec.KeyManagementService.ConnectionDetails = map[string]string{"foo": "bar"}
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty")
// Error: Data has a KMS_PROVIDER but missing details
securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault"
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"")
// Error: connection details are correct but the token secret does not exist
securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200"
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found")
@@ -87,7 +73,18 @@ func TestValidateConnectionDetails(t *testing.T) {
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.NoError(t, err, "")
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty")
securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault"
// Error: Data has a KMS_PROVIDER but missing details
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"")
// Error: connection details are correct but the token secret does not exist
securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200"
securitySpec.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v1"
// Error: TLS is configured but secrets do not exist
securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret"
+120
View File
@@ -0,0 +1,120 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kms
import (
"os"
"strings"
"github.com/libopenstorage/secrets/vault"
"github.com/libopenstorage/secrets/vault/utils"
"github.com/pkg/errors"
"github.com/hashicorp/vault/api"
)
const (
kvVersionKey = "version"
kvVersion1 = "kv"
kvVersion2 = "kv-v2"
)
// newVaultClient returns a vault client, there is no need for any secretConfig validation
// Since this is called after an already validated call InitVault()
func newVaultClient(secretConfig map[string]string) (*api.Client, error) {
// DefaultConfig uses the environment variables if present.
config := api.DefaultConfig()
// Convert map string to map interface
c := make(map[string]interface{})
for k, v := range secretConfig {
c[k] = v
}
// Configure TLS
if err := utils.ConfigureTLS(config, c); err != nil {
return nil, err
}
// Initialize the vault client
client, err := api.NewClient(config)
if err != nil {
return nil, err
}
// Set the token if provided, token should be set by ValidateConnectionDetails() if applicable
// api.NewClient() already looks up the token from the environment but we need to set it here and remove potential malformed tokens
client.SetToken(strings.TrimSuffix(os.Getenv(api.EnvVaultToken), "\n"))
// Set Vault address, was validated by ValidateConnectionDetails()
err = client.SetAddress(strings.TrimSuffix(secretConfig[api.EnvVaultAddress], "\n"))
if err != nil {
return nil, err
}
return client, nil
}
func BackendVersion(secretConfig map[string]string) (string, error) {
v1 := "v1"
v2 := "v2"
backendPath := GetParam(secretConfig, vault.VaultBackendPathKey)
if backendPath == "" {
backendPath = vault.DefaultBackendPath
}
backend := GetParam(secretConfig, vault.VaultBackendKey)
switch backend {
case kvVersion1, v1:
logger.Info("vault kv secret engine version set to v1")
return v1, nil
case kvVersion2, v2:
logger.Info("vault kv secret engine version set to v2")
return v2, nil
default:
// Initialize Vault client
vaultClient, err := newVaultClient(secretConfig)
if err != nil {
return "", errors.Wrap(err, "failed to initialize vault client")
}
mounts, err := vaultClient.Sys().ListMounts()
if err != nil {
return "", errors.Wrap(err, "failed to list vault system mounts")
}
for path, mount := range mounts {
// path is represented as 'path/'
if trimSlash(path) == trimSlash(backendPath) {
version := mount.Options[kvVersionKey]
if version == "2" {
logger.Info("vault kv secret engine version auto-detected to v2")
return v2, nil
}
logger.Info("vault kv secret engine version auto-detected to v1")
return v1, nil
}
}
}
return "", errors.Errorf("secrets engine with mount path %q not found", backendPath)
}
func trimSlash(in string) string {
return strings.Trim(in, "/")
}
+11 -4
View File
@@ -244,7 +244,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
}
kmsEnabled, err := c.CheckRGWKMS()
if err != nil {
logger.Errorf("enabling KMS failed %v", err)
logger.Errorf("failed to enable KMS. %v", err)
return v1.Container{}
}
if kmsEnabled {
@@ -444,13 +444,19 @@ func (c *clusterConfig) CheckRGWKMS() (bool, error) {
return false, err
}
secretEngine := c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]
kvVers := c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey]
// currently RGW supports kv(version 2) and transit secret engines in vault
switch secretEngine {
case kms.VaultKVSecretEngineKey:
if kvVers != "v2" {
return false, errors.New("failed to validate vault kv version, only v2 is supported")
kvVers := c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey]
if kvVers != "" {
if kvVers != "v2" {
return false, errors.New("failed to validate vault kv version, only v2 is supported")
}
} else {
// If VAUL_BACKEND is not specified let's assume it's v2
logger.Warningf("%s is not set, assuming the only supported version 2", vault.VaultBackendKey)
c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = "v2"
}
return true, nil
case kms.VaultTransitSecretEngineKey:
@@ -460,6 +466,7 @@ func (c *clusterConfig) CheckRGWKMS() (bool, error) {
}
}
return false, nil
}
+1
View File
@@ -344,6 +344,7 @@ func TestCheckRGWKMS(t *testing.T) {
// kv engine version v1, will fail
c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "kv"
c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v1"
b, err = c.CheckRGWKMS()
assert.False(t, b)
assert.Error(t, err)
-1
View File
@@ -6,6 +6,5 @@ spec:
VAULT_ADDR: https://vault.default.svc.cluster.local:8200
VAULT_BACKEND_PATH: rook/ver1
VAULT_SECRET_ENGINE: kv
VAULT_BACKEND: v1
VAULT_SKIP_VERIFY: "true"
tokenSecretName: rook-vault-token
+30 -30
View File
@@ -26,13 +26,13 @@ function install_helm {
}
if [[ "$(uname)" == "Linux" ]]; then
sudo apt-get install jq -y
install_helm
sudo apt-get install jq -y
install_helm
fi
function generate_vault_tls_config {
openssl genrsa -out "${TMPDIR}"/vault.key 2048
cat <<EOF >"${TMPDIR}"/csr.conf
[req]
req_extensions = v3_req
@@ -50,11 +50,11 @@ DNS.3 = ${SERVICE}.${NAMESPACE}.svc
DNS.4 = ${SERVICE}.${NAMESPACE}.svc.cluster.local
IP.1 = 127.0.0.1
EOF
openssl req -new -key "${TMPDIR}"/vault.key -subj "/CN=${SERVICE}.${NAMESPACE}.svc" -out "${TMPDIR}"/server.csr -config "${TMPDIR}"/csr.conf
export CSR_NAME=vault-csr
cat <<EOF >"${TMPDIR}"/csr.yaml
apiVersion: certificates.k8s.io/v1beta1
kind: CertificateSigningRequest
@@ -69,20 +69,20 @@ spec:
- key encipherment
- server auth
EOF
kubectl create -f "${TMPDIR}/"csr.yaml
kubectl certificate approve ${CSR_NAME}
serverCert=$(kubectl get csr ${CSR_NAME} -o jsonpath='{.status.certificate}')
echo "${serverCert}" | openssl base64 -d -A -out "${TMPDIR}"/vault.crt
kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "${TMPDIR}"/vault.ca
kubectl create secret generic ${SECRET_NAME} \
--namespace ${NAMESPACE} \
--from-file=vault.key="${TMPDIR}"/vault.key \
--from-file=vault.crt="${TMPDIR}"/vault.crt \
--from-file=vault.ca="${TMPDIR}"/vault.ca
--namespace ${NAMESPACE} \
--from-file=vault.key="${TMPDIR}"/vault.key \
--from-file=vault.crt="${TMPDIR}"/vault.crt \
--from-file=vault.ca="${TMPDIR}"/vault.ca
# for rook
kubectl create secret generic vault-ca-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.ca
kubectl create secret generic vault-client-cert --namespace ${ROOK_NAMESPACE} --from-file=cert="${TMPDIR}"/vault.crt
@@ -90,7 +90,7 @@ EOF
}
function vault_helm_tls {
cat <<EOF >"${TMPDIR}/"custom-values.yaml
global:
enabled: true
@@ -119,30 +119,30 @@ server:
path = "/vault/data"
}
EOF
}
function deploy_vault {
# TLS config
generate_vault_tls_config
vault_helm_tls
# Install Vault with Helm
helm repo add hashicorp https://helm.releases.hashicorp.com
helm install vault hashicorp/vault --values "${TMPDIR}/"custom-values.yaml
timeout 120 sh -c 'until kubectl get pods -l app.kubernetes.io/name=vault --field-selector=status.phase=Running|grep vault-0; do sleep 5; done'
# Unseal Vault
VAULT_INIT_TEMP_DIR=$(mktemp)
kubectl exec -ti vault-0 -- vault operator init -format "json" -ca-cert /vault/userconfig/vault-server-tls/vault.crt | tee -a "$VAULT_INIT_TEMP_DIR"
for i in $(seq 0 2); do
kubectl exec -ti vault-0 -- vault operator unseal -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$(jq -r ".unseal_keys_b64[$i]" "$VAULT_INIT_TEMP_DIR")"
kubectl exec -ti vault-0 -- vault operator unseal -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$(jq -r ".unseal_keys_b64[$i]" "$VAULT_INIT_TEMP_DIR")"
done
kubectl get pods -l app.kubernetes.io/name=vault
# Wait for vault to be ready once unsealed
while [[ $(kubectl get pods -l app.kubernetes.io/name=vault -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting vault to be ready" && sleep 1; done
# Configure Vault
ROOT_TOKEN=$(jq -r '.root_token' "$VAULT_INIT_TEMP_DIR")
kubectl exec -it vault-0 -- vault login -ca-cert /vault/userconfig/vault-server-tls/vault.crt "$ROOT_TOKEN"
@@ -151,7 +151,7 @@ function deploy_vault {
kubectl exec -ti vault-0 -- vault secrets enable -ca-cert /vault/userconfig/vault-server-tls/vault.crt -path=rook/ver2 kv-v2
kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1 || true # failure is expected
kubectl exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver2 || true # failure is expected
# Configure Vault Policy for Rook
echo '
path "rook/*" {
@@ -160,12 +160,12 @@ function deploy_vault {
path "sys/mounts" {
capabilities = ["read"]
}'| kubectl exec -i vault-0 -- vault policy write -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook -
# Create a token for Rook
ROOK_TOKEN=$(kubectl exec vault-0 -- vault token create -policy=rook -format json -ca-cert /vault/userconfig/vault-server-tls/vault.crt|jq -r '.auth.client_token'|base64)
# Configure cluster
sed -i "s|ROOK_TOKEN|$ROOK_TOKEN|" tests/manifests/test-kms-vault.yaml
sed -i "s|ROOK_TOKEN|${ROOK_TOKEN//[$'\t\r\n']}|" tests/manifests/test-kms-vault.yaml
}
function validate_rgw_token {
@@ -176,7 +176,7 @@ function validate_rgw_token {
RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=)
VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=)
VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE)
#fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW
FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key)
if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then
@@ -196,7 +196,7 @@ function validate_rgw_deployment {
function validate_osd_secret {
NB_OSD_PVC=$(kubectl -n rook-ceph get pvc|grep -c set1)
NB_VAULT_SECRET=$(kubectl -n default exec -ti vault-0 -- vault kv list -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver1|grep -c set1)
if [ "$NB_OSD_PVC" -ne "$NB_VAULT_SECRET" ]; then
echo "number of osd pvc is $NB_OSD_PVC and number of vault secret is $NB_VAULT_SECRET, mismatch"
exit 1
@@ -210,14 +210,14 @@ function validate_osd_secret {
case "$ACTION" in
deploy)
deploy_vault
;;
;;
validate_osd)
validate_osd_deployment
;;
;;
validate_rgw)
validate_rgw_deployment
;;
*)
echo "invalid action $ACTION" >&2
exit 1
esac
esac