From 16729e0e3893423d412470e4d939cc1122cb6ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 3 Nov 2021 18:20:43 +0100 Subject: [PATCH] osd: use multiple service account for vault role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can pass bound_service_account_names with a comma separated list of service accounts. Let's do this instead of remapping new values. Earlier, we thought a single service account could be added per Vault role and we were using other variables like `VAULT_AUTH_KUBERNETES_ROOK_OPERATOR_ROLE` that we were remapping to `VAULT_AUTH_KUBERNETES_ROLE` internal for the API calls to Vault. Signed-off-by: Sébastien Han --- Documentation/ceph-kms.md | 20 ++++--------- cmd/rook/secret.go | 9 ------ pkg/daemon/ceph/osd/kms/envs.go | 11 ++----- pkg/daemon/ceph/osd/kms/envs_test.go | 8 ----- pkg/daemon/ceph/osd/kms/vault.go | 29 +------------------ .../test-kms-vault-spec-k8s-auth.yaml | 3 +- tests/scripts/deploy-validate-vault.sh | 15 ++-------- 7 files changed, 12 insertions(+), 83 deletions(-) diff --git a/Documentation/ceph-kms.md b/Documentation/ceph-kms.md index eb970cdde..e1a4609ce 100644 --- a/Documentation/ceph-kms.md +++ b/Documentation/ceph-kms.md @@ -8,7 +8,7 @@ indent: true Rook has the ability to encrypt OSDs of clusters running on PVC via the flag (`encrypted: true`) in your `storageClassDeviceSets` [template](#pvc-based-cluster). By default, the Key Encryption Keys (also known as Data Encryption Keys) are stored in a Kubernetes Secret. -However, if a Key Management System exists Rook is capable of using it. +However, if a Key Management System exists Rook is capable of using it. The `security` section contains settings related to encryption of the cluster. @@ -91,8 +91,6 @@ ROOK_VAULT_SA=rook-vault-auth ROOK_SYSTEM_SA=rook-ceph-system ROOK_OSD_SA=rook-ceph-osd VAULT_POLICY_NAME=rook -VAULT_ROOK_OP_ROLE_NAME=rook-op -VAULT_ROOK_OSD_ROLE_NAME=rook-osd # create service account for vault to validate API token kubectl -n "$ROOK_NAMESPACE" create serviceaccount "$ROOK_VAULT_SA" @@ -128,16 +126,9 @@ vault write auth/kubernetes/config \ kill $proxy_pid -# configure a role for rook operator -vault write auth/kubernetes/role/"$VAULT_ROOK_OP_ROLE_NAME" \ - bound_service_account_names="$ROOK_SYSTEM_SA" \ - bound_service_account_namespaces="$ROOK_NAMESPACE" \ - policies="$VAULT_POLICY_NAME" \ - ttl=1440h - -# configure a role for rook osds -vault write auth/kubernetes/role/"$VAULT_ROOK_OSD_ROLE_NAME" \ - bound_service_account_names="$ROOK_OSD_SA" \ +# configure a role for rook +vault write auth/kubernetes/role/"$ROOK_NAMESPACE" \ + bound_service_account_names="$ROOK_SYSTEM_SA","$ROOK_OSD_SA" \ bound_service_account_namespaces="$ROOK_NAMESPACE" \ policies="$VAULT_POLICY_NAME" \ ttl=1440h @@ -154,8 +145,7 @@ security: VAULT_BACKEND_PATH: rook/ver1 VAULT_SECRET_ENGINE: kv VAULT_AUTH_METHOD: kubernetes - VAULT_AUTH_KUBERNETES_ROOK_OPERATOR_ROLE: rook-op - VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE: rook-osd + VAULT_AUTH_KUBERNETES_ROLE: rook-ceph ``` ### General Vault configuration diff --git a/cmd/rook/secret.go b/cmd/rook/secret.go index 4dbb5c510..9b975e256 100644 --- a/cmd/rook/secret.go +++ b/cmd/rook/secret.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" - "github.com/libopenstorage/secrets/vault" "github.com/pkg/errors" "github.com/rook/rook/cmd/rook/rook" "github.com/rook/rook/pkg/daemon/ceph/client" @@ -79,14 +78,6 @@ func startSecret() *kms.Config { rook.TerminateFatal(errors.Wrap(err, "failed to validate kms connection details")) } - // If Kubernetes authentication is enabled (through Service Accounts), use the role provided by - // the cluster spec - // Here we use VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE since this CLI is being called from an OSD - // init container, so the pod will use the "rook-ceph-osd" service account, different than the operator - if cephCluster.Spec.Security.KeyManagementService.IsK8sAuthEnabled() { - cephCluster.Spec.Security.KeyManagementService.ConnectionDetails[vault.AuthKubernetesRole] = cephCluster.Spec.Security.KeyManagementService.ConnectionDetails[kms.RookOSDVaultAuthKubernetesRole] - } - return kms.NewConfig(context, &cephCluster.Spec, clusterInfo) } diff --git a/pkg/daemon/ceph/osd/kms/envs.go b/pkg/daemon/ceph/osd/kms/envs.go index 5f048173a..5967869fa 100644 --- a/pkg/daemon/ceph/osd/kms/envs.go +++ b/pkg/daemon/ceph/osd/kms/envs.go @@ -73,18 +73,11 @@ func VaultConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar { } for k, v := range spec.Security.KeyManagementService.ConnectionDetails { // Skip TLS and token env var to avoid env being set multiple times - toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken, vault.AuthKubernetesRole) + toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken) if sets.NewString(toSkip...).Has(k) { continue } - // We need to apply the osd role to VAULT_AUTH_KUBERNETES_ROLE - // This is only called from the Operator but will run inside the prepare job so it's ok to - // force the vault osd role - if k == RookOSDVaultAuthKubernetesRole { - envs = append(envs, v1.EnvVar{Name: vault.AuthKubernetesRole, Value: v}) - } else { - envs = append(envs, v1.EnvVar{Name: k, Value: v}) - } + envs = append(envs, v1.EnvVar{Name: k, Value: v}) } // Add the VAULT_TOKEN diff --git a/pkg/daemon/ceph/osd/kms/envs_test.go b/pkg/daemon/ceph/osd/kms/envs_test.go index 382af396c..727b580ee 100644 --- a/pkg/daemon/ceph/osd/kms/envs_test.go +++ b/pkg/daemon/ceph/osd/kms/envs_test.go @@ -103,14 +103,6 @@ func TestVaultConfigToEnvVar(t *testing.T) { args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_BACKEND_PATH": "foo/"}}}}}, []v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "foo/"}}, }, - { - "test VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE", - args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE": "rook-osd"}}}}}, - []v1.EnvVar{ - {Name: "VAULT_AUTH_KUBERNETES_ROLE", Value: "rook-osd"}, - {Name: "VAULT_BACKEND_PATH", Value: "secret/"}, - }, - }, { "test with tls config", args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_CACERT": "my-secret-name"}}}}}, diff --git a/pkg/daemon/ceph/osd/kms/vault.go b/pkg/daemon/ceph/osd/kms/vault.go index 37392048d..182c55e1d 100644 --- a/pkg/daemon/ceph/osd/kms/vault.go +++ b/pkg/daemon/ceph/osd/kms/vault.go @@ -36,13 +36,6 @@ const ( EtcVaultDir = "/etc/vault" // VaultSecretEngineKey is the type of secret engine used (kv, transit) VaultSecretEngineKey = "VAULT_SECRET_ENGINE" - // RookOperatorVaultAuthKubernetesRole is the internal Rook mapping of the Vault role that maps - // operator's service account and namespace to the Vault policy that allows them to read the - // secrets - RookOperatorVaultAuthKubernetesRole = "VAULT_AUTH_KUBERNETES_ROOK_OPERATOR_ROLE" - // RookOSDVaultAuthKubernetesRole is the internal Rook mapping of the Vault role that maps OSD's - // service account and namespace to the Vault policy that allows them to read the secrets - RookOSDVaultAuthKubernetesRole = "VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE" // VaultKVSecretEngineKey is a kv secret engine type VaultKVSecretEngineKey = "kv" // VaultTransitSecretEngineKey is a transit secret engine type @@ -50,8 +43,7 @@ const ( ) var ( - vaultMandatoryConnectionDetails = []string{api.EnvVaultAddress} - vaultK8sAuthMethodMandatoryConnectionDetails = []string{RookOperatorVaultAuthKubernetesRole, RookOSDVaultAuthKubernetesRole} + vaultMandatoryConnectionDetails = []string{api.EnvVaultAddress} ) // Used for unit tests mocking too as well as production code @@ -258,31 +250,12 @@ func (c *Config) IsVault() bool { func validateVaultConnectionDetails(clusterdContext *clusterd.Context, ns string, kmsConfig map[string]string) error { ctx := context.TODO() - // If Kubernetes authentication is enabled, let's merge into mandatory details - isK8sAuthEnabled := GetParam(kmsConfig, vault.AuthMethod) == vault.AuthMethodKubernetes - if isK8sAuthEnabled { - vaultMandatoryConnectionDetails = append(vaultMandatoryConnectionDetails, vaultK8sAuthMethodMandatoryConnectionDetails...) - } for _, option := range vaultMandatoryConnectionDetails { if GetParam(kmsConfig, option) == "" { return errors.Errorf("failed to find connection details %q", option) } } - // When validating the connection details, we need to give a value to VAULT_AUTH_KUBERNETES_ROLE - // with the role we are currently running on. If we run from the operator we must use - // VAULT_AUTH_KUBERNETES_ROLE to VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE - // The CLI does not use the env var so VAULT_AUTH_KUBERNETES_ROLE is incorrect since it's - // setting the operator role. - if isK8sAuthEnabled { - // If ROOK_CRUSHMAP_ROOT is empty we are most likely running from the operator - if os.Getenv("ROOK_CRUSHMAP_ROOT") == "" { - kmsConfig[vault.AuthKubernetesRole] = kmsConfig[RookOperatorVaultAuthKubernetesRole] - } else { - kmsConfig[vault.AuthKubernetesRole] = kmsConfig[RookOSDVaultAuthKubernetesRole] - } - } - // We do not support a directory with multiple CA since we fetch a k8s Secret and read its content // So we operate with a single CA only if GetParam(kmsConfig, api.EnvVaultCAPath) != "" { diff --git a/tests/manifests/test-kms-vault-spec-k8s-auth.yaml b/tests/manifests/test-kms-vault-spec-k8s-auth.yaml index db911940f..a6065358a 100644 --- a/tests/manifests/test-kms-vault-spec-k8s-auth.yaml +++ b/tests/manifests/test-kms-vault-spec-k8s-auth.yaml @@ -11,5 +11,4 @@ spec: VAULT_CLIENT_CERT: "vault-client-cert" VAULT_CACERT: "vault-ca-cert" VAULT_AUTH_METHOD: kubernetes - VAULT_AUTH_KUBERNETES_ROOK_OPERATOR_ROLE: rook-op - VAULT_AUTH_KUBERNETES_ROOK_OSD_ROLE: rook-osd + VAULT_AUTH_KUBERNETES_ROLE: "rook-ceph" diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 37a3f4f32..b5c0ca861 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -29,8 +29,6 @@ ROOK_VAULT_SA=rook-vault-auth ROOK_SYSTEM_SA=rook-ceph-system ROOK_OSD_SA=rook-ceph-osd VAULT_POLICY_NAME=rook -VAULT_ROOK_OP_ROLE_NAME=rook-op -VAULT_ROOK_OSD_ROLE_NAME=rook-osd SECRET_NAME=vault-server-tls TMPDIR=$(mktemp -d) VAULT_SERVER=https://vault.default:8200 @@ -203,16 +201,9 @@ function set_up_vault_kubernetes_auth { kill $proxy_pid - # configure a role for rook operator - kubectl exec -ti vault-0 -- vault write auth/kubernetes/role/"$VAULT_ROOK_OP_ROLE_NAME" \ - bound_service_account_names="$ROOK_SYSTEM_SA" \ - bound_service_account_namespaces="$ROOK_NAMESPACE" \ - policies="$VAULT_POLICY_NAME" \ - ttl=1440h - - # configure a role for rook osds - kubectl exec -ti vault-0 -- vault write auth/kubernetes/role/"$VAULT_ROOK_OSD_ROLE_NAME" \ - bound_service_account_names="$ROOK_OSD_SA" \ + # configure a role for rook + kubectl exec -ti vault-0 -- vault write auth/kubernetes/role/"$ROOK_NAMESPACE" \ + bound_service_account_names="$ROOK_SYSTEM_SA","$ROOK_OSD_SA" \ bound_service_account_namespaces="$ROOK_NAMESPACE" \ policies="$VAULT_POLICY_NAME" \ ttl=1440h