osd: add ibm key protect kms

The cluster-wide encryption feature now has a new supported key
management system: IBM Key protect. More information about the backend
can be found in Rook documentation under the

Today's implementation stores OSD encryption keys has "Standard" keys.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2022-01-13 16:02:10 +01:00
1 parent 511c250336
commit a7acf490b6
28 files changed
+806 -221

No files matched your search

+27 -1
View File
@@ -75,7 +75,7 @@ jobs:
run: |
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done"
- name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag
run: |
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
@@ -823,3 +823,29 @@ jobs:
if: failure() && github.event_name == 'pull_request'
uses: mxschmitt/action-tmate@v3
timeout-minutes: 60
encryption-pvc-kms-ibm-kp:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: run Encryption KMS IBM Key Protect
uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
with:
ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: upload test result
uses: actions/upload-artifact@v2
if: always()
with:
name: encryption-pvc-kms-ibm-kp
path: test
- name: setup tmate session for debugging when event is PR
if: failure() && github.event_name == 'pull_request'
uses: mxschmitt/action-tmate@v3
timeout-minutes: 60
+21
View File
@@ -316,3 +316,24 @@ jobs:
with:
name: rgw-multisite-testing-ceph-${{ matrix.ceph-image-tag }}
path: test
encryption-pvc-kms-ibm-kp:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: run Encryption KMS IBM Key Protect
uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
ibm-instance-id: ${{ secrets.IBM_KP_INSTANCE_ID }}
ibm-service-api-key: ${{ secrets.IBM_KP_SERVICE_API_KEY }}
- name: upload test result
uses: actions/upload-artifact@v2
if: always()
with:
name: encryption-pvc-kms-ibm-kp
path: test
@@ -0,0 +1,82 @@
name: Encryption KMS IBM Key Protect
description: Reusable workflow to test Encryption KMS IBM Key Protect
inputs:
ibm-instance-id:
description: IBM_KP_INSTANCE_ID from the calling workflow
required: true
ibm-service-api-key:
description: IBM_KP_SERVICE_API_KEY from the calling workflow
required: true
github-token:
description: GITHUB_TOKEN from the calling workflow
required: true
runs:
using: "composite"
steps:
- name: fail if env no present
if: "env.IBM_KP_INSTANCE_ID == '' || env.IBM_KP_SERVICE_API_KEY == ''"
env:
IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }}
shell: bash --noprofile --norc -eo pipefail -x {0}
run: echo "IBM_KP_INSTANCE_ID and IBM_KP_SERVICE_API_KEY must be set in the environment" && exit 0
- name: setup cluster resources
uses: ./.github/workflows/setup-cluster-resources
with:
github-token: ${{ inputs.github-token }}
- name: use local disk and create partitions for osds
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
tests/scripts/github-action-helper.sh use_local_disk
tests/scripts/github-action-helper.sh create_partitions_for_osds
- name: create cluster prerequisites
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1)
tests/scripts/github-action-helper.sh create_cluster_prerequisites
- name: deploy cluster
shell: bash --noprofile --norc -eo pipefail -x {0}
env:
IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }}
run: |
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
envsubst < "tests/manifests/test-kms-ibm-kp-secret.in" > "tests/manifests/test-kms-ibm-kp-secret.yaml"
envsubst < "tests/manifests/test-kms-ibm-kp-spec.in" > "tests/manifests/test-kms-ibm-kp-spec.yaml"
cat tests/manifests/test-kms-ibm-kp-secret.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-ibm-kp-spec.yaml
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
- name: wait for prepare pod
shell: bash --noprofile --norc -eo pipefail -x {0}
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
- name: wait for ceph to be ready
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
kubectl -n rook-ceph get pods
kubectl -n rook-ceph get secrets
- name: validate encrypted osd
shell: bash --noprofile --norc -eo pipefail -x {0}
run: |
sudo lsblk
- name: collect common logs
if: always()
uses: ./.github/workflows/collect-logs
with:
name: encryption-pvc-kms-ibm-kp
- name: teardown cluster so that keys are removed from the KMS
shell: bash --noprofile --norc -eo pipefail -x {0}
run: kubectl -n rook-ceph delete cephcluster rook-ceph --wait
+53
View File
@@ -20,6 +20,8 @@ The `security` section contains settings related to encryption of the cluster.
Supported KMS providers:
* [Vault](#vault)
* [IBM Key Protect](#ibm-kp)
## Vault
Rook supports storing OSD encryption keys in [HashiCorp Vault KMS](https://www.vaultproject.io/).
@@ -225,3 +227,54 @@ data:
Note: if you are using self-signed certificates (not known/approved by a proper CA) you must pass `VAULT_SKIP_VERIFY: true`.
Communications will remain encrypted but the validity of the certificate will not be verified.
## IBM Key Protect
Rook supports storing OSD encryption keys in [IBM Key
Protect](https://www.ibm.com/cloud/key-protect). The current implementation stores OSD encryption
keys as [Standard Keys](https://cloud.ibm.com/docs/key-protect?topic=key-protect-envelope-encryption#key-types) using the [Bring Your Own
Key](https://cloud.ibm.com/docs/key-protect?topic=key-protect-importing-keys) (BYOK) method. This
means that the Key Protect instance policy must have Standard Imported Key enabled.
### Configuration
First, you need to [provision the Key Protect service](https://cloud.ibm.com/docs/key-protect?topic=key-protect-provision) on the IBM Cloud. Once
completed, [retrieve the instance ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID&interface=ui).
Make a record of it; we need it in the CRD.
On the IBM Cloud, the user must create a Service ID, then assign an Access Policy to this service.
Ultimately, a Service API Key needs to be generated. All the steps are summarized in the [official
documentation](https://www.ibm.com/docs/en/cloud-private/3.2.0?topic=dg-creating-service-id-by-using-cloud-private-management-console).
The Service ID must be granted access to the Key Protect Service. Once the Service API Key is
generated, store it in a Kubernetes Secret.
```yaml
apiVersion: v1
kind: Secret
metadata:
name: ibm-kp-svc-api-key
namespace: rook-ceph
data:
IBM_KP_SERVICE_API_KEY: <service API Key>
```
In order for Rook to connect to IBM Key Protect, you must configure the following in your `CephCluster` template:
```yaml
security:
kms:
# name of the k8s config map containing all the kms connection details
connectionDetails:
KMS_PROVIDER: ibm-kp
IBM_KP_INSTANCE_ID: <instance ID that was retrieved in the first paragraph>
# name of the k8s secret containing the service API Key
tokenSecretName: ibm-kp-svc-api-key
```
More options are supported such as:
* `IBM_BASE_URL`: the base URL of the Key Protect instance, depending on your
[region](https://cloud.ibm.com/docs/key-protect?topic=key-protect-regions). Defaults to `https://us-south.kms.cloud.ibm.com`.
* `IBM_TOKEN_URL`: the URL of the Key Protect instance to retrieve the token. Defaults to
`https://iam.cloud.ibm.com/oidc/token`. Only needed for private instances.
+6 -1
View File
@@ -73,7 +73,7 @@ func startSecret() *kms.Config {
}
// Validate connection details
err = kms.ValidateConnectionDetails(context, &cephCluster.Spec.Security, namespace)
err = kms.ValidateConnectionDetails(ctx, context, &cephCluster.Spec.Security, namespace)
if err != nil {
rook.TerminateFatal(errors.Wrap(err, "failed to validate kms connection details"))
}
@@ -93,9 +93,14 @@ func cliGetSecret() *cobra.Command {
}
func getSecret(cmd *cobra.Command, args []string) {
// Initialize the context
ctx, cancel := signal.NotifyContext(context.Background(), operator.ShutdownSignals...)
defer cancel()
secretName := args[0]
secretPath := args[1]
keyManagementService := startSecret()
keyManagementService.ClusterInfo.Context = ctx
// Fetch the secret
s, err := keyManagementService.GetSecret(secretName)
+1
View File
@@ -3,6 +3,7 @@ module github.com/rook/rook
go 1.16
require (
github.com/IBM/keyprotect-go-client v0.7.0
github.com/aws/aws-sdk-go v1.37.19
github.com/banzaicloud/k8s-objectmatcher v1.1.0
github.com/ceph/go-ceph v0.12.0
+4
View File
@@ -124,6 +124,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dX
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3TveR88VmL1u3njyI=
github.com/IBM/keyprotect-go-client v0.7.0 h1:JstSHD14Lp6ihwQseyPuGcs1AjOBjAmcisP0dTBA6A0=
github.com/IBM/keyprotect-go-client v0.7.0/go.mod h1:SVr2ylV/fhSQPDiUjWirN9fsyWFCNNbt8GIT8hPJVjE=
github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
@@ -731,6 +733,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
@@ -2081,6 +2084,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0=
gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+11
View File
@@ -20,6 +20,7 @@ import (
"strings"
"github.com/hashicorp/vault/api"
"github.com/libopenstorage/secrets"
"github.com/libopenstorage/secrets/vault"
)
@@ -42,6 +43,16 @@ func (kms *KeyManagementServiceSpec) IsK8sAuthEnabled() bool {
return getParam(kms.ConnectionDetails, vault.AuthMethod) == vault.AuthMethodKubernetes && kms.TokenSecretName == ""
}
// IsVaultKMS return whether Vault KMS is configured
func (kms *KeyManagementServiceSpec) IsVaultKMS() bool {
return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeVault
}
// IsIBMKeyProtectKMS return whether IBM Key Protect KMS is configured
func (kms *KeyManagementServiceSpec) IsIBMKeyProtectKMS() bool {
return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeIBM
}
// IsTLSEnabled return KMS TLS details are configured
func (kms *KeyManagementServiceSpec) IsTLSEnabled() bool {
for _, tlsOption := range VaultTLSConnectionDetails {
+39 -19
View File
@@ -63,28 +63,48 @@ func dmsetupVersion(context *clusterd.Context) error {
func setKEKinEnv(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
// KMS details are passed by the Operator as env variables in the pod
// The token if any is mounted in the provisioner pod as an env variable so the secrets lib will pick it up
kmsConfig := kms.NewConfig(context, &v1.ClusterSpec{Security: v1.SecuritySpec{KeyManagementService: v1.KeyManagementServiceSpec{ConnectionDetails: kms.ConfigEnvsToMapString()}}}, clusterInfo)
if kmsConfig.IsVault() {
// Fetch the KEK
kek, err := kmsConfig.GetSecret(os.Getenv(oposd.PVCNameEnvVarName))
if err != nil {
return errors.Wrapf(err, "failed to retrieve key encryption key from %q kms", kmsConfig.Provider)
}
// The token if any is mounted in the provisioner pod as an env variable so the secrets lib will
// pick it up
clusterSpec := &v1.ClusterSpec{Security: v1.SecuritySpec{KeyManagementService: v1.KeyManagementServiceSpec{ConnectionDetails: kms.ConfigEnvsToMapString()}}}
if kek == "" {
return errors.New("key encryption key is empty")
}
// Set the KEK as an env variable for ceph-volume
err = os.Setenv(oposd.CephVolumeEncryptedKeyEnvVarName, kek)
if err != nil {
return errors.Wrap(err, "failed to set key encryption key env variable for ceph-volume")
}
logger.Debug("successfully set vault env variables")
// If KMS is not enabled this code does not need to run since we attach the KEK value from the
// Kubernetes Secret in the provision pod spec (mounted as an environment variable)
if !clusterSpec.Security.KeyManagementService.IsEnabled() {
logger.Debug("cluster-wide encryption is enabled with kubernetes secrets and the kek is attached to the provision env spec")
return nil
}
// The ibm key protect library does not read any environment variables, so we must set the
// service api key (coming from the secret mounted as environment variable) in the KMS
// connection details. These details are used to build the client connection
if clusterSpec.Security.KeyManagementService.IsIBMKeyProtectKMS() {
ibmServiceApiKey := os.Getenv(kms.IbmKeyProtectServiceApiKey)
if ibmServiceApiKey == "" {
return errors.Errorf("ibm key protect %q environment variable is not set", kms.IbmKeyProtectServiceApiKey)
}
clusterSpec.Security.KeyManagementService.ConnectionDetails[kms.IbmKeyProtectServiceApiKey] = ibmServiceApiKey
}
kmsConfig := kms.NewConfig(context, clusterSpec, clusterInfo)
// Fetch the KEK
kek, err := kmsConfig.GetSecret(os.Getenv(oposd.PVCNameEnvVarName))
if err != nil {
return errors.Wrapf(err, "failed to retrieve key encryption key from %q kms", kmsConfig.Provider)
}
if kek == "" {
return errors.New("key encryption key is empty")
}
// Set the KEK as an env variable for ceph-volume
err = os.Setenv(oposd.CephVolumeEncryptedKeyEnvVarName, kek)
if err != nil {
return errors.Wrap(err, "failed to set key encryption key env variable for ceph-volume")
}
logger.Debug("successfully set vault env variables")
return nil
}
+52 -19
View File
@@ -30,7 +30,7 @@ import (
)
var (
knownKMSPrefix = []string{"VAULT_"}
knownKMSPrefix = []string{"VAULT_", "IBM_"}
)
// VaultTokenEnvVarFromSecret returns the kms token secret value as an env var
@@ -48,6 +48,21 @@ func vaultTokenEnvVarFromSecret(tokenSecretName string) v1.EnvVar {
}
}
// ibmKeyProtectServiceAPIKeyEnvVarFromSecret returns the kms token secret value as an env var
func ibmKeyProtectServiceAPIKeyEnvVarFromSecret(tokenSecretName string) v1.EnvVar {
return v1.EnvVar{
Name: IbmKeyProtectServiceApiKey,
ValueFrom: &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{
Name: tokenSecretName,
},
Key: IbmKeyProtectServiceApiKey,
},
},
}
}
// vaultTLSEnvVarFromSecret translates TLS env var which are set to k8s secret name to their actual path on the fs once mounted as volume
// See: VaultSecretVolumeAndMount() for more details
func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar {
@@ -63,30 +78,48 @@ func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar {
return vaultTLSEnvVar
}
// VaultConfigToEnvVar populates the kms config as env variables
func VaultConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar {
// ConfigToEnvVar populates the kms config as env variables
func ConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar {
envs := []v1.EnvVar{}
backendPath := GetParam(spec.Security.KeyManagementService.ConnectionDetails, vault.VaultBackendPathKey)
// Set BACKEND_PATH to the API's default if not passed
if backendPath == "" {
spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey] = vault.DefaultBackendPath
}
for k, v := range spec.Security.KeyManagementService.ConnectionDetails {
// Skip TLS and token env var to avoid env being set multiple times
toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken)
if sets.NewString(toSkip...).Has(k) {
continue
if spec.Security.KeyManagementService.IsVaultKMS() {
backendPath := GetParam(spec.Security.KeyManagementService.ConnectionDetails, vault.VaultBackendPathKey)
// Set BACKEND_PATH to the API's default if not passed
if backendPath == "" {
spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey] = vault.DefaultBackendPath
}
}
if spec.Security.KeyManagementService.IsIBMKeyProtectKMS() {
// We don't want to leak the IBM service API key to the container environment variables even
// the container is ephemeral.
// The IBM_KP_SERVICE_API_KEY content is mounted in the provisioner container as an
// environment variable from a secret
delete(spec.Security.KeyManagementService.ConnectionDetails, IbmKeyProtectServiceApiKey)
envs = append(envs, ibmKeyProtectServiceAPIKeyEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName))
}
for k, v := range spec.Security.KeyManagementService.ConnectionDetails {
if spec.Security.KeyManagementService.IsVaultKMS() {
// Skip TLS and token env var to avoid env being set multiple times
toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken)
if sets.NewString(toSkip...).Has(k) {
continue
}
}
envs = append(envs, v1.EnvVar{Name: k, Value: v})
}
// Add the VAULT_TOKEN
if spec.Security.KeyManagementService.IsTokenAuthEnabled() {
envs = append(envs, vaultTokenEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName))
}
if spec.Security.KeyManagementService.IsVaultKMS() {
// Add the VAULT_TOKEN
if spec.Security.KeyManagementService.IsTokenAuthEnabled() {
envs = append(envs, vaultTokenEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName))
}
// Add TLS env if any
envs = append(envs, vaultTLSEnvVarFromSecret(spec.Security.KeyManagementService.ConnectionDetails)...)
// Add TLS env if any
envs = append(envs, vaultTLSEnvVarFromSecret(spec.Security.KeyManagementService.ConnectionDetails)...)
}
logger.Debugf("kms envs are %v", envs)
+75 -39
View File
@@ -23,38 +23,58 @@ import (
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
)
func TestVaultTLSEnvVarFromSecret(t *testing.T) {
// No TLS
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{TokenSecretName: "vault-token", ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200"}}}}
envVars := VaultConfigToEnvVar(spec)
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
return envVars[i].Name < envVars[j].Name
t.Run("vault - no tls", func(t *testing.T) {
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
TokenSecretName: "vault-token",
ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200"}}},
}
envVars := ConfigToEnvVar(spec)
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
return envVars[i].Name < envVars[j].Name
})
assert.True(t, areEnvVarsSorted)
assert.Equal(t, 4, len(envVars))
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_BACKEND_PATH", Value: "secret/"})
})
assert.True(t, areEnvVarsSorted)
assert.Equal(t, 4, len(envVars))
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_BACKEND_PATH", Value: "secret/"})
// TLS
spec = cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{TokenSecretName: "vault-token", ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200", "VAULT_CACERT": "vault-ca-cert-secret"}}}}
envVars = VaultConfigToEnvVar(spec)
areEnvVarsSorted = sort.SliceIsSorted(envVars, func(i, j int) bool {
return envVars[i].Name < envVars[j].Name
t.Run("vault tls", func(t *testing.T) {
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
TokenSecretName: "vault-token",
ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200", "VAULT_CACERT": "vault-ca-cert-secret"}}},
}
envVars := ConfigToEnvVar(spec)
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
return envVars[i].Name < envVars[j].Name
})
assert.True(t, areEnvVarsSorted)
assert.Equal(t, 5, len(envVars))
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
})
t.Run("ibm kp", func(t *testing.T) {
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
TokenSecretName: "ibm-kp-token",
ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_INSTANCE_ID": "1"}}},
}
envVars := ConfigToEnvVar(spec)
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
return envVars[i].Name < envVars[j].Name
})
assert.True(t, areEnvVarsSorted)
assert.Equal(t, 3, len(envVars))
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "ibm-kp"})
assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_INSTANCE_ID", Value: "1"})
assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}})
})
assert.True(t, areEnvVarsSorted)
assert.Equal(t, 5, len(envVars))
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"})
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
}
func TestConfigEnvsToMapString(t *testing.T) {
@@ -80,7 +100,7 @@ func TestConfigEnvsToMapString(t *testing.T) {
envs = ConfigEnvsToMapString()
assert.Equal(t, 3, len(envs))
assert.True(t, envs["KMS_PROVIDER"] == "vault")
clusterSpec := &rookv1.ClusterSpec{Security: rookv1.SecuritySpec{KeyManagementService: rookv1.KeyManagementServiceSpec{ConnectionDetails: envs}}}
clusterSpec := &cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: envs}}}
assert.True(t, clusterSpec.Security.KeyManagementService.IsEnabled())
}
@@ -94,27 +114,43 @@ func TestVaultConfigToEnvVar(t *testing.T) {
want []v1.EnvVar
}{
{
"no backend path",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}}},
[]v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "secret/"}},
},
{
"with backend path",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_BACKEND_PATH": "foo/"}}}}},
[]v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "foo/"}},
},
{
"test with tls config",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_CACERT": "my-secret-name"}}}}},
"vault - no backend path",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault"}}}}},
[]v1.EnvVar{
{Name: "KMS_PROVIDER", Value: "vault"},
{Name: "VAULT_BACKEND_PATH", Value: "secret/"},
},
},
{
"vault - with backend path",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_BACKEND_PATH": "foo/"}}}}},
[]v1.EnvVar{
{Name: "KMS_PROVIDER", Value: "vault"},
{Name: "VAULT_BACKEND_PATH", Value: "foo/"},
},
},
{
"vault - test with tls config",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_CACERT": "my-secret-name"}}}}},
[]v1.EnvVar{
{Name: "KMS_PROVIDER", Value: "vault"},
{Name: "VAULT_BACKEND_PATH", Value: "secret/"},
{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"},
},
},
{
"ibm kp - IBM_KP_SERVICE_API_KEY is removed from the details",
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_SERVICE_API_KEY": "foo", "IBM_KP_INSTANCE_ID": "1"}, TokenSecretName: "ibm-kp-token"}}}},
[]v1.EnvVar{
{Name: "IBM_KP_INSTANCE_ID", Value: "1"},
{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}},
{Name: "KMS_PROVIDER", Value: "ibm-kp"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := VaultConfigToEnvVar(tt.args.spec); !reflect.DeepEqual(got, tt.want) {
if got := ConfigToEnvVar(tt.args.spec); !reflect.DeepEqual(got, tt.want) {
t.Errorf("VaultConfigToEnvVar() = %v, want %v", got, tt.want)
}
})
@@ -0,0 +1,86 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kms
import (
kp "github.com/IBM/keyprotect-go-client"
"github.com/libopenstorage/secrets"
"github.com/pkg/errors"
)
const (
// IbmKeyProtectServiceApiKey is the IBM Key Protect service API key
IbmKeyProtectServiceApiKey = "IBM_KP_SERVICE_API_KEY"
// IbmKeyProtectInstanceIdKey is the IBM Key Protect instance id
IbmKeyProtectInstanceIdKey = "IBM_KP_INSTANCE_ID"
// IbmKeyProtectBaseUrlKey is the IBM Key Protect base url
IbmKeyProtectBaseUrlKey = "IBM_KP_BASE_URL"
// IbmKeyProtectTokenUrlKey is the IBM Key Protect token url
// #nosec G101 since it's just an URL
IbmKeyProtectTokenUrlKey = "IBM_KP_TOKEN_URL"
)
var (
kmsIBMKeyProtectMandatoryTokenDetails = []string{IbmKeyProtectServiceApiKey}
kmsIBMKeyProtectMandatoryConnectionDetails = []string{IbmKeyProtectInstanceIdKey, IbmKeyProtectServiceApiKey}
// ErrIbmServiceApiKeyNotSet is returned when IBM_KP_SERVICE_API_KEY is not set
ErrIbmServiceApiKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectServiceApiKey)
// ErrIbmInstanceIdKeyNotSet is returned when IBM_KP_INSTANCE_ID is not set
ErrIbmInstanceIdKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectInstanceIdKey)
)
// InitKeyProtect initializes the KeyProtect KMS.
// With native go client directly "github.com/IBM/keyprotect-go-client"
func InitKeyProtect(config map[string]string) (*kp.Client, error) {
serviceApiKey := GetParam(config, IbmKeyProtectServiceApiKey)
if serviceApiKey == "" {
return nil, ErrIbmServiceApiKeyNotSet
}
instanceId := GetParam(config, IbmKeyProtectInstanceIdKey)
if instanceId == "" {
return nil, ErrIbmInstanceIdKeyNotSet
}
baseUrl := GetParam(config, IbmKeyProtectBaseUrlKey)
if baseUrl == "" {
baseUrl = kp.DefaultBaseURL
}
tokenUrl := GetParam(config, IbmKeyProtectTokenUrlKey)
if tokenUrl == "" {
tokenUrl = kp.DefaultTokenURL
}
cc := kp.ClientConfig{
BaseURL: baseUrl,
APIKey: serviceApiKey,
TokenURL: tokenUrl,
InstanceID: instanceId,
Verbose: kp.VerboseAll,
}
kp, err := kp.New(cc, nil)
if err != nil {
return nil, err
}
return kp, nil
}
// IsIBMKeyProtect determines whether the configured KMS is IBM Key Protect
func (c *Config) IsIBMKeyProtect() bool { return c.Provider == secrets.TypeIBM }
@@ -0,0 +1,70 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kms
import (
"testing"
kp "github.com/IBM/keyprotect-go-client"
"github.com/stretchr/testify/assert"
)
func TestInitKeyProtect(t *testing.T) {
config := map[string]string{
"foo": "1",
}
t.Run("IBM_KP_SERVICE_API_KEY not set", func(t *testing.T) {
_, err := InitKeyProtect(config)
assert.Error(t, err)
assert.ErrorIs(t, err, ErrIbmServiceApiKeyNotSet)
config[IbmKeyProtectServiceApiKey] = "foo"
})
t.Run("IBM_KP_INSTANCE_ID not set", func(t *testing.T) {
_, err := InitKeyProtect(config)
assert.Error(t, err)
assert.ErrorIs(t, err, ErrIbmInstanceIdKeyNotSet)
config[IbmKeyProtectInstanceIdKey] = "foo"
})
t.Run("default base URL", func(t *testing.T) {
c, err := InitKeyProtect(config)
assert.NoError(t, err)
assert.Equal(t, kp.DefaultBaseURL, c.Config.BaseURL)
})
t.Run("different base URL", func(t *testing.T) {
config[IbmKeyProtectBaseUrlKey] = "https://us-west.kms.cloud.ibm.com"
c, err := InitKeyProtect(config)
assert.NoError(t, err)
assert.Equal(t, "https://us-west.kms.cloud.ibm.com", c.Config.BaseURL)
})
t.Run("default base token URL", func(t *testing.T) {
c, err := InitKeyProtect(config)
assert.NoError(t, err)
assert.Equal(t, kp.DefaultTokenURL, c.Config.TokenURL)
})
t.Run("different base URL", func(t *testing.T) {
config[IbmKeyProtectTokenUrlKey] = "new"
c, err := InitKeyProtect(config)
assert.NoError(t, err)
assert.Equal(t, "new", c.Config.TokenURL)
})
}
+2 -2
View File
@@ -41,13 +41,13 @@ const (
// storeSecretInKubernetes stores the dmcrypt key in a Kubernetes Secret
func (c *Config) storeSecretInKubernetes(pvcName, key string) error {
s, err := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo)
s, err := generateOSDEncryptedKeySecret(pvcName, key, c.ClusterInfo)
if err != nil {
return err
}
// Create the Kubernetes Secret
_, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, s, metav1.CreateOptions{})
_, err = c.context.Clientset.CoreV1().Secrets(c.ClusterInfo.Namespace).Create(c.ClusterInfo.Context, s, metav1.CreateOptions{})
if err != nil && !kerrors.IsAlreadyExists(err) {
return errors.Wrapf(err, "failed to save ceph osd encryption key as a secret for pvc %q", pvcName)
}
+105 -28
View File
@@ -21,6 +21,7 @@ import (
"os"
"strings"
kp "github.com/IBM/keyprotect-go-client"
"github.com/coreos/pkg/capnslog"
"github.com/hashicorp/vault/api"
"github.com/libopenstorage/secrets"
@@ -47,14 +48,14 @@ type Config struct {
Provider string
context *clusterd.Context
clusterSpec *cephv1.ClusterSpec
clusterInfo *cephclient.ClusterInfo
ClusterInfo *cephclient.ClusterInfo
}
// NewConfig returns the selected KMS
func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) *Config {
config := &Config{
context: context,
clusterInfo: clusterInfo,
ClusterInfo: clusterInfo,
clusterSpec: clusterSpec,
}
@@ -64,6 +65,8 @@ func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clust
config.Provider = secrets.TypeK8s
case secrets.TypeVault:
config.Provider = secrets.TypeVault
case secrets.TypeIBM:
config.Provider = secrets.TypeIBM
default:
logger.Errorf("unsupported kms type %q", Provider)
}
@@ -83,16 +86,34 @@ func (c *Config) PutSecret(secretName, secretValue string) error {
}
if c.IsVault() {
// Store the secret in Vault
v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to init vault kms")
}
k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
err = put(v, GenerateOSDEncryptionSecretName(secretName), secretValue, k)
if err != nil {
return errors.Wrap(err, "failed to put secret in vault")
}
}
if c.IsIBMKeyProtect() {
kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to init ibm key protect")
}
// Create the key is not present
keyAlias := []string{secretName}
_, err = kpClient.CreateImportedKeyWithAliases(c.ClusterInfo.Context, secretName, nil, secretValue, "", "", true, keyAlias)
if err != nil {
if strings.Contains(err.Error(), "KEY_ALIAS_NOT_UNIQUE_ERR") {
logger.Debugf("key %q already exists. %v", secretName, err)
return nil
}
return errors.Wrap(err, "failed to put secret in ibm key protect")
}
}
return nil
}
@@ -102,17 +123,28 @@ func (c *Config) GetSecret(secretName string) (string, error) {
var value string
if c.IsVault() {
// Store the secret in Vault
v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
if err != nil {
return "", errors.Wrap(err, "failed to init vault")
}
k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
value, err = get(v, GenerateOSDEncryptionSecretName(secretName), k)
if err != nil {
return "", errors.Wrap(err, "failed to get secret in vault")
return "", errors.Wrap(err, "failed to get secret from vault")
}
}
if c.IsIBMKeyProtect() {
kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
if err != nil {
return "", errors.Wrap(err, "failed to init ibm key protect")
}
keyObject, err := kpClient.GetKey(c.ClusterInfo.Context, secretName)
if err != nil {
return "", errors.Wrap(err, "failed to get secret from ibm key protect")
}
value = string(keyObject.Payload)
}
return value, nil
}
@@ -121,21 +153,49 @@ func (c *Config) GetSecret(secretName string) (string, error) {
func (c *Config) DeleteSecret(secretName string) error {
if c.IsVault() {
// Store the secret in Vault
v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to delete secret in vault")
}
k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
// Force removal of all the versions of the secret on K/V version 2
k[secrets.DestroySecret] = "true"
err = delete(v, GenerateOSDEncryptionSecretName(secretName), k)
err = deleteSecret(v, GenerateOSDEncryptionSecretName(secretName), k)
if err != nil {
return errors.Wrap(err, "failed to delete secret in vault")
}
}
if c.IsIBMKeyProtect() {
kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to init ibm key protect")
}
// We use context.TODO() since the clusterInfo context has been cancelled by the CephCluster's
// deletion event
ctx := context.TODO()
// Fetch the key to get the ID
key, err := kpClient.GetKey(ctx, secretName)
if err != nil {
return errors.Wrap(err, "failed to get secret in ibm key protect")
}
// DeleteKey does not support deleting secret with the alias name so we must use the ID
// After you delete a key, the key transitions to the Destroyed state. Any data encrypted by
// keys in this state is no longer accessible. Metadata that is associated with the key,
// such as the key's deletion date, is kept in the Key Protect database. Destroyed keys can
// be recovered after up to 30 days or their expiration date, whichever is sooner. After 30
// days, keys can no longer be recovered, and become eligible to be purged after 90 days, a
// process that shreds the key material and makes its metadata inaccessible.
_, err = kpClient.DeleteKey(ctx, key.ID, kp.ReturnRepresentation, []kp.CallOpt{kp.ForceOpt{Force: true}}...)
if err != nil {
return errors.Wrap(err, "failed to delete secret in ibm key protect")
}
}
return nil
}
@@ -149,14 +209,21 @@ func GetParam(kmsConfig map[string]string, param string) string {
}
// ValidateConnectionDetails validates mandatory KMS connection details
func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error {
ctx := context.TODO()
func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error {
// Lookup mandatory connection details
for _, config := range kmsMandatoryConnectionDetails {
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
}
}
// A token must be specified if token-auth is used
if !securitySpec.KeyManagementService.IsK8sAuthEnabled() && securitySpec.KeyManagementService.TokenSecretName == "" {
if !securitySpec.KeyManagementService.IsTokenAuthEnabled() {
return errors.New("failed to validate kms configuration (missing token in spec)")
}
}
// KMS provider must be specified
provider := GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider)
@@ -167,32 +234,35 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *
return errors.Wrapf(err, "failed to fetch kms token secret %q", securitySpec.KeyManagementService.TokenSecretName)
}
// Check for empty token
token, ok := kmsToken.Data[KMSTokenSecretNameKey]
if !ok || len(token) == 0 {
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName)
}
switch provider {
case "vault":
case secrets.TypeVault:
// Check for empty token
token, ok := kmsToken.Data[KMSTokenSecretNameKey]
if !ok || len(token) == 0 {
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName)
}
// Set the env variable
err = os.Setenv(api.EnvVaultToken, string(token))
if err != nil {
return errors.Wrap(err, "failed to set vault kms token to an env var")
}
}
}
// Lookup mandatory connection details
for _, config := range kmsMandatoryConnectionDetails {
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
case secrets.TypeIBM:
for _, config := range kmsIBMKeyProtectMandatoryTokenDetails {
v, ok := kmsToken.Data[config]
if !ok || len(v) == 0 {
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", config, securitySpec.KeyManagementService.TokenSecretName)
}
// Append the token secret details to the connection details
securitySpec.KeyManagementService.ConnectionDetails[config] = strings.TrimSuffix(strings.TrimSpace(string(v)), "\n")
}
}
}
// Validate KMS provider connection details for each provider
switch provider {
case "vault":
case secrets.TypeVault:
err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails)
if err != nil {
return errors.Wrap(err, "failed to validate vault connection details")
@@ -210,6 +280,14 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *
securitySpec.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = backendVersion
}
}
case secrets.TypeIBM:
for _, config := range kmsIBMKeyProtectMandatoryConnectionDetails {
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
}
}
default:
return errors.Errorf("failed to validate kms provider connection details (provider %q not supported)", provider)
}
@@ -218,8 +296,7 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *
}
// SetTokenToEnvVar sets a KMS token as an env variable
func SetTokenToEnvVar(clusterdContext *clusterd.Context, tokenSecretName, provider, namespace string) error {
ctx := context.TODO()
func SetTokenToEnvVar(ctx context.Context, clusterdContext *clusterd.Context, tokenSecretName, provider, namespace string) error {
// Get the secret containing the kms token
kmsToken, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, tokenSecretName, metav1.GetOptions{})
if err != nil {
+114 -67
View File
@@ -37,83 +37,97 @@ func TestValidateConnectionDetails(t *testing.T) {
context := &clusterd.Context{Clientset: test.New(t, 3)}
securitySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}
ns := "rook-ceph"
// Error: no token in spec
err := ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms configuration (missing token in spec)")
securitySpec.KeyManagementService.TokenSecretName = "vault-token"
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found")
// Error: token secret present but empty content
s := &v1.Secret{
vaultSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: securitySpec.KeyManagementService.TokenSecretName,
Name: "vault-token",
Namespace: ns,
},
}
_, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
// Error: token key does not exist
s.Data = map[string][]byte{"foo": []byte("bar")}
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
// Success: token content is ok
s.Data["token"] = []byte("token")
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty")
securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault"
// Error: Data has a KMS_PROVIDER but missing details
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"")
// Error: connection details are correct but the token secret does not exist
securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200"
// Error: TLS is configured but secrets do not exist
securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret"
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"")
// Error: TLS secret exists but empty key
ibmSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "ibm-token",
Namespace: ns,
},
Data: map[string][]byte{"foo": []byte("bar")},
}
tlsSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "vault-ca-secret",
Namespace: ns,
},
}
_, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, tlsSecret, metav1.CreateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"")
ibmSecuritySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
ConnectionDetails: map[string]string{
"KMS_PROVIDER": "ibm-kp",
},
}}
// Success: TLS config is correct
tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")}
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, tlsSecret, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(context, securitySpec, ns)
assert.NoError(t, err, "")
t.Run("no kms provider given", func(t *testing.T) {
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty")
securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault"
})
// test with vauult server
t.Run("vault - no token object", func(t *testing.T) {
securitySpec.KeyManagementService.TokenSecretName = "vault-token"
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found")
})
t.Run("vault - token secret present but empty content", func(t *testing.T) {
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, vaultSecret, metav1.CreateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
})
t.Run("vault - token key does not exist", func(t *testing.T) {
vaultSecret.Data = map[string][]byte{"foo": []byte("bar")}
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, vaultSecret, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
})
// Success: token content is ok
t.Run("vault - token content is ok", func(t *testing.T) {
vaultSecret.Data["token"] = []byte("token")
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, vaultSecret, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"")
securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200"
})
t.Run("vault - TLS is configured but secrets do not exist", func(t *testing.T) {
securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret"
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"")
})
t.Run("vault - TLS secret exists but empty key", func(t *testing.T) {
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, tlsSecret, metav1.CreateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"")
})
t.Run("vault - success TLS config is correct", func(t *testing.T) {
tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")}
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, tlsSecret, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.NoError(t, err, "")
})
// test with vault server
t.Run("success - auto detect kv version and set it", func(t *testing.T) {
cluster := fakeVaultServer(t)
cluster.Start()
@@ -142,11 +156,44 @@ func TestValidateConnectionDetails(t *testing.T) {
TokenSecretName: "vault-token",
},
}
err = ValidateConnectionDetails(context, securitySpec, ns)
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
assert.NoError(t, err, "")
assert.Equal(t, securitySpec.KeyManagementService.ConnectionDetails["VAULT_BACKEND"], "v2")
})
t.Run("ibm kp - fail no token specified, only token is supported", func(t *testing.T) {
err := ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms configuration (missing token in spec)")
ibmSecuritySpec.KeyManagementService.TokenSecretName = "ibm-token"
})
t.Run("ibm kp - token present but no key for service key", func(t *testing.T) {
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, ibmSecret, metav1.CreateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to read k8s kms secret \"IBM_KP_SERVICE_API_KEY\" key \"ibm-token\" (not found or empty)")
})
t.Run("ibm kp - token ok but no instance id", func(t *testing.T) {
ibmSecret.Data["IBM_KP_SERVICE_API_KEY"] = []byte("foo")
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, ibmSecret, metav1.UpdateOptions{})
assert.NoError(t, err)
err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
assert.Error(t, err, "")
assert.EqualError(t, err, "failed to validate kms config \"IBM_KP_INSTANCE_ID\". cannot be empty")
ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_INSTANCE_ID"] = "foo"
})
t.Run("ibm kp - success", func(t *testing.T) {
err := ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
assert.NoError(t, err, "")
// IBM_KP_SERVICE_API_KEY must be appended to the details so that the client can be built with
// all the details
assert.Equal(t, ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_SERVICE_API_KEY"], "foo")
})
}
func TestSetTokenToEnvVar(t *testing.T) {
@@ -164,7 +211,7 @@ func TestSetTokenToEnvVar(t *testing.T) {
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{})
assert.NoError(t, err)
err = SetTokenToEnvVar(context, secretName, "vault", ns)
err = SetTokenToEnvVar(ctx, context, secretName, "vault", ns)
assert.NoError(t, err)
assert.Equal(t, os.Getenv("VAULT_TOKEN"), "toto")
os.Unsetenv("VAULT_TOKEN")
+3 -3
View File
@@ -222,7 +222,7 @@ func get(v secrets.Secrets, secretName string, keyContext map[string]string) (st
return s[secretName].(string), nil
}
func delete(v secrets.Secrets, secretName string, keyContext map[string]string) error {
func deleteSecret(v secrets.Secrets, secretName string, keyContext map[string]string) error {
// #nosec G104 Write the encryption key in Vault
err := v.DeleteSecret(secretName, keyContext)
if err != nil {
@@ -232,7 +232,7 @@ func delete(v secrets.Secrets, secretName string, keyContext map[string]string)
return nil
}
func buildKeyContext(config map[string]string) map[string]string {
func buildVaultKeyContext(config map[string]string) map[string]string {
// Key context is just the Vault namespace, available in the enterprise version only
keyContext := map[string]string{secrets.KeyVaultNamespace: config[api.EnvVaultNamespace]}
vaultNamespace, ok := config[api.EnvVaultNamespace]
@@ -245,7 +245,7 @@ func buildKeyContext(config map[string]string) map[string]string {
// IsVault determines whether the configured KMS is Vault
func (c *Config) IsVault() bool {
return c.Provider == "vault"
return c.Provider == secrets.TypeVault
}
func validateVaultConnectionDetails(clusterdContext *clusterd.Context, ns string, kmsConfig map[string]string) error {
+3 -3
View File
@@ -254,13 +254,13 @@ func Test_configTLS(t *testing.T) {
})
}
func Test_buildKeyContext(t *testing.T) {
func Test_buildVaultKeyContext(t *testing.T) {
t.Run("no vault namespace, return empty map and assignment is possible", func(t *testing.T) {
config := map[string]string{
"KMS_PROVIDER": "vault",
"VAULT_ADDR": "1.1.1.1",
}
context := buildKeyContext(config)
context := buildVaultKeyContext(config)
assert.Len(t, context, 0)
context["foo"] = "bar"
})
@@ -271,7 +271,7 @@ func Test_buildKeyContext(t *testing.T) {
"VAULT_ADDR": "1.1.1.1",
"VAULT_NAMESPACE": "vault-namespace",
}
context := buildKeyContext(config)
context := buildVaultKeyContext(config)
assert.Len(t, context, 1)
context["foo"] = "bar"
assert.Len(t, context, 2)
+1 -1
View File
@@ -306,7 +306,7 @@ func preClusterStartValidation(cluster *cluster) error {
// Validate on-PVC cluster encryption KMS settings
if cluster.Spec.Storage.IsOnPVCEncrypted() && cluster.Spec.Security.KeyManagementService.IsEnabled() {
// Validate the KMS details
err := kms.ValidateConnectionDetails(cluster.context, &cluster.Spec.Security, cluster.Namespace)
err := kms.ValidateConnectionDetails(cluster.ClusterInfo.Context, cluster.context, &cluster.Spec.Security, cluster.Namespace)
if err != nil {
return errors.Wrap(err, "failed to validate kms connection details")
}
+15 -2
View File
@@ -511,6 +511,10 @@ func (r *ReconcileCephCluster) removeFinalizer(client client.Client, name types.
}
func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1.CephCluster) error {
// At the point, the CephCluster has been deleted and the context cancelled, so we need to use a
// new context here
// So do NOT use another context
ctx := context.TODO()
// If the operator was stopped and we enter this code, the map is empty
if _, ok := c.clusterMap[currentCluster.Namespace]; !ok {
c.clusterMap[currentCluster.Namespace] = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}}
@@ -526,13 +530,22 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1
kmsConfig := kms.NewConfig(c.context, &currentCluster.Spec, c.clusterMap[currentCluster.Namespace].ClusterInfo)
// If token auth is used by the KMS we set it as an env variable
if currentCluster.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
err := kms.SetTokenToEnvVar(c.context, currentCluster.Spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, currentCluster.Namespace)
if currentCluster.Spec.Security.KeyManagementService.IsTokenAuthEnabled() && currentCluster.Spec.Security.KeyManagementService.IsVaultKMS() {
err := kms.SetTokenToEnvVar(ctx, c.context, currentCluster.Spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, currentCluster.Namespace)
if err != nil {
return errors.Wrapf(err, "failed to fetch kms token secret %q", currentCluster.Spec.Security.KeyManagementService.TokenSecretName)
}
}
// We need to fetch the IBM_KP_SERVICE_API_KEY value
if currentCluster.Spec.Security.KeyManagementService.IsIBMKeyProtectKMS() {
// This will validate the connection details again and will add the IBM_KP_SERVICE_API_KEY to the spec
err = kms.ValidateConnectionDetails(ctx, c.context, &currentCluster.Spec.Security, currentCluster.Namespace)
if err != nil {
return errors.Wrap(err, "failed to validate kms connection details to delete the secret")
}
}
// Delete each PV KEK
for _, osdPVC := range osdPVCs {
// Generate and store the encrypted key in whatever KMS is configured
+2 -2
View File
@@ -218,8 +218,8 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
// We could set an env var in the Operator or a global var instead of the API call?
// Hopefully, the API is cheap and we can always retrieve the token if it has changed...
if c.spec.Security.KeyManagementService.IsTokenAuthEnabled() {
err := kms.SetTokenToEnvVar(c.context, c.spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, c.clusterInfo.Namespace)
if c.spec.Security.KeyManagementService.IsTokenAuthEnabled() && c.spec.Security.KeyManagementService.IsVaultKMS() {
err := kms.SetTokenToEnvVar(c.clusterInfo.Context, c.context, c.spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, c.clusterInfo.Namespace)
if err != nil {
errMsg := fmt.Sprintf("failed to fetch kms token secret %q. %v", c.spec.Security.KeyManagementService.TokenSecretName, err)
errs.addError(errMsg)
@@ -21,7 +21,6 @@ import (
"fmt"
"path"
"github.com/libopenstorage/secrets"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
@@ -121,8 +120,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
if osdProps.encrypted {
// If a KMS is configured we populate
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if kmsProvider == secrets.TypeVault {
if c.spec.Security.KeyManagementService.IsVaultKMS() {
volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumes = append(volumes, volumeTLS)
}
@@ -280,12 +278,11 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
if osdProps.encrypted {
// If a KMS is configured we populate volume mounts and env variables
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if kmsProvider == secrets.TypeVault {
if c.spec.Security.KeyManagementService.IsVaultKMS() {
_, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumeMounts = append(volumeMounts, volumeMountsTLS)
envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...)
}
envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...)
} else {
envVars = append(envVars, cephVolumeRawEncryptedEnvVarFromSecret(osdProps))
}
+8 -12
View File
@@ -23,7 +23,6 @@ import (
"path/filepath"
"strconv"
"github.com/libopenstorage/secrets"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
@@ -867,7 +866,7 @@ func (c *Cluster) generateVaultGetKEK(osdProps osdProperties) v1.Container {
keyName := osdProps.pvc.ClaimName
keyPath := encryptionKeyPath()
envVars := c.getConfigEnvVars(osdProps, "")
envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...)
envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...)
return v1.Container{
Name: blockEncryptionKMSGetKEKInitContainer,
@@ -889,24 +888,21 @@ func (c *Cluster) getPVCEncryptionOpenInitContainerActivate(mountPath string, os
// If a KMS is enabled we need to add an init container to fetch the KEK
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
// Get Vault KEK from KMS container
if kmsProvider == secrets.TypeVault {
getKEKFromKMSContainer := c.generateVaultGetKEK(osdProps)
getKEKFromKMSContainer := c.generateVaultGetKEK(osdProps)
// Volume mount to store the encrypted key
_, volMount := c.getEncryptionVolume(osdProps)
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, volMount)
// Volume mount to store the encrypted key
_, volMount := c.getEncryptionVolume(osdProps)
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, volMount)
if c.spec.Security.KeyManagementService.IsVaultKMS() {
// Now let's see if there is a TLS config we need to mount as well
if c.spec.Security.KeyManagementService.IsTLSEnabled() {
_, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, vaultVolMount)
}
// Add the container to the list of containers
containers = append(containers, getKEKFromKMSContainer)
}
// Add the container to the list of containers
containers = append(containers, getKEKFromKMSContainer)
}
// Main block container
+2 -12
View File
@@ -21,7 +21,6 @@ import (
"path"
"path/filepath"
"github.com/libopenstorage/secrets"
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
"github.com/rook/rook/pkg/operator/ceph/config"
v1 "k8s.io/api/core/v1"
@@ -192,15 +191,6 @@ func getUdevVolume() (v1.Volume, v1.VolumeMount) {
}
func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.VolumeMount) {
// Determine whether we have a KMS configuration
var isKMS bool
if len(c.spec.Security.KeyManagementService.ConnectionDetails) != 0 {
provider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if provider == secrets.TypeVault {
isKMS = true
}
}
// Generate volume
var m int32 = 0400
volume := v1.Volume{
@@ -220,7 +210,7 @@ func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.Vol
}
// On the KMS use case, we want the volume mount to be in memory since we pass write the KEK
if isKMS {
if c.spec.Security.KeyManagementService.IsEnabled() {
volume.VolumeSource.Secret = nil
volume.VolumeSource = v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{
@@ -237,7 +227,7 @@ func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.Vol
}
// With KMS we must be able to write inside the directory to write the KEK
if isKMS {
if c.spec.Security.KeyManagementService.IsEnabled() {
volumeMounts.ReadOnly = false
}
+1 -1
View File
@@ -568,7 +568,7 @@ func (c *clusterConfig) vaultPrefixRGW() string {
func (c *clusterConfig) CheckRGWKMS() (bool, error) {
if c.store.Spec.Security != nil && c.store.Spec.Security.KeyManagementService.IsEnabled() {
err := kms.ValidateConnectionDetails(c.context, c.store.Spec.Security, c.store.Namespace)
err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, c.store.Spec.Security, c.store.Namespace)
if err != nil {
return false, err
}
+5 -3
View File
@@ -436,13 +436,15 @@ func TestDefaultReadinessProbe(t *testing.T) {
}
func TestCheckRGWKMS(t *testing.T) {
ctx := context.TODO()
setupTest := func() *clusterConfig {
context := &clusterd.Context{Clientset: test.New(t, 3)}
store := simpleStore()
store.Spec.Security = &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}
return &clusterConfig{
context: context,
store: store,
context: context,
store: store,
clusterInfo: &client.ClusterInfo{Context: ctx},
}
}
configureKMS := func(c *clusterConfig) {
@@ -458,7 +460,7 @@ func TestCheckRGWKMS(t *testing.T) {
"token": []byte("myt-otkenbenvqrev"),
},
}
_, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(context.TODO(), s, metav1.CreateOptions{})
_, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(ctx, s, metav1.CreateOptions{})
assert.NoError(t, err)
}
@@ -0,0 +1,8 @@
---
apiVersion: v1
kind: Secret
metadata:
name: rook-ibm-kp-token
namespace: rook-ceph
data:
IBM_KP_SERVICE_API_KEY: $IBM_KP_SERVICE_API_KEY
+7
View File
@@ -0,0 +1,7 @@
spec:
security:
kms:
connectionDetails:
KMS_PROVIDER: ibm-kp
IBM_KP_INSTANCE_ID: $IBM_KP_INSTANCE_ID
tokenSecretName: rook-ibm-kp-token