diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 3fd6caeba..f331e0dc6 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -75,7 +75,7 @@ jobs: run: | toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done" - + - name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag run: | toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') @@ -823,3 +823,29 @@ jobs: if: failure() && github.event_name == 'pull_request' uses: mxschmitt/action-tmate@v3 timeout-minutes: 60 + + encryption-pvc-kms-ibm-kp: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: run Encryption KMS IBM Key Protect + uses: ./.github/workflows/encryption-pvc-kms-ibm-kp + with: + ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }} + ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: upload test result + uses: actions/upload-artifact@v2 + if: always() + with: + name: encryption-pvc-kms-ibm-kp + path: test + + - name: setup tmate session for debugging when event is PR + if: failure() && github.event_name == 'pull_request' + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 60 diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index 110696a63..7c7abb156 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -316,3 +316,24 @@ jobs: with: name: rgw-multisite-testing-ceph-${{ matrix.ceph-image-tag }} path: test + + encryption-pvc-kms-ibm-kp: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: run Encryption KMS IBM Key Protect + uses: ./.github/workflows/encryption-pvc-kms-ibm-kp + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + ibm-instance-id: ${{ secrets.IBM_KP_INSTANCE_ID }} + ibm-service-api-key: ${{ secrets.IBM_KP_SERVICE_API_KEY }} + + - name: upload test result + uses: actions/upload-artifact@v2 + if: always() + with: + name: encryption-pvc-kms-ibm-kp + path: test diff --git a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml new file mode 100644 index 000000000..4b83110a8 --- /dev/null +++ b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml @@ -0,0 +1,82 @@ +name: Encryption KMS IBM Key Protect +description: Reusable workflow to test Encryption KMS IBM Key Protect +inputs: + ibm-instance-id: + description: IBM_KP_INSTANCE_ID from the calling workflow + required: true + ibm-service-api-key: + description: IBM_KP_SERVICE_API_KEY from the calling workflow + required: true + github-token: + description: GITHUB_TOKEN from the calling workflow + required: true + +runs: + using: "composite" + steps: + - name: fail if env no present + if: "env.IBM_KP_INSTANCE_ID == '' || env.IBM_KP_SERVICE_API_KEY == ''" + env: + IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }} + IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }} + shell: bash --noprofile --norc -eo pipefail -x {0} + run: echo "IBM_KP_INSTANCE_ID and IBM_KP_SERVICE_API_KEY must be set in the environment" && exit 0 + + - name: setup cluster resources + uses: ./.github/workflows/setup-cluster-resources + with: + github-token: ${{ inputs.github-token }} + + - name: use local disk and create partitions for osds + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/github-action-helper.sh create_partitions_for_osds + + - name: create cluster prerequisites + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/github-action-helper.sh create_cluster_prerequisites + + - name: deploy cluster + shell: bash --noprofile --norc -eo pipefail -x {0} + env: + IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }} + IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }} + run: | + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml + envsubst < "tests/manifests/test-kms-ibm-kp-secret.in" > "tests/manifests/test-kms-ibm-kp-secret.yaml" + envsubst < "tests/manifests/test-kms-ibm-kp-spec.in" > "tests/manifests/test-kms-ibm-kp-spec.yaml" + cat tests/manifests/test-kms-ibm-kp-secret.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml + yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-ibm-kp-spec.yaml + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi + kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml + tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml + + - name: wait for prepare pod + shell: bash --noprofile --norc -eo pipefail -x {0} + run: tests/scripts/github-action-helper.sh wait_for_prepare_pod + + - name: wait for ceph to be ready + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 + kubectl -n rook-ceph get pods + kubectl -n rook-ceph get secrets + + - name: validate encrypted osd + shell: bash --noprofile --norc -eo pipefail -x {0} + run: | + sudo lsblk + + - name: collect common logs + if: always() + uses: ./.github/workflows/collect-logs + with: + name: encryption-pvc-kms-ibm-kp + + - name: teardown cluster so that keys are removed from the KMS + shell: bash --noprofile --norc -eo pipefail -x {0} + run: kubectl -n rook-ceph delete cephcluster rook-ceph --wait diff --git a/Documentation/ceph-kms.md b/Documentation/ceph-kms.md index e1a4609ce..24f0d373b 100644 --- a/Documentation/ceph-kms.md +++ b/Documentation/ceph-kms.md @@ -20,6 +20,8 @@ The `security` section contains settings related to encryption of the cluster. Supported KMS providers: * [Vault](#vault) +* [IBM Key Protect](#ibm-kp) + ## Vault Rook supports storing OSD encryption keys in [HashiCorp Vault KMS](https://www.vaultproject.io/). @@ -225,3 +227,54 @@ data: Note: if you are using self-signed certificates (not known/approved by a proper CA) you must pass `VAULT_SKIP_VERIFY: true`. Communications will remain encrypted but the validity of the certificate will not be verified. + +## IBM Key Protect + +Rook supports storing OSD encryption keys in [IBM Key +Protect](https://www.ibm.com/cloud/key-protect). The current implementation stores OSD encryption +keys as [Standard Keys](https://cloud.ibm.com/docs/key-protect?topic=key-protect-envelope-encryption#key-types) using the [Bring Your Own +Key](https://cloud.ibm.com/docs/key-protect?topic=key-protect-importing-keys) (BYOK) method. This +means that the Key Protect instance policy must have Standard Imported Key enabled. + +### Configuration + +First, you need to [provision the Key Protect service](https://cloud.ibm.com/docs/key-protect?topic=key-protect-provision) on the IBM Cloud. Once +completed, [retrieve the instance ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID&interface=ui). +Make a record of it; we need it in the CRD. + +On the IBM Cloud, the user must create a Service ID, then assign an Access Policy to this service. +Ultimately, a Service API Key needs to be generated. All the steps are summarized in the [official +documentation](https://www.ibm.com/docs/en/cloud-private/3.2.0?topic=dg-creating-service-id-by-using-cloud-private-management-console). + +The Service ID must be granted access to the Key Protect Service. Once the Service API Key is +generated, store it in a Kubernetes Secret. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: ibm-kp-svc-api-key + namespace: rook-ceph +data: + IBM_KP_SERVICE_API_KEY: +``` + +In order for Rook to connect to IBM Key Protect, you must configure the following in your `CephCluster` template: + +```yaml +security: + kms: + # name of the k8s config map containing all the kms connection details + connectionDetails: + KMS_PROVIDER: ibm-kp + IBM_KP_INSTANCE_ID: + # name of the k8s secret containing the service API Key + tokenSecretName: ibm-kp-svc-api-key +``` + +More options are supported such as: + +* `IBM_BASE_URL`: the base URL of the Key Protect instance, depending on your + [region](https://cloud.ibm.com/docs/key-protect?topic=key-protect-regions). Defaults to `https://us-south.kms.cloud.ibm.com`. +* `IBM_TOKEN_URL`: the URL of the Key Protect instance to retrieve the token. Defaults to + `https://iam.cloud.ibm.com/oidc/token`. Only needed for private instances. diff --git a/cmd/rook/secret.go b/cmd/rook/secret.go index 9b975e256..7e5391fe6 100644 --- a/cmd/rook/secret.go +++ b/cmd/rook/secret.go @@ -73,7 +73,7 @@ func startSecret() *kms.Config { } // Validate connection details - err = kms.ValidateConnectionDetails(context, &cephCluster.Spec.Security, namespace) + err = kms.ValidateConnectionDetails(ctx, context, &cephCluster.Spec.Security, namespace) if err != nil { rook.TerminateFatal(errors.Wrap(err, "failed to validate kms connection details")) } @@ -93,9 +93,14 @@ func cliGetSecret() *cobra.Command { } func getSecret(cmd *cobra.Command, args []string) { + // Initialize the context + ctx, cancel := signal.NotifyContext(context.Background(), operator.ShutdownSignals...) + defer cancel() + secretName := args[0] secretPath := args[1] keyManagementService := startSecret() + keyManagementService.ClusterInfo.Context = ctx // Fetch the secret s, err := keyManagementService.GetSecret(secretName) diff --git a/go.mod b/go.mod index 2ae87307e..42313a507 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rook/rook go 1.16 require ( + github.com/IBM/keyprotect-go-client v0.7.0 github.com/aws/aws-sdk-go v1.37.19 github.com/banzaicloud/k8s-objectmatcher v1.1.0 github.com/ceph/go-ceph v0.12.0 diff --git a/go.sum b/go.sum index 088096cce..700c720d4 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dX github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3TveR88VmL1u3njyI= +github.com/IBM/keyprotect-go-client v0.7.0 h1:JstSHD14Lp6ihwQseyPuGcs1AjOBjAmcisP0dTBA6A0= +github.com/IBM/keyprotect-go-client v0.7.0/go.mod h1:SVr2ylV/fhSQPDiUjWirN9fsyWFCNNbt8GIT8hPJVjE= github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E= github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -731,6 +733,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= @@ -2081,6 +2084,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0= gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= diff --git a/pkg/apis/ceph.rook.io/v1/security.go b/pkg/apis/ceph.rook.io/v1/security.go index 531e8529a..01f678898 100644 --- a/pkg/apis/ceph.rook.io/v1/security.go +++ b/pkg/apis/ceph.rook.io/v1/security.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/hashicorp/vault/api" + "github.com/libopenstorage/secrets" "github.com/libopenstorage/secrets/vault" ) @@ -42,6 +43,16 @@ func (kms *KeyManagementServiceSpec) IsK8sAuthEnabled() bool { return getParam(kms.ConnectionDetails, vault.AuthMethod) == vault.AuthMethodKubernetes && kms.TokenSecretName == "" } +// IsVaultKMS return whether Vault KMS is configured +func (kms *KeyManagementServiceSpec) IsVaultKMS() bool { + return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeVault +} + +// IsIBMKeyProtectKMS return whether IBM Key Protect KMS is configured +func (kms *KeyManagementServiceSpec) IsIBMKeyProtectKMS() bool { + return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeIBM +} + // IsTLSEnabled return KMS TLS details are configured func (kms *KeyManagementServiceSpec) IsTLSEnabled() bool { for _, tlsOption := range VaultTLSConnectionDetails { diff --git a/pkg/daemon/ceph/osd/encryption.go b/pkg/daemon/ceph/osd/encryption.go index 74eea9f69..9bf2f657a 100644 --- a/pkg/daemon/ceph/osd/encryption.go +++ b/pkg/daemon/ceph/osd/encryption.go @@ -63,28 +63,48 @@ func dmsetupVersion(context *clusterd.Context) error { func setKEKinEnv(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error { // KMS details are passed by the Operator as env variables in the pod - // The token if any is mounted in the provisioner pod as an env variable so the secrets lib will pick it up - kmsConfig := kms.NewConfig(context, &v1.ClusterSpec{Security: v1.SecuritySpec{KeyManagementService: v1.KeyManagementServiceSpec{ConnectionDetails: kms.ConfigEnvsToMapString()}}}, clusterInfo) - if kmsConfig.IsVault() { - // Fetch the KEK - kek, err := kmsConfig.GetSecret(os.Getenv(oposd.PVCNameEnvVarName)) - if err != nil { - return errors.Wrapf(err, "failed to retrieve key encryption key from %q kms", kmsConfig.Provider) - } + // The token if any is mounted in the provisioner pod as an env variable so the secrets lib will + // pick it up + clusterSpec := &v1.ClusterSpec{Security: v1.SecuritySpec{KeyManagementService: v1.KeyManagementServiceSpec{ConnectionDetails: kms.ConfigEnvsToMapString()}}} - if kek == "" { - return errors.New("key encryption key is empty") - } - - // Set the KEK as an env variable for ceph-volume - err = os.Setenv(oposd.CephVolumeEncryptedKeyEnvVarName, kek) - if err != nil { - return errors.Wrap(err, "failed to set key encryption key env variable for ceph-volume") - } - - logger.Debug("successfully set vault env variables") + // If KMS is not enabled this code does not need to run since we attach the KEK value from the + // Kubernetes Secret in the provision pod spec (mounted as an environment variable) + if !clusterSpec.Security.KeyManagementService.IsEnabled() { + logger.Debug("cluster-wide encryption is enabled with kubernetes secrets and the kek is attached to the provision env spec") + return nil } + // The ibm key protect library does not read any environment variables, so we must set the + // service api key (coming from the secret mounted as environment variable) in the KMS + // connection details. These details are used to build the client connection + if clusterSpec.Security.KeyManagementService.IsIBMKeyProtectKMS() { + ibmServiceApiKey := os.Getenv(kms.IbmKeyProtectServiceApiKey) + if ibmServiceApiKey == "" { + return errors.Errorf("ibm key protect %q environment variable is not set", kms.IbmKeyProtectServiceApiKey) + } + clusterSpec.Security.KeyManagementService.ConnectionDetails[kms.IbmKeyProtectServiceApiKey] = ibmServiceApiKey + } + + kmsConfig := kms.NewConfig(context, clusterSpec, clusterInfo) + + // Fetch the KEK + kek, err := kmsConfig.GetSecret(os.Getenv(oposd.PVCNameEnvVarName)) + if err != nil { + return errors.Wrapf(err, "failed to retrieve key encryption key from %q kms", kmsConfig.Provider) + } + + if kek == "" { + return errors.New("key encryption key is empty") + } + + // Set the KEK as an env variable for ceph-volume + err = os.Setenv(oposd.CephVolumeEncryptedKeyEnvVarName, kek) + if err != nil { + return errors.Wrap(err, "failed to set key encryption key env variable for ceph-volume") + } + + logger.Debug("successfully set vault env variables") + return nil } diff --git a/pkg/daemon/ceph/osd/kms/envs.go b/pkg/daemon/ceph/osd/kms/envs.go index 028eaf5a3..3e7e82996 100644 --- a/pkg/daemon/ceph/osd/kms/envs.go +++ b/pkg/daemon/ceph/osd/kms/envs.go @@ -30,7 +30,7 @@ import ( ) var ( - knownKMSPrefix = []string{"VAULT_"} + knownKMSPrefix = []string{"VAULT_", "IBM_"} ) // VaultTokenEnvVarFromSecret returns the kms token secret value as an env var @@ -48,6 +48,21 @@ func vaultTokenEnvVarFromSecret(tokenSecretName string) v1.EnvVar { } } +// ibmKeyProtectServiceAPIKeyEnvVarFromSecret returns the kms token secret value as an env var +func ibmKeyProtectServiceAPIKeyEnvVarFromSecret(tokenSecretName string) v1.EnvVar { + return v1.EnvVar{ + Name: IbmKeyProtectServiceApiKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: tokenSecretName, + }, + Key: IbmKeyProtectServiceApiKey, + }, + }, + } +} + // vaultTLSEnvVarFromSecret translates TLS env var which are set to k8s secret name to their actual path on the fs once mounted as volume // See: VaultSecretVolumeAndMount() for more details func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar { @@ -63,30 +78,48 @@ func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar { return vaultTLSEnvVar } -// VaultConfigToEnvVar populates the kms config as env variables -func VaultConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar { +// ConfigToEnvVar populates the kms config as env variables +func ConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar { envs := []v1.EnvVar{} - backendPath := GetParam(spec.Security.KeyManagementService.ConnectionDetails, vault.VaultBackendPathKey) - // Set BACKEND_PATH to the API's default if not passed - if backendPath == "" { - spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey] = vault.DefaultBackendPath - } - for k, v := range spec.Security.KeyManagementService.ConnectionDetails { - // Skip TLS and token env var to avoid env being set multiple times - toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken) - if sets.NewString(toSkip...).Has(k) { - continue + + if spec.Security.KeyManagementService.IsVaultKMS() { + backendPath := GetParam(spec.Security.KeyManagementService.ConnectionDetails, vault.VaultBackendPathKey) + // Set BACKEND_PATH to the API's default if not passed + if backendPath == "" { + spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey] = vault.DefaultBackendPath } + } + + if spec.Security.KeyManagementService.IsIBMKeyProtectKMS() { + // We don't want to leak the IBM service API key to the container environment variables even + // the container is ephemeral. + // The IBM_KP_SERVICE_API_KEY content is mounted in the provisioner container as an + // environment variable from a secret + delete(spec.Security.KeyManagementService.ConnectionDetails, IbmKeyProtectServiceApiKey) + envs = append(envs, ibmKeyProtectServiceAPIKeyEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName)) + } + + for k, v := range spec.Security.KeyManagementService.ConnectionDetails { + if spec.Security.KeyManagementService.IsVaultKMS() { + // Skip TLS and token env var to avoid env being set multiple times + toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken) + if sets.NewString(toSkip...).Has(k) { + continue + } + } + envs = append(envs, v1.EnvVar{Name: k, Value: v}) } - // Add the VAULT_TOKEN - if spec.Security.KeyManagementService.IsTokenAuthEnabled() { - envs = append(envs, vaultTokenEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName)) - } + if spec.Security.KeyManagementService.IsVaultKMS() { + // Add the VAULT_TOKEN + if spec.Security.KeyManagementService.IsTokenAuthEnabled() { + envs = append(envs, vaultTokenEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName)) + } - // Add TLS env if any - envs = append(envs, vaultTLSEnvVarFromSecret(spec.Security.KeyManagementService.ConnectionDetails)...) + // Add TLS env if any + envs = append(envs, vaultTLSEnvVarFromSecret(spec.Security.KeyManagementService.ConnectionDetails)...) + } logger.Debugf("kms envs are %v", envs) diff --git a/pkg/daemon/ceph/osd/kms/envs_test.go b/pkg/daemon/ceph/osd/kms/envs_test.go index 727b580ee..aafb8b312 100644 --- a/pkg/daemon/ceph/osd/kms/envs_test.go +++ b/pkg/daemon/ceph/osd/kms/envs_test.go @@ -23,38 +23,58 @@ import ( "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - rookv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" ) func TestVaultTLSEnvVarFromSecret(t *testing.T) { - // No TLS - spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{TokenSecretName: "vault-token", ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200"}}}} - envVars := VaultConfigToEnvVar(spec) - areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool { - return envVars[i].Name < envVars[j].Name + t.Run("vault - no tls", func(t *testing.T) { + spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ + TokenSecretName: "vault-token", + ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200"}}}, + } + envVars := ConfigToEnvVar(spec) + areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool { + return envVars[i].Name < envVars[j].Name + }) + assert.True(t, areEnvVarsSorted) + assert.Equal(t, 4, len(envVars)) + assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}}) + assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_BACKEND_PATH", Value: "secret/"}) }) - assert.True(t, areEnvVarsSorted) - assert.Equal(t, 4, len(envVars)) - assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"}) - assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"}) - assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}}) - assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_BACKEND_PATH", Value: "secret/"}) - - // TLS - spec = cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{TokenSecretName: "vault-token", ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200", "VAULT_CACERT": "vault-ca-cert-secret"}}}} - envVars = VaultConfigToEnvVar(spec) - areEnvVarsSorted = sort.SliceIsSorted(envVars, func(i, j int) bool { - return envVars[i].Name < envVars[j].Name + t.Run("vault tls", func(t *testing.T) { + spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ + TokenSecretName: "vault-token", + ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200", "VAULT_CACERT": "vault-ca-cert-secret"}}}, + } + envVars := ConfigToEnvVar(spec) + areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool { + return envVars[i].Name < envVars[j].Name + }) + assert.True(t, areEnvVarsSorted) + assert.Equal(t, 5, len(envVars)) + assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}}) + }) + t.Run("ibm kp", func(t *testing.T) { + spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ + TokenSecretName: "ibm-kp-token", + ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_INSTANCE_ID": "1"}}}, + } + envVars := ConfigToEnvVar(spec) + areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool { + return envVars[i].Name < envVars[j].Name + }) + assert.True(t, areEnvVarsSorted) + assert.Equal(t, 3, len(envVars)) + assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "ibm-kp"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_INSTANCE_ID", Value: "1"}) + assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}}) }) - assert.True(t, areEnvVarsSorted) - assert.Equal(t, 5, len(envVars)) - assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"}) - assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"}) - assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"}) - assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}}) - } func TestConfigEnvsToMapString(t *testing.T) { @@ -80,7 +100,7 @@ func TestConfigEnvsToMapString(t *testing.T) { envs = ConfigEnvsToMapString() assert.Equal(t, 3, len(envs)) assert.True(t, envs["KMS_PROVIDER"] == "vault") - clusterSpec := &rookv1.ClusterSpec{Security: rookv1.SecuritySpec{KeyManagementService: rookv1.KeyManagementServiceSpec{ConnectionDetails: envs}}} + clusterSpec := &cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: envs}}} assert.True(t, clusterSpec.Security.KeyManagementService.IsEnabled()) } @@ -94,27 +114,43 @@ func TestVaultConfigToEnvVar(t *testing.T) { want []v1.EnvVar }{ { - "no backend path", - args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}}}, - []v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "secret/"}}, - }, - { - "with backend path", - args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_BACKEND_PATH": "foo/"}}}}}, - []v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "foo/"}}, - }, - { - "test with tls config", - args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_CACERT": "my-secret-name"}}}}}, + "vault - no backend path", + args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault"}}}}}, []v1.EnvVar{ + {Name: "KMS_PROVIDER", Value: "vault"}, + {Name: "VAULT_BACKEND_PATH", Value: "secret/"}, + }, + }, + { + "vault - with backend path", + args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_BACKEND_PATH": "foo/"}}}}}, + []v1.EnvVar{ + {Name: "KMS_PROVIDER", Value: "vault"}, + {Name: "VAULT_BACKEND_PATH", Value: "foo/"}, + }, + }, + { + "vault - test with tls config", + args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_CACERT": "my-secret-name"}}}}}, + []v1.EnvVar{ + {Name: "KMS_PROVIDER", Value: "vault"}, {Name: "VAULT_BACKEND_PATH", Value: "secret/"}, {Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"}, }, }, + { + "ibm kp - IBM_KP_SERVICE_API_KEY is removed from the details", + args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_SERVICE_API_KEY": "foo", "IBM_KP_INSTANCE_ID": "1"}, TokenSecretName: "ibm-kp-token"}}}}, + []v1.EnvVar{ + {Name: "IBM_KP_INSTANCE_ID", Value: "1"}, + {Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}}, + {Name: "KMS_PROVIDER", Value: "ibm-kp"}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := VaultConfigToEnvVar(tt.args.spec); !reflect.DeepEqual(got, tt.want) { + if got := ConfigToEnvVar(tt.args.spec); !reflect.DeepEqual(got, tt.want) { t.Errorf("VaultConfigToEnvVar() = %v, want %v", got, tt.want) } }) diff --git a/pkg/daemon/ceph/osd/kms/ibm_key_protect.go b/pkg/daemon/ceph/osd/kms/ibm_key_protect.go new file mode 100644 index 000000000..675a5380d --- /dev/null +++ b/pkg/daemon/ceph/osd/kms/ibm_key_protect.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kms + +import ( + kp "github.com/IBM/keyprotect-go-client" + "github.com/libopenstorage/secrets" + "github.com/pkg/errors" +) + +const ( + // IbmKeyProtectServiceApiKey is the IBM Key Protect service API key + IbmKeyProtectServiceApiKey = "IBM_KP_SERVICE_API_KEY" + // IbmKeyProtectInstanceIdKey is the IBM Key Protect instance id + IbmKeyProtectInstanceIdKey = "IBM_KP_INSTANCE_ID" + // IbmKeyProtectBaseUrlKey is the IBM Key Protect base url + IbmKeyProtectBaseUrlKey = "IBM_KP_BASE_URL" + // IbmKeyProtectTokenUrlKey is the IBM Key Protect token url + // #nosec G101 since it's just an URL + IbmKeyProtectTokenUrlKey = "IBM_KP_TOKEN_URL" +) + +var ( + kmsIBMKeyProtectMandatoryTokenDetails = []string{IbmKeyProtectServiceApiKey} + kmsIBMKeyProtectMandatoryConnectionDetails = []string{IbmKeyProtectInstanceIdKey, IbmKeyProtectServiceApiKey} + // ErrIbmServiceApiKeyNotSet is returned when IBM_KP_SERVICE_API_KEY is not set + ErrIbmServiceApiKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectServiceApiKey) + // ErrIbmInstanceIdKeyNotSet is returned when IBM_KP_INSTANCE_ID is not set + ErrIbmInstanceIdKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectInstanceIdKey) +) + +// InitKeyProtect initializes the KeyProtect KMS. +// With native go client directly "github.com/IBM/keyprotect-go-client" +func InitKeyProtect(config map[string]string) (*kp.Client, error) { + serviceApiKey := GetParam(config, IbmKeyProtectServiceApiKey) + if serviceApiKey == "" { + return nil, ErrIbmServiceApiKeyNotSet + } + + instanceId := GetParam(config, IbmKeyProtectInstanceIdKey) + if instanceId == "" { + return nil, ErrIbmInstanceIdKeyNotSet + } + + baseUrl := GetParam(config, IbmKeyProtectBaseUrlKey) + if baseUrl == "" { + baseUrl = kp.DefaultBaseURL + } + + tokenUrl := GetParam(config, IbmKeyProtectTokenUrlKey) + if tokenUrl == "" { + tokenUrl = kp.DefaultTokenURL + } + + cc := kp.ClientConfig{ + BaseURL: baseUrl, + APIKey: serviceApiKey, + TokenURL: tokenUrl, + InstanceID: instanceId, + Verbose: kp.VerboseAll, + } + + kp, err := kp.New(cc, nil) + if err != nil { + return nil, err + } + + return kp, nil +} + +// IsIBMKeyProtect determines whether the configured KMS is IBM Key Protect +func (c *Config) IsIBMKeyProtect() bool { return c.Provider == secrets.TypeIBM } diff --git a/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go b/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go new file mode 100644 index 000000000..5a19b164a --- /dev/null +++ b/pkg/daemon/ceph/osd/kms/ibm_key_protect_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kms + +import ( + "testing" + + kp "github.com/IBM/keyprotect-go-client" + "github.com/stretchr/testify/assert" +) + +func TestInitKeyProtect(t *testing.T) { + config := map[string]string{ + "foo": "1", + } + + t.Run("IBM_KP_SERVICE_API_KEY not set", func(t *testing.T) { + _, err := InitKeyProtect(config) + assert.Error(t, err) + assert.ErrorIs(t, err, ErrIbmServiceApiKeyNotSet) + config[IbmKeyProtectServiceApiKey] = "foo" + }) + + t.Run("IBM_KP_INSTANCE_ID not set", func(t *testing.T) { + _, err := InitKeyProtect(config) + assert.Error(t, err) + assert.ErrorIs(t, err, ErrIbmInstanceIdKeyNotSet) + config[IbmKeyProtectInstanceIdKey] = "foo" + }) + + t.Run("default base URL", func(t *testing.T) { + c, err := InitKeyProtect(config) + assert.NoError(t, err) + assert.Equal(t, kp.DefaultBaseURL, c.Config.BaseURL) + }) + + t.Run("different base URL", func(t *testing.T) { + config[IbmKeyProtectBaseUrlKey] = "https://us-west.kms.cloud.ibm.com" + c, err := InitKeyProtect(config) + assert.NoError(t, err) + assert.Equal(t, "https://us-west.kms.cloud.ibm.com", c.Config.BaseURL) + }) + + t.Run("default base token URL", func(t *testing.T) { + c, err := InitKeyProtect(config) + assert.NoError(t, err) + assert.Equal(t, kp.DefaultTokenURL, c.Config.TokenURL) + }) + + t.Run("different base URL", func(t *testing.T) { + config[IbmKeyProtectTokenUrlKey] = "new" + c, err := InitKeyProtect(config) + assert.NoError(t, err) + assert.Equal(t, "new", c.Config.TokenURL) + }) +} diff --git a/pkg/daemon/ceph/osd/kms/k8s.go b/pkg/daemon/ceph/osd/kms/k8s.go index bd4860aef..c41ff2c6a 100644 --- a/pkg/daemon/ceph/osd/kms/k8s.go +++ b/pkg/daemon/ceph/osd/kms/k8s.go @@ -41,13 +41,13 @@ const ( // storeSecretInKubernetes stores the dmcrypt key in a Kubernetes Secret func (c *Config) storeSecretInKubernetes(pvcName, key string) error { - s, err := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo) + s, err := generateOSDEncryptedKeySecret(pvcName, key, c.ClusterInfo) if err != nil { return err } // Create the Kubernetes Secret - _, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, s, metav1.CreateOptions{}) + _, err = c.context.Clientset.CoreV1().Secrets(c.ClusterInfo.Namespace).Create(c.ClusterInfo.Context, s, metav1.CreateOptions{}) if err != nil && !kerrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to save ceph osd encryption key as a secret for pvc %q", pvcName) } diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go index 48f63ca72..ab005c2bf 100644 --- a/pkg/daemon/ceph/osd/kms/kms.go +++ b/pkg/daemon/ceph/osd/kms/kms.go @@ -21,6 +21,7 @@ import ( "os" "strings" + kp "github.com/IBM/keyprotect-go-client" "github.com/coreos/pkg/capnslog" "github.com/hashicorp/vault/api" "github.com/libopenstorage/secrets" @@ -47,14 +48,14 @@ type Config struct { Provider string context *clusterd.Context clusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo + ClusterInfo *cephclient.ClusterInfo } // NewConfig returns the selected KMS func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) *Config { config := &Config{ context: context, - clusterInfo: clusterInfo, + ClusterInfo: clusterInfo, clusterSpec: clusterSpec, } @@ -64,6 +65,8 @@ func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clust config.Provider = secrets.TypeK8s case secrets.TypeVault: config.Provider = secrets.TypeVault + case secrets.TypeIBM: + config.Provider = secrets.TypeIBM default: logger.Errorf("unsupported kms type %q", Provider) } @@ -83,16 +86,34 @@ func (c *Config) PutSecret(secretName, secretValue string) error { } if c.IsVault() { // Store the secret in Vault - v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) if err != nil { return errors.Wrap(err, "failed to init vault kms") } - k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) err = put(v, GenerateOSDEncryptionSecretName(secretName), secretValue, k) if err != nil { return errors.Wrap(err, "failed to put secret in vault") } } + if c.IsIBMKeyProtect() { + kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + if err != nil { + return errors.Wrap(err, "failed to init ibm key protect") + } + + // Create the key is not present + keyAlias := []string{secretName} + _, err = kpClient.CreateImportedKeyWithAliases(c.ClusterInfo.Context, secretName, nil, secretValue, "", "", true, keyAlias) + if err != nil { + if strings.Contains(err.Error(), "KEY_ALIAS_NOT_UNIQUE_ERR") { + logger.Debugf("key %q already exists. %v", secretName, err) + return nil + } + + return errors.Wrap(err, "failed to put secret in ibm key protect") + } + } return nil } @@ -102,17 +123,28 @@ func (c *Config) GetSecret(secretName string) (string, error) { var value string if c.IsVault() { // Store the secret in Vault - v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) if err != nil { return "", errors.Wrap(err, "failed to init vault") } - k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) value, err = get(v, GenerateOSDEncryptionSecretName(secretName), k) if err != nil { - return "", errors.Wrap(err, "failed to get secret in vault") + return "", errors.Wrap(err, "failed to get secret from vault") } } + if c.IsIBMKeyProtect() { + kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + if err != nil { + return "", errors.Wrap(err, "failed to init ibm key protect") + } + keyObject, err := kpClient.GetKey(c.ClusterInfo.Context, secretName) + if err != nil { + return "", errors.Wrap(err, "failed to get secret from ibm key protect") + } + value = string(keyObject.Payload) + } return value, nil } @@ -121,21 +153,49 @@ func (c *Config) GetSecret(secretName string) (string, error) { func (c *Config) DeleteSecret(secretName string) error { if c.IsVault() { // Store the secret in Vault - v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) if err != nil { return errors.Wrap(err, "failed to delete secret in vault") } - k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) // Force removal of all the versions of the secret on K/V version 2 k[secrets.DestroySecret] = "true" - err = delete(v, GenerateOSDEncryptionSecretName(secretName), k) + err = deleteSecret(v, GenerateOSDEncryptionSecretName(secretName), k) if err != nil { return errors.Wrap(err, "failed to delete secret in vault") } } + if c.IsIBMKeyProtect() { + kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + if err != nil { + return errors.Wrap(err, "failed to init ibm key protect") + } + + // We use context.TODO() since the clusterInfo context has been cancelled by the CephCluster's + // deletion event + ctx := context.TODO() + + // Fetch the key to get the ID + key, err := kpClient.GetKey(ctx, secretName) + if err != nil { + return errors.Wrap(err, "failed to get secret in ibm key protect") + } + + // DeleteKey does not support deleting secret with the alias name so we must use the ID + // After you delete a key, the key transitions to the Destroyed state. Any data encrypted by + // keys in this state is no longer accessible. Metadata that is associated with the key, + // such as the key's deletion date, is kept in the Key Protect database. Destroyed keys can + // be recovered after up to 30 days or their expiration date, whichever is sooner. After 30 + // days, keys can no longer be recovered, and become eligible to be purged after 90 days, a + // process that shreds the key material and makes its metadata inaccessible. + _, err = kpClient.DeleteKey(ctx, key.ID, kp.ReturnRepresentation, []kp.CallOpt{kp.ForceOpt{Force: true}}...) + if err != nil { + return errors.Wrap(err, "failed to delete secret in ibm key protect") + } + } return nil } @@ -149,14 +209,21 @@ func GetParam(kmsConfig map[string]string, param string) string { } // ValidateConnectionDetails validates mandatory KMS connection details -func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error { - ctx := context.TODO() +func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error { + // Lookup mandatory connection details + for _, config := range kmsMandatoryConnectionDetails { + if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" { + return errors.Errorf("failed to validate kms config %q. cannot be empty", config) + } + } + // A token must be specified if token-auth is used if !securitySpec.KeyManagementService.IsK8sAuthEnabled() && securitySpec.KeyManagementService.TokenSecretName == "" { if !securitySpec.KeyManagementService.IsTokenAuthEnabled() { return errors.New("failed to validate kms configuration (missing token in spec)") } } + // KMS provider must be specified provider := GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider) @@ -167,32 +234,35 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec * return errors.Wrapf(err, "failed to fetch kms token secret %q", securitySpec.KeyManagementService.TokenSecretName) } - // Check for empty token - token, ok := kmsToken.Data[KMSTokenSecretNameKey] - if !ok || len(token) == 0 { - return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName) - } - switch provider { - case "vault": + case secrets.TypeVault: + // Check for empty token + token, ok := kmsToken.Data[KMSTokenSecretNameKey] + if !ok || len(token) == 0 { + return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName) + } + // Set the env variable err = os.Setenv(api.EnvVaultToken, string(token)) if err != nil { return errors.Wrap(err, "failed to set vault kms token to an env var") } - } - } - // Lookup mandatory connection details - for _, config := range kmsMandatoryConnectionDetails { - if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" { - return errors.Errorf("failed to validate kms config %q. cannot be empty", config) + case secrets.TypeIBM: + for _, config := range kmsIBMKeyProtectMandatoryTokenDetails { + v, ok := kmsToken.Data[config] + if !ok || len(v) == 0 { + return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", config, securitySpec.KeyManagementService.TokenSecretName) + } + // Append the token secret details to the connection details + securitySpec.KeyManagementService.ConnectionDetails[config] = strings.TrimSuffix(strings.TrimSpace(string(v)), "\n") + } } } // Validate KMS provider connection details for each provider switch provider { - case "vault": + case secrets.TypeVault: err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails) if err != nil { return errors.Wrap(err, "failed to validate vault connection details") @@ -210,6 +280,14 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec * securitySpec.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = backendVersion } } + + case secrets.TypeIBM: + for _, config := range kmsIBMKeyProtectMandatoryConnectionDetails { + if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" { + return errors.Errorf("failed to validate kms config %q. cannot be empty", config) + } + } + default: return errors.Errorf("failed to validate kms provider connection details (provider %q not supported)", provider) } @@ -218,8 +296,7 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec * } // SetTokenToEnvVar sets a KMS token as an env variable -func SetTokenToEnvVar(clusterdContext *clusterd.Context, tokenSecretName, provider, namespace string) error { - ctx := context.TODO() +func SetTokenToEnvVar(ctx context.Context, clusterdContext *clusterd.Context, tokenSecretName, provider, namespace string) error { // Get the secret containing the kms token kmsToken, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, tokenSecretName, metav1.GetOptions{}) if err != nil { diff --git a/pkg/daemon/ceph/osd/kms/kms_test.go b/pkg/daemon/ceph/osd/kms/kms_test.go index b84a0dbd7..180f9b32b 100644 --- a/pkg/daemon/ceph/osd/kms/kms_test.go +++ b/pkg/daemon/ceph/osd/kms/kms_test.go @@ -37,83 +37,97 @@ func TestValidateConnectionDetails(t *testing.T) { context := &clusterd.Context{Clientset: test.New(t, 3)} securitySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}} ns := "rook-ceph" - - // Error: no token in spec - err := ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate kms configuration (missing token in spec)") - - securitySpec.KeyManagementService.TokenSecretName = "vault-token" - - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found") - - // Error: token secret present but empty content - s := &v1.Secret{ + vaultSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: securitySpec.KeyManagementService.TokenSecretName, + Name: "vault-token", Namespace: ns, }, } - _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{}) - assert.NoError(t, err) - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)") - - // Error: token key does not exist - s.Data = map[string][]byte{"foo": []byte("bar")} - _, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{}) - assert.NoError(t, err) - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)") - - // Success: token content is ok - s.Data["token"] = []byte("token") - _, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{}) - assert.NoError(t, err) - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty") - securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault" - - // Error: Data has a KMS_PROVIDER but missing details - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"") - - // Error: connection details are correct but the token secret does not exist - securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200" - - // Error: TLS is configured but secrets do not exist - securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"") - - // Error: TLS secret exists but empty key + ibmSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ibm-token", + Namespace: ns, + }, + Data: map[string][]byte{"foo": []byte("bar")}, + } tlsSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "vault-ca-secret", Namespace: ns, }, } - _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, tlsSecret, metav1.CreateOptions{}) - assert.NoError(t, err) - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"") + ibmSecuritySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ + ConnectionDetails: map[string]string{ + "KMS_PROVIDER": "ibm-kp", + }, + }} - // Success: TLS config is correct - tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")} - _, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, tlsSecret, metav1.UpdateOptions{}) - assert.NoError(t, err) - err = ValidateConnectionDetails(context, securitySpec, ns) - assert.NoError(t, err, "") + t.Run("no kms provider given", func(t *testing.T) { + err := ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty") + securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault" + }) - // test with vauult server + t.Run("vault - no token object", func(t *testing.T) { + securitySpec.KeyManagementService.TokenSecretName = "vault-token" + err := ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found") + }) + + t.Run("vault - token secret present but empty content", func(t *testing.T) { + _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, vaultSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)") + }) + + t.Run("vault - token key does not exist", func(t *testing.T) { + vaultSecret.Data = map[string][]byte{"foo": []byte("bar")} + _, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, vaultSecret, metav1.UpdateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)") + }) + + // Success: token content is ok + t.Run("vault - token content is ok", func(t *testing.T) { + vaultSecret.Data["token"] = []byte("token") + _, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, vaultSecret, metav1.UpdateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"") + securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200" + }) + + t.Run("vault - TLS is configured but secrets do not exist", func(t *testing.T) { + securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" + err := ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"") + }) + + t.Run("vault - TLS secret exists but empty key", func(t *testing.T) { + _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, tlsSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"") + }) + + t.Run("vault - success TLS config is correct", func(t *testing.T) { + tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")} + _, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, tlsSecret, metav1.UpdateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, securitySpec, ns) + assert.NoError(t, err, "") + }) + + // test with vault server t.Run("success - auto detect kv version and set it", func(t *testing.T) { cluster := fakeVaultServer(t) cluster.Start() @@ -142,11 +156,44 @@ func TestValidateConnectionDetails(t *testing.T) { TokenSecretName: "vault-token", }, } - err = ValidateConnectionDetails(context, securitySpec, ns) + err := ValidateConnectionDetails(ctx, context, securitySpec, ns) assert.NoError(t, err, "") assert.Equal(t, securitySpec.KeyManagementService.ConnectionDetails["VAULT_BACKEND"], "v2") }) + t.Run("ibm kp - fail no token specified, only token is supported", func(t *testing.T) { + err := ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate kms configuration (missing token in spec)") + ibmSecuritySpec.KeyManagementService.TokenSecretName = "ibm-token" + + }) + + t.Run("ibm kp - token present but no key for service key", func(t *testing.T) { + _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, ibmSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to read k8s kms secret \"IBM_KP_SERVICE_API_KEY\" key \"ibm-token\" (not found or empty)") + }) + + t.Run("ibm kp - token ok but no instance id", func(t *testing.T) { + ibmSecret.Data["IBM_KP_SERVICE_API_KEY"] = []byte("foo") + _, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, ibmSecret, metav1.UpdateOptions{}) + assert.NoError(t, err) + err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate kms config \"IBM_KP_INSTANCE_ID\". cannot be empty") + ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_INSTANCE_ID"] = "foo" + }) + + t.Run("ibm kp - success", func(t *testing.T) { + err := ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns) + assert.NoError(t, err, "") + // IBM_KP_SERVICE_API_KEY must be appended to the details so that the client can be built with + // all the details + assert.Equal(t, ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_SERVICE_API_KEY"], "foo") + }) } func TestSetTokenToEnvVar(t *testing.T) { @@ -164,7 +211,7 @@ func TestSetTokenToEnvVar(t *testing.T) { _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{}) assert.NoError(t, err) - err = SetTokenToEnvVar(context, secretName, "vault", ns) + err = SetTokenToEnvVar(ctx, context, secretName, "vault", ns) assert.NoError(t, err) assert.Equal(t, os.Getenv("VAULT_TOKEN"), "toto") os.Unsetenv("VAULT_TOKEN") diff --git a/pkg/daemon/ceph/osd/kms/vault.go b/pkg/daemon/ceph/osd/kms/vault.go index 182c55e1d..8fe99810f 100644 --- a/pkg/daemon/ceph/osd/kms/vault.go +++ b/pkg/daemon/ceph/osd/kms/vault.go @@ -222,7 +222,7 @@ func get(v secrets.Secrets, secretName string, keyContext map[string]string) (st return s[secretName].(string), nil } -func delete(v secrets.Secrets, secretName string, keyContext map[string]string) error { +func deleteSecret(v secrets.Secrets, secretName string, keyContext map[string]string) error { // #nosec G104 Write the encryption key in Vault err := v.DeleteSecret(secretName, keyContext) if err != nil { @@ -232,7 +232,7 @@ func delete(v secrets.Secrets, secretName string, keyContext map[string]string) return nil } -func buildKeyContext(config map[string]string) map[string]string { +func buildVaultKeyContext(config map[string]string) map[string]string { // Key context is just the Vault namespace, available in the enterprise version only keyContext := map[string]string{secrets.KeyVaultNamespace: config[api.EnvVaultNamespace]} vaultNamespace, ok := config[api.EnvVaultNamespace] @@ -245,7 +245,7 @@ func buildKeyContext(config map[string]string) map[string]string { // IsVault determines whether the configured KMS is Vault func (c *Config) IsVault() bool { - return c.Provider == "vault" + return c.Provider == secrets.TypeVault } func validateVaultConnectionDetails(clusterdContext *clusterd.Context, ns string, kmsConfig map[string]string) error { diff --git a/pkg/daemon/ceph/osd/kms/vault_test.go b/pkg/daemon/ceph/osd/kms/vault_test.go index adbe16148..287abbb62 100644 --- a/pkg/daemon/ceph/osd/kms/vault_test.go +++ b/pkg/daemon/ceph/osd/kms/vault_test.go @@ -254,13 +254,13 @@ func Test_configTLS(t *testing.T) { }) } -func Test_buildKeyContext(t *testing.T) { +func Test_buildVaultKeyContext(t *testing.T) { t.Run("no vault namespace, return empty map and assignment is possible", func(t *testing.T) { config := map[string]string{ "KMS_PROVIDER": "vault", "VAULT_ADDR": "1.1.1.1", } - context := buildKeyContext(config) + context := buildVaultKeyContext(config) assert.Len(t, context, 0) context["foo"] = "bar" }) @@ -271,7 +271,7 @@ func Test_buildKeyContext(t *testing.T) { "VAULT_ADDR": "1.1.1.1", "VAULT_NAMESPACE": "vault-namespace", } - context := buildKeyContext(config) + context := buildVaultKeyContext(config) assert.Len(t, context, 1) context["foo"] = "bar" assert.Len(t, context, 2) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 6c6a7bdaa..3a6ffdb6b 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -306,7 +306,7 @@ func preClusterStartValidation(cluster *cluster) error { // Validate on-PVC cluster encryption KMS settings if cluster.Spec.Storage.IsOnPVCEncrypted() && cluster.Spec.Security.KeyManagementService.IsEnabled() { // Validate the KMS details - err := kms.ValidateConnectionDetails(cluster.context, &cluster.Spec.Security, cluster.Namespace) + err := kms.ValidateConnectionDetails(cluster.ClusterInfo.Context, cluster.context, &cluster.Spec.Security, cluster.Namespace) if err != nil { return errors.Wrap(err, "failed to validate kms connection details") } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 01c0b680b..d232a821f 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -511,6 +511,10 @@ func (r *ReconcileCephCluster) removeFinalizer(client client.Client, name types. } func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1.CephCluster) error { + // At the point, the CephCluster has been deleted and the context cancelled, so we need to use a + // new context here + // So do NOT use another context + ctx := context.TODO() // If the operator was stopped and we enter this code, the map is empty if _, ok := c.clusterMap[currentCluster.Namespace]; !ok { c.clusterMap[currentCluster.Namespace] = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}} @@ -526,13 +530,22 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1 kmsConfig := kms.NewConfig(c.context, ¤tCluster.Spec, c.clusterMap[currentCluster.Namespace].ClusterInfo) // If token auth is used by the KMS we set it as an env variable - if currentCluster.Spec.Security.KeyManagementService.IsTokenAuthEnabled() { - err := kms.SetTokenToEnvVar(c.context, currentCluster.Spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, currentCluster.Namespace) + if currentCluster.Spec.Security.KeyManagementService.IsTokenAuthEnabled() && currentCluster.Spec.Security.KeyManagementService.IsVaultKMS() { + err := kms.SetTokenToEnvVar(ctx, c.context, currentCluster.Spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, currentCluster.Namespace) if err != nil { return errors.Wrapf(err, "failed to fetch kms token secret %q", currentCluster.Spec.Security.KeyManagementService.TokenSecretName) } } + // We need to fetch the IBM_KP_SERVICE_API_KEY value + if currentCluster.Spec.Security.KeyManagementService.IsIBMKeyProtectKMS() { + // This will validate the connection details again and will add the IBM_KP_SERVICE_API_KEY to the spec + err = kms.ValidateConnectionDetails(ctx, c.context, ¤tCluster.Spec.Security, currentCluster.Namespace) + if err != nil { + return errors.Wrap(err, "failed to validate kms connection details to delete the secret") + } + } + // Delete each PV KEK for _, osdPVC := range osdPVCs { // Generate and store the encrypted key in whatever KMS is configured diff --git a/pkg/operator/ceph/cluster/osd/create.go b/pkg/operator/ceph/cluster/osd/create.go index 5b5969db4..cd357e50a 100644 --- a/pkg/operator/ceph/cluster/osd/create.go +++ b/pkg/operator/ceph/cluster/osd/create.go @@ -218,8 +218,8 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi // We could set an env var in the Operator or a global var instead of the API call? // Hopefully, the API is cheap and we can always retrieve the token if it has changed... - if c.spec.Security.KeyManagementService.IsTokenAuthEnabled() { - err := kms.SetTokenToEnvVar(c.context, c.spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, c.clusterInfo.Namespace) + if c.spec.Security.KeyManagementService.IsTokenAuthEnabled() && c.spec.Security.KeyManagementService.IsVaultKMS() { + err := kms.SetTokenToEnvVar(c.clusterInfo.Context, c.context, c.spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, c.clusterInfo.Namespace) if err != nil { errMsg := fmt.Sprintf("failed to fetch kms token secret %q. %v", c.spec.Security.KeyManagementService.TokenSecretName, err) errs.addError(errMsg) diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 3ee7d2f78..e9dbec33f 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -21,7 +21,6 @@ import ( "fmt" "path" - "github.com/libopenstorage/secrets" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" @@ -121,8 +120,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re if osdProps.encrypted { // If a KMS is configured we populate if c.spec.Security.KeyManagementService.IsEnabled() { - kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider) - if kmsProvider == secrets.TypeVault { + if c.spec.Security.KeyManagementService.IsVaultKMS() { volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") volumes = append(volumes, volumeTLS) } @@ -280,12 +278,11 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun if osdProps.encrypted { // If a KMS is configured we populate volume mounts and env variables if c.spec.Security.KeyManagementService.IsEnabled() { - kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider) - if kmsProvider == secrets.TypeVault { + if c.spec.Security.KeyManagementService.IsVaultKMS() { _, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") volumeMounts = append(volumeMounts, volumeMountsTLS) - envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...) } + envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...) } else { envVars = append(envVars, cephVolumeRawEncryptedEnvVarFromSecret(osdProps)) } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 61ef68b01..63cb3a83a 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -23,7 +23,6 @@ import ( "path/filepath" "strconv" - "github.com/libopenstorage/secrets" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" @@ -867,7 +866,7 @@ func (c *Cluster) generateVaultGetKEK(osdProps osdProperties) v1.Container { keyName := osdProps.pvc.ClaimName keyPath := encryptionKeyPath() envVars := c.getConfigEnvVars(osdProps, "") - envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...) + envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...) return v1.Container{ Name: blockEncryptionKMSGetKEKInitContainer, @@ -889,24 +888,21 @@ func (c *Cluster) getPVCEncryptionOpenInitContainerActivate(mountPath string, os // If a KMS is enabled we need to add an init container to fetch the KEK if c.spec.Security.KeyManagementService.IsEnabled() { - kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider) - // Get Vault KEK from KMS container - if kmsProvider == secrets.TypeVault { - getKEKFromKMSContainer := c.generateVaultGetKEK(osdProps) + getKEKFromKMSContainer := c.generateVaultGetKEK(osdProps) - // Volume mount to store the encrypted key - _, volMount := c.getEncryptionVolume(osdProps) - getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, volMount) + // Volume mount to store the encrypted key + _, volMount := c.getEncryptionVolume(osdProps) + getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, volMount) + if c.spec.Security.KeyManagementService.IsVaultKMS() { // Now let's see if there is a TLS config we need to mount as well if c.spec.Security.KeyManagementService.IsTLSEnabled() { _, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, vaultVolMount) } - - // Add the container to the list of containers - containers = append(containers, getKEKFromKMSContainer) } + // Add the container to the list of containers + containers = append(containers, getKEKFromKMSContainer) } // Main block container diff --git a/pkg/operator/ceph/cluster/osd/volumes.go b/pkg/operator/ceph/cluster/osd/volumes.go index c38a462cb..aee3ccbfb 100644 --- a/pkg/operator/ceph/cluster/osd/volumes.go +++ b/pkg/operator/ceph/cluster/osd/volumes.go @@ -21,7 +21,6 @@ import ( "path" "path/filepath" - "github.com/libopenstorage/secrets" kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" "github.com/rook/rook/pkg/operator/ceph/config" v1 "k8s.io/api/core/v1" @@ -192,15 +191,6 @@ func getUdevVolume() (v1.Volume, v1.VolumeMount) { } func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.VolumeMount) { - // Determine whether we have a KMS configuration - var isKMS bool - if len(c.spec.Security.KeyManagementService.ConnectionDetails) != 0 { - provider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider) - if provider == secrets.TypeVault { - isKMS = true - } - } - // Generate volume var m int32 = 0400 volume := v1.Volume{ @@ -220,7 +210,7 @@ func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.Vol } // On the KMS use case, we want the volume mount to be in memory since we pass write the KEK - if isKMS { + if c.spec.Security.KeyManagementService.IsEnabled() { volume.VolumeSource.Secret = nil volume.VolumeSource = v1.VolumeSource{ EmptyDir: &v1.EmptyDirVolumeSource{ @@ -237,7 +227,7 @@ func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.Vol } // With KMS we must be able to write inside the directory to write the KEK - if isKMS { + if c.spec.Security.KeyManagementService.IsEnabled() { volumeMounts.ReadOnly = false } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index dc6251ea9..a654abc7b 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -568,7 +568,7 @@ func (c *clusterConfig) vaultPrefixRGW() string { func (c *clusterConfig) CheckRGWKMS() (bool, error) { if c.store.Spec.Security != nil && c.store.Spec.Security.KeyManagementService.IsEnabled() { - err := kms.ValidateConnectionDetails(c.context, c.store.Spec.Security, c.store.Namespace) + err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, c.store.Spec.Security, c.store.Namespace) if err != nil { return false, err } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 18ba1ef65..e3e407e55 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -436,13 +436,15 @@ func TestDefaultReadinessProbe(t *testing.T) { } func TestCheckRGWKMS(t *testing.T) { + ctx := context.TODO() setupTest := func() *clusterConfig { context := &clusterd.Context{Clientset: test.New(t, 3)} store := simpleStore() store.Spec.Security = &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}} return &clusterConfig{ - context: context, - store: store, + context: context, + store: store, + clusterInfo: &client.ClusterInfo{Context: ctx}, } } configureKMS := func(c *clusterConfig) { @@ -458,7 +460,7 @@ func TestCheckRGWKMS(t *testing.T) { "token": []byte("myt-otkenbenvqrev"), }, } - _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(context.TODO(), s, metav1.CreateOptions{}) + _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(ctx, s, metav1.CreateOptions{}) assert.NoError(t, err) } diff --git a/tests/manifests/test-kms-ibm-kp-secret.in b/tests/manifests/test-kms-ibm-kp-secret.in new file mode 100644 index 000000000..f02c3b622 --- /dev/null +++ b/tests/manifests/test-kms-ibm-kp-secret.in @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: rook-ibm-kp-token + namespace: rook-ceph +data: + IBM_KP_SERVICE_API_KEY: $IBM_KP_SERVICE_API_KEY diff --git a/tests/manifests/test-kms-ibm-kp-spec.in b/tests/manifests/test-kms-ibm-kp-spec.in new file mode 100644 index 000000000..4070971a8 --- /dev/null +++ b/tests/manifests/test-kms-ibm-kp-spec.in @@ -0,0 +1,7 @@ +spec: + security: + kms: + connectionDetails: + KMS_PROVIDER: ibm-kp + IBM_KP_INSTANCE_ID: $IBM_KP_INSTANCE_ID + tokenSecretName: rook-ibm-kp-token