osd: add ibm key protect kms
The cluster-wide encryption feature now has a new supported key management system: IBM Key protect. More information about the backend can be found in Rook documentation under the Today's implementation stores OSD encryption keys has "Standard" keys. Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
1 parent
511c250336
commit
a7acf490b6
28 files changed
+806
-221
No files matched your search
@@ -75,7 +75,7 @@ jobs:
|
||||
run: |
|
||||
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
||||
timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done"
|
||||
|
||||
|
||||
- name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag
|
||||
run: |
|
||||
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
|
||||
@@ -823,3 +823,29 @@ jobs:
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
timeout-minutes: 60
|
||||
|
||||
encryption-pvc-kms-ibm-kp:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: run Encryption KMS IBM Key Protect
|
||||
uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
|
||||
with:
|
||||
ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }}
|
||||
ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: upload test result
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: encryption-pvc-kms-ibm-kp
|
||||
path: test
|
||||
|
||||
- name: setup tmate session for debugging when event is PR
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
timeout-minutes: 60
|
||||
@@ -316,3 +316,24 @@ jobs:
|
||||
with:
|
||||
name: rgw-multisite-testing-ceph-${{ matrix.ceph-image-tag }}
|
||||
path: test
|
||||
|
||||
encryption-pvc-kms-ibm-kp:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: run Encryption KMS IBM Key Protect
|
||||
uses: ./.github/workflows/encryption-pvc-kms-ibm-kp
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
ibm-instance-id: ${{ secrets.IBM_KP_INSTANCE_ID }}
|
||||
ibm-service-api-key: ${{ secrets.IBM_KP_SERVICE_API_KEY }}
|
||||
|
||||
- name: upload test result
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: encryption-pvc-kms-ibm-kp
|
||||
path: test
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Encryption KMS IBM Key Protect
|
||||
description: Reusable workflow to test Encryption KMS IBM Key Protect
|
||||
inputs:
|
||||
ibm-instance-id:
|
||||
description: IBM_KP_INSTANCE_ID from the calling workflow
|
||||
required: true
|
||||
ibm-service-api-key:
|
||||
description: IBM_KP_SERVICE_API_KEY from the calling workflow
|
||||
required: true
|
||||
github-token:
|
||||
description: GITHUB_TOKEN from the calling workflow
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: fail if env no present
|
||||
if: "env.IBM_KP_INSTANCE_ID == '' || env.IBM_KP_SERVICE_API_KEY == ''"
|
||||
env:
|
||||
IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
|
||||
IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }}
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: echo "IBM_KP_INSTANCE_ID and IBM_KP_SERVICE_API_KEY must be set in the environment" && exit 0
|
||||
|
||||
- name: setup cluster resources
|
||||
uses: ./.github/workflows/setup-cluster-resources
|
||||
with:
|
||||
github-token: ${{ inputs.github-token }}
|
||||
|
||||
- name: use local disk and create partitions for osds
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh use_local_disk
|
||||
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
||||
|
||||
- name: create cluster prerequisites
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1)
|
||||
tests/scripts/github-action-helper.sh create_cluster_prerequisites
|
||||
|
||||
- name: deploy cluster
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
env:
|
||||
IBM_KP_INSTANCE_ID: ${{ inputs.ibm-instance-id }}
|
||||
IBM_KP_SERVICE_API_KEY: ${{ inputs.ibm-service-api-key }}
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml
|
||||
envsubst < "tests/manifests/test-kms-ibm-kp-secret.in" > "tests/manifests/test-kms-ibm-kp-secret.yaml"
|
||||
envsubst < "tests/manifests/test-kms-ibm-kp-spec.in" > "tests/manifests/test-kms-ibm-kp-spec.yaml"
|
||||
cat tests/manifests/test-kms-ibm-kp-secret.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-ibm-kp-spec.yaml
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
|
||||
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
|
||||
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod
|
||||
|
||||
- name: wait for ceph to be ready
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
|
||||
kubectl -n rook-ceph get pods
|
||||
kubectl -n rook-ceph get secrets
|
||||
|
||||
- name: validate encrypted osd
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: |
|
||||
sudo lsblk
|
||||
|
||||
- name: collect common logs
|
||||
if: always()
|
||||
uses: ./.github/workflows/collect-logs
|
||||
with:
|
||||
name: encryption-pvc-kms-ibm-kp
|
||||
|
||||
- name: teardown cluster so that keys are removed from the KMS
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: kubectl -n rook-ceph delete cephcluster rook-ceph --wait
|
||||
@@ -20,6 +20,8 @@ The `security` section contains settings related to encryption of the cluster.
|
||||
Supported KMS providers:
|
||||
|
||||
* [Vault](#vault)
|
||||
* [IBM Key Protect](#ibm-kp)
|
||||
|
||||
## Vault
|
||||
|
||||
Rook supports storing OSD encryption keys in [HashiCorp Vault KMS](https://www.vaultproject.io/).
|
||||
@@ -225,3 +227,54 @@ data:
|
||||
|
||||
Note: if you are using self-signed certificates (not known/approved by a proper CA) you must pass `VAULT_SKIP_VERIFY: true`.
|
||||
Communications will remain encrypted but the validity of the certificate will not be verified.
|
||||
|
||||
## IBM Key Protect
|
||||
|
||||
Rook supports storing OSD encryption keys in [IBM Key
|
||||
Protect](https://www.ibm.com/cloud/key-protect). The current implementation stores OSD encryption
|
||||
keys as [Standard Keys](https://cloud.ibm.com/docs/key-protect?topic=key-protect-envelope-encryption#key-types) using the [Bring Your Own
|
||||
Key](https://cloud.ibm.com/docs/key-protect?topic=key-protect-importing-keys) (BYOK) method. This
|
||||
means that the Key Protect instance policy must have Standard Imported Key enabled.
|
||||
|
||||
### Configuration
|
||||
|
||||
First, you need to [provision the Key Protect service](https://cloud.ibm.com/docs/key-protect?topic=key-protect-provision) on the IBM Cloud. Once
|
||||
completed, [retrieve the instance ID](https://cloud.ibm.com/docs/key-protect?topic=key-protect-retrieve-instance-ID&interface=ui).
|
||||
Make a record of it; we need it in the CRD.
|
||||
|
||||
On the IBM Cloud, the user must create a Service ID, then assign an Access Policy to this service.
|
||||
Ultimately, a Service API Key needs to be generated. All the steps are summarized in the [official
|
||||
documentation](https://www.ibm.com/docs/en/cloud-private/3.2.0?topic=dg-creating-service-id-by-using-cloud-private-management-console).
|
||||
|
||||
The Service ID must be granted access to the Key Protect Service. Once the Service API Key is
|
||||
generated, store it in a Kubernetes Secret.
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ibm-kp-svc-api-key
|
||||
namespace: rook-ceph
|
||||
data:
|
||||
IBM_KP_SERVICE_API_KEY: <service API Key>
|
||||
```
|
||||
|
||||
In order for Rook to connect to IBM Key Protect, you must configure the following in your `CephCluster` template:
|
||||
|
||||
```yaml
|
||||
security:
|
||||
kms:
|
||||
# name of the k8s config map containing all the kms connection details
|
||||
connectionDetails:
|
||||
KMS_PROVIDER: ibm-kp
|
||||
IBM_KP_INSTANCE_ID: <instance ID that was retrieved in the first paragraph>
|
||||
# name of the k8s secret containing the service API Key
|
||||
tokenSecretName: ibm-kp-svc-api-key
|
||||
```
|
||||
|
||||
More options are supported such as:
|
||||
|
||||
* `IBM_BASE_URL`: the base URL of the Key Protect instance, depending on your
|
||||
[region](https://cloud.ibm.com/docs/key-protect?topic=key-protect-regions). Defaults to `https://us-south.kms.cloud.ibm.com`.
|
||||
* `IBM_TOKEN_URL`: the URL of the Key Protect instance to retrieve the token. Defaults to
|
||||
`https://iam.cloud.ibm.com/oidc/token`. Only needed for private instances.
|
||||
+6
-1
@@ -73,7 +73,7 @@ func startSecret() *kms.Config {
|
||||
}
|
||||
|
||||
// Validate connection details
|
||||
err = kms.ValidateConnectionDetails(context, &cephCluster.Spec.Security, namespace)
|
||||
err = kms.ValidateConnectionDetails(ctx, context, &cephCluster.Spec.Security, namespace)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(errors.Wrap(err, "failed to validate kms connection details"))
|
||||
}
|
||||
@@ -93,9 +93,14 @@ func cliGetSecret() *cobra.Command {
|
||||
}
|
||||
|
||||
func getSecret(cmd *cobra.Command, args []string) {
|
||||
// Initialize the context
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), operator.ShutdownSignals...)
|
||||
defer cancel()
|
||||
|
||||
secretName := args[0]
|
||||
secretPath := args[1]
|
||||
keyManagementService := startSecret()
|
||||
keyManagementService.ClusterInfo.Context = ctx
|
||||
|
||||
// Fetch the secret
|
||||
s, err := keyManagementService.GetSecret(secretName)
|
||||
|
||||
@@ -3,6 +3,7 @@ module github.com/rook/rook
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/IBM/keyprotect-go-client v0.7.0
|
||||
github.com/aws/aws-sdk-go v1.37.19
|
||||
github.com/banzaicloud/k8s-objectmatcher v1.1.0
|
||||
github.com/ceph/go-ceph v0.12.0
|
||||
|
||||
@@ -124,6 +124,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dX
|
||||
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3TveR88VmL1u3njyI=
|
||||
github.com/IBM/keyprotect-go-client v0.7.0 h1:JstSHD14Lp6ihwQseyPuGcs1AjOBjAmcisP0dTBA6A0=
|
||||
github.com/IBM/keyprotect-go-client v0.7.0/go.mod h1:SVr2ylV/fhSQPDiUjWirN9fsyWFCNNbt8GIT8hPJVjE=
|
||||
github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
|
||||
github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
|
||||
@@ -731,6 +733,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
|
||||
@@ -2081,6 +2084,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0=
|
||||
gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
|
||||
gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/vault/api"
|
||||
"github.com/libopenstorage/secrets"
|
||||
"github.com/libopenstorage/secrets/vault"
|
||||
)
|
||||
|
||||
@@ -42,6 +43,16 @@ func (kms *KeyManagementServiceSpec) IsK8sAuthEnabled() bool {
|
||||
return getParam(kms.ConnectionDetails, vault.AuthMethod) == vault.AuthMethodKubernetes && kms.TokenSecretName == ""
|
||||
}
|
||||
|
||||
// IsVaultKMS return whether Vault KMS is configured
|
||||
func (kms *KeyManagementServiceSpec) IsVaultKMS() bool {
|
||||
return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeVault
|
||||
}
|
||||
|
||||
// IsIBMKeyProtectKMS return whether IBM Key Protect KMS is configured
|
||||
func (kms *KeyManagementServiceSpec) IsIBMKeyProtectKMS() bool {
|
||||
return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeIBM
|
||||
}
|
||||
|
||||
// IsTLSEnabled return KMS TLS details are configured
|
||||
func (kms *KeyManagementServiceSpec) IsTLSEnabled() bool {
|
||||
for _, tlsOption := range VaultTLSConnectionDetails {
|
||||
|
||||
@@ -63,28 +63,48 @@ func dmsetupVersion(context *clusterd.Context) error {
|
||||
|
||||
func setKEKinEnv(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
|
||||
// KMS details are passed by the Operator as env variables in the pod
|
||||
// The token if any is mounted in the provisioner pod as an env variable so the secrets lib will pick it up
|
||||
kmsConfig := kms.NewConfig(context, &v1.ClusterSpec{Security: v1.SecuritySpec{KeyManagementService: v1.KeyManagementServiceSpec{ConnectionDetails: kms.ConfigEnvsToMapString()}}}, clusterInfo)
|
||||
if kmsConfig.IsVault() {
|
||||
// Fetch the KEK
|
||||
kek, err := kmsConfig.GetSecret(os.Getenv(oposd.PVCNameEnvVarName))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to retrieve key encryption key from %q kms", kmsConfig.Provider)
|
||||
}
|
||||
// The token if any is mounted in the provisioner pod as an env variable so the secrets lib will
|
||||
// pick it up
|
||||
clusterSpec := &v1.ClusterSpec{Security: v1.SecuritySpec{KeyManagementService: v1.KeyManagementServiceSpec{ConnectionDetails: kms.ConfigEnvsToMapString()}}}
|
||||
|
||||
if kek == "" {
|
||||
return errors.New("key encryption key is empty")
|
||||
}
|
||||
|
||||
// Set the KEK as an env variable for ceph-volume
|
||||
err = os.Setenv(oposd.CephVolumeEncryptedKeyEnvVarName, kek)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to set key encryption key env variable for ceph-volume")
|
||||
}
|
||||
|
||||
logger.Debug("successfully set vault env variables")
|
||||
// If KMS is not enabled this code does not need to run since we attach the KEK value from the
|
||||
// Kubernetes Secret in the provision pod spec (mounted as an environment variable)
|
||||
if !clusterSpec.Security.KeyManagementService.IsEnabled() {
|
||||
logger.Debug("cluster-wide encryption is enabled with kubernetes secrets and the kek is attached to the provision env spec")
|
||||
return nil
|
||||
}
|
||||
|
||||
// The ibm key protect library does not read any environment variables, so we must set the
|
||||
// service api key (coming from the secret mounted as environment variable) in the KMS
|
||||
// connection details. These details are used to build the client connection
|
||||
if clusterSpec.Security.KeyManagementService.IsIBMKeyProtectKMS() {
|
||||
ibmServiceApiKey := os.Getenv(kms.IbmKeyProtectServiceApiKey)
|
||||
if ibmServiceApiKey == "" {
|
||||
return errors.Errorf("ibm key protect %q environment variable is not set", kms.IbmKeyProtectServiceApiKey)
|
||||
}
|
||||
clusterSpec.Security.KeyManagementService.ConnectionDetails[kms.IbmKeyProtectServiceApiKey] = ibmServiceApiKey
|
||||
}
|
||||
|
||||
kmsConfig := kms.NewConfig(context, clusterSpec, clusterInfo)
|
||||
|
||||
// Fetch the KEK
|
||||
kek, err := kmsConfig.GetSecret(os.Getenv(oposd.PVCNameEnvVarName))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to retrieve key encryption key from %q kms", kmsConfig.Provider)
|
||||
}
|
||||
|
||||
if kek == "" {
|
||||
return errors.New("key encryption key is empty")
|
||||
}
|
||||
|
||||
// Set the KEK as an env variable for ceph-volume
|
||||
err = os.Setenv(oposd.CephVolumeEncryptedKeyEnvVarName, kek)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to set key encryption key env variable for ceph-volume")
|
||||
}
|
||||
|
||||
logger.Debug("successfully set vault env variables")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
knownKMSPrefix = []string{"VAULT_"}
|
||||
knownKMSPrefix = []string{"VAULT_", "IBM_"}
|
||||
)
|
||||
|
||||
// VaultTokenEnvVarFromSecret returns the kms token secret value as an env var
|
||||
@@ -48,6 +48,21 @@ func vaultTokenEnvVarFromSecret(tokenSecretName string) v1.EnvVar {
|
||||
}
|
||||
}
|
||||
|
||||
// ibmKeyProtectServiceAPIKeyEnvVarFromSecret returns the kms token secret value as an env var
|
||||
func ibmKeyProtectServiceAPIKeyEnvVarFromSecret(tokenSecretName string) v1.EnvVar {
|
||||
return v1.EnvVar{
|
||||
Name: IbmKeyProtectServiceApiKey,
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: tokenSecretName,
|
||||
},
|
||||
Key: IbmKeyProtectServiceApiKey,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// vaultTLSEnvVarFromSecret translates TLS env var which are set to k8s secret name to their actual path on the fs once mounted as volume
|
||||
// See: VaultSecretVolumeAndMount() for more details
|
||||
func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar {
|
||||
@@ -63,30 +78,48 @@ func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar {
|
||||
return vaultTLSEnvVar
|
||||
}
|
||||
|
||||
// VaultConfigToEnvVar populates the kms config as env variables
|
||||
func VaultConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar {
|
||||
// ConfigToEnvVar populates the kms config as env variables
|
||||
func ConfigToEnvVar(spec cephv1.ClusterSpec) []v1.EnvVar {
|
||||
envs := []v1.EnvVar{}
|
||||
backendPath := GetParam(spec.Security.KeyManagementService.ConnectionDetails, vault.VaultBackendPathKey)
|
||||
// Set BACKEND_PATH to the API's default if not passed
|
||||
if backendPath == "" {
|
||||
spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey] = vault.DefaultBackendPath
|
||||
}
|
||||
for k, v := range spec.Security.KeyManagementService.ConnectionDetails {
|
||||
// Skip TLS and token env var to avoid env being set multiple times
|
||||
toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken)
|
||||
if sets.NewString(toSkip...).Has(k) {
|
||||
continue
|
||||
|
||||
if spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
backendPath := GetParam(spec.Security.KeyManagementService.ConnectionDetails, vault.VaultBackendPathKey)
|
||||
// Set BACKEND_PATH to the API's default if not passed
|
||||
if backendPath == "" {
|
||||
spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey] = vault.DefaultBackendPath
|
||||
}
|
||||
}
|
||||
|
||||
if spec.Security.KeyManagementService.IsIBMKeyProtectKMS() {
|
||||
// We don't want to leak the IBM service API key to the container environment variables even
|
||||
// the container is ephemeral.
|
||||
// The IBM_KP_SERVICE_API_KEY content is mounted in the provisioner container as an
|
||||
// environment variable from a secret
|
||||
delete(spec.Security.KeyManagementService.ConnectionDetails, IbmKeyProtectServiceApiKey)
|
||||
envs = append(envs, ibmKeyProtectServiceAPIKeyEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName))
|
||||
}
|
||||
|
||||
for k, v := range spec.Security.KeyManagementService.ConnectionDetails {
|
||||
if spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
// Skip TLS and token env var to avoid env being set multiple times
|
||||
toSkip := append(cephv1.VaultTLSConnectionDetails, api.EnvVaultToken)
|
||||
if sets.NewString(toSkip...).Has(k) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
envs = append(envs, v1.EnvVar{Name: k, Value: v})
|
||||
}
|
||||
|
||||
// Add the VAULT_TOKEN
|
||||
if spec.Security.KeyManagementService.IsTokenAuthEnabled() {
|
||||
envs = append(envs, vaultTokenEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName))
|
||||
}
|
||||
if spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
// Add the VAULT_TOKEN
|
||||
if spec.Security.KeyManagementService.IsTokenAuthEnabled() {
|
||||
envs = append(envs, vaultTokenEnvVarFromSecret(spec.Security.KeyManagementService.TokenSecretName))
|
||||
}
|
||||
|
||||
// Add TLS env if any
|
||||
envs = append(envs, vaultTLSEnvVarFromSecret(spec.Security.KeyManagementService.ConnectionDetails)...)
|
||||
// Add TLS env if any
|
||||
envs = append(envs, vaultTLSEnvVarFromSecret(spec.Security.KeyManagementService.ConnectionDetails)...)
|
||||
}
|
||||
|
||||
logger.Debugf("kms envs are %v", envs)
|
||||
|
||||
|
||||
@@ -23,38 +23,58 @@ import (
|
||||
"testing"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func TestVaultTLSEnvVarFromSecret(t *testing.T) {
|
||||
// No TLS
|
||||
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{TokenSecretName: "vault-token", ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200"}}}}
|
||||
envVars := VaultConfigToEnvVar(spec)
|
||||
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
|
||||
return envVars[i].Name < envVars[j].Name
|
||||
t.Run("vault - no tls", func(t *testing.T) {
|
||||
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
|
||||
TokenSecretName: "vault-token",
|
||||
ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200"}}},
|
||||
}
|
||||
envVars := ConfigToEnvVar(spec)
|
||||
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
|
||||
return envVars[i].Name < envVars[j].Name
|
||||
})
|
||||
assert.True(t, areEnvVarsSorted)
|
||||
assert.Equal(t, 4, len(envVars))
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_BACKEND_PATH", Value: "secret/"})
|
||||
})
|
||||
assert.True(t, areEnvVarsSorted)
|
||||
assert.Equal(t, 4, len(envVars))
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_BACKEND_PATH", Value: "secret/"})
|
||||
|
||||
// TLS
|
||||
spec = cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{TokenSecretName: "vault-token", ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200", "VAULT_CACERT": "vault-ca-cert-secret"}}}}
|
||||
envVars = VaultConfigToEnvVar(spec)
|
||||
areEnvVarsSorted = sort.SliceIsSorted(envVars, func(i, j int) bool {
|
||||
return envVars[i].Name < envVars[j].Name
|
||||
t.Run("vault tls", func(t *testing.T) {
|
||||
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
|
||||
TokenSecretName: "vault-token",
|
||||
ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_ADDR": "http://1.1.1.1:8200", "VAULT_CACERT": "vault-ca-cert-secret"}}},
|
||||
}
|
||||
envVars := ConfigToEnvVar(spec)
|
||||
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
|
||||
return envVars[i].Name < envVars[j].Name
|
||||
})
|
||||
assert.True(t, areEnvVarsSorted)
|
||||
assert.Equal(t, 5, len(envVars))
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
|
||||
})
|
||||
t.Run("ibm kp", func(t *testing.T) {
|
||||
spec := cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
|
||||
TokenSecretName: "ibm-kp-token",
|
||||
ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_INSTANCE_ID": "1"}}},
|
||||
}
|
||||
envVars := ConfigToEnvVar(spec)
|
||||
areEnvVarsSorted := sort.SliceIsSorted(envVars, func(i, j int) bool {
|
||||
return envVars[i].Name < envVars[j].Name
|
||||
})
|
||||
assert.True(t, areEnvVarsSorted)
|
||||
assert.Equal(t, 3, len(envVars))
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "ibm-kp"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_INSTANCE_ID", Value: "1"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}})
|
||||
})
|
||||
assert.True(t, areEnvVarsSorted)
|
||||
assert.Equal(t, 5, len(envVars))
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "KMS_PROVIDER", Value: "vault"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_ADDR", Value: "http://1.1.1.1:8200"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"})
|
||||
assert.Contains(t, envVars, v1.EnvVar{Name: "VAULT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Key: "token"}}})
|
||||
|
||||
}
|
||||
|
||||
func TestConfigEnvsToMapString(t *testing.T) {
|
||||
@@ -80,7 +100,7 @@ func TestConfigEnvsToMapString(t *testing.T) {
|
||||
envs = ConfigEnvsToMapString()
|
||||
assert.Equal(t, 3, len(envs))
|
||||
assert.True(t, envs["KMS_PROVIDER"] == "vault")
|
||||
clusterSpec := &rookv1.ClusterSpec{Security: rookv1.SecuritySpec{KeyManagementService: rookv1.KeyManagementServiceSpec{ConnectionDetails: envs}}}
|
||||
clusterSpec := &cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: envs}}}
|
||||
assert.True(t, clusterSpec.Security.KeyManagementService.IsEnabled())
|
||||
}
|
||||
|
||||
@@ -94,27 +114,43 @@ func TestVaultConfigToEnvVar(t *testing.T) {
|
||||
want []v1.EnvVar
|
||||
}{
|
||||
{
|
||||
"no backend path",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}}},
|
||||
[]v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "secret/"}},
|
||||
},
|
||||
{
|
||||
"with backend path",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_BACKEND_PATH": "foo/"}}}}},
|
||||
[]v1.EnvVar{{Name: "VAULT_BACKEND_PATH", Value: "foo/"}},
|
||||
},
|
||||
{
|
||||
"test with tls config",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"VAULT_CACERT": "my-secret-name"}}}}},
|
||||
"vault - no backend path",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault"}}}}},
|
||||
[]v1.EnvVar{
|
||||
{Name: "KMS_PROVIDER", Value: "vault"},
|
||||
{Name: "VAULT_BACKEND_PATH", Value: "secret/"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"vault - with backend path",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_BACKEND_PATH": "foo/"}}}}},
|
||||
[]v1.EnvVar{
|
||||
{Name: "KMS_PROVIDER", Value: "vault"},
|
||||
{Name: "VAULT_BACKEND_PATH", Value: "foo/"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"vault - test with tls config",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "vault", "VAULT_CACERT": "my-secret-name"}}}}},
|
||||
[]v1.EnvVar{
|
||||
{Name: "KMS_PROVIDER", Value: "vault"},
|
||||
{Name: "VAULT_BACKEND_PATH", Value: "secret/"},
|
||||
{Name: "VAULT_CACERT", Value: "/etc/vault/vault.ca"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"ibm kp - IBM_KP_SERVICE_API_KEY is removed from the details",
|
||||
args{spec: cephv1.ClusterSpec{Security: cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{"KMS_PROVIDER": "ibm-kp", "IBM_KP_SERVICE_API_KEY": "foo", "IBM_KP_INSTANCE_ID": "1"}, TokenSecretName: "ibm-kp-token"}}}},
|
||||
[]v1.EnvVar{
|
||||
{Name: "IBM_KP_INSTANCE_ID", Value: "1"},
|
||||
{Name: "IBM_KP_SERVICE_API_KEY", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "ibm-kp-token"}, Key: "IBM_KP_SERVICE_API_KEY"}}},
|
||||
{Name: "KMS_PROVIDER", Value: "ibm-kp"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := VaultConfigToEnvVar(tt.args.spec); !reflect.DeepEqual(got, tt.want) {
|
||||
if got := ConfigToEnvVar(tt.args.spec); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("VaultConfigToEnvVar() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kms
|
||||
|
||||
import (
|
||||
kp "github.com/IBM/keyprotect-go-client"
|
||||
"github.com/libopenstorage/secrets"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
// IbmKeyProtectServiceApiKey is the IBM Key Protect service API key
|
||||
IbmKeyProtectServiceApiKey = "IBM_KP_SERVICE_API_KEY"
|
||||
// IbmKeyProtectInstanceIdKey is the IBM Key Protect instance id
|
||||
IbmKeyProtectInstanceIdKey = "IBM_KP_INSTANCE_ID"
|
||||
// IbmKeyProtectBaseUrlKey is the IBM Key Protect base url
|
||||
IbmKeyProtectBaseUrlKey = "IBM_KP_BASE_URL"
|
||||
// IbmKeyProtectTokenUrlKey is the IBM Key Protect token url
|
||||
// #nosec G101 since it's just an URL
|
||||
IbmKeyProtectTokenUrlKey = "IBM_KP_TOKEN_URL"
|
||||
)
|
||||
|
||||
var (
|
||||
kmsIBMKeyProtectMandatoryTokenDetails = []string{IbmKeyProtectServiceApiKey}
|
||||
kmsIBMKeyProtectMandatoryConnectionDetails = []string{IbmKeyProtectInstanceIdKey, IbmKeyProtectServiceApiKey}
|
||||
// ErrIbmServiceApiKeyNotSet is returned when IBM_KP_SERVICE_API_KEY is not set
|
||||
ErrIbmServiceApiKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectServiceApiKey)
|
||||
// ErrIbmInstanceIdKeyNotSet is returned when IBM_KP_INSTANCE_ID is not set
|
||||
ErrIbmInstanceIdKeyNotSet = errors.Errorf("%s not set.", IbmKeyProtectInstanceIdKey)
|
||||
)
|
||||
|
||||
// InitKeyProtect initializes the KeyProtect KMS.
|
||||
// With native go client directly "github.com/IBM/keyprotect-go-client"
|
||||
func InitKeyProtect(config map[string]string) (*kp.Client, error) {
|
||||
serviceApiKey := GetParam(config, IbmKeyProtectServiceApiKey)
|
||||
if serviceApiKey == "" {
|
||||
return nil, ErrIbmServiceApiKeyNotSet
|
||||
}
|
||||
|
||||
instanceId := GetParam(config, IbmKeyProtectInstanceIdKey)
|
||||
if instanceId == "" {
|
||||
return nil, ErrIbmInstanceIdKeyNotSet
|
||||
}
|
||||
|
||||
baseUrl := GetParam(config, IbmKeyProtectBaseUrlKey)
|
||||
if baseUrl == "" {
|
||||
baseUrl = kp.DefaultBaseURL
|
||||
}
|
||||
|
||||
tokenUrl := GetParam(config, IbmKeyProtectTokenUrlKey)
|
||||
if tokenUrl == "" {
|
||||
tokenUrl = kp.DefaultTokenURL
|
||||
}
|
||||
|
||||
cc := kp.ClientConfig{
|
||||
BaseURL: baseUrl,
|
||||
APIKey: serviceApiKey,
|
||||
TokenURL: tokenUrl,
|
||||
InstanceID: instanceId,
|
||||
Verbose: kp.VerboseAll,
|
||||
}
|
||||
|
||||
kp, err := kp.New(cc, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
// IsIBMKeyProtect determines whether the configured KMS is IBM Key Protect
|
||||
func (c *Config) IsIBMKeyProtect() bool { return c.Provider == secrets.TypeIBM }
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kms
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
kp "github.com/IBM/keyprotect-go-client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestInitKeyProtect(t *testing.T) {
|
||||
config := map[string]string{
|
||||
"foo": "1",
|
||||
}
|
||||
|
||||
t.Run("IBM_KP_SERVICE_API_KEY not set", func(t *testing.T) {
|
||||
_, err := InitKeyProtect(config)
|
||||
assert.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrIbmServiceApiKeyNotSet)
|
||||
config[IbmKeyProtectServiceApiKey] = "foo"
|
||||
})
|
||||
|
||||
t.Run("IBM_KP_INSTANCE_ID not set", func(t *testing.T) {
|
||||
_, err := InitKeyProtect(config)
|
||||
assert.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrIbmInstanceIdKeyNotSet)
|
||||
config[IbmKeyProtectInstanceIdKey] = "foo"
|
||||
})
|
||||
|
||||
t.Run("default base URL", func(t *testing.T) {
|
||||
c, err := InitKeyProtect(config)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, kp.DefaultBaseURL, c.Config.BaseURL)
|
||||
})
|
||||
|
||||
t.Run("different base URL", func(t *testing.T) {
|
||||
config[IbmKeyProtectBaseUrlKey] = "https://us-west.kms.cloud.ibm.com"
|
||||
c, err := InitKeyProtect(config)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "https://us-west.kms.cloud.ibm.com", c.Config.BaseURL)
|
||||
})
|
||||
|
||||
t.Run("default base token URL", func(t *testing.T) {
|
||||
c, err := InitKeyProtect(config)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, kp.DefaultTokenURL, c.Config.TokenURL)
|
||||
})
|
||||
|
||||
t.Run("different base URL", func(t *testing.T) {
|
||||
config[IbmKeyProtectTokenUrlKey] = "new"
|
||||
c, err := InitKeyProtect(config)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "new", c.Config.TokenURL)
|
||||
})
|
||||
}
|
||||
@@ -41,13 +41,13 @@ const (
|
||||
|
||||
// storeSecretInKubernetes stores the dmcrypt key in a Kubernetes Secret
|
||||
func (c *Config) storeSecretInKubernetes(pvcName, key string) error {
|
||||
s, err := generateOSDEncryptedKeySecret(pvcName, key, c.clusterInfo)
|
||||
s, err := generateOSDEncryptedKeySecret(pvcName, key, c.ClusterInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the Kubernetes Secret
|
||||
_, err = c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Create(c.clusterInfo.Context, s, metav1.CreateOptions{})
|
||||
_, err = c.context.Clientset.CoreV1().Secrets(c.ClusterInfo.Namespace).Create(c.ClusterInfo.Context, s, metav1.CreateOptions{})
|
||||
if err != nil && !kerrors.IsAlreadyExists(err) {
|
||||
return errors.Wrapf(err, "failed to save ceph osd encryption key as a secret for pvc %q", pvcName)
|
||||
}
|
||||
|
||||
+105
-28
@@ -21,6 +21,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
kp "github.com/IBM/keyprotect-go-client"
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/hashicorp/vault/api"
|
||||
"github.com/libopenstorage/secrets"
|
||||
@@ -47,14 +48,14 @@ type Config struct {
|
||||
Provider string
|
||||
context *clusterd.Context
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
ClusterInfo *cephclient.ClusterInfo
|
||||
}
|
||||
|
||||
// NewConfig returns the selected KMS
|
||||
func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) *Config {
|
||||
config := &Config{
|
||||
context: context,
|
||||
clusterInfo: clusterInfo,
|
||||
ClusterInfo: clusterInfo,
|
||||
clusterSpec: clusterSpec,
|
||||
}
|
||||
|
||||
@@ -64,6 +65,8 @@ func NewConfig(context *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clust
|
||||
config.Provider = secrets.TypeK8s
|
||||
case secrets.TypeVault:
|
||||
config.Provider = secrets.TypeVault
|
||||
case secrets.TypeIBM:
|
||||
config.Provider = secrets.TypeIBM
|
||||
default:
|
||||
logger.Errorf("unsupported kms type %q", Provider)
|
||||
}
|
||||
@@ -83,16 +86,34 @@ func (c *Config) PutSecret(secretName, secretValue string) error {
|
||||
}
|
||||
if c.IsVault() {
|
||||
// Store the secret in Vault
|
||||
v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to init vault kms")
|
||||
}
|
||||
k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
err = put(v, GenerateOSDEncryptionSecretName(secretName), secretValue, k)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to put secret in vault")
|
||||
}
|
||||
}
|
||||
if c.IsIBMKeyProtect() {
|
||||
kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to init ibm key protect")
|
||||
}
|
||||
|
||||
// Create the key is not present
|
||||
keyAlias := []string{secretName}
|
||||
_, err = kpClient.CreateImportedKeyWithAliases(c.ClusterInfo.Context, secretName, nil, secretValue, "", "", true, keyAlias)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "KEY_ALIAS_NOT_UNIQUE_ERR") {
|
||||
logger.Debugf("key %q already exists. %v", secretName, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.Wrap(err, "failed to put secret in ibm key protect")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -102,17 +123,28 @@ func (c *Config) GetSecret(secretName string) (string, error) {
|
||||
var value string
|
||||
if c.IsVault() {
|
||||
// Store the secret in Vault
|
||||
v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to init vault")
|
||||
}
|
||||
|
||||
k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
value, err = get(v, GenerateOSDEncryptionSecretName(secretName), k)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to get secret in vault")
|
||||
return "", errors.Wrap(err, "failed to get secret from vault")
|
||||
}
|
||||
}
|
||||
if c.IsIBMKeyProtect() {
|
||||
kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to init ibm key protect")
|
||||
}
|
||||
keyObject, err := kpClient.GetKey(c.ClusterInfo.Context, secretName)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to get secret from ibm key protect")
|
||||
}
|
||||
value = string(keyObject.Payload)
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
@@ -121,21 +153,49 @@ func (c *Config) GetSecret(secretName string) (string, error) {
|
||||
func (c *Config) DeleteSecret(secretName string) error {
|
||||
if c.IsVault() {
|
||||
// Store the secret in Vault
|
||||
v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
v, err := InitVault(c.context, c.ClusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to delete secret in vault")
|
||||
}
|
||||
|
||||
k := buildKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
k := buildVaultKeyContext(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
|
||||
// Force removal of all the versions of the secret on K/V version 2
|
||||
k[secrets.DestroySecret] = "true"
|
||||
|
||||
err = delete(v, GenerateOSDEncryptionSecretName(secretName), k)
|
||||
err = deleteSecret(v, GenerateOSDEncryptionSecretName(secretName), k)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to delete secret in vault")
|
||||
}
|
||||
}
|
||||
if c.IsIBMKeyProtect() {
|
||||
kpClient, err := InitKeyProtect(c.clusterSpec.Security.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to init ibm key protect")
|
||||
}
|
||||
|
||||
// We use context.TODO() since the clusterInfo context has been cancelled by the CephCluster's
|
||||
// deletion event
|
||||
ctx := context.TODO()
|
||||
|
||||
// Fetch the key to get the ID
|
||||
key, err := kpClient.GetKey(ctx, secretName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get secret in ibm key protect")
|
||||
}
|
||||
|
||||
// DeleteKey does not support deleting secret with the alias name so we must use the ID
|
||||
// After you delete a key, the key transitions to the Destroyed state. Any data encrypted by
|
||||
// keys in this state is no longer accessible. Metadata that is associated with the key,
|
||||
// such as the key's deletion date, is kept in the Key Protect database. Destroyed keys can
|
||||
// be recovered after up to 30 days or their expiration date, whichever is sooner. After 30
|
||||
// days, keys can no longer be recovered, and become eligible to be purged after 90 days, a
|
||||
// process that shreds the key material and makes its metadata inaccessible.
|
||||
_, err = kpClient.DeleteKey(ctx, key.ID, kp.ReturnRepresentation, []kp.CallOpt{kp.ForceOpt{Force: true}}...)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to delete secret in ibm key protect")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -149,14 +209,21 @@ func GetParam(kmsConfig map[string]string, param string) string {
|
||||
}
|
||||
|
||||
// ValidateConnectionDetails validates mandatory KMS connection details
|
||||
func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error {
|
||||
ctx := context.TODO()
|
||||
func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error {
|
||||
// Lookup mandatory connection details
|
||||
for _, config := range kmsMandatoryConnectionDetails {
|
||||
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
|
||||
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
|
||||
}
|
||||
}
|
||||
|
||||
// A token must be specified if token-auth is used
|
||||
if !securitySpec.KeyManagementService.IsK8sAuthEnabled() && securitySpec.KeyManagementService.TokenSecretName == "" {
|
||||
if !securitySpec.KeyManagementService.IsTokenAuthEnabled() {
|
||||
return errors.New("failed to validate kms configuration (missing token in spec)")
|
||||
}
|
||||
}
|
||||
|
||||
// KMS provider must be specified
|
||||
provider := GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider)
|
||||
|
||||
@@ -167,32 +234,35 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *
|
||||
return errors.Wrapf(err, "failed to fetch kms token secret %q", securitySpec.KeyManagementService.TokenSecretName)
|
||||
}
|
||||
|
||||
// Check for empty token
|
||||
token, ok := kmsToken.Data[KMSTokenSecretNameKey]
|
||||
if !ok || len(token) == 0 {
|
||||
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName)
|
||||
}
|
||||
|
||||
switch provider {
|
||||
case "vault":
|
||||
case secrets.TypeVault:
|
||||
// Check for empty token
|
||||
token, ok := kmsToken.Data[KMSTokenSecretNameKey]
|
||||
if !ok || len(token) == 0 {
|
||||
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName)
|
||||
}
|
||||
|
||||
// Set the env variable
|
||||
err = os.Setenv(api.EnvVaultToken, string(token))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to set vault kms token to an env var")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup mandatory connection details
|
||||
for _, config := range kmsMandatoryConnectionDetails {
|
||||
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
|
||||
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
|
||||
case secrets.TypeIBM:
|
||||
for _, config := range kmsIBMKeyProtectMandatoryTokenDetails {
|
||||
v, ok := kmsToken.Data[config]
|
||||
if !ok || len(v) == 0 {
|
||||
return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", config, securitySpec.KeyManagementService.TokenSecretName)
|
||||
}
|
||||
// Append the token secret details to the connection details
|
||||
securitySpec.KeyManagementService.ConnectionDetails[config] = strings.TrimSuffix(strings.TrimSpace(string(v)), "\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate KMS provider connection details for each provider
|
||||
switch provider {
|
||||
case "vault":
|
||||
case secrets.TypeVault:
|
||||
err := validateVaultConnectionDetails(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to validate vault connection details")
|
||||
@@ -210,6 +280,14 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *
|
||||
securitySpec.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = backendVersion
|
||||
}
|
||||
}
|
||||
|
||||
case secrets.TypeIBM:
|
||||
for _, config := range kmsIBMKeyProtectMandatoryConnectionDetails {
|
||||
if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
|
||||
return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return errors.Errorf("failed to validate kms provider connection details (provider %q not supported)", provider)
|
||||
}
|
||||
@@ -218,8 +296,7 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec *
|
||||
}
|
||||
|
||||
// SetTokenToEnvVar sets a KMS token as an env variable
|
||||
func SetTokenToEnvVar(clusterdContext *clusterd.Context, tokenSecretName, provider, namespace string) error {
|
||||
ctx := context.TODO()
|
||||
func SetTokenToEnvVar(ctx context.Context, clusterdContext *clusterd.Context, tokenSecretName, provider, namespace string) error {
|
||||
// Get the secret containing the kms token
|
||||
kmsToken, err := clusterdContext.Clientset.CoreV1().Secrets(namespace).Get(ctx, tokenSecretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
|
||||
@@ -37,83 +37,97 @@ func TestValidateConnectionDetails(t *testing.T) {
|
||||
context := &clusterd.Context{Clientset: test.New(t, 3)}
|
||||
securitySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}
|
||||
ns := "rook-ceph"
|
||||
|
||||
// Error: no token in spec
|
||||
err := ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate kms configuration (missing token in spec)")
|
||||
|
||||
securitySpec.KeyManagementService.TokenSecretName = "vault-token"
|
||||
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found")
|
||||
|
||||
// Error: token secret present but empty content
|
||||
s := &v1.Secret{
|
||||
vaultSecret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: securitySpec.KeyManagementService.TokenSecretName,
|
||||
Name: "vault-token",
|
||||
Namespace: ns,
|
||||
},
|
||||
}
|
||||
_, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
|
||||
|
||||
// Error: token key does not exist
|
||||
s.Data = map[string][]byte{"foo": []byte("bar")}
|
||||
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
|
||||
|
||||
// Success: token content is ok
|
||||
s.Data["token"] = []byte("token")
|
||||
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty")
|
||||
securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault"
|
||||
|
||||
// Error: Data has a KMS_PROVIDER but missing details
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"")
|
||||
|
||||
// Error: connection details are correct but the token secret does not exist
|
||||
securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200"
|
||||
|
||||
// Error: TLS is configured but secrets do not exist
|
||||
securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret"
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"")
|
||||
|
||||
// Error: TLS secret exists but empty key
|
||||
ibmSecret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ibm-token",
|
||||
Namespace: ns,
|
||||
},
|
||||
Data: map[string][]byte{"foo": []byte("bar")},
|
||||
}
|
||||
tlsSecret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "vault-ca-secret",
|
||||
Namespace: ns,
|
||||
},
|
||||
}
|
||||
_, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, tlsSecret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"")
|
||||
ibmSecuritySpec := &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{
|
||||
ConnectionDetails: map[string]string{
|
||||
"KMS_PROVIDER": "ibm-kp",
|
||||
},
|
||||
}}
|
||||
|
||||
// Success: TLS config is correct
|
||||
tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")}
|
||||
_, err = context.Clientset.CoreV1().Secrets(ns).Update(ctx, tlsSecret, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
assert.NoError(t, err, "")
|
||||
t.Run("no kms provider given", func(t *testing.T) {
|
||||
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate kms config \"KMS_PROVIDER\". cannot be empty")
|
||||
securitySpec.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault"
|
||||
})
|
||||
|
||||
// test with vauult server
|
||||
t.Run("vault - no token object", func(t *testing.T) {
|
||||
securitySpec.KeyManagementService.TokenSecretName = "vault-token"
|
||||
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to fetch kms token secret \"vault-token\": secrets \"vault-token\" not found")
|
||||
})
|
||||
|
||||
t.Run("vault - token secret present but empty content", func(t *testing.T) {
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, vaultSecret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
|
||||
})
|
||||
|
||||
t.Run("vault - token key does not exist", func(t *testing.T) {
|
||||
vaultSecret.Data = map[string][]byte{"foo": []byte("bar")}
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, vaultSecret, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to read k8s kms secret \"token\" key \"vault-token\" (not found or empty)")
|
||||
})
|
||||
|
||||
// Success: token content is ok
|
||||
t.Run("vault - token content is ok", func(t *testing.T) {
|
||||
vaultSecret.Data["token"] = []byte("token")
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, vaultSecret, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate vault connection details: failed to find connection details \"VAULT_ADDR\"")
|
||||
securitySpec.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200"
|
||||
})
|
||||
|
||||
t.Run("vault - TLS is configured but secrets do not exist", func(t *testing.T) {
|
||||
securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret"
|
||||
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"")
|
||||
})
|
||||
|
||||
t.Run("vault - TLS secret exists but empty key", func(t *testing.T) {
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, tlsSecret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"")
|
||||
})
|
||||
|
||||
t.Run("vault - success TLS config is correct", func(t *testing.T) {
|
||||
tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")}
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, tlsSecret, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.NoError(t, err, "")
|
||||
})
|
||||
|
||||
// test with vault server
|
||||
t.Run("success - auto detect kv version and set it", func(t *testing.T) {
|
||||
cluster := fakeVaultServer(t)
|
||||
cluster.Start()
|
||||
@@ -142,11 +156,44 @@ func TestValidateConnectionDetails(t *testing.T) {
|
||||
TokenSecretName: "vault-token",
|
||||
},
|
||||
}
|
||||
err = ValidateConnectionDetails(context, securitySpec, ns)
|
||||
err := ValidateConnectionDetails(ctx, context, securitySpec, ns)
|
||||
assert.NoError(t, err, "")
|
||||
assert.Equal(t, securitySpec.KeyManagementService.ConnectionDetails["VAULT_BACKEND"], "v2")
|
||||
})
|
||||
|
||||
t.Run("ibm kp - fail no token specified, only token is supported", func(t *testing.T) {
|
||||
err := ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate kms configuration (missing token in spec)")
|
||||
ibmSecuritySpec.KeyManagementService.TokenSecretName = "ibm-token"
|
||||
|
||||
})
|
||||
|
||||
t.Run("ibm kp - token present but no key for service key", func(t *testing.T) {
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, ibmSecret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to read k8s kms secret \"IBM_KP_SERVICE_API_KEY\" key \"ibm-token\" (not found or empty)")
|
||||
})
|
||||
|
||||
t.Run("ibm kp - token ok but no instance id", func(t *testing.T) {
|
||||
ibmSecret.Data["IBM_KP_SERVICE_API_KEY"] = []byte("foo")
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Update(ctx, ibmSecret, metav1.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
err = ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
|
||||
assert.Error(t, err, "")
|
||||
assert.EqualError(t, err, "failed to validate kms config \"IBM_KP_INSTANCE_ID\". cannot be empty")
|
||||
ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_INSTANCE_ID"] = "foo"
|
||||
})
|
||||
|
||||
t.Run("ibm kp - success", func(t *testing.T) {
|
||||
err := ValidateConnectionDetails(ctx, context, ibmSecuritySpec, ns)
|
||||
assert.NoError(t, err, "")
|
||||
// IBM_KP_SERVICE_API_KEY must be appended to the details so that the client can be built with
|
||||
// all the details
|
||||
assert.Equal(t, ibmSecuritySpec.KeyManagementService.ConnectionDetails["IBM_KP_SERVICE_API_KEY"], "foo")
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetTokenToEnvVar(t *testing.T) {
|
||||
@@ -164,7 +211,7 @@ func TestSetTokenToEnvVar(t *testing.T) {
|
||||
_, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = SetTokenToEnvVar(context, secretName, "vault", ns)
|
||||
err = SetTokenToEnvVar(ctx, context, secretName, "vault", ns)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, os.Getenv("VAULT_TOKEN"), "toto")
|
||||
os.Unsetenv("VAULT_TOKEN")
|
||||
|
||||
@@ -222,7 +222,7 @@ func get(v secrets.Secrets, secretName string, keyContext map[string]string) (st
|
||||
return s[secretName].(string), nil
|
||||
}
|
||||
|
||||
func delete(v secrets.Secrets, secretName string, keyContext map[string]string) error {
|
||||
func deleteSecret(v secrets.Secrets, secretName string, keyContext map[string]string) error {
|
||||
// #nosec G104 Write the encryption key in Vault
|
||||
err := v.DeleteSecret(secretName, keyContext)
|
||||
if err != nil {
|
||||
@@ -232,7 +232,7 @@ func delete(v secrets.Secrets, secretName string, keyContext map[string]string)
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildKeyContext(config map[string]string) map[string]string {
|
||||
func buildVaultKeyContext(config map[string]string) map[string]string {
|
||||
// Key context is just the Vault namespace, available in the enterprise version only
|
||||
keyContext := map[string]string{secrets.KeyVaultNamespace: config[api.EnvVaultNamespace]}
|
||||
vaultNamespace, ok := config[api.EnvVaultNamespace]
|
||||
@@ -245,7 +245,7 @@ func buildKeyContext(config map[string]string) map[string]string {
|
||||
|
||||
// IsVault determines whether the configured KMS is Vault
|
||||
func (c *Config) IsVault() bool {
|
||||
return c.Provider == "vault"
|
||||
return c.Provider == secrets.TypeVault
|
||||
}
|
||||
|
||||
func validateVaultConnectionDetails(clusterdContext *clusterd.Context, ns string, kmsConfig map[string]string) error {
|
||||
|
||||
@@ -254,13 +254,13 @@ func Test_configTLS(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func Test_buildKeyContext(t *testing.T) {
|
||||
func Test_buildVaultKeyContext(t *testing.T) {
|
||||
t.Run("no vault namespace, return empty map and assignment is possible", func(t *testing.T) {
|
||||
config := map[string]string{
|
||||
"KMS_PROVIDER": "vault",
|
||||
"VAULT_ADDR": "1.1.1.1",
|
||||
}
|
||||
context := buildKeyContext(config)
|
||||
context := buildVaultKeyContext(config)
|
||||
assert.Len(t, context, 0)
|
||||
context["foo"] = "bar"
|
||||
})
|
||||
@@ -271,7 +271,7 @@ func Test_buildKeyContext(t *testing.T) {
|
||||
"VAULT_ADDR": "1.1.1.1",
|
||||
"VAULT_NAMESPACE": "vault-namespace",
|
||||
}
|
||||
context := buildKeyContext(config)
|
||||
context := buildVaultKeyContext(config)
|
||||
assert.Len(t, context, 1)
|
||||
context["foo"] = "bar"
|
||||
assert.Len(t, context, 2)
|
||||
|
||||
@@ -306,7 +306,7 @@ func preClusterStartValidation(cluster *cluster) error {
|
||||
// Validate on-PVC cluster encryption KMS settings
|
||||
if cluster.Spec.Storage.IsOnPVCEncrypted() && cluster.Spec.Security.KeyManagementService.IsEnabled() {
|
||||
// Validate the KMS details
|
||||
err := kms.ValidateConnectionDetails(cluster.context, &cluster.Spec.Security, cluster.Namespace)
|
||||
err := kms.ValidateConnectionDetails(cluster.ClusterInfo.Context, cluster.context, &cluster.Spec.Security, cluster.Namespace)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to validate kms connection details")
|
||||
}
|
||||
|
||||
@@ -511,6 +511,10 @@ func (r *ReconcileCephCluster) removeFinalizer(client client.Client, name types.
|
||||
}
|
||||
|
||||
func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1.CephCluster) error {
|
||||
// At the point, the CephCluster has been deleted and the context cancelled, so we need to use a
|
||||
// new context here
|
||||
// So do NOT use another context
|
||||
ctx := context.TODO()
|
||||
// If the operator was stopped and we enter this code, the map is empty
|
||||
if _, ok := c.clusterMap[currentCluster.Namespace]; !ok {
|
||||
c.clusterMap[currentCluster.Namespace] = &cluster{ClusterInfo: &cephclient.ClusterInfo{Namespace: currentCluster.Namespace}}
|
||||
@@ -526,13 +530,22 @@ func (c *ClusterController) deleteOSDEncryptionKeyFromKMS(currentCluster *cephv1
|
||||
kmsConfig := kms.NewConfig(c.context, ¤tCluster.Spec, c.clusterMap[currentCluster.Namespace].ClusterInfo)
|
||||
|
||||
// If token auth is used by the KMS we set it as an env variable
|
||||
if currentCluster.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
|
||||
err := kms.SetTokenToEnvVar(c.context, currentCluster.Spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, currentCluster.Namespace)
|
||||
if currentCluster.Spec.Security.KeyManagementService.IsTokenAuthEnabled() && currentCluster.Spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
err := kms.SetTokenToEnvVar(ctx, c.context, currentCluster.Spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, currentCluster.Namespace)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to fetch kms token secret %q", currentCluster.Spec.Security.KeyManagementService.TokenSecretName)
|
||||
}
|
||||
}
|
||||
|
||||
// We need to fetch the IBM_KP_SERVICE_API_KEY value
|
||||
if currentCluster.Spec.Security.KeyManagementService.IsIBMKeyProtectKMS() {
|
||||
// This will validate the connection details again and will add the IBM_KP_SERVICE_API_KEY to the spec
|
||||
err = kms.ValidateConnectionDetails(ctx, c.context, ¤tCluster.Spec.Security, currentCluster.Namespace)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to validate kms connection details to delete the secret")
|
||||
}
|
||||
}
|
||||
|
||||
// Delete each PV KEK
|
||||
for _, osdPVC := range osdPVCs {
|
||||
// Generate and store the encrypted key in whatever KMS is configured
|
||||
|
||||
@@ -218,8 +218,8 @@ func (c *Cluster) startProvisioningOverPVCs(config *provisionConfig, errs *provi
|
||||
|
||||
// We could set an env var in the Operator or a global var instead of the API call?
|
||||
// Hopefully, the API is cheap and we can always retrieve the token if it has changed...
|
||||
if c.spec.Security.KeyManagementService.IsTokenAuthEnabled() {
|
||||
err := kms.SetTokenToEnvVar(c.context, c.spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, c.clusterInfo.Namespace)
|
||||
if c.spec.Security.KeyManagementService.IsTokenAuthEnabled() && c.spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
err := kms.SetTokenToEnvVar(c.clusterInfo.Context, c.context, c.spec.Security.KeyManagementService.TokenSecretName, kmsConfig.Provider, c.clusterInfo.Namespace)
|
||||
if err != nil {
|
||||
errMsg := fmt.Sprintf("failed to fetch kms token secret %q. %v", c.spec.Security.KeyManagementService.TokenSecretName, err)
|
||||
errs.addError(errMsg)
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/libopenstorage/secrets"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
@@ -121,8 +120,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
|
||||
if osdProps.encrypted {
|
||||
// If a KMS is configured we populate
|
||||
if c.spec.Security.KeyManagementService.IsEnabled() {
|
||||
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
|
||||
if kmsProvider == secrets.TypeVault {
|
||||
if c.spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
|
||||
volumes = append(volumes, volumeTLS)
|
||||
}
|
||||
@@ -280,12 +278,11 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
|
||||
if osdProps.encrypted {
|
||||
// If a KMS is configured we populate volume mounts and env variables
|
||||
if c.spec.Security.KeyManagementService.IsEnabled() {
|
||||
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
|
||||
if kmsProvider == secrets.TypeVault {
|
||||
if c.spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
_, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
|
||||
volumeMounts = append(volumeMounts, volumeMountsTLS)
|
||||
envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...)
|
||||
}
|
||||
envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...)
|
||||
} else {
|
||||
envVars = append(envVars, cephVolumeRawEncryptedEnvVarFromSecret(osdProps))
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/libopenstorage/secrets"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
@@ -867,7 +866,7 @@ func (c *Cluster) generateVaultGetKEK(osdProps osdProperties) v1.Container {
|
||||
keyName := osdProps.pvc.ClaimName
|
||||
keyPath := encryptionKeyPath()
|
||||
envVars := c.getConfigEnvVars(osdProps, "")
|
||||
envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...)
|
||||
envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...)
|
||||
|
||||
return v1.Container{
|
||||
Name: blockEncryptionKMSGetKEKInitContainer,
|
||||
@@ -889,24 +888,21 @@ func (c *Cluster) getPVCEncryptionOpenInitContainerActivate(mountPath string, os
|
||||
|
||||
// If a KMS is enabled we need to add an init container to fetch the KEK
|
||||
if c.spec.Security.KeyManagementService.IsEnabled() {
|
||||
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
|
||||
// Get Vault KEK from KMS container
|
||||
if kmsProvider == secrets.TypeVault {
|
||||
getKEKFromKMSContainer := c.generateVaultGetKEK(osdProps)
|
||||
getKEKFromKMSContainer := c.generateVaultGetKEK(osdProps)
|
||||
|
||||
// Volume mount to store the encrypted key
|
||||
_, volMount := c.getEncryptionVolume(osdProps)
|
||||
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, volMount)
|
||||
// Volume mount to store the encrypted key
|
||||
_, volMount := c.getEncryptionVolume(osdProps)
|
||||
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, volMount)
|
||||
|
||||
if c.spec.Security.KeyManagementService.IsVaultKMS() {
|
||||
// Now let's see if there is a TLS config we need to mount as well
|
||||
if c.spec.Security.KeyManagementService.IsTLSEnabled() {
|
||||
_, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
|
||||
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, vaultVolMount)
|
||||
}
|
||||
|
||||
// Add the container to the list of containers
|
||||
containers = append(containers, getKEKFromKMSContainer)
|
||||
}
|
||||
// Add the container to the list of containers
|
||||
containers = append(containers, getKEKFromKMSContainer)
|
||||
}
|
||||
|
||||
// Main block container
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/libopenstorage/secrets"
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -192,15 +191,6 @@ func getUdevVolume() (v1.Volume, v1.VolumeMount) {
|
||||
}
|
||||
|
||||
func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.VolumeMount) {
|
||||
// Determine whether we have a KMS configuration
|
||||
var isKMS bool
|
||||
if len(c.spec.Security.KeyManagementService.ConnectionDetails) != 0 {
|
||||
provider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
|
||||
if provider == secrets.TypeVault {
|
||||
isKMS = true
|
||||
}
|
||||
}
|
||||
|
||||
// Generate volume
|
||||
var m int32 = 0400
|
||||
volume := v1.Volume{
|
||||
@@ -220,7 +210,7 @@ func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.Vol
|
||||
}
|
||||
|
||||
// On the KMS use case, we want the volume mount to be in memory since we pass write the KEK
|
||||
if isKMS {
|
||||
if c.spec.Security.KeyManagementService.IsEnabled() {
|
||||
volume.VolumeSource.Secret = nil
|
||||
volume.VolumeSource = v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{
|
||||
@@ -237,7 +227,7 @@ func (c *Cluster) getEncryptionVolume(osdProps osdProperties) (v1.Volume, v1.Vol
|
||||
}
|
||||
|
||||
// With KMS we must be able to write inside the directory to write the KEK
|
||||
if isKMS {
|
||||
if c.spec.Security.KeyManagementService.IsEnabled() {
|
||||
volumeMounts.ReadOnly = false
|
||||
}
|
||||
|
||||
|
||||
@@ -568,7 +568,7 @@ func (c *clusterConfig) vaultPrefixRGW() string {
|
||||
|
||||
func (c *clusterConfig) CheckRGWKMS() (bool, error) {
|
||||
if c.store.Spec.Security != nil && c.store.Spec.Security.KeyManagementService.IsEnabled() {
|
||||
err := kms.ValidateConnectionDetails(c.context, c.store.Spec.Security, c.store.Namespace)
|
||||
err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, c.store.Spec.Security, c.store.Namespace)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
@@ -436,13 +436,15 @@ func TestDefaultReadinessProbe(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckRGWKMS(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
setupTest := func() *clusterConfig {
|
||||
context := &clusterd.Context{Clientset: test.New(t, 3)}
|
||||
store := simpleStore()
|
||||
store.Spec.Security = &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}}
|
||||
return &clusterConfig{
|
||||
context: context,
|
||||
store: store,
|
||||
context: context,
|
||||
store: store,
|
||||
clusterInfo: &client.ClusterInfo{Context: ctx},
|
||||
}
|
||||
}
|
||||
configureKMS := func(c *clusterConfig) {
|
||||
@@ -458,7 +460,7 @@ func TestCheckRGWKMS(t *testing.T) {
|
||||
"token": []byte("myt-otkenbenvqrev"),
|
||||
},
|
||||
}
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(context.TODO(), s, metav1.CreateOptions{})
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(ctx, s, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: rook-ibm-kp-token
|
||||
namespace: rook-ceph
|
||||
data:
|
||||
IBM_KP_SERVICE_API_KEY: $IBM_KP_SERVICE_API_KEY
|
||||
@@ -0,0 +1,7 @@
|
||||
spec:
|
||||
security:
|
||||
kms:
|
||||
connectionDetails:
|
||||
KMS_PROVIDER: ibm-kp
|
||||
IBM_KP_INSTANCE_ID: $IBM_KP_INSTANCE_ID
|
||||
tokenSecretName: rook-ibm-kp-token
|
||||
Reference in new issue
Block a user