feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #17302 from sp98/sse-s3-with-vault-agent

Browse Source 
rgw: support SSE-S3 with vault agent.
This commit is contained in:
Blaine Gardner authored and GitHub committed 2026-04-07 09:40:34 -06:00
2 parents 7e3085e67b c6836c474a
commit a9efa167cc
7 files changed
+178 -6
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
+84
View File
@@ -381,6 +381,90 @@ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine
* `tokenSecretName` can be (and often will be) the same for both kms and s3 configurations.
### SSE-S3 with Vault Agent
Instead of using a static token for SSE-S3, you can use a [Vault Agent](https://developer.hashicorp.com/vault/docs/agent-and-proxy/agent) running as a sidecar in the RGW pod. The agent authenticates to Vault using the pod's Kubernetes service account (via Vault's [Kubernetes auth method](https://developer.hashicorp.com/vault/docs/auth/kubernetes)) and acts as a local proxy, eliminating the need to manage static tokens.
This approach uses the [Vault Agent Injector](https://developer.hashicorp.com/vault/docs/platform/k8s/injector) (a mutating admission webhook) to automatically inject the Vault Agent sidecar into RGW pods.
#### Prerequisites
1. **Deploy the Vault Agent Injector** in the cluster:
```bash
helm repo add hashicorp https://helm.releases.hashicorp.com
helm install vault hashicorp/vault \
--set "injector.enabled=true" \
--set "server.enabled=false" # if using an external Vault server
```
2. **Configure Vault** with the Kubernetes auth method, a policy, and a role:
```bash
# Enable Kubernetes auth method
vault auth enable kubernetes
# Configure Kubernetes auth
vault write auth/kubernetes/config \
kubernetes_host="https://$KUBERNETES_HOST:443"
# Create a policy granting access to the transit engine
vault policy write rgw-sse-s3 - <<EOF
path "transit/keys/*" {
capabilities = ["create", "read", "update", "delete"]
}
path "transit/keys/+/rotate" {
capabilities = ["update"]
}
path "transit/datakey/plaintext/*" {
capabilities = ["update"]
}
path "transit/decrypt/*" {
capabilities = ["update"]
}
path "transit/export/encryption-key/*" {
capabilities = ["read"]
}
EOF
# Create a role bound to the RGW service account
vault write auth/kubernetes/role/rook-ceph-rgw \
bound_service_account_names=rook-ceph-rgw \
bound_service_account_namespaces=rook-ceph \
policies=rgw-sse-s3 \
ttl=1h
```
#### Configuration
Set `VAULT_AUTH_METHOD: agent` in the `s3` connection details and add Vault Agent Injector annotations to the gateway spec. The `VAULT_ADDR` should point to the local Vault Agent proxy (typically `http://127.0.0.1:8100`). No `tokenSecretName` is needed since the injected agent handles authentication.
```yaml
apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
name: my-store
namespace: rook-ceph
spec:
gateway:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: "rook-ceph-rgw"
vault.hashicorp.com/agent-cache-enable: "true"
vault.hashicorp.com/agent-cache-listener-port: "8100"
# For TLS to Vault server:
# vault.hashicorp.com/ca-cert: "/vault/tls/ca.crt"
# vault.hashicorp.com/client-cert: "/vault/tls/tls.crt"
# vault.hashicorp.com/client-key: "/vault/tls/tls.key"
security:
s3:
connectionDetails:
KMS_PROVIDER: vault
VAULT_ADDR: http://127.0.0.1:8100
VAULT_SECRET_ENGINE: transit
VAULT_AUTH_METHOD: agent
```
## Advanced configuration
!!! warning
PendingReleaseNotes.md
+1
View File
@@ -8,3 +8,4 @@
## Features
- RGW Account support: Added `CephObjectStoreAccount` CRD for managing RGW accounts, and `accountRef` field in `CephObjectStoreUser` to associate users with accounts. This feature is experimental and currently only supported with the Ceph main branch image (`quay.ceph.io/ceph-ci/ceph:main`). See the [Object Store Accounts](Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-accounts.md) documentation for more details.
- SSE-S3 with Vault Agent: Added support for server-side encryption with SSE-S3 using HashiCorp Vault Agent authentication. See the [CephObjectStore Security Settings](Documentation/CRDs/Object-Storage/ceph-object-store-crd.md#sse-s3-with-vault-agent) for more details.
pkg/apis/ceph.rook.io/v1/security.go
+5
View File
@@ -41,6 +41,11 @@ func (kms *KeyManagementServiceSpec) IsK8sAuthEnabled() bool {
return getParam(kms.ConnectionDetails, vault.AuthMethod) == vault.AuthMethodKubernetes && kms.TokenSecretName == ""
}
// IsAgentAuthEnabled return whether Vault Agent auth is enabled
func (kms *KeyManagementServiceSpec) IsAgentAuthEnabled() bool {
return getParam(kms.ConnectionDetails, vault.AuthMethod) == "agent" && kms.TokenSecretName == ""
}
// IsVaultKMS return whether Vault KMS is configured
func (kms *KeyManagementServiceSpec) IsVaultKMS() bool {
return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeVault
pkg/apis/ceph.rook.io/v1/security_test.go
+28
View File
@@ -18,6 +18,34 @@ package v1
import "testing"
func TestKeyManagementServiceSpec_IsAgentAuthEnabled(t *testing.T) {
type fields struct {
ConnectionDetails map[string]string
TokenSecretName string
}
tests := []struct {
name string
fields fields
want bool
}{
{"agent auth is disabled - everything is empty", fields{ConnectionDetails: map[string]string{}, TokenSecretName: ""}, false},
{"agent auth is disabled - token is populated", fields{ConnectionDetails: map[string]string{"VAULT_AUTH_METHOD": "agent"}, TokenSecretName: "rook-ceph-test-secret"}, false},
{"agent auth is disabled since VAULT_AUTH_METHOD is not agent", fields{ConnectionDetails: map[string]string{"VAULT_AUTH_METHOD": "token"}, TokenSecretName: ""}, false},
{"agent auth is enabled", fields{ConnectionDetails: map[string]string{"VAULT_AUTH_METHOD": "agent"}, TokenSecretName: ""}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
kms := &KeyManagementServiceSpec{
ConnectionDetails: tt.fields.ConnectionDetails,
TokenSecretName: tt.fields.TokenSecretName,
}
if got := kms.IsAgentAuthEnabled(); got != tt.want {
t.Errorf("KeyManagementServiceSpec.IsAgentAuthEnabled() = %v, want %v", got, tt.want)
}
})
}
}
func TestKeyManagementServiceSpec_IsK8sAuthEnabled(t *testing.T) {
type fields struct {
ConnectionDetails map[string]string
pkg/daemon/ceph/osd/kms/kms.go
+1 -1
View File
@@ -362,7 +362,7 @@ func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Co
// A token must be specified if token-auth is used for KMS other than Azure
if !kms.IsAzureMS() {
if !kms.IsK8sAuthEnabled() && kms.TokenSecretName == "" {
if !kms.IsK8sAuthEnabled() && !kms.IsAgentAuthEnabled() && kms.TokenSecretName == "" {
if !kms.IsTokenAuthEnabled() {
return errors.New("failed to validate kms configuration (missing token in spec)")
}
pkg/operator/ceph/object/spec.go
+32 -5
View File
@@ -285,19 +285,25 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
}
podSpec.Volumes = append(podSpec.Volumes, v)
if kmsEnabled && c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
kmsNeedsToken := kmsEnabled && c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled()
s3NeedsToken := s3Enabled && c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled()
if kmsNeedsToken {
vaultFileVol, _ := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.KeyManagementService.ConnectionDetails,
c.store.Spec.Security.KeyManagementService.TokenSecretName, sseKMS)
podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
}
if s3Enabled && c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
if s3NeedsToken {
vaultFileVol, _ := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails,
c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName, sseS3)
podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
}
podSpec.InitContainers = append(podSpec.InitContainers,
c.vaultTokenInitContainer(rgwConfig, kmsEnabled, s3Enabled))
// Only add the vault token init container if at least one of KMS or SSE-S3 needs token-based auth.
// When SSE-S3 uses agent auth, no token init container is needed for that part.
if kmsNeedsToken || s3NeedsToken {
podSpec.InitContainers = append(podSpec.InitContainers,
c.vaultTokenInitContainer(rgwConfig, kmsNeedsToken, s3NeedsToken))
}
}
c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec)
@@ -524,7 +530,9 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
log.NamedDebug(nsName, logger, "enabliing SSE-S3. %v", c.store.Spec.Security.ServerSideEncryptionS3)
container.Args = append(container.Args, c.sseS3DefaultOptions(s3EncryptionEnabled)...)
if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
if c.store.Spec.Security.ServerSideEncryptionS3.IsAgentAuthEnabled() {
container.Args = append(container.Args, c.sseS3VaultAgentOptions(s3EncryptionEnabled)...)
} else if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3EncryptionEnabled)...)
}
if c.store.Spec.Security.ServerSideEncryptionS3.IsTLSEnabled() {
@@ -896,6 +904,12 @@ func (c *clusterConfig) CheckRGWKMS() (bool, error) {
func (c *clusterConfig) CheckRGWSSES3Enabled() (bool, error) {
if c.store.Spec.Security != nil && c.store.Spec.Security.ServerSideEncryptionS3.IsEnabled() {
// When agent auth is used, tokenSecretName must not be set
if kms.GetParam(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, vault.AuthMethod) == "agent" &&
c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName != "" {
return false, errors.New("tokenSecretName must not be set when VAULT_AUTH_METHOD is agent for SSE-S3")
}
err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, &c.store.Spec.Security.ServerSideEncryptionS3, c.store.Namespace)
if err != nil {
return false, err
@@ -1087,6 +1101,19 @@ func (c *clusterConfig) sseKMSVaultTLSOptions(setOptions bool) []string {
return rgwOptions
}
func (c *clusterConfig) sseS3VaultAgentOptions(setOptions bool) []string {
if setOptions {
return []string{
cephconfig.NewFlag("rgw crypt sse s3 vault auth", "agent"),
cephconfig.NewFlag("rgw crypt sse s3 vault prefix",
path.Join(vaultPrefix, c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey])),
cephconfig.NewFlag("rgw crypt sse s3 vault secret engine",
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey]),
}
}
return []string{}
}
func (c *clusterConfig) sseS3VaultTLSOptions(setOptions bool) []string {
var rgwOptions []string
if setOptions {
pkg/operator/ceph/object/spec_test.go
+27
View File
@@ -43,6 +43,14 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func configureSSEWithVaultAgent(c *clusterConfig) {
c.store.Spec.Security = &cephv1.ObjectStoreSecuritySpec{}
c.store.Spec.Security.ServerSideEncryptionS3 = cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["KMS_PROVIDER"] = "vault"
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_ADDR"] = "http://127.0.0.1:8100"
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_AUTH_METHOD"] = "agent"
}
func configureSSE(t *testing.T, c *clusterConfig, kms bool, s3 bool) {
c.store.Spec.Security = &cephv1.ObjectStoreSecuritySpec{}
if kms {
@@ -991,6 +999,25 @@ func TestAWSServerSideEncryption(t *testing.T) {
assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseKMSVaultTLSOptions(true)))
assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultTLSOptions(true)))
})
t.Run("SSE-S3 with vault agent auth sets agent options instead of token options", func(t *testing.T) {
configureSSEWithVaultAgent(c)
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit"
rgwContainer, err := c.makeDaemonContainer(rgwConfig)
assert.NoError(t, err)
assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseS3DefaultOptions(true)))
assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultAgentOptions(true)))
assert.False(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultTokenOptions(true)))
assert.False(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultTLSOptions(false)))
})
t.Run("SSE-S3 with vault agent auth rejects tokenSecretName", func(t *testing.T) {
configureSSEWithVaultAgent(c)
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit"
c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName = "vault-token"
_, err := c.makeDaemonContainer(rgwConfig)
assert.Error(t, err)
})
}
func TestRgwCommandFlags(t *testing.T) {