From c6836c474a87944d48e4b41143e80e19b793326c Mon Sep 17 00:00:00 2001
From: Santosh <sapillai@redhat.com>
Date: Thu, 2 Apr 2026 21:56:21 +0530
Subject: [PATCH] rgw: support SSE-S3 with vault agent

Support using SSE-S3 encryption with RGW using vault Agent auth.
RGW sends requests to the agent instead of directly to Vault, and the agent transparently injects the authentication token. This eliminates using and managing a static token

Signed-off-by: Santosh <sapillai@redhat.com>
---
 .../Object-Storage/ceph-object-store-crd.md   | 84 +++++++++++++++++++
 PendingReleaseNotes.md                        |  1 +
 pkg/apis/ceph.rook.io/v1/security.go          |  5 ++
 pkg/apis/ceph.rook.io/v1/security_test.go     | 28 +++++++
 pkg/daemon/ceph/osd/kms/kms.go                |  2 +-
 pkg/operator/ceph/object/spec.go              | 37 ++++++--
 pkg/operator/ceph/object/spec_test.go         | 27 ++++++
 7 files changed, 178 insertions(+), 6 deletions(-)

diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
index 23716456e..b9c8b04bc 100644
--- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
+++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
@@ -381,6 +381,90 @@ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine
 
 * `tokenSecretName` can be (and often will be) the same for both kms and s3 configurations.
 
+### SSE-S3 with Vault Agent
+
+Instead of using a static token for SSE-S3, you can use a [Vault Agent](https://developer.hashicorp.com/vault/docs/agent-and-proxy/agent) running as a sidecar in the RGW pod. The agent authenticates to Vault using the pod's Kubernetes service account (via Vault's [Kubernetes auth method](https://developer.hashicorp.com/vault/docs/auth/kubernetes)) and acts as a local proxy, eliminating the need to manage static tokens.
+
+This approach uses the [Vault Agent Injector](https://developer.hashicorp.com/vault/docs/platform/k8s/injector) (a mutating admission webhook) to automatically inject the Vault Agent sidecar into RGW pods.
+
+#### Prerequisites
+
+1. **Deploy the Vault Agent Injector** in the cluster:
+
+    ```bash
+    helm repo add hashicorp https://helm.releases.hashicorp.com
+    helm install vault hashicorp/vault \
+        --set "injector.enabled=true" \
+        --set "server.enabled=false"   # if using an external Vault server
+    ```
+
+2. **Configure Vault** with the Kubernetes auth method, a policy, and a role:
+
+    ```bash
+    # Enable Kubernetes auth method
+    vault auth enable kubernetes
+
+    # Configure Kubernetes auth
+    vault write auth/kubernetes/config \
+        kubernetes_host="https://$KUBERNETES_HOST:443"
+
+    # Create a policy granting access to the transit engine
+    vault policy write rgw-sse-s3 - <<EOF
+    path "transit/keys/*" {
+      capabilities = ["create", "read", "update", "delete"]
+    }
+    path "transit/keys/+/rotate" {
+      capabilities = ["update"]
+    }
+    path "transit/datakey/plaintext/*" {
+      capabilities = ["update"]
+    }
+    path "transit/decrypt/*" {
+      capabilities = ["update"]
+    }
+    path "transit/export/encryption-key/*" {
+      capabilities = ["read"]
+    }
+    EOF
+
+    # Create a role bound to the RGW service account
+    vault write auth/kubernetes/role/rook-ceph-rgw \
+        bound_service_account_names=rook-ceph-rgw \
+        bound_service_account_namespaces=rook-ceph \
+        policies=rgw-sse-s3 \
+        ttl=1h
+    ```
+
+#### Configuration
+
+Set `VAULT_AUTH_METHOD: agent` in the `s3` connection details and add Vault Agent Injector annotations to the gateway spec. The `VAULT_ADDR` should point to the local Vault Agent proxy (typically `http://127.0.0.1:8100`). No `tokenSecretName` is needed since the injected agent handles authentication.
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: my-store
+  namespace: rook-ceph
+spec:
+  gateway:
+    annotations:
+      vault.hashicorp.com/agent-inject: "true"
+      vault.hashicorp.com/role: "rook-ceph-rgw"
+      vault.hashicorp.com/agent-cache-enable: "true"
+      vault.hashicorp.com/agent-cache-listener-port: "8100"
+      # For TLS to Vault server:
+      # vault.hashicorp.com/ca-cert: "/vault/tls/ca.crt"
+      # vault.hashicorp.com/client-cert: "/vault/tls/tls.crt"
+      # vault.hashicorp.com/client-key: "/vault/tls/tls.key"
+  security:
+    s3:
+      connectionDetails:
+        KMS_PROVIDER: vault
+        VAULT_ADDR: http://127.0.0.1:8100
+        VAULT_SECRET_ENGINE: transit
+        VAULT_AUTH_METHOD: agent
+```
+
 ## Advanced configuration
 
 !!! warning
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 42835d613..eeba44069 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -8,3 +8,4 @@
 ## Features
 
 - RGW Account support: Added `CephObjectStoreAccount` CRD for managing RGW accounts, and `accountRef` field in `CephObjectStoreUser` to associate users with accounts. This feature is experimental and currently only supported with the Ceph main branch image (`quay.ceph.io/ceph-ci/ceph:main`). See the [Object Store Accounts](Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-accounts.md) documentation for more details.
+- SSE-S3 with Vault Agent: Added support for server-side encryption with SSE-S3 using HashiCorp Vault Agent authentication. See the [CephObjectStore Security Settings](Documentation/CRDs/Object-Storage/ceph-object-store-crd.md#sse-s3-with-vault-agent) for more details.
diff --git a/pkg/apis/ceph.rook.io/v1/security.go b/pkg/apis/ceph.rook.io/v1/security.go
index 71379b665..42bf57443 100644
--- a/pkg/apis/ceph.rook.io/v1/security.go
+++ b/pkg/apis/ceph.rook.io/v1/security.go
@@ -41,6 +41,11 @@ func (kms *KeyManagementServiceSpec) IsK8sAuthEnabled() bool {
 	return getParam(kms.ConnectionDetails, vault.AuthMethod) == vault.AuthMethodKubernetes && kms.TokenSecretName == ""
 }
 
+// IsAgentAuthEnabled return whether Vault Agent auth is enabled
+func (kms *KeyManagementServiceSpec) IsAgentAuthEnabled() bool {
+	return getParam(kms.ConnectionDetails, vault.AuthMethod) == "agent" && kms.TokenSecretName == ""
+}
+
 // IsVaultKMS return whether Vault KMS is configured
 func (kms *KeyManagementServiceSpec) IsVaultKMS() bool {
 	return getParam(kms.ConnectionDetails, "KMS_PROVIDER") == secrets.TypeVault
diff --git a/pkg/apis/ceph.rook.io/v1/security_test.go b/pkg/apis/ceph.rook.io/v1/security_test.go
index 78348e545..85d237d4f 100644
--- a/pkg/apis/ceph.rook.io/v1/security_test.go
+++ b/pkg/apis/ceph.rook.io/v1/security_test.go
@@ -18,6 +18,34 @@ package v1
 
 import "testing"
 
+func TestKeyManagementServiceSpec_IsAgentAuthEnabled(t *testing.T) {
+	type fields struct {
+		ConnectionDetails map[string]string
+		TokenSecretName   string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   bool
+	}{
+		{"agent auth is disabled - everything is empty", fields{ConnectionDetails: map[string]string{}, TokenSecretName: ""}, false},
+		{"agent auth is disabled - token is populated", fields{ConnectionDetails: map[string]string{"VAULT_AUTH_METHOD": "agent"}, TokenSecretName: "rook-ceph-test-secret"}, false},
+		{"agent auth is disabled since VAULT_AUTH_METHOD is not agent", fields{ConnectionDetails: map[string]string{"VAULT_AUTH_METHOD": "token"}, TokenSecretName: ""}, false},
+		{"agent auth is enabled", fields{ConnectionDetails: map[string]string{"VAULT_AUTH_METHOD": "agent"}, TokenSecretName: ""}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			kms := &KeyManagementServiceSpec{
+				ConnectionDetails: tt.fields.ConnectionDetails,
+				TokenSecretName:   tt.fields.TokenSecretName,
+			}
+			if got := kms.IsAgentAuthEnabled(); got != tt.want {
+				t.Errorf("KeyManagementServiceSpec.IsAgentAuthEnabled() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestKeyManagementServiceSpec_IsK8sAuthEnabled(t *testing.T) {
 	type fields struct {
 		ConnectionDetails map[string]string
diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go
index acd07d1fc..9b162d214 100644
--- a/pkg/daemon/ceph/osd/kms/kms.go
+++ b/pkg/daemon/ceph/osd/kms/kms.go
@@ -362,7 +362,7 @@ func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Co
 
 	// A token must be specified if token-auth is used for KMS other than Azure
 	if !kms.IsAzureMS() {
-		if !kms.IsK8sAuthEnabled() && kms.TokenSecretName == "" {
+		if !kms.IsK8sAuthEnabled() && !kms.IsAgentAuthEnabled() && kms.TokenSecretName == "" {
 			if !kms.IsTokenAuthEnabled() {
 				return errors.New("failed to validate kms configuration (missing token in spec)")
 			}
diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go
index d9cec0951..27df46066 100644
--- a/pkg/operator/ceph/object/spec.go
+++ b/pkg/operator/ceph/object/spec.go
@@ -285,19 +285,25 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
 		}
 		podSpec.Volumes = append(podSpec.Volumes, v)
 
-		if kmsEnabled && c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
+		kmsNeedsToken := kmsEnabled && c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled()
+		s3NeedsToken := s3Enabled && c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled()
+		if kmsNeedsToken {
 			vaultFileVol, _ := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.KeyManagementService.ConnectionDetails,
 				c.store.Spec.Security.KeyManagementService.TokenSecretName, sseKMS)
 			podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
 		}
-		if s3Enabled && c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
+		if s3NeedsToken {
 			vaultFileVol, _ := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails,
 				c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName, sseS3)
 			podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
 		}
 
-		podSpec.InitContainers = append(podSpec.InitContainers,
-			c.vaultTokenInitContainer(rgwConfig, kmsEnabled, s3Enabled))
+		// Only add the vault token init container if at least one of KMS or SSE-S3 needs token-based auth.
+		// When SSE-S3 uses agent auth, no token init container is needed for that part.
+		if kmsNeedsToken || s3NeedsToken {
+			podSpec.InitContainers = append(podSpec.InitContainers,
+				c.vaultTokenInitContainer(rgwConfig, kmsNeedsToken, s3NeedsToken))
+		}
 	}
 	c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec)
 
@@ -524,7 +530,9 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
 		log.NamedDebug(nsName, logger, "enabliing SSE-S3. %v", c.store.Spec.Security.ServerSideEncryptionS3)
 
 		container.Args = append(container.Args, c.sseS3DefaultOptions(s3EncryptionEnabled)...)
-		if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
+		if c.store.Spec.Security.ServerSideEncryptionS3.IsAgentAuthEnabled() {
+			container.Args = append(container.Args, c.sseS3VaultAgentOptions(s3EncryptionEnabled)...)
+		} else if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
 			container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3EncryptionEnabled)...)
 		}
 		if c.store.Spec.Security.ServerSideEncryptionS3.IsTLSEnabled() {
@@ -896,6 +904,12 @@ func (c *clusterConfig) CheckRGWKMS() (bool, error) {
 
 func (c *clusterConfig) CheckRGWSSES3Enabled() (bool, error) {
 	if c.store.Spec.Security != nil && c.store.Spec.Security.ServerSideEncryptionS3.IsEnabled() {
+		// When agent auth is used, tokenSecretName must not be set
+		if kms.GetParam(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, vault.AuthMethod) == "agent" &&
+			c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName != "" {
+			return false, errors.New("tokenSecretName must not be set when VAULT_AUTH_METHOD is agent for SSE-S3")
+		}
+
 		err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, &c.store.Spec.Security.ServerSideEncryptionS3, c.store.Namespace)
 		if err != nil {
 			return false, err
@@ -1087,6 +1101,19 @@ func (c *clusterConfig) sseKMSVaultTLSOptions(setOptions bool) []string {
 	return rgwOptions
 }
 
+func (c *clusterConfig) sseS3VaultAgentOptions(setOptions bool) []string {
+	if setOptions {
+		return []string{
+			cephconfig.NewFlag("rgw crypt sse s3 vault auth", "agent"),
+			cephconfig.NewFlag("rgw crypt sse s3 vault prefix",
+				path.Join(vaultPrefix, c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey])),
+			cephconfig.NewFlag("rgw crypt sse s3 vault secret engine",
+				c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey]),
+		}
+	}
+	return []string{}
+}
+
 func (c *clusterConfig) sseS3VaultTLSOptions(setOptions bool) []string {
 	var rgwOptions []string
 	if setOptions {
diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go
index 4bd5856ba..3f675ff37 100644
--- a/pkg/operator/ceph/object/spec_test.go
+++ b/pkg/operator/ceph/object/spec_test.go
@@ -43,6 +43,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func configureSSEWithVaultAgent(c *clusterConfig) {
+	c.store.Spec.Security = &cephv1.ObjectStoreSecuritySpec{}
+	c.store.Spec.Security.ServerSideEncryptionS3 = cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}
+	c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["KMS_PROVIDER"] = "vault"
+	c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_ADDR"] = "http://127.0.0.1:8100"
+	c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_AUTH_METHOD"] = "agent"
+}
+
 func configureSSE(t *testing.T, c *clusterConfig, kms bool, s3 bool) {
 	c.store.Spec.Security = &cephv1.ObjectStoreSecuritySpec{}
 	if kms {
@@ -991,6 +999,25 @@ func TestAWSServerSideEncryption(t *testing.T) {
 		assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseKMSVaultTLSOptions(true)))
 		assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultTLSOptions(true)))
 	})
+
+	t.Run("SSE-S3 with vault agent auth sets agent options instead of token options", func(t *testing.T) {
+		configureSSEWithVaultAgent(c)
+		c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit"
+		rgwContainer, err := c.makeDaemonContainer(rgwConfig)
+		assert.NoError(t, err)
+		assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseS3DefaultOptions(true)))
+		assert.True(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultAgentOptions(true)))
+		assert.False(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultTokenOptions(true)))
+		assert.False(t, checkRGWOptions(rgwContainer.Args, c.sseS3VaultTLSOptions(false)))
+	})
+
+	t.Run("SSE-S3 with vault agent auth rejects tokenSecretName", func(t *testing.T) {
+		configureSSEWithVaultAgent(c)
+		c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit"
+		c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName = "vault-token"
+		_, err := c.makeDaemonContainer(rgwConfig)
+		assert.Error(t, err)
+	})
 }
 
 func TestRgwCommandFlags(t *testing.T) {