From f9eaa49794cf74836bd5724fc97bc85fca050880 Mon Sep 17 00:00:00 2001 From: Artem Torubarov Date: Tue, 10 Dec 2024 11:49:40 +0100 Subject: [PATCH] rgw: add rgw_enable_apis config option Signed-off-by: Artem Torubarov (cherry picked from commit 83c8ec81609a02e4aacad38af6d72131ba4fb0a6) --- .../Object-Storage/ceph-object-store-crd.md | 17 +- Documentation/CRDs/specification.md | 27 +- .../Object-Storage-RGW/object-storage.md | 114 +++++++ PendingReleaseNotes.md | 1 + .../charts/rook-ceph/templates/resources.yaml | 23 +- deploy/examples/crds.yaml | 23 +- .../examples/object-multi-instance-test.yaml | 123 ++++++++ pkg/apis/ceph.rook.io/v1/types.go | 12 + .../ceph.rook.io/v1/zz_generated.deepcopy.go | 5 + pkg/operator/ceph/object/config.go | 31 -- pkg/operator/ceph/object/rgw-probe.sh | 3 +- pkg/operator/ceph/object/spec.go | 105 ++++++- pkg/operator/ceph/object/spec_test.go | 288 ++++++++++++++++++ 13 files changed, 730 insertions(+), 42 deletions(-) create mode 100644 deploy/examples/object-multi-instance-test.yaml diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 9c928a372..d3075e2a5 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -138,11 +138,26 @@ The following options can be configured in the `keystone`-section: * `tokenCacheSize`: specifies the maximum number of entries in each Keystone token cache. * `url`: The url of the Keystone API endpoint to use. -The protocols section is divided into two parts: +### Protocols Settings +The protocols section is divided into three parts: + +- `enableAPIs` - list of APIs to be enabled in RGW instance. If no values set, all APIs will be enabled. Possible values: `s3, s3website, swift, swift_auth, admin, sts, iam, notifications`. Represents RGW [rgw_enable_apis](https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis) config parameter. - a section to configure S3 - a section to configure swift +```yaml +spec: + [...] + protocols: + enableAPIs: [] + swift: + # a section to configure swift + s3: + # a section to configure s3 + [...] +``` + #### protocols/S3 settings In the `s3` section of the `protocols` section the following options can be configured: diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 45a2c7f0f..9530f3501 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -9854,6 +9854,13 @@ If spec.sharedPools are also empty, then RGW pools (spec.dataPool and spec.metad +

ObjectStoreAPI +(string alias)

+

+(Appears on:ProtocolSpec) +

+
+

ObjectStoreHostingSpec

@@ -11422,6 +11429,23 @@ alive or ready to receive traffic.

+enableAPIs
+ + +[]ObjectStoreAPI + + + + +(Optional) +

Represents RGW ‘rgw_enable_apis’ config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis +If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications +If enabled APIs are set, all remaining APIs will be disabled. +This option overrides S3.Enabled value.

+ + + + s3
@@ -11925,7 +11949,8 @@ bool (Optional) -

Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.

+

Deprecated: use protocol.enableAPIs instead. +Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.

diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index 10123bef9..484e2424c 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -17,6 +17,7 @@ Rook can configure the Ceph Object Store for several different scenarios. See ea 3. Create [one or more object stores with pool placement targets and storage classes](#create-local-object-stores-with-pool-placements). This configuration allows Rook to provide different object placement options to object store clients. 4. Connect to an [RGW service in an external Ceph cluster](#connect-to-an-external-object-store), rather than create a local object store. 5. Configure [RGW Multisite](#object-multisite) to synchronize buckets between object stores in different clusters. +6. Create a [multi-instance RGW setup](#object-multi-instance). This option allows to have multiple `CephObjectStore` with different configurations backed by the same storage pools. For example, serving S3, Swift, or Admin-ops API by separate RGW instances. !!! note Updating the configuration of an object store between these types is not supported. @@ -669,6 +670,119 @@ Multisite also allows object stores to be independent and isolated from other ob For more information on multisite please read the [ceph multisite overview](ceph-object-multisite.md) for how to run it. +## Object Multi-instance + +This section describes how to configure multiple `CephObjectStore` backed by the same storage pools. +The setup allows using different configuration parameters for each `CephObjecStore`, like: + +- `hosting` and `gateway` configs to host object store APIs on different ports and domains. For example, having a independently-scalable deployments for internal and external traffic. +- `protocols` to host `S3`, `Swift`, and `Admin-ops` APIs on a separate `CephObjectStores`. +- having different resource limits, affinity or other configurations per `CephObjecStore` instance for other possible use-cases. + +Multi-instance setup can be described in two steps. The first step is to create `CephObjectRealm`, `CephObjectZoneGroup`, and `CephObjectZone`, where +`CephObjectZone` contains storage pools configuration. This configuration will be shared across all `CephObjectStore` instances: + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephObjectRealm +metadata: + name: multi-instance-store + namespace: rook-ceph # namespace:cluster +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZoneGroup +metadata: + name: multi-instance-store + namespace: rook-ceph # namespace:cluster +spec: + realm: multi-instance-store +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZone +metadata: + name: multi-instance-store + namespace: rook-ceph # namespace:cluster +spec: + zoneGroup: multi-instance-store + metadataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + dataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false +``` + +The second step defines multiple `CephObjectStore` with different configurations. All of the stores should refer to the same `zone`. + +```yaml +# RGW instance to host admin-ops API only +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: store-admin + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + instances: 1 + zone: + name: multi-instance-store + protocols: + enableAPIs: ["admin"] +--- +# RGW instance to host S3 API only +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: store-s3 + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + instances: 1 + zone: + name: multi-instance-store + protocols: + enableAPIs: + - s3 + - s3website + - sts + - iam + - notifications +--- +# RGW instance to host SWIFT API only +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: store-swift + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + instances: 1 + zone: + name: multi-instance-store + protocols: + enableAPIs: + - swift + - swift_auth + swift: + # if S3 API is disabled, then SWIFT can be hosted on root path without prefix + urlPrefix: "/" +``` + +!!! note + Child resources should refer to the appropriate RGW instance. + For example, a `CephObjectStoreUser` requires the Admin Ops API, + so it should refer to an instance where this API is enabled. + After the user is created, it can be used for all instances. + +See the [example configuration](https://github.com/rook/rook/blob/master/deploy/examples/object-multi-instance-test.yaml) for more details. + ## Using Swift and Keystone It is possible to access an object store using the [Swift API](https://developer.openstack.org/api-ref/object-store/index.html). diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index ef92c6650..a43b0c75c 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,3 +9,4 @@ - Enable mirroring for CephBlockPoolRadosNamespaces (see [#14701](https://github.com/rook/rook/pull/14701)). - Enable periodic monitoring for CephBlockPoolRadosNamespaces mirroring (see [#14896](https://github.com/rook/rook/pull/14896)). - Allow migration of PVC based OSDs to enable or disable encryption (see [#14776](https://github.com/rook/rook/pull/14776)). +- Support `rgw_enable_apis` option for CephObjectStore (see [#15064](https://github.com/rook/rook/pull/15064)). diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index cc9cd8917..ea44da105 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -12428,6 +12428,25 @@ spec: protocols: description: The protocol specification properties: + enableAPIs: + description: |- + Represents RGW 'rgw_enable_apis' config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis + If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications + If enabled APIs are set, all remaining APIs will be disabled. + This option overrides S3.Enabled value. + items: + enum: + - s3 + - s3website + - swift + - swift_auth + - admin + - sts + - iam + - notifications + type: string + nullable: true + type: array s3: description: The spec for S3 nullable: true @@ -12437,7 +12456,9 @@ spec: nullable: true type: boolean enabled: - description: Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. + description: |- + Deprecated: use protocol.enableAPIs instead. + Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. nullable: true type: boolean type: object diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 1c7c0ccec..1ec217174 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -12419,6 +12419,25 @@ spec: protocols: description: The protocol specification properties: + enableAPIs: + description: |- + Represents RGW 'rgw_enable_apis' config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis + If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications + If enabled APIs are set, all remaining APIs will be disabled. + This option overrides S3.Enabled value. + items: + enum: + - s3 + - s3website + - swift + - swift_auth + - admin + - sts + - iam + - notifications + type: string + nullable: true + type: array s3: description: The spec for S3 nullable: true @@ -12428,7 +12447,9 @@ spec: nullable: true type: boolean enabled: - description: Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. + description: |- + Deprecated: use protocol.enableAPIs instead. + Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. nullable: true type: boolean type: object diff --git a/deploy/examples/object-multi-instance-test.yaml b/deploy/examples/object-multi-instance-test.yaml new file mode 100644 index 000000000..740a51eda --- /dev/null +++ b/deploy/examples/object-multi-instance-test.yaml @@ -0,0 +1,123 @@ +################################################################################################################# +# Create an object store with settings for a test environment. Only a single OSD is required in this example. +# kubectl create -f object-multi-instance-test.yaml +# +# The example below provides minimalistic test configuration for multi-instance RGW deployment. +# It defines CephObjectRealm, CephObjectZoneGroup, and CephObjectZone CRs responsible for storage pools configuration. +# Then, three separate CephObjectStore resources are created. +# These stores refer to the same CephObjectZone, and therefore, to the same storage pools, representing the same ObjectStorage data. +# The CephObjectStore resources are configured to host different RGW APIs: S3, SWIFT, and Admin-ops. +# The configuration of the CephObjectStore can be further extended to include any number of RGW deployments backed by the same data. +# Deployments can be customized to host different APIs, allocate different resources, use different domains, +# and apply any other options available in the CephObjectStore CRD. +# Finally, CephObjectStoreUser is created. Child resources like user should refer to RGW instance hosting admin-ops API. +# In this case - store-admin. +################################################################################################################# +apiVersion: ceph.rook.io/v1 +kind: CephObjectRealm +metadata: + name: multi-instance-store + namespace: rook-ceph # namespace:cluster +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZoneGroup +metadata: + # Names for realm/zonegroup/zone can be arbitrary. + # For non-multisite setup, reusing the same name across realm/zonegroup/zone is advised for simplicity. + # If multiple sharedPools.poolPlacements are configured for zone, then zonegroup name will be used as a prefix + # in S3 bucket location: :. See pool placement documentation: + # https://docs.ceph.com/en/latest/radosgw/placement/#s3-bucket-placement - Ceph RGW doc + name: multi-instance-store + namespace: rook-ceph # namespace:cluster +spec: + realm: multi-instance-store +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZone +metadata: + name: multi-instance-store + namespace: rook-ceph # namespace:cluster +spec: + zoneGroup: multi-instance-store + metadataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + dataPool: + failureDomain: host + replicated: + size: 1 + requireSafeReplicaSize: false + # Alternatively, configure pool placements with pre-existing pools here. + # More details about Pool placements and storage classes: + # https://rook.io/docs/rook/latest-release/Storage-Configuration/Object-Storage-RGW/object-storage/#create-local-object-stores-with-pool-placements + # sharedPools: + # poolPlacements: + # ... +--- +# RGW instance to host admin-ops API only +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: store-admin + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + instances: 1 + zone: + name: multi-instance-store + protocols: + enableAPIs: ["admin"] +--- +# RGW instance to host S3 API only +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: store-s3 + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + instances: 1 + zone: + name: multi-instance-store + protocols: + enableAPIs: + - s3 + - s3website + - sts + - iam + - notifications +--- +# RGW instance to host SWIFT API only +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: store-swift + namespace: rook-ceph # namespace:cluster +spec: + gateway: + port: 80 + instances: 1 + zone: + name: multi-instance-store + protocols: + enableAPIs: + - swift + - swift_auth + swift: + # if S3 API is disabled, then SWIFT can be hosted on root path without prefix + urlPrefix: "/" +--- +# ObjectStore user should refer to ObjectStore instance hosting admin API. +# Created used can be used for all ObjectStore instances. +apiVersion: ceph.rook.io/v1 +kind: CephObjectStoreUser +metadata: + name: multi-instance-user + namespace: rook-ceph # namespace:cluster +spec: + store: store-admin + displayName: "my display name" diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 1e25fcebb..2ef5e2858 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1751,6 +1751,14 @@ type EndpointAddress struct { // ProtocolSpec represents a Ceph Object Store protocol specification type ProtocolSpec struct { + // Represents RGW 'rgw_enable_apis' config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis + // If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications + // If enabled APIs are set, all remaining APIs will be disabled. + // This option overrides S3.Enabled value. + // +optional + // +nullable + EnableAPIs []ObjectStoreAPI `json:"enableAPIs,omitempty"` + // The spec for S3 // +optional // +nullable @@ -1762,8 +1770,12 @@ type ProtocolSpec struct { Swift *SwiftSpec `json:"swift"` } +// +kubebuilder:validation:Enum=s3;s3website;swift;swift_auth;admin;sts;iam;notifications +type ObjectStoreAPI string + // S3Spec represents Ceph Object Store specification for the S3 API type S3Spec struct { + // Deprecated: use protocol.enableAPIs instead. // Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. // +nullable // +optional diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index c5dff621e..b63b72756 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -4261,6 +4261,11 @@ func (in *ProbeSpec) DeepCopy() *ProbeSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtocolSpec) DeepCopyInto(out *ProtocolSpec) { *out = *in + if in.EnableAPIs != nil { + in, out := &in.EnableAPIs, &out.EnableAPIs + *out = make([]ObjectStoreAPI, len(*in)) + copy(*out, *in) + } if in.S3 != nil { in, out := &in.S3, &out.S3 *out = new(S3Spec) diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index cbfc3ce99..983589d0d 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -174,53 +174,22 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { return err } - s3disabled := false if s3 := rgwConfig.Protocols.S3; s3 != nil { - if s3.Enabled != nil && !*s3.Enabled { - s3disabled = true - } - if s3.AuthUseKeystone != nil { configOptions["rgw_s3_auth_use_keystone"] = fmt.Sprintf("%t", *s3.AuthUseKeystone) } - } if swift := rgwConfig.Protocols.Swift; swift != nil { - if swift.AccountInUrl != nil { configOptions["rgw_swift_account_in_url"] = fmt.Sprintf("%t", *swift.AccountInUrl) } - if swift.UrlPrefix != nil { configOptions["rgw_swift_url_prefix"] = *swift.UrlPrefix - - if configOptions["rgw_swift_url_prefix"] == "/" { - logger.Warning("Forcefully disabled S3 as the swift prefix is given as a slash /. Ignoring any S3 options (including Enabled=true)!") - // this will later on disable the s3 api using the rgw_enable_apis setting - s3disabled = true - } - } if swift.VersioningEnabled != nil { configOptions["rgw_swift_versioning_enabled"] = fmt.Sprintf("%t", *swift.VersioningEnabled) } - - } - - if s3disabled { - // XXX: how to handle enabled APIs? We only configure s3 and - // swift in the resource, `admin` is required for the operator to - // work, `swift_auth` is required to access swift without keystone - // – not sure about the additional APIs - // https://docs.ceph.com/en/latest/radosgw/config-ref/#confval-rgw_enable_apis - // see also https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings on disabling s3 - // when using '/' as prefix - - // Swift was enabled so far already by default, so perhaps better - // not change that if someone relies on it. - - configOptions["rgw_enable_apis"] = "s3website, swift, swift_auth, admin, sts, iam, notifications" } for flag, val := range configOptions { diff --git a/pkg/operator/ceph/object/rgw-probe.sh b/pkg/operator/ceph/object/rgw-probe.sh index 853b430bc..0b7de8979 100644 --- a/pkg/operator/ceph/object/rgw-probe.sh +++ b/pkg/operator/ceph/object/rgw-probe.sh @@ -3,6 +3,7 @@ PROBE_TYPE="{{ .ProbeType }}" PROBE_PORT="{{ .Port }}" PROBE_PROTOCOL="{{ .Protocol }}" +PROBE_PATH="{{ .Path }}" # standard bash codes start at 126 and progress upward. pick error codes from 125 downward for # script as to allow curl to output new error codes and still return a distinctive number. @@ -13,7 +14,7 @@ PROBE_ERR_CODE=124 STARTUP_TYPE='startup' READINESS_TYPE='readiness' -RGW_URL="$PROBE_PROTOCOL://0.0.0.0:$PROBE_PORT" +RGW_URL="$PROBE_PROTOCOL://0.0.0.0:$PROBE_PORT$PROBE_PATH" function check() { local URL="$1" diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 1b6e5b056..b84b44b17 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -22,6 +22,7 @@ import ( "fmt" "net/url" "path" + "slices" "strings" "text/template" @@ -71,6 +72,8 @@ chown --recursive --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH var ( //go:embed rgw-probe.sh rgwProbeScriptTemplate string + + rgwAPIwithoutS3 = []string{"s3website", "swift", "swift_auth", "admin", "sts", "iam", "notifications"} ) type ProbeType string @@ -89,6 +92,7 @@ type rgwProbeConfig struct { Protocol ProtocolType Port string + Path string } func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment, error) { @@ -401,23 +405,27 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, } } - s3Enabled, err := c.CheckRGWSSES3Enabled() + if flags := buildRGWConfigFlags(c.store); len(flags) != 0 { + container.Args = append(container.Args, flags...) + } + + s3EncryptionEnabled, err := c.CheckRGWSSES3Enabled() if err != nil { return v1.Container{}, err } - if s3Enabled { + if s3EncryptionEnabled { logger.Debugf("enabliing SSE-S3. %v", c.store.Spec.Security.ServerSideEncryptionS3) - container.Args = append(container.Args, c.sseS3DefaultOptions(s3Enabled)...) + container.Args = append(container.Args, c.sseS3DefaultOptions(s3EncryptionEnabled)...) if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() { - container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3Enabled)...) + container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3EncryptionEnabled)...) } if c.store.Spec.Security.ServerSideEncryptionS3.IsTLSEnabled() { - container.Args = append(container.Args, c.sseS3VaultTLSOptions(s3Enabled)...) + container.Args = append(container.Args, c.sseS3VaultTLSOptions(s3EncryptionEnabled)...) } } - if s3Enabled || kmsEnabled { + if s3EncryptionEnabled || kmsEnabled { vaultVolMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName} container.VolumeMounts = append(container.VolumeMounts, vaultVolMount) } @@ -459,11 +467,17 @@ func noLivenessProbe() *v1.Probe { } func (c *clusterConfig) defaultReadinessProbe() (*v1.Probe, error) { + probePath, disableProbe := getRGWProbePath(c.store.Spec.Protocols) + if disableProbe { + logger.Infof("disabling startup probe for %q store", c.store.Name) + return nil, nil + } proto, port := c.endpointInfo() cfg := rgwProbeConfig{ ProbeType: ReadinessProbeType, Protocol: proto, Port: port.String(), + Path: probePath, } script, err := renderProbe(cfg) if err != nil { @@ -489,13 +503,52 @@ func (c *clusterConfig) defaultReadinessProbe() (*v1.Probe, error) { return probe, nil } +// getRGWProbePath - returns custom path for RGW probe and returns true if probe should be disabled. +func getRGWProbePath(protocolSpec cephv1.ProtocolSpec) (path string, disable bool) { + enabledAPIs := buildRGWEnableAPIsConfigVal(protocolSpec) + if len(enabledAPIs) == 0 { + // all apis including s3 are enabled + // using default s3 Probe + return "", false + } + if slices.Contains(enabledAPIs, "s3") { + // using default s3 Probe + return "", false + } + if slices.Contains(enabledAPIs, "swift") { + // using swift api for probe + // calculate path for swift probe + prefix := "/swift/" + if protocolSpec.Swift != nil && protocolSpec.Swift.UrlPrefix != nil && *protocolSpec.Swift.UrlPrefix != "" { + prefix = *protocolSpec.Swift.UrlPrefix + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + } + prefix += "info" + return prefix, false + } + // both swift and s3 are disabled - disable probe. + return "", true +} + func (c *clusterConfig) defaultStartupProbe() (*v1.Probe, error) { + probePath, disableProbe := getRGWProbePath(c.store.Spec.Protocols) + if disableProbe { + logger.Infof("disabling startup probe for %q store", c.store.Name) + return nil, nil + } proto, port := c.endpointInfo() cfg := rgwProbeConfig{ ProbeType: StartupProbeType, Protocol: proto, Port: port.String(), + Path: probePath, } + script, err := renderProbe(cfg) if err != nil { return nil, err @@ -904,6 +957,46 @@ func (c *clusterConfig) sseS3VaultTLSOptions(setOptions bool) []string { return rgwOptions } +// Builds list of rgw config parameters which should be passed as CLI flags. +// Consider set config option as flag if BOTH criteria fulfilled: +// 1. config value is not secret +// 2. config change requires RGW daemon restart +// +// Otherwise set rgw config parameter to mon database in ./config.go -> setFlagsMonConfigStore() +// CLI flags override values from mon db: see ceph config docs: https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/#config-sources +func buildRGWConfigFlags(objectStore *cephv1.CephObjectStore) []string { + var res []string + // todo: move all flags here + if enableAPIs := buildRGWEnableAPIsConfigVal(objectStore.Spec.Protocols); len(enableAPIs) != 0 { + res = append(res, cephconfig.NewFlag("rgw_enable_apis", strings.Join(enableAPIs, ","))) + logger.Debugf("Enabling APIs for RGW instance %q: %s", objectStore.Name, enableAPIs) + } + return res +} + +func buildRGWEnableAPIsConfigVal(protocolSpec cephv1.ProtocolSpec) []string { + if len(protocolSpec.EnableAPIs) != 0 { + // handle explicit enabledAPIS spec + enabledAPIs := make([]string, len(protocolSpec.EnableAPIs)) + for i, v := range protocolSpec.EnableAPIs { + enabledAPIs[i] = strings.TrimSpace(string(v)) + } + return enabledAPIs + } + + // if enabledAPIs not set, check if S3 should be disabled + if protocolSpec.S3 != nil && protocolSpec.S3.Enabled != nil && !*protocolSpec.S3.Enabled { //nolint // disable deprecation check + return rgwAPIwithoutS3 + } + // see also https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings on disabling s3 + // when using '/' as prefix + if protocolSpec.Swift != nil && protocolSpec.Swift.UrlPrefix != nil && *protocolSpec.Swift.UrlPrefix == "/" { + logger.Warning("Forcefully disabled S3 as the swift prefix is given as a slash /. Ignoring any S3 options (including Enabled=true)!") + return rgwAPIwithoutS3 + } + return nil +} + func renderProbe(cfg rgwProbeConfig) (string, error) { var writer bytes.Buffer name := string(cfg.ProbeType) + "-probe" diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index f920f5f5a..32f65f618 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -18,7 +18,9 @@ package object import ( "context" + _ "embed" "fmt" + "reflect" "testing" "github.com/pkg/errors" @@ -1151,3 +1153,289 @@ func TestGetHostnameFromEndpoint(t *testing.T) { }) } } + +func strPtr(in string) *string { + return &in +} + +func Test_buildRGWEnableAPIsConfigVal(t *testing.T) { + type args struct { + protocolSpec cephv1.ProtocolSpec + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "nothing set", + args: args{ + protocolSpec: cephv1.ProtocolSpec{}, + }, + want: nil, + }, + { + name: "only admin enabled", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "admin", + }, + }, + }, + want: []string{"admin"}, + }, + { + name: "admin and s3 enabled", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "admin", + "s3", + }, + }, + }, + want: []string{"admin", "s3"}, + }, + { + name: "whitespaces trimmed", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + " s3 ", + " swift ", + }, + }, + }, + want: []string{"s3", "swift"}, + }, + { + name: "s3 disabled", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + S3: &cephv1.S3Spec{ + Enabled: new(bool), + }, + }, + }, + want: rgwAPIwithoutS3, + }, + { + name: "s3 disabled when swift prefix is '/'", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + Swift: &cephv1.SwiftSpec{ + UrlPrefix: strPtr("/"), + }, + }, + }, + want: rgwAPIwithoutS3, + }, + { + name: "s3 is not disabled when swift prefix is not '/'", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + Swift: &cephv1.SwiftSpec{ + UrlPrefix: strPtr("object/"), + }, + }, + }, + want: nil, + }, + { + name: "enableAPIs overrides s3 option", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + " s3 ", + " swift ", + }, + S3: &cephv1.S3Spec{ + Enabled: new(bool), + }, + }, + }, + want: []string{"s3", "swift"}, + }, + { + name: "enableAPIs overrides swift prefix option", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + " s3 ", + " swift ", + }, + Swift: &cephv1.SwiftSpec{ + UrlPrefix: strPtr("/"), + }, + }, + }, + want: []string{"s3", "swift"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildRGWEnableAPIsConfigVal(tt.args.protocolSpec) + assert.ElementsMatch(t, got, tt.want) + }) + } +} + +func Test_buildRGWConfigFlags(t *testing.T) { + type args struct { + objectStore *cephv1.CephObjectStore + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "nothing set", + args: args{ + objectStore: &cephv1.CephObjectStore{}, + }, + want: nil, + }, + { + name: "enabled APIs set", + args: args{ + objectStore: &cephv1.CephObjectStore{ + Spec: cephv1.ObjectStoreSpec{ + Protocols: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "swift", + "admin", + }, + }, + }, + }, + }, + want: []string{ + "--rgw-enable-apis=swift,admin", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildRGWConfigFlags(tt.args.objectStore); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildRGWConfigFlags() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getRGWProbePathAndCode(t *testing.T) { + type args struct { + protocolSpec cephv1.ProtocolSpec + } + tests := []struct { + name string + args args + wantPath string + wantDisable bool + }{ + { + name: "all apis enabled - return s3 probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{}, + S3: &cephv1.S3Spec{}, + Swift: &cephv1.SwiftSpec{}, + }, + }, + wantPath: "", + wantDisable: false, + }, + { + name: "s3 disabled - return default swift probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{}, + S3: &cephv1.S3Spec{ + Enabled: new(bool), + }, + Swift: &cephv1.SwiftSpec{}, + }, + }, + wantPath: "/swift/info", + wantDisable: false, + }, + { + name: "s3 disabled in api list - return default swift probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "swift", + "admin", + }, + }, + }, + wantPath: "/swift/info", + wantDisable: false, + }, + { + name: "s3 disabled - return swift with custom prefix probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "swift", + "admin", + }, + Swift: &cephv1.SwiftSpec{ + UrlPrefix: strPtr("some-path"), + }, + }, + }, + wantPath: "/some-path/info", + wantDisable: false, + }, + { + name: "s3 disabled - return swift with root prefix probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + Swift: &cephv1.SwiftSpec{ + UrlPrefix: strPtr("/"), + }, + }, + }, + wantPath: "/info", + wantDisable: false, + }, + { + name: "s3 and swift disabled - disable probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "admin", + }, + }, + }, + wantPath: "", + wantDisable: true, + }, + { + name: "no suitable api enabled - disable probe", + args: args{ + protocolSpec: cephv1.ProtocolSpec{ + EnableAPIs: []cephv1.ObjectStoreAPI{ + "sts", + }, + }, + }, + wantPath: "", + wantDisable: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPath, gotDisable := getRGWProbePath(tt.args.protocolSpec) + if gotPath != tt.wantPath { + t.Errorf("getRGWProbePath() gotPath = %v, want %v", gotPath, tt.wantPath) + } + if gotDisable != tt.wantDisable { + t.Errorf("getRGWProbePath() gotDisable = %v, want %v", gotDisable, tt.wantDisable) + } + }) + } +}