diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index 54e6aeef3..d11bcb6a4 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -190,7 +190,7 @@ jobs:
- name: validate-rgw-endpoint
run: |
- rgw_endpoint=$(kubectl get service -n rook-ceph | awk '/rgw/ {print $3":80"}')
+ rgw_endpoint=$(kubectl get service -n rook-ceph -l rgw=store-a | awk '/rgw/ {print $3":80"}')
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
# pass the valid rgw-endpoint of same ceph cluster
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
@@ -264,7 +264,7 @@ jobs:
with:
name: ${{ github.job }}-${{ matrix.ceph-image }}
- raw-disk:
+ raw-disk-with-object:
runs-on: ubuntu-20.04
if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
strategy:
@@ -310,7 +310,13 @@ jobs:
run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
- name: wait for ceph to be ready
- run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
+ run: |
+ tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
+
+ - name: wait for object stores to be ready
+ run: |
+ tests/scripts/validate_cluster.sh rgw store-a
+ tests/scripts/validate_cluster.sh rgw store-b
- name: test toolbox-operator-image pod
run: |
@@ -1018,7 +1024,7 @@ jobs:
- name: wait for ceph to be ready
run: |
tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
- tests/scripts/validate_cluster.sh rgw
+ tests/scripts/validate_cluster.sh rgw my-store
kubectl -n rook-ceph get pods
kubectl -n rook-ceph get secrets
@@ -1039,7 +1045,7 @@ jobs:
echo "wait for rgw pod to be deleted"
kubectl wait --for=delete pod -l app=rook-ceph-rgw -n rook-ceph --timeout=100s
kubectl create -f tests/manifests/test-object.yaml
- tests/scripts/validate_cluster.sh rgw
+ tests/scripts/validate_cluster.sh rgw my-store
tests/scripts/deploy-validate-vault.sh validate_rgw
- name: collect common logs
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index f3795b822..62ba4f53e 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -1872,6 +1872,20 @@ PoolSpec
+
customEndpoints
[]string
diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
index 41b120fcb..409913909 100644
--- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
+++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
@@ -10,17 +10,21 @@ This guide assumes a Rook cluster as explained in the [Quickstart](../../Getting
## Configure an Object Store
-Rook has the ability to either deploy an object store in Kubernetes or to connect to an external RGW service.
-Most commonly, the object store will be configured locally by Rook.
-Alternatively, if you have an existing Ceph cluster with Rados Gateways, see the
-[external section](#connect-to-an-external-object-store) to consume it from Rook.
+Rook can configure the Ceph Object Store for several different scenarios. See each linked section for the configuration details.
+1. Create a [local object store](#create-a-local-object-store) with dedicated Ceph pools. This option is recommended if a single object store is required, and is the simplest to get started.
+2. Create [one or more object stores with shared Ceph pools](#create-local-object-stores-with-shared-pools). This option is recommended when multiple object stores are required.
+3. Connect to an [RGW service in an external Ceph cluster](#connect-to-an-external-object-store), rather than create a local object store.
+4. Configure [RGW Multisite](#object-multisite) to synchronize buckets between object stores in different clusters.
+
+!!! note
+ Updating the configuration of an object store between these types is not supported.
### Create a Local Object Store
The below sample will create a `CephObjectStore` that starts the RGW service in the cluster with an S3 API.
!!! note
- This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*.
+ This sample requires *at least 3 OSDs*, with each OSD located on a *different node*.
The OSDs must be located on different nodes, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`).
@@ -39,6 +43,7 @@ spec:
size: 3
dataPool:
failureDomain: host
+ # For production it is recommended to use more chunks, such as 4+2 or 8+4
erasureCoded:
dataChunks: 2
codingChunks: 1
@@ -64,6 +69,120 @@ To confirm the object store is configured, wait for the RGW pod(s) to start:
kubectl -n rook-ceph get pod -l app=rook-ceph-rgw
```
+To consume the object store, continue below in the section to [Create a bucket](#create-a-bucket).
+
+### Create Local Object Store(s) with Shared Pools
+
+The below sample will create one or more object stores. Shared Ceph pools will be created, which reduces the overhead of additional Ceph pools for each additional object store.
+
+Data isolation is enforced between the object stores with the use of Ceph RADOS namespaces. The separate RADOS namespaces do not allow access of the data across object stores.
+
+!!! note
+ This sample requires *at least 3 OSDs*, with each OSD located on a *different node*.
+
+The OSDs must be located on different nodes, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`).
+
+#### Shared Pools
+
+Create the shared pools that will be used by each of the object stores.
+
+!!! note
+ If object stores have been previously created, the first pool below (`.rgw.root`)
+ does not need to be defined again as it would have already been created
+ with an existing object store. There is only one `.rgw.root` pool existing
+ to store metadata for all object stores.
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-root
+ namespace: rook-ceph # namespace:cluster
+spec:
+ name: .rgw.root
+ failureDomain: host
+ replicated:
+ size: 3
+ requireSafeReplicaSize: false
+ parameters:
+ pg_num: "8"
+ application: rgw
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-meta-pool
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: host
+ replicated:
+ size: 3
+ requireSafeReplicaSize: false
+ parameters:
+ pg_num: "8"
+ application: rgw
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-data-pool
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: osd
+ erasureCoded:
+ # For production it is recommended to use more chunks, such as 4+2 or 8+4
+ dataChunks: 2
+ codingChunks: 1
+ application: rgw
+```
+
+Create the shared pools:
+
+```console
+kubectl create -f object-shared-pools.yaml
+```
+
+#### Create Each Object Store
+
+After the pools have been created above, create each object store to consume the shared pools.
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+ name: store-a
+ namespace: rook-ceph # namespace:cluster
+spec:
+ sharedPools:
+ metadataPoolName: rgw-meta-pool
+ dataPoolName: rgw-data-pool
+ preserveRadosNamespaceDataOnDelete: true
+ gateway:
+ sslCertificateRef:
+ port: 80
+ instances: 1
+```
+
+Create the object store:
+
+```console
+kubectl create -f object-a.yaml
+```
+
+To confirm the object store is configured, wait for the RGW pod(s) to start:
+
+```console
+kubectl -n rook-ceph get pod -l rgw=store-a
+```
+
+Additional object stores can be created based on the same shared pools by simply changing the
+`name` of the CephObjectStore. In the example manifests folder, two object store examples are
+provided: `object-a.yaml` and `object-b.yaml`.
+
+To consume the object store, continue below in the section to [Create a bucket](#create-a-bucket).
+Modify the default example object store name from `my-store` to the alternate name of the object store
+such as `store-a` in this example.
+
### Connect to an External Object Store
Rook can connect to existing RGW gateways to work in conjunction with the external mode of the `CephCluster` CRD. First, create a `rgw-admin-ops-user` user in the Ceph cluster with the necessary caps:
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index 8379ded52..838182e0c 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -4,13 +4,15 @@
- The removal of `CSI_ENABLE_READ_AFFINITY` option and its replacement with per-cluster
read affinity setting in cephCluster CR (CSIDriverOptions section) in [PR](https://github.com/rook/rook/pull/13665)
-- Allow setting the Ceph `application` on a pool
- updating `netNamespaceFilePath` for all clusterIDs in rook-ceph-csi-config configMap in [PR](https://github.com/rook/rook/pull/13613)
- Issue: The netNamespaceFilePath isn't updated in the CSI config map for all the clusterIDs when `CSI_ENABLE_HOST_NETWORK` is set to false in `operator.yaml`
- Impact: This results in the unintended network configurations, with pods using the host networking instead of pod networking.
+
## Features
- Kubernetes versions **v1.25** through **v1.29** are supported.
- Ceph daemon pods using the `default` service account now use a new `rook-ceph-default` service account.
+- Allow setting the Ceph `application` on a pool
+- Create object stores with shared metadata and data pools. Isolation between object stores is enabled via RADOS namespaces.
- The feature support for VolumeSnapshotGroup has been added to the RBD and CephFS CSI driver.
- Support for virtual style hosting for s3 buckets in the CephObjectStore.
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index 77336f3d1..a98c6d201 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -10105,6 +10105,29 @@ spec:
type: string
type: object
type: object
+ sharedPools:
+ description: The pool information when configuring RADOS namespaces in existing pools.
+ nullable: true
+ properties:
+ dataPoolName:
+ description: The data pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared data pool is immutable
+ rule: self == oldSelf
+ metadataPoolName:
+ description: The metadata pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared metadata pool is immutable
+ rule: self == oldSelf
+ preserveRadosNamespaceDataOnDelete:
+ description: Whether the RADOS namespaces should be preserved on deletion of the object store
+ type: boolean
+ required:
+ - dataPoolName
+ - metadataPoolName
+ type: object
zone:
description: The multisite info
nullable: true
@@ -10870,6 +10893,29 @@ spec:
default: true
description: Preserve pools on object zone deletion
type: boolean
+ sharedPools:
+ description: The pool information when configuring RADOS namespaces in existing pools.
+ nullable: true
+ properties:
+ dataPoolName:
+ description: The data pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared data pool is immutable
+ rule: self == oldSelf
+ metadataPoolName:
+ description: The metadata pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared metadata pool is immutable
+ rule: self == oldSelf
+ preserveRadosNamespaceDataOnDelete:
+ description: Whether the RADOS namespaces should be preserved on deletion of the object store
+ type: boolean
+ required:
+ - dataPoolName
+ - metadataPoolName
+ type: object
zoneGroup:
description: The display name for the ceph users
type: string
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 6889c429e..fadd17220 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -10096,6 +10096,29 @@ spec:
type: string
type: object
type: object
+ sharedPools:
+ description: The pool information when configuring RADOS namespaces in existing pools.
+ nullable: true
+ properties:
+ dataPoolName:
+ description: The data pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared data pool is immutable
+ rule: self == oldSelf
+ metadataPoolName:
+ description: The metadata pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared metadata pool is immutable
+ rule: self == oldSelf
+ preserveRadosNamespaceDataOnDelete:
+ description: Whether the RADOS namespaces should be preserved on deletion of the object store
+ type: boolean
+ required:
+ - dataPoolName
+ - metadataPoolName
+ type: object
zone:
description: The multisite info
nullable: true
@@ -10858,6 +10881,29 @@ spec:
default: true
description: Preserve pools on object zone deletion
type: boolean
+ sharedPools:
+ description: The pool information when configuring RADOS namespaces in existing pools.
+ nullable: true
+ properties:
+ dataPoolName:
+ description: The data pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared data pool is immutable
+ rule: self == oldSelf
+ metadataPoolName:
+ description: The metadata pool used for creating RADOS namespaces in the object store
+ type: string
+ x-kubernetes-validations:
+ - message: object store shared metadata pool is immutable
+ rule: self == oldSelf
+ preserveRadosNamespaceDataOnDelete:
+ description: Whether the RADOS namespaces should be preserved on deletion of the object store
+ type: boolean
+ required:
+ - dataPoolName
+ - metadataPoolName
+ type: object
zoneGroup:
description: The display name for the ceph users
type: string
diff --git a/deploy/examples/object-a.yaml b/deploy/examples/object-a.yaml
new file mode 100644
index 000000000..aea27bec8
--- /dev/null
+++ b/deploy/examples/object-a.yaml
@@ -0,0 +1,78 @@
+#################################################################################################################
+# Create an object store with settings for shared pools in a production environment. A minimum of 3 hosts with
+# OSDs are required in this example. This example shows two object stores being created with the same
+# shared metadata and data pools. The pool sharing will utilize RADOS namespaces to keep the object store
+# data independent, while avoiding the growth of PGs in the cluster.
+# kubectl create -f object-shared-pools.yaml
+# kubectl create -f object-a.yaml -f object-b.yaml
+#################################################################################################################
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+ name: store-a
+ namespace: rook-ceph # namespace:cluster
+spec:
+ # Shared pools must be defined separately from the object store.
+ # For this example, the pools are defined in object-shared-pools.yaml.
+ # Multiple object stores can be created to share these pools.
+ sharedPools:
+ metadataPoolName: rgw-meta-pool
+ dataPoolName: rgw-data-pool
+ preserveRadosNamespaceDataOnDelete: true
+ # The gateway service configuration
+ gateway:
+ # A reference to the secret in the rook namespace where the ssl certificate is stored
+ # sslCertificateRef:
+ # A reference to the secret in the rook namespace where the ca bundle is stored
+ # caBundleRef:
+ # The port that RGW pods will listen on (http)
+ port: 80
+ # The port that RGW pods will listen on (https). An ssl certificate is required.
+ # securePort: 443
+ # The number of pods in the rgw deployment
+ instances: 1
+ # The affinity rules to apply to the rgw deployment.
+ placement:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-rgw
+ # topologyKey: */zone can be used to spread RGW across different AZ
+ # Use in k8s cluster if your cluster is v1.16 or lower
+ # Use in k8s cluster is v1.17 or upper
+ topologyKey: kubernetes.io/hostname
+ # A key/value list of annotations
+ # nodeAffinity:
+ # requiredDuringSchedulingIgnoredDuringExecution:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: role
+ # operator: In
+ # values:
+ # - rgw-node
+ # topologySpreadConstraints:
+ # tolerations:
+ # - key: rgw-node
+ # operator: Exists
+ # podAffinity:
+ # podAntiAffinity:
+ # A key/value list of annotations
+ annotations:
+ # key: value
+ # A key/value list of labels
+ labels:
+ # key: value
+ resources:
+ # The requests and limits set here, allow the object store gateway Pod(s) to use half of one CPU core and 1 gigabyte of memory
+ # limits:
+ # memory: "1024Mi"
+ # requests:
+ # cpu: "500m"
+ # memory: "1024Mi"
+ priorityClassName: system-cluster-critical
diff --git a/deploy/examples/object-b.yaml b/deploy/examples/object-b.yaml
new file mode 100644
index 000000000..be0d83a14
--- /dev/null
+++ b/deploy/examples/object-b.yaml
@@ -0,0 +1,78 @@
+#################################################################################################################
+# Create an object store with settings for shared pools in a production environment. A minimum of 3 hosts with
+# OSDs are required in this example. This example shows two object stores being created with the same
+# shared metadata and data pools. The pool sharing will utilize RADOS namespaces to keep the object store
+# data independent, while avoiding the growth of PGs in the cluster.
+# kubectl create -f object-shared-pools.yaml
+# kubectl create -f object-a.yaml -f object-b.yaml
+#################################################################################################################
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+ name: store-b
+ namespace: rook-ceph # namespace:cluster
+spec:
+ # Shared pools must be defined separately from the object store.
+ # For this example, the pools are defined in object-shared-pools.yaml.
+ # Multiple object stores can be created to share these pools.
+ sharedPools:
+ metadataPoolName: rgw-meta-pool
+ dataPoolName: rgw-data-pool
+ preserveRadosNamespaceDataOnDelete: true
+ # The gateway service configuration
+ gateway:
+ # A reference to the secret in the rook namespace where the ssl certificate is stored
+ # sslCertificateRef:
+ # A reference to the secret in the rook namespace where the ca bundle is stored
+ # caBundleRef:
+ # The port that RGW pods will listen on (http)
+ port: 80
+ # The port that RGW pods will listen on (https). An ssl certificate is required.
+ # securePort: 443
+ # The number of pods in the rgw deployment
+ instances: 1
+ # The affinity rules to apply to the rgw deployment.
+ placement:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - rook-ceph-rgw
+ # topologyKey: */zone can be used to spread RGW across different AZ
+ # Use in k8s cluster if your cluster is v1.16 or lower
+ # Use in k8s cluster is v1.17 or upper
+ topologyKey: kubernetes.io/hostname
+ # A key/value list of annotations
+ # nodeAffinity:
+ # requiredDuringSchedulingIgnoredDuringExecution:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: role
+ # operator: In
+ # values:
+ # - rgw-node
+ # topologySpreadConstraints:
+ # tolerations:
+ # - key: rgw-node
+ # operator: Exists
+ # podAffinity:
+ # podAntiAffinity:
+ # A key/value list of annotations
+ annotations:
+ # key: value
+ # A key/value list of labels
+ labels:
+ # key: value
+ resources:
+ # The requests and limits set here, allow the object store gateway Pod(s) to use half of one CPU core and 1 gigabyte of memory
+ # limits:
+ # memory: "1024Mi"
+ # requests:
+ # cpu: "500m"
+ # memory: "1024Mi"
+ priorityClassName: system-cluster-critical
diff --git a/deploy/examples/object-bucket-claim-a.yaml b/deploy/examples/object-bucket-claim-a.yaml
new file mode 100644
index 000000000..59f0a95d3
--- /dev/null
+++ b/deploy/examples/object-bucket-claim-a.yaml
@@ -0,0 +1,7 @@
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+ name: ceph-bucket-a
+spec:
+ generateBucketName: ceph-bkt
+ storageClassName: rook-ceph-bucket-a
diff --git a/deploy/examples/object-shared-pools-test.yaml b/deploy/examples/object-shared-pools-test.yaml
new file mode 100644
index 000000000..4e802d892
--- /dev/null
+++ b/deploy/examples/object-shared-pools-test.yaml
@@ -0,0 +1,48 @@
+#################################################################################################################
+# Create the pools that can be shared by multiple object stores. A single OSD is required
+# in this example. This example shows two object stores being created with the same
+# shared metadata and data pools. The pool sharing will utilize RADOS namespaces to keep the object store
+# data independent, while avoiding the growth of PGs in the cluster.
+# kubectl create -f object-shared-pools.yaml
+# kubectl create -f object-a.yaml -f object-b.yaml
+#################################################################################################################
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-root
+ namespace: rook-ceph # namespace:cluster
+spec:
+ name: .rgw.root
+ failureDomain: host
+ replicated:
+ size: 1
+ requireSafeReplicaSize: false
+ parameters:
+ pg_num: "8"
+ application: rgw
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-meta-pool
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: host
+ replicated:
+ size: 1
+ requireSafeReplicaSize: false
+ parameters:
+ pg_num: "8"
+ application: rgw
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-data-pool
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: osd
+ replicated:
+ size: 1
+ requireSafeReplicaSize: false
+ application: rgw
diff --git a/deploy/examples/object-shared-pools.yaml b/deploy/examples/object-shared-pools.yaml
new file mode 100644
index 000000000..0ca505035
--- /dev/null
+++ b/deploy/examples/object-shared-pools.yaml
@@ -0,0 +1,49 @@
+#################################################################################################################
+# Create the pools that can be shared by multiple object stores. A minimum of 3 hosts with
+# OSDs are required in this example. This example shows two object stores being created with the same
+# shared metadata and data pools. The pool sharing will utilize RADOS namespaces to keep the object store
+# data independent, while avoiding the growth of PGs in the cluster.
+# kubectl create -f object-shared-pools.yaml
+# kubectl create -f object-a.yaml -f object-b.yaml
+#################################################################################################################
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-root
+ namespace: rook-ceph # namespace:cluster
+spec:
+ name: .rgw.root
+ failureDomain: host
+ replicated:
+ size: 3
+ requireSafeReplicaSize: false
+ parameters:
+ pg_num: "8"
+ application: rgw
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-meta-pool
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: host
+ replicated:
+ size: 3
+ requireSafeReplicaSize: false
+ parameters:
+ pg_num: "8"
+ application: rgw
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: rgw-data-pool
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: osd
+ erasureCoded:
+ # For production it is recommended to use more chunks, such as 4+2 or 8+4
+ dataChunks: 2
+ codingChunks: 1
+ application: rgw
diff --git a/deploy/examples/storageclass-bucket-a.yaml b/deploy/examples/storageclass-bucket-a.yaml
new file mode 100644
index 000000000..8654da650
--- /dev/null
+++ b/deploy/examples/storageclass-bucket-a.yaml
@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: rook-ceph-bucket-a
+provisioner: rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster
+reclaimPolicy: Delete
+parameters:
+ objectStoreName: store-a
+ objectStoreNamespace: rook-ceph # namespace:cluster
diff --git a/design/ceph/object/store.md b/design/ceph/object/store.md
index 937f5a236..fa70f041d 100644
--- a/design/ceph/object/store.md
+++ b/design/ceph/object/store.md
@@ -82,11 +82,11 @@ spec:
#### Pools shared by multiple CephObjectStore
-If user want to use existing pools for metadata and data, the pools must be created before the object store is created. This will be useful if multiple objectstore can share same pools. The detail of pools need to shared in `radosNamespaces` settings in object-store CRD. Now the object stores can consume same pool isolated with different namespaces. Usually RGW server itself create different [namespaces](https://docs.ceph.com/en/latest/radosgw/layout/#appendix-compendium) on the pools. User can create via [Pool CRD](/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md), this is need to present before the object store is created. Similar to `preservePoolsOnDelete` setting, `preserveRadosNamespaceDataOnDelete` is used to preserve the data in the rados namespace when the object store is deleted. It is set to 'false' by default.
+If user want to use existing pools for metadata and data, the pools must be created before the object store is created. This will be useful if multiple objectstore can share same pools. The detail of pools need to shared in `sharedPools` settings in object-store CRD. Now the object stores can consume same pool isolated with different namespaces. Usually RGW server itself create different [namespaces](https://docs.ceph.com/en/latest/radosgw/layout/#appendix-compendium) on the pools. User can create via [Pool CRD](/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md), this is need to present before the object store is created. Similar to `preservePoolsOnDelete` setting, `preserveRadosNamespaceDataOnDelete` is used to preserve the data in the rados namespace when the object store is deleted. It is set to 'false' by default.
```yaml
spec:
- radosNamespaces:
+ sharedPools:
metadataPoolName: rgw-meta-pool
dataPoolName: rgw-data-pool
preserveRadosNamespaceDataOnDelete: true
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index d0c661d5a..6a954e41f 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -1431,6 +1431,11 @@ type ObjectStoreSpec struct {
// +nullable
DataPool PoolSpec `json:"dataPool,omitempty"`
+ // The pool information when configuring RADOS namespaces in existing pools.
+ // +optional
+ // +nullable
+ SharedPools ObjectSharedPoolsSpec `json:"sharedPools"`
+
// Preserve pools on object store deletion
// +optional
PreservePoolsOnDelete bool `json:"preservePoolsOnDelete,omitempty"`
@@ -1469,6 +1474,21 @@ type ObjectStoreSpec struct {
Hosting *ObjectStoreHostingSpec `json:"hosting,omitempty"`
}
+// ObjectSharedPoolsSpec represents object store pool info when configuring RADOS namespaces in existing pools.
+type ObjectSharedPoolsSpec struct {
+ // The metadata pool used for creating RADOS namespaces in the object store
+ // +kubebuilder:validation:XValidation:message="object store shared metadata pool is immutable",rule="self == oldSelf"
+ MetadataPoolName string `json:"metadataPoolName"`
+
+ // The data pool used for creating RADOS namespaces in the object store
+ // +kubebuilder:validation:XValidation:message="object store shared data pool is immutable",rule="self == oldSelf"
+ DataPoolName string `json:"dataPoolName"`
+
+ // Whether the RADOS namespaces should be preserved on deletion of the object store
+ // +optional
+ PreserveRadosNamespaceDataOnDelete bool `json:"preserveRadosNamespaceDataOnDelete"`
+}
+
// ObjectHealthCheckSpec represents the health check of an object store
type ObjectHealthCheckSpec struct {
// livenessProbe field is no longer used
@@ -1882,6 +1902,11 @@ type ObjectZoneSpec struct {
// +nullable
DataPool PoolSpec `json:"dataPool"`
+ // The pool information when configuring RADOS namespaces in existing pools.
+ // +optional
+ // +nullable
+ SharedPools ObjectSharedPoolsSpec `json:"sharedPools"`
+
// If this zone cannot be accessed from other peer Ceph clusters via the ClusterIP Service
// endpoint created by Rook, you must set this to the externally reachable endpoint(s). You may
// include the port in the definition. For example: "https://my-object-store.my-domain.net:443".
diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go
index a5b8333f0..2d2f3214a 100644
--- a/pkg/operator/ceph/object/controller.go
+++ b/pkg/operator/ceph/object/controller.go
@@ -249,7 +249,8 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
}
err = r.deleteCOSIUser(opsCtx)
if err != nil {
- return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to delete COSI user")
+ // Allow the object store removal to proceed even if the user deletion fails
+ logger.Warningf("failed to delete COSI user. %v", err)
}
deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx)
if err != nil {
@@ -411,14 +412,14 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
logger.Info("reconciling object store deployments")
// Reconcile realm/zonegroup/zone CRs & update their names
- realmName, zoneGroupName, zoneName, zone, reconcileResponse, err := r.reconcileMultisiteCRs(cephObjectStore)
+ realmName, zoneGroupName, zoneName, zone, reconcileResponse, err := r.getMultisiteResourceNames(cephObjectStore)
if err != nil {
return reconcileResponse, err
}
- // Reconcile Ceph Zone if Multisite
+ // Reconcile Ceph Zone if Multisite to ensure it exists, or else requeue the request
if cephObjectStore.Spec.IsMultisite() {
- reconcileResponse, err := r.reconcileCephZone(cephObjectStore, zoneGroupName, realmName)
+ reconcileResponse, err := r.retrieveMultisiteZone(cephObjectStore, zoneGroupName, realmName)
if err != nil {
return reconcileResponse, err
}
@@ -427,7 +428,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
objContext.Realm = realmName
objContext.ZoneGroup = zoneGroupName
objContext.Zone = zoneName
- logger.Debugf("realm for object-store is %q, zone group for object-store is %q, zone for object-store is %q", objContext.Realm, objContext.ZoneGroup, objContext.Zone)
+ logger.Debugf("realm is %q, zone group is %q, zone is %q, for object store %q", objContext.Realm, objContext.ZoneGroup, objContext.Zone, cephObjectStore.Name)
// RECONCILE SERVICE
logger.Debug("reconciling object store service")
@@ -443,15 +444,15 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
// Reconcile Pool Creation
if !cephObjectStore.Spec.IsMultisite() {
logger.Info("reconciling object store pools")
- err = CreatePools(objContext, r.clusterSpec, cephObjectStore.Spec.MetadataPool, cephObjectStore.Spec.DataPool)
+ err = ConfigurePools(objContext, r.clusterSpec, cephObjectStore.Spec.MetadataPool, cephObjectStore.Spec.DataPool, cephObjectStore.Spec.SharedPools)
if err != nil {
return r.setFailedStatus(k8sutil.ObservedGenerationNotAvailable, namespacedName, "failed to create object pools", err)
}
}
- // Reconcile Multisite Creation
- logger.Infof("setting multisite settings for object store %q", cephObjectStore.Name)
- err = setMultisite(objContext, cephObjectStore, zone)
+ // Reconcile the object store
+ logger.Infof("configuring object store %q", cephObjectStore.Name)
+ err = configureObjectStore(objContext, cephObjectStore, zone)
if err != nil && kerrors.IsNotFound(err) {
return reconcile.Result{}, err
} else if err != nil {
@@ -470,7 +471,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
return r.reconcileCOSIUser(cephObjectStore)
}
-func (r *ReconcileCephObjectStore) reconcileCephZone(store *cephv1.CephObjectStore, zoneGroupName string, realmName string) (reconcile.Result, error) {
+func (r *ReconcileCephObjectStore) retrieveMultisiteZone(store *cephv1.CephObjectStore, zoneGroupName string, realmName string) (reconcile.Result, error) {
realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zoneGroupName)
zoneArg := fmt.Sprintf("--rgw-zone=%s", store.Spec.Zone.Name)
@@ -490,43 +491,43 @@ func (r *ReconcileCephObjectStore) reconcileCephZone(store *cephv1.CephObjectSto
return reconcile.Result{}, nil
}
-func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1.CephObjectStore) (string, string, string, *cephv1.CephObjectZone, reconcile.Result, error) {
- if cephObjectStore.Spec.IsMultisite() {
- zoneName := cephObjectStore.Spec.Zone.Name
- zone := &cephv1.CephObjectZone{}
- err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: zoneName, Namespace: cephObjectStore.Namespace}, zone)
- if err != nil {
- if kerrors.IsNotFound(err) {
- return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, err
- }
- return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZone %q", cephObjectStore.Spec.Zone.Name)
- }
- logger.Debugf("CephObjectZone resource %s found", zone.Name)
-
- zonegroup := &cephv1.CephObjectZoneGroup{}
- err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: zone.Spec.ZoneGroup, Namespace: cephObjectStore.Namespace}, zonegroup)
- if err != nil {
- if kerrors.IsNotFound(err) {
- return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, err
- }
- return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZoneGroup %q", zone.Spec.ZoneGroup)
- }
- logger.Debugf("CephObjectZoneGroup resource %s found", zonegroup.Name)
-
- realm := &cephv1.CephObjectRealm{}
- err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: zonegroup.Spec.Realm, Namespace: cephObjectStore.Namespace}, realm)
- if err != nil {
- if kerrors.IsNotFound(err) {
- return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, err
- }
- return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectRealm %q", zonegroup.Spec.Realm)
- }
- logger.Debugf("CephObjectRealm resource %s found", realm.Name)
-
- return realm.Name, zonegroup.Name, zone.Name, zone, reconcile.Result{}, nil
+func (r *ReconcileCephObjectStore) getMultisiteResourceNames(cephObjectStore *cephv1.CephObjectStore) (string, string, string, *cephv1.CephObjectZone, reconcile.Result, error) {
+ if !cephObjectStore.Spec.IsMultisite() {
+ return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil
}
- return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, nil, reconcile.Result{}, nil
+ zoneName := cephObjectStore.Spec.Zone.Name
+ zone := &cephv1.CephObjectZone{}
+ err := r.client.Get(r.opManagerContext, types.NamespacedName{Name: zoneName, Namespace: cephObjectStore.Namespace}, zone)
+ if err != nil {
+ if kerrors.IsNotFound(err) {
+ return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, err
+ }
+ return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZone %q", cephObjectStore.Spec.Zone.Name)
+ }
+ logger.Debugf("CephObjectZone resource %s found", zone.Name)
+
+ zonegroup := &cephv1.CephObjectZoneGroup{}
+ err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: zone.Spec.ZoneGroup, Namespace: cephObjectStore.Namespace}, zonegroup)
+ if err != nil {
+ if kerrors.IsNotFound(err) {
+ return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, err
+ }
+ return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZoneGroup %q", zone.Spec.ZoneGroup)
+ }
+ logger.Debugf("CephObjectZoneGroup resource %s found", zonegroup.Name)
+
+ realm := &cephv1.CephObjectRealm{}
+ err = r.client.Get(r.opManagerContext, types.NamespacedName{Name: zonegroup.Spec.Realm, Namespace: cephObjectStore.Namespace}, realm)
+ if err != nil {
+ if kerrors.IsNotFound(err) {
+ return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, err
+ }
+ return "", "", "", nil, waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectRealm %q", zonegroup.Spec.Realm)
+ }
+ logger.Debugf("CephObjectRealm resource %s found", realm.Name)
+
+ return realm.Name, zonegroup.Name, zone.Name, zone, reconcile.Result{}, nil
}
func (r *ReconcileCephObjectStore) reconcileCOSIUser(cephObjectStore *cephv1.CephObjectStore) (reconcile.Result, error) {
diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go
index 2781d7163..722313e12 100644
--- a/pkg/operator/ceph/object/objectstore.go
+++ b/pkg/operator/ceph/object/objectstore.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"os"
+ "path"
"sort"
"strconv"
"strings"
@@ -428,7 +429,7 @@ func createMultisiteConfigurations(objContext *Context, configType, configTypeAr
return nil
}
-func createMultisite(objContext *Context, endpointArg string) error {
+func createNonMultisiteStore(objContext *Context, endpointArg string, store *cephv1.CephObjectStore) error {
logger.Debugf("creating realm, zone group, zone for object-store %v", objContext.Name)
realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
@@ -450,13 +451,19 @@ func createMultisite(objContext *Context, endpointArg string) error {
return err
}
+ logger.Infof("Object store %q: realm=%s, zonegroup=%s, zone=%s", objContext.Name, objContext.Realm, objContext.ZoneGroup, objContext.Zone)
+
+ // Configure the zone for RADOS namespaces
+ err = ConfigureSharedPoolsForZone(objContext, store.Spec.SharedPools)
+ if err != nil {
+ return errors.Wrapf(err, "failed to configure rados namespaces for zone")
+ }
+
if err := commitConfigChanges(objContext); err != nil {
nsName := fmt.Sprintf("%s/%s", objContext.clusterInfo.Namespace, objContext.Name)
return errors.Wrapf(err, "failed to commit config changes after creating multisite config for CephObjectStore %q", nsName)
}
- logger.Infof("Multisite for object-store: realm=%s, zonegroup=%s, zone=%s", objContext.Realm, objContext.ZoneGroup, objContext.Zone)
-
return nil
}
@@ -544,7 +551,7 @@ func createSystemUser(objContext *Context, namespace string) error {
return nil
}
-func setMultisite(objContext *Context, store *cephv1.CephObjectStore, zone *cephv1.CephObjectZone) error {
+func configureObjectStore(objContext *Context, store *cephv1.CephObjectStore, zone *cephv1.CephObjectZone) error {
logger.Debugf("setting multisite configuration for object-store %v", store.Name)
if store.Spec.IsMultisite() {
@@ -578,13 +585,13 @@ func setMultisite(objContext *Context, store *cephv1.CephObjectStore, zone *ceph
if !store.Spec.Gateway.DisableMultisiteSyncTraffic {
endpointArg = fmt.Sprintf("--endpoints=%s", objContext.Endpoint)
}
- err := createMultisite(objContext, endpointArg)
+ err := createNonMultisiteStore(objContext, endpointArg, store)
if err != nil {
return errorOrIsNotFound(err, "failed create ceph multisite for object-store %q", objContext.Name)
}
}
- logger.Infof("multisite configuration for object-store %v is complete", store.Name)
+ logger.Infof("configuration for object-store %v is complete", store.Name)
return nil
}
@@ -716,6 +723,7 @@ func allObjectPools(storeName string) []string {
return poolsForThisStore
}
+// Detect if there are pools that do not exist for this object store
func missingPools(context *Context) ([]string, error) {
// list pools instead of querying each pool individually. querying each individually makes it
// hard to determine if an error is because the pool does not exist or because of a connection
@@ -740,7 +748,15 @@ func missingPools(context *Context) ([]string, error) {
return missingPools, nil
}
-func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool, dataPool cephv1.PoolSpec) error {
+func ConfigurePools(context *Context, cluster *cephv1.ClusterSpec, metadataPool, dataPool cephv1.PoolSpec, sharedPools cephv1.ObjectSharedPoolsSpec) error {
+ if sharedPoolsSpecified(sharedPools) {
+ if !EmptyPool(dataPool) || !EmptyPool(metadataPool) {
+ return fmt.Errorf("object store shared pools can only be specified if the metadata and data pools are not specified")
+ }
+ // Shared pools are configured elsewhere
+ return nil
+ }
+
if EmptyPool(dataPool) && EmptyPool(metadataPool) {
logger.Info("no pools specified for the CR, checking for their existence...")
missingPools, err := missingPools(context)
@@ -765,17 +781,230 @@ func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool
metadataPoolPGs = cephclient.DefaultPGCount
}
- if err := createSimilarPools(context, append(metadataPools, rootPool), clusterSpec, metadataPool, metadataPoolPGs); err != nil {
+ if err := createSimilarPools(context, append(metadataPools, rootPool), cluster, metadataPool, metadataPoolPGs); err != nil {
return errors.Wrap(err, "failed to create metadata pools")
}
- if err := createSimilarPools(context, []string{dataPoolName}, clusterSpec, dataPool, cephclient.DefaultPGCount); err != nil {
+ if err := createSimilarPools(context, []string{dataPoolName}, cluster, dataPool, cephclient.DefaultPGCount); err != nil {
return errors.Wrap(err, "failed to create data pool")
}
return nil
}
+func sharedPoolsSpecified(sharedPools cephv1.ObjectSharedPoolsSpec) bool {
+ return sharedPools.DataPoolName != "" && sharedPools.MetadataPoolName != ""
+}
+
+func ConfigureSharedPoolsForZone(objContext *Context, sharedPools cephv1.ObjectSharedPoolsSpec) error {
+ if !sharedPoolsSpecified(sharedPools) {
+ logger.Debugf("no shared pools to configure for store %q", objContext.Name)
+ return nil
+ }
+
+ if err := sharedPoolsExist(objContext, sharedPools); err != nil {
+ return errors.Wrapf(err, "object store cannot be configured until shared pools exist")
+ }
+
+ // retrieve the zone config
+ logger.Infof("Retrieving zone %q", objContext.Zone)
+ realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+ zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+ zoneArg := "--rgw-zone=" + objContext.Zone
+ args := []string{"zone", "get", realmArg, zoneGroupArg, zoneArg}
+
+ output, err := RunAdminCommandNoMultisite(objContext, true, args...)
+ if err != nil {
+ return errors.Wrap(err, "failed to get zone")
+ }
+
+ logger.Debugf("Zone config is currently:\n%s", output)
+
+ var zoneConfig map[string]interface{}
+ err = json.Unmarshal([]byte(output), &zoneConfig)
+ if err != nil {
+ return errors.Wrap(err, "failed to unmarshal zone")
+ }
+
+ metadataPrefix := fmt.Sprintf("%s:%s.", sharedPools.MetadataPoolName, objContext.Name)
+ dataPrefix := fmt.Sprintf("%s:%s.", sharedPools.DataPoolName, objContext.Name)
+ expectedDataPool := dataPrefix + "buckets.data"
+ if dataPoolIsExpected(objContext, zoneConfig, expectedDataPool) {
+ logger.Debugf("Data pool already set as expected to %q", expectedDataPool)
+ return nil
+ }
+
+ logger.Infof("Updating rados namespace configuration for zone %q", objContext.Zone)
+ if err := applyExpectedRadosNamespaceSettings(zoneConfig, metadataPrefix, dataPrefix, expectedDataPool); err != nil {
+ return errors.Wrap(err, "failed to configure rados namespaces")
+ }
+
+ configBytes, err := json.Marshal(zoneConfig)
+ if err != nil {
+ return errors.Wrap(err, "failed to serialize zone config")
+ }
+ logger.Debugf("Raw zone settings to apply: %s", string(configBytes))
+
+ configFilename := path.Join(objContext.Context.ConfigDir, objContext.Name+".zonecfg")
+ if err := os.WriteFile(configFilename, configBytes, 0600); err != nil {
+ return errors.Wrap(err, "failed to write zonfig config file")
+ }
+ defer os.Remove(configFilename)
+
+ args = []string{"zone", "set", zoneArg, "--infile=" + configFilename, realmArg, zoneGroupArg}
+ output, err = RunAdminCommandNoMultisite(objContext, false, args...)
+ if err != nil {
+ return errors.Wrap(err, "failed to set zone config")
+ }
+ logger.Debugf("Zone set results=%s", output)
+
+ if err = zoneUpdateWorkaround(objContext, output, expectedDataPool); err != nil {
+ return errors.Wrap(err, "failed to apply zone set workaround")
+ }
+
+ logger.Infof("Successfully configured RADOS namespaces for object store %q", objContext.Name)
+ return nil
+}
+
+func sharedPoolsExist(objContext *Context, sharedPools cephv1.ObjectSharedPoolsSpec) error {
+ existingPools, err := cephclient.ListPoolSummaries(objContext.Context, objContext.clusterInfo)
+ if err != nil {
+ return errors.Wrapf(err, "failed to list pools")
+ }
+ foundMetadataPool := false
+ foundDataPool := false
+ for _, pool := range existingPools {
+ if pool.Name == sharedPools.MetadataPoolName {
+ foundMetadataPool = true
+ }
+ if pool.Name == sharedPools.DataPoolName {
+ foundDataPool = true
+ }
+ }
+
+ if !foundMetadataPool && !foundDataPool {
+ return fmt.Errorf("pools do not exist: %q and %q", sharedPools.MetadataPoolName, sharedPools.DataPoolName)
+ }
+ if !foundMetadataPool {
+ return fmt.Errorf("metadata pool does not exist: %q", sharedPools.MetadataPoolName)
+ }
+ if !foundDataPool {
+ return fmt.Errorf("data pool does not exist: %q", sharedPools.DataPoolName)
+ }
+
+ logger.Info("verified shared pools exist")
+ return nil
+}
+
+func applyExpectedRadosNamespaceSettings(zoneConfig map[string]interface{}, metadataPrefix, dataPrefix, dataPool string) error {
+ // Update the necessary fields for RAODS namespaces
+ zoneConfig["domain_root"] = metadataPrefix + "meta.root"
+ zoneConfig["control_pool"] = metadataPrefix + "control"
+ zoneConfig["gc_pool"] = metadataPrefix + "log.gc"
+ zoneConfig["lc_pool"] = metadataPrefix + "log.lc"
+ zoneConfig["log_pool"] = metadataPrefix + "log"
+ zoneConfig["intent_log_pool"] = metadataPrefix + "log.intent"
+ zoneConfig["usage_log_pool"] = metadataPrefix + "log.usage"
+ zoneConfig["roles_pool"] = metadataPrefix + "meta.roles"
+ zoneConfig["reshard_pool"] = metadataPrefix + "log.reshard"
+ zoneConfig["user_keys_pool"] = metadataPrefix + "meta.users.keys"
+ zoneConfig["user_email_pool"] = metadataPrefix + "meta.users.email"
+ zoneConfig["user_swift_pool"] = metadataPrefix + "meta.users.swift"
+ zoneConfig["user_uid_pool"] = metadataPrefix + "meta.users.uid"
+ zoneConfig["otp_pool"] = metadataPrefix + "otp"
+ zoneConfig["notif_pool"] = metadataPrefix + "log.notif"
+
+ placementPools, ok := zoneConfig["placement_pools"].([]interface{})
+ if !ok {
+ return fmt.Errorf("failed to parse placement_pools")
+ }
+ if len(placementPools) == 0 {
+ return fmt.Errorf("no placement pools")
+ }
+ placementPool, ok := placementPools[0].(map[string]interface{})
+ if !ok {
+ return fmt.Errorf("failed to parse placement_pools[0]")
+ }
+ placementVals, ok := placementPool["val"].(map[string]interface{})
+ if !ok {
+ return fmt.Errorf("failed to parse placement_pools[0].val")
+ }
+ placementVals["index_pool"] = metadataPrefix + "buckets.index"
+ placementVals["data_extra_pool"] = dataPrefix + "buckets.non-ec"
+ storageClasses, ok := placementVals["storage_classes"].(map[string]interface{})
+ if !ok {
+ return fmt.Errorf("failed to parse storage_classes")
+ }
+ stdStorageClass, ok := storageClasses["STANDARD"].(map[string]interface{})
+ if !ok {
+ return fmt.Errorf("failed to parse storage_classes.STANDARD")
+ }
+ stdStorageClass["data_pool"] = dataPool
+ return nil
+}
+
+func dataPoolIsExpected(objContext *Context, zoneConfig map[string]interface{}, expectedDataPool string) bool {
+ placementPools, ok := zoneConfig["placement_pools"].([]interface{})
+ if !ok {
+ return false
+ }
+ placementPool, ok := placementPools[0].(map[string]interface{})
+ if !ok {
+ return false
+ }
+ placementVals, ok := placementPool["val"].(map[string]interface{})
+ if !ok {
+ return false
+ }
+ storageClasses, ok := placementVals["storage_classes"].(map[string]interface{})
+ if !ok {
+ return false
+ }
+ stdStorageClass, ok := storageClasses["STANDARD"].(map[string]interface{})
+ if !ok {
+ return false
+ }
+ logger.Infof("data pool is currently set to %q", stdStorageClass["data_pool"])
+ return stdStorageClass["data_pool"] == expectedDataPool
+}
+
+// There was a radosgw-admin bug that was preventing the RADOS namespace from being applied
+// for the data pool. The fix is included in Reef v18.2.3 or newer, and v19.2.0.
+// The workaround is to run a "radosgw-admin zone placement modify" command to apply
+// the desired data pool config.
+// After Reef (v18) support is removed, this method will be dead code.
+func zoneUpdateWorkaround(objContext *Context, zoneOutput, expectedDataPool string) error {
+ var zoneConfig map[string]interface{}
+ err := json.Unmarshal([]byte(zoneOutput), &zoneConfig)
+ if err != nil {
+ return errors.Wrap(err, "failed to unmarshal zone")
+ }
+ // Update the necessary fields for RAODS namespaces
+ // If the radosgw-admin fix is in the release, the data pool is already applied and we skip the workaround.
+ if dataPoolIsExpected(objContext, zoneConfig, expectedDataPool) {
+ logger.Infof("data pool was already set as expected to %q, workaround not needed", expectedDataPool)
+ return nil
+ }
+
+ logger.Infof("Setting data pool to %q", expectedDataPool)
+ args := []string{"zone", "placement", "modify",
+ "--rgw-realm=" + objContext.Realm,
+ "--rgw-zonegroup=" + objContext.ZoneGroup,
+ "--rgw-zone=" + objContext.Name,
+ "--placement-id", "default-placement",
+ "--storage-class", "STANDARD",
+ "--data-pool=" + expectedDataPool,
+ }
+
+ output, err := RunAdminCommandNoMultisite(objContext, false, args...)
+ if err != nil {
+ return errors.Wrap(err, "failed to set zone config")
+ }
+ logger.Debugf("zone placement modify output=%s", output)
+ logger.Info("zone placement for the data pool was applied successfully")
+ return nil
+}
+
// Check if this is a recent release of ceph where the legacy rgw_rados_pool_pg_num_min
// is no longer available.
func rgwRadosPGNumIsNew(cephVer cephver.CephVersion) bool {
@@ -792,7 +1021,7 @@ func configurePoolsConcurrently() bool {
return true
}
-func createSimilarPools(ctx *Context, pools []string, clusterSpec *cephv1.ClusterSpec, poolSpec cephv1.PoolSpec, pgCount string) error {
+func createSimilarPools(ctx *Context, pools []string, cluster *cephv1.ClusterSpec, poolSpec cephv1.PoolSpec, pgCount string) error {
// We have concurrency
if configurePoolsConcurrently() {
waitGroup, _ := errgroup.WithContext(ctx.clusterInfo.Context)
@@ -800,14 +1029,14 @@ func createSimilarPools(ctx *Context, pools []string, clusterSpec *cephv1.Cluste
// Avoid the loop re-using the same value with a closure
pool := pool
- waitGroup.Go(func() error { return createRGWPool(ctx, clusterSpec, poolSpec, pgCount, pool) })
+ waitGroup.Go(func() error { return createRGWPool(ctx, cluster, poolSpec, pgCount, pool) })
}
return waitGroup.Wait()
}
// No concurrency!
for _, pool := range pools {
- err := createRGWPool(ctx, clusterSpec, poolSpec, pgCount, pool)
+ err := createRGWPool(ctx, cluster, poolSpec, pgCount, pool)
if err != nil {
return err
}
@@ -816,14 +1045,14 @@ func createSimilarPools(ctx *Context, pools []string, clusterSpec *cephv1.Cluste
return nil
}
-func createRGWPool(ctx *Context, clusterSpec *cephv1.ClusterSpec, poolSpec cephv1.PoolSpec, pgCount, requestedName string) error {
+func createRGWPool(ctx *Context, cluster *cephv1.ClusterSpec, poolSpec cephv1.PoolSpec, pgCount, requestedName string) error {
// create the pool if it doesn't exist yet
poolSpec.Application = rgwApplication
pool := cephv1.NamedPoolSpec{
Name: poolName(ctx.Name, requestedName),
PoolSpec: poolSpec,
}
- if err := cephclient.CreatePoolWithPGs(ctx.Context, ctx.clusterInfo, clusterSpec, pool, pgCount); err != nil {
+ if err := cephclient.CreatePoolWithPGs(ctx.Context, ctx.clusterInfo, cluster, pool, pgCount); err != nil {
return errors.Wrapf(err, "failed to create pool %q", pool.Name)
}
// Set the pg_num_min if not the default so the autoscaler won't immediately increase the pg count
diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go
index 625083ed7..96e7d63d5 100644
--- a/pkg/operator/ceph/object/objectstore_test.go
+++ b/pkg/operator/ceph/object/objectstore_test.go
@@ -18,7 +18,9 @@ package object
import (
"context"
+ "encoding/json"
"fmt"
+ "strings"
"syscall"
"testing"
"time"
@@ -70,6 +72,47 @@ const (
"max_objects": -1
}
}`
+ objectZoneJson = `{
+ "id": "c1a20ed9-6370-4abd-b78c-bdf0da2a8dbb",
+ "name": "store-a",
+ "domain_root": "rgw-meta-pool:store-a.meta.root",
+ "control_pool": "rgw-meta-pool:store-a.control",
+ "gc_pool": "rgw-meta-pool:store-a.log.gc",
+ "lc_pool": "rgw-meta-pool:store-a.log.lc",
+ "log_pool": "rgw-meta-pool:store-a.log",
+ "intent_log_pool": "rgw-meta-pool:store-a.log.intent",
+ "usage_log_pool": "rgw-meta-pool:store-a.log.usage",
+ "roles_pool": "rgw-meta-pool:store-a.meta.roles",
+ "reshard_pool": "rgw-meta-pool:store-a.log.reshard",
+ "user_keys_pool": "rgw-meta-pool:store-a.meta.users.keys",
+ "user_email_pool": "rgw-meta-pool:store-a.meta.users.email",
+ "user_swift_pool": "rgw-meta-pool:store-a.meta.users.swift",
+ "user_uid_pool": "rgw-meta-pool:store-a.meta.users.uid",
+ "otp_pool": "rgw-meta-pool:store-a.otp",
+ "system_key": {
+ "access_key": "",
+ "secret_key": ""
+ },
+ "placement_pools": [
+ {
+ "key": "default-placement",
+ "val": {
+ "index_pool": "rgw-meta-pool:store-a.buckets.index",
+ "storage_classes": {
+ "STANDARD": {
+ "data_pool": "rgw-data-pool:store-a.buckets.data"
+ }
+ },
+ "data_extra_pool": "rgw-data-pool:store-a.buckets.non-ec",
+ "index_type": 0,
+ "inline_data": true
+ }
+ }
+ ],
+ "realm_id": "e7f176c6-d207-459c-aa04-c3334300ddc6",
+ "notif_pool": "rgw-meta-pool:store-a.log.notif"
+ }`
+
//#nosec G101 -- The credentials are just for the unit tests
access_key = "VFKF8SSU9L3L2UR03Z8C"
//#nosec G101 -- The credentials are just for the unit tests
@@ -98,14 +141,246 @@ func TestReconcileRealm(t *testing.T) {
objContext := NewContext(context, &client.ClusterInfo{Namespace: "mycluster"}, storeName)
// create the first realm, marked as default
store := cephv1.CephObjectStore{}
- err := setMultisite(objContext, &store, nil)
+ err := configureObjectStore(objContext, &store, nil)
assert.Nil(t, err)
// create the second realm, not marked as default
- err = setMultisite(objContext, &store, nil)
+ err = configureObjectStore(objContext, &store, nil)
assert.Nil(t, err)
}
+func TestApplyExpectedRadosNamespaceSettings(t *testing.T) {
+ dataPoolName := "testdatapool"
+ metaPrefix := "testmeta"
+ dataPrefix := "testdata"
+ var zoneConfig map[string]interface{}
+
+ t.Run("fail when input empty", func(t *testing.T) {
+ input := map[string]interface{}{}
+ err := applyExpectedRadosNamespaceSettings(input, metaPrefix, dataPrefix, dataPoolName)
+ assert.Error(t, err)
+ assert.True(t, strings.Contains(err.Error(), "placement_pools"))
+ })
+ t.Run("valid input", func(t *testing.T) {
+ assert.NoError(t, json.Unmarshal([]byte(objectZoneJson), &zoneConfig))
+ assert.NoError(t, applyExpectedRadosNamespaceSettings(zoneConfig, metaPrefix, dataPrefix, dataPoolName))
+ // validate a sampling of the updated fields
+ assert.Equal(t, metaPrefix+"log.notif", zoneConfig["notif_pool"])
+ placementPools := zoneConfig["placement_pools"].([]interface{})
+ placementPool := placementPools[0].(map[string]interface{})
+ placementVals := placementPool["val"].(map[string]interface{})
+ storageClasses := placementVals["storage_classes"].(map[string]interface{})
+ stdStorageClass := storageClasses["STANDARD"].(map[string]interface{})
+ assert.Equal(t, dataPoolName, stdStorageClass["data_pool"])
+ })
+ t.Run("placement pools empty", func(t *testing.T) {
+ // remove expected sections of the json and confirm that it returns an error without throwing an exception
+ emptyPlacementPoolsJson := `{
+ "otp_pool": "rgw-meta-pool:store-a.otp",
+ "placement_pools": []
+ }`
+ assert.NoError(t, json.Unmarshal([]byte(emptyPlacementPoolsJson), &zoneConfig))
+ err := applyExpectedRadosNamespaceSettings(zoneConfig, metaPrefix, dataPrefix, dataPoolName)
+ assert.Error(t, err)
+ assert.True(t, strings.Contains(err.Error(), "no placement pools"))
+ })
+ t.Run("placement pool value missing", func(t *testing.T) {
+ missingPoolValueJson := `{
+ "otp_pool": "rgw-meta-pool:store-a.otp",
+ "placement_pools": [
+ {
+ "key": "default-placement"
+ }
+ ]
+ }`
+ assert.NoError(t, json.Unmarshal([]byte(missingPoolValueJson), &zoneConfig))
+ err := applyExpectedRadosNamespaceSettings(zoneConfig, metaPrefix, dataPrefix, dataPoolName)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "placement_pools[0].val")
+ })
+ t.Run("storage classes missing", func(t *testing.T) {
+ storageClassesMissing := `{
+ "otp_pool": "rgw-meta-pool:store-a.otp",
+ "placement_pools": [
+ {
+ "key": "default-placement",
+ "val": {
+ "index_pool": "rgw-meta-pool:store-a.buckets.index"
+ }
+ }
+ ]
+ }`
+ assert.NoError(t, json.Unmarshal([]byte(storageClassesMissing), &zoneConfig))
+ err := applyExpectedRadosNamespaceSettings(zoneConfig, metaPrefix, dataPrefix, dataPoolName)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "storage_classes")
+ })
+ t.Run("standard storage class missing", func(t *testing.T) {
+ standardSCMissing := `{
+ "otp_pool": "rgw-meta-pool:store-a.otp",
+ "placement_pools": [
+ {
+ "key": "default-placement",
+ "val": {
+ "index_pool": "rgw-meta-pool:store-a.buckets.index",
+ "storage_classes": {
+ "BAD": {
+ "data_pool": "rgw-data-pool:store-a.buckets.data"
+ }
+ }
+ }
+ }
+ ]
+ }`
+ assert.NoError(t, json.Unmarshal([]byte(standardSCMissing), &zoneConfig))
+ err := applyExpectedRadosNamespaceSettings(zoneConfig, metaPrefix, dataPrefix, dataPoolName)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "storage_classes.STANDARD")
+ })
+ t.Run("no config missing", func(t *testing.T) {
+ nothingMissing := `{
+ "otp_pool": "rgw-meta-pool:store-a.otp",
+ "placement_pools": [
+ {
+ "key": "default-placement",
+ "val": {
+ "index_pool": "rgw-meta-pool:store-a.buckets.index",
+ "storage_classes": {
+ "STANDARD": {
+ "data_pool": "rgw-data-pool:store-a.buckets.data"
+ }
+ }
+ }
+ }
+ ]
+ }`
+ assert.NoError(t, json.Unmarshal([]byte(nothingMissing), &zoneConfig))
+ err := applyExpectedRadosNamespaceSettings(zoneConfig, metaPrefix, dataPrefix, dataPoolName)
+ assert.NoError(t, err)
+ })
+}
+
+func TestSharedPoolsExist(t *testing.T) {
+ executor := &exectest.MockExecutor{}
+ poolJson := ""
+ mockExecutorFuncOutput := func(command string, args ...string) (string, error) {
+ logger.Infof("Command: %s %v", command, args)
+ if args[0] == "osd" && args[1] == "lspools" {
+ return poolJson, nil
+ }
+ return "", errors.Errorf("unexpected ceph command %q", args)
+ }
+ executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+ return mockExecutorFuncOutput(command, args...)
+ }
+ context := &Context{Context: &clusterd.Context{Executor: executor}, Name: "myobj", clusterInfo: client.AdminTestClusterInfo("mycluster")}
+ sharedPools := cephv1.ObjectSharedPoolsSpec{
+ MetadataPoolName: "metapool",
+ DataPoolName: "datapool",
+ }
+ poolJson = `[{"poolnum":1,"poolname":".mgr"},{"poolnum":13,"poolname":".rgw.root"},
+ {"poolnum":14,"poolname":"rgw-meta-pool"},{"poolnum":15,"poolname":"rgw-data-pool"}]`
+ err := sharedPoolsExist(context, sharedPools)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "pools do not exist")
+
+ sharedPools.MetadataPoolName = "rgw-meta-pool"
+ err = sharedPoolsExist(context, sharedPools)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "data pool does not exist")
+
+ sharedPools.DataPoolName = "rgw-data-pool"
+ sharedPools.MetadataPoolName = "bad-pool"
+ err = sharedPoolsExist(context, sharedPools)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "metadata pool does not exist")
+
+ sharedPools.MetadataPoolName = "rgw-meta-pool"
+ err = sharedPoolsExist(context, sharedPools)
+ assert.NoError(t, err)
+}
+
+func TestConfigureStoreWithSharedPools(t *testing.T) {
+ dataPoolAlreadySet := "datapool:store-a.buckets.data"
+ zoneGetCalled := false
+ zoneSetCalled := false
+ placementModifyCalled := false
+ mockExecutorFuncOutput := func(command string, args ...string) (string, error) {
+ logger.Infof("Command: %s %v", command, args)
+ if args[0] == "osd" && args[1] == "lspools" {
+ return `[{"poolnum":14,"poolname":"test-meta"},{"poolnum":15,"poolname":"test-data"}]`, nil
+ }
+ return "", errors.Errorf("unexpected ceph command %q", args)
+ }
+ executorFuncTimeout := func(timeout time.Duration, command string, args ...string) (string, error) {
+ logger.Infof("CommandTimeout: %s %v", command, args)
+ if args[0] == "zone" {
+ if args[1] == "get" {
+ zoneGetCalled = true
+ replaceDataPool := "rgw-data-pool:store-a.buckets.data"
+ return strings.Replace(objectZoneJson, replaceDataPool, dataPoolAlreadySet, -1), nil
+ } else if args[1] == "set" {
+ zoneSetCalled = true
+ return objectZoneJson, nil
+ } else if args[1] == "placement" && args[2] == "modify" {
+ placementModifyCalled = true
+ return objectZoneJson, nil
+ }
+ }
+ return "", errors.Errorf("unexpected ceph command %q", args)
+ }
+ executor := &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: mockExecutorFuncOutput,
+ MockExecuteCommandWithCombinedOutput: mockExecutorFuncOutput,
+ MockExecuteCommandWithTimeout: executorFuncTimeout,
+ }
+ context := &Context{
+ Context: &clusterd.Context{Executor: executor},
+ Name: "myobj",
+ Realm: "myobj",
+ ZoneGroup: "myobj",
+ Zone: "myobj",
+ clusterInfo: client.AdminTestClusterInfo("mycluster"),
+ }
+
+ t.Run("no shared pools", func(t *testing.T) {
+ // No shared pools specified, so skip the config
+ sharedPools := cephv1.ObjectSharedPoolsSpec{}
+ err := ConfigureSharedPoolsForZone(context, sharedPools)
+ assert.NoError(t, err)
+ assert.False(t, zoneGetCalled)
+ assert.False(t, zoneSetCalled)
+ assert.False(t, placementModifyCalled)
+ })
+ t.Run("configure the zone", func(t *testing.T) {
+ sharedPools := cephv1.ObjectSharedPoolsSpec{
+ MetadataPoolName: "test-meta",
+ DataPoolName: "test-data",
+ }
+ err := ConfigureSharedPoolsForZone(context, sharedPools)
+ assert.NoError(t, err)
+ assert.True(t, zoneGetCalled)
+ assert.True(t, zoneSetCalled)
+ assert.True(t, placementModifyCalled)
+ })
+ t.Run("data pool already set", func(t *testing.T) {
+ // Simulate that the data pool has already been set and the zone update can be skipped
+ sharedPools := cephv1.ObjectSharedPoolsSpec{
+ MetadataPoolName: "test-meta",
+ DataPoolName: "test-data",
+ }
+ dataPoolAlreadySet = fmt.Sprintf("%s:%s.buckets.data", sharedPools.DataPoolName, context.Zone)
+ zoneGetCalled = false
+ zoneSetCalled = false
+ placementModifyCalled = false
+ err := ConfigureSharedPoolsForZone(context, sharedPools)
+ assert.True(t, zoneGetCalled)
+ assert.False(t, zoneSetCalled)
+ assert.False(t, placementModifyCalled)
+ assert.NoError(t, err)
+ })
+}
+
func TestDeleteStore(t *testing.T) {
deleteStore(t, "myobj", `"mystore","myobj"`, false)
deleteStore(t, "myobj", `"myobj"`, true)
@@ -652,7 +927,8 @@ func Test_createMultisite(t *testing.T) {
objContext := NewContext(ctx, &client.ClusterInfo{Namespace: "my-cluster"}, "my-store")
// assumption: endpointArg is sufficiently tested by integration tests
- err := createMultisite(objContext, "")
+ store := &cephv1.CephObjectStore{}
+ err := createNonMultisiteStore(objContext, "", store)
assert.Equal(t, tt.expectCommands.getRealm, calledGetRealm)
assert.Equal(t, tt.expectCommands.createRealm, calledCreateRealm)
assert.Equal(t, tt.expectCommands.getZoneGroup, calledGetZoneGroup)
diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go
index 36ff48144..50cdeeaae 100644
--- a/pkg/operator/ceph/object/zone/controller.go
+++ b/pkg/operator/ceph/object/zone/controller.go
@@ -268,21 +268,20 @@ func (r *ReconcileObjectZone) createorUpdateCephZone(zone *cephv1.CephObjectZone
return reconcile.Result{}, nil
}
- if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
- logger.Debugf("ceph zone %q not found, running `radosgw-admin zone create`", zone.Name)
-
- zoneIsMaster := false
- if zoneGroupJson.MasterZoneID == "" {
- zoneIsMaster = true
- }
-
- err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster)
- if err != nil {
- return reconcile.Result{}, err
- }
- } else {
+ if code, ok := exec.ExitStatus(err); ok && code != int(syscall.ENOENT) {
return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zone get failed with code %d for reason %q", code, output)
}
+ logger.Debugf("ceph zone %q not found, running `radosgw-admin zone create`", zone.Name)
+
+ zoneIsMaster := false
+ if zoneGroupJson.MasterZoneID == "" {
+ zoneIsMaster = true
+ }
+
+ err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster)
+ if err != nil {
+ return reconcile.Result{}, err
+ }
return reconcile.Result{}, nil
}
@@ -294,7 +293,7 @@ func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zon
zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup)
zoneArg := fmt.Sprintf("--rgw-zone=%s", zone.Name)
- err := object.CreatePools(objContext, r.clusterSpec, zone.Spec.MetadataPool, zone.Spec.DataPool)
+ err := object.ConfigurePools(objContext, r.clusterSpec, zone.Spec.MetadataPool, zone.Spec.DataPool, zone.Spec.SharedPools)
if err != nil {
return errors.Wrapf(err, "failed to create pools for zone %v", zone.Name)
}
@@ -321,6 +320,12 @@ func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zon
}
logger.Debugf("created ceph zone %q", zone.Name)
+ // Configure the zone for RADOS namespaces
+ err = object.ConfigureSharedPoolsForZone(objContext, zone.Spec.SharedPools)
+ if err != nil {
+ return errors.Wrapf(err, "failed to configure rados namespaces for zone")
+ }
+
return nil
}
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 9af42bae4..6de8cf89e 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -301,7 +301,9 @@ function deploy_cluster() {
# create the cluster resources
kubectl create -f cluster-test.yaml
- kubectl create -f object-test.yaml
+ kubectl create -f object-shared-pools-test.yaml
+ kubectl create -f object-a.yaml
+ kubectl create -f object-b.yaml
kubectl create -f pool-test.yaml
kubectl create -f filesystem-test.yaml
sed -i "/resources:/,/ # priorityClassName:/d" rbdmirror.yaml
diff --git a/tests/scripts/validate_cluster.sh b/tests/scripts/validate_cluster.sh
index 306a1eece..091e83caa 100755
--- a/tests/scripts/validate_cluster.sh
+++ b/tests/scripts/validate_cluster.sh
@@ -20,7 +20,14 @@ set -xEe
if [ -z "$DAEMON_TO_VALIDATE" ]; then
DAEMON_TO_VALIDATE=all
fi
-OSD_COUNT=$2
+# The second script arg is optional and depends on the daemon
+if [ "$DAEMON_TO_VALIDATE" == "rgw" ]; then
+ export OBJECT_STORE_NAME=$2
+else
+ export OSD_COUNT=$2
+ # default to the name of the object store from object-a.yaml
+ export OBJECT_STORE_NAME=store-a
+fi
#############
# FUNCTIONS #
@@ -63,7 +70,7 @@ function test_demo_osd {
function test_demo_rgw {
timeout 360 bash -x <<-'EOF'
- until [[ "$(kubectl -n rook-ceph get pods -l app=rook-ceph-rgw -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" == "True" ]]; do
+ until [[ "$(kubectl -n rook-ceph get pods -l rgw=$OBJECT_STORE_NAME -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" == "True" ]]; do
echo "waiting for rgw pods to be ready"
sleep 5
done
|