From 572f4ead4907ac94bd48871dc25acb627a07c327 Mon Sep 17 00:00:00 2001 From: Oded Viner Date: Sun, 29 Jun 2025 12:08:53 +0300 Subject: [PATCH] object: mark realm as default to avoid zone creation add defaultRealm field to cephobjectstore to allow rook to mark the created realm as default. this prevents ceph from auto-creating a "default" zonegroup and zone when the default is not explicitly set. affects only non-multisite setups. Signed-off-by: Oded Viner (cherry picked from commit 182cd211ca0c23156fe04a6a4b05b1a0a5b30403) --- .../Object-Storage/ceph-object-realm-crd.md | 1 + .../Object-Storage/ceph-object-store-crd.md | 8 ++- Documentation/CRDs/specification.md | 54 +++++++++++++++++++ .../charts/rook-ceph/templates/resources.yaml | 13 +++++ deploy/examples/crds.yaml | 13 +++++ pkg/apis/ceph.rook.io/v1/types.go | 12 +++++ pkg/operator/ceph/object/controller.go | 6 +++ pkg/operator/ceph/object/objectstore.go | 24 +++++++-- pkg/operator/ceph/object/objectstore_test.go | 3 +- pkg/operator/ceph/object/realm/controller.go | 9 ++++ 10 files changed, 136 insertions(+), 7 deletions(-) diff --git a/Documentation/CRDs/Object-Storage/ceph-object-realm-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-realm-crd.md index e14f0bb73..7853bfaa3 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-realm-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-realm-crd.md @@ -30,3 +30,4 @@ spec: * `pull`: This optional section is for the pulling the realm for another ceph cluster. * `endpoint`: The endpoint in the realm from another ceph cluster you want to pull from. This endpoint must be in the master zone of the master zone group of the realm. +* `defaultRealm`: When set to true, Rook will mark the CephObjectStore's realm as the default realm in the Ceph cluster. Only one realm can be marked default. Ceph does not allow default to be unassigned after it is assigned; a different realm can be marked default instead. diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 57b335fea..7ef151e19 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -96,6 +96,10 @@ When the `zone` section is set pools with the object stores name will not be cre This is useful for applications that need object store credentials to be created in their own namespace, where neither OBCs nor COSI is being used to create buckets. The default is empty. +### Realm + +* `defaultRealm`: When set to true, Rook will mark the CephObjectStore's realm as the default realm in the Ceph cluster. Only one realm can be marked default. Ceph does not allow default to be unassigned after it is assigned; a different realm can be marked default instead. + ## Auth Settings The `auth`-section allows the configuration of authentication providers in addition to the regular authentication mechanism. @@ -399,7 +403,7 @@ spec: rgw_s3_auth_use_ldap: "true" # bool rgwConfigFromSecret: rgw_keystone_barbican_password: # name of rgw option with secret value - name: "barbican-secret" # name of K8s secret + name: "barbican-secret" # name of K8s secret key: "password" # key of secret value in K8s secret.Data map[string]string rgwCommandFlags: rgw_dmclock_auth_res: "100.0" # float @@ -478,7 +482,7 @@ The sample configuration below demonstrates how to securely handle secret config rgwConfig: rgw_barbican_url: "http://barbican.example.com:9311" rgw_keystone_barbican_domain: "domain" - rgw_keystone_barbican_project: "project" + rgw_keystone_barbican_project: "project" rgw_keystone_barbican_user: "user" rgwConfigFromSecret: rgw_keystone_barbican_password: diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index a7f4b61c0..fe70ba186 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -1752,6 +1752,18 @@ PullSpec + + +defaultRealm
+ +bool + + + +(Optional) +

Set this realm as the default in Ceph. Only one realm should be default.

+ + @@ -1998,6 +2010,21 @@ A common use case for hosting configuration is to inform Rook of endpoints that wildcards, which in turn allows virtual host-style bucket addressing.

+ + +defaultRealm
+ +bool + + + +(Optional) +

Set this realm as the default in Ceph. Only one realm should be default. +Do not set this true on more than one CephObjectStore. +This may not be set when zone is also specified; in this case, the realm +referenced by the zone’s zonegroup should configure defaulting behavior.

+ + @@ -10125,6 +10152,18 @@ PullSpec + + +defaultRealm
+ +bool + + + +(Optional) +

Set this realm as the default in Ceph. Only one realm should be default.

+ +

ObjectSharedPoolsSpec @@ -10498,6 +10537,21 @@ A common use case for hosting configuration is to inform Rook of endpoints that wildcards, which in turn allows virtual host-style bucket addressing.

+ + +defaultRealm
+ +bool + + + +(Optional) +

Set this realm as the default in Ceph. Only one realm should be default. +Do not set this true on more than one CephObjectStore. +This may not be set when zone is also specified; in this case, the realm +referenced by the zone’s zonegroup should configure defaulting behavior.

+ +

ObjectStoreStatus diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index a83d651a9..b5db99099 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -10952,6 +10952,9 @@ spec: description: ObjectRealmSpec represent the spec of an ObjectRealm nullable: true properties: + defaultRealm: + description: Set this realm as the default in Ceph. Only one realm should be default. + type: boolean pull: description: PullSpec represents the pulling specification of a Ceph Object Storage Gateway Realm properties: @@ -11292,6 +11295,13 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + defaultRealm: + description: |- + Set this realm as the default in Ceph. Only one realm should be default. + Do not set this true on more than one CephObjectStore. + This may not be set when zone is also specified; in this case, the realm + referenced by the zone's zonegroup should configure defaulting behavior. + type: boolean gateway: description: The rgw pod info nullable: true @@ -13102,6 +13112,9 @@ spec: - name type: object type: object + x-kubernetes-validations: + - message: defaultRealm must not be true when zone.name is set (multisite configuration) + rule: '!(has(self.defaultRealm) && self.defaultRealm == true && has(self.zone) && size(self.zone.name) > 0)' status: description: ObjectStoreStatus represents the status of a Ceph Object Store resource properties: diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 30821799d..6811f2802 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -10944,6 +10944,9 @@ spec: description: ObjectRealmSpec represent the spec of an ObjectRealm nullable: true properties: + defaultRealm: + description: Set this realm as the default in Ceph. Only one realm should be default. + type: boolean pull: description: PullSpec represents the pulling specification of a Ceph Object Storage Gateway Realm properties: @@ -11283,6 +11286,13 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + defaultRealm: + description: |- + Set this realm as the default in Ceph. Only one realm should be default. + Do not set this true on more than one CephObjectStore. + This may not be set when zone is also specified; in this case, the realm + referenced by the zone's zonegroup should configure defaulting behavior. + type: boolean gateway: description: The rgw pod info nullable: true @@ -13093,6 +13103,9 @@ spec: - name type: object type: object + x-kubernetes-validations: + - message: defaultRealm must not be true when zone.name is set (multisite configuration) + rule: '!(has(self.defaultRealm) && self.defaultRealm == true && has(self.zone) && size(self.zone.name) > 0)' status: description: ObjectStoreStatus represents the status of a Ceph Object Store resource properties: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 0b00ffb20..0b4638971 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1534,6 +1534,7 @@ type CephObjectStoreList struct { } // ObjectStoreSpec represent the spec of a pool +// +kubebuilder:validation:XValidation:rule="!(has(self.defaultRealm) && self.defaultRealm == true && has(self.zone) && size(self.zone.name) > 0)",message="defaultRealm must not be true when zone.name is set (multisite configuration)" type ObjectStoreSpec struct { // The metadata pool settings // +optional @@ -1597,6 +1598,13 @@ type ObjectStoreSpec struct { // +nullable // +optional Hosting *ObjectStoreHostingSpec `json:"hosting,omitempty"` + + // Set this realm as the default in Ceph. Only one realm should be default. + // Do not set this true on more than one CephObjectStore. + // This may not be set when zone is also specified; in this case, the realm + // referenced by the zone's zonegroup should configure defaulting behavior. + // +optional + DefaultRealm bool `json:"defaultRealm,omitempty"` } // ObjectSharedPoolsSpec represents object store pool info when configuring RADOS namespaces in existing pools. @@ -2220,6 +2228,10 @@ type CephObjectRealmList struct { // ObjectRealmSpec represent the spec of an ObjectRealm type ObjectRealmSpec struct { Pull PullSpec `json:"pull,omitempty"` + + // Set this realm as the default in Ceph. Only one realm should be default. + // +optional + DefaultRealm bool `json:"defaultRealm,omitempty"` } // PullSpec represents the pulling specification of a Ceph Object Storage Gateway Realm diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 282ef1519..36b26510c 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -596,6 +596,12 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create object store %q", cephObjectStore.Name) } + if cephObjectStore.Spec.DefaultRealm { + logger.Debugf("marking realm %q as default for object store %q", realmName, cephObjectStore.Name) + if err := SetDefaultRealm(objContext, realmName); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to set realm %q as default", realmName) + } + } } return reconcile.Result{}, nil diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 927a3736d..3963fecf5 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -31,6 +31,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/exec" @@ -385,7 +386,7 @@ func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, bo return zoneEndpointsList, isEndpointAlreadyExists, nil } -func createMultisiteConfigurations(objContext *Context, configType, configTypeArg string, args ...string) error { +func createMultisiteConfigurations(objContext *Context, store *cephv1.CephObjectStore, configType, configTypeArg string, args ...string) error { args = append([]string{configType}, args...) args = append(args, configTypeArg) // get the multisite config before creating @@ -415,6 +416,10 @@ func createMultisiteConfigurations(objContext *Context, configType, configTypeAr } // create the object if it doesn't exist yet + if store.Spec.DefaultRealm && objContext.clusterInfo.CephVersion.IsAtLeast(cephver.Squid) { + logger.Infof("marking object store %q as default realm", store.Namespace+"/"+store.Name) + args = append(args, "--default") + } output, err = RunAdminCommandNoMultisite(objContext, false, args...) if err != nil { return errorOrIsNotFound(err, "failed to create ceph %q %q, for reason %q", configType, configTypeArg, output) @@ -431,17 +436,17 @@ func createNonMultisiteStore(objContext *Context, endpointArg string, store *cep zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone) - err := createMultisiteConfigurations(objContext, "realm", realmArg, "create") + err := createMultisiteConfigurations(objContext, store, "realm", realmArg, "create") if err != nil { return err } - err = createMultisiteConfigurations(objContext, "zonegroup", zoneGroupArg, "create", "--master", realmArg, endpointArg) + err = createMultisiteConfigurations(objContext, store, "zonegroup", zoneGroupArg, "create", "--master", realmArg, endpointArg) if err != nil { return err } - err = createMultisiteConfigurations(objContext, "zone", zoneArg, "create", "--master", endpointArg, realmArg, zoneGroupArg) + err = createMultisiteConfigurations(objContext, store, "zone", zoneArg, "create", "--master", endpointArg, realmArg, zoneGroupArg) if err != nil { return err } @@ -1407,3 +1412,14 @@ func ValidateObjectStorePoolsConfig(metadataPool, dataPool cephv1.PoolSpec, shar } return nil } + +func SetDefaultRealm(objContext *Context, realmName string) error { + args := []string{"realm", "default", fmt.Sprintf("--rgw-realm=%s", realmName)} + output, err := RunAdminCommandNoMultisite(objContext, false, args...) + if err != nil { + return errors.Wrapf(err, "failed to set realm %q as default, reason: %q", realmName, output) + } + + logger.Infof("successfully set realm %q as default", realmName+"/"+objContext.clusterInfo.Namespace) + return nil +} diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 76532f867..3c134eaea 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -1054,6 +1054,7 @@ func getreturnErrString() []string { func Test_createMultisiteConfigurations(t *testing.T) { executor := getExecutor() returnErrString := getreturnErrString() + store := &cephv1.CephObjectStore{} for i := 0; i < 4; i++ { ctx := &clusterd.Context{ Executor: executor[i], @@ -1061,7 +1062,7 @@ func Test_createMultisiteConfigurations(t *testing.T) { objContext := NewContext(ctx, &client.ClusterInfo{Namespace: "my-cluster"}, "my-store") realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) - err := createMultisiteConfigurations(objContext, "realm", realmArg, "create") + err := createMultisiteConfigurations(objContext, store, "realm", realmArg, "create") if i == 0 { assert.NoError(t, err) } else { diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 270b4e713..f7cc47e90 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -210,6 +210,15 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R } } + // Set the realm as default if specified and supported + if cephObjectRealm.Spec.DefaultRealm { + objCtx := object.NewContext(r.context, r.clusterInfo, cephObjectRealm.Namespace) + if err := object.SetDefaultRealm(objCtx, cephObjectRealm.Name); err != nil { + return reconcile.Result{}, *cephObjectRealm, errors.Wrapf(err, + "failed to set realm %q as default", cephObjectRealm.Name) + } + } + // update ObservedGeneration in status at the end of reconcile // Set Ready status, we are done reconciling r.updateStatus(observedGeneration, request.NamespacedName, k8sutil.ReadyStatus)