From f6a25b0c4bbcaaa475bd057c7107fac2b20418d9 Mon Sep 17 00:00:00 2001 From: Artem Torubarov Date: Tue, 22 Oct 2024 11:23:42 +0200 Subject: [PATCH] rgw: support v19 zone pools add rgw shared pools rados ns to topics_pool, account_pool, and group_pool Signed-off-by: Artem Torubarov --- pkg/operator/ceph/object/json_helpers.go | 15 +- pkg/operator/ceph/object/json_helpers_test.go | 259 +++++++----------- pkg/operator/ceph/object/objectstore.go | 23 +- pkg/operator/ceph/object/shared_pools.go | 16 +- pkg/operator/ceph/object/shared_pools_test.go | 68 +++++ 5 files changed, 198 insertions(+), 183 deletions(-) diff --git a/pkg/operator/ceph/object/json_helpers.go b/pkg/operator/ceph/object/json_helpers.go index 719339e7d..5558272cd 100644 --- a/pkg/operator/ceph/object/json_helpers.go +++ b/pkg/operator/ceph/object/json_helpers.go @@ -43,14 +43,9 @@ func getObjProperty[T string | map[string]interface{} | []interface{}](obj map[s return res, fmt.Errorf("json property %q not found", strings.Join(path, ".")) } -// setObjProperty - helper function to manipulate JSON Objects. -// sets value to json object nested field and returns previous value if presented. -// Example: -// -// obj = {"a":{"b":"foo"}} -// // will replace "foo" with "bar" and return "foo" -// setObjProperty(obj,"bar","a","b") -func setObjProperty[T string | []string | map[string]interface{} | []interface{}](obj map[string]interface{}, val T, path ...string) (T, error) { +// updateObjProperty - helper function to manipulate JSON Objects. +// sets new value to json object nested field only if it is already exists in json and returns previous value. +func updateObjProperty[T string | []string | map[string]interface{} | []interface{}](obj map[string]interface{}, val T, path ...string) (T, error) { var prev T if len(path) == 0 { return prev, fmt.Errorf("json property path is empty") @@ -60,6 +55,10 @@ func setObjProperty[T string | []string | map[string]interface{} | []interface{} if last { // last path segment: set result and return prev value prevVal, ok := obj[p] + if !ok { + // not exists: + return prev, fmt.Errorf("json property %q not exists", strings.Join(path, ".")) + } if ok { prevRes, ok := prevVal.(T) if ok { diff --git a/pkg/operator/ceph/object/json_helpers_test.go b/pkg/operator/ceph/object/json_helpers_test.go index 9448c1070..6ba5dac40 100644 --- a/pkg/operator/ceph/object/json_helpers_test.go +++ b/pkg/operator/ceph/object/json_helpers_test.go @@ -219,7 +219,24 @@ func Test_getObjPropertyObjArr(t *testing.T) { } } -func Test_setObjProperty(t *testing.T) { +func Test_deepCopyJson(t *testing.T) { + in := map[string]interface{}{ + "key": []interface{}{"1", "2", "3"}, + } + inCopy, err := deepCopyJson(in) + assert.NoError(t, err) + assert.EqualValues(t, in, inCopy) + + assert.EqualValues(t, []interface{}{"1", "2", "3"}, in["key"]) + assert.EqualValues(t, []interface{}{"1", "2", "3"}, inCopy["key"]) + + inCopy["key"].([]interface{})[1] = "7" + + assert.EqualValues(t, []interface{}{"1", "2", "3"}, in["key"]) + assert.EqualValues(t, []interface{}{"1", "7", "3"}, inCopy["key"]) +} + +func Test_updateObjProperty(t *testing.T) { type args struct { json string val string @@ -233,7 +250,7 @@ func Test_setObjProperty(t *testing.T) { wantErr bool }{ { - name: "replace val", + name: "exists", args: args{ json: `{"a":{"b":"val"}}`, val: "new val", @@ -259,7 +276,7 @@ func Test_setObjProperty(t *testing.T) { wantErr: false, }, { - name: "add val", + name: "not exists", args: args{ json: `{"a":{"b":"val"}}`, val: "val2", @@ -268,21 +285,8 @@ func Test_setObjProperty(t *testing.T) { }, }, wantPrev: "", - wantJSON: `{"a":{"b":"val","c":"val2"}}`, - wantErr: false, - }, - { - name: "add root val", - args: args{ - json: `{"a":{"b":"val"}}`, - val: "val2", - path: []string{ - "c", - }, - }, - wantPrev: "", - wantJSON: `{"a":{"b":"val"},"c":"val2"}`, - wantErr: false, + wantJSON: `{"a":{"b":"val"}}`, + wantErr: true, }, } for _, tt := range tests { @@ -290,66 +294,7 @@ func Test_setObjProperty(t *testing.T) { obj := map[string]interface{}{} err := json.Unmarshal([]byte(tt.args.json), &obj) assert.NoError(t, err) - prev, err := setObjProperty(obj, tt.args.val, tt.args.path...) - if tt.wantErr { - assert.Error(t, err) - return - } else { - assert.NoError(t, err) - } - assert.EqualValues(t, tt.wantPrev, prev) - bytes, err := json.Marshal(obj) - assert.NoError(t, err) - assert.JSONEq(t, tt.wantJSON, string(bytes)) - }) - } -} -func Test_setObjPropertyObj(t *testing.T) { - type args struct { - json string - val map[string]interface{} - path []string - } - tests := []struct { - name string - args args - wantPrev map[string]interface{} - wantJSON string - wantErr bool - }{ - { - name: "add obj", - args: args{ - json: `{"a":{"b":{}}}`, - val: map[string]interface{}{"c": "val1"}, - path: []string{ - "a", "b", - }, - }, - wantPrev: map[string]interface{}{}, - wantJSON: `{"a":{"b":{"c":"val1"}}}`, - wantErr: false, - }, - { - name: "set obj", - args: args{ - json: `{"a":{"b":{"c": "val1"}}}`, - val: map[string]interface{}{"d": "val2"}, - path: []string{ - "a", "b", - }, - }, - wantPrev: map[string]interface{}{"c": "val1"}, - wantJSON: `{"a":{"b":{"d":"val2"}}}`, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - obj := map[string]interface{}{} - err := json.Unmarshal([]byte(tt.args.json), &obj) - assert.NoError(t, err) - prev, err := setObjProperty(obj, tt.args.val, tt.args.path...) + prev, err := updateObjProperty(obj, tt.args.val, tt.args.path...) if tt.wantErr { assert.Error(t, err) return @@ -364,7 +309,67 @@ func Test_setObjPropertyObj(t *testing.T) { } } -func Test_setObjPropertyArr(t *testing.T) { +func Test_updateObjPropertyObj(t *testing.T) { + type args struct { + json string + val map[string]interface{} + path []string + } + tests := []struct { + name string + args args + wantPrev map[string]interface{} + wantJSON string + wantErr bool + }{ + { + name: "exists", + args: args{ + json: `{"a":{"b":{"c": "val1"}}}`, + val: map[string]interface{}{"d": "val2"}, + path: []string{ + "a", "b", + }, + }, + wantPrev: map[string]interface{}{"c": "val1"}, + wantJSON: `{"a":{"b":{"d":"val2"}}}`, + wantErr: false, + }, + { + name: "not exists", + args: args{ + json: `{"a":{"b":{}}}`, + val: map[string]interface{}{"c": "val1"}, + path: []string{ + "a", "c", + }, + }, + wantPrev: nil, + wantJSON: `{"a":{"b":{}}}`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := map[string]interface{}{} + err := json.Unmarshal([]byte(tt.args.json), &obj) + assert.NoError(t, err) + prev, err := updateObjProperty(obj, tt.args.val, tt.args.path...) + if tt.wantErr { + assert.Error(t, err) + return + } else { + assert.NoError(t, err) + } + assert.EqualValues(t, tt.wantPrev, prev) + bytes, err := json.Marshal(obj) + assert.NoError(t, err) + assert.JSONEq(t, tt.wantJSON, string(bytes)) + }) + } +} + +func Test_updateObjPropertyArr(t *testing.T) { type args struct { json string val []interface{} @@ -378,23 +383,7 @@ func Test_setObjPropertyArr(t *testing.T) { wantErr bool }{ { - name: "set obj arr", - args: args{ - json: `{"a":{"b":{}}}`, - val: []interface{}{ - map[string]interface{}{"c": "val1"}, - map[string]interface{}{"d": "val2"}, - }, - path: []string{ - "a", "b", - }, - }, - wantPrev: nil, - wantJSON: `{"a":{"b":[{"c":"val1"},{"d":"val2"}]}}`, - wantErr: false, - }, - { - name: "add obj arr", + name: "exists", args: args{ json: `{"a":{"b":[{"c": "val"}]}}`, val: []interface{}{ @@ -411,64 +400,21 @@ func Test_setObjPropertyArr(t *testing.T) { wantJSON: `{"a":{"b":[{"d":"val1"},{"e":"val2"}]}}`, wantErr: false, }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - obj := map[string]interface{}{} - err := json.Unmarshal([]byte(tt.args.json), &obj) - assert.NoError(t, err) - prev, err := setObjProperty(obj, tt.args.val, tt.args.path...) - if tt.wantErr { - assert.Error(t, err) - return - } else { - assert.NoError(t, err) - } - assert.EqualValues(t, tt.wantPrev, prev) - bytes, err := json.Marshal(obj) - assert.NoError(t, err) - assert.JSONEq(t, tt.wantJSON, string(bytes)) - }) - } -} -func Test_setObjPropertyStrArr(t *testing.T) { - type args struct { - json string - val []string - path []string - } - tests := []struct { - name string - args args - wantPrev []string - wantJSON string - wantErr bool - }{ { - name: "add str arr", + name: "not exists", args: args{ json: `{"a":{"b":{}}}`, - val: []string{"c", "d"}, + val: []interface{}{ + map[string]interface{}{"c": "val1"}, + map[string]interface{}{"d": "val2"}, + }, path: []string{ - "a", "b", + "a", "c", }, }, wantPrev: nil, - wantJSON: `{"a":{"b":["c","d"]}}`, - wantErr: false, - }, - { - name: "set str arr", - args: args{ - json: `{"a":{"b":["val"]}}`, - val: []string{"c", "d"}, - path: []string{ - "a", "b", - }, - }, - wantPrev: []string{"val"}, - wantJSON: `{"a":{"b":["c","d"]}}`, - wantErr: false, + wantJSON: `{"a":{"b":{}}}`, + wantErr: true, }, } for _, tt := range tests { @@ -476,7 +422,7 @@ func Test_setObjPropertyStrArr(t *testing.T) { obj := map[string]interface{}{} err := json.Unmarshal([]byte(tt.args.json), &obj) assert.NoError(t, err) - prev, err := setObjProperty(obj, tt.args.val, tt.args.path...) + prev, err := updateObjProperty(obj, tt.args.val, tt.args.path...) if tt.wantErr { assert.Error(t, err) return @@ -490,20 +436,3 @@ func Test_setObjPropertyStrArr(t *testing.T) { }) } } - -func Test_deepCopyJson(t *testing.T) { - in := map[string]interface{}{ - "key": []interface{}{"1", "2", "3"}, - } - inCopy, err := deepCopyJson(in) - assert.NoError(t, err) - assert.EqualValues(t, in, inCopy) - - assert.EqualValues(t, []interface{}{"1", "2", "3"}, in["key"]) - assert.EqualValues(t, []interface{}{"1", "2", "3"}, inCopy["key"]) - - inCopy["key"].([]interface{})[1] = "7" - - assert.EqualValues(t, []interface{}{"1", "2", "3"}, in["key"]) - assert.EqualValues(t, []interface{}{"1", "7", "3"}, inCopy["key"]) -} diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index b80806845..63f28dbed 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -903,18 +903,37 @@ func adjustZoneDefaultPools(zone map[string]interface{}, spec cephv1.ObjectShare "user_uid_pool": ".meta.users.uid", "otp_pool": ".otp", "notif_pool": ".log.notif", + "topics_pool": ".meta.topics", // introduced in Ceph v19 + "account_pool": ".meta.account", // introduced in Ceph v19 + "group_pool": ".meta.group", // introduced in Ceph v19 } for pool, nsSuffix := range zonePoolNSSuffix { // replace rgw internal index pools with namespaced metadata pool namespacedPool := defaultMetaPool + nsSuffix - prev, err := setObjProperty(zone, namespacedPool, pool) + prev, err := updateObjProperty(zone, namespacedPool, pool) if err != nil { - return nil, fmt.Errorf("unable to set pool %s for zone %s: %w", pool, name, err) + logger.Infof("unable to apply rados namespace to shared pool: %v", err) } if namespacedPool != prev { logger.Debugf("update shared pool %s for zone %s: %s -> %s", pool, name, prev, namespacedPool) } } + + // check for unknown pool properties in zone json + for field, val := range zone { + if _, ok := val.(string); !ok { + // not a string property + continue + } + if !strings.HasSuffix(field, "_pool") { + // not a pool property + continue + } + if _, ok := zonePoolNSSuffix[field]; !ok { + logger.Warningf("zone config %q contains unknown pool %q", name, field) + } + } + return zone, nil } diff --git a/pkg/operator/ceph/object/shared_pools.go b/pkg/operator/ceph/object/shared_pools.go index 0ab62b8d7..7627a08a8 100644 --- a/pkg/operator/ceph/object/shared_pools.go +++ b/pkg/operator/ceph/object/shared_pools.go @@ -98,11 +98,11 @@ func adjustZonePlacementPools(zone map[string]interface{}, spec cephv1.ObjectSha } // update placement with values from spec: if pSpec, inSpec := fromSpec[placementID]; inSpec { - _, err = setObjProperty(pObj, pSpec.Val.IndexPool, "val", "index_pool") + _, err = updateObjProperty(pObj, pSpec.Val.IndexPool, "val", "index_pool") if err != nil { return nil, fmt.Errorf("unable to set index pool to pool placement %q for zone %q: %w", placementID, name, err) } - _, err = setObjProperty(pObj, pSpec.Val.DataExtraPool, "val", "data_extra_pool") + _, err = updateObjProperty(pObj, pSpec.Val.DataExtraPool, "val", "data_extra_pool") if err != nil { return nil, fmt.Errorf("unable to set data extra pool to pool placement %q for zone %q: %w", placementID, name, err) } @@ -111,7 +111,7 @@ func adjustZonePlacementPools(zone map[string]interface{}, spec cephv1.ObjectSha return nil, fmt.Errorf("unable convert to pool placement %q storage class for zone %q: %w", placementID, name, err) } - _, err = setObjProperty(pObj, scObj, "val", "storage_classes") + _, err = updateObjProperty(pObj, scObj, "val", "storage_classes") if err != nil { return nil, fmt.Errorf("unable to set storage classes to pool placement %q for zone %q: %w", placementID, name, err) } @@ -144,7 +144,7 @@ func adjustZonePlacementPools(zone map[string]interface{}, spec cephv1.ObjectSha placements = append(placements, pObj) } - _, err = setObjProperty(zone, placements, "placement_pools") + _, err = updateObjProperty(zone, placements, "placement_pools") if err != nil { return nil, fmt.Errorf("unable to set pool placements for zone %q: %w", name, err) } @@ -247,7 +247,7 @@ func adjustZoneGroupPlacementTargets(group, zone map[string]interface{}) (map[st return nil, fmt.Errorf("unable to deep copy config for zonegroup %s: %w", name, err) } - _, err = setObjProperty(group, defaultPlacementCephConfigName, "default_placement") + _, err = updateObjProperty(group, defaultPlacementCephConfigName, "default_placement") if err != nil { return nil, fmt.Errorf("unable to set default_placement for zonegroup %s: %w", name, err) } @@ -277,9 +277,9 @@ func adjustZoneGroupPlacementTargets(group, zone map[string]interface{}) (map[st sc := []interface{}{} ok = castJson(desired.StorageClasses, &sc) if ok { - _, err = setObjProperty(tObj, sc, "storage_classes") + _, err = updateObjProperty(tObj, sc, "storage_classes") } else { - _, err = setObjProperty(tObj, desired.StorageClasses, "storage_classes") + _, err = updateObjProperty(tObj, desired.StorageClasses, "storage_classes") } if err != nil { return nil, fmt.Errorf("unable to set storage classes to pool placement target %q for zonegroup %q: %w", tName, name, err) @@ -317,7 +317,7 @@ func adjustZoneGroupPlacementTargets(group, zone map[string]interface{}) (map[st currentTargets = append(currentTargets, tObj) } - _, err = setObjProperty(group, currentTargets, "placement_targets") + _, err = updateObjProperty(group, currentTargets, "placement_targets") if err != nil { return nil, fmt.Errorf("unable to set placement targets for zonegroup %q: %w", name, err) } diff --git a/pkg/operator/ceph/object/shared_pools_test.go b/pkg/operator/ceph/object/shared_pools_test.go index 33f3e6ed2..4ef29275f 100644 --- a/pkg/operator/ceph/object/shared_pools_test.go +++ b/pkg/operator/ceph/object/shared_pools_test.go @@ -993,6 +993,74 @@ func Test_adjustZoneDefaultPools(t *testing.T) { }, "placement_pools": [], "realm_id": "29e28253-be54-4581-90dd-206020d2fcdd" +}`, + wantChanged: true, + wantErr: false, + }, + { + name: "v19 shared pool set", + args: args{ + beforeJSON: `{ + "id": "f539c2c0-e1ed-4c42-9294-41742352eeae", + "name": "test", + "topics_pool": "TopicsPool", + "account_pool": "AccountPool", + "group_pool": "GroupPool", + "domain_root": "DomainRoot", + "control_pool": "ControlPool", + "gc_pool": "GcPool", + "lc_pool": "LcPool", + "log_pool": "LogPool", + "intent_log_pool": "IntentLogPool", + "usage_log_pool": "UsageLogPool", + "roles_pool": "RolesPool", + "reshard_pool": "ReshardPool", + "user_keys_pool": "UserKeysPool", + "user_email_pool": "UserEmailPool", + "user_swift_pool": "UserSwiftPool", + "user_uid_pool": "UserUIDPool", + "otp_pool": "OtpPool", + "notif_pool": "NotifPool", + "system_key": { + "access_key": "AccessKey", + "secret_key": "SecretKey" + }, + "placement_pools": [], + "realm_id": "29e28253-be54-4581-90dd-206020d2fcdd" +}`, + spec: cephv1.ObjectSharedPoolsSpec{ + MetadataPoolName: "meta-pool", + DataPoolName: "data-pool", + PreserveRadosNamespaceDataOnDelete: false, + }, + }, + wantJSON: `{ + "id": "f539c2c0-e1ed-4c42-9294-41742352eeae", + "name": "test", + "topics_pool": "meta-pool:test.meta.topics", + "account_pool": "meta-pool:test.meta.account", + "group_pool": "meta-pool:test.meta.group", + "domain_root": "meta-pool:test.meta.root", + "control_pool": "meta-pool:test.control", + "gc_pool": "meta-pool:test.log.gc", + "lc_pool": "meta-pool:test.log.lc", + "log_pool": "meta-pool:test.log", + "intent_log_pool": "meta-pool:test.log.intent", + "usage_log_pool": "meta-pool:test.log.usage", + "roles_pool": "meta-pool:test.meta.roles", + "reshard_pool": "meta-pool:test.log.reshard", + "user_keys_pool": "meta-pool:test.meta.users.keys", + "user_email_pool": "meta-pool:test.meta.users.email", + "user_swift_pool": "meta-pool:test.meta.users.swift", + "user_uid_pool": "meta-pool:test.meta.users.uid", + "otp_pool": "meta-pool:test.otp", + "notif_pool": "meta-pool:test.log.notif", + "system_key": { + "access_key": "AccessKey", + "secret_key": "SecretKey" + }, + "placement_pools": [], + "realm_id": "29e28253-be54-4581-90dd-206020d2fcdd" }`, wantChanged: true, wantErr: false,