my-rook-config/pkg/operator/ceph/object/shared_pools.go

package object

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"sort"

	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/util/log"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
)

const (
	defaultPlacementCephConfigName = "default-placement"
	defaultPlacementStorageClass   = "STANDARD"
)

func IsNeedToCreateObjectStorePools(sharedPools cephv1.ObjectSharedPoolsSpec) bool {
	for _, pp := range sharedPools.PoolPlacements {
		if pp.Default {
			// No need to create pools. External pools from default placement will be used
			return false
		}
	}
	if sharedPools.MetadataPoolName != "" && sharedPools.DataPoolName != "" {
		// No need to create pools. Shared pools will be used
		return false
	}
	return true
}

func validatePoolPlacements(placements []cephv1.PoolPlacementSpec) error {
	hasDefault := false
	names := make(map[string]struct{}, len(placements))
	for _, p := range placements {
		if hasDefault && p.Default {
			return fmt.Errorf("invalidObjStorePoolConfig: only one placement can be set as default")
		}
		hasDefault = hasDefault || p.Default

		if _, ok := names[p.Name]; ok {
			return fmt.Errorf("invalidObjStorePoolConfig: invalid placement %s: placement names must be unique", p.Name)
		}

		if p.Name == defaultPlacementCephConfigName && !p.Default {
			return fmt.Errorf("invalidObjStorePoolConfig: placement with name %s must be marked as default", defaultPlacementCephConfigName)
		}

		names[p.Name] = struct{}{}
		if err := validatePoolPlacementStorageClasses(p.StorageClasses); err != nil {
			return fmt.Errorf("invalidObjStorePoolConfig: invalid placement %s: %w", p.Name, err)
		}
	}
	return nil
}

func validatePoolPlacementStorageClasses(scList []cephv1.PlacementStorageClassSpec) error {
	names := make(map[string]struct{}, len(scList))
	for _, sc := range scList {
		if sc.Name == defaultPlacementStorageClass {
			return fmt.Errorf("invalid placement StorageClass %q: %q name is reserved", sc.Name, defaultPlacementStorageClass)
		}
		if _, ok := names[sc.Name]; ok {
			return fmt.Errorf("invalid placement StorageClass %q: name must be unique", sc.Name)
		}
		names[sc.Name] = struct{}{}
	}
	return nil
}

func adjustZonePlacementPools(zone map[string]interface{}, spec cephv1.ObjectSharedPoolsSpec) (map[string]interface{}, error) {
	name, err := getObjProperty[string](zone, "name")
	if err != nil {
		return nil, fmt.Errorf("unable to get zone name: %w", err)
	}

	// deep copy source zone
	zone, err = deepCopyJson(zone)
	if err != nil {
		return nil, fmt.Errorf("unable to deep copy config for zone %s: %w", name, err)
	}

	placements, err := getObjProperty[[]interface{}](zone, "placement_pools")
	if err != nil {
		return nil, fmt.Errorf("unable to get pool placements for zone %s: %w", name, err)
	}

	fromSpec := toZonePlacementPools(spec, name)

	inConfig := map[string]struct{}{}
	idxToRemove := map[int]struct{}{}
	for i, p := range placements {
		pObj, ok := p.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("unable to cast pool placement to object for zone %s: %+v", name, p)
		}
		placementID, err := getObjProperty[string](pObj, "key")
		if err != nil {
			return nil, fmt.Errorf("unable to get pool placement name for zone %s: %w", name, err)
		}
		// check if placement should be removed
		if _, inSpec := fromSpec[placementID]; !inSpec {
			if placementID == defaultPlacementCephConfigName {
				// 'default-placement' should always be kept as a workaround for https://tracker.ceph.com/issues/68775.
				// if user specified other placement as default, then copy pool names to 'default-placement' from it:
				if userDefault, inSpec := fromSpec[getDefaultPlacementName(spec)]; inSpec {
					// duplicate user defined default placement under 'default-placement' name in spec to update pools on the next step
					fromSpec[defaultPlacementCephConfigName] = userDefault
				}
			} else {
				// remove placement if it is not in spec
				idxToRemove[i] = struct{}{}
				continue
			}
		}
		// update placement with values from spec:
		if pSpec, inSpec := fromSpec[placementID]; inSpec {
			_, err = updateObjProperty(pObj, pSpec.Val.IndexPool, "val", "index_pool")
			if err != nil {
				return nil, fmt.Errorf("unable to set index pool to pool placement %q for zone %q: %w", placementID, name, err)
			}
			_, err = updateObjProperty(pObj, pSpec.Val.DataExtraPool, "val", "data_extra_pool")
			if err != nil {
				return nil, fmt.Errorf("unable to set data extra pool to pool placement %q for zone %q: %w", placementID, name, err)
			}
			scObj, err := toObj(pSpec.Val.StorageClasses)
			if err != nil {
				return nil, fmt.Errorf("unable convert to pool placement %q storage class for zone %q: %w", placementID, name, err)
			}

			_, err = updateObjProperty(pObj, scObj, "val", "storage_classes")
			if err != nil {
				return nil, fmt.Errorf("unable to set storage classes to pool placement %q for zone %q: %w", placementID, name, err)
			}
			inConfig[placementID] = struct{}{}
		}
	}
	if len(idxToRemove) != 0 {
		// delete placements from slice
		updated := make([]interface{}, 0, len(placements)-len(idxToRemove))
		for i := range placements {
			if _, ok := idxToRemove[i]; ok {
				// remove
				continue
			}
			updated = append(updated, placements[i])
		}
		placements = updated
	}

	// add new placements from spec:
	for placementID, p := range fromSpec {
		if _, ok := inConfig[placementID]; ok {
			// already in config
			continue
		}
		pObj, err := toObj(p)
		if err != nil {
			return nil, fmt.Errorf("unable convert pool placement %q for zone %q: %w", placementID, name, err)
		}
		placements = append(placements, pObj)
	}

	// sort placements array.
	// Reason: 'radosgw-admin zone set --infile' sorts placement_pools by key before storing it in ceph
	// and returns JSON with sorted placement_pools array. So we sort input array for easy comparison with applied JSON.
	sort.Slice(placements, func(i, j int) bool {
		pI, ok := placements[i].(map[string]interface{})
		if !ok {
			return false
		}
		nameI, err := getObjProperty[string](pI, "key")
		if err != nil {
			return false
		}
		pJ, ok := placements[j].(map[string]interface{})
		if !ok {
			return false
		}
		nameJ, err := getObjProperty[string](pJ, "key")
		if err != nil {
			return false
		}
		return nameI < nameJ
	})

	_, err = updateObjProperty(zone, placements, "placement_pools")
	if err != nil {
		return nil, fmt.Errorf("unable to set pool placements for zone %q: %w", name, err)
	}
	return zone, nil
}

func getDefaultPlacementName(spec cephv1.ObjectSharedPoolsSpec) string {
	for _, p := range spec.PoolPlacements {
		if p.Default {
			return p.Name
		}
	}
	return defaultPlacementCephConfigName
}

func getDefaultMetadataPool(spec cephv1.ObjectSharedPoolsSpec) string {
	for _, p := range spec.PoolPlacements {
		if p.Default {
			return p.MetadataPoolName
		}
	}
	return spec.MetadataPoolName
}

// toZonePlacementPools converts pool placement CRD definition to zone config json format structures
func toZonePlacementPools(spec cephv1.ObjectSharedPoolsSpec, ns string) map[string]ZonePlacementPool {
	res := make(map[string]ZonePlacementPool, len(spec.PoolPlacements)+1)
	// map sharedPools if presented:
	if spec.DataPoolName != "" && spec.MetadataPoolName != "" {
		res[defaultPlacementCephConfigName] = ZonePlacementPool{
			Key: defaultPlacementCephConfigName,
			Val: ZonePlacementPoolVal{
				// The extra pool is for omap data for multi-part uploads, so we use
				// the metadata pool instead of the data pool.
				DataExtraPool: spec.MetadataPoolName + ":" + ns + ".buckets.non-ec",
				IndexPool:     spec.MetadataPoolName + ":" + ns + ".buckets.index",
				StorageClasses: map[string]ZonePlacementStorageClass{
					defaultPlacementStorageClass: {
						DataPool: spec.DataPoolName + ":" + ns + ".buckets.data",
					},
				},
				// Workaround: radosgw-admin set zone json command sets incorrect default value for placement inline_data field.
				// So we should set default value (true) explicitly.
				// See: https://tracker.ceph.com/issues/67933
				InlineData: true,
			},
		}
	}
	// map pool placements:
	for _, pp := range spec.PoolPlacements {
		res[pp.Name] = toZonePlacementPool(pp, ns)
	}
	return res
}

func toZonePlacementPool(spec cephv1.PoolPlacementSpec, ns string) ZonePlacementPool {
	placementNS := ns + "." + spec.Name
	// The extra pool is for omap data for multi-part uploads, so we use
	// the metadata pool instead of the data pool.
	nonECPool := spec.MetadataPoolName + ":" + placementNS + ".data.non-ec"
	if spec.DataNonECPoolName != "" {
		nonECPool = spec.DataNonECPoolName + ":" + placementNS + ".data.non-ec"
	}

	res := ZonePlacementPool{
		Key: spec.Name,
		Val: ZonePlacementPoolVal{
			DataExtraPool: nonECPool,
			IndexPool:     spec.MetadataPoolName + ":" + placementNS + ".index",
			StorageClasses: map[string]ZonePlacementStorageClass{
				defaultPlacementStorageClass: {
					DataPool: spec.DataPoolName + ":" + placementNS + ".data",
				},
			},
			// Workaround: 'radosgw-admin set zone json' command sets incorrect default value for placement inline_data field.
			// So we should set default value (true) explicitly.
			// See: https://tracker.ceph.com/issues/67933
			InlineData: true,
		},
	}
	for _, v := range spec.StorageClasses {
		res.Val.StorageClasses[v.Name] = ZonePlacementStorageClass{
			DataPool: v.DataPoolName + ":" + ns + "." + v.Name,
		}
	}
	return res
}

func adjustZoneGroupPlacementTargets(group, zone map[string]interface{}, defaultPlacement string) (map[string]interface{}, error) {
	name, err := getObjProperty[string](group, "name")
	if err != nil {
		return nil, fmt.Errorf("unable to get zonegroup name: %w", err)
	}

	// deep copy source group
	group, err = deepCopyJson(group)
	if err != nil {
		return nil, fmt.Errorf("unable to deep copy config for zonegroup %s: %w", name, err)
	}

	_, err = updateObjProperty(group, defaultPlacement, "default_placement")
	if err != nil {
		return nil, fmt.Errorf("unable to set default_placement for zonegroup %s: %w", name, err)
	}

	desiredTargets, err := createPlacementTargetsFromZonePoolPlacements(zone)
	if err != nil {
		return nil, fmt.Errorf("unable to create targets from placements for zonegroup %q: %w", name, err)
	}
	currentTargets, err := getObjProperty[[]interface{}](group, "placement_targets")
	if err != nil {
		return nil, fmt.Errorf("unable to get targets from placements for zonegroup %q: %w", name, err)
	}

	applied := map[string]struct{}{}
	idxToRemove := map[int]struct{}{}
	for i, target := range currentTargets {
		tObj, ok := target.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("unable to cast placement target to object for zonegroup %q: %+v", name, target)
		}
		tName, err := getObjProperty[string](tObj, "name")
		if err != nil {
			return nil, fmt.Errorf("unable to get placement target name for zonegroup %q: %w", name, err)
		}
		// update target:
		if desired, ok := desiredTargets[tName]; ok {
			sc := []interface{}{}
			ok = castJson(desired.StorageClasses, &sc)
			if ok {
				_, err = updateObjProperty(tObj, sc, "storage_classes")
			} else {
				_, err = updateObjProperty(tObj, desired.StorageClasses, "storage_classes")
			}
			if err != nil {
				return nil, fmt.Errorf("unable to set storage classes to pool placement target %q for zonegroup %q: %w", tName, name, err)
			}
			applied[tName] = struct{}{}
		} else {
			// remove target
			idxToRemove[i] = struct{}{}
			continue
		}
	}
	if len(idxToRemove) != 0 {
		// delete targets from slice
		updated := make([]interface{}, 0, len(currentTargets)-len(idxToRemove))
		for i := range currentTargets {
			if _, ok := idxToRemove[i]; ok {
				// remove
				continue
			}
			updated = append(updated, currentTargets[i])
		}
		currentTargets = updated
	}

	// add new targets:
	for targetName, target := range desiredTargets {
		if _, ok := applied[targetName]; ok {
			// already in config
			continue
		}
		tObj, err := toObj(target)
		if err != nil {
			return nil, fmt.Errorf("unable convert placement target %q for zonegroup %q: %w", targetName, name, err)
		}
		currentTargets = append(currentTargets, tObj)
	}

	_, err = updateObjProperty(group, currentTargets, "placement_targets")
	if err != nil {
		return nil, fmt.Errorf("unable to set placement targets for zonegroup %q: %w", name, err)
	}

	return group, nil
}

func createPlacementTargetsFromZonePoolPlacements(zone map[string]interface{}) (map[string]ZonegroupPlacementTarget, error) {
	zoneName, err := getObjProperty[string](zone, "name")
	if err != nil {
		return nil, fmt.Errorf("unable to get zone name: %w", err)
	}

	zonePoolPlacements, err := getObjProperty[[]interface{}](zone, "placement_pools")
	if err != nil {
		return nil, fmt.Errorf("unable to get pool placements for zone %q: %w", zoneName, err)
	}

	res := make(map[string]ZonegroupPlacementTarget, len(zonePoolPlacements))
	for _, pp := range zonePoolPlacements {
		ppObj, ok := pp.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("unable to cast zone pool placement to json obj for zone %q: %+v", zoneName, pp)
		}
		name, err := getObjProperty[string](ppObj, "key")
		if err != nil {
			return nil, fmt.Errorf("unable to get pool placement key for zone %q: %w", zoneName, err)
		}
		storClasses, err := getObjProperty[map[string]interface{}](ppObj, "val", "storage_classes")
		if err != nil {
			return nil, fmt.Errorf("unable to get pool placement storage classes for zone %q: %w", zoneName, err)
		}
		target := ZonegroupPlacementTarget{
			Name: name,
		}
		for sc := range storClasses {
			target.StorageClasses = append(target.StorageClasses, sc)
		}
		sort.Strings(target.StorageClasses)
		res[name] = target
	}
	return res, nil
}

func getZoneJSON(objContext *Context) (map[string]interface{}, error) {
	if objContext.Realm == "" {
		return nil, fmt.Errorf("get zone: object store realm is missing from context")
	}
	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)

	if objContext.Zone == "" {
		return nil, fmt.Errorf("get zone: object store zone is missing from context")
	}
	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)

	log.NamedDebug(objContext.NsName(), logger, "get zone: rgw-realm=%s, rgw-zone=%s", objContext.Realm, objContext.Zone)

	jsonStr, err := RunAdminCommandNoMultisite(objContext, true, "zone", "get", realmArg, zoneArg)
	if err != nil {
		// This handles the case where the pod we use to exec command (act as a proxy) is not found/ready yet
		// The caller can nicely handle the error and not overflow the op logs with misleading error messages
		if kerrors.IsNotFound(err) {
			return nil, err
		}
		return nil, errors.Wrap(err, "failed to get rgw zone group")
	}
	log.NamedDebug(objContext.NsName(), logger, "get zone success: rgw-realm=%s, rgw-zone=%s, res=%s", objContext.Realm, objContext.Zone, jsonStr)
	res := map[string]interface{}{}
	return res, json.Unmarshal([]byte(jsonStr), &res)
}

func getZoneGroupJSON(objContext *Context) (map[string]interface{}, error) {
	if objContext.Realm == "" {
		return nil, fmt.Errorf("get zonegroup: object store realm is missing from context")
	}
	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)

	if objContext.Zone == "" {
		return nil, fmt.Errorf("get zonegroup: object store zone is missing from context")
	}
	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)

	if objContext.ZoneGroup == "" {
		return nil, fmt.Errorf("get zonegroup: object store zonegroup is missing from context")
	}
	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)

	log.NamedDebug(objContext.NsName(), logger, "get zonegroup: rgw-realm=%s, rgw-zone=%s, rgw-zonegroup=%s", objContext.Realm, objContext.Zone, objContext.ZoneGroup)
	jsonStr, err := RunAdminCommandNoMultisite(objContext, true, "zonegroup", "get", realmArg, zoneGroupArg, zoneArg)
	if err != nil {
		// This handles the case where the pod we use to exec command (act as a proxy) is not found/ready yet
		// The caller can nicely handle the error and not overflow the op logs with misleading error messages
		if kerrors.IsNotFound(err) {
			return nil, err
		}
		return nil, errors.Wrap(err, "failed to get rgw zone group")
	}
	log.NamedDebug(objContext.NsName(), logger, "get zonegroup success: rgw-realm=%s, rgw-zone=%s, rgw-zonegroup=%s, res=%s", objContext.Realm, objContext.Zone, objContext.ZoneGroup, jsonStr)
	res := map[string]interface{}{}
	return res, json.Unmarshal([]byte(jsonStr), &res)
}

func updateZoneJSON(objContext *Context, zone map[string]interface{}) (map[string]interface{}, error) {
	if objContext.Realm == "" {
		return nil, fmt.Errorf("update zone: object store realm is missing from context")
	}
	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)

	if objContext.Zone == "" {
		return nil, fmt.Errorf("update zone: object store zone is missing from context")
	}
	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)

	configBytes, err := json.Marshal(zone)
	if err != nil {
		return nil, err
	}
	configFilename := path.Join(objContext.Context.ConfigDir, objContext.Name+".zonecfg")
	if err := os.WriteFile(configFilename, configBytes, 0o600); err != nil {
		return nil, errors.Wrap(err, "failed to write zone config file")
	}
	defer os.Remove(configFilename)

	args := []string{"zone", "set", zoneArg, "--infile=" + configFilename, realmArg}
	updatedBytes, err := RunAdminCommandNoMultisite(objContext, false, args...)
	if err != nil {
		return nil, errors.Wrap(err, "failed to set zone config")
	}
	log.NamedDebug(objContext.NsName(), logger, "update zone: %s json config updated value from %q to %q", objContext.Zone, string(configBytes), string(updatedBytes))
	updated := map[string]interface{}{}
	err = json.Unmarshal([]byte(updatedBytes), &updated)
	return updated, err
}

func updateZoneGroupJSON(objContext *Context, group map[string]interface{}) (map[string]interface{}, error) {
	if objContext.Realm == "" {
		return nil, fmt.Errorf("update zonegroup: object store realm is missing from context")
	}
	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)

	if objContext.Zone == "" {
		return nil, fmt.Errorf("update zonegroup: object store zone is missing from context")
	}
	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)

	if objContext.ZoneGroup == "" {
		return nil, fmt.Errorf("update zonegroup: object store zonegroup is missing from context")
	}
	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)

	configBytes, err := json.Marshal(group)
	if err != nil {
		return nil, err
	}
	configFilename := path.Join(objContext.Context.ConfigDir, objContext.Name+".zonegroupcfg")
	if err := os.WriteFile(configFilename, configBytes, 0o600); err != nil {
		return nil, errors.Wrap(err, "failed to write zonegroup config file")
	}
	defer os.Remove(configFilename)

	args := []string{"zonegroup", "set", zoneArg, "--infile=" + configFilename, realmArg, zoneGroupArg}
	updatedBytes, err := RunAdminCommandNoMultisite(objContext, false, args...)
	if err != nil {
		return nil, errors.Wrap(err, fmt.Sprintf("failed to set zonegroup config %s", updatedBytes))
	}
	updated := map[string]interface{}{}
	err = json.Unmarshal([]byte(updatedBytes), &updated)
	return updated, err
}

type ZonegroupPlacementTarget struct {
	Name           string   `json:"name"`
	StorageClasses []string `json:"storage_classes"`
}

type ZonePlacementPool struct {
	Key string               `json:"key"`
	Val ZonePlacementPoolVal `json:"val"`
}

type ZonePlacementPoolVal struct {
	DataExtraPool  string                               `json:"data_extra_pool"`
	IndexPool      string                               `json:"index_pool"`
	StorageClasses map[string]ZonePlacementStorageClass `json:"storage_classes"`
	InlineData     bool                                 `json:"inline_data"`
}

type ZonePlacementStorageClass struct {
	DataPool string `json:"data_pool"`
}