my-rook-config/pkg/operator/ceph/csi/peermap/config.go

/*
Copyright 2021 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package peermap

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"strconv"

	"github.com/coreos/pkg/capnslog"
	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/pkg/util"
	"github.com/rook/rook/pkg/util/exec"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
)

const (
	mappingConfigName = "rook-ceph-csi-mapping-config"
	mappingConfigkey  = "csi-mapping-config-json"
)

var logger = capnslog.NewPackageLogger("github.com/rook/rook", "peer-map")

var peerPoolTempFile = "peerPool"

type PeerIDMapping struct {
	ClusterIDMapping map[string]string
	RBDPoolIDMapping []map[string]string
}

type PeerIDMappings []PeerIDMapping

// addClusterIDMapping adds cluster ID map if not present already
func (m *PeerIDMappings) addClusterIDMapping(newClusterIDMap map[string]string) {
	if m.clusterIDMapIndex(newClusterIDMap) == -1 {
		newDRMap := PeerIDMapping{
			ClusterIDMapping: newClusterIDMap,
		}
		*m = append(*m, newDRMap)
	}
}

// addRBDPoolIDMapping adds all the pool ID maps for a given cluster ID map
func (m *PeerIDMappings) addRBDPoolIDMapping(clusterIDMap, newPoolIDMap map[string]string) {
	for i := 0; i < len(*m); i++ {
		if reflect.DeepEqual((*m)[i].ClusterIDMapping, clusterIDMap) {
			(*m)[i].RBDPoolIDMapping = append((*m)[i].RBDPoolIDMapping, newPoolIDMap)
		}
	}
}

// updateRBDPoolIDMapping updates the Pool ID mappings between local and peer cluster.
// It adds the cluster and Pool ID mappings if not present, else updates the pool ID map if required.
func (m *PeerIDMappings) updateRBDPoolIDMapping(newMappings PeerIDMapping) {
	newClusterIDMap := newMappings.ClusterIDMapping
	newPoolIDMap := newMappings.RBDPoolIDMapping[0]
	peerPoolID, localPoolID := getMapKV(newPoolIDMap)

	// Append new mappings if no existing mappings are available
	if len(*m) == 0 {
		*m = append(*m, newMappings)
		return
	}
	clusterIDMapExists := false
	for i := 0; i < len(*m); i++ {
		if reflect.DeepEqual((*m)[i].ClusterIDMapping, newClusterIDMap) {
			clusterIDMapExists = true
			poolIDMapUpdated := false
			for j := 0; j < len((*m)[i].RBDPoolIDMapping); j++ {
				existingPoolMap := (*m)[i].RBDPoolIDMapping[j]
				if _, ok := existingPoolMap[peerPoolID]; ok {
					poolIDMapUpdated = true
					existingPoolMap[peerPoolID] = localPoolID
				}
			}
			if !poolIDMapUpdated {
				(*m)[i].RBDPoolIDMapping = append((*m)[i].RBDPoolIDMapping, newPoolIDMap)
			}
		}
	}

	if !clusterIDMapExists {
		*m = append(*m, newMappings)
	}
}

func (m *PeerIDMappings) clusterIDMapIndex(newClusterIDMap map[string]string) int {
	for i, mapping := range *m {
		if reflect.DeepEqual(mapping.ClusterIDMapping, newClusterIDMap) {
			return i
		}
	}
	return -1
}

func (m *PeerIDMappings) String() (string, error) {
	mappingInBytes, err := json.Marshal(m)
	if err != nil {
		return "", errors.Wrap(err, "failed to marshal peer cluster mapping config")
	}

	return string(mappingInBytes), nil
}

func toObj(in string) (PeerIDMappings, error) {
	var mappings PeerIDMappings
	err := json.Unmarshal([]byte(in), &mappings)
	if err != nil {
		return mappings, errors.Wrap(err, "failed to unmarshal peer cluster mapping config")
	}

	return mappings, nil
}

func ReconcilePoolIDMap(ctx context.Context, clusterContext *clusterd.Context, clusterInfo *cephclient.ClusterInfo, pool *cephv1.CephBlockPool) error {
	if pool.Spec.Mirroring.Peers == nil {
		logger.Infof("no peer secrets added in ceph block pool %q. skipping pool ID mappings with peer cluster", pool.Name)
		return nil
	}

	mappings, err := getClusterPoolIDMap(clusterContext, clusterInfo, pool)
	if err != nil {
		return errors.Wrapf(err, "failed to get peer pool ID mappings for the pool %q", pool.Name)
	}

	err = CreateOrUpdateConfig(ctx, clusterContext, mappings)
	if err != nil {
		return errors.Wrapf(err, "failed to create or update peer pool ID mappings configMap for the pool %q", pool.Name)
	}

	logger.Infof("successfully updated config map with cluster and RDB pool ID mappings for the pool %q", pool.Name)
	return nil
}

// getClusterPoolIDMap returns a mapping between local and peer cluster ID, and between local and peer pool ID
func getClusterPoolIDMap(clusterContext *clusterd.Context, clusterInfo *cephclient.ClusterInfo, pool *cephv1.CephBlockPool) (*PeerIDMappings, error) {
	mappings := &PeerIDMappings{}

	// Get local cluster pool details
	localPoolDetails, err := cephclient.GetPoolDetails(clusterContext, clusterInfo, pool.Name)
	if err != nil {
		return mappings, errors.Wrapf(err, "failed to get details for the pool %q", pool.Name)
	}

	logger.Debugf("pool details of local cluster %+v", localPoolDetails)

	for _, peerSecret := range pool.Spec.Mirroring.Peers.SecretNames {
		s, err := clusterContext.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(clusterInfo.Context, peerSecret, metav1.GetOptions{})
		if err != nil {
			return mappings, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret)
		}

		token := s.Data["token"]
		decodedTokenToGo, err := decodePeerToken(string(token))
		if err != nil {
			return mappings, errors.Wrap(err, "failed to decode bootstrap peer token")
		}

		peerClientName := fmt.Sprintf("client.%s", decodedTokenToGo.ClientID)
		credentials := cephclient.CephCred{
			Username: peerClientName,
			Secret:   decodedTokenToGo.Key,
		}

		// Add cluster ID mappings
		clusterIDMapping := map[string]string{
			decodedTokenToGo.Namespace: clusterInfo.Namespace,
		}

		mappings.addClusterIDMapping(clusterIDMapping)

		// Generate peer cluster keyring in a temporary file
		keyring := cephclient.CephKeyring(credentials)
		keyringFile, err := util.CreateTempFile(keyring)
		if err != nil {
			return mappings, errors.Wrap(err, "failed to create a temp keyring file")
		}
		defer os.Remove(keyringFile.Name())

		// Generate an empty config file to be passed as `--conf`argument in ceph CLI
		configFile, err := util.CreateTempFile("")
		if err != nil {
			return mappings, errors.Wrap(err, "failed to create a temp config file")
		}
		defer os.Remove(configFile.Name())

		// Build command
		args := []string{
			"osd", "pool", "get", pool.Name, "all",
			fmt.Sprintf("--cluster=%s", decodedTokenToGo.Namespace),
			fmt.Sprintf("--conf=%s", configFile.Name()),
			fmt.Sprintf("--fsid=%s", decodedTokenToGo.ClusterFSID),
			fmt.Sprintf("--mon-host=%s", decodedTokenToGo.MonHost),
			fmt.Sprintf("--keyring=%s", keyringFile.Name()),
			fmt.Sprintf("--name=%s", peerClientName),
			"--format", "json",
		}

		// Get peer cluster pool details
		peerPoolDetails, err := getPeerPoolDetails(clusterContext, args...)
		if err != nil {
			return mappings, errors.Wrapf(err, "failed to get pool details from peer cluster %q", decodedTokenToGo.Namespace)
		}

		logger.Debugf("pool details from peer cluster %+v", peerPoolDetails)

		// Add Pool ID mappings
		poolIDMapping := map[string]string{
			strconv.Itoa(peerPoolDetails.Number): strconv.Itoa(localPoolDetails.Number),
		}
		mappings.addRBDPoolIDMapping(clusterIDMapping, poolIDMapping)
	}

	return mappings, nil
}

func CreateOrUpdateConfig(ctx context.Context, clusterContext *clusterd.Context, mappings *PeerIDMappings) error {
	data, err := mappings.String()
	if err != nil {
		return errors.Wrap(err, "failed to convert peer cluster mappings struct to string")
	}

	opNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
	request := types.NamespacedName{Name: mappingConfigName, Namespace: opNamespace}
	existingConfigMap := &v1.ConfigMap{}

	err = clusterContext.Client.Get(ctx, request, existingConfigMap)
	if err != nil {
		if kerrors.IsNotFound(err) {
			// Create new configMap
			return createConfig(ctx, clusterContext, request, data)
		}
		return errors.Wrapf(err, "failed to get existing mapping config map %q", existingConfigMap.Name)
	}

	existingCMData := existingConfigMap.Data[mappingConfigkey]
	if existingCMData == "[]" {
		existingConfigMap.Data[mappingConfigkey] = data
	} else {
		existingMappings, err := toObj(existingCMData)
		if err != nil {
			return errors.Wrapf(err, "failed to extract existing mapping data from the config map %q", existingConfigMap.Name)
		}
		updatedCMData, err := UpdateExistingData(&existingMappings, mappings)
		if err != nil {
			return errors.Wrapf(err, "failed to update existing mapping data from the config map %q", existingConfigMap.Name)
		}
		existingConfigMap.Data[mappingConfigkey] = updatedCMData
	}

	// Update existing configMap
	if err := clusterContext.Client.Update(ctx, existingConfigMap); err != nil {
		return errors.Wrapf(err, "failed to update existing mapping config map %q", existingConfigMap.Name)
	}

	return nil
}

func UpdateExistingData(existingMappings, newMappings *PeerIDMappings) (string, error) {
	for i, mapping := range *newMappings {
		if len(mapping.RBDPoolIDMapping) == 0 {
			logger.Warning("no pool ID mapping available between local and peer cluster")
			continue
		}
		existingMappings.updateRBDPoolIDMapping((*newMappings)[i])
	}

	data, err := existingMappings.String()
	if err != nil {
		return "", errors.Wrap(err, "failed to convert peer cluster mappings struct to string")
	}
	return data, nil
}

func createConfig(ctx context.Context, clusterContext *clusterd.Context, request types.NamespacedName, data string) error {
	newConfigMap := &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      request.Name,
			Namespace: request.Namespace,
		},
		Data: map[string]string{
			mappingConfigkey: data,
		},
	}

	// Get Operator owner reference
	operatorPodName := os.Getenv(k8sutil.PodNameEnvVar)
	ownerRef, err := k8sutil.GetDeploymentOwnerReference(ctx, clusterContext.Clientset, operatorPodName, request.Namespace)
	if err != nil {
		return errors.Wrap(err, "failed to get operator owner reference")
	}
	if ownerRef != nil {
		blockOwnerDeletion := false
		ownerRef.BlockOwnerDeletion = &blockOwnerDeletion
	}

	ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, request.Namespace)

	// Set controller reference only when creating the configMap for the first time
	err = ownerInfo.SetControllerReference(newConfigMap)
	if err != nil {
		return errors.Wrapf(err, "failed to set owner reference on configMap %q", newConfigMap.Name)
	}

	err = clusterContext.Client.Create(ctx, newConfigMap)
	if err != nil {
		return errors.Wrapf(err, "failed to create mapping configMap %q", newConfigMap.Name)
	}
	return nil
}

func decodePeerToken(token string) (*cephclient.PeerToken, error) {
	// decode the base64 encoded token
	decodedToken, err := base64.StdEncoding.DecodeString(string(token))
	if err != nil {
		return nil, errors.Wrap(err, "failed to decode bootstrap peer token")
	}

	// Unmarshal the decoded token to a Go type
	var decodedTokenToGo cephclient.PeerToken
	err = json.Unmarshal(decodedToken, &decodedTokenToGo)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal decoded token")
	}

	logger.Debugf("peer cluster info %+v", decodedTokenToGo)

	return &decodedTokenToGo, nil
}

func getPeerPoolDetails(ctx *clusterd.Context, args ...string) (cephclient.CephStoragePoolDetails, error) {
	poolDetailsTempFile, err := os.CreateTemp("", peerPoolTempFile)
	if err != nil {
		return cephclient.CephStoragePoolDetails{}, errors.Wrap(err, "failed to generate temporary file")
	}

	defer func() {
		err := os.Remove(poolDetailsTempFile.Name())
		if err != nil {
			logger.Errorf("failed to remove file %q. %v", poolDetailsTempFile.Name(), err)
		}
	}()

	args = append(args, "--out-file", poolDetailsTempFile.Name())

	_, err = ctx.Executor.ExecuteCommandWithTimeout(exec.CephCommandsTimeout, "ceph", args...)
	if err != nil {
		return cephclient.CephStoragePoolDetails{}, errors.Wrap(err, "failed to get pool details from peer cluster")
	}

	data, err := os.ReadFile(poolDetailsTempFile.Name())
	if err != nil {
		return cephclient.CephStoragePoolDetails{}, errors.Wrapf(err, "failed to read from the temporary json output file %q", poolDetailsTempFile.Name())
	}

	return cephclient.ParsePoolDetails(data)
}

func getMapKV(input map[string]string) (string, string) {
	for k, v := range input {
		return k, v
	}
	return "", ""
}