From 3f8abec403684ee12775b2ee38e52051ad743c51 Mon Sep 17 00:00:00 2001 From: Santosh Pillai Date: Mon, 9 Aug 2021 13:32:02 +0530 Subject: [PATCH] ceph: add ClusterID and PoolID mappings between local and peer cluster During disaster recovery/migration of a cluster, as part of the failover, the kubernetes artifacts like deployment, PVC, PV, etc will be restored to a new cluster by the admin. Even if the kubernetes objects are restored the corresponding RBD/CephFS subvolume cannot be retrieved during CSI operations as the clusterID and poolID are not the same in both clusters This PR creates a mapping between Cluster ID and RBD Pool ID between local cluster and peer cluster. Signed-off-by: Santosh Pillai --- .github/workflows/canary-integration-test.yml | 14 + pkg/daemon/ceph/client/pool.go | 9 +- pkg/operator/ceph/cluster/mgr/dashboard.go | 19 +- pkg/operator/ceph/csi/peermap/config.go | 368 ++++++++++++++ pkg/operator/ceph/csi/peermap/config_test.go | 471 ++++++++++++++++++ .../rbd/csi-rbdplugin-provisioner-dep.yaml | 25 +- .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 23 +- pkg/operator/ceph/object/objectstore.go | 5 +- pkg/operator/ceph/operator.go | 40 +- pkg/operator/ceph/pool/controller.go | 7 + pkg/operator/ceph/pool/controller_test.go | 38 ++ pkg/operator/k8sutil/deployment.go | 28 ++ pkg/util/file.go | 17 + 13 files changed, 996 insertions(+), 68 deletions(-) create mode 100644 pkg/operator/ceph/csi/peermap/config.go create mode 100644 pkg/operator/ceph/csi/peermap/config_test.go diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 964d12e01..1f1607288 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -817,6 +817,20 @@ jobs: kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test + - name: copy block mirror peer secret into the primary cluster for replicapool + run: | + kubectl -n rook-ceph-secondary get secret pool-peer-token-replicapool -o yaml |\ + sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: pool-peer-token-replicapool/name: pool-peer-token-replicapool-config/g' |\ + kubectl create --namespace=rook-ceph -f - + + - name: add block mirror peer secret to the primary cluster for replicapool + run: | + kubectl -n rook-ceph patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}' + + - name: wait for rook-ceph-csi-mapping-config to be updated with cluster ID + run: | + timeout 60 sh -c 'until [ "$(kubectl get cm -n rook-ceph rook-ceph-csi-mapping-config -o jsonpath='{.data.csi-mapping-config-json}' | grep -c "rook-ceph-secondary")" -eq 1 ]; do echo "waiting for rook-ceph-csi-mapping-config to be created with cluster ID mappings" && sleep 1; done' + - name: create replicated mirrored filesystem on cluster 1 run: | PRIMARY_YAML=cluster/examples/kubernetes/ceph/filesystem-test-primary.yaml diff --git a/pkg/daemon/ceph/client/pool.go b/pkg/daemon/ceph/client/pool.go index e1500b0d0..f35a8720f 100644 --- a/pkg/daemon/ceph/client/pool.go +++ b/pkg/daemon/ceph/client/pool.go @@ -124,6 +124,11 @@ func GetPoolDetails(context *clusterd.Context, clusterInfo *ClusterInfo, name st return CephStoragePoolDetails{}, errors.Wrapf(err, "failed to get pool %s details. %s", name, string(output)) } + return ParsePoolDetails(output) +} + +func ParsePoolDetails(in []byte) (CephStoragePoolDetails, error) { + // The response for osd pool get when passing var=all is actually malformed JSON similar to: // {"pool":"rbd","size":1}{"pool":"rbd","min_size":2}... // Note the multiple top level entities, one for each property returned. To workaround this, @@ -132,7 +137,7 @@ func GetPoolDetails(context *clusterd.Context, clusterInfo *ClusterInfo, name st // Since previously set fields remain intact if they are not overwritten, the result is the JSON // unmarshalling of all properties in the response. var poolDetails CephStoragePoolDetails - poolDetailsUnits := strings.Split(string(output), "}{") + poolDetailsUnits := strings.Split(string(in), "}{") for i := range poolDetailsUnits { pdu := poolDetailsUnits[i] if !strings.HasPrefix(pdu, "{") { @@ -143,7 +148,7 @@ func GetPoolDetails(context *clusterd.Context, clusterInfo *ClusterInfo, name st } err := json.Unmarshal([]byte(pdu), &poolDetails) if err != nil { - return CephStoragePoolDetails{}, errors.Wrapf(err, "unmarshal failed raw buffer response %s", string(output)) + return CephStoragePoolDetails{}, errors.Wrapf(err, "unmarshal failed raw buffer response %s", string(in)) } } diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 9c055b6c0..ec340326d 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -21,7 +21,6 @@ import ( "context" "crypto/rand" "fmt" - "io/ioutil" "os" "strconv" "syscall" @@ -31,6 +30,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/exec" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -214,21 +214,6 @@ func FileBasedPasswordSupported(c *client.ClusterInfo) bool { return false } -func CreateTempPasswordFile(password string) (*os.File, error) { - // Generate a temp file - file, err := ioutil.TempFile("", "") - if err != nil { - return nil, errors.Wrap(err, "failed to generate temp file") - } - - // Write password into file - err = ioutil.WriteFile(file.Name(), []byte(password), 0440) - if err != nil { - return nil, errors.Wrap(err, "failed to write dashboard password into file") - } - return file, nil -} - func (c *Cluster) setLoginCredentials(password string) error { // Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log. logger.Infof("setting ceph dashboard %q login creds", dashboardUsername) @@ -237,7 +222,7 @@ func (c *Cluster) setLoginCredentials(password string) error { // for latest Ceph versions if FileBasedPasswordSupported(c.clusterInfo) { // Generate a temp file - file, err := CreateTempPasswordFile(password) + file, err := util.CreateTempFile(password) if err != nil { return errors.Wrap(err, "failed to create a temporary dashboard password file") } diff --git a/pkg/operator/ceph/csi/peermap/config.go b/pkg/operator/ceph/csi/peermap/config.go new file mode 100644 index 000000000..b7a8d6d7d --- /dev/null +++ b/pkg/operator/ceph/csi/peermap/config.go @@ -0,0 +1,368 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package peermap + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "reflect" + "strconv" + + "github.com/coreos/pkg/capnslog" + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/exec" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + mappingConfigName = "rook-ceph-csi-mapping-config" + mappingConfigkey = "csi-mapping-config-json" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "peer-map") + +type PeerIDMapping struct { + ClusterIDMapping map[string]string + RBDPoolIDMapping []map[string]string +} + +type PeerIDMappings []PeerIDMapping + +// addClusterIDMapping adds cluster ID map if not present already +func (m *PeerIDMappings) addClusterIDMapping(newClusterIDMap map[string]string) { + if m.clusterIDMapIndex(newClusterIDMap) == -1 { + newDRMap := PeerIDMapping{ + ClusterIDMapping: newClusterIDMap, + } + *m = append(*m, newDRMap) + } +} + +// addRBDPoolIDMapping adds all the pool ID maps for a given cluster ID map +func (m *PeerIDMappings) addRBDPoolIDMapping(clusterIDMap, newPoolIDMap map[string]string) { + for i := 0; i < len(*m); i++ { + if reflect.DeepEqual((*m)[i].ClusterIDMapping, clusterIDMap) { + (*m)[i].RBDPoolIDMapping = append((*m)[i].RBDPoolIDMapping, newPoolIDMap) + } + } +} + +// updateRBDPoolIDMapping updates the Pool ID mappings between local and peer cluster. +// It adds the cluster and Pool ID mappings if not present, else updates the pool ID map if required. +func (m *PeerIDMappings) updateRBDPoolIDMapping(newMappings PeerIDMapping) { + newClusterIDMap := newMappings.ClusterIDMapping + newPoolIDMap := newMappings.RBDPoolIDMapping[0] + peerPoolID, localPoolID := getMapKV(newPoolIDMap) + + // Append new mappings if no existing mappings are available + if len(*m) == 0 { + *m = append(*m, newMappings) + return + } + clusterIDMapExists := false + for i := 0; i < len(*m); i++ { + if reflect.DeepEqual((*m)[i].ClusterIDMapping, newClusterIDMap) { + clusterIDMapExists = true + poolIDMapUpdated := false + for j := 0; j < len((*m)[i].RBDPoolIDMapping); j++ { + existingPoolMap := (*m)[i].RBDPoolIDMapping[j] + if _, ok := existingPoolMap[peerPoolID]; ok { + poolIDMapUpdated = true + existingPoolMap[peerPoolID] = localPoolID + } + } + if !poolIDMapUpdated { + (*m)[i].RBDPoolIDMapping = append((*m)[i].RBDPoolIDMapping, newPoolIDMap) + } + } + } + + if !clusterIDMapExists { + *m = append(*m, newMappings) + } +} + +func (m *PeerIDMappings) clusterIDMapIndex(newClusterIDMap map[string]string) int { + for i, mapping := range *m { + if reflect.DeepEqual(mapping.ClusterIDMapping, newClusterIDMap) { + return i + } + } + return -1 +} + +func (m *PeerIDMappings) String() (string, error) { + mappingInBytes, err := json.Marshal(m) + if err != nil { + return "", errors.Wrap(err, "failed to marshal peer cluster mapping config") + } + + return string(mappingInBytes), nil +} + +func toObj(in string) (PeerIDMappings, error) { + var mappings PeerIDMappings + err := json.Unmarshal([]byte(in), &mappings) + if err != nil { + return mappings, errors.Wrap(err, "failed to unmarshal peer cluster mapping config") + } + + return mappings, nil +} + +func ReconcilePoolIDMap(clusterContext *clusterd.Context, clusterInfo *cephclient.ClusterInfo, pool *cephv1.CephBlockPool) error { + if pool.Spec.Mirroring.Peers == nil { + logger.Infof("no peer secrets added in ceph block pool %q. skipping pool ID mappings with peer cluster", pool.Name) + return nil + } + + mappings, err := getClusterPoolIDMap(clusterContext, clusterInfo, pool) + if err != nil { + return errors.Wrapf(err, "failed to get peer pool ID mappings for the pool %q", pool.Name) + } + + err = CreateOrUpdateConfig(clusterContext, mappings) + if err != nil { + return errors.Wrapf(err, "failed to create or update peer pool ID mappings configMap for the pool %q", pool.Name) + } + + logger.Infof("successfully updated config map with cluster and RDB pool ID mappings for the pool %q", pool.Name) + return nil +} + +// getClusterPoolIDMap returns a mapping between local and peer cluster ID, and between local and peer pool ID +func getClusterPoolIDMap(clusterContext *clusterd.Context, clusterInfo *cephclient.ClusterInfo, pool *cephv1.CephBlockPool) (*PeerIDMappings, error) { + mappings := &PeerIDMappings{} + + // Get local cluster pool details + localPoolDetails, err := cephclient.GetPoolDetails(clusterContext, clusterInfo, pool.Name) + if err != nil { + return mappings, errors.Wrapf(err, "failed to get details for the pool %q", pool.Name) + } + + logger.Debugf("pool details of local cluster %+v", localPoolDetails) + + for _, peerSecret := range pool.Spec.Mirroring.Peers.SecretNames { + s, err := clusterContext.Clientset.CoreV1().Secrets(clusterInfo.Namespace).Get(context.TODO(), peerSecret, metav1.GetOptions{}) + if err != nil { + return mappings, errors.Wrapf(err, "failed to fetch kubernetes secret %q bootstrap peer", peerSecret) + } + + token := s.Data["token"] + decodedTokenToGo, err := decodePeerToken(string(token)) + if err != nil { + return mappings, errors.Wrap(err, "failed to decode bootstrap peer token") + } + + peerClientName := fmt.Sprintf("client.%s", decodedTokenToGo.ClientID) + credentials := cephclient.CephCred{ + Username: peerClientName, + Secret: decodedTokenToGo.Key, + } + + // Add cluster ID mappings + clusterIDMapping := map[string]string{ + decodedTokenToGo.Namespace: clusterInfo.Namespace, + } + + mappings.addClusterIDMapping(clusterIDMapping) + + // Generate peer cluster keyring in a temporary file + keyring := cephclient.CephKeyring(credentials) + keyringFile, err := util.CreateTempFile(keyring) + if err != nil { + return mappings, errors.Wrap(err, "failed to create a temp keyring file") + } + defer os.Remove(keyringFile.Name()) + + // Generate an empty config file to be passed as `--conf`argument in ceph CLI + configFile, err := util.CreateTempFile("") + if err != nil { + return mappings, errors.Wrap(err, "failed to create a temp config file") + } + defer os.Remove(configFile.Name()) + + // Build command + args := []string{"osd", "pool", "get", pool.Name, "all", + fmt.Sprintf("--cluster=%s", decodedTokenToGo.Namespace), + fmt.Sprintf("--conf=%s", configFile.Name()), + fmt.Sprintf("--fsid=%s", decodedTokenToGo.ClusterFSID), + fmt.Sprintf("--mon-host=%s", decodedTokenToGo.MonHost), + fmt.Sprintf("--keyring=%s", keyringFile.Name()), + fmt.Sprintf("--name=%s", peerClientName), + "--format", "json", + } + + // Get peer cluster pool details + peerPoolDetails, err := getPeerPoolDetails(clusterContext, args...) + if err != nil { + return mappings, errors.Wrapf(err, "failed to get pool details from peer cluster %q", decodedTokenToGo.Namespace) + } + + logger.Debugf("pool details from peer cluster %+v", peerPoolDetails) + + // Add Pool ID mappings + poolIDMapping := map[string]string{ + strconv.Itoa(peerPoolDetails.Number): strconv.Itoa(localPoolDetails.Number), + } + mappings.addRBDPoolIDMapping(clusterIDMapping, poolIDMapping) + } + + return mappings, nil +} + +func CreateOrUpdateConfig(clusterContext *clusterd.Context, mappings *PeerIDMappings) error { + ctx := context.TODO() + data, err := mappings.String() + if err != nil { + return errors.Wrap(err, "failed to convert peer cluster mappings struct to string") + } + + opNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) + request := types.NamespacedName{Name: mappingConfigName, Namespace: opNamespace} + existingConfigMap := &v1.ConfigMap{} + + err = clusterContext.Client.Get(ctx, request, existingConfigMap) + if err != nil { + if kerrors.IsNotFound(err) { + // Create new configMap + return createConfig(clusterContext, request, data) + } + return errors.Wrapf(err, "failed to get existing mapping config map %q", existingConfigMap.Name) + } + + existingCMData := existingConfigMap.Data[mappingConfigkey] + if existingCMData == "[]" { + existingConfigMap.Data[mappingConfigkey] = data + } else { + existingMappings, err := toObj(existingCMData) + if err != nil { + return errors.Wrapf(err, "failed to extract existing mapping data from the config map %q", existingConfigMap.Name) + } + updatedCMData, err := UpdateExistingData(&existingMappings, mappings) + if err != nil { + return errors.Wrapf(err, "failed to update existing mapping data from the config map %q", existingConfigMap.Name) + } + existingConfigMap.Data[mappingConfigkey] = updatedCMData + } + + // Update existing configMap + if err := clusterContext.Client.Update(ctx, existingConfigMap); err != nil { + return errors.Wrapf(err, "failed to update existing mapping config map %q", existingConfigMap.Name) + } + + return nil +} + +func UpdateExistingData(existingMappings, newMappings *PeerIDMappings) (string, error) { + for i, mapping := range *newMappings { + if len(mapping.RBDPoolIDMapping) == 0 { + logger.Warning("no pool ID mapping available between local and peer cluster") + continue + } + existingMappings.updateRBDPoolIDMapping((*newMappings)[i]) + } + + data, err := existingMappings.String() + if err != nil { + return "", errors.Wrap(err, "failed to convert peer cluster mappings struct to string") + } + return data, nil +} + +func createConfig(clusterContext *clusterd.Context, request types.NamespacedName, data string) error { + newConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.Name, + Namespace: request.Namespace, + }, + Data: map[string]string{ + mappingConfigkey: data, + }, + } + + // Get Operator owner reference + operatorPodName := os.Getenv(k8sutil.PodNameEnvVar) + ownerRef, err := k8sutil.GetDeploymentOwnerReference(clusterContext.Clientset, operatorPodName, request.Namespace) + if err != nil { + return errors.Wrap(err, "failed to get operator owner reference") + } + if ownerRef != nil { + blockOwnerDeletion := false + ownerRef.BlockOwnerDeletion = &blockOwnerDeletion + } + + ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, request.Namespace) + + // Set controller reference only when creating the configMap for the first time + err = ownerInfo.SetControllerReference(newConfigMap) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference on configMap %q", newConfigMap.Name) + } + + err = clusterContext.Client.Create(context.TODO(), newConfigMap) + if err != nil { + return errors.Wrapf(err, "failed to create mapping configMap %q", newConfigMap.Name) + } + return nil +} + +func decodePeerToken(token string) (*cephclient.PeerToken, error) { + // decode the base64 encoded token + decodedToken, err := base64.StdEncoding.DecodeString(string(token)) + if err != nil { + return nil, errors.Wrap(err, "failed to decode bootstrap peer token") + } + + // Unmarshal the decoded token to a Go type + var decodedTokenToGo cephclient.PeerToken + err = json.Unmarshal(decodedToken, &decodedTokenToGo) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal decoded token") + } + + logger.Debugf("peer cluster info %+v", decodedTokenToGo) + + return &decodedTokenToGo, nil +} + +func getPeerPoolDetails(ctx *clusterd.Context, args ...string) (cephclient.CephStoragePoolDetails, error) { + peerPoolDetails, err := ctx.Executor.ExecuteCommandWithTimeout(exec.CephCommandsTimeout, "ceph", args...) + if err != nil { + return cephclient.CephStoragePoolDetails{}, errors.Wrap(err, "failed to get pool details from peer cluster") + } + + return cephclient.ParsePoolDetails([]byte(peerPoolDetails)) +} + +func getMapKV(input map[string]string) (string, string) { + for k, v := range input { + return k, v + } + return "", "" +} diff --git a/pkg/operator/ceph/csi/peermap/config_test.go b/pkg/operator/ceph/csi/peermap/config_test.go new file mode 100644 index 000000000..0b1d4fdb7 --- /dev/null +++ b/pkg/operator/ceph/csi/peermap/config_test.go @@ -0,0 +1,471 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package peermap + +import ( + "context" + "os" + "reflect" + "testing" + "time" + + "strings" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestAddClusterIDMapping(t *testing.T) { + clusterMap1 := map[string]string{"cluster-1": "cluster-2"} + m := &PeerIDMappings{} + m.addClusterIDMapping(clusterMap1) + assert.Equal(t, 1, len(*m)) + + // Add same cluster map again and assert that it didn't get added. + m.addClusterIDMapping(clusterMap1) + assert.Equal(t, 1, len(*m)) + + // Add new cluster map and assert that it got added. + clusterMap2 := map[string]string{"cluster-1": "cluster-3"} + m.addClusterIDMapping(clusterMap2) + assert.Equal(t, 2, len(*m)) +} + +func TestUpdateClusterPoolIDMap(t *testing.T) { + m := &PeerIDMappings{} + + // Ensure only local:peer-1 mapping should be present + newMappings := PeerIDMapping{ + ClusterIDMapping: map[string]string{"local": "peer-1"}, + RBDPoolIDMapping: []map[string]string{{"1": "2"}}, + } + m.updateRBDPoolIDMapping(newMappings) + assert.Equal(t, len(*m), 1) + assert.Equal(t, (*m)[0].ClusterIDMapping["local"], "peer-1") + assert.Equal(t, len((*m)[0].RBDPoolIDMapping), 1) + assert.Equal(t, (*m)[0].RBDPoolIDMapping[0]["1"], "2") + + // Ensure RBD pool ID mappings get updated + newMappings = PeerIDMapping{ + ClusterIDMapping: map[string]string{"local": "peer-1"}, + RBDPoolIDMapping: []map[string]string{{"1": "3"}}, + } + m.updateRBDPoolIDMapping(newMappings) + assert.Equal(t, len(*m), 1) + assert.Equal(t, (*m)[0].ClusterIDMapping["local"], "peer-1") + assert.Equal(t, len((*m)[0].RBDPoolIDMapping), 1) + assert.Equal(t, (*m)[0].RBDPoolIDMapping[0]["1"], "3") + + // Ensure that new pool ID mappings got added + newMappings = PeerIDMapping{ + ClusterIDMapping: map[string]string{"local": "peer-1"}, + RBDPoolIDMapping: []map[string]string{{"2": "4"}}, + } + m.updateRBDPoolIDMapping(newMappings) + assert.Equal(t, len(*m), 1) + assert.Equal(t, (*m)[0].ClusterIDMapping["local"], "peer-1") + assert.Equal(t, len((*m)[0].RBDPoolIDMapping), 2) + assert.Equal(t, (*m)[0].RBDPoolIDMapping[0]["1"], "3") + assert.Equal(t, (*m)[0].RBDPoolIDMapping[1]["2"], "4") + + // Ensure that new pool ID mappings got added + newMappings = PeerIDMapping{ + ClusterIDMapping: map[string]string{"local": "peer-1"}, + RBDPoolIDMapping: []map[string]string{{"3": "5"}}, + } + m.updateRBDPoolIDMapping(newMappings) + assert.Equal(t, len(*m), 1) + assert.Equal(t, (*m)[0].ClusterIDMapping["local"], "peer-1") + assert.Equal(t, len((*m)[0].RBDPoolIDMapping), 3) + assert.Equal(t, (*m)[0].RBDPoolIDMapping[0]["1"], "3") + assert.Equal(t, (*m)[0].RBDPoolIDMapping[1]["2"], "4") + assert.Equal(t, (*m)[0].RBDPoolIDMapping[2]["3"], "5") + + // Ensure that new Cluster ID mappings got added + newMappings = PeerIDMapping{ + ClusterIDMapping: map[string]string{"local": "peer-2"}, + RBDPoolIDMapping: []map[string]string{{"1": "3"}}, + } + m.updateRBDPoolIDMapping(newMappings) + assert.Equal(t, len(*m), 2) + assert.Equal(t, (*m)[0].ClusterIDMapping["local"], "peer-1") + assert.Equal(t, len((*m)[0].RBDPoolIDMapping), 3) + assert.Equal(t, (*m)[0].RBDPoolIDMapping[0]["1"], "3") + assert.Equal(t, (*m)[0].RBDPoolIDMapping[1]["2"], "4") + assert.Equal(t, (*m)[0].RBDPoolIDMapping[2]["3"], "5") + + assert.Equal(t, (*m)[1].ClusterIDMapping["local"], "peer-2") + assert.Equal(t, len((*m)[1].RBDPoolIDMapping), 1) + assert.Equal(t, (*m)[0].RBDPoolIDMapping[0]["1"], "3") +} + +func TestAddPoolIDMapping(t *testing.T) { + clusterMap1 := map[string]string{"cluster-1": "cluster-2"} + m := &PeerIDMappings{} + m.addClusterIDMapping(clusterMap1) + assert.Equal(t, 1, len(*m)) + + // Add two Pool ID mapping + poolIDMap1 := map[string]string{"1": "2"} + poolIDMap2 := map[string]string{"2": "3"} + + m.addRBDPoolIDMapping(clusterMap1, poolIDMap1) + m.addRBDPoolIDMapping(clusterMap1, poolIDMap2) + + assert.Equal(t, 2, len((*m)[0].RBDPoolIDMapping)) + + // Add another cluster ID mapping + clusterMap2 := map[string]string{"cluster-1": "cluster-3"} + m.addClusterIDMapping(clusterMap2) + + // Add one Pool ID mapping + poolIDMap3 := map[string]string{"2": "4"} + m.addRBDPoolIDMapping(clusterMap2, poolIDMap3) + + // Assert total of two mappings are added + assert.Equal(t, 2, len(*m)) + + // Assert two pool ID mappings are available for first cluster mapping + assert.Equal(t, 2, len((*m)[0].RBDPoolIDMapping)) + + // Assert one pool ID mapping is available for second cluster mapping + assert.Equal(t, 1, len((*m)[1].RBDPoolIDMapping)) +} + +const ( + ns = "rook-ceph-primary" +) + +// #nosec G101 fake token for peer cluster "peer1" +var fakeTokenPeer1 = "eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0iLCAibmFtZXNwYWNlIjogInBlZXIxIn0=" + +// #nosec G101 fake token for peer cluster "peer2" +var fakeTokenPeer2 = "eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0iLCAibmFtZXNwYWNlIjogInBlZXIyIn0=" + +// #nosec G101 fake token for peer cluster "peer3" +var fakeTokenPeer3 = "eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0iLCAibmFtZXNwYWNlIjogInBlZXIzIn0=" + +var peer1Secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "peer1Secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "token": []byte(fakeTokenPeer1), + }, +} + +var peer2Secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "peer2Secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "token": []byte(fakeTokenPeer2), + }, +} + +var peer3Secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "peer3Secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "token": []byte(fakeTokenPeer3), + }, +} + +var fakeSinglePeerCephBlockPool = cephv1.CephBlockPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mirrorPool1", + Namespace: ns, + }, + Spec: cephv1.PoolSpec{ + Mirroring: cephv1.MirroringSpec{ + Peers: &cephv1.MirroringPeerSpec{ + SecretNames: []string{ + "peer1Secret", + }, + }, + }, + }, +} + +var fakeMultiPeerCephBlockPool = cephv1.CephBlockPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mirrorPool1", + Namespace: ns, + }, + Spec: cephv1.PoolSpec{ + Mirroring: cephv1.MirroringSpec{ + Peers: &cephv1.MirroringPeerSpec{ + SecretNames: []string{ + "peer1Secret", + "peer2Secret", + "peer3Secret", + }, + }, + }, + }, +} + +var mockExecutor = &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + // Fake pool details for "rook-ceph-primary" cluster + if args[0] == "osd" && args[1] == "pool" && args[2] == "get" && strings.HasSuffix(args[6], ns) { + if args[3] == "mirrorPool1" { + return `{"pool_id": 1}`, nil + } else if args[3] == "mirrorPool2" { + return `{"pool_id": 2}`, nil + } + + } + return "", nil + }, + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if args[0] == "osd" && args[1] == "pool" && args[2] == "get" && strings.HasSuffix(args[5], "peer1") { + if args[3] == "mirrorPool1" { + return `{"pool_id": 2}`, nil + } else if args[3] == "mirrorPool2" { + return `{"pool_id": 3}`, nil + } + } + if args[0] == "osd" && args[1] == "pool" && args[2] == "get" && strings.HasSuffix(args[5], "peer2") { + if args[3] == "mirrorPool1" { + return `{"pool_id": 3}`, nil + } else if args[3] == "mirrorPool2" { + return `{"pool_id": 4}`, nil + } + } + if args[0] == "osd" && args[1] == "pool" && args[2] == "get" && strings.HasSuffix(args[5], "peer3") { + if args[3] == "mirrorPool1" { + return `{"pool_id": 4}`, nil + } else if args[3] == "mirrorPool2" { + return `{"pool_id": 5}`, nil + } + } + return "", nil + }, +} + +func TestSinglePeerMappings(t *testing.T) { + clusterInfo := &cephclient.ClusterInfo{Namespace: ns} + fakeContext := &clusterd.Context{ + Executor: mockExecutor, + Clientset: test.New(t, 3), + } + + // create fake secret with "peer1" cluster token + _, err := fakeContext.Clientset.CoreV1().Secrets(ns).Create(context.TODO(), &peer1Secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + //expected: &[{ClusterIDMapping:{peer1:rook-ceph-primary}. RBDPoolIDMapping:[{2:1}]}] + actualMappings, err := getClusterPoolIDMap( + fakeContext, + clusterInfo, + &fakeSinglePeerCephBlockPool, + ) + assert.NoError(t, err) + mappings := *actualMappings + assert.Equal(t, 1, len(mappings)) + assert.Equal(t, ns, mappings[0].ClusterIDMapping["peer1"]) + assert.Equal(t, "1", mappings[0].RBDPoolIDMapping[0]["2"]) +} + +func TestMultiPeerMappings(t *testing.T) { + clusterInfo := &cephclient.ClusterInfo{Namespace: ns} + fakeContext := &clusterd.Context{ + Executor: mockExecutor, + Clientset: test.New(t, 3), + } + + // create fake secret with "peer1" cluster token + _, err := fakeContext.Clientset.CoreV1().Secrets(ns).Create(context.TODO(), &peer1Secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + // create fake secret with "peer2" cluster token + _, err = fakeContext.Clientset.CoreV1().Secrets(ns).Create(context.TODO(), &peer2Secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + // create fake secret with "peer3" cluster token + _, err = fakeContext.Clientset.CoreV1().Secrets(ns).Create(context.TODO(), &peer3Secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + actualMappings, err := getClusterPoolIDMap( + fakeContext, + clusterInfo, + &fakeMultiPeerCephBlockPool, + ) + assert.NoError(t, err) + mappings := *actualMappings + /* Expected: + [ + {ClusterIDMapping:{peer1:rook-ceph-primary}, RBDPoolIDMapping:[{2:1}]} + {ClusterIDMapping:{peer2:rook-ceph-primary}, RBDPoolIDMapping:[{3:1}]} + {ClusterIDMapping:map{peer3:rook-ceph-primary} RBDPoolIDMapping:[{4:1}]} + ] + */ + + assert.Equal(t, 3, len(mappings)) + + assert.Equal(t, 1, len(mappings[0].ClusterIDMapping)) + assert.Equal(t, ns, mappings[0].ClusterIDMapping["peer1"]) + assert.Equal(t, "1", mappings[0].RBDPoolIDMapping[0]["2"]) + + assert.Equal(t, 1, len(mappings[1].ClusterIDMapping)) + assert.Equal(t, ns, mappings[1].ClusterIDMapping["peer2"]) + assert.Equal(t, "1", mappings[1].RBDPoolIDMapping[0]["3"]) + + assert.Equal(t, 1, len(mappings[2].ClusterIDMapping)) + assert.Equal(t, ns, mappings[2].ClusterIDMapping["peer3"]) + assert.Equal(t, "1", mappings[2].RBDPoolIDMapping[0]["4"]) +} + +func TestDecodePeerToken(t *testing.T) { + // Valid token + decodedToken, err := decodePeerToken(fakeTokenPeer1) + assert.NoError(t, err) + assert.Equal(t, "peer1", decodedToken.Namespace) + + // Invalid token + _, err = decodePeerToken("invalidToken") + assert.Error(t, err) +} + +func fakeOperatorPod() *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: ns, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testReplicaSet", + }, + }, + }, + Spec: corev1.PodSpec{}, + } + return p +} + +func fakeReplicaSet() *appsv1.ReplicaSet { + r := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testReplicaSet", + Namespace: ns, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + }, + }, + }, + } + + return r +} + +func TestCreateOrUpdateConfig(t *testing.T) { + os.Setenv("POD_NAME", "test") + defer os.Setenv("POD_NAME", "") + os.Setenv("POD_NAMESPACE", ns) + defer os.Setenv("POD_NAMESPACE", "") + + scheme := scheme.Scheme + err := cephv1.AddToScheme(scheme) + assert.NoError(t, err) + + err = appsv1.AddToScheme(scheme) + assert.NoError(t, err) + + err = corev1.AddToScheme(scheme) + assert.NoError(t, err) + + fakeContext := &clusterd.Context{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build(), + Executor: mockExecutor, + Clientset: test.New(t, 3), + } + + // Create fake pod + _, err = fakeContext.Clientset.CoreV1().Pods(ns).Create(context.TODO(), fakeOperatorPod(), metav1.CreateOptions{}) + assert.NoError(t, err) + + // Create fake replicaset + _, err = fakeContext.Clientset.AppsV1().ReplicaSets(ns).Create(context.TODO(), fakeReplicaSet(), metav1.CreateOptions{}) + assert.NoError(t, err) + + // Create empty ID mapping configMap + err = CreateOrUpdateConfig(fakeContext, &PeerIDMappings{}) + assert.NoError(t, err) + validateConfig(t, fakeContext, PeerIDMappings{}) + + // Create ID mapping configMap with data + actualMappings := &PeerIDMappings{ + { + ClusterIDMapping: map[string]string{"peer1": ns}, + RBDPoolIDMapping: []map[string]string{ + { + "2": "1", + }, + }, + }, + } + + err = CreateOrUpdateConfig(fakeContext, actualMappings) + assert.NoError(t, err) + //validateConfig(t, fakeContext, actualMappings) + + //Update existing mapping config + mappings := *actualMappings + mappings = append(mappings, PeerIDMapping{ + ClusterIDMapping: map[string]string{"peer2": ns}, + RBDPoolIDMapping: []map[string]string{ + { + "3": "1", + }, + }, + }) + + err = CreateOrUpdateConfig(fakeContext, &mappings) + assert.NoError(t, err) + validateConfig(t, fakeContext, mappings) +} + +func validateConfig(t *testing.T, c *clusterd.Context, mappings PeerIDMappings) { + cm := &corev1.ConfigMap{} + err := c.Client.Get(context.TODO(), types.NamespacedName{Name: mappingConfigName, Namespace: ns}, cm) + assert.NoError(t, err) + + data := cm.Data[mappingConfigkey] + expectedMappings, err := toObj(data) + + assert.NoError(t, err) + assert.True(t, reflect.DeepEqual(mappings, expectedMappings)) +} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 03b962a3f..8ac752b3d 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -103,7 +103,7 @@ spec: fieldPath: metadata.namespace imagePullPolicy: "IfNotPresent" volumeMounts: - - name: ceph-csi-config + - name: ceph-csi-configs mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys @@ -166,7 +166,7 @@ spec: - mountPath: /lib/modules name: lib-modules readOnly: true - - name: ceph-csi-config + - name: ceph-csi-configs mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys @@ -204,12 +204,21 @@ spec: emptyDir: { medium: "Memory" } - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json + - name: ceph-csi-configs + projected: + sources: + - name: ceph-csi-config + configMap: + name: rook-ceph-csi-config + items: + - key: csi-cluster-config-json + path: config.json + - name: ceph-csi-mapping-config + configMap: + name: rook-ceph-csi-mapping-config + items: + - key: csi-mapping-config-json + path: cluster-mapping.json - name: keys-tmp-dir emptyDir: { medium: "Memory" diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index 4ac97ff7b..fe79929f2 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -100,7 +100,7 @@ spec: - mountPath: /lib/modules name: lib-modules readOnly: true - - name: ceph-csi-config + - name: ceph-csi-configs mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys @@ -154,12 +154,21 @@ spec: - name: lib-modules hostPath: path: /lib/modules - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json + - name: ceph-csi-configs + projected: + sources: + - name: ceph-csi-config + configMap: + name: rook-ceph-csi-config + items: + - key: csi-cluster-config-json + path: config.json + - name: ceph-csi-mapping-config + configMap: + name: rook-ceph-csi-mapping-config + items: + - key: csi-mapping-config-json + path: cluster-mapping.json - name: keys-tmp-dir emptyDir: { medium: "Memory" diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 351b2ef85..32f1070a7 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -33,6 +33,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util" "github.com/rook/rook/pkg/util/exec" "golang.org/x/sync/errgroup" v1 "k8s.io/api/core/v1" @@ -875,7 +876,7 @@ func enableRGWDashboard(context *Context) error { // for latest Ceph versions if mgr.FileBasedPasswordSupported(context.clusterInfo) { - accessFile, err := mgr.CreateTempPasswordFile(*u.AccessKey) + accessFile, err := util.CreateTempFile(*u.AccessKey) if err != nil { return errors.Wrap(err, "failed to create a temporary dashboard access-key file") } @@ -887,7 +888,7 @@ func enableRGWDashboard(context *Context) error { } }() - secretFile, err = mgr.CreateTempPasswordFile(*u.SecretKey) + secretFile, err = util.CreateTempFile(*u.SecretKey) if err != nil { return errors.Wrap(err, "failed to create a temporary dashboard secret-key file") } diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index 06603fe6b..7f4e6cdd7 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -33,12 +33,11 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/csi" + "github.com/rook/rook/pkg/operator/ceph/csi/peermap" "github.com/rook/rook/pkg/operator/ceph/provisioner" "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/sig-storage-lib-external-provisioner/v6/controller" ) @@ -253,7 +252,8 @@ func (o *Operator) updateDrivers() error { return nil } - ownerRef, err := getDeploymentOwnerReference(o.context.Clientset, o.config.OperatorNamespace) + operatorPodName := os.Getenv(k8sutil.PodNameEnvVar) + ownerRef, err := k8sutil.GetDeploymentOwnerReference(o.context.Clientset, operatorPodName, o.config.OperatorNamespace) if err != nil { logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err) } @@ -270,35 +270,11 @@ func (o *Operator) updateDrivers() error { return errors.Wrap(err, "failed creating csi config map") } + err = peermap.CreateOrUpdateConfig(o.context, &peermap.PeerIDMappings{}) + if err != nil { + return errors.Wrap(err, "failed to create pool ID mapping config map") + } + go csi.ValidateAndConfigureDrivers(o.context, o.config.OperatorNamespace, o.config.Image, o.config.ServiceAccount, serverVersion, ownerInfo) return nil } - -// getDeploymentOwnerReference returns an OwnerReference to the rook-ceph-operator deployment -func getDeploymentOwnerReference(clientset kubernetes.Interface, namespace string) (*metav1.OwnerReference, error) { - ctx := context.TODO() - var deploymentRef *metav1.OwnerReference - podName := os.Getenv(k8sutil.PodNameEnvVar) - pod, err := clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - return nil, errors.Wrapf(err, "could not find pod %q to find deployment owner reference", podName) - } - for _, podOwner := range pod.OwnerReferences { - if podOwner.Kind == "ReplicaSet" { - replicaset, err := clientset.AppsV1().ReplicaSets(namespace).Get(ctx, podOwner.Name, metav1.GetOptions{}) - if err != nil { - return nil, errors.Wrapf(err, "could not find replicaset %q to find deployment owner reference", podOwner.Name) - } - for _, replicasetOwner := range replicaset.OwnerReferences { - if replicasetOwner.Kind == "Deployment" { - localreplicasetOwner := replicasetOwner - deploymentRef = &localreplicasetOwner - } - } - } - } - if deploymentRef == nil { - return nil, errors.New("could not find owner reference for rook-ceph deployment") - } - return deploymentRef, nil -} diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index d88b58ea6..b56aed1bf 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -32,6 +32,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/csi/peermap" "github.com/rook/rook/pkg/operator/k8sutil" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -304,6 +305,12 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile return reconcileResponse, errors.Wrap(err, "failed to add ceph rbd mirror peer") } + // ReconcilePoolIDMap updates the `rook-ceph-csi-mapping-config` with local and peer cluster pool ID map + err = peermap.ReconcilePoolIDMap(r.context, r.clusterInfo, cephBlockPool) + if err != nil { + return reconcileResponse, errors.Wrapf(err, "failed to update pool ID mapping config for the pool %q", cephBlockPool.Name) + } + // Set Ready status, we are done reconciling updateStatus(r.client, request.NamespacedName, cephv1.ConditionReady, opcontroller.GenerateStatusInfo(cephBlockPool)) diff --git a/pkg/operator/ceph/pool/controller_test.go b/pkg/operator/ceph/pool/controller_test.go index 0cd9f46f4..954b28e0b 100644 --- a/pkg/operator/ceph/pool/controller_test.go +++ b/pkg/operator/ceph/pool/controller_test.go @@ -33,6 +33,8 @@ import ( exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" "github.com/tevino/abool" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -347,6 +349,42 @@ func TestCephBlockPoolController(t *testing.T) { blockPoolChannels: make(map[string]*blockPoolHealth), } + os.Setenv("POD_NAME", "test") + defer os.Setenv("POD_NAME", "") + os.Setenv("POD_NAMESPACE", namespace) + defer os.Setenv("POD_NAMESPACE", "") + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testReplicaSet", + }, + }, + }, + } + // Create fake pod + _, err = r.context.Clientset.CoreV1().Pods(namespace).Create(context.TODO(), p, metav1.CreateOptions{}) + assert.NoError(t, err) + + replicaSet := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testReplicaSet", + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Deployment", + }, + }, + }, + } + + // Create fake replicaset + _, err = r.context.Clientset.AppsV1().ReplicaSets(namespace).Create(context.TODO(), replicaSet, metav1.CreateOptions{}) + assert.NoError(t, err) + pool.Spec.Mirroring.Mode = "image" pool.Spec.Mirroring.Peers.SecretNames = []string{} err = r.client.Update(context.TODO(), pool) diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index f84ed8ad0..34d62c0bb 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -394,6 +394,34 @@ func DeleteDeployment(clientset kubernetes.Interface, namespace, name string) er return deleteResourceAndWait(namespace, name, "deployment", deleteAction, getAction) } +// GetDeploymentOwnerReference returns an OwnerReference to the deployment that is running the given pod name +func GetDeploymentOwnerReference(clientset kubernetes.Interface, podName, namespace string) (*metav1.OwnerReference, error) { + ctx := context.TODO() + var deploymentRef *metav1.OwnerReference + pod, err := clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "could not find pod %q in namespace %q to find deployment owner reference", podName, namespace) + } + for _, podOwner := range pod.OwnerReferences { + if podOwner.Kind == "ReplicaSet" { + replicaset, err := clientset.AppsV1().ReplicaSets(namespace).Get(ctx, podOwner.Name, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "could not find replicaset %q in namespace %q to find deployment owner reference", podOwner.Name, namespace) + } + for _, replicasetOwner := range replicaset.OwnerReferences { + if replicasetOwner.Kind == "Deployment" { + localreplicasetOwner := replicasetOwner + deploymentRef = &localreplicasetOwner + } + } + } + } + if deploymentRef == nil { + return nil, errors.New("could not find owner reference for rook-ceph deployment") + } + return deploymentRef, nil +} + // WaitForDeploymentImage waits for all deployments with the given labels are running. // WARNING:This is currently only useful for testing! func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, container string, initContainer bool, desiredImage string) error { diff --git a/pkg/util/file.go b/pkg/util/file.go index 3e034b278..ac815b3f4 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -25,6 +25,7 @@ import ( "runtime" "github.com/coreos/pkg/capnslog" + "github.com/pkg/errors" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "util") @@ -60,3 +61,19 @@ func PathToProjectRoot() string { root := filepath.Dir(pkg) // return root } + +// CreateTempFile creates a temporary file with content passed as an argument +func CreateTempFile(content string) (*os.File, error) { + // Generate a temp file + file, err := ioutil.TempFile("", "") + if err != nil { + return nil, errors.Wrap(err, "failed to generate temp file") + } + + // Write content into file + err = ioutil.WriteFile(file.Name(), []byte(content), 0440) + if err != nil { + return nil, errors.Wrap(err, "failed to write content into file") + } + return file, nil +}