From dc0985c7f63dfd21039adfb5a95041d13d99c951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 18 May 2021 19:39:00 +0200 Subject: [PATCH] ceph: rehydrate the bootstrap peer token secret on monitor changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the monitors addresses change, we must update the peer secret with the new addresses. For this, each time the configmap "rook-ceph-mon-endpoints" is updated, we trigger a reconcile on the CephBlockPool CRs, which will update the token secret. Signed-off-by: Sébastien Han --- pkg/daemon/ceph/client/mirror_test.go | 2 +- pkg/operator/ceph/pool/config.go | 7 +- pkg/operator/ceph/pool/controller.go | 14 +++ pkg/operator/ceph/pool/predicate.go | 110 +++++++++++++++++++++++ pkg/operator/ceph/pool/predicate_test.go | 46 ++++++++++ pkg/operator/k8sutil/secret.go | 48 ++++++++++ 6 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 pkg/operator/ceph/pool/predicate.go create mode 100644 pkg/operator/ceph/pool/predicate_test.go create mode 100644 pkg/operator/k8sutil/secret.go diff --git a/pkg/daemon/ceph/client/mirror_test.go b/pkg/daemon/ceph/client/mirror_test.go index a2a0347dc..930bcf8dc 100644 --- a/pkg/daemon/ceph/client/mirror_test.go +++ b/pkg/daemon/ceph/client/mirror_test.go @@ -46,7 +46,7 @@ func TestCreateRBDMirrorBootstrapPeer(t *testing.T) { assert.Equal(t, "create", args[4]) assert.Equal(t, pool, args[5]) assert.Equal(t, "--site-name", args[6]) - assert.Equal(t, "4fe04ebb-ec0c-46c2-ac55-9eb52ebbfb82-mycluster", args[7]) + assert.Equal(t, "4fe04ebb-ec0c-46c2-ac55-9eb52ebbfb82", args[7]) return bootstrapPeerToken, nil } return "", errors.New("unknown command") diff --git a/pkg/operator/ceph/pool/config.go b/pkg/operator/ceph/pool/config.go index 65756e34b..9b716995a 100644 --- a/pkg/operator/ceph/pool/config.go +++ b/pkg/operator/ceph/pool/config.go @@ -18,7 +18,6 @@ limitations under the License. package pool import ( - "context" "fmt" "github.com/pkg/errors" @@ -42,7 +41,6 @@ const ( ) func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1.CephBlockPool, namespacedName types.NamespacedName) (reconcile.Result, error) { - ctx := context.TODO() // Create rbd mirror bootstrap peer token boostrapToken, err := cephclient.CreateRBDMirrorBootstrapPeer(r.context, r.clusterInfo, cephBlockPool.Name) if err != nil { @@ -60,11 +58,12 @@ func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1 // Create Secret logger.Debugf("store rbd-mirror bootstrap token in a Kubernetes Secret %q", s.Name) - _, err = r.context.Clientset.CoreV1().Secrets(cephBlockPool.Namespace).Create(ctx, s, metav1.CreateOptions{}) + _, err = k8sutil.CreateOrUpdateSecret(r.context.Clientset, s) if err != nil && !kerrors.IsAlreadyExists(err) { - return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to create rbd-mirror bootstrap peer %q secret", s.Name) + return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to create or update rbd-mirror bootstrap peer %q secret", s.Name) } + logger.Infof("successfully created bootstrap peer token secret for pool %q", cephBlockPool.Name) return reconcile.Result{}, nil } diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 19a1e87e8..4e079b976 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -32,6 +32,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -109,6 +110,19 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } + // Build Handler function to return the list of ceph block pool + // This is used by the watchers below + handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephBlockPoolList{}, mgr.GetScheme()) + if err != nil { + return err + } + + // Watch for ConfigMap "rook-ceph-mon-endpoints" update and reconcile, which will reconcile update the bootstrap peer token + err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: corev1.SchemeGroupVersion.String()}}}, handler.EnqueueRequestsFromMapFunc(handlerFunc), predicateMonEndpointChanges()) + if err != nil { + return err + } + return nil } diff --git a/pkg/operator/ceph/pool/predicate.go b/pkg/operator/ceph/pool/predicate.go new file mode 100644 index 000000000..06b62eb82 --- /dev/null +++ b/pkg/operator/ceph/pool/predicate.go @@ -0,0 +1,110 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package pool to manage a rook pool. +package pool + +import ( + "encoding/json" + "reflect" + "sort" + + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +func predicateMonEndpointChanges() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + cmNew, ok := e.ObjectNew.(*corev1.ConfigMap) + if !ok { + return false + } + cmOld, ok := e.ObjectOld.(*corev1.ConfigMap) + if !ok { + return false + } + if cmNew.GetName() == mon.EndpointConfigMapName { + if wereMonEndpointsUpdated(cmOld.Data, cmNew.Data) { + logger.Info("monitor endpoints changed, updating the bootstrap peer token") + return true + } + } + return false + }, + } +} + +func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool { + // Check the mapping key first + if oldMapping, ok := oldCMData["mapping"]; ok { + if newMapping, ok := newCMData["mapping"]; ok { + // Unmarshal both into a type + var oldMappingToGo mon.Mapping + err := json.Unmarshal([]byte(oldMapping), &oldMappingToGo) + if err != nil { + logger.Debugf("failed to unmarshal new. %v", err) + return false + } + + var newMappingToGo mon.Mapping + err = json.Unmarshal([]byte(newMapping), &newMappingToGo) + if err != nil { + logger.Debugf("failed to unmarshal new. %v", err) + return false + } + + // If length is different, monitors are different + if len(oldMappingToGo.Schedule) != len(newMappingToGo.Schedule) { + logger.Debugf("mons were added or removed from the endpoints cm") + return true + } + // Since Schedule is map, it's unordered, so let's order it + oldKeys := make([]string, 0, len(oldMappingToGo.Schedule)) + for k := range oldMappingToGo.Schedule { + oldKeys = append(oldKeys, k) + } + sort.Strings(oldKeys) + + newKeys := make([]string, 0, len(newMappingToGo.Schedule)) + for k := range oldMappingToGo.Schedule { + newKeys = append(newKeys, k) + } + sort.Strings(newKeys) + + // Iterate over the map and compare the values + for _, v := range oldKeys { + if !reflect.DeepEqual(oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) { + logger.Debugf("oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) + return true + } + } + } + } + + return false +} diff --git a/pkg/operator/ceph/pool/predicate_test.go b/pkg/operator/ceph/pool/predicate_test.go new file mode 100644 index 000000000..d441a9751 --- /dev/null +++ b/pkg/operator/ceph/pool/predicate_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package pool to manage a rook pool. +package pool + +import "testing" + +func TestWereMonEndpointsUpdated(t *testing.T) { + type args struct { + oldCMData map[string]string + newCMData map[string]string + } + tests := []struct { + name string + args args + want bool + }{ + {"no old mapping key", args{oldCMData: map[string]string{}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false}, + {"no new mapping key", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{}}, false}, + {"identical content", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false}, + {"identical content but different order", args{oldCMData: map[string]string{"mapping": `{"node":{"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false}, + {"same length but different mons IP for 'i'", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.187"}}}`}}, true}, + {"different length", args{oldCMData: map[string]string{"mapping": `{"node":{"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.187"}}}`}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := wereMonEndpointsUpdated(tt.args.oldCMData, tt.args.newCMData); got != tt.want { + t.Errorf("whereMonEndpointsUpdated() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/operator/k8sutil/secret.go b/pkg/operator/k8sutil/secret.go new file mode 100644 index 000000000..e63f98332 --- /dev/null +++ b/pkg/operator/k8sutil/secret.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sutil + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// CreateOrUpdateSecret creates a secret or updates the secret declaratively if it already exists. +func CreateOrUpdateSecret(clientset kubernetes.Interface, secretDefinition *v1.Secret) (*v1.Secret, error) { + ctx := context.TODO() + name := secretDefinition.Name + logger.Debugf("creating secret %s", name) + + s, err := clientset.CoreV1().Secrets(secretDefinition.Namespace).Create(ctx, secretDefinition, metav1.CreateOptions{}) + if err != nil { + if !errors.IsAlreadyExists(err) { + return nil, fmt.Errorf("failed to create secret %s. %+v", name, err) + } + s, err = clientset.CoreV1().Secrets(secretDefinition.Namespace).Update(ctx, secretDefinition, metav1.UpdateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to update secret %s. %+v", name, err) + } + } else { + logger.Debugf("created secret %s", s.Name) + } + return s, err +}