ceph: rehydrate the bootstrap peer token secret on monitor changes

If the monitors addresses change, we must update the peer secret with
the new addresses. For this, each time the configmap
"rook-ceph-mon-endpoints" is updated, we trigger a reconcile on the
CephBlockPool CRs, which will update the token secret.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2021-05-20 08:50:22 +02:00
1 parent 0459a20173
commit dc0985c7f6
6 files changed
+222 -5

No files matched your search

+1 -1
View File
@@ -46,7 +46,7 @@ func TestCreateRBDMirrorBootstrapPeer(t *testing.T) {
assert.Equal(t, "create", args[4])
assert.Equal(t, pool, args[5])
assert.Equal(t, "--site-name", args[6])
assert.Equal(t, "4fe04ebb-ec0c-46c2-ac55-9eb52ebbfb82-mycluster", args[7])
assert.Equal(t, "4fe04ebb-ec0c-46c2-ac55-9eb52ebbfb82", args[7])
return bootstrapPeerToken, nil
}
return "", errors.New("unknown command")
+3 -4
View File
@@ -18,7 +18,6 @@ limitations under the License.
package pool
import (
"context"
"fmt"
"github.com/pkg/errors"
@@ -42,7 +41,6 @@ const (
)
func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1.CephBlockPool, namespacedName types.NamespacedName) (reconcile.Result, error) {
ctx := context.TODO()
// Create rbd mirror bootstrap peer token
boostrapToken, err := cephclient.CreateRBDMirrorBootstrapPeer(r.context, r.clusterInfo, cephBlockPool.Name)
if err != nil {
@@ -60,11 +58,12 @@ func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1
// Create Secret
logger.Debugf("store rbd-mirror bootstrap token in a Kubernetes Secret %q", s.Name)
_, err = r.context.Clientset.CoreV1().Secrets(cephBlockPool.Namespace).Create(ctx, s, metav1.CreateOptions{})
_, err = k8sutil.CreateOrUpdateSecret(r.context.Clientset, s)
if err != nil && !kerrors.IsAlreadyExists(err) {
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to create rbd-mirror bootstrap peer %q secret", s.Name)
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to create or update rbd-mirror bootstrap peer %q secret", s.Name)
}
logger.Infof("successfully created bootstrap peer token secret for pool %q", cephBlockPool.Name)
return reconcile.Result{}, nil
}
+14
View File
@@ -32,6 +32,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -109,6 +110,19 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
return err
}
// Build Handler function to return the list of ceph block pool
// This is used by the watchers below
handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephBlockPoolList{}, mgr.GetScheme())
if err != nil {
return err
}
// Watch for ConfigMap "rook-ceph-mon-endpoints" update and reconcile, which will reconcile update the bootstrap peer token
err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: corev1.SchemeGroupVersion.String()}}}, handler.EnqueueRequestsFromMapFunc(handlerFunc), predicateMonEndpointChanges())
if err != nil {
return err
}
return nil
}
+110
View File
@@ -0,0 +1,110 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package pool to manage a rook pool.
package pool
import (
"encoding/json"
"reflect"
"sort"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)
func predicateMonEndpointChanges() predicate.Funcs {
return predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool {
return false
},
DeleteFunc: func(e event.DeleteEvent) bool {
return false
},
GenericFunc: func(e event.GenericEvent) bool {
return false
},
UpdateFunc: func(e event.UpdateEvent) bool {
cmNew, ok := e.ObjectNew.(*corev1.ConfigMap)
if !ok {
return false
}
cmOld, ok := e.ObjectOld.(*corev1.ConfigMap)
if !ok {
return false
}
if cmNew.GetName() == mon.EndpointConfigMapName {
if wereMonEndpointsUpdated(cmOld.Data, cmNew.Data) {
logger.Info("monitor endpoints changed, updating the bootstrap peer token")
return true
}
}
return false
},
}
}
func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
// Check the mapping key first
if oldMapping, ok := oldCMData["mapping"]; ok {
if newMapping, ok := newCMData["mapping"]; ok {
// Unmarshal both into a type
var oldMappingToGo mon.Mapping
err := json.Unmarshal([]byte(oldMapping), &oldMappingToGo)
if err != nil {
logger.Debugf("failed to unmarshal new. %v", err)
return false
}
var newMappingToGo mon.Mapping
err = json.Unmarshal([]byte(newMapping), &newMappingToGo)
if err != nil {
logger.Debugf("failed to unmarshal new. %v", err)
return false
}
// If length is different, monitors are different
if len(oldMappingToGo.Schedule) != len(newMappingToGo.Schedule) {
logger.Debugf("mons were added or removed from the endpoints cm")
return true
}
// Since Schedule is map, it's unordered, so let's order it
oldKeys := make([]string, 0, len(oldMappingToGo.Schedule))
for k := range oldMappingToGo.Schedule {
oldKeys = append(oldKeys, k)
}
sort.Strings(oldKeys)
newKeys := make([]string, 0, len(newMappingToGo.Schedule))
for k := range oldMappingToGo.Schedule {
newKeys = append(newKeys, k)
}
sort.Strings(newKeys)
// Iterate over the map and compare the values
for _, v := range oldKeys {
if !reflect.DeepEqual(oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) {
logger.Debugf("oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v])
return true
}
}
}
}
return false
}
+46
View File
@@ -0,0 +1,46 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package pool to manage a rook pool.
package pool
import "testing"
func TestWereMonEndpointsUpdated(t *testing.T) {
type args struct {
oldCMData map[string]string
newCMData map[string]string
}
tests := []struct {
name string
args args
want bool
}{
{"no old mapping key", args{oldCMData: map[string]string{}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false},
{"no new mapping key", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{}}, false},
{"identical content", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false},
{"identical content but different order", args{oldCMData: map[string]string{"mapping": `{"node":{"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false},
{"same length but different mons IP for 'i'", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.187"}}}`}}, true},
{"different length", args{oldCMData: map[string]string{"mapping": `{"node":{"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.187"}}}`}}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := wereMonEndpointsUpdated(tt.args.oldCMData, tt.args.newCMData); got != tt.want {
t.Errorf("whereMonEndpointsUpdated() = %v, want %v", got, tt.want)
}
})
}
}
+48
View File
@@ -0,0 +1,48 @@
/*
Copyright 2021 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8sutil
import (
"context"
"fmt"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
// CreateOrUpdateSecret creates a secret or updates the secret declaratively if it already exists.
func CreateOrUpdateSecret(clientset kubernetes.Interface, secretDefinition *v1.Secret) (*v1.Secret, error) {
ctx := context.TODO()
name := secretDefinition.Name
logger.Debugf("creating secret %s", name)
s, err := clientset.CoreV1().Secrets(secretDefinition.Namespace).Create(ctx, secretDefinition, metav1.CreateOptions{})
if err != nil {
if !errors.IsAlreadyExists(err) {
return nil, fmt.Errorf("failed to create secret %s. %+v", name, err)
}
s, err = clientset.CoreV1().Secrets(secretDefinition.Namespace).Update(ctx, secretDefinition, metav1.UpdateOptions{})
if err != nil {
return nil, fmt.Errorf("failed to update secret %s. %+v", name, err)
}
} else {
logger.Debugf("created secret %s", s.Name)
}
return s, err
}