ceph: rehydrate the bootstrap peer token secret on monitor changes
If the monitors addresses change, we must update the peer secret with the new addresses. For this, each time the configmap "rook-ceph-mon-endpoints" is updated, we trigger a reconcile on the CephBlockPool CRs, which will update the token secret. Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
1 parent
0459a20173
commit
dc0985c7f6
6 files changed
+222
-5
No files matched your search
@@ -46,7 +46,7 @@ func TestCreateRBDMirrorBootstrapPeer(t *testing.T) {
|
||||
assert.Equal(t, "create", args[4])
|
||||
assert.Equal(t, pool, args[5])
|
||||
assert.Equal(t, "--site-name", args[6])
|
||||
assert.Equal(t, "4fe04ebb-ec0c-46c2-ac55-9eb52ebbfb82-mycluster", args[7])
|
||||
assert.Equal(t, "4fe04ebb-ec0c-46c2-ac55-9eb52ebbfb82", args[7])
|
||||
return bootstrapPeerToken, nil
|
||||
}
|
||||
return "", errors.New("unknown command")
|
||||
|
||||
@@ -18,7 +18,6 @@ limitations under the License.
|
||||
package pool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -42,7 +41,6 @@ const (
|
||||
)
|
||||
|
||||
func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1.CephBlockPool, namespacedName types.NamespacedName) (reconcile.Result, error) {
|
||||
ctx := context.TODO()
|
||||
// Create rbd mirror bootstrap peer token
|
||||
boostrapToken, err := cephclient.CreateRBDMirrorBootstrapPeer(r.context, r.clusterInfo, cephBlockPool.Name)
|
||||
if err != nil {
|
||||
@@ -60,11 +58,12 @@ func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1
|
||||
|
||||
// Create Secret
|
||||
logger.Debugf("store rbd-mirror bootstrap token in a Kubernetes Secret %q", s.Name)
|
||||
_, err = r.context.Clientset.CoreV1().Secrets(cephBlockPool.Namespace).Create(ctx, s, metav1.CreateOptions{})
|
||||
_, err = k8sutil.CreateOrUpdateSecret(r.context.Clientset, s)
|
||||
if err != nil && !kerrors.IsAlreadyExists(err) {
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to create rbd-mirror bootstrap peer %q secret", s.Name)
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to create or update rbd-mirror bootstrap peer %q secret", s.Name)
|
||||
}
|
||||
|
||||
logger.Infof("successfully created bootstrap peer token secret for pool %q", cephBlockPool.Name)
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
@@ -109,6 +110,19 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Build Handler function to return the list of ceph block pool
|
||||
// This is used by the watchers below
|
||||
handlerFunc, err := opcontroller.ObjectToCRMapper(mgr.GetClient(), &cephv1.CephBlockPoolList{}, mgr.GetScheme())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Watch for ConfigMap "rook-ceph-mon-endpoints" update and reconcile, which will reconcile update the bootstrap peer token
|
||||
err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: corev1.SchemeGroupVersion.String()}}}, handler.EnqueueRequestsFromMapFunc(handlerFunc), predicateMonEndpointChanges())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package pool to manage a rook pool.
|
||||
package pool
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
||||
)
|
||||
|
||||
func predicateMonEndpointChanges() predicate.Funcs {
|
||||
return predicate.Funcs{
|
||||
CreateFunc: func(e event.CreateEvent) bool {
|
||||
return false
|
||||
},
|
||||
DeleteFunc: func(e event.DeleteEvent) bool {
|
||||
return false
|
||||
},
|
||||
GenericFunc: func(e event.GenericEvent) bool {
|
||||
return false
|
||||
},
|
||||
UpdateFunc: func(e event.UpdateEvent) bool {
|
||||
cmNew, ok := e.ObjectNew.(*corev1.ConfigMap)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
cmOld, ok := e.ObjectOld.(*corev1.ConfigMap)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if cmNew.GetName() == mon.EndpointConfigMapName {
|
||||
if wereMonEndpointsUpdated(cmOld.Data, cmNew.Data) {
|
||||
logger.Info("monitor endpoints changed, updating the bootstrap peer token")
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func wereMonEndpointsUpdated(oldCMData, newCMData map[string]string) bool {
|
||||
// Check the mapping key first
|
||||
if oldMapping, ok := oldCMData["mapping"]; ok {
|
||||
if newMapping, ok := newCMData["mapping"]; ok {
|
||||
// Unmarshal both into a type
|
||||
var oldMappingToGo mon.Mapping
|
||||
err := json.Unmarshal([]byte(oldMapping), &oldMappingToGo)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to unmarshal new. %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
var newMappingToGo mon.Mapping
|
||||
err = json.Unmarshal([]byte(newMapping), &newMappingToGo)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to unmarshal new. %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// If length is different, monitors are different
|
||||
if len(oldMappingToGo.Schedule) != len(newMappingToGo.Schedule) {
|
||||
logger.Debugf("mons were added or removed from the endpoints cm")
|
||||
return true
|
||||
}
|
||||
// Since Schedule is map, it's unordered, so let's order it
|
||||
oldKeys := make([]string, 0, len(oldMappingToGo.Schedule))
|
||||
for k := range oldMappingToGo.Schedule {
|
||||
oldKeys = append(oldKeys, k)
|
||||
}
|
||||
sort.Strings(oldKeys)
|
||||
|
||||
newKeys := make([]string, 0, len(newMappingToGo.Schedule))
|
||||
for k := range oldMappingToGo.Schedule {
|
||||
newKeys = append(newKeys, k)
|
||||
}
|
||||
sort.Strings(newKeys)
|
||||
|
||||
// Iterate over the map and compare the values
|
||||
for _, v := range oldKeys {
|
||||
if !reflect.DeepEqual(oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v]) {
|
||||
logger.Debugf("oldMappingToGo.Schedule[v] AND newMappingToGo.Schedule[v]: %v | %v", oldMappingToGo.Schedule[v], newMappingToGo.Schedule[v])
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package pool to manage a rook pool.
|
||||
package pool
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestWereMonEndpointsUpdated(t *testing.T) {
|
||||
type args struct {
|
||||
oldCMData map[string]string
|
||||
newCMData map[string]string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want bool
|
||||
}{
|
||||
{"no old mapping key", args{oldCMData: map[string]string{}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false},
|
||||
{"no new mapping key", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{}}, false},
|
||||
{"identical content", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false},
|
||||
{"identical content but different order", args{oldCMData: map[string]string{"mapping": `{"node":{"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}}, false},
|
||||
{"same length but different mons IP for 'i'", args{oldCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.187"}}}`}}, true},
|
||||
{"different length", args{oldCMData: map[string]string{"mapping": `{"node":{"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"}}}`}, newCMData: map[string]string{"mapping": `{"node":{"g":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"h":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.121"},"i":{"Name":"minikube","Hostname":"minikube","Address":"192.168.39.187"}}}`}}, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := wereMonEndpointsUpdated(tt.args.oldCMData, tt.args.newCMData); got != tt.want {
|
||||
t.Errorf("whereMonEndpointsUpdated() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
Copyright 2021 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// CreateOrUpdateSecret creates a secret or updates the secret declaratively if it already exists.
|
||||
func CreateOrUpdateSecret(clientset kubernetes.Interface, secretDefinition *v1.Secret) (*v1.Secret, error) {
|
||||
ctx := context.TODO()
|
||||
name := secretDefinition.Name
|
||||
logger.Debugf("creating secret %s", name)
|
||||
|
||||
s, err := clientset.CoreV1().Secrets(secretDefinition.Namespace).Create(ctx, secretDefinition, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("failed to create secret %s. %+v", name, err)
|
||||
}
|
||||
s, err = clientset.CoreV1().Secrets(secretDefinition.Namespace).Update(ctx, secretDefinition, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to update secret %s. %+v", name, err)
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("created secret %s", s.Name)
|
||||
}
|
||||
return s, err
|
||||
}
|
||||
Reference in new issue
Block a user