diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index 33c6e5e93..3807ad030 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -18,8 +18,6 @@ package rbd import ( "context" - "encoding/base64" - "encoding/json" "fmt" "github.com/pkg/errors" @@ -29,7 +27,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config/keyring" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -43,20 +40,6 @@ const ( caps mon = "profile rbd-mirror" caps osd = "profile rbd" ` - keyringPeerTemplate = ` -[client.%s] - key = %s -` - cephConfPeerTemplate = ` -[global] - fsid = %s - mon_host = %s -` - peerCephConfigKey = "peerCephConfig" - peerCephKeyringKey = "peerCephKeyring" - peerCephConfigFileConfigMapName = "rbd-mirror-peer-config" - // #nosec G101 since this is not leaking any hardcoded credentials, it's just the secret name - peerCephKeyringSecretName = "rbd-mirror-peer-keyring" ) // daemonConfig for a single rbd-mirror @@ -108,7 +91,7 @@ func (r *ReconcileCephRBDMirror) reconcileAddBoostrapPeer(cephRBDMirror *cephv1. ctx := context.TODO() // List all the peers secret, we can have more than one peer we might want to configure // For each, get the Kubernetes Secret and import the "peer token" so that we can configure the mirroring - for k, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames { + for _, peerSecret := range cephRBDMirror.Spec.Peers.SecretNames { logger.Debugf("fetching bootstrap peer kubernetes secret %q", peerSecret) s, err := r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Get(ctx, peerSecret, metav1.GetOptions{}) // We don't care about IsNotFound here, we still need to fail @@ -125,24 +108,11 @@ func (r *ReconcileCephRBDMirror) reconcileAddBoostrapPeer(cephRBDMirror *cephv1. // Add Peer detail to the Struct r.peers[peerSecret] = peerSpec - // Get controller owner ref - ownerInfo := k8sutil.NewOwnerInfo(cephRBDMirror, r.scheme) - // Add rbd-mirror peer err = r.addPeer(peerSecret, s.Data) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to add rbd-mirror bootstrap peer") } - - // Only create the config and key Secret and ConfigMap for the first peer - // Peers are always identical, at least this is the supported way today (Ceph Octopus), only the pool name differs - if k == 0 { - // Create the token ceph config file and key - err = r.createBootstrapPeerConfigAndKey(peerSecret, string(s.Data["token"]), ownerInfo) - if err != nil { - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create bootstrap peer config and key") - } - } } return reconcile.Result{}, nil @@ -189,91 +159,3 @@ func validateSpec(r *cephv1.RBDMirroringSpec) error { return nil } - -func (r *ReconcileCephRBDMirror) createBootstrapPeerConfigAndKey(peerSecret string, tokenBase64 string, ownerInfo *k8sutil.OwnerInfo) error { - ctx := context.TODO() - // Decode the base64 token - decodeToken, err := base64.StdEncoding.DecodeString(tokenBase64) - if err != nil { - return errors.Wrap(err, "failed to decode bootstrap peer token") - } - - // Unmarshal JSON into PeerToken struct - var token PeerToken - if err := json.Unmarshal([]byte(decodeToken), &token); err != nil { - return errors.Wrap(err, "failed to unmarshal bootstrap peer token") - } - - // Build peer ceph conf - cephPeerConfig := fmt.Sprintf(cephConfPeerTemplate, token.ClusterFSID, token.MonHost) - - // Put it in a ConfigMap - cm, err := generatePeerCephConfigFileConfigMap(r.peers[peerSecret].info.Peers[0].UUID, cephPeerConfig, r.clusterInfo.Namespace, ownerInfo) - if err != nil { - return errors.Wrap(err, "failed to generate peer ceph config file configmap") - } - - _, err = r.context.Clientset.CoreV1().ConfigMaps(r.clusterInfo.Namespace).Create(ctx, cm, metav1.CreateOptions{}) - if err != nil && !kerrors.IsNotFound(err) && !kerrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create kubernetes config map %q bootstrap peer config", cm.Name) - } - - // Build the key file - cephPeerKey := fmt.Sprintf(keyringPeerTemplate, token.ClientID, token.Key) - - // Put it in a Secret - s, err := generatePeerKeyringSecret(r.peers[peerSecret].info.Peers[0].UUID, cephPeerKey, r.clusterInfo.Namespace, ownerInfo) - if err != nil { - return errors.Wrap(err, "failed to generate peer keyring secret") - } - _, err = r.context.Clientset.CoreV1().Secrets(r.clusterInfo.Namespace).Create(ctx, s, metav1.CreateOptions{}) - if err != nil && !kerrors.IsNotFound(err) && !kerrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create kubernetes secret %q bootstrap peer key", s.Name) - } - - return nil -} - -func generatePeerCephConfigFileConfigMapName(siteName string) string { - return fmt.Sprintf("%s-%s", peerCephConfigFileConfigMapName, siteName) -} - -func generatePeerKeyringSecretName(siteName string) string { - return fmt.Sprintf("%s-%s", peerCephKeyringSecretName, siteName) -} - -func generatePeerCephConfigFileConfigMap(peerSiteUUID, peerCephConfig string, namespace string, ownerInfo *k8sutil.OwnerInfo) (*v1.ConfigMap, error) { - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: generatePeerCephConfigFileConfigMapName(peerSiteUUID), - Namespace: namespace, - }, - Data: map[string]string{ - peerCephConfigKey: peerCephConfig, - }, - } - err := ownerInfo.SetControllerReference(cm) - if err != nil { - return nil, errors.Wrapf(err, "failed to set owner reference to configmap %q", cm.Name) - } - - return cm, nil -} - -func generatePeerKeyringSecret(peerSiteUUID, peerCephKey string, namespace string, ownerInfo *k8sutil.OwnerInfo) (*v1.Secret, error) { - s := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: generatePeerKeyringSecretName(peerSiteUUID), - Namespace: namespace, - }, - Data: map[string][]byte{ - peerCephKeyringKey: []byte(peerCephKey), - }, - Type: k8sutil.RookType, - } - err := ownerInfo.SetControllerReference(s) - if err != nil { - return nil, errors.Wrapf(err, "failed to set owner reference to secret %q", s.Name) - } - return s, nil -} diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index 45033ed72..da5044fd8 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -69,16 +69,6 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM } rbdMirror.Spec.Placement.ApplyToPodSpec(&podSpec.Spec) - // If the rbd mirror has a peer we must add the relevant ceph config file and key to connect to it - // Both cm and secret have been created already, so it's fine to just reference them - if rbdMirror.Spec.Peers.HasPeers() { - // Add the config map and secret - // We only use the first peer in the list because peers are all the same, just the pool differs - firstPeer := rbdMirror.Spec.Peers.SecretNames[0] - volProjection := peerConfigMapAndSecretVolumeAndMount(r.peers[firstPeer].info.Peers[0].SiteName, r.peers[firstPeer].info.Peers[0].ClientName, r.peers[firstPeer].info.Peers[0].UUID) - podSpec.Spec.Volumes[0].VolumeSource.Projected.Sources = append(podSpec.Spec.Volumes[0].VolumeSource.Projected.Sources, volProjection...) - } - replicas := int32(rbdMirror.Spec.Count) d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -139,33 +129,3 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem return container } - -// return the volume and matching volume mount for mounting the config map into /etc/ceph/ -func peerConfigMapAndSecretVolumeAndMount(siteName, clientName, peerSiteUUID string) []v1.VolumeProjection { - // Projection list - secretAndConfigMapVolumeProjections := []v1.VolumeProjection{} - - // CM - configMapName := generatePeerCephConfigFileConfigMapName(peerSiteUUID) // configmap name and name of volume - configMapFile := fmt.Sprintf("%s.conf", siteName) - mode := int32(0444) - projectionConfigMap := &v1.ConfigMapProjection{Items: []v1.KeyToPath{{Key: peerCephConfigKey, Path: configMapFile, Mode: &mode}}} - projectionConfigMap.Name = configMapName - - // Secret - secretName := generatePeerKeyringSecretName(peerSiteUUID) // secret name and name of volume - secretFile := fmt.Sprintf("%s.%s.keyring", siteName, clientName) - projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: peerCephKeyringKey, Path: secretFile, Mode: &mode}}} - projectionSecret.Name = secretName - - configMapProjection := v1.VolumeProjection{ - ConfigMap: projectionConfigMap, - } - secretAndConfigMapVolumeProjections = append(secretAndConfigMapVolumeProjections, configMapProjection) - secretProjection := v1.VolumeProjection{ - Secret: projectionSecret, - } - secretAndConfigMapVolumeProjections = append(secretAndConfigMapVolumeProjections, secretProjection) - - return secretAndConfigMapVolumeProjections -} diff --git a/pkg/operator/ceph/cluster/rbd/spec_test.go b/pkg/operator/ceph/cluster/rbd/spec_test.go index c529f96f8..8345d88b6 100644 --- a/pkg/operator/ceph/cluster/rbd/spec_test.go +++ b/pkg/operator/ceph/cluster/rbd/spec_test.go @@ -99,15 +99,4 @@ func TestPodSpec(t *testing.T) { podTemplate.RunFullSuite(config.RbdMirrorType, "a", AppName, "ns", "ceph/ceph:myceph", "200", "100", "600", "300", /* resources */ "my-priority-class") - - // Test with peer - rbdMirror.Spec.Peers.SecretNames = append(rbdMirror.Spec.Peers.SecretNames, "foo") - p := cephv1.PeersSpec{UUID: "c9838c14-d9a1-4e69-b51e-09ff0a4d617c", SiteName: "foo", ClientName: "client.rbd-mirror-peer"} - r.peers["foo"] = &peerSpec{poolName: "foo", info: &cephv1.PoolMirroringInfo{Peers: []cephv1.PeersSpec{p}}} - d, err = r.makeDeployment(&daemonConf, rbdMirror) - assert.NoError(t, err) - // We now have the volume for the ConfigMap and the Secret - assert.Equal(t, 4, len(d.Spec.Template.Spec.Volumes)) - assert.Equal(t, 3, len(d.Spec.Template.Spec.Volumes[0].Projected.Sources)) - assert.Equal(t, 4, len(d.Spec.Template.Spec.Containers[0].VolumeMounts)) }