csi: update port to 3300 if msgr2 is required

When msgr2 is required, ensure the mon endpoints passed to
the csi configmap are on port 3300. The mons will be
listening on both 6789 and 3300. Existing volumes can
continue using port 6789 while new volumes will use
port 3300.

Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
travisn committed 2023-03-10 10:34:56 -07:00
1 parent eeb9403774
commit 0484a3973f
8 files changed
+78 -7

No files matched your search

@@ -113,7 +113,8 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
}
// Save CSI configmap
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: csi.MonEndpoints(cluster.ClusterInfo.Monitors)})
monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.Monitors, cluster.Spec.RequireMsgr2())
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: monEndpoints})
if err != nil {
return errors.Wrap(err, "failed to update csi cluster config")
}
+2 -1
View File
@@ -1078,7 +1078,8 @@ func (c *Cluster) saveMonConfig() error {
return errors.Wrap(err, "failed to write connection config for new mons")
}
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: csi.MonEndpoints(c.ClusterInfo.Monitors)}); err != nil {
monEndpoints := csi.MonEndpoints(c.ClusterInfo.Monitors, c.spec.RequireMsgr2())
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: monEndpoints}); err != nil {
return errors.Wrap(err, "failed to update csi cluster config")
}
+18 -2
View File
@@ -19,11 +19,14 @@ package csi
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
@@ -99,10 +102,23 @@ func formatCsiClusterConfig(cc csiClusterConfig) (string, error) {
return string(ccJson), nil
}
func MonEndpoints(mons map[string]*cephclient.MonInfo) []string {
func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []string {
endpoints := make([]string, 0)
for _, m := range mons {
endpoints = append(endpoints, m.Endpoint)
endpoint := m.Endpoint
if requireMsgr2 {
logger.Debugf("evaluating mon %q for msgr1 on endpoint %q", m.Name, m.Endpoint)
if strings.HasSuffix(m.Endpoint, fmt.Sprintf(":%d", client.Msgr1port)) {
parts := strings.Split(m.Endpoint, ":")
if len(parts) != 2 {
logger.Errorf("endpoint %q does not contain two parts to extract the port", m.Endpoint)
continue
}
endpoint = fmt.Sprintf("%s:%d", parts[0], client.Msgr2port)
logger.Debugf("mon %q will use the msgrv2 port: %q", m.Name, endpoint)
}
}
endpoints = append(endpoints, endpoint)
}
return endpoints
}
@@ -17,8 +17,10 @@ limitations under the License.
package csi
import (
"strings"
"testing"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/stretchr/testify/assert"
)
@@ -336,3 +338,51 @@ func contains(src, dest []string) bool {
return false
}
func TestMonEndpoints(t *testing.T) {
monInfo := map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: "1.2.3.4:6789"},
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
}
t.Run("msgrv1 when not require msgr2", func(t *testing.T) {
endpoints := MonEndpoints(monInfo, false)
assert.Equal(t, 3, len(endpoints))
verifyEndpointPort(t, endpoints, "6789")
})
t.Run("convert to msgr2", func(t *testing.T) {
endpoints := MonEndpoints(monInfo, true)
assert.Equal(t, 3, len(endpoints))
verifyEndpointPort(t, endpoints, "3300")
})
t.Run("remains msgr2", func(t *testing.T) {
monInfo := map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: "1.2.3.4:3300"},
"b": {Name: "b", Endpoint: "1.2.3.5:3300"},
"c": {Name: "c", Endpoint: "1.2.3.6:3300"},
}
endpoints := MonEndpoints(monInfo, false)
assert.Equal(t, 3, len(endpoints))
verifyEndpointPort(t, endpoints, "3300")
})
t.Run("bad endpoint", func(t *testing.T) {
monInfo := map[string]*cephclient.MonInfo{
"a": {Name: "a", Endpoint: "1.2.3.4:5:6789"},
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
}
endpoints := MonEndpoints(monInfo, true)
assert.Equal(t, 2, len(endpoints))
verifyEndpointPort(t, endpoints, "3300")
})
}
func verifyEndpointPort(t *testing.T, endpoints []string, expectedPort string) {
for _, endpoint := range endpoints {
assert.True(t, strings.HasSuffix(endpoint, expectedPort))
}
}
+1 -1
View File
@@ -874,7 +874,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp
}
clusterConfigEntry := &CsiClusterConfigEntry{
Monitors: MonEndpoints(c.clusterInfo.Monitors),
Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()),
RBD: &CsiRBDSpec{},
CephFS: &CsiCephFSSpec{},
NFS: &CsiNFSSpec{},
@@ -267,7 +267,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesyst
// config map, so no special care is needed in this controller
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
Namespace: r.clusterInfo.Namespace,
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
CephFS: &csi.CsiCephFSSpec{
SubvolumeGroup: cephFilesystemSubVolumeGroup.Name,
},
@@ -259,7 +259,7 @@ func (r *ReconcileCephBlockPoolRadosNamespace) updateClusterConfig(cephBlockPool
// config map, so no special care is needed in this controller
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
Namespace: r.clusterInfo.Namespace,
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
RBD: &csi.CsiRBDSpec{
RadosNamespace: cephBlockPoolRadosNamespace.Name,
},
+3
View File
@@ -437,6 +437,9 @@ func (s *UpgradeSuite) upgradeToMaster() {
s.installer.Manifests = installer.NewCephManifests(s.settings)
if s.settings.UseHelm {
logger.Info("Requiring msgr2 during helm upgrade to test the port conversion from 6789 to 3300")
s.settings.RequireMsgr2 = true
// Upgrade the operator chart
err := s.installer.UpgradeRookOperatorViaHelm()
require.NoError(s.T(), err, "failed to upgrade the operator chart")