csi: update port to 3300 if msgr2 is required
When msgr2 is required, ensure the mon endpoints passed to the csi configmap are on port 3300. The mons will be listening on both 6789 and 3300. Existing volumes can continue using port 6789 while new volumes will use port 3300. Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
8 files changed
+78
-7
No files matched your search
@@ -113,7 +113,8 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
}
|
||||
|
||||
// Save CSI configmap
|
||||
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: csi.MonEndpoints(cluster.ClusterInfo.Monitors)})
|
||||
monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.Monitors, cluster.Spec.RequireMsgr2())
|
||||
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: monEndpoints})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update csi cluster config")
|
||||
}
|
||||
|
||||
@@ -1078,7 +1078,8 @@ func (c *Cluster) saveMonConfig() error {
|
||||
return errors.Wrap(err, "failed to write connection config for new mons")
|
||||
}
|
||||
|
||||
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: csi.MonEndpoints(c.ClusterInfo.Monitors)}); err != nil {
|
||||
monEndpoints := csi.MonEndpoints(c.ClusterInfo.Monitors, c.spec.RequireMsgr2())
|
||||
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: monEndpoints}); err != nil {
|
||||
return errors.Wrap(err, "failed to update csi cluster config")
|
||||
}
|
||||
|
||||
|
||||
@@ -19,11 +19,14 @@ package csi
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -99,10 +102,23 @@ func formatCsiClusterConfig(cc csiClusterConfig) (string, error) {
|
||||
return string(ccJson), nil
|
||||
}
|
||||
|
||||
func MonEndpoints(mons map[string]*cephclient.MonInfo) []string {
|
||||
func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []string {
|
||||
endpoints := make([]string, 0)
|
||||
for _, m := range mons {
|
||||
endpoints = append(endpoints, m.Endpoint)
|
||||
endpoint := m.Endpoint
|
||||
if requireMsgr2 {
|
||||
logger.Debugf("evaluating mon %q for msgr1 on endpoint %q", m.Name, m.Endpoint)
|
||||
if strings.HasSuffix(m.Endpoint, fmt.Sprintf(":%d", client.Msgr1port)) {
|
||||
parts := strings.Split(m.Endpoint, ":")
|
||||
if len(parts) != 2 {
|
||||
logger.Errorf("endpoint %q does not contain two parts to extract the port", m.Endpoint)
|
||||
continue
|
||||
}
|
||||
endpoint = fmt.Sprintf("%s:%d", parts[0], client.Msgr2port)
|
||||
logger.Debugf("mon %q will use the msgrv2 port: %q", m.Name, endpoint)
|
||||
}
|
||||
}
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
return endpoints
|
||||
}
|
||||
|
||||
@@ -17,8 +17,10 @@ limitations under the License.
|
||||
package csi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -336,3 +338,51 @@ func contains(src, dest []string) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func TestMonEndpoints(t *testing.T) {
|
||||
monInfo := map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: "1.2.3.4:6789"},
|
||||
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
|
||||
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
|
||||
}
|
||||
|
||||
t.Run("msgrv1 when not require msgr2", func(t *testing.T) {
|
||||
endpoints := MonEndpoints(monInfo, false)
|
||||
assert.Equal(t, 3, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "6789")
|
||||
})
|
||||
|
||||
t.Run("convert to msgr2", func(t *testing.T) {
|
||||
endpoints := MonEndpoints(monInfo, true)
|
||||
assert.Equal(t, 3, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "3300")
|
||||
})
|
||||
|
||||
t.Run("remains msgr2", func(t *testing.T) {
|
||||
monInfo := map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: "1.2.3.4:3300"},
|
||||
"b": {Name: "b", Endpoint: "1.2.3.5:3300"},
|
||||
"c": {Name: "c", Endpoint: "1.2.3.6:3300"},
|
||||
}
|
||||
endpoints := MonEndpoints(monInfo, false)
|
||||
assert.Equal(t, 3, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "3300")
|
||||
})
|
||||
|
||||
t.Run("bad endpoint", func(t *testing.T) {
|
||||
monInfo := map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: "1.2.3.4:5:6789"},
|
||||
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
|
||||
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
|
||||
}
|
||||
endpoints := MonEndpoints(monInfo, true)
|
||||
assert.Equal(t, 2, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "3300")
|
||||
})
|
||||
}
|
||||
|
||||
func verifyEndpointPort(t *testing.T, endpoints []string, expectedPort string) {
|
||||
for _, endpoint := range endpoints {
|
||||
assert.True(t, strings.HasSuffix(endpoint, expectedPort))
|
||||
}
|
||||
}
|
||||
@@ -874,7 +874,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp
|
||||
}
|
||||
|
||||
clusterConfigEntry := &CsiClusterConfigEntry{
|
||||
Monitors: MonEndpoints(c.clusterInfo.Monitors),
|
||||
Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()),
|
||||
RBD: &CsiRBDSpec{},
|
||||
CephFS: &CsiCephFSSpec{},
|
||||
NFS: &CsiNFSSpec{},
|
||||
|
||||
@@ -267,7 +267,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesyst
|
||||
// config map, so no special care is needed in this controller
|
||||
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
|
||||
Namespace: r.clusterInfo.Namespace,
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
|
||||
CephFS: &csi.CsiCephFSSpec{
|
||||
SubvolumeGroup: cephFilesystemSubVolumeGroup.Name,
|
||||
},
|
||||
|
||||
@@ -259,7 +259,7 @@ func (r *ReconcileCephBlockPoolRadosNamespace) updateClusterConfig(cephBlockPool
|
||||
// config map, so no special care is needed in this controller
|
||||
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
|
||||
Namespace: r.clusterInfo.Namespace,
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
|
||||
RBD: &csi.CsiRBDSpec{
|
||||
RadosNamespace: cephBlockPoolRadosNamespace.Name,
|
||||
},
|
||||
|
||||
@@ -437,6 +437,9 @@ func (s *UpgradeSuite) upgradeToMaster() {
|
||||
s.installer.Manifests = installer.NewCephManifests(s.settings)
|
||||
|
||||
if s.settings.UseHelm {
|
||||
logger.Info("Requiring msgr2 during helm upgrade to test the port conversion from 6789 to 3300")
|
||||
s.settings.RequireMsgr2 = true
|
||||
|
||||
// Upgrade the operator chart
|
||||
err := s.installer.UpgradeRookOperatorViaHelm()
|
||||
require.NoError(s.T(), err, "failed to upgrade the operator chart")
|
||||
|
||||
Reference in new issue
Block a user