csi: update port to 3300 if msgr2 is required
When msgr2 is required, ensure the mon endpoints passed to
the csi configmap are on port 3300. The mons will be
listening on both 6789 and 3300. Existing volumes can
continue using port 6789 while new volumes will use
port 3300.
Signed-off-by: travisn <tnielsen@redhat.com>
(cherry picked from commit 0484a3973f)
This commit is contained in:
1 parent
dadde56367
commit
12d79edd1d
8 files changed
+78
-7
No files matched your search
@@ -113,7 +113,8 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
|
||||
}
|
||||
|
||||
// Save CSI configmap
|
||||
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: csi.MonEndpoints(cluster.ClusterInfo.Monitors)})
|
||||
monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.Monitors, cluster.Spec.RequireMsgr2())
|
||||
err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: monEndpoints})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update csi cluster config")
|
||||
}
|
||||
|
||||
@@ -1078,7 +1078,8 @@ func (c *Cluster) saveMonConfig() error {
|
||||
return errors.Wrap(err, "failed to write connection config for new mons")
|
||||
}
|
||||
|
||||
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: csi.MonEndpoints(c.ClusterInfo.Monitors)}); err != nil {
|
||||
monEndpoints := csi.MonEndpoints(c.ClusterInfo.Monitors, c.spec.RequireMsgr2())
|
||||
if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: monEndpoints}); err != nil {
|
||||
return errors.Wrap(err, "failed to update csi cluster config")
|
||||
}
|
||||
|
||||
|
||||
@@ -19,11 +19,14 @@ package csi
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -99,10 +102,23 @@ func formatCsiClusterConfig(cc csiClusterConfig) (string, error) {
|
||||
return string(ccJson), nil
|
||||
}
|
||||
|
||||
func MonEndpoints(mons map[string]*cephclient.MonInfo) []string {
|
||||
func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []string {
|
||||
endpoints := make([]string, 0)
|
||||
for _, m := range mons {
|
||||
endpoints = append(endpoints, m.Endpoint)
|
||||
endpoint := m.Endpoint
|
||||
if requireMsgr2 {
|
||||
logger.Debugf("evaluating mon %q for msgr1 on endpoint %q", m.Name, m.Endpoint)
|
||||
if strings.HasSuffix(m.Endpoint, fmt.Sprintf(":%d", client.Msgr1port)) {
|
||||
parts := strings.Split(m.Endpoint, ":")
|
||||
if len(parts) != 2 {
|
||||
logger.Errorf("endpoint %q does not contain two parts to extract the port", m.Endpoint)
|
||||
continue
|
||||
}
|
||||
endpoint = fmt.Sprintf("%s:%d", parts[0], client.Msgr2port)
|
||||
logger.Debugf("mon %q will use the msgrv2 port: %q", m.Name, endpoint)
|
||||
}
|
||||
}
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
return endpoints
|
||||
}
|
||||
|
||||
@@ -17,8 +17,10 @@ limitations under the License.
|
||||
package csi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -336,3 +338,51 @@ func contains(src, dest []string) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func TestMonEndpoints(t *testing.T) {
|
||||
monInfo := map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: "1.2.3.4:6789"},
|
||||
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
|
||||
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
|
||||
}
|
||||
|
||||
t.Run("msgrv1 when not require msgr2", func(t *testing.T) {
|
||||
endpoints := MonEndpoints(monInfo, false)
|
||||
assert.Equal(t, 3, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "6789")
|
||||
})
|
||||
|
||||
t.Run("convert to msgr2", func(t *testing.T) {
|
||||
endpoints := MonEndpoints(monInfo, true)
|
||||
assert.Equal(t, 3, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "3300")
|
||||
})
|
||||
|
||||
t.Run("remains msgr2", func(t *testing.T) {
|
||||
monInfo := map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: "1.2.3.4:3300"},
|
||||
"b": {Name: "b", Endpoint: "1.2.3.5:3300"},
|
||||
"c": {Name: "c", Endpoint: "1.2.3.6:3300"},
|
||||
}
|
||||
endpoints := MonEndpoints(monInfo, false)
|
||||
assert.Equal(t, 3, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "3300")
|
||||
})
|
||||
|
||||
t.Run("bad endpoint", func(t *testing.T) {
|
||||
monInfo := map[string]*cephclient.MonInfo{
|
||||
"a": {Name: "a", Endpoint: "1.2.3.4:5:6789"},
|
||||
"b": {Name: "b", Endpoint: "1.2.3.5:6789"},
|
||||
"c": {Name: "c", Endpoint: "1.2.3.6:6789"},
|
||||
}
|
||||
endpoints := MonEndpoints(monInfo, true)
|
||||
assert.Equal(t, 2, len(endpoints))
|
||||
verifyEndpointPort(t, endpoints, "3300")
|
||||
})
|
||||
}
|
||||
|
||||
func verifyEndpointPort(t *testing.T, endpoints []string, expectedPort string) {
|
||||
for _, endpoint := range endpoints {
|
||||
assert.True(t, strings.HasSuffix(endpoint, expectedPort))
|
||||
}
|
||||
}
|
||||
@@ -874,7 +874,7 @@ func (r *ReconcileCSI) configureHolder(driver driverDetails, c ClusterDetail, tp
|
||||
}
|
||||
|
||||
clusterConfigEntry := &CsiClusterConfigEntry{
|
||||
Monitors: MonEndpoints(c.clusterInfo.Monitors),
|
||||
Monitors: MonEndpoints(c.clusterInfo.Monitors, c.cluster.Spec.RequireMsgr2()),
|
||||
RBD: &CsiRBDSpec{},
|
||||
CephFS: &CsiCephFSSpec{},
|
||||
NFS: &CsiNFSSpec{},
|
||||
|
||||
@@ -267,7 +267,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) updateClusterConfig(cephFilesyst
|
||||
// config map, so no special care is needed in this controller
|
||||
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
|
||||
Namespace: r.clusterInfo.Namespace,
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
|
||||
CephFS: &csi.CsiCephFSSpec{
|
||||
SubvolumeGroup: cephFilesystemSubVolumeGroup.Name,
|
||||
},
|
||||
|
||||
@@ -259,7 +259,7 @@ func (r *ReconcileCephBlockPoolRadosNamespace) updateClusterConfig(cephBlockPool
|
||||
// config map, so no special care is needed in this controller
|
||||
csiClusterConfigEntry := csi.CsiClusterConfigEntry{
|
||||
Namespace: r.clusterInfo.Namespace,
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors),
|
||||
Monitors: csi.MonEndpoints(r.clusterInfo.Monitors, cephCluster.Spec.RequireMsgr2()),
|
||||
RBD: &csi.CsiRBDSpec{
|
||||
RadosNamespace: cephBlockPoolRadosNamespace.Name,
|
||||
},
|
||||
|
||||
@@ -437,6 +437,9 @@ func (s *UpgradeSuite) upgradeToMaster() {
|
||||
s.installer.Manifests = installer.NewCephManifests(s.settings)
|
||||
|
||||
if s.settings.UseHelm {
|
||||
logger.Info("Requiring msgr2 during helm upgrade to test the port conversion from 6789 to 3300")
|
||||
s.settings.RequireMsgr2 = true
|
||||
|
||||
// Upgrade the operator chart
|
||||
err := s.installer.UpgradeRookOperatorViaHelm()
|
||||
require.NoError(s.T(), err, "failed to upgrade the operator chart")
|
||||
|
||||
Reference in new issue
Block a user