mon: options for enabling msgr2 encryption and compression

msgr2 allows for encryption and/or compression across the wire.
Settings for enabling the encryption and compression are now
available in the cluster CR to that ceph will be automatically
configured with these settings when desired.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2022-03-10 11:50:42 -07:00
1 parent cec3942d72
commit 03c7164b15
24 files changed
+364 -50

No files matched your search

+13
View File
@@ -305,6 +305,19 @@ Configure the network that will be enabled for the cluster and services.
* `selectors`: List the network selector(s) that will be used associated by a key.
* `ipFamily`: Specifies the network stack Ceph daemons should listen on.
* `dualStack`: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks.
* `connections`: Settings for network connections using Ceph's msgr2 protocol
* `encryption`: Settings for encryption on the wire to Ceph daemons
* `enabled`: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network.
The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between
Ceph daemons will be encrypted. When encryption is not enabled, clients still establish a strong initial authentication
and data integrity is still validated with a crc check.
IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only,
set "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class.
The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes.
* `compression`:
* `enabled`: Whether to compress the data in transit across the wire. The default is false.
Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption.
> **NOTE:** Changing networking configuration after a Ceph cluster has been deployed is NOT
> supported and will result in a non-functioning cluster.
+2
View File
@@ -9,4 +9,6 @@
* The number of mgr daemons for example clusters is increased to 2 from 1, resulting in a standby mgr daemon.
If the active mgr goes down, Ceph will update the passive mgr to be active, and rook will update all the services
with the label app=rook-ceph-mgr to direct traffic to the new active mgr.
* Network encryption is configurable with settings in the CephCluster CR. Requires the 5.11 kernel or newer.
* Network compression is configurable with settings in the CephCluster CR. Requires Ceph Quincy (v17) or newer.
* Add support for custom ceph.conf for csi pods. See #9567
@@ -1653,6 +1653,27 @@ spec:
description: Network related configuration
nullable: true
properties:
connections:
description: Settings for network connections such as compression and encryption across the wire.
nullable: true
properties:
compression:
description: Compression settings for the network connections.
nullable: true
properties:
enabled:
description: Whether to compress the data in transit across the wire. The default is not set. Requires Ceph Quincy (v17) or newer.
type: boolean
type: object
encryption:
description: Encryption settings for the network connections.
nullable: true
properties:
enabled:
description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is not set. Even if encryption is not enabled, clients still establish a strong initial authentication for the connection and data integrity is still validated with a crc check. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted.
type: boolean
type: object
type: object
dualStack:
description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6
type: boolean
+13
View File
@@ -81,6 +81,19 @@ spec:
# deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions.
rulesNamespace: rook-ceph
network:
connections:
# Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network.
# The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted.
# When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check.
# IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only,
# you can set the "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class.
# The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes.
encryption:
enabled: false
# Whether to compress the data in transit across the wire. The default is false.
# Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption.
compression:
enabled: false
# enable host networking
#provider: host
# enable the Multus network provider
+21
View File
@@ -1652,6 +1652,27 @@ spec:
description: Network related configuration
nullable: true
properties:
connections:
description: Settings for network connections such as compression and encryption across the wire.
nullable: true
properties:
compression:
description: Compression settings for the network connections.
nullable: true
properties:
enabled:
description: Whether to compress the data in transit across the wire. The default is not set. Requires Ceph Quincy (v17) or newer.
type: boolean
type: object
encryption:
description: Encryption settings for the network connections.
nullable: true
properties:
enabled:
description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is not set. Even if encryption is not enabled, clients still establish a strong initial authentication for the connection and data integrity is still validated with a crc check. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted.
type: boolean
type: object
type: object
dualStack:
description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6
type: boolean
+14
View File
@@ -29,6 +29,20 @@ import (
// will be registered for the validating webhook.
var _ webhook.Validator = &CephCluster{}
// RequireMsgr2 checks if the network settings require the msgr2 protocol
func (c *ClusterSpec) RequireMsgr2() bool {
if c.Network.Connections == nil {
return false
}
if c.Network.Connections.Compression != nil && c.Network.Connections.Compression.Enabled {
return true
}
if c.Network.Connections.Encryption != nil && c.Network.Connections.Encryption.Enabled {
return true
}
return false
}
func (c *ClusterSpec) IsStretchCluster() bool {
return c.Mon.StretchCluster != nil && len(c.Mon.StretchCluster.Zones) > 0
}
+36
View File
@@ -1916,6 +1916,12 @@ type NetworkSpec struct {
// +optional
Selectors map[string]string `json:"selectors,omitempty"`
// Settings for network connections such as compression and encryption across the
// wire.
// +nullable
// +optional
Connections *ConnectionsSpec `json:"connections,omitempty"`
// HostNetwork to enable host network
// +optional
HostNetwork bool `json:"hostNetwork,omitempty"`
@@ -1931,6 +1937,36 @@ type NetworkSpec struct {
DualStack bool `json:"dualStack,omitempty"`
}
type ConnectionsSpec struct {
// Encryption settings for the network connections.
// +nullable
// +optional
Encryption *EncryptionSpec `json:"encryption,omitempty"`
// Compression settings for the network connections.
// +nullable
// +optional
Compression *CompressionSpec `json:"compression,omitempty"`
}
type EncryptionSpec struct {
// Whether to encrypt the data in transit across the wire to prevent eavesdropping
// the data on the network. The default is not set. Even if encryption is not enabled,
// clients still establish a strong initial authentication for the connection
// and data integrity is still validated with a crc check. When encryption is enabled,
// all communication between clients and Ceph daemons, or between Ceph daemons will
// be encrypted.
// +optional
Enabled bool `json:"enabled,omitempty"`
}
type CompressionSpec struct {
// Whether to compress the data in transit across the wire.
// The default is not set. Requires Ceph Quincy (v17) or newer.
// +optional
Enabled bool `json:"enabled,omitempty"`
}
// DisruptionManagementSpec configures management of daemon disruptions
type DisruptionManagementSpec struct {
// This enables management of poddisruptionbudgets
@@ -1715,6 +1715,22 @@ func (in *ClusterVersion) DeepCopy() *ClusterVersion {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CompressionSpec) DeepCopyInto(out *CompressionSpec) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CompressionSpec.
func (in *CompressionSpec) DeepCopy() *CompressionSpec {
if in == nil {
return nil
}
out := new(CompressionSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Condition) DeepCopyInto(out *Condition) {
*out = *in
@@ -1733,6 +1749,32 @@ func (in *Condition) DeepCopy() *Condition {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ConnectionsSpec) DeepCopyInto(out *ConnectionsSpec) {
*out = *in
if in.Encryption != nil {
in, out := &in.Encryption, &out.Encryption
*out = new(EncryptionSpec)
**out = **in
}
if in.Compression != nil {
in, out := &in.Compression, &out.Compression
*out = new(CompressionSpec)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionsSpec.
func (in *ConnectionsSpec) DeepCopy() *ConnectionsSpec {
if in == nil {
return nil
}
out := new(ConnectionsSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CrashCollectorSpec) DeepCopyInto(out *CrashCollectorSpec) {
*out = *in
@@ -1839,6 +1881,22 @@ func (in *DisruptionManagementSpec) DeepCopy() *DisruptionManagementSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *EncryptionSpec) DeepCopyInto(out *EncryptionSpec) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionSpec.
func (in *EncryptionSpec) DeepCopy() *EncryptionSpec {
if in == nil {
return nil
}
out := new(EncryptionSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ErasureCodedSpec) DeepCopyInto(out *ErasureCodedSpec) {
*out = *in
@@ -2660,6 +2718,11 @@ func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) {
(*out)[key] = val
}
}
if in.Connections != nil {
in, out := &in.Connections, &out.Connections
*out = new(ConnectionsSpec)
(*in).DeepCopyInto(*out)
}
return
}
+23 -6
View File
@@ -20,9 +20,11 @@ package client
import (
"fmt"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -44,6 +46,8 @@ const (
DefaultKeyringFile = "keyring"
// Msgr2port is the listening port of the messenger v2 protocol
Msgr2port = 3300
// Msgr1port is the listening port of the messenger v1 protocol
Msgr1port = 6789
)
var (
@@ -186,7 +190,7 @@ func CreateDefaultCephConfig(context *clusterd.Context, clusterInfo *ClusterInfo
// extract a list of just the monitor names, which will populate the "mon initial members"
// and "mon hosts" global config field
monMembers, monHosts := PopulateMonHostMembers(clusterInfo.Monitors)
monMembers, monHosts := PopulateMonHostMembers(clusterInfo)
conf := &CephConfig{
GlobalConfig: &GlobalConfig{
@@ -242,15 +246,28 @@ func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath st
// PopulateMonHostMembers extracts a list of just the monitor names, which will populate the "mon initial members"
// and "mon hosts" global config field
func PopulateMonHostMembers(monitors map[string]*MonInfo) ([]string, []string) {
monMembers := make([]string, len(monitors))
monHosts := make([]string, len(monitors))
func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) {
monMembers := make([]string, len(clusterInfo.Monitors))
monHosts := make([]string, len(clusterInfo.Monitors))
i := 0
for _, monitor := range monitors {
for _, monitor := range clusterInfo.Monitors {
monMembers[i] = monitor.Name
monIP := cephutil.GetIPFromEndpoint(monitor.Endpoint)
monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port)
if clusterInfo.RequireMsgr2 {
monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port)
} else {
// Detect the current port if the mon already exists
// so the same msgr1 port can be preserved if needed (6789 or 6790)
currentMonPort := cephutil.GetPortFromEndpoint(monitor.Endpoint)
// Ensure we're setting a msgr1 port, rather than duplicating msgr2
if currentMonPort == Msgr2port {
currentMonPort = Msgr1port
}
msgr2Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(Msgr2port)))
msgr1Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(currentMonPort)))
monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]"
}
i++
}
+1
View File
@@ -42,6 +42,7 @@ type ClusterInfo struct {
CephVersion cephver.CephVersion
Namespace string
OwnerInfo *k8sutil.OwnerInfo
RequireMsgr2 bool
// Hide the name of the cluster since in 99% of uses we want to use the cluster namespace.
// If the CR name is needed, access it through the NamespacedName() method.
name string
+1 -1
View File
@@ -61,7 +61,7 @@ func CreateTestClusterInfo(monCount int) *client.ClusterInfo {
id := mons[i]
c.Monitors[id] = &client.MonInfo{
Name: id,
Endpoint: fmt.Sprintf("1.2.3.%d:6789", (i + 1)),
Endpoint: fmt.Sprintf("1.2.3.%d:3300", (i + 1)),
}
}
c.SetName(c.Namespace)
+48
View File
@@ -191,6 +191,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error {
} else {
clusterInfo.OwnerInfo = cluster.ownerInfo
clusterInfo.SetName(c.namespacedName.Name)
clusterInfo.RequireMsgr2 = cluster.Spec.RequireMsgr2()
cluster.ClusterInfo = clusterInfo
}
// If the local cluster has already been configured, immediately start monitoring the cluster.
@@ -505,6 +506,10 @@ func (c *cluster) postMonStartupActions() error {
}
}
if err := c.configureMsgr2(); err != nil {
return errors.Wrap(err, "failed to configured msgr2")
}
crushRoot := client.GetCrushRootFromSpec(c.Spec)
if crushRoot != "default" {
// Remove the root=default and replicated_rule which are created by
@@ -523,3 +528,46 @@ func (c *cluster) postMonStartupActions() error {
return nil
}
func (c *cluster) configureMsgr2() error {
if c.Spec.Network.Connections == nil {
return nil
}
// Set network encryption
monStore := config.GetMonStore(c.context, c.ClusterInfo)
if c.Spec.Network.Connections.Encryption != nil {
encryptionSetting := "crc secure"
if c.Spec.Network.Connections.Encryption.Enabled {
encryptionSetting = "secure"
}
logger.Infof("setting msgr2 encryption mode to %q", encryptionSetting)
if err := monStore.Set("global", "ms_cluster_mode", encryptionSetting); err != nil {
return errors.Wrapf(err, "failed to set ms_cluster_mode to %q", encryptionSetting)
}
if err := monStore.Set("global", "ms_service_mode", encryptionSetting); err != nil {
return errors.Wrapf(err, "failed to set ms_service_mode to %q", encryptionSetting)
}
if err := monStore.Set("global", "ms_client_mode", encryptionSetting); err != nil {
return errors.Wrapf(err, "failed to set ms_client_mode to %q", encryptionSetting)
}
}
// Set network compression
if c.Spec.Network.Connections.Compression != nil {
if c.ClusterInfo.CephVersion.IsAtLeastQuincy() {
compressionSetting := "none"
if c.Spec.Network.Connections.Compression.Enabled {
compressionSetting = "force"
}
logger.Infof("setting msgr2 compression mode to %q", compressionSetting)
if err := monStore.Set("global", "ms_osd_compress_mode", compressionSetting); err != nil {
return errors.Wrapf(err, "failed to set ms_osd_compress_mode to %q", compressionSetting)
}
} else {
logger.Warningf("network compression requires Ceph Quincy (v17) or newer, skipping for current ceph %q", c.ClusterInfo.CephVersion.String())
}
}
return nil
}
+69
View File
@@ -26,6 +26,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/version"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -149,3 +150,71 @@ func TestPreMonChecks(t *testing.T) {
assert.False(t, unsetSkipSanity)
})
}
func TestConfigureMsgr2(t *testing.T) {
type fields struct {
encryptionExpected bool
compressionExpected bool
disabledBothExpected bool
cephVersion version.CephVersion
Spec *cephv1.ClusterSpec
}
tests := []struct {
name string
fields fields
}{
{"default settings", fields{false, false, false, version.CephVersion{Major: 15}, &cephv1.ClusterSpec{}}},
{"encryption enabled", fields{true, false, false, version.CephVersion{Major: 16}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Encryption: &cephv1.EncryptionSpec{Enabled: true}}}}}},
{"compression enabled old version", fields{false, false, false, version.CephVersion{Major: 16}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Compression: &cephv1.CompressionSpec{Enabled: true}}}}}},
{"compression enabled good version", fields{false, true, false, version.CephVersion{Major: 17}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Compression: &cephv1.CompressionSpec{Enabled: true}}}}}},
{"both enabled", fields{true, true, false, version.CephVersion{Major: 17}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Encryption: &cephv1.EncryptionSpec{Enabled: true}, Compression: &cephv1.CompressionSpec{Enabled: true}}}}}},
{"both disabled", fields{false, false, true, version.CephVersion{Major: 17}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Encryption: &cephv1.EncryptionSpec{Enabled: false}, Compression: &cephv1.CompressionSpec{Enabled: false}}}}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
enabledCompression := false
disabledCompression := false
enabledEncryption := false
disabledEncryption := false
executor := &exectest.MockExecutor{
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
logger.Infof("Execute: %s %v", command, args)
if args[0] == "config" && args[1] == "set" {
if args[3] == "ms_osd_compress_mode" {
if args[4] == "force" {
enabledCompression = true
} else if args[4] == "none" {
disabledCompression = true
}
}
if args[3] == "ms_cluster_mode" {
if args[4] == "secure" {
enabledEncryption = true
} else if args[4] == "crc secure" {
disabledEncryption = true
}
}
return "", nil
}
return "", errors.Errorf("unrecognized command")
},
}
clusterInfo := client.AdminTestClusterInfo("rook-ceph")
clusterInfo.CephVersion = tt.fields.cephVersion
context := &clusterd.Context{Clientset: testop.New(t, 3), Executor: executor}
c := &cluster{
ClusterInfo: clusterInfo,
Spec: tt.fields.Spec,
context: context,
}
err := c.configureMsgr2()
assert.NoError(t, err)
assert.Equal(t, tt.fields.compressionExpected, enabledCompression)
assert.Equal(t, tt.fields.encryptionExpected, enabledEncryption)
assert.Equal(t, tt.fields.disabledBothExpected, disabledCompression)
assert.Equal(t, tt.fields.disabledBothExpected, disabledEncryption)
})
}
}
+4 -4
View File
@@ -319,8 +319,8 @@ func TestCheckHealthNotFound(t *testing.T) {
// Check if the two mons are found in the configmap
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
assert.Nil(t, err)
if cm.Data[EndpointDataKey] != "a=1.2.3.1:6789,b=1.2.3.2:6789" {
assert.Equal(t, "b=1.2.3.2:6789,a=1.2.3.1:6789", cm.Data[EndpointDataKey])
if cm.Data[EndpointDataKey] != "a=1.2.3.1:3300,b=1.2.3.2:3300" {
assert.Equal(t, "b=1.2.3.2:3300,a=1.2.3.1:3300", cm.Data[EndpointDataKey])
}
// Because the mon a isn't in the MonInQuorumResponse() this will create a new mon
@@ -334,8 +334,8 @@ func TestCheckHealthNotFound(t *testing.T) {
// recheck that the "not found" mon has been replaced with a new one
cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
assert.Nil(t, err)
if cm.Data[EndpointDataKey] != "a=1.2.3.1:6789,f=:3300" {
assert.Equal(t, "f=:3300,a=1.2.3.1:6789", cm.Data[EndpointDataKey])
if cm.Data[EndpointDataKey] != "a=1.2.3.1:3300,f=:3300" {
assert.Equal(t, "f=:3300,a=1.2.3.1:3300", cm.Data[EndpointDataKey])
}
}
+1
View File
@@ -500,6 +500,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
c.ClusterInfo.OwnerInfo = c.ownerInfo
c.ClusterInfo.Context = context
c.ClusterInfo.SetName(clusterName)
c.ClusterInfo.RequireMsgr2 = c.spec.RequireMsgr2()
// save cluster monitor config
if err = c.saveMonConfig(); err != nil {
+2 -2
View File
@@ -279,7 +279,7 @@ func TestPersistMons(t *testing.T) {
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey])
assert.Equal(t, "a=1.2.3.1:3300", cm.Data[EndpointDataKey])
assert.Equal(t, map[string]string{"key": "value"}, cm.Annotations)
// Persist mon b, and remove mon a for simply testing the configmap is updated
@@ -307,7 +307,7 @@ func TestSaveMonEndpoints(t *testing.T) {
cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{})
assert.NoError(t, err)
assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey])
assert.Equal(t, "a=1.2.3.1:3300", cm.Data[EndpointDataKey])
assert.Equal(t, `{"node":{}}`, cm.Data[MappingKey])
assert.Equal(t, "-1", cm.Data[MaxMonIDKey])
+5 -19
View File
@@ -17,14 +17,11 @@ limitations under the License.
package mon
import (
"strconv"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
func (c *Cluster) createService(mon *monConfig) (string, error) {
@@ -35,16 +32,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
Labels: c.getLabels(mon, false, true),
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: "tcp-msgr1",
Port: mon.Port,
// --public-bind-addr=IP with no IP:port has the mon listen on port 6789
// regardless of what port the mon advertises (--public-addr) to the outside.
TargetPort: intstr.FromInt(int(DefaultMsgr1Port)),
Protocol: v1.ProtocolTCP,
},
},
Selector: c.getLabels(mon, false, false),
},
}
@@ -53,7 +40,10 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
return "", errors.Wrapf(err, "failed to set owner reference to mon service %q", svcDef.Name)
}
// If deploying Nautilus or newer we need a new port for the monitor service
// If the cluster doesn't require msgr2 we allow msgr1 as well
if !c.ClusterInfo.RequireMsgr2 {
addServicePort(svcDef, "tcp-msgr1", DefaultMsgr1Port)
}
addServicePort(svcDef, "tcp-msgr2", DefaultMsgr2Port)
// Set the ClusterIP if the service does not exist and we expect a certain cluster IP
@@ -77,10 +67,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) {
return "", nil
}
// mon endpoint are not actually like, they remain with the mgrs1 format
// however it's interesting to show that monitors can be addressed via 2 different ports
// in the end the service has msgr1 and msgr2 ports configured so it's not entirely wrong
logger.Infof("mon %q endpoint is [v2:%s:%s,v1:%s:%d]", mon.DaemonName, s.Spec.ClusterIP, strconv.Itoa(int(DefaultMsgr2Port)), s.Spec.ClusterIP, mon.Port)
logger.Infof("mon %q ip is %s", mon.DaemonName, s.Spec.ClusterIP)
return s.Spec.ClusterIP, nil
}
+11 -5
View File
@@ -303,8 +303,8 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
SecurityContext: controller.PodSecurityContext(),
Ports: []corev1.ContainerPort{
{
Name: "tcp-msgr1",
ContainerPort: monConfig.Port,
Name: "tcp-msgr2",
ContainerPort: DefaultMsgr2Port,
Protocol: corev1.ProtocolTCP,
},
},
@@ -318,6 +318,15 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
WorkingDir: config.VarLogCephDir,
}
if !c.spec.RequireMsgr2() {
// Add messenger 1 port
container.Ports = append(container.Ports, v1.ContainerPort{
Name: "tcp-msgr1",
ContainerPort: DefaultMsgr1Port,
Protocol: v1.ProtocolTCP,
})
}
if monConfig.Zone != "" {
desiredLocation := fmt.Sprintf("%s=%s", c.stretchFailureDomainName(), monConfig.Zone)
container.Args = append(container.Args, []string{"--set-crush-location", desiredLocation}...)
@@ -338,9 +347,6 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
config.NewFlag("public-bind-addr", controller.ContainerEnvVarReference(podIPEnvVar)))
}
// Add messenger 2 port
addContainerPort(container, "tcp-msgr2", 3300)
return container
}
-12
View File
@@ -55,15 +55,3 @@ func addServicePort(service *v1.Service, name string, port int32) {
Protocol: v1.ProtocolTCP,
})
}
// addContainerPort adds a port to a container
func addContainerPort(container v1.Container, name string, port int32) {
if port == 0 {
return
}
container.Ports = append(container.Ports, v1.ContainerPort{
Name: name,
ContainerPort: port,
Protocol: v1.ProtocolTCP,
})
}
+1 -1
View File
@@ -74,7 +74,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephclient.ClusterInfo
// extract a list of just the monitor names, which will populate the "mon initial members"
// and "mon hosts" global config field
members, hosts := cephclient.PopulateMonHostMembers(clusterInfo.Monitors)
members, hosts := cephclient.PopulateMonHostMembers(clusterInfo)
// store these in a secret instead of the configmap; secrets are required by CSI drivers
secret := &v1.Secret{
@@ -156,6 +156,11 @@ spec:
enabled: true
network:
hostNetwork: false
connections:
encryption:
enabled: ` + strconv.FormatBool(m.settings.ConnectionsEncrypted) + `
compression:
enabled: ` + strconv.FormatBool(m.settings.ConnectionsCompressed) + `
crashCollector:
disable: false
` + pruner + `
@@ -199,6 +204,11 @@ spec:
dataDirHostPath: ` + m.settings.DataDirHostPath + `
network:
hostNetwork: false
connections:
encryption:
enabled: ` + strconv.FormatBool(m.settings.ConnectionsEncrypted) + `
compression:
enabled: ` + strconv.FormatBool(m.settings.ConnectionsCompressed) + `
crashCollector:
disable: false
` + pruner + `
@@ -44,6 +44,8 @@ type TestCephSettings struct {
SkipClusterCleanup bool
SkipCleanupPolicy bool
DirectMountToolbox bool
ConnectionsEncrypted bool
ConnectionsCompressed bool
EnableVolumeReplication bool
ChangeHostName bool
RookVersion string
+1
View File
@@ -69,6 +69,7 @@ func (h *HelmSuite) SetupSuite() {
EnableAdmissionController: false,
EnableDiscovery: true,
ChangeHostName: true,
ConnectionsEncrypted: true,
RookVersion: installer.LocalBuildTag,
CephVersion: installer.OctopusVersion,
}
+2
View File
@@ -88,6 +88,8 @@ func (s *SmokeSuite) SetupSuite() {
Mons: 3,
SkipOSDCreation: false,
EnableAdmissionController: true,
ConnectionsEncrypted: true,
ConnectionsCompressed: true,
UseCrashPruner: true,
EnableVolumeReplication: true,
ChangeHostName: true,