diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 8298c9fc7..32037483f 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -305,6 +305,19 @@ Configure the network that will be enabled for the cluster and services. * `selectors`: List the network selector(s) that will be used associated by a key. * `ipFamily`: Specifies the network stack Ceph daemons should listen on. * `dualStack`: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks. +* `connections`: Settings for network connections using Ceph's msgr2 protocol + * `encryption`: Settings for encryption on the wire to Ceph daemons + * `enabled`: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. + The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between + Ceph daemons will be encrypted. When encryption is not enabled, clients still establish a strong initial authentication + and data integrity is still validated with a crc check. + IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, + set "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. + The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes. + * `compression`: + * `enabled`: Whether to compress the data in transit across the wire. The default is false. + Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption. + > **NOTE:** Changing networking configuration after a Ceph cluster has been deployed is NOT > supported and will result in a non-functioning cluster. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 23963e183..e706b04a5 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,4 +9,6 @@ * The number of mgr daemons for example clusters is increased to 2 from 1, resulting in a standby mgr daemon. If the active mgr goes down, Ceph will update the passive mgr to be active, and rook will update all the services with the label app=rook-ceph-mgr to direct traffic to the new active mgr. +* Network encryption is configurable with settings in the CephCluster CR. Requires the 5.11 kernel or newer. +* Network compression is configurable with settings in the CephCluster CR. Requires Ceph Quincy (v17) or newer. * Add support for custom ceph.conf for csi pods. See #9567 diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 085660012..ab4534973 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -1653,6 +1653,27 @@ spec: description: Network related configuration nullable: true properties: + connections: + description: Settings for network connections such as compression and encryption across the wire. + nullable: true + properties: + compression: + description: Compression settings for the network connections. + nullable: true + properties: + enabled: + description: Whether to compress the data in transit across the wire. The default is not set. Requires Ceph Quincy (v17) or newer. + type: boolean + type: object + encryption: + description: Encryption settings for the network connections. + nullable: true + properties: + enabled: + description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is not set. Even if encryption is not enabled, clients still establish a strong initial authentication for the connection and data integrity is still validated with a crc check. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. + type: boolean + type: object + type: object dualStack: description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 type: boolean diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 0d522b922..067eb424b 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -81,6 +81,19 @@ spec: # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. rulesNamespace: rook-ceph network: + connections: + # Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. + # The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. + # When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check. + # IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, + # you can set the "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. + # The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes. + encryption: + enabled: false + # Whether to compress the data in transit across the wire. The default is false. + # Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption. + compression: + enabled: false # enable host networking #provider: host # enable the Multus network provider diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 61545f381..b24f255ef 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -1652,6 +1652,27 @@ spec: description: Network related configuration nullable: true properties: + connections: + description: Settings for network connections such as compression and encryption across the wire. + nullable: true + properties: + compression: + description: Compression settings for the network connections. + nullable: true + properties: + enabled: + description: Whether to compress the data in transit across the wire. The default is not set. Requires Ceph Quincy (v17) or newer. + type: boolean + type: object + encryption: + description: Encryption settings for the network connections. + nullable: true + properties: + enabled: + description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is not set. Even if encryption is not enabled, clients still establish a strong initial authentication for the connection and data integrity is still validated with a crc check. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. + type: boolean + type: object + type: object dualStack: description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 type: boolean diff --git a/pkg/apis/ceph.rook.io/v1/cluster.go b/pkg/apis/ceph.rook.io/v1/cluster.go index a1d8ad48f..94763d46d 100644 --- a/pkg/apis/ceph.rook.io/v1/cluster.go +++ b/pkg/apis/ceph.rook.io/v1/cluster.go @@ -29,6 +29,20 @@ import ( // will be registered for the validating webhook. var _ webhook.Validator = &CephCluster{} +// RequireMsgr2 checks if the network settings require the msgr2 protocol +func (c *ClusterSpec) RequireMsgr2() bool { + if c.Network.Connections == nil { + return false + } + if c.Network.Connections.Compression != nil && c.Network.Connections.Compression.Enabled { + return true + } + if c.Network.Connections.Encryption != nil && c.Network.Connections.Encryption.Enabled { + return true + } + return false +} + func (c *ClusterSpec) IsStretchCluster() bool { return c.Mon.StretchCluster != nil && len(c.Mon.StretchCluster.Zones) > 0 } diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 683c501a7..60f0358ce 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1916,6 +1916,12 @@ type NetworkSpec struct { // +optional Selectors map[string]string `json:"selectors,omitempty"` + // Settings for network connections such as compression and encryption across the + // wire. + // +nullable + // +optional + Connections *ConnectionsSpec `json:"connections,omitempty"` + // HostNetwork to enable host network // +optional HostNetwork bool `json:"hostNetwork,omitempty"` @@ -1931,6 +1937,36 @@ type NetworkSpec struct { DualStack bool `json:"dualStack,omitempty"` } +type ConnectionsSpec struct { + // Encryption settings for the network connections. + // +nullable + // +optional + Encryption *EncryptionSpec `json:"encryption,omitempty"` + + // Compression settings for the network connections. + // +nullable + // +optional + Compression *CompressionSpec `json:"compression,omitempty"` +} + +type EncryptionSpec struct { + // Whether to encrypt the data in transit across the wire to prevent eavesdropping + // the data on the network. The default is not set. Even if encryption is not enabled, + // clients still establish a strong initial authentication for the connection + // and data integrity is still validated with a crc check. When encryption is enabled, + // all communication between clients and Ceph daemons, or between Ceph daemons will + // be encrypted. + // +optional + Enabled bool `json:"enabled,omitempty"` +} + +type CompressionSpec struct { + // Whether to compress the data in transit across the wire. + // The default is not set. Requires Ceph Quincy (v17) or newer. + // +optional + Enabled bool `json:"enabled,omitempty"` +} + // DisruptionManagementSpec configures management of daemon disruptions type DisruptionManagementSpec struct { // This enables management of poddisruptionbudgets diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 5fe49663f..f4b17eb67 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1715,6 +1715,22 @@ func (in *ClusterVersion) DeepCopy() *ClusterVersion { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CompressionSpec) DeepCopyInto(out *CompressionSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CompressionSpec. +func (in *CompressionSpec) DeepCopy() *CompressionSpec { + if in == nil { + return nil + } + out := new(CompressionSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Condition) DeepCopyInto(out *Condition) { *out = *in @@ -1733,6 +1749,32 @@ func (in *Condition) DeepCopy() *Condition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionsSpec) DeepCopyInto(out *ConnectionsSpec) { + *out = *in + if in.Encryption != nil { + in, out := &in.Encryption, &out.Encryption + *out = new(EncryptionSpec) + **out = **in + } + if in.Compression != nil { + in, out := &in.Compression, &out.Compression + *out = new(CompressionSpec) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionsSpec. +func (in *ConnectionsSpec) DeepCopy() *ConnectionsSpec { + if in == nil { + return nil + } + out := new(ConnectionsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CrashCollectorSpec) DeepCopyInto(out *CrashCollectorSpec) { *out = *in @@ -1839,6 +1881,22 @@ func (in *DisruptionManagementSpec) DeepCopy() *DisruptionManagementSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionSpec) DeepCopyInto(out *EncryptionSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionSpec. +func (in *EncryptionSpec) DeepCopy() *EncryptionSpec { + if in == nil { + return nil + } + out := new(EncryptionSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ErasureCodedSpec) DeepCopyInto(out *ErasureCodedSpec) { *out = *in @@ -2660,6 +2718,11 @@ func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { (*out)[key] = val } } + if in.Connections != nil { + in, out := &in.Connections, &out.Connections + *out = new(ConnectionsSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/daemon/ceph/client/config.go b/pkg/daemon/ceph/client/config.go index 1ad96d111..ea0c594b5 100644 --- a/pkg/daemon/ceph/client/config.go +++ b/pkg/daemon/ceph/client/config.go @@ -20,9 +20,11 @@ package client import ( "fmt" "io/ioutil" + "net" "os" "path" "path/filepath" + "strconv" "strings" "github.com/rook/rook/pkg/operator/k8sutil" @@ -44,6 +46,8 @@ const ( DefaultKeyringFile = "keyring" // Msgr2port is the listening port of the messenger v2 protocol Msgr2port = 3300 + // Msgr1port is the listening port of the messenger v1 protocol + Msgr1port = 6789 ) var ( @@ -186,7 +190,7 @@ func CreateDefaultCephConfig(context *clusterd.Context, clusterInfo *ClusterInfo // extract a list of just the monitor names, which will populate the "mon initial members" // and "mon hosts" global config field - monMembers, monHosts := PopulateMonHostMembers(clusterInfo.Monitors) + monMembers, monHosts := PopulateMonHostMembers(clusterInfo) conf := &CephConfig{ GlobalConfig: &GlobalConfig{ @@ -242,15 +246,28 @@ func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath st // PopulateMonHostMembers extracts a list of just the monitor names, which will populate the "mon initial members" // and "mon hosts" global config field -func PopulateMonHostMembers(monitors map[string]*MonInfo) ([]string, []string) { - monMembers := make([]string, len(monitors)) - monHosts := make([]string, len(monitors)) +func PopulateMonHostMembers(clusterInfo *ClusterInfo) ([]string, []string) { + monMembers := make([]string, len(clusterInfo.Monitors)) + monHosts := make([]string, len(clusterInfo.Monitors)) i := 0 - for _, monitor := range monitors { + for _, monitor := range clusterInfo.Monitors { monMembers[i] = monitor.Name monIP := cephutil.GetIPFromEndpoint(monitor.Endpoint) - monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port) + if clusterInfo.RequireMsgr2 { + monHosts[i] = fmt.Sprintf("[v2:%s:%d]", monIP, Msgr2port) + } else { + // Detect the current port if the mon already exists + // so the same msgr1 port can be preserved if needed (6789 or 6790) + currentMonPort := cephutil.GetPortFromEndpoint(monitor.Endpoint) + // Ensure we're setting a msgr1 port, rather than duplicating msgr2 + if currentMonPort == Msgr2port { + currentMonPort = Msgr1port + } + msgr2Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(Msgr2port))) + msgr1Endpoint := net.JoinHostPort(monIP, strconv.Itoa(int(currentMonPort))) + monHosts[i] = "[v2:" + msgr2Endpoint + ",v1:" + msgr1Endpoint + "]" + } i++ } diff --git a/pkg/daemon/ceph/client/info.go b/pkg/daemon/ceph/client/info.go index b27bbbbf1..3bfb1ea92 100644 --- a/pkg/daemon/ceph/client/info.go +++ b/pkg/daemon/ceph/client/info.go @@ -42,6 +42,7 @@ type ClusterInfo struct { CephVersion cephver.CephVersion Namespace string OwnerInfo *k8sutil.OwnerInfo + RequireMsgr2 bool // Hide the name of the cluster since in 99% of uses we want to use the cluster namespace. // If the CR name is needed, access it through the NamespacedName() method. name string diff --git a/pkg/daemon/ceph/client/test/info.go b/pkg/daemon/ceph/client/test/info.go index 052fe696b..f48b9008d 100644 --- a/pkg/daemon/ceph/client/test/info.go +++ b/pkg/daemon/ceph/client/test/info.go @@ -61,7 +61,7 @@ func CreateTestClusterInfo(monCount int) *client.ClusterInfo { id := mons[i] c.Monitors[id] = &client.MonInfo{ Name: id, - Endpoint: fmt.Sprintf("1.2.3.%d:6789", (i + 1)), + Endpoint: fmt.Sprintf("1.2.3.%d:3300", (i + 1)), } } c.SetName(c.Namespace) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index cd7484a4f..88eb585b7 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -191,6 +191,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster) error { } else { clusterInfo.OwnerInfo = cluster.ownerInfo clusterInfo.SetName(c.namespacedName.Name) + clusterInfo.RequireMsgr2 = cluster.Spec.RequireMsgr2() cluster.ClusterInfo = clusterInfo } // If the local cluster has already been configured, immediately start monitoring the cluster. @@ -505,6 +506,10 @@ func (c *cluster) postMonStartupActions() error { } } + if err := c.configureMsgr2(); err != nil { + return errors.Wrap(err, "failed to configured msgr2") + } + crushRoot := client.GetCrushRootFromSpec(c.Spec) if crushRoot != "default" { // Remove the root=default and replicated_rule which are created by @@ -523,3 +528,46 @@ func (c *cluster) postMonStartupActions() error { return nil } + +func (c *cluster) configureMsgr2() error { + if c.Spec.Network.Connections == nil { + return nil + } + + // Set network encryption + monStore := config.GetMonStore(c.context, c.ClusterInfo) + if c.Spec.Network.Connections.Encryption != nil { + encryptionSetting := "crc secure" + if c.Spec.Network.Connections.Encryption.Enabled { + encryptionSetting = "secure" + } + logger.Infof("setting msgr2 encryption mode to %q", encryptionSetting) + + if err := monStore.Set("global", "ms_cluster_mode", encryptionSetting); err != nil { + return errors.Wrapf(err, "failed to set ms_cluster_mode to %q", encryptionSetting) + } + if err := monStore.Set("global", "ms_service_mode", encryptionSetting); err != nil { + return errors.Wrapf(err, "failed to set ms_service_mode to %q", encryptionSetting) + } + if err := monStore.Set("global", "ms_client_mode", encryptionSetting); err != nil { + return errors.Wrapf(err, "failed to set ms_client_mode to %q", encryptionSetting) + } + } + + // Set network compression + if c.Spec.Network.Connections.Compression != nil { + if c.ClusterInfo.CephVersion.IsAtLeastQuincy() { + compressionSetting := "none" + if c.Spec.Network.Connections.Compression.Enabled { + compressionSetting = "force" + } + logger.Infof("setting msgr2 compression mode to %q", compressionSetting) + if err := monStore.Set("global", "ms_osd_compress_mode", compressionSetting); err != nil { + return errors.Wrapf(err, "failed to set ms_osd_compress_mode to %q", compressionSetting) + } + } else { + logger.Warningf("network compression requires Ceph Quincy (v17) or newer, skipping for current ceph %q", c.ClusterInfo.CephVersion.String()) + } + } + return nil +} diff --git a/pkg/operator/ceph/cluster/cluster_test.go b/pkg/operator/ceph/cluster/cluster_test.go index cdbef0528..392f7494c 100644 --- a/pkg/operator/ceph/cluster/cluster_test.go +++ b/pkg/operator/ceph/cluster/cluster_test.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/version" cephver "github.com/rook/rook/pkg/operator/ceph/version" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -149,3 +150,71 @@ func TestPreMonChecks(t *testing.T) { assert.False(t, unsetSkipSanity) }) } + +func TestConfigureMsgr2(t *testing.T) { + type fields struct { + encryptionExpected bool + compressionExpected bool + disabledBothExpected bool + cephVersion version.CephVersion + Spec *cephv1.ClusterSpec + } + tests := []struct { + name string + fields fields + }{ + {"default settings", fields{false, false, false, version.CephVersion{Major: 15}, &cephv1.ClusterSpec{}}}, + {"encryption enabled", fields{true, false, false, version.CephVersion{Major: 16}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Encryption: &cephv1.EncryptionSpec{Enabled: true}}}}}}, + {"compression enabled old version", fields{false, false, false, version.CephVersion{Major: 16}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Compression: &cephv1.CompressionSpec{Enabled: true}}}}}}, + {"compression enabled good version", fields{false, true, false, version.CephVersion{Major: 17}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Compression: &cephv1.CompressionSpec{Enabled: true}}}}}}, + {"both enabled", fields{true, true, false, version.CephVersion{Major: 17}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Encryption: &cephv1.EncryptionSpec{Enabled: true}, Compression: &cephv1.CompressionSpec{Enabled: true}}}}}}, + {"both disabled", fields{false, false, true, version.CephVersion{Major: 17}, &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{Connections: &cephv1.ConnectionsSpec{Encryption: &cephv1.EncryptionSpec{Enabled: false}, Compression: &cephv1.CompressionSpec{Enabled: false}}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + enabledCompression := false + disabledCompression := false + enabledEncryption := false + disabledEncryption := false + executor := &exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + logger.Infof("Execute: %s %v", command, args) + if args[0] == "config" && args[1] == "set" { + if args[3] == "ms_osd_compress_mode" { + if args[4] == "force" { + enabledCompression = true + } else if args[4] == "none" { + disabledCompression = true + } + } + if args[3] == "ms_cluster_mode" { + if args[4] == "secure" { + enabledEncryption = true + } else if args[4] == "crc secure" { + disabledEncryption = true + } + } + + return "", nil + } + return "", errors.Errorf("unrecognized command") + }, + } + clusterInfo := client.AdminTestClusterInfo("rook-ceph") + clusterInfo.CephVersion = tt.fields.cephVersion + context := &clusterd.Context{Clientset: testop.New(t, 3), Executor: executor} + c := &cluster{ + ClusterInfo: clusterInfo, + Spec: tt.fields.Spec, + context: context, + } + + err := c.configureMsgr2() + assert.NoError(t, err) + assert.Equal(t, tt.fields.compressionExpected, enabledCompression) + assert.Equal(t, tt.fields.encryptionExpected, enabledEncryption) + assert.Equal(t, tt.fields.disabledBothExpected, disabledCompression) + assert.Equal(t, tt.fields.disabledBothExpected, disabledEncryption) + }) + } +} diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go index 8c1d92101..136f28eb5 100644 --- a/pkg/operator/ceph/cluster/mon/health_test.go +++ b/pkg/operator/ceph/cluster/mon/health_test.go @@ -319,8 +319,8 @@ func TestCheckHealthNotFound(t *testing.T) { // Check if the two mons are found in the configmap cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) - if cm.Data[EndpointDataKey] != "a=1.2.3.1:6789,b=1.2.3.2:6789" { - assert.Equal(t, "b=1.2.3.2:6789,a=1.2.3.1:6789", cm.Data[EndpointDataKey]) + if cm.Data[EndpointDataKey] != "a=1.2.3.1:3300,b=1.2.3.2:3300" { + assert.Equal(t, "b=1.2.3.2:3300,a=1.2.3.1:3300", cm.Data[EndpointDataKey]) } // Because the mon a isn't in the MonInQuorumResponse() this will create a new mon @@ -334,8 +334,8 @@ func TestCheckHealthNotFound(t *testing.T) { // recheck that the "not found" mon has been replaced with a new one cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) - if cm.Data[EndpointDataKey] != "a=1.2.3.1:6789,f=:3300" { - assert.Equal(t, "f=:3300,a=1.2.3.1:6789", cm.Data[EndpointDataKey]) + if cm.Data[EndpointDataKey] != "a=1.2.3.1:3300,f=:3300" { + assert.Equal(t, "f=:3300,a=1.2.3.1:3300", cm.Data[EndpointDataKey]) } } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 6f99c0487..50ab45654 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -500,6 +500,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s c.ClusterInfo.OwnerInfo = c.ownerInfo c.ClusterInfo.Context = context c.ClusterInfo.SetName(clusterName) + c.ClusterInfo.RequireMsgr2 = c.spec.RequireMsgr2() // save cluster monitor config if err = c.saveMonConfig(); err != nil { diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 068c77cea..402fac229 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -279,7 +279,7 @@ func TestPersistMons(t *testing.T) { cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(context.TODO(), EndpointConfigMapName, metav1.GetOptions{}) assert.NoError(t, err) - assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey]) + assert.Equal(t, "a=1.2.3.1:3300", cm.Data[EndpointDataKey]) assert.Equal(t, map[string]string{"key": "value"}, cm.Annotations) // Persist mon b, and remove mon a for simply testing the configmap is updated @@ -307,7 +307,7 @@ func TestSaveMonEndpoints(t *testing.T) { cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, EndpointConfigMapName, metav1.GetOptions{}) assert.NoError(t, err) - assert.Equal(t, "a=1.2.3.1:6789", cm.Data[EndpointDataKey]) + assert.Equal(t, "a=1.2.3.1:3300", cm.Data[EndpointDataKey]) assert.Equal(t, `{"node":{}}`, cm.Data[MappingKey]) assert.Equal(t, "-1", cm.Data[MaxMonIDKey]) diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 73df857c2..b01d4a314 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -17,14 +17,11 @@ limitations under the License. package mon import ( - "strconv" - "github.com/pkg/errors" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) func (c *Cluster) createService(mon *monConfig) (string, error) { @@ -35,16 +32,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { Labels: c.getLabels(mon, false, true), }, Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "tcp-msgr1", - Port: mon.Port, - // --public-bind-addr=IP with no IP:port has the mon listen on port 6789 - // regardless of what port the mon advertises (--public-addr) to the outside. - TargetPort: intstr.FromInt(int(DefaultMsgr1Port)), - Protocol: v1.ProtocolTCP, - }, - }, Selector: c.getLabels(mon, false, false), }, } @@ -53,7 +40,10 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { return "", errors.Wrapf(err, "failed to set owner reference to mon service %q", svcDef.Name) } - // If deploying Nautilus or newer we need a new port for the monitor service + // If the cluster doesn't require msgr2 we allow msgr1 as well + if !c.ClusterInfo.RequireMsgr2 { + addServicePort(svcDef, "tcp-msgr1", DefaultMsgr1Port) + } addServicePort(svcDef, "tcp-msgr2", DefaultMsgr2Port) // Set the ClusterIP if the service does not exist and we expect a certain cluster IP @@ -77,10 +67,6 @@ func (c *Cluster) createService(mon *monConfig) (string, error) { return "", nil } - // mon endpoint are not actually like, they remain with the mgrs1 format - // however it's interesting to show that monitors can be addressed via 2 different ports - // in the end the service has msgr1 and msgr2 ports configured so it's not entirely wrong - logger.Infof("mon %q endpoint is [v2:%s:%s,v1:%s:%d]", mon.DaemonName, s.Spec.ClusterIP, strconv.Itoa(int(DefaultMsgr2Port)), s.Spec.ClusterIP, mon.Port) - + logger.Infof("mon %q ip is %s", mon.DaemonName, s.Spec.ClusterIP) return s.Spec.ClusterIP, nil } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index c817697dd..10b4e91fb 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -303,8 +303,8 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container SecurityContext: controller.PodSecurityContext(), Ports: []corev1.ContainerPort{ { - Name: "tcp-msgr1", - ContainerPort: monConfig.Port, + Name: "tcp-msgr2", + ContainerPort: DefaultMsgr2Port, Protocol: corev1.ProtocolTCP, }, }, @@ -318,6 +318,15 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container WorkingDir: config.VarLogCephDir, } + if !c.spec.RequireMsgr2() { + // Add messenger 1 port + container.Ports = append(container.Ports, v1.ContainerPort{ + Name: "tcp-msgr1", + ContainerPort: DefaultMsgr1Port, + Protocol: v1.ProtocolTCP, + }) + } + if monConfig.Zone != "" { desiredLocation := fmt.Sprintf("%s=%s", c.stretchFailureDomainName(), monConfig.Zone) container.Args = append(container.Args, []string{"--set-crush-location", desiredLocation}...) @@ -338,9 +347,6 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container config.NewFlag("public-bind-addr", controller.ContainerEnvVarReference(podIPEnvVar))) } - // Add messenger 2 port - addContainerPort(container, "tcp-msgr2", 3300) - return container } diff --git a/pkg/operator/ceph/cluster/mon/util.go b/pkg/operator/ceph/cluster/mon/util.go index 50e571607..f688a3794 100644 --- a/pkg/operator/ceph/cluster/mon/util.go +++ b/pkg/operator/ceph/cluster/mon/util.go @@ -55,15 +55,3 @@ func addServicePort(service *v1.Service, name string, port int32) { Protocol: v1.ProtocolTCP, }) } - -// addContainerPort adds a port to a container -func addContainerPort(container v1.Container, name string, port int32) { - if port == 0 { - return - } - container.Ports = append(container.Ports, v1.ContainerPort{ - Name: name, - ContainerPort: port, - Protocol: v1.ProtocolTCP, - }) -} diff --git a/pkg/operator/ceph/config/store.go b/pkg/operator/ceph/config/store.go index d7da7e30a..e0988eaa3 100644 --- a/pkg/operator/ceph/config/store.go +++ b/pkg/operator/ceph/config/store.go @@ -74,7 +74,7 @@ func (s *Store) createOrUpdateMonHostSecrets(clusterInfo *cephclient.ClusterInfo // extract a list of just the monitor names, which will populate the "mon initial members" // and "mon hosts" global config field - members, hosts := cephclient.PopulateMonHostMembers(clusterInfo.Monitors) + members, hosts := cephclient.PopulateMonHostMembers(clusterInfo) // store these in a secret instead of the configmap; secrets are required by CSI drivers secret := &v1.Secret{ diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index c0333cd42..eacc17b64 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -156,6 +156,11 @@ spec: enabled: true network: hostNetwork: false + connections: + encryption: + enabled: ` + strconv.FormatBool(m.settings.ConnectionsEncrypted) + ` + compression: + enabled: ` + strconv.FormatBool(m.settings.ConnectionsCompressed) + ` crashCollector: disable: false ` + pruner + ` @@ -199,6 +204,11 @@ spec: dataDirHostPath: ` + m.settings.DataDirHostPath + ` network: hostNetwork: false + connections: + encryption: + enabled: ` + strconv.FormatBool(m.settings.ConnectionsEncrypted) + ` + compression: + enabled: ` + strconv.FormatBool(m.settings.ConnectionsCompressed) + ` crashCollector: disable: false ` + pruner + ` diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 7ac21093d..b05ffa6df 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -44,6 +44,8 @@ type TestCephSettings struct { SkipClusterCleanup bool SkipCleanupPolicy bool DirectMountToolbox bool + ConnectionsEncrypted bool + ConnectionsCompressed bool EnableVolumeReplication bool ChangeHostName bool RookVersion string diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 91c46b45e..664d460d0 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -69,6 +69,7 @@ func (h *HelmSuite) SetupSuite() { EnableAdmissionController: false, EnableDiscovery: true, ChangeHostName: true, + ConnectionsEncrypted: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.OctopusVersion, } diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 7b4847837..5907b8191 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -88,6 +88,8 @@ func (s *SmokeSuite) SetupSuite() { Mons: 3, SkipOSDCreation: false, EnableAdmissionController: true, + ConnectionsEncrypted: true, + ConnectionsCompressed: true, UseCrashPruner: true, EnableVolumeReplication: true, ChangeHostName: true,