diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 127948bf6..78b8913b3 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -164,6 +164,9 @@ Configure the network that will be enabled for the cluster and services. * `ipFamily`: Specifies the network stack Ceph daemons should listen on. * `dualStack`: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks. * `connections`: Settings for network connections using Ceph's msgr2 protocol + * `requireMsgr2`: Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled + and clients will be required to connect to the Ceph cluster with the v2 port (3300). + Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). Default is false. * `encryption`: Settings for encryption on the wire to Ceph daemons * `enabled`: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index d19cedb84..cfea99b62 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -8,6 +8,9 @@ ## Features +- Added setting `requireMsgr2` on the CephCluster CR to allow clusters with a kernel of 5.11 or newer + to fully communicate with msgr2 and disable the msgr1 port. This allows for more flexibility to enable + msgr2 features such as encryption and compression on the wire. - Change `pspEnable` default value to `false` in helm charts. - [Bucket notifications and topics](https://rook.io/docs/rook/latest/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications/) for object stores moved to stable from experimental. diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index 4477a406a..a010ec1a8 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -145,7 +145,24 @@ cephClusterSpec: ssl: true # Network configuration, see: https://github.com/rook/rook/blob/master/Documentation/CRDs/ceph-cluster-crd.md#network-configuration-settings - # network: + network: + connections: + # Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. + # The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. + # When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check. + # IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, + # you can set the "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. + # The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes. + encryption: + enabled: false + # Whether to compress the data in transit across the wire. The default is false. + # Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption. + compression: + enabled: false + # Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled + # and clients will be required to connect to the Ceph cluster with the v2 port (3300). + # Requires a kernel that supports msgr v2 (kernel 5.11 or CentOS 8.4 or newer). + requireMsgr2: false # # enable host networking # provider: host # # EXPERIMENTAL: enable the Multus network provider diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index e488a68dc..2d485b29a 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -1875,6 +1875,9 @@ spec: description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is not set. Even if encryption is not enabled, clients still establish a strong initial authentication for the connection and data integrity is still validated with a crc check. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. type: boolean type: object + requireMsgr2: + description: Whether to require msgr2 (port 3300) even if compression or encryption are not enabled. If true, the msgr1 port (6789) will be disabled. Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). + type: boolean type: object dualStack: description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 19e31214b..061f99bf5 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -88,6 +88,10 @@ spec: # Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption. compression: enabled: false + # Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled + # and clients will be required to connect to the Ceph cluster with the v2 port (3300). + # Requires a kernel that supports msgr v2 (kernel 5.11 or CentOS 8.4 or newer). + requireMsgr2: false # enable host networking #provider: host # enable the Multus network provider diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 66f7317f3..ace1a61c1 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -1873,6 +1873,9 @@ spec: description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is not set. Even if encryption is not enabled, clients still establish a strong initial authentication for the connection and data integrity is still validated with a crc check. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. type: boolean type: object + requireMsgr2: + description: Whether to require msgr2 (port 3300) even if compression or encryption are not enabled. If true, the msgr1 port (6789) will be disabled. Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). + type: boolean type: object dualStack: description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 diff --git a/pkg/apis/ceph.rook.io/v1/cluster.go b/pkg/apis/ceph.rook.io/v1/cluster.go index 3ff422def..985ebd75d 100644 --- a/pkg/apis/ceph.rook.io/v1/cluster.go +++ b/pkg/apis/ceph.rook.io/v1/cluster.go @@ -33,6 +33,9 @@ func (c *ClusterSpec) RequireMsgr2() bool { if c.Network.Connections == nil { return false } + if c.Network.Connections.RequireMsgr2 { + return true + } if c.Network.Connections.Compression != nil && c.Network.Connections.Compression.Enabled { return true } diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 7666a171b..acbc14805 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2170,6 +2170,12 @@ type ConnectionsSpec struct { // +nullable // +optional Compression *CompressionSpec `json:"compression,omitempty"` + + // Whether to require msgr2 (port 3300) even if compression or encryption are not enabled. + // If true, the msgr1 port (6789) will be disabled. + // Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). + // +optional + RequireMsgr2 bool `json:"requireMsgr2,omitempty"` } type EncryptionSpec struct { diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index b120838ad..9fcf77903 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -159,6 +159,7 @@ spec: network: hostNetwork: false connections: + requireMsgr2: ` + strconv.FormatBool(m.settings.RequireMsgr2) + ` encryption: enabled: ` + strconv.FormatBool(m.settings.ConnectionsEncrypted) + ` compression: diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 76300c3e0..6e345d24c 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -46,6 +46,7 @@ type TestCephSettings struct { DirectMountToolbox bool ConnectionsEncrypted bool ConnectionsCompressed bool + RequireMsgr2 bool EnableVolumeReplication bool TestNFSCSI bool ChangeHostName bool diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index d5056dfe0..9f18cf23a 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -83,6 +83,7 @@ func (s *MultiClusterDeploySuite) SetupSuite() { EnableAdmissionController: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.PacificVersion, + RequireMsgr2: true, } s.settings.ApplyEnvVars() externalSettings := &installer.TestCephSettings{