From 51e7310e09601a3deb45d60eb9b05aa71201d94a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 31 Mar 2021 16:28:04 +0200 Subject: [PATCH] ceph: add dualstack support on pacific MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With Pacific comes the support for dualstack where ceph daemons can listen on both ipv4 and ipv6 stacks. A new field in the network spec has been added: `dualStack` Signed-off-by: Sébastien Han --- Documentation/ceph-cluster-crd.md | 7 +++- PendingReleaseNotes.md | 8 ++-- .../charts/rook-ceph/templates/resources.yaml | 7 ++++ cluster/examples/kubernetes/ceph/cluster.yaml | 2 + cluster/examples/kubernetes/ceph/crds.yaml | 8 ++++ pkg/apis/ceph.rook.io/v1/types.go | 6 +++ pkg/operator/ceph/cluster/osd/spec.go | 5 +-- pkg/operator/ceph/controller/spec.go | 36 ++++++++++++++++-- pkg/operator/ceph/controller/spec_test.go | 38 ++++++++++++++++++- 9 files changed, 102 insertions(+), 15 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index bcabca5d1..777e9a3d4 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -293,6 +293,8 @@ Configure the network that will be enabled for the cluster and services. * `provider`: Specifies the network provider that will be used to connect the network interface. You can choose between `host`, and `multus`. * `selectors`: List the network selector(s) that will be used associated by a key. +* `ipFamily`: Specifies the network stack Ceph daemons should listen on. +* `dualStack`: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks. > **NOTE:** Changing networking configuration after a Ceph cluster has been deployed is NOT > supported and will result in a non-functioning cluster. @@ -366,7 +368,8 @@ This is required in order to use the NAD across namespaces. #### IPFamily -Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is not supported by ceph. +Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is supported as of ceph Pacific. +To turn on dual stack see the [network configuration section](#network-configuration-settings). ### Node Settings @@ -1267,7 +1270,7 @@ export ROOK_EXTERNAL_ADMIN_SECRET=AQC6Ylxdja+NDBAAB7qy9MEAr4VLLq4dCIvxtg== If the Ceph admin key is not provided, the following script needs to be executed on a machine that can connect to the Ceph cluster using the Ceph admin key. On that machine, run `cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh`. -> **WARNING**: Since only Ceph admin key can create CRs in the external cluster, please make sure that rgw pools have been prepared. You can get existing pools by running `ceph osd pool ls`. +> **WARNING**: Since only Ceph admin key can create CRs in the external cluster, please make sure that rgw pools have been prepared. You can get existing pools by running `ceph osd pool ls`. **Example**: diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index d60ee95b5..83c19245e 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -20,17 +20,17 @@ v1.6... ### Ceph -* Ceph Pacific support -* Multiple Ceph Filesystems (with Pacific only) +* Ceph Pacific support and along with it: + * Multiple Ceph Filesystems + * Networking dualstack * CephClient CRD has been converted to use the controller-runtime library * Extending the support of vault KMS configuration for Ceph RGW * Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons * Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon * Multiple Ceph mgr daemons are supported for stretch clusters and other clusters where HA of the mgr is more critical -* OSDs: +* OSDs: * as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore * for Pacific (16.2.x), Rook is able to update multiple OSD Deployments at the same time to speed up updates and upgrades for larger Ceph clusters * Disable CSI GRPC metrics by default * Ceph daemon pods using the `default` service account now use a new `rook-ceph-default` service account. - diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 733cc5d57..79713a7f4 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -1172,11 +1172,18 @@ spec: description: Network related configuration nullable: true properties: + dualStack: + description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 + type: boolean hostNetwork: description: HostNetwork to enable host network type: boolean ipFamily: + default: IPv4 description: IPFamily is the single stack IPv6 or IPv4 protocol + enum: + - IPv4 + - IPv6 nullable: true type: string provider: diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 0d50ce60e..971aca5fc 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -95,6 +95,8 @@ spec: #cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus # Provide internet protocol version. IPv6, IPv4 or empty string are valid options. Empty string would mean IPv4 #ipFamily: "IPv6" + # Ceph daemons to listen on both IPv4 and Ipv6 networks + #dualStack: false # enable the crash collector for ceph daemon crash collection crashCollector: disable: false diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 9b9e852fb..981b9941d 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -1462,11 +1462,19 @@ spec: description: Network related configuration nullable: true properties: + dualStack: + description: DualStack determines whether Ceph daemons should + listen on both IPv4 and IPv6 + type: boolean hostNetwork: description: HostNetwork to enable host network type: boolean ipFamily: + default: IPv4 description: IPFamily is the single stack IPv6 or IPv4 protocol + enum: + - IPv4 + - IPv6 nullable: true type: string provider: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 5c6c3545c..a537cc2c0 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1353,9 +1353,15 @@ type NetworkSpec struct { HostNetwork bool `json:"hostNetwork,omitempty"` // IPFamily is the single stack IPv6 or IPv4 protocol + // +kubebuilder:validation:Enum=IPv4;IPv6 + // +kubebuilder:default=IPv4 // +nullable // +optional IPFamily IPFamilyType `json:"ipFamily,omitempty"` + + // DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 + // +optional + DualStack bool `json:"dualStack,omitempty"` } // DisruptionManagementSpec configures management of daemon disruptions diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 550e7a9fb..b465b082c 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -452,10 +452,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC args = append(args, opconfig.LoggingFlags()...) args = append(args, osdOnSDNFlag(c.spec.Network)...) - - if c.spec.Network.IPFamily == cephv1.IPv6 { - args = append(args, opconfig.NewFlag("ms-bind-ipv6", "true")) - } + args = append(args, controller.NetworkBindingFlags(c.clusterInfo, &c.spec)...) osdDataDirPath := activateOSDMountPath + osdID if osdProps.onPVC() && osd.CVMode == "lvm" { diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index ff397f8de..b15c4846d 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -314,10 +314,7 @@ func DaemonFlags(cluster *cephclient.ClusterInfo, spec *cephv1.ClusterSpec, daem // run ceph daemon process under the 'ceph' group config.NewFlag("setgroup", "ceph"), ) - - if spec.Network.IPFamily == cephv1.IPv6 { - flags = append(flags, config.NewFlag("ms-bind-ipv6", "true")) - } + flags = append(flags, NetworkBindingFlags(cluster, spec)...) return flags } @@ -331,6 +328,37 @@ func AdminFlags(cluster *cephclient.ClusterInfo) []string { ) } +func NetworkBindingFlags(cluster *cephclient.ClusterInfo, spec *cephv1.ClusterSpec) []string { + var args []string + + // As of Pacific, Ceph supports dual-stack, so setting IPv6 family without disabling IPv4 binding actually enables dual-stack + // This is likely not user's intent, so on Pacific let's make sure to disable IPv4 when IPv6 is selected + if !spec.Network.DualStack { + switch spec.Network.IPFamily { + case cephv1.IPv4: + args = append(args, config.NewFlag("ms-bind-ipv4", "true")) + args = append(args, config.NewFlag("ms-bind-ipv6", "false")) + + case cephv1.IPv6: + args = append(args, config.NewFlag("ms-bind-ipv4", "false")) + args = append(args, config.NewFlag("ms-bind-ipv6", "true")) + } + } else { + if cluster.CephVersion.IsAtLeastPacific() { + args = append(args, config.NewFlag("ms-bind-ipv4", "true")) + args = append(args, config.NewFlag("ms-bind-ipv6", "true")) + } else { + logger.Info("dual-stack is only supported on ceph pacific") + // Still acknowledge IPv6, nothing to do for IPv4 since it will always be "on" + if spec.Network.IPFamily == cephv1.IPv6 { + args = append(args, config.NewFlag("ms-bind-ipv6", "true")) + } + } + } + + return args +} + // ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name // which can be used in command or argument fields. func ContainerEnvVarReference(envVarName string) string { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 153c83efb..3c76461e4 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -18,12 +18,15 @@ package controller import ( "math" + "reflect" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" opconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" @@ -179,7 +182,7 @@ func TestDaemonFlags(t *testing.T) { expected: []string{"--fsid=id", "--keyring=/etc/ceph/keyring-store/keyring", "--log-to-stderr=true", "--err-to-stderr=true", "--mon-cluster-log-to-stderr=true", "--log-stderr-prefix=debug ", "--default-log-to-file=false", "--default-mon-cluster-log-to-file=false", "--mon-host=$(ROOK_CEPH_MON_HOST)", "--mon-initial-members=$(ROOK_CEPH_MON_INITIAL_MEMBERS)", "--id=daemon-id", "--setuser=ceph", "--setgroup=ceph", - "--ms-bind-ipv6=true"}, + "--ms-bind-ipv4=false", "--ms-bind-ipv6=true"}, }, { label: "case 2: IPv6 disabled", @@ -199,3 +202,36 @@ func TestDaemonFlags(t *testing.T) { assert.Equalf(t, tc.expected, actual, "[%s]: failed to get correct daemonset flags", tc.label) } } + +func TestNetworkBindingFlags(t *testing.T) { + ipv4FlagTrue := "--ms-bind-ipv4=true" + ipv4FlagFalse := "--ms-bind-ipv4=false" + ipv6FlagTrue := "--ms-bind-ipv6=true" + ipv6FlagFalse := "--ms-bind-ipv6=false" + type args struct { + cluster *cephclient.ClusterInfo + spec *cephv1.ClusterSpec + } + tests := []struct { + name string + args args + want []string + }{ + {"octopus-ipv4", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4}}}, []string{ipv4FlagTrue, ipv6FlagFalse}}, + {"octopus-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6}}}, []string{ipv4FlagFalse, ipv6FlagTrue}}, + {"octopus-dualstack-unsupported", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4, DualStack: true}}}, []string{}}, + {"octopus-dualstack-unsupported-by-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6, DualStack: true}}}, []string{ipv6FlagTrue}}, + {"pacific-ipv4", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4}}}, []string{ipv4FlagTrue, ipv6FlagFalse}}, + {"pacific-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6}}}, []string{ipv4FlagFalse, ipv6FlagTrue}}, + {"pacific-dualstack-supported", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6, DualStack: true}}}, []string{ipv4FlagTrue, ipv6FlagTrue}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NetworkBindingFlags(tt.args.cluster, tt.args.spec); !reflect.DeepEqual(got, tt.want) { + if len(got) != 0 && len(tt.want) != 0 { + t.Errorf("NetworkBindingFlags() = %+v, want %+v", got, tt.want) + } + } + }) + } +}