ceph: add dualstack support on pacific

With Pacific comes the support for dualstack where ceph daemons can
listen on both ipv4 and ipv6 stacks.
A new field in the network spec has been added: `dualStack`

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han committed 2021-03-31 18:04:29 +02:00
1 parent 8c157cf64b
commit 51e7310e09
9 files changed
+102 -15

No files matched your search

+5 -2
View File
@@ -293,6 +293,8 @@ Configure the network that will be enabled for the cluster and services.
* `provider`: Specifies the network provider that will be used to connect the network interface. You can choose between `host`, and `multus`.
* `selectors`: List the network selector(s) that will be used associated by a key.
* `ipFamily`: Specifies the network stack Ceph daemons should listen on.
* `dualStack`: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks.
> **NOTE:** Changing networking configuration after a Ceph cluster has been deployed is NOT
> supported and will result in a non-functioning cluster.
@@ -366,7 +368,8 @@ This is required in order to use the NAD across namespaces.
#### IPFamily
Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is not supported by ceph.
Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is supported as of ceph Pacific.
To turn on dual stack see the [network configuration section](#network-configuration-settings).
### Node Settings
@@ -1267,7 +1270,7 @@ export ROOK_EXTERNAL_ADMIN_SECRET=AQC6Ylxdja+NDBAAB7qy9MEAr4VLLq4dCIvxtg==
If the Ceph admin key is not provided, the following script needs to be executed on a machine that can connect to the Ceph cluster using the Ceph admin key.
On that machine, run `cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh`.
> **WARNING**: Since only Ceph admin key can create CRs in the external cluster, please make sure that rgw pools have been prepared. You can get existing pools by running `ceph osd pool ls`.
> **WARNING**: Since only Ceph admin key can create CRs in the external cluster, please make sure that rgw pools have been prepared. You can get existing pools by running `ceph osd pool ls`.
**Example**:
+4 -4
View File
@@ -20,17 +20,17 @@ v1.6...
### Ceph
* Ceph Pacific support
* Multiple Ceph Filesystems (with Pacific only)
* Ceph Pacific support and along with it:
* Multiple Ceph Filesystems
* Networking dualstack
* CephClient CRD has been converted to use the controller-runtime library
* Extending the support of vault KMS configuration for Ceph RGW
* Enable disruption budgets (PDBs) by default for Mon, RGW, MDS, and OSD daemons
* Add CephFilesystemMirror CRD to deploy cephfs-mirror daemon
* Multiple Ceph mgr daemons are supported for stretch clusters and other clusters where HA of the mgr is more critical
* OSDs:
* OSDs:
* as of Nautilus 14.2.14 and Octopus 15.2.9 if the OSD scenario is simple (one OSD per disk) we won't use LVM to prepare the disk anymore
* for Pacific (16.2.x), Rook is able to update multiple OSD Deployments at the same time to speed
up updates and upgrades for larger Ceph clusters
* Disable CSI GRPC metrics by default
* Ceph daemon pods using the `default` service account now use a new `rook-ceph-default` service account.
@@ -1172,11 +1172,18 @@ spec:
description: Network related configuration
nullable: true
properties:
dualStack:
description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6
type: boolean
hostNetwork:
description: HostNetwork to enable host network
type: boolean
ipFamily:
default: IPv4
description: IPFamily is the single stack IPv6 or IPv4 protocol
enum:
- IPv4
- IPv6
nullable: true
type: string
provider:
@@ -95,6 +95,8 @@ spec:
#cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus
# Provide internet protocol version. IPv6, IPv4 or empty string are valid options. Empty string would mean IPv4
#ipFamily: "IPv6"
# Ceph daemons to listen on both IPv4 and Ipv6 networks
#dualStack: false
# enable the crash collector for ceph daemon crash collection
crashCollector:
disable: false
@@ -1462,11 +1462,19 @@ spec:
description: Network related configuration
nullable: true
properties:
dualStack:
description: DualStack determines whether Ceph daemons should
listen on both IPv4 and IPv6
type: boolean
hostNetwork:
description: HostNetwork to enable host network
type: boolean
ipFamily:
default: IPv4
description: IPFamily is the single stack IPv6 or IPv4 protocol
enum:
- IPv4
- IPv6
nullable: true
type: string
provider:
+6
View File
@@ -1353,9 +1353,15 @@ type NetworkSpec struct {
HostNetwork bool `json:"hostNetwork,omitempty"`
// IPFamily is the single stack IPv6 or IPv4 protocol
// +kubebuilder:validation:Enum=IPv4;IPv6
// +kubebuilder:default=IPv4
// +nullable
// +optional
IPFamily IPFamilyType `json:"ipFamily,omitempty"`
// DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6
// +optional
DualStack bool `json:"dualStack,omitempty"`
}
// DisruptionManagementSpec configures management of daemon disruptions
+1 -4
View File
@@ -452,10 +452,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
args = append(args, opconfig.LoggingFlags()...)
args = append(args, osdOnSDNFlag(c.spec.Network)...)
if c.spec.Network.IPFamily == cephv1.IPv6 {
args = append(args, opconfig.NewFlag("ms-bind-ipv6", "true"))
}
args = append(args, controller.NetworkBindingFlags(c.clusterInfo, &c.spec)...)
osdDataDirPath := activateOSDMountPath + osdID
if osdProps.onPVC() && osd.CVMode == "lvm" {
+32 -4
View File
@@ -314,10 +314,7 @@ func DaemonFlags(cluster *cephclient.ClusterInfo, spec *cephv1.ClusterSpec, daem
// run ceph daemon process under the 'ceph' group
config.NewFlag("setgroup", "ceph"),
)
if spec.Network.IPFamily == cephv1.IPv6 {
flags = append(flags, config.NewFlag("ms-bind-ipv6", "true"))
}
flags = append(flags, NetworkBindingFlags(cluster, spec)...)
return flags
}
@@ -331,6 +328,37 @@ func AdminFlags(cluster *cephclient.ClusterInfo) []string {
)
}
func NetworkBindingFlags(cluster *cephclient.ClusterInfo, spec *cephv1.ClusterSpec) []string {
var args []string
// As of Pacific, Ceph supports dual-stack, so setting IPv6 family without disabling IPv4 binding actually enables dual-stack
// This is likely not user's intent, so on Pacific let's make sure to disable IPv4 when IPv6 is selected
if !spec.Network.DualStack {
switch spec.Network.IPFamily {
case cephv1.IPv4:
args = append(args, config.NewFlag("ms-bind-ipv4", "true"))
args = append(args, config.NewFlag("ms-bind-ipv6", "false"))
case cephv1.IPv6:
args = append(args, config.NewFlag("ms-bind-ipv4", "false"))
args = append(args, config.NewFlag("ms-bind-ipv6", "true"))
}
} else {
if cluster.CephVersion.IsAtLeastPacific() {
args = append(args, config.NewFlag("ms-bind-ipv4", "true"))
args = append(args, config.NewFlag("ms-bind-ipv6", "true"))
} else {
logger.Info("dual-stack is only supported on ceph pacific")
// Still acknowledge IPv6, nothing to do for IPv4 since it will always be "on"
if spec.Network.IPFamily == cephv1.IPv6 {
args = append(args, config.NewFlag("ms-bind-ipv6", "true"))
}
}
}
return args
}
// ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name
// which can be used in command or argument fields.
func ContainerEnvVarReference(envVarName string) string {
+37 -1
View File
@@ -18,12 +18,15 @@ package controller
import (
"math"
"reflect"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/config"
opconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
@@ -179,7 +182,7 @@ func TestDaemonFlags(t *testing.T) {
expected: []string{"--fsid=id", "--keyring=/etc/ceph/keyring-store/keyring", "--log-to-stderr=true", "--err-to-stderr=true",
"--mon-cluster-log-to-stderr=true", "--log-stderr-prefix=debug ", "--default-log-to-file=false", "--default-mon-cluster-log-to-file=false",
"--mon-host=$(ROOK_CEPH_MON_HOST)", "--mon-initial-members=$(ROOK_CEPH_MON_INITIAL_MEMBERS)", "--id=daemon-id", "--setuser=ceph", "--setgroup=ceph",
"--ms-bind-ipv6=true"},
"--ms-bind-ipv4=false", "--ms-bind-ipv6=true"},
},
{
label: "case 2: IPv6 disabled",
@@ -199,3 +202,36 @@ func TestDaemonFlags(t *testing.T) {
assert.Equalf(t, tc.expected, actual, "[%s]: failed to get correct daemonset flags", tc.label)
}
}
func TestNetworkBindingFlags(t *testing.T) {
ipv4FlagTrue := "--ms-bind-ipv4=true"
ipv4FlagFalse := "--ms-bind-ipv4=false"
ipv6FlagTrue := "--ms-bind-ipv6=true"
ipv6FlagFalse := "--ms-bind-ipv6=false"
type args struct {
cluster *cephclient.ClusterInfo
spec *cephv1.ClusterSpec
}
tests := []struct {
name string
args args
want []string
}{
{"octopus-ipv4", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4}}}, []string{ipv4FlagTrue, ipv6FlagFalse}},
{"octopus-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6}}}, []string{ipv4FlagFalse, ipv6FlagTrue}},
{"octopus-dualstack-unsupported", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4, DualStack: true}}}, []string{}},
{"octopus-dualstack-unsupported-by-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Octopus}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6, DualStack: true}}}, []string{ipv6FlagTrue}},
{"pacific-ipv4", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4}}}, []string{ipv4FlagTrue, ipv6FlagFalse}},
{"pacific-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6}}}, []string{ipv4FlagFalse, ipv6FlagTrue}},
{"pacific-dualstack-supported", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6, DualStack: true}}}, []string{ipv4FlagTrue, ipv6FlagTrue}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := NetworkBindingFlags(tt.args.cluster, tt.args.spec); !reflect.DeepEqual(got, tt.want) {
if len(got) != 0 && len(tt.want) != 0 {
t.Errorf("NetworkBindingFlags() = %+v, want %+v", got, tt.want)
}
}
})
}
}