From b2f2ff6b9dd91a3f3f8f4016cb276df36517c987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Mon, 17 Jun 2019 13:59:16 +0200 Subject: [PATCH] ceph: osd: fix startup on sdn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a new flag to the osd startup so that on msgr2 (default on Nautilus and above) the osd is able to find the IP address in the container to bind to. This requires this Ceph patch https://github.com/ceph/ceph/pull/28589 and is already present in Octopus. Closes: https://github.com/rook/rook/issues/3140 Signed-off-by: Sébastien Han --- pkg/operator/ceph/cluster/osd/spec.go | 17 +++++++++++++++++ pkg/operator/ceph/cluster/osd/spec_test.go | 15 +++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index bb070667f..bd5db0ca4 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/ceph/version" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" batch "k8s.io/api/batch/v1" @@ -172,6 +173,8 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, commonArgs = append(commonArgs, "--default-log-to-file", "false") } + commonArgs = append(commonArgs, osdOnSDNFlag(c.HostNetwork, c.clusterInfo.CephVersion)...) + // Add the volume to the spec and the mount to the daemon container copyBinariesVolume, copyBinariesContainer := c.getCopyBinariesContainer() volumes = append(volumes, copyBinariesVolume) @@ -222,6 +225,8 @@ func (c *Cluster) makeDeployment(nodeName string, selection rookalpha.Selection, args = append(args, "--default-log-to-file", "false") } + args = append(args, osdOnSDNFlag(c.HostNetwork, c.clusterInfo.CephVersion)...) + // mount /run/udev in the container so ceph-volume (via `lvs`) // can access the udev database volumes = append(volumes, v1.Volume{ @@ -619,3 +624,15 @@ func getConfigFromContainer(osdContainer v1.Container) map[string]string { return cfg } + +func osdOnSDNFlag(hostnetwork bool, v cephver.CephVersion) []string { + var args []string + // OSD fails to find the right IP to bind to when running on SDN + // for more details: https://github.com/rook/rook/issues/3140 + if !hostnetwork { + if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 2}) { + args = append(args, "--ms-learn-addr-from-peer", "false") + } + } + return args +} diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 25aabe6da..4f1a5b4b5 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -328,3 +328,18 @@ func TestHostNetwork(t *testing.T) { assert.Equal(t, true, r.Spec.Template.Spec.HostNetwork) assert.Equal(t, v1.DNSClusterFirstWithHostNet, r.Spec.Template.Spec.DNSPolicy) } + +func TestOsdOnSDNFlag(t *testing.T) { + hostnetwork := false + v := cephver.Mimic + args := osdOnSDNFlag(hostnetwork, v) + assert.Empty(t, args) + + v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 2} + args = osdOnSDNFlag(hostnetwork, v) + assert.NotEmpty(t, args) + + v = cephver.Octopus + args = osdOnSDNFlag(hostnetwork, v) + assert.NotEmpty(t, args) +}