From f5bbc8c5c52193c0b03ffded0b94d9ea2aaa8a7d Mon Sep 17 00:00:00 2001 From: rohan47 Date: Tue, 21 Jul 2020 20:20:48 +0530 Subject: [PATCH] ceph: fix networkAttachmentConfig to be compatible with whereabouts cni In the whereabouts config, the network range is determined by range option unlike other cni plugins that use Subnet option. Signed-off-by: rohan47 --- Documentation/ceph-cluster-crd.md | 1 + pkg/operator/ceph/config/network.go | 16 ++++++- pkg/operator/ceph/config/network_test.go | 58 ++++++++++++++++++++++++ pkg/operator/k8sutil/network.go | 1 + 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 6305d06b6..a1339ac2b 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -239,6 +239,7 @@ For example, you can do: * `cluster`: "my-replication-storage-network" For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD. +The Network attachment definitions should be using whereabouts cni. If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods. You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`. diff --git a/pkg/operator/ceph/config/network.go b/pkg/operator/ceph/config/network.go index 6ecbfd432..97a54709f 100644 --- a/pkg/operator/ceph/config/network.go +++ b/pkg/operator/ceph/config/network.go @@ -32,6 +32,8 @@ const ( PublicNetworkSelectorKeyName = "public" // ClusterNetworkSelectorKeyName is the network selector key for the ceph cluster network ClusterNetworkSelectorKeyName = "cluster" + // Whereabouts Ipam type + WhereaboutsIpamType = "whereabouts" ) var ( @@ -65,8 +67,9 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ return []Option{}, errors.Wrapf(err, "failed to get network attachment definition configuration for selector %q", selectorKey) } - if netConfig.Ipam.Subnet != "" { - cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), netConfig.Ipam.Subnet)) + networkRange := getNetworkRange(netConfig) + if networkRange != "" { + cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), networkRange)) } else { return []Option{}, errors.Errorf("empty subnet from network attachment definition %q", networkSelectors[selectorKey]) } @@ -74,3 +77,12 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ return cephNetworks, nil } + +func getNetworkRange(netConfig k8sutil.NetworkAttachmentConfig) string { + if netConfig.Ipam.Subnet != "" { + return netConfig.Ipam.Subnet + } else if netConfig.Ipam.Range != "" && netConfig.Ipam.Type == WhereaboutsIpamType { + return netConfig.Ipam.Range + } + return "" +} diff --git a/pkg/operator/ceph/config/network_test.go b/pkg/operator/ceph/config/network_test.go index c1c1beeac..6b9c0e8a6 100644 --- a/pkg/operator/ceph/config/network_test.go +++ b/pkg/operator/ceph/config/network_test.go @@ -23,6 +23,7 @@ import ( networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" fakenetclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,6 +63,7 @@ func TestGenerateNetworkSettings(t *testing.T) { }, } + // this nad uses whereabouts cni network := &networkv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{ Name: "public-network-attach-def", @@ -136,3 +138,59 @@ func TestGenerateNetworkSettings(t *testing.T) { assert.NoError(t, err) assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork)) } + +func TestGetNetworkRange(t *testing.T) { + ns := "rook-ceph" + nad := &networkv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "public-network-attach-def", + Namespace: ns, + }, + Spec: networkv1.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "", + "gateway": "172.18.8.1" + } + }`, + }, + } + + netConfig, err := k8sutil.GetNetworkAttachmentConfig(*nad) + assert.NoError(t, err) + + // + // TEST 1: subnet/range is empty + // + networkRange := getNetworkRange(netConfig) + assert.Empty(t, networkRange) + + // + // TEST 2: subnet is not empty + // + netConfig.Ipam.Type = "host-local" + netConfig.Ipam.Subnet = "192.168.0.0/24" + networkRange = getNetworkRange(netConfig) + assert.Equal(t, "192.168.0.0/24", networkRange) + + // + // TEST 3: range is empty and ipam type is whereabouts + // + netConfig.Ipam.Type = "whereabouts" + netConfig.Ipam.Subnet = "" + networkRange = getNetworkRange(netConfig) + assert.Empty(t, networkRange) + + // + // TEST 4: range is not empty and ipam type is whereabouts + // + netConfig.Ipam.Type = "whereabouts" + netConfig.Ipam.Range = "192.168.0.0/24" + networkRange = getNetworkRange(netConfig) + assert.Equal(t, "192.168.0.0/24", networkRange) +} diff --git a/pkg/operator/k8sutil/network.go b/pkg/operator/k8sutil/network.go index 8be0a20bb..0eb62860b 100644 --- a/pkg/operator/k8sutil/network.go +++ b/pkg/operator/k8sutil/network.go @@ -35,6 +35,7 @@ type NetworkAttachmentConfig struct { Ipam struct { Type string `json:"type"` Subnet string `json:"subnet"` + Range string `json:"range"` RangeStart string `json:"rangeStart"` RangeEnd string `json:"rangeEnd"` Routes []struct {