ceph: fix networkAttachmentConfig to be compatible with whereabouts cni

In the whereabouts config, the network range is determined by range
option unlike other cni plugins that use Subnet option.

Signed-off-by: rohan47 <rohgupta@redhat.com>

Documentation/ceph-cluster-crd.md

@@ -239,6 +239,7 @@ For example, you can do:
 * `cluster`: "my-replication-storage-network"
 
 For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD.
+The Network attachment definitions should be using whereabouts cni.
 If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods.
 You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`.
 

pkg/operator/ceph/config/network.go

@@ -32,6 +32,8 @@ const (
 	PublicNetworkSelectorKeyName = "public"
 	// ClusterNetworkSelectorKeyName is the network selector key for the ceph cluster network
 	ClusterNetworkSelectorKeyName = "cluster"
+	// Whereabouts Ipam type
+	WhereaboutsIpamType = "whereabouts"
 )
 
 var (
@@ -65,8 +67,9 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ
 			return []Option{}, errors.Wrapf(err, "failed to get network attachment definition configuration for selector %q", selectorKey)
 		}
 
-		if netConfig.Ipam.Subnet != "" {
-			cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), netConfig.Ipam.Subnet))
+		networkRange := getNetworkRange(netConfig)
+		if networkRange != "" {
+			cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), networkRange))
 		} else {
 			return []Option{}, errors.Errorf("empty subnet from network attachment definition %q", networkSelectors[selectorKey])
 		}
@@ -74,3 +77,12 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ
 
 	return cephNetworks, nil
 }
+
+func getNetworkRange(netConfig k8sutil.NetworkAttachmentConfig) string {
+	if netConfig.Ipam.Subnet != "" {
+		return netConfig.Ipam.Subnet
+	} else if netConfig.Ipam.Range != "" && netConfig.Ipam.Type == WhereaboutsIpamType {
+		return netConfig.Ipam.Range
+	}
+	return ""
+}

pkg/operator/ceph/config/network_test.go

@@ -23,6 +23,7 @@ import (
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	fakenetclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
 	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/operator/k8sutil"
 	testop "github.com/rook/rook/pkg/operator/test"
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -62,6 +63,7 @@ func TestGenerateNetworkSettings(t *testing.T) {
 		},
 	}
 
+	// this nad uses whereabouts cni
 	network := &networkv1.NetworkAttachmentDefinition{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "public-network-attach-def",
@@ -136,3 +138,59 @@ func TestGenerateNetworkSettings(t *testing.T) {
 	assert.NoError(t, err)
 	assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork))
 }
+
+func TestGetNetworkRange(t *testing.T) {
+	ns := "rook-ceph"
+	nad := &networkv1.NetworkAttachmentDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "public-network-attach-def",
+			Namespace: ns,
+		},
+		Spec: networkv1.NetworkAttachmentDefinitionSpec{
+			Config: `{
+				"cniVersion": "0.3.0",
+				"type": "macvlan",
+				"master": "eth2",
+				"mode": "bridge",
+				"ipam": {
+				  "type": "host-local",
+				  "subnet": "",
+				  "gateway": "172.18.8.1"
+				}
+			  }`,
+		},
+	}
+
+	netConfig, err := k8sutil.GetNetworkAttachmentConfig(*nad)
+	assert.NoError(t, err)
+
+	//
+	// TEST 1: subnet/range is empty
+	//
+	networkRange := getNetworkRange(netConfig)
+	assert.Empty(t, networkRange)
+
+	//
+	// TEST 2: subnet is not empty
+	//
+	netConfig.Ipam.Type = "host-local"
+	netConfig.Ipam.Subnet = "192.168.0.0/24"
+	networkRange = getNetworkRange(netConfig)
+	assert.Equal(t, "192.168.0.0/24", networkRange)
+
+	//
+	// TEST 3: range is empty and ipam type is whereabouts
+	//
+	netConfig.Ipam.Type = "whereabouts"
+	netConfig.Ipam.Subnet = ""
+	networkRange = getNetworkRange(netConfig)
+	assert.Empty(t, networkRange)
+
+	//
+	// TEST 4: range is not empty and ipam type is whereabouts
+	//
+	netConfig.Ipam.Type = "whereabouts"
+	netConfig.Ipam.Range = "192.168.0.0/24"
+	networkRange = getNetworkRange(netConfig)
+	assert.Equal(t, "192.168.0.0/24", networkRange)
+}

pkg/operator/k8sutil/network.go

@@ -35,6 +35,7 @@ type NetworkAttachmentConfig struct {
 	Ipam       struct {
 		Type       string `json:"type"`
 		Subnet     string `json:"subnet"`
+		Range      string `json:"range"`
 		RangeStart string `json:"rangeStart"`
 		RangeEnd   string `json:"rangeEnd"`
 		Routes     []struct {