ceph: support NAD from diffrent namespaces

Added the support of NAD from diffrent namespaces.
they can be referrenced as <namespace>/<name-of-nad> e.g.,
default/public-nw.
Updated the multus doc to explain the same.

Signed-off-by: rohan47 <rohgupta@redhat.com>

Documentation/ceph-cluster-crd.md

@@ -304,7 +304,9 @@ The selector keys are required to be `public` and `cluster` where each represent
 * `public`: client communications with the cluster (reads/writes)
 * `cluster`: internal Ceph replication network
 
-If you want to learn more, please read [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/).
+If you want to learn more, please read
+* [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/).
+* [Multus documentation](https://intel.github.io/multus-cni/doc/how-to-use.html)
 
 Based on the configuration, the operator will do the following:
 
@@ -314,14 +316,47 @@ Based on the configuration, the operator will do the following:
 In order to work, each selector value must match a `NetworkAttachmentDefinition` object name in Multus.
 For example, you can do:
 
-* `public`: "my-public-storage-network"
-* `cluster`: "my-replication-storage-network"
+* `public`: "rook-ceph/my-public-storage-network"
+* `cluster`: "rook-ceph/my-replication-storage-network"
 
 For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD.
 The Network attachment definitions should be using whereabouts cni.
 If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods.
 You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`.
 
+A valid NetworkAttachmentDefinition will look like following:
+
+```yaml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: rook-public-nw
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "name": "public-nad",
+      "type": "macvlan",
+      "master": "ens5",
+      "mode": "bridge",
+      "ipam": {
+        "type": "whereabouts",
+        "range": "192.168.1.0/24"
+      }
+    }'
+```
+
+* Ensure that `master` matches the network interface of the host that you want to use.
+* The NAD should be referenced along with the namespace in which it is present like `public: <namespace>/<name of NAD>`.
+  e.g., the network attachment definition are in `rook-multus` namespace:
+
+```yaml
+  public: rook-multus/rook-public-nw
+  cluster: rook-multus/rook-cluster-nw
+```
+
+This is required in order to use the NAD across namespaces.
+* In Openshift, to use the NetworkAttachmentDefinition across namespaces, the NAD must be deployed in the default namespace and it can be referenced as `default/myNAD` where `default` is the namespace and `myNAD` is the network attachment definition.
+
 #### IPFamily
 
 Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is not supported by ceph.

cluster/examples/kubernetes/ceph/common.yaml

@@ -129,12 +129,6 @@ rules:
   - create
   - update
   - delete
-- apiGroups:
-  - k8s.cni.cncf.io
-  resources:
-  - network-attachment-definitions
-  verbs:
-  - get
 ---
 # The cluster role for managing the Rook CRDs
 apiVersion: rbac.authorization.k8s.io/v1
@@ -248,6 +242,12 @@ rules:
   - delete
   - get
   - update
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - network-attachment-definitions
+  verbs:
+  - get
 ---
 # Aspects of ceph-mgr that require cluster-wide access
 kind: ClusterRole

pkg/operator/ceph/cluster/cluster.go

@@ -371,8 +371,13 @@ func (c *ClusterController) preClusterStartValidation(cluster *cluster) error {
 				continue
 			}
 
+			multusNamespace, nad := config.GetMultusNamespace(cluster.Spec.Network.Selectors[selector])
+			if multusNamespace == "" {
+				multusNamespace = cluster.Namespace
+			}
+
 			// Get network attachment definition
-			_, err := c.context.NetworkClient.NetworkAttachmentDefinitions(cluster.Namespace).Get(cluster.Spec.Network.Selectors[selector], metav1.GetOptions{})
+			_, err := c.context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(nad, metav1.GetOptions{})
 			if err != nil {
 				if kerrors.IsNotFound(err) {
 					return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector)

pkg/operator/ceph/config/network.go

@@ -19,6 +19,7 @@ package config
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
@@ -52,11 +53,15 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ
 			continue
 		}
 
+		multusNamespace, nad := GetMultusNamespace(networkSelectors[selectorKey])
+		if multusNamespace == "" {
+			multusNamespace = namespace
+		}
 		// Get network attachment definition
-		netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(namespace).Get(networkSelectors[selectorKey], metav1.GetOptions{})
+		netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(nad, metav1.GetOptions{})
 		if err != nil {
 			if kerrors.IsNotFound(err) {
-				return []Option{}, errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selectorKey)
+				return []Option{}, errors.Wrapf(err, "specified network attachment definition %q in namespace %q for selector %q does not exist", nad, namespace, selectorKey)
 			}
 			return []Option{}, errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selectorKey)
 		}
@@ -78,6 +83,14 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ
 	return cephNetworks, nil
 }
 
+func GetMultusNamespace(nad string) (string, string) {
+	tmp := strings.Split(nad, "/")
+	if len(tmp) == 2 {
+		return tmp[0], tmp[1]
+	}
+	return "", nad
+}
+
 func getNetworkRange(netConfig k8sutil.NetworkAttachmentConfig) string {
 	if netConfig.Ipam.Subnet != "" {
 		return netConfig.Ipam.Subnet

pkg/operator/ceph/config/network_test.go

@@ -196,3 +196,15 @@ func TestGetNetworkRange(t *testing.T) {
 	networkRange = getNetworkRange(netConfig)
 	assert.Equal(t, "192.168.0.0/24", networkRange)
 }
+
+func TestGetMultusNamespace(t *testing.T) {
+	// TEST 1: When namespace is specified with the NAD
+	namespace, nad := GetMultusNamespace("multus-ns/public-nad")
+	assert.Equal(t, "multus-ns", namespace)
+	assert.Equal(t, "public-nad", nad)
+
+	// TEST 2: When only NAD is specified
+	namespace, nad = GetMultusNamespace("public-nad")
+	assert.Empty(t, namespace)
+	assert.Equal(t, "public-nad", nad)
+}

pkg/operator/ceph/csi/spec.go

@@ -542,7 +542,7 @@ func applyCephClusterNetworkConfig(ctx *clusterd.Context, objectMeta *metav1.Obj
 	cephClusters := &v1.CephClusterList{}
 	err := ctx.Client.List(context.TODO(), cephClusters, &client.ListOptions{})
 	if err != nil {
-		return false, errors.Wrapf(err, "failed to find CephClusters")
+		return false, errors.Wrap(err, "failed to find CephClusters")
 	}
 	for _, cephCluster := range cephClusters.Items {
 		if cephCluster.Spec.Network.IsMultus() {