diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 4a6a25a1f..db652eab3 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -304,7 +304,9 @@ The selector keys are required to be `public` and `cluster` where each represent * `public`: client communications with the cluster (reads/writes) * `cluster`: internal Ceph replication network -If you want to learn more, please read [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/). +If you want to learn more, please read +* [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/). +* [Multus documentation](https://intel.github.io/multus-cni/doc/how-to-use.html) Based on the configuration, the operator will do the following: @@ -314,14 +316,47 @@ Based on the configuration, the operator will do the following: In order to work, each selector value must match a `NetworkAttachmentDefinition` object name in Multus. For example, you can do: -* `public`: "my-public-storage-network" -* `cluster`: "my-replication-storage-network" +* `public`: "rook-ceph/my-public-storage-network" +* `cluster`: "rook-ceph/my-replication-storage-network" For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD. The Network attachment definitions should be using whereabouts cni. If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods. You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`. +A valid NetworkAttachmentDefinition will look like following: + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: rook-public-nw +spec: + config: '{ + "cniVersion": "0.3.0", + "name": "public-nad", + "type": "macvlan", + "master": "ens5", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "192.168.1.0/24" + } + }' +``` + +* Ensure that `master` matches the network interface of the host that you want to use. +* The NAD should be referenced along with the namespace in which it is present like `public: /`. + e.g., the network attachment definition are in `rook-multus` namespace: + +```yaml + public: rook-multus/rook-public-nw + cluster: rook-multus/rook-cluster-nw +``` + +This is required in order to use the NAD across namespaces. +* In Openshift, to use the NetworkAttachmentDefinition across namespaces, the NAD must be deployed in the default namespace and it can be referenced as `default/myNAD` where `default` is the namespace and `myNAD` is the network attachment definition. + #### IPFamily Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is not supported by ceph. diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 8114bdd1f..9c68238f1 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -129,12 +129,6 @@ rules: - create - update - delete -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get --- # The cluster role for managing the Rook CRDs apiVersion: rbac.authorization.k8s.io/v1 @@ -248,6 +242,12 @@ rules: - delete - get - update +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- # Aspects of ceph-mgr that require cluster-wide access kind: ClusterRole diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 23383f1d9..83c00e074 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -371,8 +371,13 @@ func (c *ClusterController) preClusterStartValidation(cluster *cluster) error { continue } + multusNamespace, nad := config.GetMultusNamespace(cluster.Spec.Network.Selectors[selector]) + if multusNamespace == "" { + multusNamespace = cluster.Namespace + } + // Get network attachment definition - _, err := c.context.NetworkClient.NetworkAttachmentDefinitions(cluster.Namespace).Get(cluster.Spec.Network.Selectors[selector], metav1.GetOptions{}) + _, err := c.context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(nad, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector) diff --git a/pkg/operator/ceph/config/network.go b/pkg/operator/ceph/config/network.go index 97a54709f..cc75bc928 100644 --- a/pkg/operator/ceph/config/network.go +++ b/pkg/operator/ceph/config/network.go @@ -19,6 +19,7 @@ package config import ( "fmt" + "strings" "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" @@ -52,11 +53,15 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ continue } + multusNamespace, nad := GetMultusNamespace(networkSelectors[selectorKey]) + if multusNamespace == "" { + multusNamespace = namespace + } // Get network attachment definition - netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(namespace).Get(networkSelectors[selectorKey], metav1.GetOptions{}) + netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(multusNamespace).Get(nad, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { - return []Option{}, errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selectorKey) + return []Option{}, errors.Wrapf(err, "specified network attachment definition %q in namespace %q for selector %q does not exist", nad, namespace, selectorKey) } return []Option{}, errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selectorKey) } @@ -78,6 +83,14 @@ func generateNetworkSettings(context *clusterd.Context, namespace string, networ return cephNetworks, nil } +func GetMultusNamespace(nad string) (string, string) { + tmp := strings.Split(nad, "/") + if len(tmp) == 2 { + return tmp[0], tmp[1] + } + return "", nad +} + func getNetworkRange(netConfig k8sutil.NetworkAttachmentConfig) string { if netConfig.Ipam.Subnet != "" { return netConfig.Ipam.Subnet diff --git a/pkg/operator/ceph/config/network_test.go b/pkg/operator/ceph/config/network_test.go index 6eb4896c1..6f0c2b62c 100644 --- a/pkg/operator/ceph/config/network_test.go +++ b/pkg/operator/ceph/config/network_test.go @@ -196,3 +196,15 @@ func TestGetNetworkRange(t *testing.T) { networkRange = getNetworkRange(netConfig) assert.Equal(t, "192.168.0.0/24", networkRange) } + +func TestGetMultusNamespace(t *testing.T) { + // TEST 1: When namespace is specified with the NAD + namespace, nad := GetMultusNamespace("multus-ns/public-nad") + assert.Equal(t, "multus-ns", namespace) + assert.Equal(t, "public-nad", nad) + + // TEST 2: When only NAD is specified + namespace, nad = GetMultusNamespace("public-nad") + assert.Empty(t, namespace) + assert.Equal(t, "public-nad", nad) +} diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 53096e171..b7fbd5a40 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -542,7 +542,7 @@ func applyCephClusterNetworkConfig(ctx *clusterd.Context, objectMeta *metav1.Obj cephClusters := &v1.CephClusterList{} err := ctx.Client.List(context.TODO(), cephClusters, &client.ListOptions{}) if err != nil { - return false, errors.Wrapf(err, "failed to find CephClusters") + return false, errors.Wrap(err, "failed to find CephClusters") } for _, cephCluster := range cephClusters.Items { if cephCluster.Spec.Network.IsMultus() {