From 20d154350795742dd7e286398b7616371d6b67d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 1 Apr 2020 09:58:04 +0200 Subject: [PATCH] ceph: add multus support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit You can now use Rook along with Multus. Multus must be up and running and the right ressources must exist such as NetworkAttachmentDefinition CR. The Cluster CR spec has new fields to work with multus: network: provider: multus (or 'host' for hostNetworking) selectors: public: NetworkAttachmentDefinition name cluster: NetworkAttachmentDefinition name If only a single NetworkAttachmentDefinition is provided Rook will use both anyway for the Ceph traffic. Please refer to the doc to learn more. Closes: https://github.com/rook/rook/issues/4716 Signed-off-by: Sébastien Han --- Documentation/ceph-cluster-crd.md | 41 +++++- PendingReleaseNotes.md | 1 + cluster/charts/rook-ceph/templates/role.yaml | 6 + .../kubernetes/ceph/cluster-on-pvc.yaml | 2 - cluster/examples/kubernetes/ceph/cluster.yaml | 16 +- cluster/examples/kubernetes/ceph/common.yaml | 6 + .../ceph/upgrade-from-v1.2-apply.yaml | 44 ++++++ cmd/rook/rook/rook.go | 4 + go.mod | 2 + go.sum | 16 +- pkg/clusterd/context.go | 4 + pkg/operator/ceph/cluster/cluster.go | 2 +- pkg/operator/ceph/cluster/controller.go | 41 +++++- pkg/operator/ceph/cluster/mgr/spec.go | 3 + pkg/operator/ceph/cluster/mon/mon.go | 8 +- pkg/operator/ceph/cluster/mon/spec.go | 10 +- pkg/operator/ceph/cluster/osd/spec.go | 72 ++++----- pkg/operator/ceph/cluster/rbd/spec.go | 2 + pkg/operator/ceph/config/config.go | 16 ++ pkg/operator/ceph/config/network.go | 76 ++++++++++ pkg/operator/ceph/config/network_test.go | 138 ++++++++++++++++++ pkg/operator/ceph/file/mds/spec.go | 10 +- pkg/operator/ceph/nfs/spec.go | 7 + pkg/operator/ceph/object/rgw.go | 1 + pkg/operator/ceph/object/rgw_test.go | 7 +- pkg/operator/ceph/object/spec.go | 10 +- pkg/operator/k8sutil/network.go | 36 ++++- pkg/operator/k8sutil/network_test.go | 30 ++++ tests/framework/installer/ceph_manifests.go | 6 + 29 files changed, 549 insertions(+), 68 deletions(-) create mode 100644 pkg/operator/ceph/config/network.go create mode 100644 pkg/operator/ceph/config/network_test.go diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 2a2f87bcf..7547f1589 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -123,8 +123,7 @@ If this value is empty, each pod will get an ephemeral directory to store their Recommended: * If you have a single Rook Ceph cluster, set the `rulesNamespace` to the same namespace as the cluster or keep it empty. * If you have multiple Rook Ceph clusters in the same Kubernetes cluster, choose the same namespace to set `rulesNamespace` for all the clusters (ideally, namespace with prometheus deployed). Otherwise, you will get duplicate alerts with duplicate alert definitions. -* `network`: The network settings for the cluster - * `hostNetwork`: uses network of the hosts instead of using the SDN below the containers. +* `network`: For the network settings for the cluster, refer to the [network configuration settings](#network-configuration-settings) * `mon`: contains mon related options [mon settings](#mon-settings) For more details on the mons and when to choose a number other than `3`, see the [mon health design doc](https://github.com/rook/rook/blob/master/design/ceph/mon-health.md). * `mgr`: manager top level section @@ -208,6 +207,44 @@ Some modules will have special configuration to ensure the module is fully funct * `pg_autoscaler`: Rook will configure all new pools with PG autoscaling by setting: `osd_pool_default_pg_autoscale_mode = on` +### Network Configuration Settings + +If not specified, the default SDN will be used. +Configure the network that will be enabled for the cluster and services. + +* `provider`: Specifies the network provider that will be used to connect the network interface. You can choose between `host`, and `multus`. +* `selectors`: List the network selector(s) that will be used associated by a key. + +#### Host Networking + +To use host networking, set `provider: host`. + +#### Multus (EXPERIMENTAL) + +Rook has experimental support for Multus. + +The selector keys are required to be `public` and `cluster` where each represent: + +* `public`: client communications with the cluster (reads/writes) +* `cluster`: internal Ceph replication network + +If you want to learn more, please read [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/). + +Based on the configuration, the operator will do the following: + + 1. if only the `public` selector is specified both communication and replication will happen on that network + 2. if both `public` and `cluster` selectors are specified the first one will run the communication network and the second the replication network + +In order to work, each selector value must match a `NetworkAttachmentDefinition` object name in Multus. +For example, you can do: + +* `public`: "my-public-storage-network" +* `cluster`: "my-replication-storage-network" + +For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD. +If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods. +You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`. + ### Node Settings In addition to the cluster level settings specified above, each individual node can also specify configuration to override the cluster level settings and defaults. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index a9bf0c8d8..6ce22e5b3 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -31,6 +31,7 @@ ConfigMap can be used in mix with the already existing Env Vars defined in opera - Pools can now be configured to inline compress the data using the `compressionMode` parameter. Support added [here](https://github.com/rook/rook/pull/5124) - Ceph OSDs do not use the host PID, but the PID namespace of the pod (more security). The OSD does not see host running processes anymore. - placement of all the ceph daemons now supports [topologySpreadConstraints](Documentation/ceph-cluster-crd.md#placement-configuration-settings). +- Rook is now capable of working with Multus to expose dedicated interfaces to pods, for more information please refer to the [network configuration doc](Documentation/ceph-cluster-crd.html#network-configuration-settings). ### EdgeFS diff --git a/cluster/charts/rook-ceph/templates/role.yaml b/cluster/charts/rook-ceph/templates/role.yaml index 84743d729..c8e64915a 100644 --- a/cluster/charts/rook-ceph/templates/role.yaml +++ b/cluster/charts/rook-ceph/templates/role.yaml @@ -35,6 +35,12 @@ rules: - create - update - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index 5ff8d7279..0571c37bb 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -39,8 +39,6 @@ spec: dashboard: enabled: true ssl: true - network: - hostNetwork: false crashCollector: disable: false storage: diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 891083cfb..e052b0320 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -68,8 +68,20 @@ spec: # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. rulesNamespace: rook-ceph network: - # toggle to use hostNetwork - hostNetwork: false + # enable host networking + #provider: host + # EXPERIMENTAL: enable the Multus network provider + #provider: multus + #selectors: + # The selector keys are required to be `public` and `cluster`. + # Based on the configuration, the operator will do the following: + # 1. if only the `public` selector key is specified both public_network and cluster_network Ceph settings will listen on that interface + # 2. if both `public` and `cluster` selector keys are specified the first one will point to 'public_network' flag and the second one to 'cluster_network' + # + # In order to work, each selector value must match a NetworkAttachmentDefinition object in Multus + # + #public: public-conf --> NetworkAttachmentDefinition object name in Multus + #cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus rbdMirroring: # The number of daemons that will perform the rbd mirroring. # rbd mirroring must be configured with "rbd mirror" from the rook toolbox. diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 3c831be8f..0c2988597 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -709,6 +709,12 @@ rules: - create - update - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- # The cluster role for managing the Rook CRDs apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml index 860aa315e..09886d998 100644 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml +++ b/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml @@ -235,3 +235,47 @@ metadata: # should be in the namespace of the operator namespace: rook-ceph data: +--- +# The role for the operator to manage resources in its own namespace +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + namespace: rook-ceph + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - apps + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get diff --git a/cmd/rook/rook/rook.go b/cmd/rook/rook/rook.go index 1fded786a..40f0bde6b 100644 --- a/cmd/rook/rook/rook.go +++ b/cmd/rook/rook/rook.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/coreos/pkg/capnslog" + netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" @@ -154,6 +155,9 @@ func NewContext() *clusterd.Context { context.RookClientset, err = rookclient.NewForConfig(context.KubeConfig) TerminateOnError(err, "failed to create rook clientset") + context.NetworkClient, err = netclient.NewForConfig(context.KubeConfig) + TerminateOnError(err, "failed to create network clientset") + return context } diff --git a/go.mod b/go.mod index 9b3ef66d4..214e8f3df 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/google/go-cmp v0.3.1 github.com/google/uuid v1.1.1 github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428 + github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8 github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20200107223247-51020689f1fb github.com/openshift/cluster-api v0.0.0-20191129101638-b09907ac6668 github.com/openshift/machine-api-operator v0.2.1-0.20190903202259-474e14e4965a @@ -55,6 +56,7 @@ replace ( k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2 k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2 k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2 + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a // forces sigs.k8s.io/structured-merge-diff to v1 and not v3 k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.2 k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.2 k8s.io/kubectl => k8s.io/kubectl v0.17.2 diff --git a/go.sum b/go.sum index c0fef4624..129943759 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93C github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coredns/corefile-migration v1.0.4/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -144,6 +145,8 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.10.0+incompatible h1:l6Soi8WCOOVAeCo4W98iBFC6Og7/X8bpRt51oNLZ2C8= +github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -262,6 +265,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -271,6 +275,7 @@ github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -395,6 +400,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8 h1:/+OIW+inkJRBJlIHQqMUXRbYHmQLZwj7lqIV/TUecsE= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8/go.mod h1:bDRYxMNuVGINaUROT+51ToYoimT8cqwYoRHSVGdQyRg= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -691,6 +698,7 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE= golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -783,6 +791,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -827,6 +836,8 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0 h1:7+F62GGWUowoiJOUDivedlBECd/fTeUDJnCu0JetQO0= +golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458 h1:DgonIcqC7u+gVZX7lpuReBil5B/i8fvW/hAQdhT6/ao= golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= @@ -937,11 +948,6 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-aggregator v0.17.2/go.mod h1:8xQTzaH0GrcKPiSB4YYWwWbeQ0j/4zRsbQt8usEMbRg= k8s.io/kube-controller-manager v0.17.2 h1:YqE2AM6YzSppRHjmxHSZn6FiXNy6hTpRPEa2SYpHV5s= k8s.io/kube-controller-manager v0.17.2/go.mod h1:xznSbCHdVODF5StxiBMh3s6HenyCBdsedazlsh6/J3M= -k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6 h1:s9IxTKe9GwDH0S/WaX62nFYr0or32DsTWex9AileL7U= -k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6/go.mod h1:RZvgC8MSN6DjiMV6oIfEE9pDL9CYXokkfaCKZeHm3nc= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-proxy v0.17.2/go.mod h1:PVY+Cqds8qa/TLEqiSgDPgwWBiRHYjeS4kvp/C5dYjc= diff --git a/pkg/clusterd/context.go b/pkg/clusterd/context.go index ee360937b..40bb7179f 100644 --- a/pkg/clusterd/context.go +++ b/pkg/clusterd/context.go @@ -17,6 +17,7 @@ package clusterd import ( "github.com/coreos/pkg/capnslog" + netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/sys" @@ -55,6 +56,9 @@ type Context struct { // Information about the network for this machine and its cluster NetworkInfo NetworkInfo + // NetworkClient is a connection to the CNI plugin API + NetworkClient netclient.K8sCniCncfIoV1Interface + // The local devices detected on the node Devices []*sys.LocalDisk } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index c4488bd8b..ce157e58c 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -246,7 +246,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers if c.Spec.External.Enable { // Apply CRD ConfigOverrideName to the external cluster - err = config.SetDefaultConfigs(c.context, c.Namespace, c.Info) + err = config.SetDefaultConfigs(c.context, c.Namespace, c.Info, cephv1.NetworkSpec{}) if err != nil { // Mons are up, so something else is wrong return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options on the external cluster monitors") diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 297980c0a..6f8c3813d 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -375,7 +375,9 @@ func (c *ClusterController) configureExternalCephCluster(namespace, name string, return nil } -func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error { +// Validate the cluster Specs +func (c *ClusterController) preClusterStartValidation(cluster *cluster, clusterObj *cephv1.CephCluster) error { + if cluster.Spec.Mon.Count == 0 { logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) cluster.Spec.Mon.Count = mon.DefaultMonCount @@ -386,11 +388,46 @@ func (c *ClusterController) configureLocalCephCluster(namespace, name string, cl if len(cluster.Spec.Storage.Directories) != 0 { logger.Warning("running osds on directory is not supported anymore, use devices instead.") } + if cluster.Spec.Network.IsMultus() { + _, isPublic := cluster.Spec.Network.Selectors[config.PublicNetworkSelectorKeyName] + _, isCluster := cluster.Spec.Network.Selectors[config.ClusterNetworkSelectorKeyName] + if !isPublic && !isCluster { + return errors.New("both network selector values for public and cluster selector cannot be empty for multus provider") + } + + for _, selector := range config.NetworkSelectors { + // If one selector is empty, we continue + // This means a single interface is used both public and cluster network + if _, ok := cluster.Spec.Network.Selectors[selector]; !ok { + continue + } + + // Get network attachment definition + _, err := c.context.NetworkClient.NetworkAttachmentDefinitions(cluster.Namespace).Get(cluster.Spec.Network.Selectors[selector], metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector) + } + return errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selector) + } + } + } + + logger.Debug("cluster spec successfully validated") + return nil +} + +func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error { + // Cluster Spec validation + err := c.preClusterStartValidation(cluster, clusterObj) + if err != nil { + return errors.Wrap(err, "failed to perform validation before cluster creation") + } // Start the Rook cluster components. Retry several times in case of failure. failedMessage := "" - err := wait.Poll(clusterCreateInterval, clusterCreateTimeout, + err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) { cephVersion, canRetry, err := c.detectAndValidateCephVersion(cluster, cluster.Spec.CephVersion.Image) if err != nil { diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 90bf39169..7d3fe6677 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -76,7 +76,10 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *apps.Deployment { keyring.Volume().Admin()) if c.Network.IsHost() { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &podSpec.ObjectMeta) } + c.annotations.ApplyToObjectMeta(&podSpec.ObjectMeta) c.applyPrometheusAnnotations(&podSpec.ObjectMeta) c.placement.ApplyToPodSpec(&podSpec.Spec) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index fe08df8c9..1225bcf07 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -231,7 +231,7 @@ func (c *Cluster) startMons(targetCount int) error { // only once and do it as early as possible in the mon orchestration. setConfigsNeedsRetry := false if existingCount > 0 { - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { // If we fail here, it could be because the mons are not healthy, and this might be // fixed by updating the mon deployments. Instead of returning error here, log a @@ -252,7 +252,7 @@ func (c *Cluster) startMons(targetCount int) error { // values in the config database. Do this only when the existing count is zero so that // this is only done once when the cluster is created. if existingCount == 0 { - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after creating the first mon") } @@ -260,7 +260,7 @@ func (c *Cluster) startMons(targetCount int) error { // Or if we need to retry, only do this when we are on the first iteration of the // loop. This could be in the same if statement as above, but separate it to get a // different error message. - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after updating the existing mons") } @@ -274,7 +274,7 @@ func (c *Cluster) startMons(targetCount int) error { } if setConfigsNeedsRetry { - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after forcefully updating the existing mons") } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 3992ad1b2..0ef9444bd 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -154,9 +154,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string) HostNetwork: c.Network.IsHost(), PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames), } - if c.Network.IsHost() { - podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } + // Replace default unreachable node toleration if c.spec.Mon.VolumeClaimTemplate != nil { k8sutil.AddUnreachableNodeToleration(&podSpec) @@ -172,6 +170,12 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string) } cephv1.GetMonAnnotations(c.spec.Annotations).ApplyToObjectMeta(&pod.ObjectMeta) + if c.Network.IsHost() { + pod.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &pod.ObjectMeta) + } + return pod } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index d22a5c1d1..86f78e619 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -361,11 +361,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC // needed for luksOpen synchronization when devices are encrypted and the osd is prepared with LVM hostIPC := osdProps.storeConfig.EncryptedDevice - DNSPolicy := v1.DNSClusterFirst - if c.Network.IsHost() { - DNSPolicy = v1.DNSClusterFirstWithHostNet - } - initContainers := make([]v1.Container, 0, 4) if doConfigInit { initContainers = append(initContainers, @@ -423,6 +418,42 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC securityContext, )) + podTemplateSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: AppName, + Labels: c.getOSDLabels(osd.ID, failureDomainValue, osdProps.portable), + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + ServiceAccountName: serviceAccountName, + HostNetwork: c.Network.IsHost(), + HostPID: hostPID, + HostIPC: hostIPC, + PriorityClassName: c.priorityClassName, + InitContainers: initContainers, + Containers: []v1.Container{ + { + Command: command, + Args: args, + Name: "osd", + Image: c.cephVersion.Image, + VolumeMounts: volumeMounts, + Env: envVars, + Resources: osdProps.resources, + SecurityContext: securityContext, + LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID), + }, + }, + Volumes: volumes, + }, + } + + if c.Network.IsHost() { + podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &podTemplateSpec.ObjectMeta) + } + deployment := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, @@ -440,36 +471,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC Strategy: apps.DeploymentStrategy{ Type: apps.RecreateDeploymentStrategyType, }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: AppName, - Labels: c.getOSDLabels(osd.ID, failureDomainValue, osdProps.portable), - }, - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyAlways, - ServiceAccountName: serviceAccountName, - HostNetwork: c.Network.IsHost(), - HostPID: hostPID, - HostIPC: hostIPC, - DNSPolicy: DNSPolicy, - PriorityClassName: c.priorityClassName, - InitContainers: initContainers, - Containers: []v1.Container{ - { - Command: command, - Args: args, - Name: "osd", - Image: c.cephVersion.Image, - VolumeMounts: volumeMounts, - Env: envVars, - Resources: osdProps.resources, - SecurityContext: securityContext, - LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID), - }, - }, - Volumes: volumes, - }, - }, + Template: podTemplateSpec, Replicas: &replicaCount, }, } diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index 67c2abaac..d6ae7f303 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -50,6 +50,8 @@ func (m *Mirroring) makeDeployment(daemonConfig *daemonConfig) *apps.Deployment if m.Network.IsHost() { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if m.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(m.Network.NetworkSpec, &podSpec.ObjectMeta) } m.placement.ApplyToPodSpec(&podSpec.Spec) diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index 537e2e985..647a106f8 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -26,6 +26,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" ) @@ -103,6 +104,7 @@ func SetDefaultConfigs( context *clusterd.Context, namespace string, clusterInfo *cephconfig.ClusterInfo, + networkSpec cephv1.NetworkSpec, ) error { // ceph.conf is never used. All configurations are made in the centralized mon config database, // or they are specified on the commandline when daemons are called. @@ -116,5 +118,19 @@ func SetDefaultConfigs( return errors.Wrapf(err, "failed to apply legacy config overrides") } + // Apply Multus if needed + if networkSpec.IsMultus() { + logger.Info("configuring ceph network(s) with multus") + cephNetworks, err := generateNetworkSettings(context, namespace, networkSpec.Selectors) + if err != nil { + errors.Wrap(err, "failed to generate network settings") + } + + // Apply ceph network settings to the mon config store + if err := monStore.SetAll(cephNetworks...); err != nil { + return errors.Wrap(err, "failed to network config overrides") + } + } + return nil } diff --git a/pkg/operator/ceph/config/network.go b/pkg/operator/ceph/config/network.go new file mode 100644 index 000000000..6ecbfd432 --- /dev/null +++ b/pkg/operator/ceph/config/network.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package config provides default configurations which Rook will set in Ceph clusters. +package config + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // PublicNetworkSelectorKeyName is the network selector key for the ceph public network + PublicNetworkSelectorKeyName = "public" + // ClusterNetworkSelectorKeyName is the network selector key for the ceph cluster network + ClusterNetworkSelectorKeyName = "cluster" +) + +var ( + // NetworkSelectors is a slice of ceph network selector key name + NetworkSelectors = []string{PublicNetworkSelectorKeyName, ClusterNetworkSelectorKeyName} +) + +func generateNetworkSettings(context *clusterd.Context, namespace string, networkSelectors map[string]string) ([]Option, error) { + cephNetworks := []Option{} + + for _, selectorKey := range NetworkSelectors { + // This means only "public" was specified and thus we use the same subnet for cluster too + if _, ok := networkSelectors[selectorKey]; !ok { + cephNetworks = append(cephNetworks, cephNetworks[0]) + cephNetworks[1].Option = fmt.Sprintf("%s_network", selectorKey) + continue + } + + // Get network attachment definition + netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(namespace).Get(networkSelectors[selectorKey], metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return []Option{}, errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selectorKey) + } + return []Option{}, errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selectorKey) + } + + // Get network attachment definition configuration + netConfig, err := k8sutil.GetNetworkAttachmentConfig(*netDefinition) + if err != nil { + return []Option{}, errors.Wrapf(err, "failed to get network attachment definition configuration for selector %q", selectorKey) + } + + if netConfig.Ipam.Subnet != "" { + cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), netConfig.Ipam.Subnet)) + } else { + return []Option{}, errors.Errorf("empty subnet from network attachment definition %q", networkSelectors[selectorKey]) + } + } + + return cephNetworks, nil +} diff --git a/pkg/operator/ceph/config/network_test.go b/pkg/operator/ceph/config/network_test.go new file mode 100644 index 000000000..c1c1beeac --- /dev/null +++ b/pkg/operator/ceph/config/network_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "testing" + + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + fakenetclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/clusterd" + testop "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGenerateNetworkSettings(t *testing.T) { + ns := "rook-ceph" + clientset := testop.New(t, 1) + ctx := &clusterd.Context{ + Clientset: clientset, + NetworkClient: fakenetclient.NewSimpleClientset().K8sCniCncfIoV1(), + } + + // + // TEST 1: network definition does not exist + // + netSelector := map[string]string{ + "public": "public-network-attach-def", + } + + cephNetwork, err := generateNetworkSettings(ctx, ns, netSelector) + assert.Error(t, err) + + // + // TEST 2: single dedicated networks + // + expectedNetworks := []Option{ + { + Who: "global", + Option: "public_network", + Value: "192.168.0.0/24", + }, + { + Who: "global", + Option: "cluster_network", + Value: "192.168.0.0/24", + }, + } + + network := &networkv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "public-network-attach-def", + Namespace: ns, + }, + Spec: networkv1.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "192.168.0.0/24", + "gateway": "172.18.8.1" + } + }`, + }, + } + + // Create public network definition + ctx.NetworkClient.NetworkAttachmentDefinitions(ns).Create(network) + + cephNetwork, err = generateNetworkSettings(ctx, ns, netSelector) + assert.NoError(t, err) + assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork)) + + // + // TEST 3: two dedicated networks + // + expectedNetworks = []Option{ + { + Who: "global", + Option: "public_network", + Value: "192.168.0.0/24", + }, + { + Who: "global", + Option: "cluster_network", + Value: "172.18.0.0/16", + }, + } + + netSelector = map[string]string{ + "public": "public-network-attach-def", + "cluster": "cluster-network-attach-def", + } + network2 := &networkv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-network-attach-def", + Namespace: ns, + }, + Spec: networkv1.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "172.18.0.0/16", + "gateway": "172.18.0.1" + } + }`, + }, + } + + // Create cluster network definition + ctx.NetworkClient.NetworkAttachmentDefinitions(ns).Create(network2) + + cephNetwork, err = generateNetworkSettings(ctx, ns, netSelector) + assert.NoError(t, err) + assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork)) +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 6a5f3195e..cbadd7e5c 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -60,9 +60,6 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { // Replace default unreachable node toleration k8sutil.AddUnreachableNodeToleration(&podSpec.Spec) - if c.clusterSpec.Network.IsHost() { - podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&podSpec.ObjectMeta) c.fs.Spec.MetadataServer.Placement.ApplyToPodSpec(&podSpec.Spec) @@ -84,6 +81,13 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { }, }, } + + if c.clusterSpec.Network.IsHost() { + d.Spec.Template.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.clusterSpec.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.clusterSpec.Network.NetworkSpec, &podSpec.ObjectMeta) + } + k8sutil.AddRookVersionLabelToDeployment(d) c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&d.ObjectMeta) controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d) diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index edd44860a..062312843 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -143,6 +143,13 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) }, Spec: podSpec, } + + if r.cephClusterSpec.Network.IsHost() { + podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if r.cephClusterSpec.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(r.cephClusterSpec.Network.NetworkSpec, &podTemplateSpec.ObjectMeta) + } + nfs.Spec.Server.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) // Multiple replicas of the nfs service would be handled by creating a service and a new deployment for each one, rather than increasing the pod count here diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index d0304fdf2..94ffcff56 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -51,6 +51,7 @@ type clusterConfig struct { skipUpgradeChecks bool client client.Client scheme *runtime.Scheme + Network cephv1.NetworkSpec } type rgwConfig struct { diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 5e9e52692..e664d5653 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -61,7 +61,7 @@ func TestStartRGW(t *testing.T) { r := &ReconcileCephObjectStore{client: cl, scheme: s} // start a basic cluster - c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s} + c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s, cephv1.NetworkSpec{}} err := c.startRGWPods() assert.Nil(t, err) @@ -107,7 +107,7 @@ func TestCreateObjectStore(t *testing.T) { object := []runtime.Object{&cephv1.CephObjectStore{}} cl := fake.NewFakeClientWithScheme(s, object...) r := &ReconcileCephObjectStore{client: cl, scheme: s} - c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s} + c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s, cephv1.NetworkSpec{}} err := c.createOrUpdateStore() assert.Nil(t, err) } @@ -136,7 +136,8 @@ func TestGenerateSecretName(t *testing.T) { &cephconfig.DataPathMap{}, false, cl, - scheme.Scheme} + scheme.Scheme, + cephv1.NetworkSpec{}} secret := c.generateSecretName("a") assert.Equal(t, "rook-ceph-rgw-default-a-keyring", secret) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 8dba5efc2..311dfe270 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -83,10 +83,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec // Replace default unreachable node toleration k8sutil.AddUnreachableNodeToleration(&podSpec) - if c.clusterSpec.Network.IsHost() { - podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } - // Set the ssl cert if specified if c.store.Spec.Gateway.SSLCertificateRef != "" { // Keep the SSL secret as secure as possible in the container. Give only user read perms. @@ -114,6 +110,12 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec } c.store.Spec.Gateway.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) + if c.clusterSpec.Network.IsHost() { + podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.clusterSpec.Network.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &podTemplateSpec.ObjectMeta) + } + return podTemplateSpec } diff --git a/pkg/operator/k8sutil/network.go b/pkg/operator/k8sutil/network.go index 22af146ed..8be0a20bb 100644 --- a/pkg/operator/k8sutil/network.go +++ b/pkg/operator/k8sutil/network.go @@ -21,12 +21,31 @@ import ( "fmt" "strings" + netapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// NetworkAttachmentConfig represents the configuration of the NetworkAttachmentDefinitions object +type NetworkAttachmentConfig struct { + CniVersion string `json:"cniVersion"` + Type string `json:"type"` + Master string `json:"master"` + Mode string `json:"mode"` + Ipam struct { + Type string `json:"type"` + Subnet string `json:"subnet"` + RangeStart string `json:"rangeStart"` + RangeEnd string `json:"rangeEnd"` + Routes []struct { + Dst string `json:"dst"` + } `json:"routes"` + Gateway string `json:"gateway"` + } `json:"ipam"` +} + // parseMultusSelector will parse short and JSON form of individual multus -// network attachment selection annotation. Valid JSON will be unmarshaled and +// network attachment selection annotation. Valid JSON will be unmarshalled and // return as is, while invalid JSON will be tried using // /@ short syntax. func parseMultusSelector(selector string) (map[string]string, error) { @@ -75,7 +94,7 @@ func GetMultusIfName(selector string) (string, error) { ifName = name } - // fail selector without inteface name + // fail selector without interface name if ifName == "" { return "", fmt.Errorf("GetMultusIfname: missing interface") } @@ -119,3 +138,16 @@ func ApplyMultus(net rookv1.NetworkSpec, objectMeta *metav1.ObjectMeta) error { return nil } + +// GetNetworkAttachmentConfig returns the NetworkAttachmentDefinitions configuration +func GetNetworkAttachmentConfig(n netapi.NetworkAttachmentDefinition) (NetworkAttachmentConfig, error) { + netConfigJSON := n.Spec.Config + var netConfig NetworkAttachmentConfig + + err := json.Unmarshal([]byte(netConfigJSON), &netConfig) + if err != nil { + return netConfig, fmt.Errorf("failed to unmarshal netconfig json %q. %v", netConfigJSON, err) + } + + return netConfig, nil +} diff --git a/pkg/operator/k8sutil/network_test.go b/pkg/operator/k8sutil/network_test.go index 0454ed503..ac2c6946a 100644 --- a/pkg/operator/k8sutil/network_test.go +++ b/pkg/operator/k8sutil/network_test.go @@ -19,6 +19,7 @@ package k8sutil import ( "testing" + netapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -124,3 +125,32 @@ func TestNetwork_ApplyMultusMixedError(t *testing.T) { assert.Error(t, err) } + +func TestGetNetworkAttachmentConfig(t *testing.T) { + dummyNetAttachDef := netapi.NetworkAttachmentDefinition{ + Spec: netapi.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "172.18.8.0/24", + "rangeStart": "172.18.8.200", + "rangeEnd": "172.18.8.216", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "172.18.8.1" + } + }`, + }, + } + + config, err := GetNetworkAttachmentConfig(dummyNetAttachDef) + assert.NoError(t, err) + assert.Equal(t, "172.18.8.0/24", config.Ipam.Subnet) +} diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 3b5608951..21663cd7f 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -686,6 +686,12 @@ rules: - create - update - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole