diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 2a2f87bcf..7547f1589 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -123,8 +123,7 @@ If this value is empty, each pod will get an ephemeral directory to store their Recommended: * If you have a single Rook Ceph cluster, set the `rulesNamespace` to the same namespace as the cluster or keep it empty. * If you have multiple Rook Ceph clusters in the same Kubernetes cluster, choose the same namespace to set `rulesNamespace` for all the clusters (ideally, namespace with prometheus deployed). Otherwise, you will get duplicate alerts with duplicate alert definitions. -* `network`: The network settings for the cluster - * `hostNetwork`: uses network of the hosts instead of using the SDN below the containers. +* `network`: For the network settings for the cluster, refer to the [network configuration settings](#network-configuration-settings) * `mon`: contains mon related options [mon settings](#mon-settings) For more details on the mons and when to choose a number other than `3`, see the [mon health design doc](https://github.com/rook/rook/blob/master/design/ceph/mon-health.md). * `mgr`: manager top level section @@ -208,6 +207,44 @@ Some modules will have special configuration to ensure the module is fully funct * `pg_autoscaler`: Rook will configure all new pools with PG autoscaling by setting: `osd_pool_default_pg_autoscale_mode = on` +### Network Configuration Settings + +If not specified, the default SDN will be used. +Configure the network that will be enabled for the cluster and services. + +* `provider`: Specifies the network provider that will be used to connect the network interface. You can choose between `host`, and `multus`. +* `selectors`: List the network selector(s) that will be used associated by a key. + +#### Host Networking + +To use host networking, set `provider: host`. + +#### Multus (EXPERIMENTAL) + +Rook has experimental support for Multus. + +The selector keys are required to be `public` and `cluster` where each represent: + +* `public`: client communications with the cluster (reads/writes) +* `cluster`: internal Ceph replication network + +If you want to learn more, please read [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/). + +Based on the configuration, the operator will do the following: + + 1. if only the `public` selector is specified both communication and replication will happen on that network + 2. if both `public` and `cluster` selectors are specified the first one will run the communication network and the second the replication network + +In order to work, each selector value must match a `NetworkAttachmentDefinition` object name in Multus. +For example, you can do: + +* `public`: "my-public-storage-network" +* `cluster`: "my-replication-storage-network" + +For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD. +If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods. +You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`. + ### Node Settings In addition to the cluster level settings specified above, each individual node can also specify configuration to override the cluster level settings and defaults. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index a9bf0c8d8..6ce22e5b3 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -31,6 +31,7 @@ ConfigMap can be used in mix with the already existing Env Vars defined in opera - Pools can now be configured to inline compress the data using the `compressionMode` parameter. Support added [here](https://github.com/rook/rook/pull/5124) - Ceph OSDs do not use the host PID, but the PID namespace of the pod (more security). The OSD does not see host running processes anymore. - placement of all the ceph daemons now supports [topologySpreadConstraints](Documentation/ceph-cluster-crd.md#placement-configuration-settings). +- Rook is now capable of working with Multus to expose dedicated interfaces to pods, for more information please refer to the [network configuration doc](Documentation/ceph-cluster-crd.html#network-configuration-settings). ### EdgeFS diff --git a/cluster/charts/rook-ceph/templates/role.yaml b/cluster/charts/rook-ceph/templates/role.yaml index 66de05a8b..cc005564e 100644 --- a/cluster/charts/rook-ceph/templates/role.yaml +++ b/cluster/charts/rook-ceph/templates/role.yaml @@ -27,6 +27,12 @@ rules: - create - update - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index 5ff8d7279..0571c37bb 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -39,8 +39,6 @@ spec: dashboard: enabled: true ssl: true - network: - hostNetwork: false crashCollector: disable: false storage: diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 891083cfb..e052b0320 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -68,8 +68,20 @@ spec: # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. rulesNamespace: rook-ceph network: - # toggle to use hostNetwork - hostNetwork: false + # enable host networking + #provider: host + # EXPERIMENTAL: enable the Multus network provider + #provider: multus + #selectors: + # The selector keys are required to be `public` and `cluster`. + # Based on the configuration, the operator will do the following: + # 1. if only the `public` selector key is specified both public_network and cluster_network Ceph settings will listen on that interface + # 2. if both `public` and `cluster` selector keys are specified the first one will point to 'public_network' flag and the second one to 'cluster_network' + # + # In order to work, each selector value must match a NetworkAttachmentDefinition object in Multus + # + #public: public-conf --> NetworkAttachmentDefinition object name in Multus + #cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus rbdMirroring: # The number of daemons that will perform the rbd mirroring. # rbd mirroring must be configured with "rbd mirror" from the rook toolbox. diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 03d169121..8d49d6cff 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -702,6 +702,12 @@ rules: - create - update - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- # The cluster role for managing the Rook CRDs apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml index 860aa315e..09886d998 100644 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml +++ b/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml @@ -235,3 +235,47 @@ metadata: # should be in the namespace of the operator namespace: rook-ceph data: +--- +# The role for the operator to manage resources in its own namespace +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + namespace: rook-ceph + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - apps + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get diff --git a/cmd/rook/rook/rook.go b/cmd/rook/rook/rook.go index 1fded786a..40f0bde6b 100644 --- a/cmd/rook/rook/rook.go +++ b/cmd/rook/rook/rook.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/coreos/pkg/capnslog" + netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" @@ -154,6 +155,9 @@ func NewContext() *clusterd.Context { context.RookClientset, err = rookclient.NewForConfig(context.KubeConfig) TerminateOnError(err, "failed to create rook clientset") + context.NetworkClient, err = netclient.NewForConfig(context.KubeConfig) + TerminateOnError(err, "failed to create network clientset") + return context } diff --git a/go.mod b/go.mod index 9b3ef66d4..214e8f3df 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/google/go-cmp v0.3.1 github.com/google/uuid v1.1.1 github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428 + github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8 github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20200107223247-51020689f1fb github.com/openshift/cluster-api v0.0.0-20191129101638-b09907ac6668 github.com/openshift/machine-api-operator v0.2.1-0.20190903202259-474e14e4965a @@ -55,6 +56,7 @@ replace ( k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2 k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2 k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2 + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a // forces sigs.k8s.io/structured-merge-diff to v1 and not v3 k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.2 k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.2 k8s.io/kubectl => k8s.io/kubectl v0.17.2 diff --git a/go.sum b/go.sum index c0fef4624..129943759 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93C github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coredns/corefile-migration v1.0.4/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -144,6 +145,8 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.10.0+incompatible h1:l6Soi8WCOOVAeCo4W98iBFC6Og7/X8bpRt51oNLZ2C8= +github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -262,6 +265,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -271,6 +275,7 @@ github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -395,6 +400,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8 h1:/+OIW+inkJRBJlIHQqMUXRbYHmQLZwj7lqIV/TUecsE= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8/go.mod h1:bDRYxMNuVGINaUROT+51ToYoimT8cqwYoRHSVGdQyRg= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -691,6 +698,7 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE= golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -783,6 +791,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -827,6 +836,8 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0 h1:7+F62GGWUowoiJOUDivedlBECd/fTeUDJnCu0JetQO0= +golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458 h1:DgonIcqC7u+gVZX7lpuReBil5B/i8fvW/hAQdhT6/ao= golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= @@ -937,11 +948,6 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-aggregator v0.17.2/go.mod h1:8xQTzaH0GrcKPiSB4YYWwWbeQ0j/4zRsbQt8usEMbRg= k8s.io/kube-controller-manager v0.17.2 h1:YqE2AM6YzSppRHjmxHSZn6FiXNy6hTpRPEa2SYpHV5s= k8s.io/kube-controller-manager v0.17.2/go.mod h1:xznSbCHdVODF5StxiBMh3s6HenyCBdsedazlsh6/J3M= -k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6 h1:s9IxTKe9GwDH0S/WaX62nFYr0or32DsTWex9AileL7U= -k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6/go.mod h1:RZvgC8MSN6DjiMV6oIfEE9pDL9CYXokkfaCKZeHm3nc= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-proxy v0.17.2/go.mod h1:PVY+Cqds8qa/TLEqiSgDPgwWBiRHYjeS4kvp/C5dYjc= diff --git a/pkg/clusterd/context.go b/pkg/clusterd/context.go index ee360937b..40bb7179f 100644 --- a/pkg/clusterd/context.go +++ b/pkg/clusterd/context.go @@ -17,6 +17,7 @@ package clusterd import ( "github.com/coreos/pkg/capnslog" + netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/sys" @@ -55,6 +56,9 @@ type Context struct { // Information about the network for this machine and its cluster NetworkInfo NetworkInfo + // NetworkClient is a connection to the CNI plugin API + NetworkClient netclient.K8sCniCncfIoV1Interface + // The local devices detected on the node Devices []*sys.LocalDisk } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index c4488bd8b..ce157e58c 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -246,7 +246,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers if c.Spec.External.Enable { // Apply CRD ConfigOverrideName to the external cluster - err = config.SetDefaultConfigs(c.context, c.Namespace, c.Info) + err = config.SetDefaultConfigs(c.context, c.Namespace, c.Info, cephv1.NetworkSpec{}) if err != nil { // Mons are up, so something else is wrong return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options on the external cluster monitors") diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 297980c0a..6f8c3813d 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -375,7 +375,9 @@ func (c *ClusterController) configureExternalCephCluster(namespace, name string, return nil } -func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error { +// Validate the cluster Specs +func (c *ClusterController) preClusterStartValidation(cluster *cluster, clusterObj *cephv1.CephCluster) error { + if cluster.Spec.Mon.Count == 0 { logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) cluster.Spec.Mon.Count = mon.DefaultMonCount @@ -386,11 +388,46 @@ func (c *ClusterController) configureLocalCephCluster(namespace, name string, cl if len(cluster.Spec.Storage.Directories) != 0 { logger.Warning("running osds on directory is not supported anymore, use devices instead.") } + if cluster.Spec.Network.IsMultus() { + _, isPublic := cluster.Spec.Network.Selectors[config.PublicNetworkSelectorKeyName] + _, isCluster := cluster.Spec.Network.Selectors[config.ClusterNetworkSelectorKeyName] + if !isPublic && !isCluster { + return errors.New("both network selector values for public and cluster selector cannot be empty for multus provider") + } + + for _, selector := range config.NetworkSelectors { + // If one selector is empty, we continue + // This means a single interface is used both public and cluster network + if _, ok := cluster.Spec.Network.Selectors[selector]; !ok { + continue + } + + // Get network attachment definition + _, err := c.context.NetworkClient.NetworkAttachmentDefinitions(cluster.Namespace).Get(cluster.Spec.Network.Selectors[selector], metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector) + } + return errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selector) + } + } + } + + logger.Debug("cluster spec successfully validated") + return nil +} + +func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error { + // Cluster Spec validation + err := c.preClusterStartValidation(cluster, clusterObj) + if err != nil { + return errors.Wrap(err, "failed to perform validation before cluster creation") + } // Start the Rook cluster components. Retry several times in case of failure. failedMessage := "" - err := wait.Poll(clusterCreateInterval, clusterCreateTimeout, + err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) { cephVersion, canRetry, err := c.detectAndValidateCephVersion(cluster, cluster.Spec.CephVersion.Image) if err != nil { diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 90bf39169..7d3fe6677 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -76,7 +76,10 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *apps.Deployment { keyring.Volume().Admin()) if c.Network.IsHost() { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &podSpec.ObjectMeta) } + c.annotations.ApplyToObjectMeta(&podSpec.ObjectMeta) c.applyPrometheusAnnotations(&podSpec.ObjectMeta) c.placement.ApplyToPodSpec(&podSpec.Spec) diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 1bc18b65d..3f209dd29 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -231,7 +231,7 @@ func (c *Cluster) startMons(targetCount int) error { // only once and do it as early as possible in the mon orchestration. setConfigsNeedsRetry := false if existingCount > 0 { - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { // If we fail here, it could be because the mons are not healthy, and this might be // fixed by updating the mon deployments. Instead of returning error here, log a @@ -252,7 +252,7 @@ func (c *Cluster) startMons(targetCount int) error { // values in the config database. Do this only when the existing count is zero so that // this is only done once when the cluster is created. if existingCount == 0 { - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after creating the first mon") } @@ -260,7 +260,7 @@ func (c *Cluster) startMons(targetCount int) error { // Or if we need to retry, only do this when we are on the first iteration of the // loop. This could be in the same if statement as above, but separate it to get a // different error message. - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after updating the existing mons") } @@ -274,7 +274,7 @@ func (c *Cluster) startMons(targetCount int) error { } if setConfigsNeedsRetry { - err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo) + err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network) if err != nil { return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after forcefully updating the existing mons") } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 3992ad1b2..0ef9444bd 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -154,9 +154,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string) HostNetwork: c.Network.IsHost(), PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames), } - if c.Network.IsHost() { - podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } + // Replace default unreachable node toleration if c.spec.Mon.VolumeClaimTemplate != nil { k8sutil.AddUnreachableNodeToleration(&podSpec) @@ -172,6 +170,12 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string) } cephv1.GetMonAnnotations(c.spec.Annotations).ApplyToObjectMeta(&pod.ObjectMeta) + if c.Network.IsHost() { + pod.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &pod.ObjectMeta) + } + return pod } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index d22a5c1d1..86f78e619 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -361,11 +361,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC // needed for luksOpen synchronization when devices are encrypted and the osd is prepared with LVM hostIPC := osdProps.storeConfig.EncryptedDevice - DNSPolicy := v1.DNSClusterFirst - if c.Network.IsHost() { - DNSPolicy = v1.DNSClusterFirstWithHostNet - } - initContainers := make([]v1.Container, 0, 4) if doConfigInit { initContainers = append(initContainers, @@ -423,6 +418,42 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC securityContext, )) + podTemplateSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: AppName, + Labels: c.getOSDLabels(osd.ID, failureDomainValue, osdProps.portable), + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + ServiceAccountName: serviceAccountName, + HostNetwork: c.Network.IsHost(), + HostPID: hostPID, + HostIPC: hostIPC, + PriorityClassName: c.priorityClassName, + InitContainers: initContainers, + Containers: []v1.Container{ + { + Command: command, + Args: args, + Name: "osd", + Image: c.cephVersion.Image, + VolumeMounts: volumeMounts, + Env: envVars, + Resources: osdProps.resources, + SecurityContext: securityContext, + LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID), + }, + }, + Volumes: volumes, + }, + } + + if c.Network.IsHost() { + podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &podTemplateSpec.ObjectMeta) + } + deployment := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, @@ -440,36 +471,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC Strategy: apps.DeploymentStrategy{ Type: apps.RecreateDeploymentStrategyType, }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: AppName, - Labels: c.getOSDLabels(osd.ID, failureDomainValue, osdProps.portable), - }, - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyAlways, - ServiceAccountName: serviceAccountName, - HostNetwork: c.Network.IsHost(), - HostPID: hostPID, - HostIPC: hostIPC, - DNSPolicy: DNSPolicy, - PriorityClassName: c.priorityClassName, - InitContainers: initContainers, - Containers: []v1.Container{ - { - Command: command, - Args: args, - Name: "osd", - Image: c.cephVersion.Image, - VolumeMounts: volumeMounts, - Env: envVars, - Resources: osdProps.resources, - SecurityContext: securityContext, - LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID), - }, - }, - Volumes: volumes, - }, - }, + Template: podTemplateSpec, Replicas: &replicaCount, }, } diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index 67c2abaac..d6ae7f303 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -50,6 +50,8 @@ func (m *Mirroring) makeDeployment(daemonConfig *daemonConfig) *apps.Deployment if m.Network.IsHost() { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if m.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(m.Network.NetworkSpec, &podSpec.ObjectMeta) } m.placement.ApplyToPodSpec(&podSpec.Spec) diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index 537e2e985..647a106f8 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -26,6 +26,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" ) @@ -103,6 +104,7 @@ func SetDefaultConfigs( context *clusterd.Context, namespace string, clusterInfo *cephconfig.ClusterInfo, + networkSpec cephv1.NetworkSpec, ) error { // ceph.conf is never used. All configurations are made in the centralized mon config database, // or they are specified on the commandline when daemons are called. @@ -116,5 +118,19 @@ func SetDefaultConfigs( return errors.Wrapf(err, "failed to apply legacy config overrides") } + // Apply Multus if needed + if networkSpec.IsMultus() { + logger.Info("configuring ceph network(s) with multus") + cephNetworks, err := generateNetworkSettings(context, namespace, networkSpec.Selectors) + if err != nil { + errors.Wrap(err, "failed to generate network settings") + } + + // Apply ceph network settings to the mon config store + if err := monStore.SetAll(cephNetworks...); err != nil { + return errors.Wrap(err, "failed to network config overrides") + } + } + return nil } diff --git a/pkg/operator/ceph/config/network.go b/pkg/operator/ceph/config/network.go new file mode 100644 index 000000000..6ecbfd432 --- /dev/null +++ b/pkg/operator/ceph/config/network.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package config provides default configurations which Rook will set in Ceph clusters. +package config + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // PublicNetworkSelectorKeyName is the network selector key for the ceph public network + PublicNetworkSelectorKeyName = "public" + // ClusterNetworkSelectorKeyName is the network selector key for the ceph cluster network + ClusterNetworkSelectorKeyName = "cluster" +) + +var ( + // NetworkSelectors is a slice of ceph network selector key name + NetworkSelectors = []string{PublicNetworkSelectorKeyName, ClusterNetworkSelectorKeyName} +) + +func generateNetworkSettings(context *clusterd.Context, namespace string, networkSelectors map[string]string) ([]Option, error) { + cephNetworks := []Option{} + + for _, selectorKey := range NetworkSelectors { + // This means only "public" was specified and thus we use the same subnet for cluster too + if _, ok := networkSelectors[selectorKey]; !ok { + cephNetworks = append(cephNetworks, cephNetworks[0]) + cephNetworks[1].Option = fmt.Sprintf("%s_network", selectorKey) + continue + } + + // Get network attachment definition + netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(namespace).Get(networkSelectors[selectorKey], metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return []Option{}, errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selectorKey) + } + return []Option{}, errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selectorKey) + } + + // Get network attachment definition configuration + netConfig, err := k8sutil.GetNetworkAttachmentConfig(*netDefinition) + if err != nil { + return []Option{}, errors.Wrapf(err, "failed to get network attachment definition configuration for selector %q", selectorKey) + } + + if netConfig.Ipam.Subnet != "" { + cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), netConfig.Ipam.Subnet)) + } else { + return []Option{}, errors.Errorf("empty subnet from network attachment definition %q", networkSelectors[selectorKey]) + } + } + + return cephNetworks, nil +} diff --git a/pkg/operator/ceph/config/network_test.go b/pkg/operator/ceph/config/network_test.go new file mode 100644 index 000000000..c1c1beeac --- /dev/null +++ b/pkg/operator/ceph/config/network_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "testing" + + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + fakenetclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/clusterd" + testop "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGenerateNetworkSettings(t *testing.T) { + ns := "rook-ceph" + clientset := testop.New(t, 1) + ctx := &clusterd.Context{ + Clientset: clientset, + NetworkClient: fakenetclient.NewSimpleClientset().K8sCniCncfIoV1(), + } + + // + // TEST 1: network definition does not exist + // + netSelector := map[string]string{ + "public": "public-network-attach-def", + } + + cephNetwork, err := generateNetworkSettings(ctx, ns, netSelector) + assert.Error(t, err) + + // + // TEST 2: single dedicated networks + // + expectedNetworks := []Option{ + { + Who: "global", + Option: "public_network", + Value: "192.168.0.0/24", + }, + { + Who: "global", + Option: "cluster_network", + Value: "192.168.0.0/24", + }, + } + + network := &networkv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "public-network-attach-def", + Namespace: ns, + }, + Spec: networkv1.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "192.168.0.0/24", + "gateway": "172.18.8.1" + } + }`, + }, + } + + // Create public network definition + ctx.NetworkClient.NetworkAttachmentDefinitions(ns).Create(network) + + cephNetwork, err = generateNetworkSettings(ctx, ns, netSelector) + assert.NoError(t, err) + assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork)) + + // + // TEST 3: two dedicated networks + // + expectedNetworks = []Option{ + { + Who: "global", + Option: "public_network", + Value: "192.168.0.0/24", + }, + { + Who: "global", + Option: "cluster_network", + Value: "172.18.0.0/16", + }, + } + + netSelector = map[string]string{ + "public": "public-network-attach-def", + "cluster": "cluster-network-attach-def", + } + network2 := &networkv1.NetworkAttachmentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-network-attach-def", + Namespace: ns, + }, + Spec: networkv1.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "172.18.0.0/16", + "gateway": "172.18.0.1" + } + }`, + }, + } + + // Create cluster network definition + ctx.NetworkClient.NetworkAttachmentDefinitions(ns).Create(network2) + + cephNetwork, err = generateNetworkSettings(ctx, ns, netSelector) + assert.NoError(t, err) + assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork)) +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 6a5f3195e..cbadd7e5c 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -60,9 +60,6 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { // Replace default unreachable node toleration k8sutil.AddUnreachableNodeToleration(&podSpec.Spec) - if c.clusterSpec.Network.IsHost() { - podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&podSpec.ObjectMeta) c.fs.Spec.MetadataServer.Placement.ApplyToPodSpec(&podSpec.Spec) @@ -84,6 +81,13 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { }, }, } + + if c.clusterSpec.Network.IsHost() { + d.Spec.Template.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.clusterSpec.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(c.clusterSpec.Network.NetworkSpec, &podSpec.ObjectMeta) + } + k8sutil.AddRookVersionLabelToDeployment(d) c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&d.ObjectMeta) controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d) diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index edd44860a..062312843 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -143,6 +143,13 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) }, Spec: podSpec, } + + if r.cephClusterSpec.Network.IsHost() { + podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if r.cephClusterSpec.Network.NetworkSpec.IsMultus() { + k8sutil.ApplyMultus(r.cephClusterSpec.Network.NetworkSpec, &podTemplateSpec.ObjectMeta) + } + nfs.Spec.Server.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) // Multiple replicas of the nfs service would be handled by creating a service and a new deployment for each one, rather than increasing the pod count here diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index d0304fdf2..94ffcff56 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -51,6 +51,7 @@ type clusterConfig struct { skipUpgradeChecks bool client client.Client scheme *runtime.Scheme + Network cephv1.NetworkSpec } type rgwConfig struct { diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 5e9e52692..e664d5653 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -61,7 +61,7 @@ func TestStartRGW(t *testing.T) { r := &ReconcileCephObjectStore{client: cl, scheme: s} // start a basic cluster - c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s} + c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s, cephv1.NetworkSpec{}} err := c.startRGWPods() assert.Nil(t, err) @@ -107,7 +107,7 @@ func TestCreateObjectStore(t *testing.T) { object := []runtime.Object{&cephv1.CephObjectStore{}} cl := fake.NewFakeClientWithScheme(s, object...) r := &ReconcileCephObjectStore{client: cl, scheme: s} - c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s} + c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s, cephv1.NetworkSpec{}} err := c.createOrUpdateStore() assert.Nil(t, err) } @@ -136,7 +136,8 @@ func TestGenerateSecretName(t *testing.T) { &cephconfig.DataPathMap{}, false, cl, - scheme.Scheme} + scheme.Scheme, + cephv1.NetworkSpec{}} secret := c.generateSecretName("a") assert.Equal(t, "rook-ceph-rgw-default-a-keyring", secret) } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 8dba5efc2..311dfe270 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -83,10 +83,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec // Replace default unreachable node toleration k8sutil.AddUnreachableNodeToleration(&podSpec) - if c.clusterSpec.Network.IsHost() { - podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet - } - // Set the ssl cert if specified if c.store.Spec.Gateway.SSLCertificateRef != "" { // Keep the SSL secret as secure as possible in the container. Give only user read perms. @@ -114,6 +110,12 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec } c.store.Spec.Gateway.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) + if c.clusterSpec.Network.IsHost() { + podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.clusterSpec.Network.IsMultus() { + k8sutil.ApplyMultus(c.Network.NetworkSpec, &podTemplateSpec.ObjectMeta) + } + return podTemplateSpec } diff --git a/pkg/operator/k8sutil/network.go b/pkg/operator/k8sutil/network.go index 22af146ed..8be0a20bb 100644 --- a/pkg/operator/k8sutil/network.go +++ b/pkg/operator/k8sutil/network.go @@ -21,12 +21,31 @@ import ( "fmt" "strings" + netapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// NetworkAttachmentConfig represents the configuration of the NetworkAttachmentDefinitions object +type NetworkAttachmentConfig struct { + CniVersion string `json:"cniVersion"` + Type string `json:"type"` + Master string `json:"master"` + Mode string `json:"mode"` + Ipam struct { + Type string `json:"type"` + Subnet string `json:"subnet"` + RangeStart string `json:"rangeStart"` + RangeEnd string `json:"rangeEnd"` + Routes []struct { + Dst string `json:"dst"` + } `json:"routes"` + Gateway string `json:"gateway"` + } `json:"ipam"` +} + // parseMultusSelector will parse short and JSON form of individual multus -// network attachment selection annotation. Valid JSON will be unmarshaled and +// network attachment selection annotation. Valid JSON will be unmarshalled and // return as is, while invalid JSON will be tried using // /@ short syntax. func parseMultusSelector(selector string) (map[string]string, error) { @@ -75,7 +94,7 @@ func GetMultusIfName(selector string) (string, error) { ifName = name } - // fail selector without inteface name + // fail selector without interface name if ifName == "" { return "", fmt.Errorf("GetMultusIfname: missing interface") } @@ -119,3 +138,16 @@ func ApplyMultus(net rookv1.NetworkSpec, objectMeta *metav1.ObjectMeta) error { return nil } + +// GetNetworkAttachmentConfig returns the NetworkAttachmentDefinitions configuration +func GetNetworkAttachmentConfig(n netapi.NetworkAttachmentDefinition) (NetworkAttachmentConfig, error) { + netConfigJSON := n.Spec.Config + var netConfig NetworkAttachmentConfig + + err := json.Unmarshal([]byte(netConfigJSON), &netConfig) + if err != nil { + return netConfig, fmt.Errorf("failed to unmarshal netconfig json %q. %v", netConfigJSON, err) + } + + return netConfig, nil +} diff --git a/pkg/operator/k8sutil/network_test.go b/pkg/operator/k8sutil/network_test.go index 0454ed503..ac2c6946a 100644 --- a/pkg/operator/k8sutil/network_test.go +++ b/pkg/operator/k8sutil/network_test.go @@ -19,6 +19,7 @@ package k8sutil import ( "testing" + netapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -124,3 +125,32 @@ func TestNetwork_ApplyMultusMixedError(t *testing.T) { assert.Error(t, err) } + +func TestGetNetworkAttachmentConfig(t *testing.T) { + dummyNetAttachDef := netapi.NetworkAttachmentDefinition{ + Spec: netapi.NetworkAttachmentDefinitionSpec{ + Config: `{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth2", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "172.18.8.0/24", + "rangeStart": "172.18.8.200", + "rangeEnd": "172.18.8.216", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "172.18.8.1" + } + }`, + }, + } + + config, err := GetNetworkAttachmentConfig(dummyNetAttachDef) + assert.NoError(t, err) + assert.Equal(t, "172.18.8.0/24", config.Ipam.Subnet) +} diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 0f4b4462b..38e7e3d56 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -675,6 +675,12 @@ rules: - create - update - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole