Merge pull request #4077 from leseb/rook-multus
ceph: add multus support
This commit is contained in:
29 files changed
+549
-68
No files matched your search
@@ -123,8 +123,7 @@ If this value is empty, each pod will get an ephemeral directory to store their
|
||||
Recommended:
|
||||
* If you have a single Rook Ceph cluster, set the `rulesNamespace` to the same namespace as the cluster or keep it empty.
|
||||
* If you have multiple Rook Ceph clusters in the same Kubernetes cluster, choose the same namespace to set `rulesNamespace` for all the clusters (ideally, namespace with prometheus deployed). Otherwise, you will get duplicate alerts with duplicate alert definitions.
|
||||
* `network`: The network settings for the cluster
|
||||
* `hostNetwork`: uses network of the hosts instead of using the SDN below the containers.
|
||||
* `network`: For the network settings for the cluster, refer to the [network configuration settings](#network-configuration-settings)
|
||||
* `mon`: contains mon related options [mon settings](#mon-settings)
|
||||
For more details on the mons and when to choose a number other than `3`, see the [mon health design doc](https://github.com/rook/rook/blob/master/design/ceph/mon-health.md).
|
||||
* `mgr`: manager top level section
|
||||
@@ -208,6 +207,44 @@ Some modules will have special configuration to ensure the module is fully funct
|
||||
|
||||
* `pg_autoscaler`: Rook will configure all new pools with PG autoscaling by setting: `osd_pool_default_pg_autoscale_mode = on`
|
||||
|
||||
### Network Configuration Settings
|
||||
|
||||
If not specified, the default SDN will be used.
|
||||
Configure the network that will be enabled for the cluster and services.
|
||||
|
||||
* `provider`: Specifies the network provider that will be used to connect the network interface. You can choose between `host`, and `multus`.
|
||||
* `selectors`: List the network selector(s) that will be used associated by a key.
|
||||
|
||||
#### Host Networking
|
||||
|
||||
To use host networking, set `provider: host`.
|
||||
|
||||
#### Multus (EXPERIMENTAL)
|
||||
|
||||
Rook has experimental support for Multus.
|
||||
|
||||
The selector keys are required to be `public` and `cluster` where each represent:
|
||||
|
||||
* `public`: client communications with the cluster (reads/writes)
|
||||
* `cluster`: internal Ceph replication network
|
||||
|
||||
If you want to learn more, please read [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/).
|
||||
|
||||
Based on the configuration, the operator will do the following:
|
||||
|
||||
1. if only the `public` selector is specified both communication and replication will happen on that network
|
||||
2. if both `public` and `cluster` selectors are specified the first one will run the communication network and the second the replication network
|
||||
|
||||
In order to work, each selector value must match a `NetworkAttachmentDefinition` object name in Multus.
|
||||
For example, you can do:
|
||||
|
||||
* `public`: "my-public-storage-network"
|
||||
* `cluster`: "my-replication-storage-network"
|
||||
|
||||
For `multus` network provider, an already working cluster with Multus networking is required. Network attachment definition that later will be attached to the cluster needs to be created before the Cluster CRD.
|
||||
If Rook cannot find the provided Network attachment definition it will fail running the Ceph OSD pods.
|
||||
You can add the Multus network attachment selection annotation selecting the created network attachment definition on `selectors`.
|
||||
|
||||
### Node Settings
|
||||
|
||||
In addition to the cluster level settings specified above, each individual node can also specify configuration to override the cluster level settings and defaults.
|
||||
|
||||
@@ -31,6 +31,7 @@ ConfigMap can be used in mix with the already existing Env Vars defined in opera
|
||||
- Pools can now be configured to inline compress the data using the `compressionMode` parameter. Support added [here](https://github.com/rook/rook/pull/5124)
|
||||
- Ceph OSDs do not use the host PID, but the PID namespace of the pod (more security). The OSD does not see host running processes anymore.
|
||||
- placement of all the ceph daemons now supports [topologySpreadConstraints](Documentation/ceph-cluster-crd.md#placement-configuration-settings).
|
||||
- Rook is now capable of working with Multus to expose dedicated interfaces to pods, for more information please refer to the [network configuration doc](Documentation/ceph-cluster-crd.html#network-configuration-settings).
|
||||
|
||||
### EdgeFS
|
||||
|
||||
|
||||
@@ -27,6 +27,12 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- k8s.cni.cncf.io
|
||||
resources:
|
||||
- network-attachment-definitions
|
||||
verbs:
|
||||
- get
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
|
||||
@@ -39,8 +39,6 @@ spec:
|
||||
dashboard:
|
||||
enabled: true
|
||||
ssl: true
|
||||
network:
|
||||
hostNetwork: false
|
||||
crashCollector:
|
||||
disable: false
|
||||
storage:
|
||||
|
||||
@@ -68,8 +68,20 @@ spec:
|
||||
# deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions.
|
||||
rulesNamespace: rook-ceph
|
||||
network:
|
||||
# toggle to use hostNetwork
|
||||
hostNetwork: false
|
||||
# enable host networking
|
||||
#provider: host
|
||||
# EXPERIMENTAL: enable the Multus network provider
|
||||
#provider: multus
|
||||
#selectors:
|
||||
# The selector keys are required to be `public` and `cluster`.
|
||||
# Based on the configuration, the operator will do the following:
|
||||
# 1. if only the `public` selector key is specified both public_network and cluster_network Ceph settings will listen on that interface
|
||||
# 2. if both `public` and `cluster` selector keys are specified the first one will point to 'public_network' flag and the second one to 'cluster_network'
|
||||
#
|
||||
# In order to work, each selector value must match a NetworkAttachmentDefinition object in Multus
|
||||
#
|
||||
#public: public-conf --> NetworkAttachmentDefinition object name in Multus
|
||||
#cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus
|
||||
rbdMirroring:
|
||||
# The number of daemons that will perform the rbd mirroring.
|
||||
# rbd mirroring must be configured with "rbd mirror" from the rook toolbox.
|
||||
|
||||
@@ -702,6 +702,12 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- k8s.cni.cncf.io
|
||||
resources:
|
||||
- network-attachment-definitions
|
||||
verbs:
|
||||
- get
|
||||
---
|
||||
# The cluster role for managing the Rook CRDs
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
|
||||
@@ -235,3 +235,47 @@ metadata:
|
||||
# should be in the namespace of the operator
|
||||
namespace: rook-ceph
|
||||
data:
|
||||
---
|
||||
# The role for the operator to manage resources in its own namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
- services
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- daemonsets
|
||||
- statefulsets
|
||||
- deployments
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- k8s.cni.cncf.io
|
||||
resources:
|
||||
- network-attachment-definitions
|
||||
verbs:
|
||||
- get
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -154,6 +155,9 @@ func NewContext() *clusterd.Context {
|
||||
context.RookClientset, err = rookclient.NewForConfig(context.KubeConfig)
|
||||
TerminateOnError(err, "failed to create rook clientset")
|
||||
|
||||
context.NetworkClient, err = netclient.NewForConfig(context.KubeConfig)
|
||||
TerminateOnError(err, "failed to create network clientset")
|
||||
|
||||
return context
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ require (
|
||||
github.com/google/go-cmp v0.3.1
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428
|
||||
github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8
|
||||
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20200107223247-51020689f1fb
|
||||
github.com/openshift/cluster-api v0.0.0-20191129101638-b09907ac6668
|
||||
github.com/openshift/machine-api-operator v0.2.1-0.20190903202259-474e14e4965a
|
||||
@@ -55,6 +56,7 @@ replace (
|
||||
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2
|
||||
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2
|
||||
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2
|
||||
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a // forces sigs.k8s.io/structured-merge-diff to v1 and not v3
|
||||
k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.2
|
||||
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.2
|
||||
k8s.io/kubectl => k8s.io/kubectl v0.17.2
|
||||
|
||||
@@ -96,6 +96,7 @@ github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93C
|
||||
github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
|
||||
github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
|
||||
github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
|
||||
github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE=
|
||||
github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
|
||||
github.com/coredns/corefile-migration v1.0.4/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
@@ -144,6 +145,8 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg
|
||||
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
|
||||
github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
|
||||
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
|
||||
github.com/emicklei/go-restful v2.10.0+incompatible h1:l6Soi8WCOOVAeCo4W98iBFC6Og7/X8bpRt51oNLZ2C8=
|
||||
github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
|
||||
github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
|
||||
github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
|
||||
github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
|
||||
@@ -262,6 +265,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
|
||||
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34=
|
||||
github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
@@ -271,6 +275,7 @@ github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4er
|
||||
github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg=
|
||||
github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
|
||||
github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
@@ -395,6 +400,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8 h1:/+OIW+inkJRBJlIHQqMUXRbYHmQLZwj7lqIV/TUecsE=
|
||||
github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200401090632-ee27f62faef8/go.mod h1:bDRYxMNuVGINaUROT+51ToYoimT8cqwYoRHSVGdQyRg=
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
@@ -691,6 +698,7 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8U
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE=
|
||||
golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
@@ -783,6 +791,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -827,6 +836,8 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw
|
||||
golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws=
|
||||
golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0 h1:7+F62GGWUowoiJOUDivedlBECd/fTeUDJnCu0JetQO0=
|
||||
golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458 h1:DgonIcqC7u+gVZX7lpuReBil5B/i8fvW/hAQdhT6/ao=
|
||||
golang.org/x/tools v0.0.0-20200319210407-521f4a0cd458/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
@@ -937,11 +948,6 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
|
||||
k8s.io/kube-aggregator v0.17.2/go.mod h1:8xQTzaH0GrcKPiSB4YYWwWbeQ0j/4zRsbQt8usEMbRg=
|
||||
k8s.io/kube-controller-manager v0.17.2 h1:YqE2AM6YzSppRHjmxHSZn6FiXNy6hTpRPEa2SYpHV5s=
|
||||
k8s.io/kube-controller-manager v0.17.2/go.mod h1:xznSbCHdVODF5StxiBMh3s6HenyCBdsedazlsh6/J3M=
|
||||
k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
|
||||
k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6 h1:s9IxTKe9GwDH0S/WaX62nFYr0or32DsTWex9AileL7U=
|
||||
k8s.io/kube-openapi v0.0.0-20190722073852-5e22f3d471e6/go.mod h1:RZvgC8MSN6DjiMV6oIfEE9pDL9CYXokkfaCKZeHm3nc=
|
||||
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ=
|
||||
k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
|
||||
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
|
||||
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
|
||||
k8s.io/kube-proxy v0.17.2/go.mod h1:PVY+Cqds8qa/TLEqiSgDPgwWBiRHYjeS4kvp/C5dYjc=
|
||||
|
||||
@@ -17,6 +17,7 @@ package clusterd
|
||||
|
||||
import (
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
netclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
"github.com/rook/rook/pkg/util/sys"
|
||||
@@ -55,6 +56,9 @@ type Context struct {
|
||||
// Information about the network for this machine and its cluster
|
||||
NetworkInfo NetworkInfo
|
||||
|
||||
// NetworkClient is a connection to the CNI plugin API
|
||||
NetworkClient netclient.K8sCniCncfIoV1Interface
|
||||
|
||||
// The local devices detected on the node
|
||||
Devices []*sys.LocalDisk
|
||||
}
|
||||
@@ -246,7 +246,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers
|
||||
|
||||
if c.Spec.External.Enable {
|
||||
// Apply CRD ConfigOverrideName to the external cluster
|
||||
err = config.SetDefaultConfigs(c.context, c.Namespace, c.Info)
|
||||
err = config.SetDefaultConfigs(c.context, c.Namespace, c.Info, cephv1.NetworkSpec{})
|
||||
if err != nil {
|
||||
// Mons are up, so something else is wrong
|
||||
return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options on the external cluster monitors")
|
||||
|
||||
@@ -375,7 +375,9 @@ func (c *ClusterController) configureExternalCephCluster(namespace, name string,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error {
|
||||
// Validate the cluster Specs
|
||||
func (c *ClusterController) preClusterStartValidation(cluster *cluster, clusterObj *cephv1.CephCluster) error {
|
||||
|
||||
if cluster.Spec.Mon.Count == 0 {
|
||||
logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount)
|
||||
cluster.Spec.Mon.Count = mon.DefaultMonCount
|
||||
@@ -386,11 +388,46 @@ func (c *ClusterController) configureLocalCephCluster(namespace, name string, cl
|
||||
if len(cluster.Spec.Storage.Directories) != 0 {
|
||||
logger.Warning("running osds on directory is not supported anymore, use devices instead.")
|
||||
}
|
||||
if cluster.Spec.Network.IsMultus() {
|
||||
_, isPublic := cluster.Spec.Network.Selectors[config.PublicNetworkSelectorKeyName]
|
||||
_, isCluster := cluster.Spec.Network.Selectors[config.ClusterNetworkSelectorKeyName]
|
||||
if !isPublic && !isCluster {
|
||||
return errors.New("both network selector values for public and cluster selector cannot be empty for multus provider")
|
||||
}
|
||||
|
||||
for _, selector := range config.NetworkSelectors {
|
||||
// If one selector is empty, we continue
|
||||
// This means a single interface is used both public and cluster network
|
||||
if _, ok := cluster.Spec.Network.Selectors[selector]; !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Get network attachment definition
|
||||
_, err := c.context.NetworkClient.NetworkAttachmentDefinitions(cluster.Namespace).Get(cluster.Spec.Network.Selectors[selector], metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
return errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selector)
|
||||
}
|
||||
return errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selector)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debug("cluster spec successfully validated")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) configureLocalCephCluster(namespace, name string, cluster *cluster, clusterObj *cephv1.CephCluster) error {
|
||||
// Cluster Spec validation
|
||||
err := c.preClusterStartValidation(cluster, clusterObj)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to perform validation before cluster creation")
|
||||
}
|
||||
|
||||
// Start the Rook cluster components. Retry several times in case of failure.
|
||||
failedMessage := ""
|
||||
|
||||
err := wait.Poll(clusterCreateInterval, clusterCreateTimeout,
|
||||
err = wait.Poll(clusterCreateInterval, clusterCreateTimeout,
|
||||
func() (bool, error) {
|
||||
cephVersion, canRetry, err := c.detectAndValidateCephVersion(cluster, cluster.Spec.CephVersion.Image)
|
||||
if err != nil {
|
||||
|
||||
@@ -76,7 +76,10 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) *apps.Deployment {
|
||||
keyring.Volume().Admin())
|
||||
if c.Network.IsHost() {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if c.Network.NetworkSpec.IsMultus() {
|
||||
k8sutil.ApplyMultus(c.Network.NetworkSpec, &podSpec.ObjectMeta)
|
||||
}
|
||||
|
||||
c.annotations.ApplyToObjectMeta(&podSpec.ObjectMeta)
|
||||
c.applyPrometheusAnnotations(&podSpec.ObjectMeta)
|
||||
c.placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
@@ -231,7 +231,7 @@ func (c *Cluster) startMons(targetCount int) error {
|
||||
// only once and do it as early as possible in the mon orchestration.
|
||||
setConfigsNeedsRetry := false
|
||||
if existingCount > 0 {
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo)
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network)
|
||||
if err != nil {
|
||||
// If we fail here, it could be because the mons are not healthy, and this might be
|
||||
// fixed by updating the mon deployments. Instead of returning error here, log a
|
||||
@@ -252,7 +252,7 @@ func (c *Cluster) startMons(targetCount int) error {
|
||||
// values in the config database. Do this only when the existing count is zero so that
|
||||
// this is only done once when the cluster is created.
|
||||
if existingCount == 0 {
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo)
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after creating the first mon")
|
||||
}
|
||||
@@ -260,7 +260,7 @@ func (c *Cluster) startMons(targetCount int) error {
|
||||
// Or if we need to retry, only do this when we are on the first iteration of the
|
||||
// loop. This could be in the same if statement as above, but separate it to get a
|
||||
// different error message.
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo)
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after updating the existing mons")
|
||||
}
|
||||
@@ -274,7 +274,7 @@ func (c *Cluster) startMons(targetCount int) error {
|
||||
}
|
||||
|
||||
if setConfigsNeedsRetry {
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo)
|
||||
err := config.SetDefaultConfigs(c.context, c.Namespace, c.ClusterInfo, c.spec.Network)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to set Rook and/or user-defined Ceph config options after forcefully updating the existing mons")
|
||||
}
|
||||
|
||||
@@ -154,9 +154,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string)
|
||||
HostNetwork: c.Network.IsHost(),
|
||||
PriorityClassName: cephv1.GetMonPriorityClassName(c.spec.PriorityClassNames),
|
||||
}
|
||||
if c.Network.IsHost() {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// Replace default unreachable node toleration
|
||||
if c.spec.Mon.VolumeClaimTemplate != nil {
|
||||
k8sutil.AddUnreachableNodeToleration(&podSpec)
|
||||
@@ -172,6 +170,12 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string)
|
||||
}
|
||||
cephv1.GetMonAnnotations(c.spec.Annotations).ApplyToObjectMeta(&pod.ObjectMeta)
|
||||
|
||||
if c.Network.IsHost() {
|
||||
pod.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if c.Network.NetworkSpec.IsMultus() {
|
||||
k8sutil.ApplyMultus(c.Network.NetworkSpec, &pod.ObjectMeta)
|
||||
}
|
||||
|
||||
return pod
|
||||
}
|
||||
|
||||
|
||||
@@ -361,11 +361,6 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
|
||||
// needed for luksOpen synchronization when devices are encrypted and the osd is prepared with LVM
|
||||
hostIPC := osdProps.storeConfig.EncryptedDevice
|
||||
|
||||
DNSPolicy := v1.DNSClusterFirst
|
||||
if c.Network.IsHost() {
|
||||
DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
initContainers := make([]v1.Container, 0, 4)
|
||||
if doConfigInit {
|
||||
initContainers = append(initContainers,
|
||||
@@ -423,6 +418,42 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
|
||||
securityContext,
|
||||
))
|
||||
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: AppName,
|
||||
Labels: c.getOSDLabels(osd.ID, failureDomainValue, osdProps.portable),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
ServiceAccountName: serviceAccountName,
|
||||
HostNetwork: c.Network.IsHost(),
|
||||
HostPID: hostPID,
|
||||
HostIPC: hostIPC,
|
||||
PriorityClassName: c.priorityClassName,
|
||||
InitContainers: initContainers,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Command: command,
|
||||
Args: args,
|
||||
Name: "osd",
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
Resources: osdProps.resources,
|
||||
SecurityContext: securityContext,
|
||||
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID),
|
||||
},
|
||||
},
|
||||
Volumes: volumes,
|
||||
},
|
||||
}
|
||||
|
||||
if c.Network.IsHost() {
|
||||
podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if c.Network.NetworkSpec.IsMultus() {
|
||||
k8sutil.ApplyMultus(c.Network.NetworkSpec, &podTemplateSpec.ObjectMeta)
|
||||
}
|
||||
|
||||
deployment := &apps.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: deploymentName,
|
||||
@@ -440,36 +471,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
|
||||
Strategy: apps.DeploymentStrategy{
|
||||
Type: apps.RecreateDeploymentStrategyType,
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: AppName,
|
||||
Labels: c.getOSDLabels(osd.ID, failureDomainValue, osdProps.portable),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
ServiceAccountName: serviceAccountName,
|
||||
HostNetwork: c.Network.IsHost(),
|
||||
HostPID: hostPID,
|
||||
HostIPC: hostIPC,
|
||||
DNSPolicy: DNSPolicy,
|
||||
PriorityClassName: c.priorityClassName,
|
||||
InitContainers: initContainers,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Command: command,
|
||||
Args: args,
|
||||
Name: "osd",
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
Resources: osdProps.resources,
|
||||
SecurityContext: securityContext,
|
||||
LivenessProbe: controller.GenerateLivenessProbeExecDaemon(opconfig.OsdType, osdID),
|
||||
},
|
||||
},
|
||||
Volumes: volumes,
|
||||
},
|
||||
},
|
||||
Template: podTemplateSpec,
|
||||
Replicas: &replicaCount,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -50,6 +50,8 @@ func (m *Mirroring) makeDeployment(daemonConfig *daemonConfig) *apps.Deployment
|
||||
|
||||
if m.Network.IsHost() {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if m.Network.NetworkSpec.IsMultus() {
|
||||
k8sutil.ApplyMultus(m.Network.NetworkSpec, &podSpec.ObjectMeta)
|
||||
}
|
||||
m.placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
)
|
||||
@@ -103,6 +104,7 @@ func SetDefaultConfigs(
|
||||
context *clusterd.Context,
|
||||
namespace string,
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
networkSpec cephv1.NetworkSpec,
|
||||
) error {
|
||||
// ceph.conf is never used. All configurations are made in the centralized mon config database,
|
||||
// or they are specified on the commandline when daemons are called.
|
||||
@@ -116,5 +118,19 @@ func SetDefaultConfigs(
|
||||
return errors.Wrapf(err, "failed to apply legacy config overrides")
|
||||
}
|
||||
|
||||
// Apply Multus if needed
|
||||
if networkSpec.IsMultus() {
|
||||
logger.Info("configuring ceph network(s) with multus")
|
||||
cephNetworks, err := generateNetworkSettings(context, namespace, networkSpec.Selectors)
|
||||
if err != nil {
|
||||
errors.Wrap(err, "failed to generate network settings")
|
||||
}
|
||||
|
||||
// Apply ceph network settings to the mon config store
|
||||
if err := monStore.SetAll(cephNetworks...); err != nil {
|
||||
return errors.Wrap(err, "failed to network config overrides")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
Copyright 2020 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package config provides default configurations which Rook will set in Ceph clusters.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
// PublicNetworkSelectorKeyName is the network selector key for the ceph public network
|
||||
PublicNetworkSelectorKeyName = "public"
|
||||
// ClusterNetworkSelectorKeyName is the network selector key for the ceph cluster network
|
||||
ClusterNetworkSelectorKeyName = "cluster"
|
||||
)
|
||||
|
||||
var (
|
||||
// NetworkSelectors is a slice of ceph network selector key name
|
||||
NetworkSelectors = []string{PublicNetworkSelectorKeyName, ClusterNetworkSelectorKeyName}
|
||||
)
|
||||
|
||||
func generateNetworkSettings(context *clusterd.Context, namespace string, networkSelectors map[string]string) ([]Option, error) {
|
||||
cephNetworks := []Option{}
|
||||
|
||||
for _, selectorKey := range NetworkSelectors {
|
||||
// This means only "public" was specified and thus we use the same subnet for cluster too
|
||||
if _, ok := networkSelectors[selectorKey]; !ok {
|
||||
cephNetworks = append(cephNetworks, cephNetworks[0])
|
||||
cephNetworks[1].Option = fmt.Sprintf("%s_network", selectorKey)
|
||||
continue
|
||||
}
|
||||
|
||||
// Get network attachment definition
|
||||
netDefinition, err := context.NetworkClient.NetworkAttachmentDefinitions(namespace).Get(networkSelectors[selectorKey], metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
return []Option{}, errors.Wrapf(err, "specified network attachment definition for selector %q does not exist", selectorKey)
|
||||
}
|
||||
return []Option{}, errors.Wrapf(err, "failed to fetch network attachment definition for selector %q", selectorKey)
|
||||
}
|
||||
|
||||
// Get network attachment definition configuration
|
||||
netConfig, err := k8sutil.GetNetworkAttachmentConfig(*netDefinition)
|
||||
if err != nil {
|
||||
return []Option{}, errors.Wrapf(err, "failed to get network attachment definition configuration for selector %q", selectorKey)
|
||||
}
|
||||
|
||||
if netConfig.Ipam.Subnet != "" {
|
||||
cephNetworks = append(cephNetworks, configOverride("global", fmt.Sprintf("%s_network", selectorKey), netConfig.Ipam.Subnet))
|
||||
} else {
|
||||
return []Option{}, errors.Errorf("empty subnet from network attachment definition %q", networkSelectors[selectorKey])
|
||||
}
|
||||
}
|
||||
|
||||
return cephNetworks, nil
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
Copyright 2020 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
|
||||
fakenetclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestGenerateNetworkSettings(t *testing.T) {
|
||||
ns := "rook-ceph"
|
||||
clientset := testop.New(t, 1)
|
||||
ctx := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
NetworkClient: fakenetclient.NewSimpleClientset().K8sCniCncfIoV1(),
|
||||
}
|
||||
|
||||
//
|
||||
// TEST 1: network definition does not exist
|
||||
//
|
||||
netSelector := map[string]string{
|
||||
"public": "public-network-attach-def",
|
||||
}
|
||||
|
||||
cephNetwork, err := generateNetworkSettings(ctx, ns, netSelector)
|
||||
assert.Error(t, err)
|
||||
|
||||
//
|
||||
// TEST 2: single dedicated networks
|
||||
//
|
||||
expectedNetworks := []Option{
|
||||
{
|
||||
Who: "global",
|
||||
Option: "public_network",
|
||||
Value: "192.168.0.0/24",
|
||||
},
|
||||
{
|
||||
Who: "global",
|
||||
Option: "cluster_network",
|
||||
Value: "192.168.0.0/24",
|
||||
},
|
||||
}
|
||||
|
||||
network := &networkv1.NetworkAttachmentDefinition{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "public-network-attach-def",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: networkv1.NetworkAttachmentDefinitionSpec{
|
||||
Config: `{
|
||||
"cniVersion": "0.3.0",
|
||||
"type": "macvlan",
|
||||
"master": "eth2",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "192.168.0.0/24",
|
||||
"gateway": "172.18.8.1"
|
||||
}
|
||||
}`,
|
||||
},
|
||||
}
|
||||
|
||||
// Create public network definition
|
||||
ctx.NetworkClient.NetworkAttachmentDefinitions(ns).Create(network)
|
||||
|
||||
cephNetwork, err = generateNetworkSettings(ctx, ns, netSelector)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork))
|
||||
|
||||
//
|
||||
// TEST 3: two dedicated networks
|
||||
//
|
||||
expectedNetworks = []Option{
|
||||
{
|
||||
Who: "global",
|
||||
Option: "public_network",
|
||||
Value: "192.168.0.0/24",
|
||||
},
|
||||
{
|
||||
Who: "global",
|
||||
Option: "cluster_network",
|
||||
Value: "172.18.0.0/16",
|
||||
},
|
||||
}
|
||||
|
||||
netSelector = map[string]string{
|
||||
"public": "public-network-attach-def",
|
||||
"cluster": "cluster-network-attach-def",
|
||||
}
|
||||
network2 := &networkv1.NetworkAttachmentDefinition{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "cluster-network-attach-def",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: networkv1.NetworkAttachmentDefinitionSpec{
|
||||
Config: `{
|
||||
"cniVersion": "0.3.0",
|
||||
"type": "macvlan",
|
||||
"master": "eth2",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "172.18.0.0/16",
|
||||
"gateway": "172.18.0.1"
|
||||
}
|
||||
}`,
|
||||
},
|
||||
}
|
||||
|
||||
// Create cluster network definition
|
||||
ctx.NetworkClient.NetworkAttachmentDefinitions(ns).Create(network2)
|
||||
|
||||
cephNetwork, err = generateNetworkSettings(ctx, ns, netSelector)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, cephNetwork, expectedNetworks, fmt.Sprintf("networks: %+v", cephNetwork))
|
||||
}
|
||||
@@ -60,9 +60,6 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
|
||||
// Replace default unreachable node toleration
|
||||
k8sutil.AddUnreachableNodeToleration(&podSpec.Spec)
|
||||
|
||||
if c.clusterSpec.Network.IsHost() {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&podSpec.ObjectMeta)
|
||||
c.fs.Spec.MetadataServer.Placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
@@ -84,6 +81,13 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if c.clusterSpec.Network.IsHost() {
|
||||
d.Spec.Template.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if c.clusterSpec.Network.NetworkSpec.IsMultus() {
|
||||
k8sutil.ApplyMultus(c.clusterSpec.Network.NetworkSpec, &podSpec.ObjectMeta)
|
||||
}
|
||||
|
||||
k8sutil.AddRookVersionLabelToDeployment(d)
|
||||
c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&d.ObjectMeta)
|
||||
controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d)
|
||||
|
||||
@@ -143,6 +143,13 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
|
||||
if r.cephClusterSpec.Network.IsHost() {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if r.cephClusterSpec.Network.NetworkSpec.IsMultus() {
|
||||
k8sutil.ApplyMultus(r.cephClusterSpec.Network.NetworkSpec, &podTemplateSpec.ObjectMeta)
|
||||
}
|
||||
|
||||
nfs.Spec.Server.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
||||
|
||||
// Multiple replicas of the nfs service would be handled by creating a service and a new deployment for each one, rather than increasing the pod count here
|
||||
|
||||
@@ -51,6 +51,7 @@ type clusterConfig struct {
|
||||
skipUpgradeChecks bool
|
||||
client client.Client
|
||||
scheme *runtime.Scheme
|
||||
Network cephv1.NetworkSpec
|
||||
}
|
||||
|
||||
type rgwConfig struct {
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestStartRGW(t *testing.T) {
|
||||
r := &ReconcileCephObjectStore{client: cl, scheme: s}
|
||||
|
||||
// start a basic cluster
|
||||
c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s}
|
||||
c := &clusterConfig{info, context, store, version, &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s, cephv1.NetworkSpec{}}
|
||||
err := c.startRGWPods()
|
||||
assert.Nil(t, err)
|
||||
|
||||
@@ -107,7 +107,7 @@ func TestCreateObjectStore(t *testing.T) {
|
||||
object := []runtime.Object{&cephv1.CephObjectStore{}}
|
||||
cl := fake.NewFakeClientWithScheme(s, object...)
|
||||
r := &ReconcileCephObjectStore{client: cl, scheme: s}
|
||||
c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s}
|
||||
c := &clusterConfig{info, context, store, "1.2.3.4", &cephv1.ClusterSpec{}, &metav1.OwnerReference{}, data, false, r.client, s, cephv1.NetworkSpec{}}
|
||||
err := c.createOrUpdateStore()
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
@@ -136,7 +136,8 @@ func TestGenerateSecretName(t *testing.T) {
|
||||
&cephconfig.DataPathMap{},
|
||||
false,
|
||||
cl,
|
||||
scheme.Scheme}
|
||||
scheme.Scheme,
|
||||
cephv1.NetworkSpec{}}
|
||||
secret := c.generateSecretName("a")
|
||||
assert.Equal(t, "rook-ceph-rgw-default-a-keyring", secret)
|
||||
}
|
||||
|
||||
@@ -83,10 +83,6 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec
|
||||
// Replace default unreachable node toleration
|
||||
k8sutil.AddUnreachableNodeToleration(&podSpec)
|
||||
|
||||
if c.clusterSpec.Network.IsHost() {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// Set the ssl cert if specified
|
||||
if c.store.Spec.Gateway.SSLCertificateRef != "" {
|
||||
// Keep the SSL secret as secure as possible in the container. Give only user read perms.
|
||||
@@ -114,6 +110,12 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec
|
||||
}
|
||||
c.store.Spec.Gateway.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
||||
|
||||
if c.clusterSpec.Network.IsHost() {
|
||||
podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if c.clusterSpec.Network.IsMultus() {
|
||||
k8sutil.ApplyMultus(c.Network.NetworkSpec, &podTemplateSpec.ObjectMeta)
|
||||
}
|
||||
|
||||
return podTemplateSpec
|
||||
}
|
||||
|
||||
|
||||
@@ -21,12 +21,31 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
netapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// NetworkAttachmentConfig represents the configuration of the NetworkAttachmentDefinitions object
|
||||
type NetworkAttachmentConfig struct {
|
||||
CniVersion string `json:"cniVersion"`
|
||||
Type string `json:"type"`
|
||||
Master string `json:"master"`
|
||||
Mode string `json:"mode"`
|
||||
Ipam struct {
|
||||
Type string `json:"type"`
|
||||
Subnet string `json:"subnet"`
|
||||
RangeStart string `json:"rangeStart"`
|
||||
RangeEnd string `json:"rangeEnd"`
|
||||
Routes []struct {
|
||||
Dst string `json:"dst"`
|
||||
} `json:"routes"`
|
||||
Gateway string `json:"gateway"`
|
||||
} `json:"ipam"`
|
||||
}
|
||||
|
||||
// parseMultusSelector will parse short and JSON form of individual multus
|
||||
// network attachment selection annotation. Valid JSON will be unmarshaled and
|
||||
// network attachment selection annotation. Valid JSON will be unmarshalled and
|
||||
// return as is, while invalid JSON will be tried using
|
||||
// <namespace>/<name>@<interface> short syntax.
|
||||
func parseMultusSelector(selector string) (map[string]string, error) {
|
||||
@@ -75,7 +94,7 @@ func GetMultusIfName(selector string) (string, error) {
|
||||
ifName = name
|
||||
}
|
||||
|
||||
// fail selector without inteface name
|
||||
// fail selector without interface name
|
||||
if ifName == "" {
|
||||
return "", fmt.Errorf("GetMultusIfname: missing interface")
|
||||
}
|
||||
@@ -119,3 +138,16 @@ func ApplyMultus(net rookv1.NetworkSpec, objectMeta *metav1.ObjectMeta) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetNetworkAttachmentConfig returns the NetworkAttachmentDefinitions configuration
|
||||
func GetNetworkAttachmentConfig(n netapi.NetworkAttachmentDefinition) (NetworkAttachmentConfig, error) {
|
||||
netConfigJSON := n.Spec.Config
|
||||
var netConfig NetworkAttachmentConfig
|
||||
|
||||
err := json.Unmarshal([]byte(netConfigJSON), &netConfig)
|
||||
if err != nil {
|
||||
return netConfig, fmt.Errorf("failed to unmarshal netconfig json %q. %v", netConfigJSON, err)
|
||||
}
|
||||
|
||||
return netConfig, nil
|
||||
}
|
||||
@@ -19,6 +19,7 @@ package k8sutil
|
||||
import (
|
||||
"testing"
|
||||
|
||||
netapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
|
||||
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -124,3 +125,32 @@ func TestNetwork_ApplyMultusMixedError(t *testing.T) {
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGetNetworkAttachmentConfig(t *testing.T) {
|
||||
dummyNetAttachDef := netapi.NetworkAttachmentDefinition{
|
||||
Spec: netapi.NetworkAttachmentDefinitionSpec{
|
||||
Config: `{
|
||||
"cniVersion": "0.3.0",
|
||||
"type": "macvlan",
|
||||
"master": "eth2",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "172.18.8.0/24",
|
||||
"rangeStart": "172.18.8.200",
|
||||
"rangeEnd": "172.18.8.216",
|
||||
"routes": [
|
||||
{
|
||||
"dst": "0.0.0.0/0"
|
||||
}
|
||||
],
|
||||
"gateway": "172.18.8.1"
|
||||
}
|
||||
}`,
|
||||
},
|
||||
}
|
||||
|
||||
config, err := GetNetworkAttachmentConfig(dummyNetAttachDef)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "172.18.8.0/24", config.Ipam.Subnet)
|
||||
}
|
||||
@@ -675,6 +675,12 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- k8s.cni.cncf.io
|
||||
resources:
|
||||
- network-attachment-definitions
|
||||
verbs:
|
||||
- get
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
|
||||
Reference in new issue
Block a user