diff --git a/deploy/examples/networkpolicy.yaml b/deploy/examples/networkpolicy.yaml new file mode 100644 index 000000000..6d31e8ac3 --- /dev/null +++ b/deploy/examples/networkpolicy.yaml @@ -0,0 +1,361 @@ +################################################################################################################# +# NetworkPolicy definitions for the Rook-Ceph operator and operand pods. +# These policies control egress (and where applicable, ingress) traffic for each Ceph daemon type. +# +# Egress-only rationale: CSI node plugins use hostNetwork, so their traffic arrives from node IPs +# which cannot be matched by namespaceSelector/podSelector ingress rules. Restricting ingress would +# require ipBlock CIDRs that are cluster-specific and fragile, so we only enforce egress on most +# Ceph daemon pods. +################################################################################################################# +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-mon + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-mon + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-osd + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 + - protocol: TCP + port: 6789 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mgr + ports: + - protocol: TCP + port: 6800 + endPort: 7300 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-osd + ports: + - protocol: TCP + port: 6800 + endPort: 7300 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-mgr + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 + - protocol: TCP + port: 6789 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-mds + - rook-ceph-osd + ports: + - protocol: TCP + port: 6800 + endPort: 7300 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-mds + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-mds + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 + - protocol: TCP + port: 6789 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-osd + - rook-ceph-mds + ports: + - protocol: TCP + port: 6800 + endPort: 7300 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-exporter + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-exporter + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring # namespace:monitoring + ports: + - protocol: TCP + port: 9926 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 +--- +# The OSD prepare job needs node topology labels via the K8s API. +# The API server is host-networked so it cannot be targeted by a pod/namespace +# selector; therefore egress is unrestricted. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-osd-prepare + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-osd-prepare + policyTypes: + - Ingress + - Egress + egress: + - {} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-crashcollector + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-crashcollector + policyTypes: + - Ingress + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rook-ceph-tools + namespace: rook-ceph # namespace:cluster +spec: + podSelector: + matchLabels: + app: rook-ceph-tools + policyTypes: + - Ingress + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system # namespace:dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mon + ports: + - protocol: TCP + port: 3300 + - protocol: TCP + port: 6789 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-mgr + ports: + - protocol: TCP + port: 6800 + endPort: 7300 + - protocol: TCP + port: 9283 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-osd + ports: + - protocol: TCP + port: 6800 + endPort: 7300 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: rook-ceph # namespace:cluster + podSelector: + matchLabels: + app: rook-ceph-rgw + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 443 + - protocol: TCP + port: 8080 + - protocol: TCP + port: 8443 diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 1e178175b..1d464b594 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -129,6 +129,13 @@ func (h *CephInstaller) CreateCephOperator() (err error) { return errors.Errorf("Failed to create rook-operator pod: %v ", err) } + if networkPolicy := h.Manifests.GetNetworkPolicy(); networkPolicy != "" { + logger.Info("Creating NetworkPolicies") + if _, err = h.k8shelper.KubectlWithStdin(networkPolicy, createFromStdinArgs...); err != nil { + return errors.Wrap(err, "failed to create network policies") + } + } + if err := h.CreateVolumeReplicationCRDs(); err != nil { return errors.Wrap(err, "failed to create volume replication CRDs") } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index f6a38b7af..8ee7c8fa5 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -33,6 +33,7 @@ type CephManifests interface { GetOperator() string GetCommon() string GetCommonExternal() string + GetNetworkPolicy() string GetCephCluster() string GetExternalCephCluster() string GetToolbox() string @@ -109,6 +110,10 @@ func (m *CephManifestsMaster) GetCommon() string { return m.settings.readManifest("common.yaml") } +func (m *CephManifestsMaster) GetNetworkPolicy() string { + return m.settings.readManifest("networkpolicy.yaml") +} + func (m *CephManifestsMaster) GetToolbox() string { if m.settings.DirectMountToolbox { manifest := strings.ReplaceAll(m.settings.readManifest("direct-mount.yaml"), "name: rook-direct-mount", "name: rook-ceph-tools") diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go index cecb0146c..74c83503e 100644 --- a/tests/framework/installer/ceph_manifests_previous.go +++ b/tests/framework/installer/ceph_manifests_previous.go @@ -75,6 +75,10 @@ func (m *CephManifestsPreviousVersion) GetCommonExternal() string { return m.settings.readManifestFromGitHub("common-external.yaml") } +func (m *CephManifestsPreviousVersion) GetNetworkPolicy() string { + return "" +} + //********************************************************************************** //********************************************************************************** // Methods in this section may need to be customized depending on new diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 315048d55..4c2b44a29 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -358,6 +358,7 @@ function replace_ceph_image() { function deploy_cluster() { cd "${REPO_DIR}/deploy/examples" + kubectl create -f networkpolicy.yaml deploy_manifest_with_local_build operator.yaml if [ $# == 0 ]; then