From b872557ebbf6feeb372e00afbf95f5858d04aba4 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 28 May 2019 17:20:08 -0600 Subject: [PATCH 1/2] ceph example: add psp to common.yaml Add Rook-Ceph PSP requirements to the `common.yaml` file. This is commented for users to inspect and make informed decisions about their own clusters just as with the rest of the file. For user clusters without pod security policies enabled, this part of the config will be a don't-care and not harm anything. Because the PSP setup is now codified in `common.yaml`, it is no longer necessary to have documentation for pod security policies, and the `Documentation/ceph-psp.md` file is deleted. Signed-off-by: Blaine Gardner --- Documentation/ceph-psp.md | 145 ----------------- Documentation/psp.md | 12 +- cluster/examples/kubernetes/ceph/common.yaml | 156 +++++++++++++++++++ cluster/examples/kubernetes/ceph/psp.yaml | 36 ----- 4 files changed, 166 insertions(+), 183 deletions(-) delete mode 100644 Documentation/ceph-psp.md delete mode 100644 cluster/examples/kubernetes/ceph/psp.yaml diff --git a/Documentation/ceph-psp.md b/Documentation/ceph-psp.md deleted file mode 100644 index d3226ceac..000000000 --- a/Documentation/ceph-psp.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -title: Ceph cluster Pod Security Policies -weight: 1400 -indent: true ---- - -# Using Rook-Ceph with Pod Security Policies (PSPs) - -See the [Rook overall PSP document](./psp.md) before continuing on here with Ceph specifics. - -##### PodSecurityPolicy - -You need at least one `PodSecurityPolicy` that allows privileged `Pod` execution. Please look at [https://github.com/rook/rook/cluster/examples/kubernetes/ceph/psp.yaml](psp.yaml) for a reasonably pared down for Ceph. - -**Hint**: Allowing `hostNetwork` usage is required when using `hostNetwork: true` in the Cluster -Resource Definition! You are then also required to allow the usage of `hostPorts` in the -`PodSecurityPolicy`. The given port range is a minimal working recommendation for a Rook Ceph cluster: - ```yaml - hostPorts: - # Ceph msgr2 port - - min: 3300 - max: 3300 - # Ceph ports - - min: 6789 - max: 7300 - # Ceph MGR Prometheus Metrics - - min: 9283 - max: 9283 -``` - -##### ClusterRole and ClusterRoleBinding - -Next up you require a `ClusterRole` and a corresponding `ClusterRoleBinding`, which enables the Rook Agent `ServiceAccount` to run the rook-ceph-agent `Pods` on all nodes -with privileged rights. Here are the definitions: - -```yaml -# privilegedPSP grants access to use the privileged PSP. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: psp:rook -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - privileged - verbs: - - use - -``` -and -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: rook-ceph-system ---- -# Allow the rook-ceph-system serviceAccount to use the privileged PSP -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph -``` - -Save these definitions to one or multiple yaml files and create them by executing `kubectl apply -f .yaml` - -You will also require two more `RoleBindings` for each Rook Cluster you deploy: -Create these two `RoleBindings` in the Namespace you plan to deploy your Rook Cluster into (default is "rook" namespace): - -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: rook-ceph ---- -# Allow the default serviceAccount to use the privileged PSP -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-default-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: default - namespace: rook-ceph ---- -# Allow the rook-ceph-osd serviceAccount to use the privileged PSP -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph ---- -# Allow the rook-ceph-mgr serviceAccount to use the privileged PSP -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph ---- -# Allow the rook-ceph-cmd-reporter serviceAccount to use the privileged PSP -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph -``` diff --git a/Documentation/psp.md b/Documentation/psp.md index 8e241d147..55558e2dc 100644 --- a/Documentation/psp.md +++ b/Documentation/psp.md @@ -3,6 +3,13 @@ title: Pod Security Policies weight: 1400 indent: true --- +{% assign url = page.url | split: '/' %} +{% assign currentVersion = url[3] %} +{% if currentVersion != 'master' %} +{% assign branchName = currentVersion | replace: 'v', '' | prepend: 'release-' %} +{% else %} +{% assign branchName = currentVersion %} +{% endif %} # Using Rook with Pod Security Policies @@ -27,8 +34,9 @@ If you have activated the [PodSecurityPolicy Admission Controller](https://kuber using [PodSecurityPolicies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/), you will require additional `(Cluster)RoleBindings` for the different `ServiceAccounts` Rook uses to start the Rook Storage Pods. -Security policies will differ for different backends. See -[Ceph's Pod Security Policy document](./ceph-psp.md) for an example of how this is done in practice. +Security policies will differ for different backends. See Ceph's Pod Security Policies set up in its +[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/common.yaml) +for an example of how this is done in practice. **Note**: You do not have to perform these steps if you do not have the `PodSecurityPolicy` Admission Controller activated! diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index d04e41ba7..d18f40090 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -691,3 +691,159 @@ subjects: namespace: rook-ceph # OLM: END CMD REPORTER ROLEBINDING --- +################################################################################################################# +# Beginning of pod security policy resources. The example will assume the cluster will be created in the +# "rook-ceph" namespace. If you want to create the cluster in a different namespace, you will need to modify +# the roles and bindings accordingly. +################################################################################################################# +# OLM: BEGIN CLUSTER POD SECURITY POLICY +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: rook-privileged +spec: + privileged: true + allowedCapabilities: + # required by CSI + - SYS_ADMIN + # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group + fsGroup: + rule: RunAsAny + # runAsUser, supplementalGroups - Rook needs to run some pods as root + # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + # seLinux - seLinux context is unknown ahead of time; set if this is well-known + seLinux: + rule: RunAsAny + volumes: + # recommended minimum set + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret + - projected + # required for Rook + - hostPath + - flexVolume + # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known + # directory-based OSDs make this hard to nail down + # allowedHostPaths: + # - /run/udev # for OSD prep + # - /dev # for OSD prep + # - /var/lib/rook # or whatever the dataDirHostPath value is set to + # Ceph requires host IPC for setting up encrypted devices + hostIPC: true + # Ceph OSDs need to share the same PID namespace + hostPID: true + # hostNetwork can be set to 'false' if host networking isn't used + hostNetwork: true + hostPorts: + # Ceph messenger protocol v1 + - min: 6789 + max: 6790 # <- support old default port + # Ceph messenger protocol v2 + - min: 3300 + max: 3300 + # Ceph RADOS ports for OSDs, MDSes + - min: 6800 + max: 7300 + # # Ceph dashboard port HTTP (not recommended) + # - min: 7000 + # max: 7000 + # Ceph dashboard port HTTPS + - min: 8443 + max: 8443 + # Ceph mgr Prometheus Metrics + - min: 9283 + max: 9283 +# OLM: END CLUSTER POD SECURITY POLICY +--- +# OLM: BEGIN POD SECURITY POLICY BINDINGS +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: 'psp:rook' +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - rook-privileged + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-system-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-default-psp + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: default + namespace: rook-ceph +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-osd-psp + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr-psp + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter-psp + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph +# OLM: END CLUSTER POD SECURITY POLICY BINDINGS +--- diff --git a/cluster/examples/kubernetes/ceph/psp.yaml b/cluster/examples/kubernetes/ceph/psp.yaml deleted file mode 100644 index 2626f2069..000000000 --- a/cluster/examples/kubernetes/ceph/psp.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: PodSecurityPolicy -metadata: - name: rook -spec: - fsGroup: - rule: RunAsAny - privileged: true - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - - 'hostPath' - - 'flexVolume' - hostPID: true - hostNetwork: true - hostIPC: true - # The following section is only needed when hostNetwork: true - hostPorts: - # Ceph msgr2 port - - min: 3300 - max: 3300 - # Ceph msgr1 ports - - min: 6789 - max: 7300 - # Ceph MGR Prometheus Metrics - - min: 9283 - max: 9283 From f54dcbb5f5f5c4645727f12477c43ac8fc06038c Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Fri, 12 Jul 2019 16:35:38 -0600 Subject: [PATCH 2/2] ceph integration: add psp to integration tests Signed-off-by: Blaine Gardner --- .../csi/rbac/cephfs/csi-node-plugin-psp.yaml | 14 ++ .../csi/rbac/cephfs/csi-provisioner-psp.yaml | 14 ++ .../csi/rbac/rbd/csi-node-plugin-psp.yaml | 14 ++ .../csi/rbac/rbd/csi-provisioner-psp.yaml | 14 ++ tests/framework/installer/ceph_installer.go | 8 + tests/framework/installer/ceph_manifests.go | 202 ++++++++++++++++++ .../installer/ceph_manifests_v1.0.go | 189 ++++++++++++++++ tests/integration/ceph_upgrade_test.go | 15 ++ 8 files changed, 470 insertions(+) create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-node-plugin-psp.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-psp.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-node-plugin-psp.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-psp.yaml diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-node-plugin-psp.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-node-plugin-psp.yaml new file mode 100644 index 000000000..061cdfc2e --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-node-plugin-psp.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph +--- diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-psp.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-psp.yaml new file mode 100644 index 000000000..cfa4b9469 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-psp.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph +--- diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-node-plugin-psp.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-node-plugin-psp.yaml new file mode 100644 index 000000000..e9740f87f --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-node-plugin-psp.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph +--- diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-psp.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-psp.yaml new file mode 100644 index 000000000..a1dee566c --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-psp.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph +--- diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 9119a3310..1fdf2acd1 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -433,6 +433,14 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner-rules", nil) + h.k8shelper.Clientset.PolicyV1beta1().PodSecurityPolicies().Delete("rook-privileged", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("psp:rook", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-system-psp", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-csi-rbd-provisioner-sa-psp", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-csi-rbd-plugin-sa-psp", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-csi-cephfs-provisioner-sa-psp", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-csi-cephfs-plugin-sa-psp", nil) + h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-rbd-config", nil) h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-cephfs-config", nil) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index bde0e1a38..e94758ddc 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -875,7 +875,154 @@ roleRef: kind: Role name: cephfs-external-provisioner-cfg apiGroup: rbac.authorization.k8s.io + --- + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: rook-privileged +spec: + privileged: true + allowedCapabilities: + # required by CSI + - SYS_ADMIN + # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group + fsGroup: + rule: RunAsAny + # runAsUser, supplementalGroups - Rook needs to run some pods as root + # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + # seLinux - seLinux context is unknown ahead of time; set if this is well-known + seLinux: + rule: RunAsAny + volumes: + # recommended minimum set + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret + - projected + # required for Rook + - hostPath + - flexVolume + # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known + # directory-based OSDs make this hard to nail down + # allowedHostPaths: + # - /run/udev # for OSD prep + # - /dev # for OSD prep + # - /var/lib/rook # or whatever the dataDirHostPath value is set to + # Ceph requires host IPC for setting up encrypted devices + hostIPC: true + # Ceph OSDs need to share the same PID namespace + hostPID: true + # hostNetwork can be set to 'false' if host networking isn't used + hostNetwork: true + hostPorts: + # Ceph messenger protocol v1 + - min: 6789 + max: 6790 # <- support old default port + # Ceph messenger protocol v2 + - min: 3300 + max: 3300 + # Ceph RADOS ports for OSDs, MDSes + - min: 6800 + max: 7300 + # # Ceph dashboard port HTTP (not recommended) + # - min: 7000 + # max: 7000 + # Ceph dashboard port HTTPS + - min: 8443 + max: 8443 + # Ceph mgr Prometheus Metrics + - min: 9283 + max: 9283 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: 'psp:rook' +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - rook-privileged + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-system-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: ` + namespace + ` + +--- + apiVersion: apps/v1 kind: Deployment metadata: @@ -1107,6 +1254,61 @@ subjects: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-default-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: default + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-osd-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-osd + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-mgr + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` ` } diff --git a/tests/framework/installer/ceph_manifests_v1.0.go b/tests/framework/installer/ceph_manifests_v1.0.go index 4cf26b656..3bf13e5a6 100644 --- a/tests/framework/installer/ceph_manifests_v1.0.go +++ b/tests/framework/installer/ceph_manifests_v1.0.go @@ -780,7 +780,154 @@ roleRef: kind: ClusterRole name: cephfs-external-provisioner-runner apiGroup: rbac.authorization.k8s.io + --- + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: rook-privileged +spec: + privileged: true + allowedCapabilities: + # required by CSI + - SYS_ADMIN + # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group + fsGroup: + rule: RunAsAny + # runAsUser, supplementalGroups - Rook needs to run some pods as root + # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + # seLinux - seLinux context is unknown ahead of time; set if this is well-known + seLinux: + rule: RunAsAny + volumes: + # recommended minimum set + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret + - projected + # required for Rook + - hostPath + - flexVolume + # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known + # directory-based OSDs make this hard to nail down + # allowedHostPaths: + # - /run/udev # for OSD prep + # - /dev # for OSD prep + # - /var/lib/rook # or whatever the dataDirHostPath value is set to + # Ceph requires host IPC for setting up encrypted devices + hostIPC: true + # Ceph OSDs need to share the same PID namespace + hostPID: true + # hostNetwork can be set to 'false' if host networking isn't used + hostNetwork: true + hostPorts: + # Ceph messenger protocol v1 + - min: 6789 + max: 6790 # <- support old default port + # Ceph messenger protocol v2 + - min: 3300 + max: 3300 + # Ceph RADOS ports for OSDs, MDSes + - min: 6800 + max: 7300 + # # Ceph dashboard port HTTP (not recommended) + # - min: 7000 + # max: 7000 + # Ceph dashboard port HTTPS + - min: 8443 + max: 8443 + # Ceph mgr Prometheus Metrics + - min: 9283 + max: 9283 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: 'psp:rook' +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - rook-privileged + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-system-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: ` + namespace + ` + +--- + apiVersion: apps/v1 kind: Deployment metadata: @@ -971,6 +1118,48 @@ roleRef: kind: ClusterRole name: rook-ceph-mgr-system subjects: +- kind: ServiceAccount + name: rook-ceph-mgr + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-default-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: default + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-osd-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-osd + namespace: ` + namespace + ` +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: - kind: ServiceAccount name: rook-ceph-mgr namespace: ` + namespace + ` diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 1ed43a205..5300cd9b0 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -198,6 +198,21 @@ subjects: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter-psp + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` + +--- ` logger.Infof("creating the new resources that have been added since 1.0") return s.k8sh.ResourceOperation("create", newResources)