Merge pull request #10797 from BlaineEXE/psp-move

docs: move PSPs from common.yaml to psp.yaml

Documentation/Getting-Started/example-configurations.md

@@ -11,9 +11,11 @@ See the **[example yaml files](https://github.com/rook/rook/blob/master/deploy/e
 The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments.
 The [crds.yaml](https://github.com/rook/rook/blob/master/deploy/examples/crds.yaml) and
 [common.yaml](https://github.com/rook/rook/blob/master/deploy/examples/common.yaml) sets these resources up.
+You may also need [psp.yaml](https://github.com/rook/rook/blob/master/deploy/examples/psp.yaml) if
+your cluster is protected by Pod Security Policies.
 
 ```console
-kubectl create -f crds.yaml -f common.yaml
+kubectl create -f crds.yaml -f common.yaml # -f psp.yaml
 ```
 
 The examples all assume the operator and all Ceph daemons will be started in the same namespace. If you want to deploy the operator in a separate namespace, see the comments throughout `common.yaml`.

Documentation/Getting-Started/quickstart.md

@@ -50,6 +50,7 @@ The first step is to deploy the Rook operator. Check that you are using the [exa
 
 ```console
 cd deploy/examples
+# kubectl create -f psp.yaml # if your cluster is protected by pod security policies
 kubectl create -f crds.yaml -f common.yaml -f operator.yaml
 
 # verify the rook-ceph-operator is in the `Running` state before proceeding

Documentation/Storage-Configuration/ceph-teardown.md

@@ -74,6 +74,7 @@ This includes related resources such as the agent and discover daemonsets with t
 ```console
 kubectl delete -f operator.yaml
 kubectl delete -f common.yaml
+kubectl delete -f psp.yaml
 kubectl delete -f crds.yaml
 ```
 

Documentation/Upgrade/rook-upgrade.md

@@ -52,7 +52,6 @@ those releases.
 * The minimum supported version of Ceph-CSI is v3.6.0. You must update to at least this version of
   Ceph-CSI before or at the same time you update the Rook operator image to v1.10
 
-
 ## Considerations
 
 With this upgrade guide, there are a few notes to consider:

Makefile

@@ -194,6 +194,7 @@ gen-rbac: $(HELM) $(YQ) ## Generate RBAC from Helm charts
 	@# output only stdout to the file; stderr for debugging should keep going to stderr
 	HELM=$(HELM) ./build/rbac/gen-common.sh
 	HELM=$(HELM) ./build/rbac/gen-nfs-rbac.sh
+	HELM=$(HELM) ./build/rbac/gen-psp.sh
 
 docs-preview: ## Preview the documentation through mkdocs
 	mkdocs serve

build/rbac/gen-psp.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -xeEuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+pushd "$SCRIPT_DIR" &>/dev/stderr
+
+PSP_YAML_FILE="$SCRIPT_DIR/../../deploy/examples/psp.yaml"
+
+tmpdir="$(mktemp -d)"
+WITHOUT_FILE="${tmpdir}"/without-psp.yaml # intermediate file of yaml that doesn't include PSP
+WITH_FILE="${tmpdir}"/with-psp.yaml       # intermediate file of yaml that includes previous plus PSP
+
+export ADDITIONAL_HELM_CLI_OPTIONS="--set pspEnable=false"
+./get-helm-rbac.sh >"$WITHOUT_FILE"
+
+export ADDITIONAL_HELM_CLI_OPTIONS="--set pspEnable=true"
+./get-helm-rbac.sh >"$WITH_FILE"
+
+rm -f "$PSP_YAML_FILE"
+cat psp.yaml.header >>"$PSP_YAML_FILE"
+./keep-added.sh "$WITHOUT_FILE" "$WITH_FILE" >>"$PSP_YAML_FILE"
+
+rm -rf "$tmpdir"
+popd &>/dev/stderr

build/rbac/get-helm-rbac.sh

@@ -8,15 +8,11 @@ if ! command -v "$HELM" &>/dev/null; then
   exit 1
 fi
 
-# Whether to include Pod Security Policy (PSP) related resources in the RBAC output.
-# Empty string means DO include PSP resources. Any other value means do NOT include PSP resources.
-: "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES:=""}"
-
 # Supply additional CLI options to the helm command used for generating RBAC.
 # e.g., '--set key=value'
 : "${ADDITIONAL_HELM_CLI_OPTIONS:=""}"
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 pushd "$SCRIPT_DIR" &>/dev/stderr
 
 options=(
@@ -24,11 +20,6 @@ options=(
   --set crds.enabled=false
   --set csi.csiAddons.enabled=true
 )
-if [[ -z "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES}" ]]; then
-  options+=(--set pspEnable=true)
-else
-  options+=(--set pspEnable=false)
-fi
 
 for option in ${ADDITIONAL_HELM_CLI_OPTIONS}; do
   options+=("$option")

build/rbac/psp.yaml.header

@@ -0,0 +1,3 @@
+####################################################################################################
+# This Pod Security Policy (PSP) allows Rook to run in Kubernetes environments using PSPs
+####################################################################################################

deploy/charts/library/templates/_cluster-psp.tpl

@@ -2,6 +2,8 @@
 RoleBindings needed to enable Pod Security Policies for a CephCluster.
 */}}
 {{- define "library.cluster.psp.rolebindings" }}
+{{- if semverCompare "<1.25.0-0" .Capabilities.KubeVersion.GitVersion }}
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
@@ -90,3 +92,4 @@ subjects:
     name: rook-ceph-purge-osd
     namespace: {{ .Release.Namespace }} # namespace:cluster
 {{- end }}
+{{- end }}

deploy/charts/rook-ceph-cluster/templates/rbac.yaml

@@ -12,7 +12,6 @@ clusterrolebindings
 {{ include "library.cluster.clusterrolebindings" . }}
 
 {{- if .Values.pspEnable }}
----
 {{ include "library.cluster.psp.rolebindings" . }}
 {{- end }}
 

deploy/charts/rook-ceph-cluster/values.yaml

@@ -368,7 +368,8 @@ cephClusterSpec:
         disabled: false
 
 ingress:
-  dashboard: {}
+  dashboard:
+    {}
     # annotations:
     #   external-dns.alpha.kubernetes.io/hostname: dashboard.example.com
     #   nginx.ingress.kubernetes.io/rewrite-target: /ceph-dashboard/$2

deploy/charts/rook-ceph/templates/cluster-rbac.yaml

@@ -16,7 +16,6 @@ clusterrolebindings
 {{ include "library.cluster.clusterrolebindings" . }}
 
 {{- if .Values.pspEnable }}
----
 {{ include "library.cluster.psp.rolebindings" . }}
 {{- end }}
 

deploy/charts/rook-ceph/templates/psp.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.pspEnable }}
+{{- if semverCompare "<1.25.0-0" .Capabilities.KubeVersion.GitVersion }}
 # We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs.
 # * privileged (for kube-system namespace)
 # * restricted (for all logged in users)
@@ -169,3 +170,4 @@ subjects:
     namespace: {{ .Release.Namespace }} # namespace:operator
 {{- end }}
 {{- end }}
+{{- end }}

deploy/charts/rook-ceph/values.yaml

@@ -316,22 +316,22 @@ csi:
   # CEPH CSI NFS plugin resource requirement list, Put here list of resource
   # requests and limits you want to apply for plugin pod
   csiNFSPluginResource: |
-   - name : driver-registrar
-     resource:
-       requests:
-         memory: 128Mi
-         cpu: 50m
-       limits:
-         memory: 256Mi
-         cpu: 100m
-   - name : csi-nfsplugin
-     resource:
-       requests:
-         memory: 512Mi
-         cpu: 250m
-       limits:
-         memory: 1Gi
-         cpu: 500m
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 100m
+    - name : csi-nfsplugin
+      resource:
+        requests:
+          memory: 512Mi
+          cpu: 250m
+        limits:
+          memory: 1Gi
+          cpu: 500m
 
   # Set provisionerTolerations and provisionerNodeAffinity for provisioner pod.
   # The CSI provisioner would be best to start on the same nodes as other ceph daemons.
@@ -360,17 +360,17 @@ csi:
   #rbdLivenessMetricsPort: 9080
   #kubeletDirPath: /var/lib/kubelet
   #cephcsi:
-    #image: quay.io/cephcsi/cephcsi:v3.7.0
+  #  image: quay.io/cephcsi/cephcsi:v3.7.0
   #registrar:
-    #image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
+  #  image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
   #provisioner:
-    #image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0
+  #  image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0
   #snapshotter:
-    #image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1
+  #  image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1
   #attacher:
-    #image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0
+  #  image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0
   #resizer:
-    #image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0
+  #  image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0
   # Labels to add to the CSI CephFS Deployments and DaemonSets Pods.
   #cephfsPodLabels: "key1=value1,key2=value2"
   # Labels to add to the CSI NFS Deployments and DaemonSets Pods.

deploy/examples/psp.yaml

@@ -0,0 +1,259 @@
+####################################################################################################
+# This Pod Security Policy (PSP) allows Rook to run in Kubernetes environments using PSPs
+####################################################################################################
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: 'psp:rook'
+  labels:
+    operator: rook
+    storage-backend: ceph
+    app.kubernetes.io/part-of: rook-ceph-operator
+rules:
+  - apiGroups:
+      - policy
+    resources:
+      - podsecuritypolicies
+    resourceNames:
+      - 00-rook-privileged
+    verbs:
+      - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rook-ceph-system-psp
+  labels:
+    operator: rook
+    storage-backend: ceph
+    app.kubernetes.io/part-of: rook-ceph-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: 'psp:rook'
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-system
+    namespace: rook-ceph # namespace:operator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rook-csi-cephfs-plugin-sa-psp
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: 'psp:rook'
+subjects:
+  - kind: ServiceAccount
+    name: rook-csi-cephfs-plugin-sa
+    namespace: rook-ceph # namespace:operator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rook-csi-cephfs-provisioner-sa-psp
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: 'psp:rook'
+subjects:
+  - kind: ServiceAccount
+    name: rook-csi-cephfs-provisioner-sa
+    namespace: rook-ceph # namespace:operator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rook-csi-rbd-plugin-sa-psp
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: 'psp:rook'
+subjects:
+  - kind: ServiceAccount
+    name: rook-csi-rbd-plugin-sa
+    namespace: rook-ceph # namespace:operator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rook-csi-rbd-provisioner-sa-psp
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: 'psp:rook'
+subjects:
+  - kind: ServiceAccount
+    name: rook-csi-rbd-provisioner-sa
+    namespace: rook-ceph # namespace:operator
+---
+# We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs.
+# * privileged (for kube-system namespace)
+# * restricted (for all logged in users)
+#
+# PSPs are applied based on the first match alphabetically. `rook-ceph-operator` comes after
+# `restricted` alphabetically, so we name this `00-rook-privileged`, so it stays somewhere
+# close to the top and so `rook-system` gets the intended PSP. This may need to be renamed in
+# environments with other `00`-prefixed PSPs.
+#
+# More on PSP ordering: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#policy-order
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: 00-rook-privileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default'
+spec:
+  privileged: true
+  allowedCapabilities:
+    # required by CSI
+    - SYS_ADMIN
+    - MKNOD
+  fsGroup:
+    rule: RunAsAny
+  # runAsUser, supplementalGroups - Rook needs to run some pods as root
+  # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time
+  runAsUser:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  # seLinux - seLinux context is unknown ahead of time; set if this is well-known
+  seLinux:
+    rule: RunAsAny
+  volumes:
+    # recommended minimum set
+    - configMap
+    - downwardAPI
+    - emptyDir
+    - persistentVolumeClaim
+    - secret
+    - projected
+    # required for Rook
+    - hostPath
+  # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known
+  # allowedHostPaths:
+  #   - pathPrefix: "/run/udev"  # for OSD prep
+  #     readOnly: false
+  #   - pathPrefix: "/dev"  # for OSD prep
+  #     readOnly: false
+  #   - pathPrefix: "/var/lib/rook"  # or whatever the dataDirHostPath value is set to
+  #     readOnly: false
+  # Ceph requires host IPC for setting up encrypted devices
+  hostIPC: true
+  # Ceph OSDs need to share the same PID namespace
+  hostPID: true
+  # hostNetwork can be set to 'false' if host networking isn't used
+  hostNetwork: true
+  hostPorts:
+    # Ceph messenger protocol v1
+    - min: 6789
+      max: 6790 # <- support old default port
+    # Ceph messenger protocol v2
+    - min: 3300
+      max: 3300
+    # Ceph RADOS ports for OSDs, MDSes
+    - min: 6800
+      max: 7300
+    # # Ceph dashboard port HTTP (not recommended)
+    # - min: 7000
+    #   max: 7000
+    # Ceph dashboard port HTTPS
+    - min: 8443
+      max: 8443
+    # Ceph mgr Prometheus Metrics
+    - min: 9283
+      max: 9283
+    # port for CSIAddons
+    - min: 9070
+      max: 9070
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rook-ceph-cmd-reporter-psp
+  namespace: rook-ceph # namespace:cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:rook
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-cmd-reporter
+    namespace: rook-ceph # namespace:cluster
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rook-ceph-default-psp
+  namespace: rook-ceph # namespace:cluster
+  labels:
+    operator: rook
+    storage-backend: ceph
+    app.kubernetes.io/part-of: rook-ceph-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:rook
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: rook-ceph # namespace:cluster
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rook-ceph-mgr-psp
+  namespace: rook-ceph # namespace:cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:rook
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-mgr
+    namespace: rook-ceph # namespace:cluster
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rook-ceph-osd-psp
+  namespace: rook-ceph # namespace:cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:rook
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-osd
+    namespace: rook-ceph # namespace:cluster
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rook-ceph-purge-osd-psp
+  namespace: rook-ceph # namespace:cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:rook
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-purge-osd
+    namespace: rook-ceph # namespace:cluster
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rook-ceph-rgw-psp
+  namespace: rook-ceph # namespace:cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:rook
+subjects:
+  - kind: ServiceAccount
+    name: rook-ceph-rgw
+    namespace: rook-ceph # namespace:cluster

deploy/olm/generate-rook-csv.sh

@@ -85,8 +85,8 @@ ROOK_OP_VERSION=$3
 : "${SED_IN_PLACE:="build/sed-in-place"}"
 YQ_CMD_DELETE=($yq delete -i)
 YQ_CMD_MERGE_OVERWRITE=($yq merge --inplace --overwrite --prettyPrint)
-YQ_CMD_MERGE=($yq merge --inplace --append -P )
-YQ_CMD_WRITE=($yq write --inplace -P )
+YQ_CMD_MERGE=($yq merge --inplace --append -P)
+YQ_CMD_WRITE=($yq write --inplace -P)
 OPERATOR_YAML_FILE_K8S="deploy/examples/operator.yaml"
 OPERATOR_YAML_FILE_OCP="deploy/examples/operator-openshift.yaml"
 CSV_PATH="$OLM_CATALOG_DIR/deploy/olm-catalog/${PLATFORM}/${VERSION}"
@@ -105,7 +105,7 @@ fi
 #############
 # FUNCTIONS #
 #############
-function create_directories(){
+function create_directories() {
     mkdir -p "$CSV_PATH"
     mkdir -p "$OLM_CATALOG_DIR/deploy/crds"
 }
@@ -115,10 +115,10 @@ function cleanup() {
     "${YQ_CMD_DELETE[@]}" "$CSV_FILE_NAME" 'spec.install.spec.deployments[0].spec.template.metadata.creationTimestamp'
 }
 
-function generate_csv(){
-    pushd "$OLM_CATALOG_DIR" &> /dev/null
+function generate_csv() {
+    pushd "$OLM_CATALOG_DIR" &>/dev/null
     "${OP_SDK_CMD[@]}" "$VERSION"
-    popd &> /dev/null
+    popd &>/dev/null
 
     mv "$CSV_BUNDLE_PATH/olm.clusterserviceversion.yaml" "$CSV_FILE_NAME"
 
@@ -159,7 +159,7 @@ function generate_operator_yaml() {
         operator_file=$OPERATOR_YAML_FILE_OCP
     fi
 
-    sed -n '/^# OLM: BEGIN OPERATOR DEPLOYMENT$/,/# OLM: END OPERATOR DEPLOYMENT$/p' "$operator_file" > "$OLM_OPERATOR_YAML_FILE"
+    sed -n '/^# OLM: BEGIN OPERATOR DEPLOYMENT$/,/# OLM: END OPERATOR DEPLOYMENT$/p' "$operator_file" >"$OLM_OPERATOR_YAML_FILE"
 }
 
 function hack_csv() {
@@ -210,7 +210,7 @@ function generate_package() {
     "${YQ_CMD_WRITE[@]}" "$PACKAGE_FILE" channels[0].currentCSV "rook-ceph.v${VERSION}"
 }
 
-function apply_rook_op_img(){
+function apply_rook_op_img() {
     "${YQ_CMD_WRITE[@]}" "$CSV_FILE_NAME" metadata.annotations.containerImage "$ROOK_OP_VERSION"
     "${YQ_CMD_WRITE[@]}" "$CSV_FILE_NAME" spec.install.spec.deployments[0].spec.template.spec.containers[0].image "$ROOK_OP_VERSION"
 }
@@ -232,12 +232,9 @@ function validate_crds() {
 create_directories
 generate_operator_yaml "$@"
 
-# Do not include Pod Security Policy (PSP) resources for CSV generation since OLM uses
-# Security Context Constraints (SCC).
-export DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES=true
 # Generate csi nfs rbac too.
-export ADDITIONAL_HELM_CLI_OPTIONS="--set csi.nfs.enabled=true"
-./build/rbac/get-helm-rbac.sh > "$OLM_RBAC_YAML_FILE"
+export ADDITIONAL_HELM_CLI_OPTIONS="--set csi.nfs.enabled=true --set pspEnable=false"
+./build/rbac/get-helm-rbac.sh >"$OLM_RBAC_YAML_FILE"
 
 # TODO: do we need separate clusterrole/clusterrolebinding/role/rolebinding/serviceaccount files, or
 # can these just stay in rbac.yaml? If they need to be separate, we can do that here with YQ.

tests/scripts/github-action-helper.sh

@@ -100,7 +100,7 @@ function create_bluestore_partitions_and_pvcs() {
   tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART"
 }
 
-function create_bluestore_partitions_and_pvcs_for_wal(){
+function create_bluestore_partitions_and_pvcs_for_wal() {
   BLOCK_PART="$BLOCK"3
   DB_PART="$BLOCK"1
   WAL_PART="$BLOCK"2
@@ -111,9 +111,9 @@ function create_bluestore_partitions_and_pvcs_for_wal(){
 function collect_udev_logs_in_background() {
   local log_dir="${1:-"/home/runner/work/rook/rook/tests/integration/_output/tests"}"
   mkdir -p "${log_dir}"
-  udevadm monitor --property &> "${log_dir}"/udev-monitor-property.txt &
-  udevadm monitor --kernel &> "${log_dir}"/udev-monitor-kernel.txt &
-  udevadm monitor --udev &> "${log_dir}"/udev-monitor-udev.txt &
+  udevadm monitor --property &>"${log_dir}"/udev-monitor-property.txt &
+  udevadm monitor --kernel &>"${log_dir}"/udev-monitor-kernel.txt &
+  udevadm monitor --udev &>"${log_dir}"/udev-monitor-udev.txt &
 }
 
 function build_rook() {
@@ -125,25 +125,26 @@ function build_rook() {
   for _ in $(seq 1 3); do
     if ! o=$(make -j"$(nproc)" IMAGES='ceph' "$build_type"); then
       case "$o" in
-        *"$NETWORK_ERROR"*)
-          echo "network failure occurred, retrying..."
-          continue
+      *"$NETWORK_ERROR"*)
+        echo "network failure occurred, retrying..."
+        continue
         ;;
-        *"$SERVICE_UNAVAILABLE_ERROR"*)
-          echo "network failure occurred, retrying..."
-          continue
+      *"$SERVICE_UNAVAILABLE_ERROR"*)
+        echo "network failure occurred, retrying..."
+        continue
         ;;
-        *"$INTERNAL_ERROR"*)
-          echo "network failure occurred, retrying..."
-          continue
+      *"$INTERNAL_ERROR"*)
+        echo "network failure occurred, retrying..."
+        continue
         ;;
-        *"$INTERNAL_SERVER_ERROR"*)
-          echo "network failure occurred, retrying..."
-          continue
+      *"$INTERNAL_SERVER_ERROR"*)
+        echo "network failure occurred, retrying..."
+        continue
+        ;;
+      *)
+        # valid failure
+        exit 1
         ;;
-        *)
-          # valid failure
-          exit 1
       esac
     fi
     # no errors so we break the loop after the first iteration
@@ -165,6 +166,12 @@ function validate_yaml() {
   cd deploy/examples
   kubectl create -f crds.yaml -f common.yaml
 
+  # create and use PSPs on k8s versions older than 1.21
+  kube_minor_ver="$(kubectl version -o json | jq -r '.serverVersion.minor')"
+  if [[ "$kube_minor_ver" -lt 21 ]]; then
+    kubectl create -f psp.yaml
+  fi
+
   # create the volume replication CRDs
   replication_version=v0.3.0
   replication_url="https://raw.githubusercontent.com/csi-addons/volume-replication-operator/${replication_version}/config/crd/bases"
@@ -177,7 +184,7 @@ function validate_yaml() {
   kubectl apply -f "${keda_url}"
 
   # skipping folders and some yamls that are only for openshift.
-  manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*')"
+  manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*' -and -not -name 'psp*')"
   with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline
   # shellcheck disable=SC2086 # '-f manifest1.yaml -f manifest2.yaml etc.' should not be quoted
   kubectl create ${with_f_arg} --dry-run=client
@@ -185,7 +192,7 @@ function validate_yaml() {
 
 function create_cluster_prerequisites() {
   # this might be called from another function that has already done a cd
-  ( cd deploy/examples && kubectl create -f crds.yaml -f common.yaml )
+  (cd deploy/examples && kubectl create -f crds.yaml -f common.yaml)
 }
 
 function deploy_manifest_with_local_build() {
@@ -196,8 +203,8 @@ function deploy_manifest_with_local_build() {
 }
 
 function replace_ceph_image() {
-  local file="$1"  # parameter 1: the file in which to replace the ceph image
-  local ceph_image="${2:-}"  # parameter 2: the new ceph image to use
+  local file="$1"           # parameter 1: the file in which to replace the ceph image
+  local ceph_image="${2:-}" # parameter 2: the new ceph image to use
   if [[ -z ${ceph_image} ]]; then
     echo "No Ceph image given. Not adjusting manifests."
     return 0
@@ -209,14 +216,14 @@ function deploy_cluster() {
   cd deploy/examples
   deploy_manifest_with_local_build operator.yaml
   if [ $# == 0 ]; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-test.yaml
-  elif [ "$1" = "two_osds_in_device" ] ; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n    config:\n      osdsPerDevice: \"2\"|g" cluster-test.yaml
-  elif [ "$1" = "osd_with_metadata_device" ] ; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n    config:\n      metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml
-  elif [ "$1" = "encryption" ] ; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n    config:\n      encryptedDevice: \"true\"|g" cluster-test.yaml
-  elif [ "$1" = "lvm" ] ; then
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-test.yaml
+  elif [ "$1" = "two_osds_in_device" ]; then
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n    config:\n      osdsPerDevice: \"2\"|g" cluster-test.yaml
+  elif [ "$1" = "osd_with_metadata_device" ]; then
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n    config:\n      metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml
+  elif [ "$1" = "encryption" ]; then
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n    config:\n      encryptedDevice: \"true\"|g" cluster-test.yaml
+  elif [ "$1" = "lvm" ]; then
     sed -i "s|#deviceFilter:|devices:\n      - name: \"/dev/test-rook-vg/test-rook-lv\"|g" cluster-test.yaml
   else
     echo "invalid argument: $*" >&2
@@ -241,11 +248,11 @@ function deploy_csi_hostnetwork_disabled_cluster() {
   sed -i 's/.*CSI_ENABLE_HOST_NETWORK:.*/  CSI_ENABLE_HOST_NETWORK: \"false\"/g' operator.yaml
   deploy_manifest_with_local_build operator.yaml
   if [ $# == 0 ]; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-test.yaml
-  elif [ "$1" = "two_osds_in_device" ] ; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n    config:\n      osdsPerDevice: \"2\"|g" cluster-test.yaml
-  elif [ "$1" = "osd_with_metadata_device" ] ; then
-    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n    config:\n      metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-test.yaml
+  elif [ "$1" = "two_osds_in_device" ]; then
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n    config:\n      osdsPerDevice: \"2\"|g" cluster-test.yaml
+  elif [ "$1" = "osd_with_metadata_device" ]; then
+    sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n    config:\n      metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml
   fi
   kubectl create -f cluster-test.yaml
   kubectl create -f filesystem-test.yaml
@@ -256,7 +263,7 @@ function wait_for_prepare_pod() {
   get_pod_cmd=(kubectl --namespace rook-ceph get pod --no-headers)
   timeout=450
   start_time="${SECONDS}"
-  while [[ $(( SECONDS - start_time )) -lt $timeout ]]; do
+  while [[ $((SECONDS - start_time)) -lt $timeout ]]; do
     pod="$("${get_pod_cmd[@]}" --selector=app=rook-ceph-osd-prepare --output custom-columns=NAME:.metadata.name,PHASE:status.phase | awk 'FNR <= 1')"
     if echo "$pod" | grep 'Running\|Succeeded\|Failed'; then break; fi
     echo 'waiting for at least one osd prepare pod to be running or finished'
@@ -266,7 +273,7 @@ function wait_for_prepare_pod() {
   kubectl --namespace rook-ceph logs --follow "$pod"
   timeout=60
   start_time="${SECONDS}"
-  while [[ $(( SECONDS - start_time )) -lt $timeout ]]; do
+  while [[ $((SECONDS - start_time)) -lt $timeout ]]; do
     pod="$("${get_pod_cmd[@]}" --selector app=rook-ceph-osd,ceph_daemon_id=0 --output custom-columns=NAME:.metadata.name,PHASE:status.phase)"
     if echo "$pod" | grep 'Running'; then break; fi
     echo 'waiting for OSD 0 pod to be running'
@@ -315,7 +322,7 @@ function create_LV_on_disk() {
 }
 
 function deploy_first_rook_cluster() {
-  BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1)
+  BLOCK=$(sudo lsblk | awk '/14G/ {print $1}' | head -1)
   create_cluster_prerequisites
   cd deploy/examples/
 
@@ -328,9 +335,9 @@ function deploy_first_rook_cluster() {
 }
 
 function deploy_second_rook_cluster() {
-  BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1)
+  BLOCK=$(sudo lsblk | awk '/14G/ {print $1}' | head -1)
   cd deploy/examples/
-  NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f -
+  NAMESPACE=rook-ceph-secondary envsubst <common-second-cluster.yaml | kubectl create -f -
   sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' cluster-test.yaml
   yq w -i -d1 cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"2
   yq w -i -d1 cluster-test.yaml spec.dataDirHostPath "/var/lib/rook-external"
@@ -341,17 +348,17 @@ function deploy_second_rook_cluster() {
 
 function wait_for_rgw() {
   for _ in {1..120}; do
-    if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --no-headers --field-selector=status.phase=Running|wc -l)" -ge 1 ] ; then
-        echo "rgw pod is found"
-        break
+    if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --no-headers --field-selector=status.phase=Running | wc -l)" -ge 1 ]; then
+      echo "rgw pod is found"
+      break
     fi
     echo "waiting for rgw pods"
     sleep 5
   done
   for _ in {1..120}; do
-    if [ "$(kubectl -n "$1" get deployment -l app=rook-ceph-rgw -o yaml | yq read - 'items[0].status.readyReplicas')" -ge 1 ] ; then
-        echo "rgw is ready"
-        break
+    if [ "$(kubectl -n "$1" get deployment -l app=rook-ceph-rgw -o yaml | yq read - 'items[0].status.readyReplicas')" -ge 1 ]; then
+      echo "rgw is ready"
+      break
     fi
     echo "waiting for rgw becomes ready"
     sleep 5
@@ -359,25 +366,25 @@ function wait_for_rgw() {
 }
 
 function verify_operator_log_message() {
-  local message="$1"  # param 1: the message to verify exists
-  local namespace="${2:-rook-ceph}"  # optional param 2: the namespace of the CephCluster (default: rook-ceph)
+  local message="$1"                # param 1: the message to verify exists
+  local namespace="${2:-rook-ceph}" # optional param 2: the namespace of the CephCluster (default: rook-ceph)
   kubectl --namespace "$namespace" logs deployment/rook-ceph-operator | grep "$message"
 }
 
 function wait_for_operator_log_message() {
-  local message="$1"  # param 1: the message to look for
-  local timeout="$2"  # param 2: the timeout for waiting for the message to exist
-  local namespace="${3:-rook-ceph}"  # optional param 3: the namespace of the CephCluster (default: rook-ceph)
+  local message="$1"                # param 1: the message to look for
+  local timeout="$2"                # param 2: the timeout for waiting for the message to exist
+  local namespace="${3:-rook-ceph}" # optional param 3: the namespace of the CephCluster (default: rook-ceph)
   start_time="${SECONDS}"
-  while [[ $(( SECONDS - start_time )) -lt $timeout ]]; do
+  while [[ $((SECONDS - start_time)) -lt $timeout ]]; do
     if verify_operator_log_message "$message" "$namespace"; then return 0; fi
     sleep 5
   done
   echo "timed out" >&2 && return 1
 }
 
-function restart_operator () {
-  local namespace="${1:-rook-ceph}"  # optional param 1: the namespace of the CephCluster (default: rook-ceph)
+function restart_operator() {
+  local namespace="${1:-rook-ceph}" # optional param 1: the namespace of the CephCluster (default: rook-ceph)
   kubectl --namespace "$namespace" delete pod --selector app=rook-ceph-operator
   # wait for new pod to be running
   get_pod_cmd=(kubectl --namespace "$namespace" get pod --selector app=rook-ceph-operator --no-headers)
@@ -388,11 +395,11 @@ function restart_operator () {
 
 function write_object_to_cluster1_read_from_cluster2() {
   cd deploy/examples/
-  echo "[default]" > s3cfg
-  echo "host_bucket = no.way.in.hell" >> ./s3cfg
-  echo "use_https = False" >> ./s3cfg
+  echo "[default]" >s3cfg
+  echo "host_bucket = no.way.in.hell" >>./s3cfg
+  echo "use_https = False" >>./s3cfg
   fallocate -l 1M ./1M.dat
-  echo "hello world" >> ./1M.dat
+  echo "hello world" >>./1M.dat
   CLUSTER_1_IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}")
   BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}")
   BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}")
@@ -418,9 +425,9 @@ function create_helm_tag() {
 
 function deploy_multus() {
   # download the multus daemonset, and remove mem and cpu limits that cause it to crash on minikube
-  curl https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml \
-    | sed -e 's/cpu: /# cpu: /g' -e 's/memory: /# memory: /g' \
-    | kubectl apply -f -
+  curl https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml |
+    sed -e 's/cpu: /# cpu: /g' -e 's/memory: /# memory: /g' |
+    kubectl apply -f -
 
   # install whereabouts
   kubectl apply \
@@ -462,20 +469,20 @@ function deploy_multus_cluster() {
   cd deploy/examples
   deploy_manifest_with_local_build operator.yaml
   deploy_manifest_with_local_build toolbox.yaml
-  sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-multus-test.yaml
+  sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-multus-test.yaml
   kubectl create -f cluster-multus-test.yaml
   kubectl create -f filesystem-test.yaml
 }
 
 function wait_for_ceph_csi_configmap_to_be_updated {
   timeout 60 bash <<EOF
-until [[ $(kubectl -n rook-ceph get configmap rook-ceph-csi-config  -o jsonpath="{.data.csi-cluster-config-json}" | jq .[0].rbd.netNamespaceFilePath) != "null" ]]; do
+until [[ $(kubectl -n rook-ceph get configmap rook-ceph-csi-config -o jsonpath="{.data.csi-cluster-config-json}" | jq .[0].rbd.netNamespaceFilePath) != "null" ]]; do
   echo "waiting for ceph csi configmap to be updated with rbd.netNamespaceFilePath"
   sleep 5
 done
 EOF
   timeout 60 bash <<EOF
-until [[ $(kubectl -n rook-ceph get configmap rook-ceph-csi-config  -o jsonpath="{.data.csi-cluster-config-json}" | jq .[0].cephFS.netNamespaceFilePath) != "null" ]]; do
+until [[ $(kubectl -n rook-ceph get configmap rook-ceph-csi-config -o jsonpath="{.data.csi-cluster-config-json}" | jq .[0].cephFS.netNamespaceFilePath) != "null" ]]; do
   echo "waiting for ceph csi configmap to be updated with cephFS.netNamespaceFilePath"
   sleep 5
 done