diff --git a/Documentation/Getting-Started/example-configurations.md b/Documentation/Getting-Started/example-configurations.md index e5cf46729..ee2da9e88 100644 --- a/Documentation/Getting-Started/example-configurations.md +++ b/Documentation/Getting-Started/example-configurations.md @@ -11,9 +11,11 @@ See the **[example yaml files](https://github.com/rook/rook/blob/master/deploy/e The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments. The [crds.yaml](https://github.com/rook/rook/blob/master/deploy/examples/crds.yaml) and [common.yaml](https://github.com/rook/rook/blob/master/deploy/examples/common.yaml) sets these resources up. +You may also need [psp.yaml](https://github.com/rook/rook/blob/master/deploy/examples/psp.yaml) if +your cluster is protected by Pod Security Policies. ```console -kubectl create -f crds.yaml -f common.yaml +kubectl create -f crds.yaml -f common.yaml # -f psp.yaml ``` The examples all assume the operator and all Ceph daemons will be started in the same namespace. If you want to deploy the operator in a separate namespace, see the comments throughout `common.yaml`. diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md index 24ec41ef6..f7200493c 100644 --- a/Documentation/Getting-Started/quickstart.md +++ b/Documentation/Getting-Started/quickstart.md @@ -50,6 +50,7 @@ The first step is to deploy the Rook operator. Check that you are using the [exa ```console cd deploy/examples +# kubectl create -f psp.yaml # if your cluster is protected by pod security policies kubectl create -f crds.yaml -f common.yaml -f operator.yaml # verify the rook-ceph-operator is in the `Running` state before proceeding diff --git a/Documentation/Storage-Configuration/ceph-teardown.md b/Documentation/Storage-Configuration/ceph-teardown.md index 0d550b608..2b0eb63a8 100644 --- a/Documentation/Storage-Configuration/ceph-teardown.md +++ b/Documentation/Storage-Configuration/ceph-teardown.md @@ -74,6 +74,7 @@ This includes related resources such as the agent and discover daemonsets with t ```console kubectl delete -f operator.yaml kubectl delete -f common.yaml +kubectl delete -f psp.yaml kubectl delete -f crds.yaml ``` diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md index ce7fb83b3..f7a3fd62c 100644 --- a/Documentation/Upgrade/rook-upgrade.md +++ b/Documentation/Upgrade/rook-upgrade.md @@ -52,7 +52,6 @@ those releases. * The minimum supported version of Ceph-CSI is v3.6.0. You must update to at least this version of Ceph-CSI before or at the same time you update the Rook operator image to v1.10 - ## Considerations With this upgrade guide, there are a few notes to consider: diff --git a/Makefile b/Makefile index d8cdaf692..ed128c371 100644 --- a/Makefile +++ b/Makefile @@ -194,6 +194,7 @@ gen-rbac: $(HELM) $(YQ) ## Generate RBAC from Helm charts @# output only stdout to the file; stderr for debugging should keep going to stderr HELM=$(HELM) ./build/rbac/gen-common.sh HELM=$(HELM) ./build/rbac/gen-nfs-rbac.sh + HELM=$(HELM) ./build/rbac/gen-psp.sh docs-preview: ## Preview the documentation through mkdocs mkdocs serve diff --git a/build/rbac/gen-psp.sh b/build/rbac/gen-psp.sh new file mode 100755 index 000000000..99142b4b0 --- /dev/null +++ b/build/rbac/gen-psp.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -xeEuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +pushd "$SCRIPT_DIR" &>/dev/stderr + +PSP_YAML_FILE="$SCRIPT_DIR/../../deploy/examples/psp.yaml" + +tmpdir="$(mktemp -d)" +WITHOUT_FILE="${tmpdir}"/without-psp.yaml # intermediate file of yaml that doesn't include PSP +WITH_FILE="${tmpdir}"/with-psp.yaml # intermediate file of yaml that includes previous plus PSP + +export ADDITIONAL_HELM_CLI_OPTIONS="--set pspEnable=false" +./get-helm-rbac.sh >"$WITHOUT_FILE" + +export ADDITIONAL_HELM_CLI_OPTIONS="--set pspEnable=true" +./get-helm-rbac.sh >"$WITH_FILE" + +rm -f "$PSP_YAML_FILE" +cat psp.yaml.header >>"$PSP_YAML_FILE" +./keep-added.sh "$WITHOUT_FILE" "$WITH_FILE" >>"$PSP_YAML_FILE" + +rm -rf "$tmpdir" +popd &>/dev/stderr diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index 82286217d..c5ec28285 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -8,15 +8,11 @@ if ! command -v "$HELM" &>/dev/null; then exit 1 fi -# Whether to include Pod Security Policy (PSP) related resources in the RBAC output. -# Empty string means DO include PSP resources. Any other value means do NOT include PSP resources. -: "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES:=""}" - # Supply additional CLI options to the helm command used for generating RBAC. # e.g., '--set key=value' : "${ADDITIONAL_HELM_CLI_OPTIONS:=""}" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" pushd "$SCRIPT_DIR" &>/dev/stderr options=( @@ -24,11 +20,6 @@ options=( --set crds.enabled=false --set csi.csiAddons.enabled=true ) -if [[ -z "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES}" ]]; then - options+=(--set pspEnable=true) -else - options+=(--set pspEnable=false) -fi for option in ${ADDITIONAL_HELM_CLI_OPTIONS}; do options+=("$option") diff --git a/build/rbac/psp.yaml.header b/build/rbac/psp.yaml.header new file mode 100644 index 000000000..049a2ec11 --- /dev/null +++ b/build/rbac/psp.yaml.header @@ -0,0 +1,3 @@ +#################################################################################################### +# This Pod Security Policy (PSP) allows Rook to run in Kubernetes environments using PSPs +#################################################################################################### diff --git a/deploy/charts/library/templates/_cluster-psp.tpl b/deploy/charts/library/templates/_cluster-psp.tpl index b586a2a9e..1bf140d6e 100644 --- a/deploy/charts/library/templates/_cluster-psp.tpl +++ b/deploy/charts/library/templates/_cluster-psp.tpl @@ -2,6 +2,8 @@ RoleBindings needed to enable Pod Security Policies for a CephCluster. */}} {{- define "library.cluster.psp.rolebindings" }} +{{- if semverCompare "<1.25.0-0" .Capabilities.KubeVersion.GitVersion }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -90,3 +92,4 @@ subjects: name: rook-ceph-purge-osd namespace: {{ .Release.Namespace }} # namespace:cluster {{- end }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/templates/rbac.yaml b/deploy/charts/rook-ceph-cluster/templates/rbac.yaml index 1e3540572..7e3b658e8 100644 --- a/deploy/charts/rook-ceph-cluster/templates/rbac.yaml +++ b/deploy/charts/rook-ceph-cluster/templates/rbac.yaml @@ -12,7 +12,6 @@ clusterrolebindings {{ include "library.cluster.clusterrolebindings" . }} {{- if .Values.pspEnable }} ---- {{ include "library.cluster.psp.rolebindings" . }} {{- end }} diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index 7e833a15d..443040fb0 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -368,7 +368,8 @@ cephClusterSpec: disabled: false ingress: - dashboard: {} + dashboard: + {} # annotations: # external-dns.alpha.kubernetes.io/hostname: dashboard.example.com # nginx.ingress.kubernetes.io/rewrite-target: /ceph-dashboard/$2 diff --git a/deploy/charts/rook-ceph/templates/cluster-rbac.yaml b/deploy/charts/rook-ceph/templates/cluster-rbac.yaml index 06dcca401..7605d7c04 100644 --- a/deploy/charts/rook-ceph/templates/cluster-rbac.yaml +++ b/deploy/charts/rook-ceph/templates/cluster-rbac.yaml @@ -16,7 +16,6 @@ clusterrolebindings {{ include "library.cluster.clusterrolebindings" . }} {{- if .Values.pspEnable }} ---- {{ include "library.cluster.psp.rolebindings" . }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/psp.yaml b/deploy/charts/rook-ceph/templates/psp.yaml index 920299873..5e5385a12 100644 --- a/deploy/charts/rook-ceph/templates/psp.yaml +++ b/deploy/charts/rook-ceph/templates/psp.yaml @@ -1,4 +1,5 @@ {{- if .Values.pspEnable }} +{{- if semverCompare "<1.25.0-0" .Capabilities.KubeVersion.GitVersion }} # We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs. # * privileged (for kube-system namespace) # * restricted (for all logged in users) @@ -169,3 +170,4 @@ subjects: namespace: {{ .Release.Namespace }} # namespace:operator {{- end }} {{- end }} +{{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 9b21fc54b..29e61a00e 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -316,22 +316,22 @@ csi: # CEPH CSI NFS plugin resource requirement list, Put here list of resource # requests and limits you want to apply for plugin pod csiNFSPluginResource: | - - name : driver-registrar - resource: - requests: - memory: 128Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 100m - - name : csi-nfsplugin - resource: - requests: - memory: 512Mi - cpu: 250m - limits: - memory: 1Gi - cpu: 500m + - name : driver-registrar + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 100m + - name : csi-nfsplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 500m # Set provisionerTolerations and provisionerNodeAffinity for provisioner pod. # The CSI provisioner would be best to start on the same nodes as other ceph daemons. @@ -360,17 +360,17 @@ csi: #rbdLivenessMetricsPort: 9080 #kubeletDirPath: /var/lib/kubelet #cephcsi: - #image: quay.io/cephcsi/cephcsi:v3.7.0 + # image: quay.io/cephcsi/cephcsi:v3.7.0 #registrar: - #image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + # image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 #provisioner: - #image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 + # image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 #snapshotter: - #image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1 + # image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1 #attacher: - #image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 + # image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 #resizer: - #image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 + # image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 # Labels to add to the CSI CephFS Deployments and DaemonSets Pods. #cephfsPodLabels: "key1=value1,key2=value2" # Labels to add to the CSI NFS Deployments and DaemonSets Pods. diff --git a/deploy/examples/psp.yaml b/deploy/examples/psp.yaml new file mode 100644 index 000000000..a290544c2 --- /dev/null +++ b/deploy/examples/psp.yaml @@ -0,0 +1,259 @@ +#################################################################################################### +# This Pod Security Policy (PSP) allows Rook to run in Kubernetes environments using PSPs +#################################################################################################### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: 'psp:rook' + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - 00-rook-privileged + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-ceph-system-psp + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph # namespace:operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-cephfs-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph # namespace:operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-csi-rbd-provisioner-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator +--- +# We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs. +# * privileged (for kube-system namespace) +# * restricted (for all logged in users) +# +# PSPs are applied based on the first match alphabetically. `rook-ceph-operator` comes after +# `restricted` alphabetically, so we name this `00-rook-privileged`, so it stays somewhere +# close to the top and so `rook-system` gets the intended PSP. This may need to be renamed in +# environments with other `00`-prefixed PSPs. +# +# More on PSP ordering: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#policy-order +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: 00-rook-privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' +spec: + privileged: true + allowedCapabilities: + # required by CSI + - SYS_ADMIN + - MKNOD + fsGroup: + rule: RunAsAny + # runAsUser, supplementalGroups - Rook needs to run some pods as root + # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + # seLinux - seLinux context is unknown ahead of time; set if this is well-known + seLinux: + rule: RunAsAny + volumes: + # recommended minimum set + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret + - projected + # required for Rook + - hostPath + # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known + # allowedHostPaths: + # - pathPrefix: "/run/udev" # for OSD prep + # readOnly: false + # - pathPrefix: "/dev" # for OSD prep + # readOnly: false + # - pathPrefix: "/var/lib/rook" # or whatever the dataDirHostPath value is set to + # readOnly: false + # Ceph requires host IPC for setting up encrypted devices + hostIPC: true + # Ceph OSDs need to share the same PID namespace + hostPID: true + # hostNetwork can be set to 'false' if host networking isn't used + hostNetwork: true + hostPorts: + # Ceph messenger protocol v1 + - min: 6789 + max: 6790 # <- support old default port + # Ceph messenger protocol v2 + - min: 3300 + max: 3300 + # Ceph RADOS ports for OSDs, MDSes + - min: 6800 + max: 7300 + # # Ceph dashboard port HTTP (not recommended) + # - min: 7000 + # max: 7000 + # Ceph dashboard port HTTPS + - min: 8443 + max: 8443 + # Ceph mgr Prometheus Metrics + - min: 9283 + max: 9283 + # port for CSIAddons + - min: 9070 + max: 9070 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter-psp + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-default-psp + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: default + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr-psp + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-osd-psp + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-purge-osd-psp + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-rgw-psp + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-rgw + namespace: rook-ceph # namespace:cluster diff --git a/deploy/olm/generate-rook-csv.sh b/deploy/olm/generate-rook-csv.sh index 1d1b4f578..b3ff8210f 100755 --- a/deploy/olm/generate-rook-csv.sh +++ b/deploy/olm/generate-rook-csv.sh @@ -85,8 +85,8 @@ ROOK_OP_VERSION=$3 : "${SED_IN_PLACE:="build/sed-in-place"}" YQ_CMD_DELETE=($yq delete -i) YQ_CMD_MERGE_OVERWRITE=($yq merge --inplace --overwrite --prettyPrint) -YQ_CMD_MERGE=($yq merge --inplace --append -P ) -YQ_CMD_WRITE=($yq write --inplace -P ) +YQ_CMD_MERGE=($yq merge --inplace --append -P) +YQ_CMD_WRITE=($yq write --inplace -P) OPERATOR_YAML_FILE_K8S="deploy/examples/operator.yaml" OPERATOR_YAML_FILE_OCP="deploy/examples/operator-openshift.yaml" CSV_PATH="$OLM_CATALOG_DIR/deploy/olm-catalog/${PLATFORM}/${VERSION}" @@ -105,7 +105,7 @@ fi ############# # FUNCTIONS # ############# -function create_directories(){ +function create_directories() { mkdir -p "$CSV_PATH" mkdir -p "$OLM_CATALOG_DIR/deploy/crds" } @@ -115,10 +115,10 @@ function cleanup() { "${YQ_CMD_DELETE[@]}" "$CSV_FILE_NAME" 'spec.install.spec.deployments[0].spec.template.metadata.creationTimestamp' } -function generate_csv(){ - pushd "$OLM_CATALOG_DIR" &> /dev/null +function generate_csv() { + pushd "$OLM_CATALOG_DIR" &>/dev/null "${OP_SDK_CMD[@]}" "$VERSION" - popd &> /dev/null + popd &>/dev/null mv "$CSV_BUNDLE_PATH/olm.clusterserviceversion.yaml" "$CSV_FILE_NAME" @@ -159,7 +159,7 @@ function generate_operator_yaml() { operator_file=$OPERATOR_YAML_FILE_OCP fi - sed -n '/^# OLM: BEGIN OPERATOR DEPLOYMENT$/,/# OLM: END OPERATOR DEPLOYMENT$/p' "$operator_file" > "$OLM_OPERATOR_YAML_FILE" + sed -n '/^# OLM: BEGIN OPERATOR DEPLOYMENT$/,/# OLM: END OPERATOR DEPLOYMENT$/p' "$operator_file" >"$OLM_OPERATOR_YAML_FILE" } function hack_csv() { @@ -210,7 +210,7 @@ function generate_package() { "${YQ_CMD_WRITE[@]}" "$PACKAGE_FILE" channels[0].currentCSV "rook-ceph.v${VERSION}" } -function apply_rook_op_img(){ +function apply_rook_op_img() { "${YQ_CMD_WRITE[@]}" "$CSV_FILE_NAME" metadata.annotations.containerImage "$ROOK_OP_VERSION" "${YQ_CMD_WRITE[@]}" "$CSV_FILE_NAME" spec.install.spec.deployments[0].spec.template.spec.containers[0].image "$ROOK_OP_VERSION" } @@ -232,12 +232,9 @@ function validate_crds() { create_directories generate_operator_yaml "$@" -# Do not include Pod Security Policy (PSP) resources for CSV generation since OLM uses -# Security Context Constraints (SCC). -export DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES=true # Generate csi nfs rbac too. -export ADDITIONAL_HELM_CLI_OPTIONS="--set csi.nfs.enabled=true" -./build/rbac/get-helm-rbac.sh > "$OLM_RBAC_YAML_FILE" +export ADDITIONAL_HELM_CLI_OPTIONS="--set csi.nfs.enabled=true --set pspEnable=false" +./build/rbac/get-helm-rbac.sh >"$OLM_RBAC_YAML_FILE" # TODO: do we need separate clusterrole/clusterrolebinding/role/rolebinding/serviceaccount files, or # can these just stay in rbac.yaml? If they need to be separate, we can do that here with YQ. diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 67fd8e260..0c000094b 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -100,7 +100,7 @@ function create_bluestore_partitions_and_pvcs() { tests/scripts/localPathPV.sh "$BLOCK_PART" "$DB_PART" } -function create_bluestore_partitions_and_pvcs_for_wal(){ +function create_bluestore_partitions_and_pvcs_for_wal() { BLOCK_PART="$BLOCK"3 DB_PART="$BLOCK"1 WAL_PART="$BLOCK"2 @@ -111,9 +111,9 @@ function create_bluestore_partitions_and_pvcs_for_wal(){ function collect_udev_logs_in_background() { local log_dir="${1:-"/home/runner/work/rook/rook/tests/integration/_output/tests"}" mkdir -p "${log_dir}" - udevadm monitor --property &> "${log_dir}"/udev-monitor-property.txt & - udevadm monitor --kernel &> "${log_dir}"/udev-monitor-kernel.txt & - udevadm monitor --udev &> "${log_dir}"/udev-monitor-udev.txt & + udevadm monitor --property &>"${log_dir}"/udev-monitor-property.txt & + udevadm monitor --kernel &>"${log_dir}"/udev-monitor-kernel.txt & + udevadm monitor --udev &>"${log_dir}"/udev-monitor-udev.txt & } function build_rook() { @@ -125,25 +125,26 @@ function build_rook() { for _ in $(seq 1 3); do if ! o=$(make -j"$(nproc)" IMAGES='ceph' "$build_type"); then case "$o" in - *"$NETWORK_ERROR"*) - echo "network failure occurred, retrying..." - continue + *"$NETWORK_ERROR"*) + echo "network failure occurred, retrying..." + continue ;; - *"$SERVICE_UNAVAILABLE_ERROR"*) - echo "network failure occurred, retrying..." - continue + *"$SERVICE_UNAVAILABLE_ERROR"*) + echo "network failure occurred, retrying..." + continue ;; - *"$INTERNAL_ERROR"*) - echo "network failure occurred, retrying..." - continue + *"$INTERNAL_ERROR"*) + echo "network failure occurred, retrying..." + continue ;; - *"$INTERNAL_SERVER_ERROR"*) - echo "network failure occurred, retrying..." - continue + *"$INTERNAL_SERVER_ERROR"*) + echo "network failure occurred, retrying..." + continue + ;; + *) + # valid failure + exit 1 ;; - *) - # valid failure - exit 1 esac fi # no errors so we break the loop after the first iteration @@ -165,6 +166,12 @@ function validate_yaml() { cd deploy/examples kubectl create -f crds.yaml -f common.yaml + # create and use PSPs on k8s versions older than 1.21 + kube_minor_ver="$(kubectl version -o json | jq -r '.serverVersion.minor')" + if [[ "$kube_minor_ver" -lt 21 ]]; then + kubectl create -f psp.yaml + fi + # create the volume replication CRDs replication_version=v0.3.0 replication_url="https://raw.githubusercontent.com/csi-addons/volume-replication-operator/${replication_version}/config/crd/bases" @@ -177,7 +184,7 @@ function validate_yaml() { kubectl apply -f "${keda_url}" # skipping folders and some yamls that are only for openshift. - manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*')" + manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*' -and -not -name 'psp*')" with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline # shellcheck disable=SC2086 # '-f manifest1.yaml -f manifest2.yaml etc.' should not be quoted kubectl create ${with_f_arg} --dry-run=client @@ -185,7 +192,7 @@ function validate_yaml() { function create_cluster_prerequisites() { # this might be called from another function that has already done a cd - ( cd deploy/examples && kubectl create -f crds.yaml -f common.yaml ) + (cd deploy/examples && kubectl create -f crds.yaml -f common.yaml) } function deploy_manifest_with_local_build() { @@ -196,8 +203,8 @@ function deploy_manifest_with_local_build() { } function replace_ceph_image() { - local file="$1" # parameter 1: the file in which to replace the ceph image - local ceph_image="${2:-}" # parameter 2: the new ceph image to use + local file="$1" # parameter 1: the file in which to replace the ceph image + local ceph_image="${2:-}" # parameter 2: the new ceph image to use if [[ -z ${ceph_image} ]]; then echo "No Ceph image given. Not adjusting manifests." return 0 @@ -209,14 +216,14 @@ function deploy_cluster() { cd deploy/examples deploy_manifest_with_local_build operator.yaml if [ $# == 0 ]; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-test.yaml - elif [ "$1" = "two_osds_in_device" ] ; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n config:\n osdsPerDevice: \"2\"|g" cluster-test.yaml - elif [ "$1" = "osd_with_metadata_device" ] ; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n config:\n metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml - elif [ "$1" = "encryption" ] ; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n config:\n encryptedDevice: \"true\"|g" cluster-test.yaml - elif [ "$1" = "lvm" ] ; then + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-test.yaml + elif [ "$1" = "two_osds_in_device" ]; then + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n osdsPerDevice: \"2\"|g" cluster-test.yaml + elif [ "$1" = "osd_with_metadata_device" ]; then + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml + elif [ "$1" = "encryption" ]; then + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n encryptedDevice: \"true\"|g" cluster-test.yaml + elif [ "$1" = "lvm" ]; then sed -i "s|#deviceFilter:|devices:\n - name: \"/dev/test-rook-vg/test-rook-lv\"|g" cluster-test.yaml else echo "invalid argument: $*" >&2 @@ -241,11 +248,11 @@ function deploy_csi_hostnetwork_disabled_cluster() { sed -i 's/.*CSI_ENABLE_HOST_NETWORK:.*/ CSI_ENABLE_HOST_NETWORK: \"false\"/g' operator.yaml deploy_manifest_with_local_build operator.yaml if [ $# == 0 ]; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-test.yaml - elif [ "$1" = "two_osds_in_device" ] ; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n config:\n osdsPerDevice: \"2\"|g" cluster-test.yaml - elif [ "$1" = "osd_with_metadata_device" ] ; then - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}\n config:\n metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-test.yaml + elif [ "$1" = "two_osds_in_device" ]; then + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n osdsPerDevice: \"2\"|g" cluster-test.yaml + elif [ "$1" = "osd_with_metadata_device" ]; then + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}\n config:\n metadataDevice: /dev/test-rook-vg/test-rook-lv|g" cluster-test.yaml fi kubectl create -f cluster-test.yaml kubectl create -f filesystem-test.yaml @@ -256,7 +263,7 @@ function wait_for_prepare_pod() { get_pod_cmd=(kubectl --namespace rook-ceph get pod --no-headers) timeout=450 start_time="${SECONDS}" - while [[ $(( SECONDS - start_time )) -lt $timeout ]]; do + while [[ $((SECONDS - start_time)) -lt $timeout ]]; do pod="$("${get_pod_cmd[@]}" --selector=app=rook-ceph-osd-prepare --output custom-columns=NAME:.metadata.name,PHASE:status.phase | awk 'FNR <= 1')" if echo "$pod" | grep 'Running\|Succeeded\|Failed'; then break; fi echo 'waiting for at least one osd prepare pod to be running or finished' @@ -266,7 +273,7 @@ function wait_for_prepare_pod() { kubectl --namespace rook-ceph logs --follow "$pod" timeout=60 start_time="${SECONDS}" - while [[ $(( SECONDS - start_time )) -lt $timeout ]]; do + while [[ $((SECONDS - start_time)) -lt $timeout ]]; do pod="$("${get_pod_cmd[@]}" --selector app=rook-ceph-osd,ceph_daemon_id=0 --output custom-columns=NAME:.metadata.name,PHASE:status.phase)" if echo "$pod" | grep 'Running'; then break; fi echo 'waiting for OSD 0 pod to be running' @@ -315,7 +322,7 @@ function create_LV_on_disk() { } function deploy_first_rook_cluster() { - BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk | awk '/14G/ {print $1}' | head -1) create_cluster_prerequisites cd deploy/examples/ @@ -328,9 +335,9 @@ function deploy_first_rook_cluster() { } function deploy_second_rook_cluster() { - BLOCK=$(sudo lsblk|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk | awk '/14G/ {print $1}' | head -1) cd deploy/examples/ - NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - + NAMESPACE=rook-ceph-secondary envsubst &2 && return 1 } -function restart_operator () { - local namespace="${1:-rook-ceph}" # optional param 1: the namespace of the CephCluster (default: rook-ceph) +function restart_operator() { + local namespace="${1:-rook-ceph}" # optional param 1: the namespace of the CephCluster (default: rook-ceph) kubectl --namespace "$namespace" delete pod --selector app=rook-ceph-operator # wait for new pod to be running get_pod_cmd=(kubectl --namespace "$namespace" get pod --selector app=rook-ceph-operator --no-headers) @@ -388,11 +395,11 @@ function restart_operator () { function write_object_to_cluster1_read_from_cluster2() { cd deploy/examples/ - echo "[default]" > s3cfg - echo "host_bucket = no.way.in.hell" >> ./s3cfg - echo "use_https = False" >> ./s3cfg + echo "[default]" >s3cfg + echo "host_bucket = no.way.in.hell" >>./s3cfg + echo "use_https = False" >>./s3cfg fallocate -l 1M ./1M.dat - echo "hello world" >> ./1M.dat + echo "hello world" >>./1M.dat CLUSTER_1_IP_ADDR=$(kubectl -n rook-ceph get svc rook-ceph-rgw-multisite-store -o jsonpath="{.spec.clusterIP}") BASE64_ACCESS_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.access-key}") BASE64_SECRET_KEY=$(kubectl -n rook-ceph get secrets realm-a-keys -o jsonpath="{.data.secret-key}") @@ -418,9 +425,9 @@ function create_helm_tag() { function deploy_multus() { # download the multus daemonset, and remove mem and cpu limits that cause it to crash on minikube - curl https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml \ - | sed -e 's/cpu: /# cpu: /g' -e 's/memory: /# memory: /g' \ - | kubectl apply -f - + curl https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml | + sed -e 's/cpu: /# cpu: /g' -e 's/memory: /# memory: /g' | + kubectl apply -f - # install whereabouts kubectl apply \ @@ -462,20 +469,20 @@ function deploy_multus_cluster() { cd deploy/examples deploy_manifest_with_local_build operator.yaml deploy_manifest_with_local_build toolbox.yaml - sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\/}|g" cluster-multus-test.yaml + sed -i "s|#deviceFilter:|deviceFilter: ${BLOCK/\/dev\//}|g" cluster-multus-test.yaml kubectl create -f cluster-multus-test.yaml kubectl create -f filesystem-test.yaml } function wait_for_ceph_csi_configmap_to_be_updated { timeout 60 bash <