From 113e2f849da9aac5a82fc8ac1a147d706e90445f Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Mon, 22 Nov 2021 17:31:19 -0700 Subject: [PATCH 1/2] build: use yq for RBAC yaml parsing Use yq instead of Python for parsing RBAC from the Helm chart. We need to use yq v4.14.1 or higher to fix yq's handling of the yaml header markers ('---'). Update the Makefile's yq version to v4, which also requires updating the script to update the CRDs. This was quite easy. It is very difficult, however, to change the version of yq used by the CSV generating/parsing scripts, which already used their own yq download. Continue using yq v3 for this. In order to make sure the scripts are using the right version of yq, add basic validation to them to verify they are running v3 or v4 as required for their operation. Signed-off-by: Blaine Gardner --- .github/workflows/build.yml | 11 - .github/workflows/rbac-gen.yaml | 10 - Makefile | 2 +- build/crds/build-crds.sh | 20 +- build/makelib/golang.mk | 15 +- build/rbac/get-helm-rbac.sh | 2 +- build/rbac/keep-rbac-yaml.py | 77 ------ build/rbac/keep-rbac-yaml.sh | 47 ++++ build/rbac/rbac.yaml | 447 ++++++++++++++++---------------- deploy/olm/generate-rook-csv.sh | 7 +- images/ceph/Makefile | 20 +- 11 files changed, 310 insertions(+), 348 deletions(-) delete mode 100755 build/rbac/keep-rbac-yaml.py create mode 100755 build/rbac/keep-rbac-yaml.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 199813572..beb6c51b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,17 +54,6 @@ jobs: working-directory: /Users/runner/go/src/github.com/rook/rook run: tests/scripts/validate_modified_files.sh crd - # needed for gen-rbac - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - - name: Install pip dependencies - run: | - python -m pip install --upgrade pip - pip install ruamel.yaml==0.17.16 - - name: run gen-rbac working-directory: /Users/runner/go/src/github.com/rook/rook run: GOPATH=$(go env GOPATH) make gen-rbac diff --git a/.github/workflows/rbac-gen.yaml b/.github/workflows/rbac-gen.yaml index 3c31fb437..7b8fff777 100644 --- a/.github/workflows/rbac-gen.yaml +++ b/.github/workflows/rbac-gen.yaml @@ -32,16 +32,6 @@ jobs: - name: copy working directory to GOPATH run: sudo mkdir -p /home/runner/go/src/github.com && sudo cp -a /home/runner/work/rook /home/runner/go/src/github.com/ - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - - name: Install pip dependencies - run: | - python -m pip install --upgrade pip - pip install ruamel.yaml==0.17.16 - - name: run gen-rbac working-directory: /home/runner/go/src/github.com/rook/rook run: GOPATH=$(go env GOPATH) make gen-rbac diff --git a/Makefile b/Makefile index 441482055..5e52acafd 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,7 @@ crds: $(CONTROLLER_GEN) $(YQ) @echo Updating CRD manifests @build/crds/build-crds.sh $(CONTROLLER_GEN) $(YQ) -gen-rbac: $(HELM) ## generate RBAC from Helm charts +gen-rbac: $(HELM) $(YQ) ## generate RBAC from Helm charts HELM=$(HELM) ./build/rbac/get-helm-rbac.sh .PHONY: all build.common cross.build.parallel diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 898904763..58a064254 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -36,6 +36,11 @@ OLM_CATALOG_DIR="${DESTINATION_ROOT}/deploy/olm/deploy/crds" CEPH_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/examples/crds.yaml" CEPH_HELM_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/charts/rook-ceph/templates/resources.yaml" +if [[ "$($YQ_BIN_PATH --version)" != "yq (https://github.com/mikefarah/yq/) version 4."* ]]; then + echo "yq must be version 4.x" + exit 1 +fi + ############# # FUNCTIONS # ############# @@ -50,7 +55,7 @@ generating_crds_v1() { echo "Generating ceph crds" "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/ceph.rook.io/v1" output:crd:artifacts:config="$OLM_CATALOG_DIR" # the csv upgrade is failing on the volumeClaimTemplate.metadata.annotations.crushDeviceClass unless we preserve the annotations as an unknown field - $YQ_BIN_PATH w -i "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields true + $YQ_BIN_PATH eval --inplace '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields = true' "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml } generating_main_crd() { @@ -70,8 +75,9 @@ build_helm_resources() { # add header echo "{{- if .Values.crds.enabled }}" - # Add helm annotations to all CRDS and skip the first 4 lines of crds.yaml - "$YQ_BIN_PATH" w -d'*' "$CEPH_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep | tail -n +5 + # Add helm annotations to all CRDS, remove empty lines in the output + # skip the comment lines of crds.yaml as well as the yaml doc header + "$YQ_BIN_PATH" eval-all '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$CEPH_CRDS_FILE_PATH" | tail -n +6 # DO NOT REMOVE the empty line, it is necessary echo "" @@ -91,11 +97,7 @@ fi generating_main_crd -for crd in "$OLM_CATALOG_DIR/"*.yaml; do - echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output doc separators - # Process each intermediate CRD file with yq to enforce consistent formatting in the final product - # regardless of whether yq was used in previous steps to alter CRD intermediate files. - $YQ_BIN_PATH read "$crd" >> "$CEPH_CRDS_FILE_PATH" -done +echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output the first doc separator +$YQ_BIN_PATH eval-all '.' "$OLM_CATALOG_DIR/"*.yaml >> "$CEPH_CRDS_FILE_PATH" build_helm_resources diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk index bfeb2ee65..05fda05e1 100644 --- a/build/makelib/golang.mk +++ b/build/makelib/golang.mk @@ -199,8 +199,7 @@ $(GOJUNIT): @$(GOHOST) clean -modcache export CONTROLLER_GEN=$(TOOLS_HOST_DIR)/controller-gen-$(CONTROLLER_GEN_VERSION) -export YQ=$(TOOLS_HOST_DIR)/yq-v3 -$(CONTROLLER_GEN) $(YQ): +$(CONTROLLER_GEN): { \ set -e ;\ mkdir -p $(TOOLS_HOST_DIR) ;\ @@ -213,12 +212,18 @@ $(CONTROLLER_GEN) $(YQ): echo === installing controller-gen ;\ go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION);\ mv $$CONTROLLER_GEN_TMP_DIR/controller-gen $(CONTROLLER_GEN) ;\ - echo === installing yq ;\ - go get github.com/mikefarah/yq/v3;\ - mv $$CONTROLLER_GEN_TMP_DIR/yq $(YQ) ;\ rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ } +YQ_VERSION = v4.14.2 +YQ := $(TOOLS_HOST_DIR)/yq-$(YQ_VERSION) +export YQ +$(YQ): + @echo === installing yq $(YQ_VERSION) $(REAL_HOST_PLATFORM) + @mkdir -p $(TOOLS_HOST_DIR) + @curl -JL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQ) + @chmod +x $(YQ) + export CODE_GENERATOR_VERSION=0.20.0 export CODE_GENERATOR=$(TOOLS_HOST_DIR)/code-generator-$(CODE_GENERATOR_VERSION) $(CODE_GENERATOR): diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index d54464426..d2b178a9e 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -8,6 +8,6 @@ pushd "$SCRIPT_DIR" ${HELM} template ../../deploy/charts/rook-ceph \ --namespace rook-ceph \ - --set crds.enabled=false | ./keep-rbac-yaml.py > rbac.yaml + --set crds.enabled=false | ./keep-rbac-yaml.sh > rbac.yaml popd diff --git a/build/rbac/keep-rbac-yaml.py b/build/rbac/keep-rbac-yaml.py deleted file mode 100755 index c63c18f46..000000000 --- a/build/rbac/keep-rbac-yaml.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -# Read any number of YAML documents from stdin, and output RBAC-related documents to stdout sorted -# by Kubernetes Kind then Name. - -import sys - -# ruamel.yaml is a small fork from the python standard yaml library that preserves comments -import ruamel.yaml - -# All the Kubernetes Kinds that we want to keep as RBAC -rbac_kinds = [ - "PodSecurityPolicy", - "ServiceAccount", - "ClusterRole", - "ClusterRoleBinding", - "Role", - "RoleBinding", -] - -# Log to stderr -def log(*values): - print(*values, file=sys.stderr, flush=True) - -# Return / for a Kubernetes resource from a yaml doc -def kind_and_name(doc): - return doc["kind"] + "/" + doc["metadata"]["name"] - - -# Set up and configure the yaml parser/dumper -yaml=ruamel.yaml.YAML() -# output lists in the form that is indented from the parent like below -# parent: -# - list -# - items -yaml.indent(sequence=4, offset=2) - -all_docs = yaml.load_all(sys.stdin.read()) - -kept_docs = [] -docs_processed = 0 -for doc in all_docs: - docs_processed += 1 - kind = doc["kind"] - if kind not in rbac_kinds: - # we don't want non-RBAC resources - log("discarding doc:", kind_and_name(doc)) - continue - log("keeping doc:", kind_and_name(doc)) - - # helm adds '# Source: ' comments to the top of each yaml doc. Strip these. - if doc.ca is not None and doc.ca.comment is not None: - comments = doc.ca.comment[1] - for comment in comments: - if comment.value.startswith("# Source: ") and comment.value.endswith(".yaml\n"): - log(" dropping comment:", comment.value.strip()) - comments.remove(comment) - - # helm-managed resources have a "chart" label, but we remove those for rendered RBAC - if "labels" in doc["metadata"] and "chart" in doc["metadata"]["labels"]: - log(" dropping 'chart' label") - del doc["metadata"]["labels"]["chart"] - - kept_docs.append(doc) - - -kept_docs.sort(key=kind_and_name) - -# Log to stderr the overall list of docs kept and a summary -for doc in kept_docs: - log(kind_and_name(doc)) -log("docs processed:", docs_processed) -log("docs kept :", len(kept_docs)) - - -# Dump to stdout (this should be the only time this script writes to stdout) -yaml.dump_all(kept_docs, sys.stdout) diff --git a/build/rbac/keep-rbac-yaml.sh b/build/rbac/keep-rbac-yaml.sh new file mode 100755 index 000000000..31ddb84d6 --- /dev/null +++ b/build/rbac/keep-rbac-yaml.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eEuo pipefail + +# READS FROM STDIN +# WRITES TO STDOUT +# DEBUGS TO STDERR + +: ${YQ:=yq} + +if [[ "$($YQ --version)" != "yq (https://github.com/mikefarah/yq/) version 4."* ]]; then + echo "yq must be version 4.x" + exit 1 +fi + +temp_dir="$(mktemp -d)" +pushd "${temp_dir}" &>/dev/stderr + +# Output the RBAC into separate temporary files named with Kind and Name so that the filesystem can +# sort the files, and we can keep the same resource ordering as before for easy diffing. Then we +# just read in the files, sorted by the fs for final output. + +$YQ eval ' + select(.kind == "PodSecurityPolicy"), + select(.kind == "ServiceAccount"), + select(.kind == "ClusterRole"), + select(.kind == "ClusterRoleBinding"), + select(.kind == "Role"), + select(.kind == "RoleBinding") + ' - | # select all RBAC resource Kinds +$YQ eval 'del(.metadata.labels.chart)' - | # remove the 'chart' label that only applies to Helm-managed resources +sed '/^$/d' | # remove empty lines caused by yq's display of header/footer comments +sed '/^# Source: /d' | # helm adds '# Source: ' comments atop of each yaml doc. Strip these +$YQ eval --split-exp '.kind + " " + .metadata.name + " "' - # split into files by .yaml +# outputting the filenames with spaces after kind and name keeps the same sorting from before + +# For debugging, output the resource kinds and names we processed and the number we are keeping +for file in *.yml; do + echo "${file%.yml}" >/dev/stderr +done +# shellcheck disable=SC2012 # we know filenames are alphanumeric from being k8s resources +echo "Number of RBAC resources: $(ls "${temp_dir}" | wc -l)" >/dev/stderr + +$YQ eval-all '.' ./*.yml | # output all files, now sorted by Kind and Name by the fs +sed '/^$/d' # remove empty lines caused by yq's display of header/footer comments + +rm -rf "${temp_dir}" +popd &>/dev/stderr diff --git a/build/rbac/rbac.yaml b/build/rbac/rbac.yaml index 611d7a961..2095060f2 100644 --- a/build/rbac/rbac.yaml +++ b/build/rbac/rbac.yaml @@ -1,76 +1,77 @@ +--- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin rules: - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, update] - - apiGroups: [''] - resources: [namespaces] - verbs: [get, list] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner rules: - - apiGroups: [''] - resources: [secrets] - verbs: [get, list] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, create, delete, update, patch] - - apiGroups: [''] - resources: [persistentvolumeclaims] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [storageclasses] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [events] - verbs: [list, watch, create, update, patch] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update, patch] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments/status] - verbs: [patch] - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [persistentvolumeclaims/status] - verbs: [update, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots] - verbs: [get, list, watch, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents] - verbs: [create, get, list, watch, update, delete] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotclasses] - verbs: [get, list, watch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents/status] - verbs: [update] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots/status] - verbs: [update] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: psp:rook + name: 'psp:rook' labels: operator: rook storage-backend: ceph @@ -89,93 +90,93 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin rules: - - apiGroups: [''] - resources: [secrets] - verbs: [get, list] - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, update] - - apiGroups: [''] - resources: [namespaces] - verbs: [get, list] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list] - - apiGroups: [''] - resources: [serviceaccounts] - verbs: [get] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner rules: - - apiGroups: [''] - resources: [secrets] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, create, delete, update, patch] - - apiGroups: [''] - resources: [persistentvolumeclaims] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update, patch] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments/status] - verbs: [patch] - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, watch] - - apiGroups: [storage.k8s.io] - resources: [storageclasses] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [events] - verbs: [list, watch, create, update, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots] - verbs: [get, list, watch, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents] - verbs: [create, get, list, watch, update, delete] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotclasses] - verbs: [get, list, watch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents/status] - verbs: [update] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots/status] - verbs: [update] - - apiGroups: [''] - resources: [persistentvolumeclaims/status] - verbs: [update, patch] - - apiGroups: [''] - resources: [configmaps] - verbs: [get] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplications, volumereplicationclasses] - verbs: [create, delete, get, list, patch, update, watch] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplications/finalizers] - verbs: [update] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplications/status] - verbs: [get, patch, update] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplicationclasses/status] - verbs: [get] - - apiGroups: [''] - resources: [serviceaccounts] - verbs: [get] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications", "volumereplicationclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/finalizers"] + verbs: ["update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplicationclasses/status"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- # The cluster role for managing all the cluster-specific resources in a namespace apiVersion: rbac.authorization.k8s.io/v1 @@ -187,7 +188,7 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" - apps - extensions resources: @@ -219,28 +220,28 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" resources: - # Pod access is needed for fencing + # Pod access is needed for fencing - pods - # Node access is needed for determining nodes where mons should run + # Node access is needed for determining nodes where mons should run - nodes - nodes/proxy - services - # Rook watches secrets which it uses to configure access to external resources. - # e.g., external Ceph cluster; TLS certificates for the admission controller or object store + # Rook watches secrets which it uses to configure access to external resources. + # e.g., external Ceph cluster; TLS certificates for the admission controller or object store - secrets - # Rook watches for changes to the rook-operator-config configmap + # Rook watches for changes to the rook-operator-config configmap - configmaps verbs: - get - list - watch - apiGroups: - - '' + - "" resources: - events - # PVs and PVCs are managed by the Rook provisioner + # PVs and PVCs are managed by the Rook provisioner - persistentvolumes - persistentvolumeclaims - endpoints @@ -275,27 +276,27 @@ rules: - apiGroups: - ceph.rook.io resources: - - '*' + - "*" verbs: - - '*' + - "*" - apiGroups: - rook.io resources: - - '*' + - "*" verbs: - - '*' + - "*" - apiGroups: - policy - apps - extensions resources: - # This is for the clusterdisruption controller + # This is for the clusterdisruption controller - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers + # This is for both clusterdisruption and nodedrain controllers - deployments - replicasets verbs: - - '*' + - "*" - apiGroups: - healthchecking.openshift.io resources: @@ -344,7 +345,7 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" resources: - configmaps - nodes @@ -355,7 +356,7 @@ rules: - list - watch - apiGroups: - - '' + - "" resources: - events verbs: @@ -380,7 +381,7 @@ metadata: name: rook-ceph-mgr-system rules: - apiGroups: - - '' + - "" resources: - configmaps verbs: @@ -399,8 +400,8 @@ metadata: operator: rook storage-backend: ceph rules: - - apiGroups: [''] - resources: [secrets, configmaps] + - apiGroups: [""] + resources: ["secrets", "configmaps"] verbs: # OBC controller creates secrets and configmaps containing information for users about how to # connect to object buckets. It deletes them when an OBC is deleted. @@ -408,14 +409,14 @@ rules: - create - update - delete - - apiGroups: [storage.k8s.io] - resources: [storageclasses] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] verbs: # OBC controller gets parameters from the OBC's storageclass # Rook gets additional parameters from the OBC's storageclass - get - - apiGroups: [objectbucket.io] - resources: [objectbucketclaims] + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims"] verbs: # OBC controller needs to list/watch OBCs and get latest version of a reconciled OBC - list @@ -425,8 +426,8 @@ rules: # information outside of the status subresource - update # OBC controller does not delete OBCs; users do this - - apiGroups: [objectbucket.io] - resources: [objectbuckets] + - apiGroups: ["objectbucket.io"] + resources: ["objectbuckets"] verbs: # OBC controller needs to list/watch OBs and get latest version of a reconciled OB - list @@ -437,8 +438,8 @@ rules: - create - update - delete - - apiGroups: [objectbucket.io] - resources: [objectbucketclaims/status, objectbuckets/status] + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims/status", "objectbuckets/status"] verbs: # OBC controller updates OBC and OB statuses - update @@ -449,7 +450,7 @@ metadata: name: rook-ceph-osd rules: - apiGroups: - - '' + - "" resources: - nodes verbs: @@ -468,12 +469,12 @@ rules: # However, some Kubernetes APIs involve a "subresource", such as the logs for a pod. [...] # To represent this in an RBAC role, use a slash to delimit the resource and subresource. # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources - - apiGroups: [''] - resources: [pods, pods/log] - verbs: [get, list] - - apiGroups: [''] - resources: [pods/exec] - verbs: [create] + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -612,7 +613,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-ceph-system @@ -625,7 +626,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-cephfs-plugin-sa @@ -638,7 +639,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-cephfs-provisioner-sa @@ -651,7 +652,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-rbd-plugin-sa @@ -664,7 +665,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-rbd-provisioner-sa @@ -685,8 +686,8 @@ kind: PodSecurityPolicy metadata: name: 00-rook-privileged annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: runtime/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: runtime/default + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: true allowedCapabilities: @@ -753,15 +754,15 @@ metadata: name: cephfs-external-provisioner-cfg namespace: rook-ceph # namespace:operator rules: - - apiGroups: [''] - resources: [endpoints] - verbs: [get, watch, list, delete, update, create] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list, create, delete] - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: [get, watch, list, delete, update, create] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -769,15 +770,15 @@ metadata: name: rbd-external-provisioner-cfg namespace: rook-ceph # namespace:operator rules: - - apiGroups: [''] - resources: [endpoints] - verbs: [get, watch, list, delete, update, create] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list, watch, create, delete, update] - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: [get, watch, list, delete, update, create] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -786,7 +787,7 @@ metadata: namespace: rook-ceph # namespace:cluster rules: - apiGroups: - - '' + - "" resources: - pods - configmaps @@ -806,7 +807,7 @@ metadata: namespace: rook-ceph # namespace:cluster rules: - apiGroups: - - '' + - "" resources: - pods - services @@ -832,9 +833,9 @@ rules: - apiGroups: - ceph.rook.io resources: - - '*' + - "*" verbs: - - '*' + - "*" - apiGroups: - apps resources: @@ -856,12 +857,12 @@ metadata: name: rook-ceph-osd namespace: rook-ceph # namespace:cluster rules: - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list, watch, create, update, delete] - - apiGroups: [ceph.rook.io] - resources: [cephclusters, cephclusters/finalizers] - verbs: [get, list, create, update, delete] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["ceph.rook.io"] + resources: ["cephclusters", "cephclusters/finalizers"] + verbs: ["get", "list", "create", "update", "delete"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -869,18 +870,18 @@ metadata: name: rook-ceph-purge-osd namespace: rook-ceph # namespace:cluster rules: - - apiGroups: [''] - resources: [configmaps] - verbs: [get] - - apiGroups: [apps] - resources: [deployments] - verbs: [get, delete] - - apiGroups: [batch] - resources: [jobs] - verbs: [get, list, delete] - - apiGroups: [''] - resources: [persistentvolumeclaims] - verbs: [get, update, delete] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "update", "delete"] --- # Allow the operator to manage resources in its own namespace apiVersion: rbac.authorization.k8s.io/v1 @@ -893,7 +894,7 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" resources: - pods - configmaps diff --git a/deploy/olm/generate-rook-csv.sh b/deploy/olm/generate-rook-csv.sh index 82d6eea38..e59162b91 100755 --- a/deploy/olm/generate-rook-csv.sh +++ b/deploy/olm/generate-rook-csv.sh @@ -13,7 +13,7 @@ PACKAGE_FILE="$OLM_CATALOG_DIR/assemble/rook-ceph.package.yaml" SUPPORTED_PLATFORMS='k8s|ocp|okd' operator_sdk="${OPERATOR_SDK:-operator-sdk}" -yq="${YQ:-yq}" +yq="${YQv3:-yq}" # Default CSI to true : "${OLM_INCLUDE_CEPHFS_CSI:=true}" @@ -35,6 +35,11 @@ if ! command -v "$yq" >/dev/null && [ ! -f "$yq" ]; then exit 1 fi +if [[ "$("$yq" --version)" != "yq version 3."* ]]; then + echo "yq must be version 3" >/dev/stderr + exit 1 +fi + if [[ -z "$1" ]]; then echo "Please provide a version, e.g:" echo "" diff --git a/images/ceph/Makefile b/images/ceph/Makefile index bf1a2dfc3..978d7335b 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -27,7 +27,7 @@ BASEIMAGE = $(REGISTRY_NAME)/ceph/ceph-$(GOARCH):$(CEPH_VERSION) CEPH_IMAGE = $(BUILD_REGISTRY)/ceph-$(GOARCH) OPERATOR_SDK_VERSION = v0.17.1 # TODO: update to yq v4 - v3 end of life in Aug 2021 ; v4 removes the 'yq delete' cmd and changes syntax -YQ_VERSION = 3.3.0 +YQv3_VERSION = 3.3.0 GOHOST := GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) go MANIFESTS_DIR=../../deploy/examples @@ -65,8 +65,8 @@ VOLUME_REPLICATION_CRD = replication.storage.openshift.io_volumereplications.yam VOLUME_REPLICATION_CLASS_CRD = replication.storage.openshift.io_volumereplicationclasses.yaml OPERATOR_SDK := $(TOOLS_HOST_DIR)/operator-sdk-$(OPERATOR_SDK_VERSION) -YQ := $(TOOLS_HOST_DIR)/yq-$(YQ_VERSION) -export OPERATOR_SDK YQ +YQv3 := $(TOOLS_HOST_DIR)/yq-$(YQv3_VERSION) +export OPERATOR_SDK YQv3 # ==================================================================================== # Build Rook @@ -101,7 +101,7 @@ endif # generate CSV template files into the directory defined by the env var CSV_TEMPLATE_DIR # CSV_TEMPLATE_DIR will be created if it doesn't already exist -generate-csv-templates: $(OPERATOR_SDK) $(YQ) ## Generate CSV templates for OLM into CSV_TEMPLATE_DIR +generate-csv-templates: $(OPERATOR_SDK) $(YQv3) ## Generate CSV templates for OLM into CSV_TEMPLATE_DIR @if [[ -z "$(CSV_TEMPLATE_DIR)" ]]; then echo "CSV_TEMPLATE_DIR is not set"; exit 1; fi @# first, copy the existing CRDs and OLM catalog directory to CSV_TEMPLATE_DIR @# then, generate or copy all prerequisites into CSV_TEMPLATE_DIR (e.g., CRDs) @@ -130,11 +130,11 @@ get-volume-replication-crds: fi @cp $(CACHE_DIR)/crds/* $(CRD_TEMPLATE_DIR) -$(YQ): - @echo === installing yq $(REAL_HOST_PLATFORM) +$(YQv3): + @echo === installing yq $(YQv3_VERSION) $(REAL_HOST_PLATFORM) @mkdir -p $(TOOLS_HOST_DIR) - @curl -JL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQ) - @chmod +x $(YQ) + @curl -JL https://github.com/mikefarah/yq/releases/download/$(YQv3_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQv3) + @chmod +x $(YQv3) $(OPERATOR_SDK): @echo === installing operator-sdk $(REAL_HOST_PLATFORM) @@ -144,11 +144,11 @@ $(OPERATOR_SDK): @chmod +x $(OPERATOR_SDK) @$(OPERATOR_SDK) version -csv: $(OPERATOR_SDK) $(YQ) ## Generate a CSV file for OLM. +csv: $(OPERATOR_SDK) $(YQv3) ## Generate a CSV file for OLM. @echo Generating CSV manifests @cd ../.. && deploy/olm/generate-rook-csv.sh $(CSV_VERSION) $(CSV_PLATFORM) $(ROOK_OP_VERSION) -csv-clean: $(OPERATOR_SDK) $(YQ) ## Remove existing OLM files. +csv-clean: ## Remove existing OLM files. @rm -fr ../../deploy/olm/deploy/* ../../deploy/olm/templates/* # reading from a file and outputting to the same file can have undefined results, so use this intermediate From 51fc36d59142e83837d103c5d5d3f6af9fcd4f2e Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Mon, 29 Nov 2021 11:26:38 -0700 Subject: [PATCH 2/2] build: prevent parallel build collisions for ceph Prevent parallel build collisions in the 'ceph' image by building prerequisites for the builds before running the build targets in parallel. Build targets can collide and try to create the same target at nearly the same time, causing failures. Signed-off-by: Blaine Gardner --- Makefile | 2 +- images/Makefile | 4 +++- images/ceph/Makefile | 11 +++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5e52acafd..9675d6a9a 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ build.version: @mkdir -p $(OUTPUT_DIR) @echo "$(VERSION)" > $(OUTPUT_DIR)/version -build.common: build.version helm.build mod.check +build.common: build.version helm.build mod.check crds @$(MAKE) go.init @$(MAKE) go.validate @$(MAKE) -C images/ceph list-image diff --git a/images/Makefile b/images/Makefile index 040e9d3f9..d3d153a5f 100644 --- a/images/Makefile +++ b/images/Makefile @@ -30,7 +30,9 @@ ceph.%: @$(MAKE) -C ceph PLATFORM=$* do.build: ceph.$(PLATFORM) ; -build.all: $(foreach p,$(PLATFORMS), ceph.$(p)) ; ## Build images for all platforms. +build.all: ## Build images for all platforms. + $(MAKE) -C ceph prerequisites + $(MAKE) $(foreach p,$(PLATFORMS), ceph.$(p)) ; # ==================================================================================== # Help diff --git a/images/ceph/Makefile b/images/ceph/Makefile index 978d7335b..2d71cf7c5 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -84,7 +84,8 @@ do.build: ifeq ($(INCLUDE_CSV_TEMPLATES),true) @$(MAKE) CSV_TEMPLATE_DIR=$(TEMP) generate-csv-templates - @$(MAKE) CRD_TEMPLATE_DIR=$(TEMP)/deploy/olm/templates/crds/ get-volume-replication-crds + @$(MAKE) get-volume-replication-crds + cp $(CACHE_DIR)/crds/* $(TEMP)/deploy/olm/templates/crds/. @cp -r $(TEMP)/deploy/olm/templates $(TEMP)/ceph-csv-templates else mkdir $(TEMP)/ceph-csv-templates @@ -99,6 +100,10 @@ endif fi @rm -fr $(TEMP) +# call this before building multiple arches in parallel to prevent parallel build processes from +# conflicting +prerequisites: $(OPERATOR_SDK) $(YQv3) get-volume-replication-crds + # generate CSV template files into the directory defined by the env var CSV_TEMPLATE_DIR # CSV_TEMPLATE_DIR will be created if it doesn't already exist generate-csv-templates: $(OPERATOR_SDK) $(YQv3) ## Generate CSV templates for OLM into CSV_TEMPLATE_DIR @@ -122,13 +127,11 @@ generate-csv-templates: $(OPERATOR_SDK) $(YQv3) ## Generate CSV templates for OL @echo " === Generated CSV templates can be found at $(CSV_TEMPLATE_DIR)/deploy/olm/templates" get-volume-replication-crds: - @if [[ -z "$(CRD_TEMPLATE_DIR)" ]]; then echo "CRD_TEMPLATE_DIR is not set"; exit 1; fi - @if [[ ! -d "$(CACHE_DIR)/crds" ]]; then\ + if [[ ! -d "$(CACHE_DIR)/crds" ]]; then\ mkdir -p $(CACHE_DIR)/crds;\ curl -L $(VOL_REPL_URL)/$(VOLUME_REPLICATION_CRD) -o $(CACHE_DIR)/crds/$(VOLUME_REPLICATION_CRD);\ curl -L $(VOL_REPL_URL)/$(VOLUME_REPLICATION_CLASS_CRD) -o $(CACHE_DIR)/crds/$(VOLUME_REPLICATION_CLASS_CRD);\ fi - @cp $(CACHE_DIR)/crds/* $(CRD_TEMPLATE_DIR) $(YQv3): @echo === installing yq $(YQv3_VERSION) $(REAL_HOST_PLATFORM)