diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 199813572..beb6c51b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,17 +54,6 @@ jobs: working-directory: /Users/runner/go/src/github.com/rook/rook run: tests/scripts/validate_modified_files.sh crd - # needed for gen-rbac - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - - name: Install pip dependencies - run: | - python -m pip install --upgrade pip - pip install ruamel.yaml==0.17.16 - - name: run gen-rbac working-directory: /Users/runner/go/src/github.com/rook/rook run: GOPATH=$(go env GOPATH) make gen-rbac diff --git a/.github/workflows/rbac-gen.yaml b/.github/workflows/rbac-gen.yaml index 3c31fb437..7b8fff777 100644 --- a/.github/workflows/rbac-gen.yaml +++ b/.github/workflows/rbac-gen.yaml @@ -32,16 +32,6 @@ jobs: - name: copy working directory to GOPATH run: sudo mkdir -p /home/runner/go/src/github.com && sudo cp -a /home/runner/work/rook /home/runner/go/src/github.com/ - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - - name: Install pip dependencies - run: | - python -m pip install --upgrade pip - pip install ruamel.yaml==0.17.16 - - name: run gen-rbac working-directory: /home/runner/go/src/github.com/rook/rook run: GOPATH=$(go env GOPATH) make gen-rbac diff --git a/Makefile b/Makefile index 441482055..9675d6a9a 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ build.version: @mkdir -p $(OUTPUT_DIR) @echo "$(VERSION)" > $(OUTPUT_DIR)/version -build.common: build.version helm.build mod.check +build.common: build.version helm.build mod.check crds @$(MAKE) go.init @$(MAKE) go.validate @$(MAKE) -C images/ceph list-image @@ -183,7 +183,7 @@ crds: $(CONTROLLER_GEN) $(YQ) @echo Updating CRD manifests @build/crds/build-crds.sh $(CONTROLLER_GEN) $(YQ) -gen-rbac: $(HELM) ## generate RBAC from Helm charts +gen-rbac: $(HELM) $(YQ) ## generate RBAC from Helm charts HELM=$(HELM) ./build/rbac/get-helm-rbac.sh .PHONY: all build.common cross.build.parallel diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 898904763..58a064254 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -36,6 +36,11 @@ OLM_CATALOG_DIR="${DESTINATION_ROOT}/deploy/olm/deploy/crds" CEPH_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/examples/crds.yaml" CEPH_HELM_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/charts/rook-ceph/templates/resources.yaml" +if [[ "$($YQ_BIN_PATH --version)" != "yq (https://github.com/mikefarah/yq/) version 4."* ]]; then + echo "yq must be version 4.x" + exit 1 +fi + ############# # FUNCTIONS # ############# @@ -50,7 +55,7 @@ generating_crds_v1() { echo "Generating ceph crds" "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/ceph.rook.io/v1" output:crd:artifacts:config="$OLM_CATALOG_DIR" # the csv upgrade is failing on the volumeClaimTemplate.metadata.annotations.crushDeviceClass unless we preserve the annotations as an unknown field - $YQ_BIN_PATH w -i "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields true + $YQ_BIN_PATH eval --inplace '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields = true' "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml } generating_main_crd() { @@ -70,8 +75,9 @@ build_helm_resources() { # add header echo "{{- if .Values.crds.enabled }}" - # Add helm annotations to all CRDS and skip the first 4 lines of crds.yaml - "$YQ_BIN_PATH" w -d'*' "$CEPH_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep | tail -n +5 + # Add helm annotations to all CRDS, remove empty lines in the output + # skip the comment lines of crds.yaml as well as the yaml doc header + "$YQ_BIN_PATH" eval-all '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$CEPH_CRDS_FILE_PATH" | tail -n +6 # DO NOT REMOVE the empty line, it is necessary echo "" @@ -91,11 +97,7 @@ fi generating_main_crd -for crd in "$OLM_CATALOG_DIR/"*.yaml; do - echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output doc separators - # Process each intermediate CRD file with yq to enforce consistent formatting in the final product - # regardless of whether yq was used in previous steps to alter CRD intermediate files. - $YQ_BIN_PATH read "$crd" >> "$CEPH_CRDS_FILE_PATH" -done +echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output the first doc separator +$YQ_BIN_PATH eval-all '.' "$OLM_CATALOG_DIR/"*.yaml >> "$CEPH_CRDS_FILE_PATH" build_helm_resources diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk index bfeb2ee65..05fda05e1 100644 --- a/build/makelib/golang.mk +++ b/build/makelib/golang.mk @@ -199,8 +199,7 @@ $(GOJUNIT): @$(GOHOST) clean -modcache export CONTROLLER_GEN=$(TOOLS_HOST_DIR)/controller-gen-$(CONTROLLER_GEN_VERSION) -export YQ=$(TOOLS_HOST_DIR)/yq-v3 -$(CONTROLLER_GEN) $(YQ): +$(CONTROLLER_GEN): { \ set -e ;\ mkdir -p $(TOOLS_HOST_DIR) ;\ @@ -213,12 +212,18 @@ $(CONTROLLER_GEN) $(YQ): echo === installing controller-gen ;\ go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION);\ mv $$CONTROLLER_GEN_TMP_DIR/controller-gen $(CONTROLLER_GEN) ;\ - echo === installing yq ;\ - go get github.com/mikefarah/yq/v3;\ - mv $$CONTROLLER_GEN_TMP_DIR/yq $(YQ) ;\ rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ } +YQ_VERSION = v4.14.2 +YQ := $(TOOLS_HOST_DIR)/yq-$(YQ_VERSION) +export YQ +$(YQ): + @echo === installing yq $(YQ_VERSION) $(REAL_HOST_PLATFORM) + @mkdir -p $(TOOLS_HOST_DIR) + @curl -JL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQ) + @chmod +x $(YQ) + export CODE_GENERATOR_VERSION=0.20.0 export CODE_GENERATOR=$(TOOLS_HOST_DIR)/code-generator-$(CODE_GENERATOR_VERSION) $(CODE_GENERATOR): diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index d54464426..d2b178a9e 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -8,6 +8,6 @@ pushd "$SCRIPT_DIR" ${HELM} template ../../deploy/charts/rook-ceph \ --namespace rook-ceph \ - --set crds.enabled=false | ./keep-rbac-yaml.py > rbac.yaml + --set crds.enabled=false | ./keep-rbac-yaml.sh > rbac.yaml popd diff --git a/build/rbac/keep-rbac-yaml.py b/build/rbac/keep-rbac-yaml.py deleted file mode 100755 index c63c18f46..000000000 --- a/build/rbac/keep-rbac-yaml.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -# Read any number of YAML documents from stdin, and output RBAC-related documents to stdout sorted -# by Kubernetes Kind then Name. - -import sys - -# ruamel.yaml is a small fork from the python standard yaml library that preserves comments -import ruamel.yaml - -# All the Kubernetes Kinds that we want to keep as RBAC -rbac_kinds = [ - "PodSecurityPolicy", - "ServiceAccount", - "ClusterRole", - "ClusterRoleBinding", - "Role", - "RoleBinding", -] - -# Log to stderr -def log(*values): - print(*values, file=sys.stderr, flush=True) - -# Return / for a Kubernetes resource from a yaml doc -def kind_and_name(doc): - return doc["kind"] + "/" + doc["metadata"]["name"] - - -# Set up and configure the yaml parser/dumper -yaml=ruamel.yaml.YAML() -# output lists in the form that is indented from the parent like below -# parent: -# - list -# - items -yaml.indent(sequence=4, offset=2) - -all_docs = yaml.load_all(sys.stdin.read()) - -kept_docs = [] -docs_processed = 0 -for doc in all_docs: - docs_processed += 1 - kind = doc["kind"] - if kind not in rbac_kinds: - # we don't want non-RBAC resources - log("discarding doc:", kind_and_name(doc)) - continue - log("keeping doc:", kind_and_name(doc)) - - # helm adds '# Source: ' comments to the top of each yaml doc. Strip these. - if doc.ca is not None and doc.ca.comment is not None: - comments = doc.ca.comment[1] - for comment in comments: - if comment.value.startswith("# Source: ") and comment.value.endswith(".yaml\n"): - log(" dropping comment:", comment.value.strip()) - comments.remove(comment) - - # helm-managed resources have a "chart" label, but we remove those for rendered RBAC - if "labels" in doc["metadata"] and "chart" in doc["metadata"]["labels"]: - log(" dropping 'chart' label") - del doc["metadata"]["labels"]["chart"] - - kept_docs.append(doc) - - -kept_docs.sort(key=kind_and_name) - -# Log to stderr the overall list of docs kept and a summary -for doc in kept_docs: - log(kind_and_name(doc)) -log("docs processed:", docs_processed) -log("docs kept :", len(kept_docs)) - - -# Dump to stdout (this should be the only time this script writes to stdout) -yaml.dump_all(kept_docs, sys.stdout) diff --git a/build/rbac/keep-rbac-yaml.sh b/build/rbac/keep-rbac-yaml.sh new file mode 100755 index 000000000..31ddb84d6 --- /dev/null +++ b/build/rbac/keep-rbac-yaml.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eEuo pipefail + +# READS FROM STDIN +# WRITES TO STDOUT +# DEBUGS TO STDERR + +: ${YQ:=yq} + +if [[ "$($YQ --version)" != "yq (https://github.com/mikefarah/yq/) version 4."* ]]; then + echo "yq must be version 4.x" + exit 1 +fi + +temp_dir="$(mktemp -d)" +pushd "${temp_dir}" &>/dev/stderr + +# Output the RBAC into separate temporary files named with Kind and Name so that the filesystem can +# sort the files, and we can keep the same resource ordering as before for easy diffing. Then we +# just read in the files, sorted by the fs for final output. + +$YQ eval ' + select(.kind == "PodSecurityPolicy"), + select(.kind == "ServiceAccount"), + select(.kind == "ClusterRole"), + select(.kind == "ClusterRoleBinding"), + select(.kind == "Role"), + select(.kind == "RoleBinding") + ' - | # select all RBAC resource Kinds +$YQ eval 'del(.metadata.labels.chart)' - | # remove the 'chart' label that only applies to Helm-managed resources +sed '/^$/d' | # remove empty lines caused by yq's display of header/footer comments +sed '/^# Source: /d' | # helm adds '# Source: ' comments atop of each yaml doc. Strip these +$YQ eval --split-exp '.kind + " " + .metadata.name + " "' - # split into files by .yaml +# outputting the filenames with spaces after kind and name keeps the same sorting from before + +# For debugging, output the resource kinds and names we processed and the number we are keeping +for file in *.yml; do + echo "${file%.yml}" >/dev/stderr +done +# shellcheck disable=SC2012 # we know filenames are alphanumeric from being k8s resources +echo "Number of RBAC resources: $(ls "${temp_dir}" | wc -l)" >/dev/stderr + +$YQ eval-all '.' ./*.yml | # output all files, now sorted by Kind and Name by the fs +sed '/^$/d' # remove empty lines caused by yq's display of header/footer comments + +rm -rf "${temp_dir}" +popd &>/dev/stderr diff --git a/build/rbac/rbac.yaml b/build/rbac/rbac.yaml index 611d7a961..2095060f2 100644 --- a/build/rbac/rbac.yaml +++ b/build/rbac/rbac.yaml @@ -1,76 +1,77 @@ +--- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin rules: - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, update] - - apiGroups: [''] - resources: [namespaces] - verbs: [get, list] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner rules: - - apiGroups: [''] - resources: [secrets] - verbs: [get, list] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, create, delete, update, patch] - - apiGroups: [''] - resources: [persistentvolumeclaims] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [storageclasses] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [events] - verbs: [list, watch, create, update, patch] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update, patch] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments/status] - verbs: [patch] - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [persistentvolumeclaims/status] - verbs: [update, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots] - verbs: [get, list, watch, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents] - verbs: [create, get, list, watch, update, delete] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotclasses] - verbs: [get, list, watch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents/status] - verbs: [update] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots/status] - verbs: [update] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: psp:rook + name: 'psp:rook' labels: operator: rook storage-backend: ceph @@ -89,93 +90,93 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin rules: - - apiGroups: [''] - resources: [secrets] - verbs: [get, list] - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, update] - - apiGroups: [''] - resources: [namespaces] - verbs: [get, list] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list] - - apiGroups: [''] - resources: [serviceaccounts] - verbs: [get] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner rules: - - apiGroups: [''] - resources: [secrets] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [persistentvolumes] - verbs: [get, list, watch, create, delete, update, patch] - - apiGroups: [''] - resources: [persistentvolumeclaims] - verbs: [get, list, watch, update] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments] - verbs: [get, list, watch, update, patch] - - apiGroups: [storage.k8s.io] - resources: [volumeattachments/status] - verbs: [patch] - - apiGroups: [''] - resources: [nodes] - verbs: [get, list, watch] - - apiGroups: [storage.k8s.io] - resources: [storageclasses] - verbs: [get, list, watch] - - apiGroups: [''] - resources: [events] - verbs: [list, watch, create, update, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots] - verbs: [get, list, watch, patch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents] - verbs: [create, get, list, watch, update, delete] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotclasses] - verbs: [get, list, watch] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshotcontents/status] - verbs: [update] - - apiGroups: [snapshot.storage.k8s.io] - resources: [volumesnapshots/status] - verbs: [update] - - apiGroups: [''] - resources: [persistentvolumeclaims/status] - verbs: [update, patch] - - apiGroups: [''] - resources: [configmaps] - verbs: [get] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplications, volumereplicationclasses] - verbs: [create, delete, get, list, patch, update, watch] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplications/finalizers] - verbs: [update] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplications/status] - verbs: [get, patch, update] - - apiGroups: [replication.storage.openshift.io] - resources: [volumereplicationclasses/status] - verbs: [get] - - apiGroups: [''] - resources: [serviceaccounts] - verbs: [get] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications", "volumereplicationclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/finalizers"] + verbs: ["update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplicationclasses/status"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- # The cluster role for managing all the cluster-specific resources in a namespace apiVersion: rbac.authorization.k8s.io/v1 @@ -187,7 +188,7 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" - apps - extensions resources: @@ -219,28 +220,28 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" resources: - # Pod access is needed for fencing + # Pod access is needed for fencing - pods - # Node access is needed for determining nodes where mons should run + # Node access is needed for determining nodes where mons should run - nodes - nodes/proxy - services - # Rook watches secrets which it uses to configure access to external resources. - # e.g., external Ceph cluster; TLS certificates for the admission controller or object store + # Rook watches secrets which it uses to configure access to external resources. + # e.g., external Ceph cluster; TLS certificates for the admission controller or object store - secrets - # Rook watches for changes to the rook-operator-config configmap + # Rook watches for changes to the rook-operator-config configmap - configmaps verbs: - get - list - watch - apiGroups: - - '' + - "" resources: - events - # PVs and PVCs are managed by the Rook provisioner + # PVs and PVCs are managed by the Rook provisioner - persistentvolumes - persistentvolumeclaims - endpoints @@ -275,27 +276,27 @@ rules: - apiGroups: - ceph.rook.io resources: - - '*' + - "*" verbs: - - '*' + - "*" - apiGroups: - rook.io resources: - - '*' + - "*" verbs: - - '*' + - "*" - apiGroups: - policy - apps - extensions resources: - # This is for the clusterdisruption controller + # This is for the clusterdisruption controller - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers + # This is for both clusterdisruption and nodedrain controllers - deployments - replicasets verbs: - - '*' + - "*" - apiGroups: - healthchecking.openshift.io resources: @@ -344,7 +345,7 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" resources: - configmaps - nodes @@ -355,7 +356,7 @@ rules: - list - watch - apiGroups: - - '' + - "" resources: - events verbs: @@ -380,7 +381,7 @@ metadata: name: rook-ceph-mgr-system rules: - apiGroups: - - '' + - "" resources: - configmaps verbs: @@ -399,8 +400,8 @@ metadata: operator: rook storage-backend: ceph rules: - - apiGroups: [''] - resources: [secrets, configmaps] + - apiGroups: [""] + resources: ["secrets", "configmaps"] verbs: # OBC controller creates secrets and configmaps containing information for users about how to # connect to object buckets. It deletes them when an OBC is deleted. @@ -408,14 +409,14 @@ rules: - create - update - delete - - apiGroups: [storage.k8s.io] - resources: [storageclasses] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] verbs: # OBC controller gets parameters from the OBC's storageclass # Rook gets additional parameters from the OBC's storageclass - get - - apiGroups: [objectbucket.io] - resources: [objectbucketclaims] + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims"] verbs: # OBC controller needs to list/watch OBCs and get latest version of a reconciled OBC - list @@ -425,8 +426,8 @@ rules: # information outside of the status subresource - update # OBC controller does not delete OBCs; users do this - - apiGroups: [objectbucket.io] - resources: [objectbuckets] + - apiGroups: ["objectbucket.io"] + resources: ["objectbuckets"] verbs: # OBC controller needs to list/watch OBs and get latest version of a reconciled OB - list @@ -437,8 +438,8 @@ rules: - create - update - delete - - apiGroups: [objectbucket.io] - resources: [objectbucketclaims/status, objectbuckets/status] + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims/status", "objectbuckets/status"] verbs: # OBC controller updates OBC and OB statuses - update @@ -449,7 +450,7 @@ metadata: name: rook-ceph-osd rules: - apiGroups: - - '' + - "" resources: - nodes verbs: @@ -468,12 +469,12 @@ rules: # However, some Kubernetes APIs involve a "subresource", such as the logs for a pod. [...] # To represent this in an RBAC role, use a slash to delimit the resource and subresource. # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources - - apiGroups: [''] - resources: [pods, pods/log] - verbs: [get, list] - - apiGroups: [''] - resources: [pods/exec] - verbs: [create] + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -612,7 +613,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-ceph-system @@ -625,7 +626,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-cephfs-plugin-sa @@ -638,7 +639,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-cephfs-provisioner-sa @@ -651,7 +652,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-rbd-plugin-sa @@ -664,7 +665,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: psp:rook + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-rbd-provisioner-sa @@ -685,8 +686,8 @@ kind: PodSecurityPolicy metadata: name: 00-rook-privileged annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: runtime/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: runtime/default + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: true allowedCapabilities: @@ -753,15 +754,15 @@ metadata: name: cephfs-external-provisioner-cfg namespace: rook-ceph # namespace:operator rules: - - apiGroups: [''] - resources: [endpoints] - verbs: [get, watch, list, delete, update, create] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list, create, delete] - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: [get, watch, list, delete, update, create] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -769,15 +770,15 @@ metadata: name: rbd-external-provisioner-cfg namespace: rook-ceph # namespace:operator rules: - - apiGroups: [''] - resources: [endpoints] - verbs: [get, watch, list, delete, update, create] - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list, watch, create, delete, update] - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: [get, watch, list, delete, update, create] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -786,7 +787,7 @@ metadata: namespace: rook-ceph # namespace:cluster rules: - apiGroups: - - '' + - "" resources: - pods - configmaps @@ -806,7 +807,7 @@ metadata: namespace: rook-ceph # namespace:cluster rules: - apiGroups: - - '' + - "" resources: - pods - services @@ -832,9 +833,9 @@ rules: - apiGroups: - ceph.rook.io resources: - - '*' + - "*" verbs: - - '*' + - "*" - apiGroups: - apps resources: @@ -856,12 +857,12 @@ metadata: name: rook-ceph-osd namespace: rook-ceph # namespace:cluster rules: - - apiGroups: [''] - resources: [configmaps] - verbs: [get, list, watch, create, update, delete] - - apiGroups: [ceph.rook.io] - resources: [cephclusters, cephclusters/finalizers] - verbs: [get, list, create, update, delete] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["ceph.rook.io"] + resources: ["cephclusters", "cephclusters/finalizers"] + verbs: ["get", "list", "create", "update", "delete"] --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -869,18 +870,18 @@ metadata: name: rook-ceph-purge-osd namespace: rook-ceph # namespace:cluster rules: - - apiGroups: [''] - resources: [configmaps] - verbs: [get] - - apiGroups: [apps] - resources: [deployments] - verbs: [get, delete] - - apiGroups: [batch] - resources: [jobs] - verbs: [get, list, delete] - - apiGroups: [''] - resources: [persistentvolumeclaims] - verbs: [get, update, delete] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "update", "delete"] --- # Allow the operator to manage resources in its own namespace apiVersion: rbac.authorization.k8s.io/v1 @@ -893,7 +894,7 @@ metadata: storage-backend: ceph rules: - apiGroups: - - '' + - "" resources: - pods - configmaps diff --git a/deploy/olm/generate-rook-csv.sh b/deploy/olm/generate-rook-csv.sh index 82d6eea38..e59162b91 100755 --- a/deploy/olm/generate-rook-csv.sh +++ b/deploy/olm/generate-rook-csv.sh @@ -13,7 +13,7 @@ PACKAGE_FILE="$OLM_CATALOG_DIR/assemble/rook-ceph.package.yaml" SUPPORTED_PLATFORMS='k8s|ocp|okd' operator_sdk="${OPERATOR_SDK:-operator-sdk}" -yq="${YQ:-yq}" +yq="${YQv3:-yq}" # Default CSI to true : "${OLM_INCLUDE_CEPHFS_CSI:=true}" @@ -35,6 +35,11 @@ if ! command -v "$yq" >/dev/null && [ ! -f "$yq" ]; then exit 1 fi +if [[ "$("$yq" --version)" != "yq version 3."* ]]; then + echo "yq must be version 3" >/dev/stderr + exit 1 +fi + if [[ -z "$1" ]]; then echo "Please provide a version, e.g:" echo "" diff --git a/images/Makefile b/images/Makefile index 040e9d3f9..d3d153a5f 100644 --- a/images/Makefile +++ b/images/Makefile @@ -30,7 +30,9 @@ ceph.%: @$(MAKE) -C ceph PLATFORM=$* do.build: ceph.$(PLATFORM) ; -build.all: $(foreach p,$(PLATFORMS), ceph.$(p)) ; ## Build images for all platforms. +build.all: ## Build images for all platforms. + $(MAKE) -C ceph prerequisites + $(MAKE) $(foreach p,$(PLATFORMS), ceph.$(p)) ; # ==================================================================================== # Help diff --git a/images/ceph/Makefile b/images/ceph/Makefile index bf1a2dfc3..2d71cf7c5 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -27,7 +27,7 @@ BASEIMAGE = $(REGISTRY_NAME)/ceph/ceph-$(GOARCH):$(CEPH_VERSION) CEPH_IMAGE = $(BUILD_REGISTRY)/ceph-$(GOARCH) OPERATOR_SDK_VERSION = v0.17.1 # TODO: update to yq v4 - v3 end of life in Aug 2021 ; v4 removes the 'yq delete' cmd and changes syntax -YQ_VERSION = 3.3.0 +YQv3_VERSION = 3.3.0 GOHOST := GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) go MANIFESTS_DIR=../../deploy/examples @@ -65,8 +65,8 @@ VOLUME_REPLICATION_CRD = replication.storage.openshift.io_volumereplications.yam VOLUME_REPLICATION_CLASS_CRD = replication.storage.openshift.io_volumereplicationclasses.yaml OPERATOR_SDK := $(TOOLS_HOST_DIR)/operator-sdk-$(OPERATOR_SDK_VERSION) -YQ := $(TOOLS_HOST_DIR)/yq-$(YQ_VERSION) -export OPERATOR_SDK YQ +YQv3 := $(TOOLS_HOST_DIR)/yq-$(YQv3_VERSION) +export OPERATOR_SDK YQv3 # ==================================================================================== # Build Rook @@ -84,7 +84,8 @@ do.build: ifeq ($(INCLUDE_CSV_TEMPLATES),true) @$(MAKE) CSV_TEMPLATE_DIR=$(TEMP) generate-csv-templates - @$(MAKE) CRD_TEMPLATE_DIR=$(TEMP)/deploy/olm/templates/crds/ get-volume-replication-crds + @$(MAKE) get-volume-replication-crds + cp $(CACHE_DIR)/crds/* $(TEMP)/deploy/olm/templates/crds/. @cp -r $(TEMP)/deploy/olm/templates $(TEMP)/ceph-csv-templates else mkdir $(TEMP)/ceph-csv-templates @@ -99,9 +100,13 @@ endif fi @rm -fr $(TEMP) +# call this before building multiple arches in parallel to prevent parallel build processes from +# conflicting +prerequisites: $(OPERATOR_SDK) $(YQv3) get-volume-replication-crds + # generate CSV template files into the directory defined by the env var CSV_TEMPLATE_DIR # CSV_TEMPLATE_DIR will be created if it doesn't already exist -generate-csv-templates: $(OPERATOR_SDK) $(YQ) ## Generate CSV templates for OLM into CSV_TEMPLATE_DIR +generate-csv-templates: $(OPERATOR_SDK) $(YQv3) ## Generate CSV templates for OLM into CSV_TEMPLATE_DIR @if [[ -z "$(CSV_TEMPLATE_DIR)" ]]; then echo "CSV_TEMPLATE_DIR is not set"; exit 1; fi @# first, copy the existing CRDs and OLM catalog directory to CSV_TEMPLATE_DIR @# then, generate or copy all prerequisites into CSV_TEMPLATE_DIR (e.g., CRDs) @@ -122,19 +127,17 @@ generate-csv-templates: $(OPERATOR_SDK) $(YQ) ## Generate CSV templates for OLM @echo " === Generated CSV templates can be found at $(CSV_TEMPLATE_DIR)/deploy/olm/templates" get-volume-replication-crds: - @if [[ -z "$(CRD_TEMPLATE_DIR)" ]]; then echo "CRD_TEMPLATE_DIR is not set"; exit 1; fi - @if [[ ! -d "$(CACHE_DIR)/crds" ]]; then\ + if [[ ! -d "$(CACHE_DIR)/crds" ]]; then\ mkdir -p $(CACHE_DIR)/crds;\ curl -L $(VOL_REPL_URL)/$(VOLUME_REPLICATION_CRD) -o $(CACHE_DIR)/crds/$(VOLUME_REPLICATION_CRD);\ curl -L $(VOL_REPL_URL)/$(VOLUME_REPLICATION_CLASS_CRD) -o $(CACHE_DIR)/crds/$(VOLUME_REPLICATION_CLASS_CRD);\ fi - @cp $(CACHE_DIR)/crds/* $(CRD_TEMPLATE_DIR) -$(YQ): - @echo === installing yq $(REAL_HOST_PLATFORM) +$(YQv3): + @echo === installing yq $(YQv3_VERSION) $(REAL_HOST_PLATFORM) @mkdir -p $(TOOLS_HOST_DIR) - @curl -JL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQ) - @chmod +x $(YQ) + @curl -JL https://github.com/mikefarah/yq/releases/download/$(YQv3_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQv3) + @chmod +x $(YQv3) $(OPERATOR_SDK): @echo === installing operator-sdk $(REAL_HOST_PLATFORM) @@ -144,11 +147,11 @@ $(OPERATOR_SDK): @chmod +x $(OPERATOR_SDK) @$(OPERATOR_SDK) version -csv: $(OPERATOR_SDK) $(YQ) ## Generate a CSV file for OLM. +csv: $(OPERATOR_SDK) $(YQv3) ## Generate a CSV file for OLM. @echo Generating CSV manifests @cd ../.. && deploy/olm/generate-rook-csv.sh $(CSV_VERSION) $(CSV_PLATFORM) $(ROOK_OP_VERSION) -csv-clean: $(OPERATOR_SDK) $(YQ) ## Remove existing OLM files. +csv-clean: ## Remove existing OLM files. @rm -fr ../../deploy/olm/deploy/* ../../deploy/olm/templates/* # reading from a file and outputting to the same file can have undefined results, so use this intermediate