From 2c4b66957cf78ca474e6af35103cf057b10759a2 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 17 Nov 2021 15:53:30 -0700 Subject: [PATCH] build: generate CSV from helm charts In order to generate common.yaml from Helm charts, we have to have a different way of generating CSV than from meta-comments in common.yaml. This implementation changes what appears in the CSV's RBAC somewhat, but these changes could be considered bugs fixed by using the new Helm generation method. - a few PSP related resources are removed from CSV - resources related to 'rook-ceph-purge-osd' Job are added to CSV - ClusterRoleBinding 'rook-ceph-object-bucket' is added to CSV Signed-off-by: Blaine Gardner --- Makefile | 3 +- build/rbac/get-helm-rbac.sh | 26 ++++++--- deploy/examples/common.yaml | 46 ---------------- deploy/olm/generate-rook-csv-templates.sh | 3 -- deploy/olm/generate-rook-csv.sh | 65 ++++------------------- images/ceph/Makefile | 2 +- 6 files changed, 33 insertions(+), 112 deletions(-) diff --git a/Makefile b/Makefile index 9675d6a9a..21e8256da 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,8 @@ crds: $(CONTROLLER_GEN) $(YQ) @build/crds/build-crds.sh $(CONTROLLER_GEN) $(YQ) gen-rbac: $(HELM) $(YQ) ## generate RBAC from Helm charts - HELM=$(HELM) ./build/rbac/get-helm-rbac.sh + @# output only stdout to the file; stderr for debugging should keep going to stderr + HELM=$(HELM) ./build/rbac/get-helm-rbac.sh > build/rbac/rbac.yaml .PHONY: all build.common cross.build.parallel .PHONY: build build.all install test check vet fmt codegen mod.check clean distclean prune diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index d2b178a9e..98e6b62be 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -1,13 +1,27 @@ #!/usr/bin/env bash set -eEuox pipefail -: ${HELM:=helm} +: "${HELM:=helm}" + +# Whether to include Pod Security Policy (PSP) related resources in the RBAC output. +# Empty string means DO include PSP resources. Any other value means do NOT include PSP resources. +: "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES:=""}" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -pushd "$SCRIPT_DIR" +pushd "$SCRIPT_DIR" &>/dev/stderr -${HELM} template ../../deploy/charts/rook-ceph \ - --namespace rook-ceph \ - --set crds.enabled=false | ./keep-rbac-yaml.sh > rbac.yaml +options=( + --namespace rook-ceph + --set crds.enabled=false +) +if [[ -z "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES}" ]]; then + options+=(--set pspEnable=true) +else + options+=(--set pspEnable=false) +fi -popd +echo "generating Helm template with options: ${options[*]}" &>/dev/stderr + +${HELM} template ../../deploy/charts/rook-ceph "${options[@]}" | ./keep-rbac-yaml.sh + +popd &>/dev/stderr diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index 54dbce993..5b0765896 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -15,7 +15,6 @@ apiVersion: v1 kind: Namespace metadata: name: rook-ceph # namespace:cluster -# OLM: BEGIN OBJECTBUCKET ROLEBINDING --- # Give Rook-Ceph Operator permissions to provision ObjectBuckets in response to ObjectBucketClaims. kind: ClusterRoleBinding @@ -30,8 +29,6 @@ subjects: - kind: ServiceAccount name: rook-ceph-system namespace: rook-ceph # namespace:operator -# OLM: END OBJECTBUCKET ROLEBINDING -# OLM: BEGIN OPERATOR ROLE --- # The cluster role for managing all the cluster-specific resources in a namespace apiVersion: rbac.authorization.k8s.io/v1 @@ -347,8 +344,6 @@ rules: verbs: # OBC controller updates OBC and OB statuses - update -# OLM: END OPERATOR ROLE -# OLM: BEGIN SERVICE ACCOUNT SYSTEM --- # The rook system service account used by the operator, agent, and discovery pods apiVersion: v1 @@ -362,8 +357,6 @@ metadata: # imagePullSecrets: # - name: my-registry-secret -# OLM: END SERVICE ACCOUNT SYSTEM -# OLM: BEGIN OPERATOR ROLEBINDING --- # Grant the operator, agent, and discovery agents access to resources in the namespace kind: RoleBinding @@ -415,14 +408,12 @@ subjects: - kind: ServiceAccount name: rook-ceph-system namespace: rook-ceph # namespace:operator -# OLM: END OPERATOR ROLEBINDING ################################################################################################################# # Beginning of cluster-specific resources. The example will assume the cluster will be created in the "rook-ceph" # namespace. If you want to create the cluster in a different namespace, you will need to modify these roles # and bindings accordingly. ################################################################################################################# # Service account for the Ceph OSDs. Must exist and cannot be renamed. -# OLM: BEGIN SERVICE ACCOUNT OSD --- apiVersion: v1 kind: ServiceAccount @@ -432,8 +423,6 @@ metadata: # imagePullSecrets: # - name: my-registry-secret -# OLM: END SERVICE ACCOUNT OSD -# OLM: BEGIN SERVICE ACCOUNT MGR --- # Service account for the Ceph Mgr. Must exist and cannot be renamed. apiVersion: v1 @@ -444,16 +433,12 @@ metadata: # imagePullSecrets: # - name: my-registry-secret -# OLM: END SERVICE ACCOUNT MGR -# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT --- apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster -# OLM: END CMD REPORTER SERVICE ACCOUNT -# OLM: BEGIN CLUSTER ROLE --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -552,8 +537,6 @@ rules: - persistentvolumeclaims verbs: - delete -# OLM: END CLUSTER ROLE -# OLM: BEGIN CMD REPORTER ROLE --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -573,8 +556,6 @@ rules: - create - update - delete -# OLM: END CMD REPORTER ROLE -# OLM: BEGIN CLUSTER ROLEBINDING --- # Allow the operator to create resources in this cluster's namespace kind: RoleBinding @@ -665,8 +646,6 @@ subjects: name: rook-ceph-osd namespace: rook-ceph # namespace:cluster -# OLM: END CLUSTER ROLEBINDING -# OLM: BEGIN CMD REPORTER ROLEBINDING --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -681,13 +660,11 @@ subjects: - kind: ServiceAccount name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster -# OLM: END CMD REPORTER ROLEBINDING ################################################################################################################# # Beginning of pod security policy resources. The example will assume the cluster will be created in the # "rook-ceph" namespace. If you want to create the cluster in a different namespace, you will need to modify # the roles and bindings accordingly. ################################################################################################################# -# OLM: BEGIN CLUSTER POD SECURITY POLICY --- apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -757,8 +734,6 @@ spec: # Ceph mgr Prometheus Metrics - min: 9283 max: 9283 -# OLM: END CLUSTER POD SECURITY POLICY -# OLM: BEGIN POD SECURITY POLICY BINDINGS --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -842,8 +817,6 @@ subjects: - kind: ServiceAccount name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster -# OLM: END CLUSTER POD SECURITY POLICY BINDINGS -# OLM: BEGIN CSI CEPHFS SERVICE ACCOUNT --- apiVersion: v1 kind: ServiceAccount @@ -856,8 +829,6 @@ kind: ServiceAccount metadata: name: rook-csi-cephfs-provisioner-sa namespace: rook-ceph # namespace:operator -# OLM: END CSI CEPHFS SERVICE ACCOUNT -# OLM: BEGIN CSI CEPHFS ROLE --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -874,8 +845,6 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] -# OLM: END CSI CEPHFS ROLE -# OLM: BEGIN CSI CEPHFS ROLEBINDING --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -890,8 +859,6 @@ roleRef: kind: Role name: cephfs-external-provisioner-cfg apiGroup: rbac.authorization.k8s.io -# OLM: END CSI CEPHFS ROLEBINDING -# OLM: BEGIN CSI CEPHFS CLUSTER ROLE --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -961,8 +928,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] -# OLM: END CSI CEPHFS CLUSTER ROLE -# OLM: BEGIN CSI CEPHFS CLUSTER ROLEBINDING --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -1016,8 +981,6 @@ roleRef: kind: ClusterRole name: cephfs-external-provisioner-runner apiGroup: rbac.authorization.k8s.io -# OLM: END CSI CEPHFS CLUSTER ROLEBINDING -# OLM: BEGIN CSI RBD SERVICE ACCOUNT --- apiVersion: v1 kind: ServiceAccount @@ -1030,8 +993,6 @@ kind: ServiceAccount metadata: name: rook-csi-rbd-provisioner-sa namespace: rook-ceph # namespace:operator -# OLM: END CSI RBD SERVICE ACCOUNT -# OLM: BEGIN CSI RBD ROLE --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -1048,8 +1009,6 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] -# OLM: END CSI RBD ROLE -# OLM: BEGIN CSI RBD ROLEBINDING --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1064,8 +1023,6 @@ roleRef: kind: Role name: rbd-external-provisioner-cfg apiGroup: rbac.authorization.k8s.io -# OLM: END CSI RBD ROLEBINDING -# OLM: BEGIN CSI RBD CLUSTER ROLE --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -1159,8 +1116,6 @@ rules: - apiGroups: [""] resources: ["serviceaccounts"] verbs: ["get"] -# OLM: END CSI RBD CLUSTER ROLE -# OLM: BEGIN CSI RBD CLUSTER ROLEBINDING --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -1213,7 +1168,6 @@ roleRef: kind: ClusterRole name: rbd-external-provisioner-runner apiGroup: rbac.authorization.k8s.io -# OLM: END CSI RBD CLUSTER ROLEBINDING --- # Aspects of ceph osd purge job that require access to the operator/cluster namespace kind: Role diff --git a/deploy/olm/generate-rook-csv-templates.sh b/deploy/olm/generate-rook-csv-templates.sh index 9ff09b6a8..e11f98081 100755 --- a/deploy/olm/generate-rook-csv-templates.sh +++ b/deploy/olm/generate-rook-csv-templates.sh @@ -2,9 +2,6 @@ set -e export OLM_SKIP_PKG_FILE_GEN="true" -export OLM_INCLUDE_CEPHFS_CSI="true" -export OLM_INCLUDE_RBD_CSI="true" -export OLM_INCLUDE_REPORTER="true" if [ -f "Dockerfile" ]; then # if this is being executed from the images/ceph/ dir, diff --git a/deploy/olm/generate-rook-csv.sh b/deploy/olm/generate-rook-csv.sh index e59162b91..5d62de1b7 100755 --- a/deploy/olm/generate-rook-csv.sh +++ b/deploy/olm/generate-rook-csv.sh @@ -88,15 +88,12 @@ YQ_CMD_MERGE=($yq merge --inplace --append -P ) YQ_CMD_WRITE=($yq write --inplace -P ) OPERATOR_YAML_FILE_K8S="deploy/examples/operator.yaml" OPERATOR_YAML_FILE_OCP="deploy/examples/operator-openshift.yaml" -COMMON_YAML_FILE="deploy/examples/common.yaml" CSV_PATH="$OLM_CATALOG_DIR/deploy/olm-catalog/${PLATFORM}/${VERSION}" CSV_BUNDLE_PATH="${CSV_PATH}/manifests" CSV_FILE_NAME="$CSV_BUNDLE_PATH/ceph.clusterserviceversion.yaml" OP_SDK_CMD=($operator_sdk generate csv --output-dir="deploy/olm-catalog/${PLATFORM}/${VERSION}" --csv-version) OLM_OPERATOR_YAML_FILE="$OLM_CATALOG_DIR/deploy/operator.yaml" -OLM_ROLE_YAML_FILE="$OLM_CATALOG_DIR/deploy/role.yaml" -OLM_ROLE_BINDING_YAML_FILE="$OLM_CATALOG_DIR/deploy/role_binding.yaml" -OLM_SERVICE_ACCOUNT_YAML_FILE="$OLM_CATALOG_DIR/deploy/service_account.yaml" +OLM_RBAC_YAML_FILE="$OLM_CATALOG_DIR/deploy/rbac.yaml" CEPH_EXTERNAL_SCRIPT_FILE="deploy/examples/create-external-cluster-resources.py" if [[ -d "$CSV_BUNDLE_PATH" ]]; then @@ -164,54 +161,6 @@ function generate_operator_yaml() { sed -n '/^# OLM: BEGIN OPERATOR DEPLOYMENT$/,/# OLM: END OPERATOR DEPLOYMENT$/p' "$operator_file" > "$OLM_OPERATOR_YAML_FILE" } -function generate_role_yaml() { - sed -n '/^# OLM: BEGIN OPERATOR ROLE$/,/# OLM: END OPERATOR ROLE$/p' "$COMMON_YAML_FILE" > "$OLM_ROLE_YAML_FILE" - sed -n '/^# OLM: BEGIN CLUSTER ROLE$/,/# OLM: END CLUSTER ROLE$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_YAML_FILE" - - if [ "$OLM_INCLUDE_CEPHFS_CSI" = true ]; then - sed -n '/^# OLM: BEGIN CSI CEPHFS ROLE$/,/# OLM: END CSI CEPHFS ROLE$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_YAML_FILE" - sed -n '/^# OLM: BEGIN CSI CEPHFS CLUSTER ROLE$/,/# OLM: END CSI CEPHFS CLUSTER ROLE$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_YAML_FILE" - fi - if [ "$OLM_INCLUDE_RBD_CSI" = true ]; then - sed -n '/^# OLM: BEGIN CSI RBD ROLE$/,/# OLM: END CSI RBD ROLE$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_YAML_FILE" - sed -n '/^# OLM: BEGIN CSI RBD CLUSTER ROLE$/,/# OLM: END CSI RBD CLUSTER ROLE$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_YAML_FILE" - fi - if [ "$OLM_INCLUDE_REPORTER" = true ] ; then - sed -n '/^# OLM: BEGIN CMD REPORTER ROLE$/,/# OLM: END CMD REPORTER ROLE$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_YAML_FILE" - fi -} - -function generate_role_binding_yaml() { - sed -n '/^# OLM: BEGIN OPERATOR ROLEBINDING$/,/# OLM: END OPERATOR ROLEBINDING$/p' "$COMMON_YAML_FILE" > "$OLM_ROLE_BINDING_YAML_FILE" - sed -n '/^# OLM: BEGIN CLUSTER ROLEBINDING$/,/# OLM: END CLUSTER ROLEBINDING$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_BINDING_YAML_FILE" - if [ "$OLM_INCLUDE_CEPHFS_CSI" = true ]; then - sed -n '/^# OLM: BEGIN CSI CEPHFS ROLEBINDING$/,/# OLM: END CSI CEPHFS ROLEBINDING$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_BINDING_YAML_FILE" - sed -n '/^# OLM: BEGIN CSI CEPHFS CLUSTER ROLEBINDING$/,/# OLM: END CSI CEPHFS CLUSTER ROLEBINDING$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_BINDING_YAML_FILE" - fi - if [ "$OLM_INCLUDE_RBD_CSI" = true ]; then - sed -n '/^# OLM: BEGIN CSI RBD ROLEBINDING$/,/# OLM: END CSI RBD ROLEBINDING$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_BINDING_YAML_FILE" - sed -n '/^# OLM: BEGIN CSI RBD CLUSTER ROLEBINDING$/,/# OLM: END CSI RBD CLUSTER ROLEBINDING$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_BINDING_YAML_FILE" - fi - if [ "$OLM_INCLUDE_REPORTER" = true ] ; then - sed -n '/^# OLM: BEGIN CMD REPORTER ROLEBINDING$/,/# OLM: END CMD REPORTER ROLEBINDING$/p' "$COMMON_YAML_FILE" >> "$OLM_ROLE_BINDING_YAML_FILE" - fi -} - -function generate_service_account_yaml() { - sed -n '/^# OLM: BEGIN SERVICE ACCOUNT SYSTEM$/,/# OLM: END SERVICE ACCOUNT SYSTEM$/p' "$COMMON_YAML_FILE" > "$OLM_SERVICE_ACCOUNT_YAML_FILE" - sed -n '/^# OLM: BEGIN SERVICE ACCOUNT OSD$/,/# OLM: END SERVICE ACCOUNT OSD$/p' "$COMMON_YAML_FILE" >> "$OLM_SERVICE_ACCOUNT_YAML_FILE" - sed -n '/^# OLM: BEGIN SERVICE ACCOUNT MGR$/,/# OLM: END SERVICE ACCOUNT MGR$/p' "$COMMON_YAML_FILE" >> "$OLM_SERVICE_ACCOUNT_YAML_FILE" - if [ "$OLM_INCLUDE_CEPHFS_CSI" = true ]; then - sed -n '/^# OLM: BEGIN CSI CEPHFS SERVICE ACCOUNT$/,/# OLM: END CSI CEPHFS SERVICE ACCOUNT$/p' "$COMMON_YAML_FILE" >> "$OLM_SERVICE_ACCOUNT_YAML_FILE" - fi - if [ "$OLM_INCLUDE_RBD_CSI" = true ]; then - sed -n '/^# OLM: BEGIN CSI RBD SERVICE ACCOUNT$/,/# OLM: END CSI RBD SERVICE ACCOUNT$/p' "$COMMON_YAML_FILE" >> "$OLM_SERVICE_ACCOUNT_YAML_FILE" - fi - if [ "$OLM_INCLUDE_REPORTER" = true ] ; then - sed -n '/^# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT$/,/# OLM: END CMD REPORTER SERVICE ACCOUNT$/p' "$COMMON_YAML_FILE" >> "$OLM_SERVICE_ACCOUNT_YAML_FILE" - fi -} - function hack_csv() { # Let's respect the following mapping # somehow the operator-sdk command generates serviceAccountNames suffixed with '-rules' @@ -267,9 +216,15 @@ function apply_rook_op_img(){ ######## create_directories generate_operator_yaml "$@" -generate_role_yaml -generate_role_binding_yaml -generate_service_account_yaml + +# Do not include Pod Security Policy (PSP) resources for CSV generation since OLM uses +# Security Context Constraints (SCC). +export DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES=true +./build/rbac/get-helm-rbac.sh > "$OLM_RBAC_YAML_FILE" + +# TODO: do we need separate clusterrole/clusterrolebinding/role/rolebinding/servicaccount files, or +# can these just stay in rbac.yaml? If they need to be separate, we can do that here with YQ. + generate_csv "$@" hack_csv if [ -z "${OLM_SKIP_PKG_FILE_GEN}" ]; then diff --git a/images/ceph/Makefile b/images/ceph/Makefile index 2d71cf7c5..85667571b 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -147,7 +147,7 @@ $(OPERATOR_SDK): @chmod +x $(OPERATOR_SDK) @$(OPERATOR_SDK) version -csv: $(OPERATOR_SDK) $(YQv3) ## Generate a CSV file for OLM. +csv: $(OPERATOR_SDK) $(YQv3) $(HELM) ## Generate a CSV file for OLM. @echo Generating CSV manifests @cd ../.. && deploy/olm/generate-rook-csv.sh $(CSV_VERSION) $(CSV_PLATFORM) $(ROOK_OP_VERSION)