From bcc7d3e54103dc8ef9a9e64cd107d0e4feff0e98 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Mon, 6 Dec 2021 11:15:06 -0700 Subject: [PATCH] build: generate common.yaml from helm charts Generate deploy/examples/common.yaml from Helm charts. There were still a few differences between common.yaml and the Helm charts. Where there were differences we assume common.yaml was the most up-to-date. Signed-off-by: Blaine Gardner --- Makefile | 4 +- build/rbac/gen-common.sh | 13 ++ .../library/templates/_cluster-role.tpl | 6 + .../templates/_cluster-serviceaccount.tpl | 1 - .../library/templates/_imagepullsecret.tpl | 4 + .../rook-ceph/templates/clusterrole.yaml | 8 +- .../templates/clusterrolebinding.yaml | 8 +- deploy/charts/rook-ceph/templates/role.yaml | 12 -- deploy/examples/common.yaml | 145 +++++++++++------- 9 files changed, 121 insertions(+), 80 deletions(-) create mode 100755 build/rbac/gen-common.sh diff --git a/Makefile b/Makefile index 21e8256da..4ece242dc 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ build.version: @mkdir -p $(OUTPUT_DIR) @echo "$(VERSION)" > $(OUTPUT_DIR)/version -build.common: build.version helm.build mod.check crds +build.common: build.version helm.build mod.check crds gen-rbac @$(MAKE) go.init @$(MAKE) go.validate @$(MAKE) -C images/ceph list-image @@ -185,7 +185,7 @@ crds: $(CONTROLLER_GEN) $(YQ) gen-rbac: $(HELM) $(YQ) ## generate RBAC from Helm charts @# output only stdout to the file; stderr for debugging should keep going to stderr - HELM=$(HELM) ./build/rbac/get-helm-rbac.sh > build/rbac/rbac.yaml + HELM=$(HELM) ./build/rbac/gen-common.sh .PHONY: all build.common cross.build.parallel .PHONY: build build.all install test check vet fmt codegen mod.check clean distclean prune diff --git a/build/rbac/gen-common.sh b/build/rbac/gen-common.sh new file mode 100755 index 000000000..547510a7e --- /dev/null +++ b/build/rbac/gen-common.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -xeEuo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +pushd "$SCRIPT_DIR" &>/dev/stderr + +COMMON_YAML_FILE="$SCRIPT_DIR/../../deploy/examples/common.yaml" + +rm -f "$COMMON_YAML_FILE" +cat common.yaml.header >> "$COMMON_YAML_FILE" +./get-helm-rbac.sh >> "$COMMON_YAML_FILE" + +popd &>/dev/stderr diff --git a/deploy/charts/library/templates/_cluster-role.tpl b/deploy/charts/library/templates/_cluster-role.tpl index d25c4d01f..06b61439c 100644 --- a/deploy/charts/library/templates/_cluster-role.tpl +++ b/deploy/charts/library/templates/_cluster-role.tpl @@ -8,6 +8,11 @@ metadata: name: rook-ceph-osd namespace: {{ .Release.Namespace }} # namespace:cluster rules: + # this is needed for rook's "key-management" CLI to fetch the vault token from the secret when + # validating the connection details + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "update", "delete"] @@ -86,6 +91,7 @@ rules: - update - delete --- +# Aspects of ceph osd purge job that require access to the cluster namespace kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/charts/library/templates/_cluster-serviceaccount.tpl b/deploy/charts/library/templates/_cluster-serviceaccount.tpl index 8ca0ec806..c5adc886d 100644 --- a/deploy/charts/library/templates/_cluster-serviceaccount.tpl +++ b/deploy/charts/library/templates/_cluster-serviceaccount.tpl @@ -11,7 +11,6 @@ metadata: labels: operator: rook storage-backend: ceph - i-am-a-new-label: delete-me {{- include "library.rook-ceph.labels" . | nindent 4 }} {{ include "library.imagePullSecrets" . }} --- diff --git a/deploy/charts/library/templates/_imagepullsecret.tpl b/deploy/charts/library/templates/_imagepullsecret.tpl index dc877b3af..87231a887 100644 --- a/deploy/charts/library/templates/_imagepullsecret.tpl +++ b/deploy/charts/library/templates/_imagepullsecret.tpl @@ -5,5 +5,9 @@ Define imagePullSecrets option to pass to all service accounts {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets }} +{{- else }} + {{/* if the secrets are not included, include a comment for generating common.yaml */}} +# imagePullSecrets: +# - name: my-registry-secret {{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index 30312a922..89570cffb 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -357,7 +357,7 @@ rules: verbs: ["update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] @@ -375,6 +375,10 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin + labels: + operator: rook + storage-backend: ceph + {{- include "library.rook-ceph.labels" . | nindent 4 }} rules: - apiGroups: [""] resources: ["secrets"] @@ -429,7 +433,7 @@ rules: verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] diff --git a/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml b/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml index 1e484b25b..766379795 100644 --- a/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml @@ -52,14 +52,14 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: rook-csi-rbd-plugin-sa namespace: {{ .Release.Namespace }} # namespace:operator +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/charts/rook-ceph/templates/role.yaml b/deploy/charts/rook-ceph/templates/role.yaml index 74cefb7cc..9fa89d843 100644 --- a/deploy/charts/rook-ceph/templates/role.yaml +++ b/deploy/charts/rook-ceph/templates/role.yaml @@ -38,18 +38,6 @@ rules: - create - update - delete -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - - prometheusrules - verbs: - - get - - list - - watch - - create - - update - - delete - apiGroups: - batch resources: diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index bcc3fc57b..6fae3544d 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -52,6 +52,18 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] @@ -67,29 +79,14 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: "psp:rook" + name: 'psp:rook' labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: @@ -108,9 +105,6 @@ metadata: labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: [""] @@ -209,9 +203,6 @@ metadata: labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: @@ -245,9 +236,6 @@ metadata: labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: @@ -374,9 +362,6 @@ metadata: labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: @@ -434,9 +419,6 @@ metadata: labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: [""] @@ -496,16 +478,13 @@ rules: - get - list --- -apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-system labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator rules: # Most resources are represented by a string representation of their name, such as "pods", just as it appears in the URL for the relevant API endpoint. @@ -579,9 +558,6 @@ metadata: labels: operator: rook storage-backend: ceph - app.kubernetes.io/name: rook-ceph - app.kubernetes.io/instance: rook-ceph - app.kubernetes.io/component: ceph-csi app.kubernetes.io/part-of: rook-ceph-operator roleRef: apiGroup: rbac.authorization.k8s.io @@ -606,8 +582,8 @@ subjects: name: rook-ceph-mgr namespace: rook-ceph # namespace:cluster --- -# Give Rook-Ceph Operator permissions to provision ObjectBuckets in response to ObjectBucketClaims. kind: ClusterRoleBinding +# Give Rook-Ceph Operator permissions to provision ObjectBuckets in response to ObjectBucketClaims. apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-object-bucket @@ -641,6 +617,7 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -654,10 +631,14 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: rook-ceph-system-psp + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "psp:rook" + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-ceph-system @@ -670,7 +651,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "psp:rook" + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-cephfs-plugin-sa @@ -683,7 +664,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "psp:rook" + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-cephfs-provisioner-sa @@ -696,7 +677,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "psp:rook" + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-rbd-plugin-sa @@ -709,21 +690,29 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "psp:rook" + name: 'psp:rook' subjects: - kind: ServiceAccount name: rook-csi-rbd-provisioner-sa namespace: rook-ceph # namespace:operator --- +# We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs. +# * privileged (for kube-system namespace) +# * restricted (for all logged in users) +# +# PSPs are applied based on the first match alphabetically. `rook-ceph-operator` comes after +# `restricted` alphabetically, so we name this `00-rook-privileged`, so it stays somewhere +# close to the top and so `rook-system` gets the intended PSP. This may need to be renamed in +# environments with other `00`-prefixed PSPs. +# +# More on PSP ordering: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#policy-order apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: - # Note: Kubernetes matches PSPs to deployments alphabetically. In some environments, this PSP may - # need to be renamed with a value that will match before others. name: 00-rook-privileged annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: "runtime/default" - seccomp.security.alpha.kubernetes.io/defaultProfileName: "runtime/default" + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: true allowedCapabilities: @@ -881,7 +870,7 @@ rules: - patch - delete - apiGroups: - - "" + - '' resources: - persistentvolumeclaims verbs: @@ -905,7 +894,7 @@ rules: resources: ["cephclusters", "cephclusters/finalizers"] verbs: ["get", "list", "create", "update", "delete"] --- -# Aspects of ceph osd purge job that require access to the operator/cluster namespace +# Aspects of ceph osd purge job that require access to the cluster namespace kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -925,7 +914,7 @@ rules: resources: ["persistentvolumeclaims"] verbs: ["get", "update", "delete"] --- -# The role for the operator to manage resources in its own namespace +# Allow the operator to manage resources in its own namespace apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -934,6 +923,7 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: - "" @@ -1046,6 +1036,10 @@ kind: RoleBinding metadata: name: rook-ceph-default-psp namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -1055,7 +1049,7 @@ subjects: name: default namespace: rook-ceph # namespace:cluster --- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules +# Allow the ceph mgr to access resources scoped to the CephCluster namespace necessary for mgr modules kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -1084,7 +1078,7 @@ subjects: name: rook-ceph-mgr namespace: rook-ceph # namespace:cluster --- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules +# Allow the ceph mgr to access resources in the Rook operator namespace necessary for mgr modules kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -1143,7 +1137,7 @@ subjects: name: rook-ceph-purge-osd namespace: rook-ceph # namespace:cluster --- -# Grant the operator, agent, and discovery agents access to resources in the namespace +# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -1152,6 +1146,7 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -1161,36 +1156,55 @@ subjects: name: rook-ceph-system namespace: rook-ceph # namespace:operator --- +# Service account for the job that reports the Ceph version in an image apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret --- -# Service account for the Ceph Mgr. Must exist and cannot be renamed. +# Service account for Ceph mgrs apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-mgr namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator # imagePullSecrets: -# - name: my-registry-secret +# - name: my-registry-secret --- +# Service account for Ceph OSDs apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-osd namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator # imagePullSecrets: -# - name: my-registry-secret +# - name: my-registry-secret --- +# Service account for job that purges OSDs from a Rook-Ceph cluster apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-purge-osd namespace: rook-ceph # namespace:cluster +# imagePullSecrets: +# - name: my-registry-secret --- -# The rook system service account used by the operator, agent, and discovery pods +# Service account for the Rook-Ceph operator apiVersion: v1 kind: ServiceAccount metadata: @@ -1199,29 +1213,42 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator # imagePullSecrets: -# - name: my-registry-secret +# - name: my-registry-secret --- +# Service account for the CephFS CSI driver apiVersion: v1 kind: ServiceAccount metadata: name: rook-csi-cephfs-plugin-sa namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret --- +# Service account for the CephFS CSI provisioner apiVersion: v1 kind: ServiceAccount metadata: name: rook-csi-cephfs-provisioner-sa namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret --- +# Service account for the RBD CSI driver apiVersion: v1 kind: ServiceAccount metadata: name: rook-csi-rbd-plugin-sa namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret --- +# Service account for the RBD CSI provisioner apiVersion: v1 kind: ServiceAccount metadata: name: rook-csi-rbd-provisioner-sa namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret