Merge pull request #16043 from subhamkrai/csi-op-helm-support

helm: add helm support for new ceph-csi-operator

.gitignore

@@ -5,6 +5,12 @@
 
 /_output
 
+deploy/charts/rook-ceph/charts/ceph-csi-operator-*
+deploy/charts/rook-ceph/charts/library-*
+deploy/charts/rook-ceph-cluster/charts/library-*
+deploy/charts/rook-ceph/Chart.lock
+deploy/charts/rook-ceph-cluster/Chart.lock
+
 /tests/integration/rook-test/
 
 # OLM related stuff

Documentation/Helm-Charts/operator-chart.md

@@ -46,6 +46,9 @@ The following table lists the configurable parameters of the rook-operator chart
 |-----------|-------------|---------|
 | `allowLoopDevices` | If true, loop devices are allowed to be used for osds in test clusters | `false` |
 | `annotations` | Pod annotations | `{}` |
+| `ceph-csi-operator.controllerManager.manager.env.csiServiceAccountPrefix` |  | `"ceph-csi-"` |
+| `ceph-csi-operator.fullnameOverride` |  | `"ceph-csi"` |
+| `ceph-csi-operator.nameOverride` |  | `"ceph-csi"` |
 | `cephCommandsTimeoutSeconds` | The timeout for ceph commands in seconds | `"15"` |
 | `containerSecurityContext` | Set the container security context for the operator | `{"capabilities":{"drop":["ALL"]},"runAsGroup":2016,"runAsNonRoot":true,"runAsUser":2016}` |
 | `crds.enabled` | Whether the helm chart should create and update the CRDs. If false, the CRDs must be managed independently with deploy/examples/crds.yaml. **WARNING** Only set during first deployment. If later disabled the cluster may be DESTROYED. If the CRDs are deleted in this case, see [the disaster recovery guide](https://rook.io/docs/rook/latest/Troubleshooting/disaster-recovery/#restoring-crds-after-deletion) to restore them. | `true` |
@@ -126,6 +129,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.registrar.tag` | Registrar image tag | `"v2.13.0"` |
 | `csi.resizer.repository` | Kubernetes CSI resizer image repository | `"registry.k8s.io/sig-storage/csi-resizer"` |
 | `csi.resizer.tag` | Resizer image tag | `"v1.13.2"` |
+| `csi.rookUseCsiOperator` |  | `false` |
 | `csi.serviceMonitor.enabled` | Enable ServiceMonitor for Ceph CSI drivers | `false` |
 | `csi.serviceMonitor.interval` | Service monitor scrape interval | `"10s"` |
 | `csi.serviceMonitor.labels` | ServiceMonitor additional labels | `{}` |

Makefile

@@ -205,6 +205,7 @@ mod.check: go.mod.check ## Check if any go modules changed.
 mod.update: go.mod.update ## Update all go modules.
 
 clean: ## Remove all files that are created by building.
+	@$(MAKE) helm.dependency.clean
 	@$(MAKE) go.mod.clean
 	@$(MAKE) -C images clean
 	@rm -fr $(OUTPUT_DIR) $(WORK_DIR)
@@ -222,7 +223,7 @@ crds: $(CONTROLLER_GEN) $(YQ)
 	@GOBIN=$(GOBIN) build/crds/generate-crd-docs.sh
 
 gen.rbac: gen-rbac
-gen-rbac: $(HELM) $(YQ) ## Generate RBAC from Helm charts
+gen-rbac: $(HELM) $(YQ) helm.dependency.build ## Generate RBAC from Helm charts
 	@# output only stdout to the file; stderr for debugging should keep going to stderr
 	HELM=$(HELM) ./build/rbac/gen-common.sh
 	HELM=$(HELM) ./build/rbac/gen-nfs-rbac.sh

build/makelib/helm.mk

@@ -38,6 +38,11 @@ $(HELM):
 define helm.chart
 $(HELM_OUTPUT_DIR)/$(1)-$(VERSION).tgz: $(HELM) $(HELM_OUTPUT_DIR) $(shell find $(HELM_CHARTS_DIR)/$(1) -type f)
 	@echo === helm package $(1)
+	@if [ ! -f $(HELM_CHARTS_DIR)/$(1)/charts/ceph-csi-operator-*.tgz ]; then \
+		echo "=== helm repo add (dependencies missing)"; \
+		$(HELM) repo add ceph-csi-operator https://ceph.github.io/ceph-csi-operator --force-update; \
+		$(MAKE) helm.dependency.update.$(1); \
+	fi
 	@rm -rf $(OUTPUT_DIR)/$(1)
 	@cp -aL $(HELM_CHARTS_DIR)/$(1) $(OUTPUT_DIR)
 	@$(SED_IN_PLACE) 's|master|$(VERSION)|g' $(OUTPUT_DIR)/$(1)/values.yaml
@@ -53,6 +58,38 @@ $(HELM_INDEX): $(HELM) $(HELM_OUTPUT_DIR)
 
 helm.build: $(HELM_INDEX)
 
+# Update helm chart dependencies
+define helm.dependency.update
+helm.dependency.update.$(1): $(HELM)
+	@echo === updating helm dependencies for $(1)
+	@cd $(HELM_CHARTS_DIR)/$(1) && $(HELM) dependency update
+helm.dependency.update: helm.dependency.update.$(1)
+endef
+$(foreach p,$(HELM_CHARTS),$(eval $(call helm.dependency.update,$(p))))
+
+# Build helm chart dependencies
+define helm.dependency.build
+helm.dependency.build.$(1): $(HELM)
+	@echo === building helm dependencies for $(1)
+	@cd $(HELM_CHARTS_DIR)/$(1) && $(HELM) dependency build
+helm.dependency.build: helm.dependency.build.$(1)
+endef
+$(foreach p,$(HELM_CHARTS),$(eval $(call helm.dependency.build,$(p))))
+
+# Clean up helm dependency artifacts from source directories
+define helm.dependency.clean
+helm.dependency.clean.$(1):
+	@echo === cleaning up dependency artifacts for $(1)
+	@rm -f $(HELM_CHARTS_DIR)/$(1)/charts/*.tgz
+helm.dependency.clean: helm.dependency.clean.$(1)
+endef
+$(foreach p,$(HELM_CHARTS),$(eval $(call helm.dependency.clean,$(p))))
+
+.PHONY: helm.dependency.update helm.dependency.build helm.dependency.clean
+helm.dependency.update: ## Update helm chart dependencies for all charts
+helm.dependency.build: ## Build helm chart dependencies for all charts
+helm.dependency.clean: ## Clean up helm dependency artifacts (.tgz files) from source directories
+
 # ====================================================================================
 # Makefile helper functions for helm-docs: https://github.com/norwoodj/helm-docs
 #

deploy/charts/rook-ceph/Chart.yaml

@@ -7,6 +7,11 @@ icon: https://rook.io/images/rook-logo.svg
 sources:
   - https://github.com/rook/rook
 dependencies:
-- name: library
-  version: "0.0.1"
-  repository: "file://../library"
+  - name: library
+    version: "0.0.1"
+    repository: "file://../library"
+  - name: ceph-csi-operator
+    version: 0.4.0
+    repository: https://ceph.github.io/ceph-csi-operator
+    alias: ceph-csi-operator
+    condition: csi.rookUseCsiOperator

deploy/charts/rook-ceph/templates/NOTES.txt

@@ -1,5 +1,4 @@
-The Rook Operator has been installed. Check its status by running:
-  kubectl --namespace {{ .Release.Namespace }} get pods -l "app=rook-ceph-operator"
+The Rook Operator has been installed.
 
 Visit https://rook.io/docs/rook/latest for instructions on how to create and configure Rook clusters
 
@@ -9,3 +8,6 @@ Important Notes:
 - The sample manifests assume you also installed the rook-ceph operator in the `rook-ceph` namespace.
 - The helm chart includes all the RBAC required to create a CephCluster CRD in the same namespace.
 - Any disk devices you add to the cluster in the 'CephCluster' must be empty (no filesystem and no partitions).
+{{- if .Values.csi.rookUseCsiOperator }}
+- The CSI operator will manage the CSI driver lifecycle for RBD, CephFS, and NFS drivers.
+{{- end }}

deploy/charts/rook-ceph/templates/configmap.yaml

@@ -31,6 +31,7 @@ data:
 {{- end }}
 
 {{- if .Values.csi }}
+  ROOK_USE_CSI_OPERATOR: {{ .Values.csi.rookUseCsiOperator | quote }}
   ROOK_CSI_ENABLE_RBD: {{ .Values.csi.enableRbdDriver | quote }}
   ROOK_CSI_ENABLE_CEPHFS: {{ .Values.csi.enableCephfsDriver | quote }}
   ROOK_CSI_DISABLE_DRIVER: {{ .Values.csi.disableCsiDriver | quote }}

deploy/charts/rook-ceph/values.yaml

@@ -77,9 +77,18 @@ containerSecurityContext:
 # -- If true, loop devices are allowed to be used for osds in test clusters
 allowLoopDevices: false
 
+ceph-csi-operator:
+  nameOverride: ceph-csi
+  fullnameOverride: ceph-csi
+  controllerManager:
+    manager:
+      env:
+        csiServiceAccountPrefix: "ceph-csi-"
+
 # Settings for whether to disable the drivers or other daemons if they are not
 # needed
 csi:
+  rookUseCsiOperator: false
   # -- Enable Ceph CSI RBD driver
   enableRbdDriver: true
   # -- Enable Ceph CSI CephFS driver