From ba4460247710124d00d8768614f7dd2dcd8c8758 Mon Sep 17 00:00:00 2001 From: Tom Hellier Date: Fri, 19 Nov 2021 14:54:57 +0000 Subject: [PATCH] helm: allow further configurability of ingress version The ingress api version changed when it went to v1, and this has caused some upheaval throughout the kubernetes ecosystem. This commit uses a common method of deciding which ingress api to use, and allows the optional override of the kubernetes version presented to helm using the helm build-in capabilities. also add an ingress into the helm integration tests so any regressions to how ingresses are handled in the future are caught easier. Closes rook#9174 Signed-off-by: Tom Hellier --- .../integration-test-setup/action.yaml | 4 ++-- .../setup-cluster-resources/action.yaml | 4 ++-- Documentation/helm-ceph-cluster.md | 1 + .../rook-ceph-cluster/templates/_helpers.tpl | 20 ++++++++++++++++ .../rook-ceph-cluster/templates/ingress.yaml | 24 +++++++------------ cluster/charts/rook-ceph-cluster/values.yaml | 3 +++ .../installer/ceph_helm_installer.go | 14 ++++++++++- tests/integration/ceph_base_deploy_test.go | 6 +++++ tests/integration/ceph_helm_test.go | 1 + 9 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.github/workflows/integration-test-setup/action.yaml b/.github/workflows/integration-test-setup/action.yaml index 4c9f2f659..6cdd6b5ea 100644 --- a/.github/workflows/integration-test-setup/action.yaml +++ b/.github/workflows/integration-test-setup/action.yaml @@ -19,9 +19,9 @@ runs: - name: setup minikube uses: manusa/actions-setup-minikube@v2.4.2 with: - minikube version: "v1.23.2" + minikube version: "v1.24.0" kubernetes version: ${{ inputs.kubernetes-version }} - start args: --memory 6g --cpus=2 + start args: --memory 6g --cpus=2 --addons ingress github token: ${{ inputs.github-token }} - name: print k8s cluster status diff --git a/.github/workflows/setup-cluster-resources/action.yaml b/.github/workflows/setup-cluster-resources/action.yaml index cb33f214e..d209d98ff 100644 --- a/.github/workflows/setup-cluster-resources/action.yaml +++ b/.github/workflows/setup-cluster-resources/action.yaml @@ -16,9 +16,9 @@ runs: - name: setup minikube uses: manusa/actions-setup-minikube@v2.4.2 with: - minikube version: 'v1.23.2' + minikube version: 'v1.24.0' kubernetes version: 'v1.22.2' - start args: --memory 6g --cpus=2 + start args: --memory 6g --cpus=2 --addons ingress github token: ${{ inputs.github-token }} - name: install deps diff --git a/Documentation/helm-ceph-cluster.md b/Documentation/helm-ceph-cluster.md index 07fc35a4a..ff3707748 100644 --- a/Documentation/helm-ceph-cluster.md +++ b/Documentation/helm-ceph-cluster.md @@ -57,6 +57,7 @@ The following tables lists the configurable parameters of the rook-operator char | Parameter | Description | Default | | ---------------------- | -------------------------------------------------------------------- | ----------- | | `operatorNamespace` | Namespace of the Rook Operator | `rook-ceph` | +| `kubeVersion` | Optional override of the target kubernetes version | `` | | `configOverride` | Cluster ceph.conf override | | | `toolbox.enabled` | Enable Ceph debugging pod deployment. See [toolbox](ceph-toolbox.md) | `false` | | `toolbox.tolerations` | Toolbox tolerations | `[]` | diff --git a/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl b/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl index 5bbd35b90..96b5fdbfd 100644 --- a/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl +++ b/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl @@ -4,3 +4,23 @@ Define the clusterName as defaulting to the release namespace {{- define "clusterName" -}} {{ .Values.clusterName | default .Release.Namespace }} {{- end -}} + +{{/* +Return the target Kubernetes version. +*/}} +{{- define "capabilities.kubeVersion" -}} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "capabilities.ingress.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} diff --git a/cluster/charts/rook-ceph-cluster/templates/ingress.yaml b/cluster/charts/rook-ceph-cluster/templates/ingress.yaml index d665de5f2..078693288 100644 --- a/cluster/charts/rook-ceph-cluster/templates/ingress.yaml +++ b/cluster/charts/rook-ceph-cluster/templates/ingress.yaml @@ -1,12 +1,6 @@ {{- if .Values.ingress.dashboard.host }} --- -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }} -apiVersion: networking.k8s.io/v1 -{{ else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} -apiVersion: networking.k8s.io/v1beta1 -{{ else }} -apiVersion: extensions/v1beta1 -{{ end -}} +apiVersion: {{ include "capabilities.ingress.apiVersion" . }} kind: Ingress metadata: name: {{ template "clusterName" . }}-dashboard @@ -20,7 +14,14 @@ spec: paths: - path: {{ .Values.ingress.dashboard.host.path | default "/" }} backend: -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }} +{{- if (semverCompare "<1.18-0" (include "capabilities.kubeVersion" .)) }} + serviceName: rook-ceph-mgr-dashboard + {{- if .Values.cephClusterSpec.dashboard.ssl }} + servicePort: https-dashboard + {{- else }} + servicePort: http-dashboard + {{- end }} +{{- else }} service: name: rook-ceph-mgr-dashboard port: @@ -30,13 +31,6 @@ spec: name: http-dashboard {{- end }} pathType: Prefix -{{- else }} - serviceName: rook-ceph-mgr-dashboard - {{- if .Values.cephClusterSpec.dashboard.ssl }} - servicePort: https-dashboard - {{- else }} - servicePort: http-dashboard - {{- end }} {{- end }} {{- if .Values.ingress.dashboard.tls }} tls: {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }} diff --git a/cluster/charts/rook-ceph-cluster/values.yaml b/cluster/charts/rook-ceph-cluster/values.yaml index 1ed9b4791..7edc3aae3 100644 --- a/cluster/charts/rook-ceph-cluster/values.yaml +++ b/cluster/charts/rook-ceph-cluster/values.yaml @@ -8,6 +8,9 @@ operatorNamespace: rook-ceph # The metadata.name of the CephCluster CR. The default name is the same as the namespace. # clusterName: rook-ceph +# Ability to override the kubernetes version used in rendering the helm chart +# kubeVersion: 1.21 + # Ability to override ceph.conf # configOverride: | # [global] diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index ec0fa60c4..7c9ecf03c 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -72,6 +72,7 @@ func (h *CephInstaller) CreateRookCephClusterViaHelm(values map[string]interface if err := yaml.Unmarshal([]byte(h.Manifests.GetCephCluster()), &clusterCRD); err != nil { return err } + values["cephClusterSpec"] = clusterCRD["spec"] values["operatorNamespace"] = h.settings.OperatorNamespace values["configOverride"] = clusterCustomSettings @@ -79,7 +80,18 @@ func (h *CephInstaller) CreateRookCephClusterViaHelm(values map[string]interface "enabled": true, "image": "rook/ceph:" + LocalBuildTag, } - values["cephClusterSpec"] = clusterCRD["spec"] + values["ingress"] = map[string]interface{}{ + "dashboard": map[string]interface{}{ + "annotations": map[string]interface{}{ + "kubernetes.io/ingress-class": "nginx", + "nginx.ingress.kubernetes.io/rewrite-target": "/ceph-dashboard/$2", + }, + "host": map[string]interface{}{ + "name": "localhost", + "path": "/ceph-dashboard(/|$)(.*)", + }, + }, + } if err := h.CreateBlockPoolConfiguration(values, blockPoolName, blockPoolSCName); err != nil { return err diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index b366033f5..66b11f817 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -74,6 +74,12 @@ func checkIfRookClusterIsHealthy(s suite.Suite, testClient *clients.TestClient, require.Nil(s.T(), err) } +func checkIfRookClusterHasHealthyIngress(s suite.Suite, k8sh *utils.K8sHelper, clusterNamespace string) { + logger.Infof("Testing ingress %s health", clusterNamespace) + _, err := k8sh.GetResourceStatus("Ingress", clusterNamespace + "-dashboard", clusterNamespace) + assert.NoError(s.T(), err) +} + func HandlePanics(r interface{}, uninstaller func(), t func() *testing.T) { if r != nil { logger.Infof("unexpected panic occurred during test %s, --> %v", t().Name(), r) diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 2093e48f2..7455423d6 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -91,6 +91,7 @@ func (h *HelmSuite) AfterTest(suiteName, testName string) { // Test to make sure all rook components are installed and Running func (h *HelmSuite) TestARookInstallViaHelm() { checkIfRookClusterIsInstalled(h.Suite, h.k8shelper, h.settings.Namespace, h.settings.Namespace, 1) + checkIfRookClusterHasHealthyIngress(h.Suite, h.k8shelper, h.settings.Namespace) } // Test BlockCreation on Rook that was installed via Helm