From 59f6cd00ff06fc05a180dfe45cb388d4f3f08a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Antoine-Gombeaud?= Date: Fri, 7 Oct 2022 19:39:00 +0200 Subject: [PATCH] helm: fix toolbox command, default to cluster image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Loïc Antoine-Gombeaud Co-authored-by: Travis Nielsen --- .../Helm-Charts/ceph-cluster-chart.md | 4 +- .../templates/deployment.yaml | 71 ++++++++++++++++++- deploy/charts/rook-ceph-cluster/values.yaml | 6 +- .../installer/ceph_helm_installer.go | 1 - 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.md b/Documentation/Helm-Charts/ceph-cluster-chart.md index fa5a8a7dc..e6dd4f07b 100644 --- a/Documentation/Helm-Charts/ceph-cluster-chart.md +++ b/Documentation/Helm-Charts/ceph-cluster-chart.md @@ -72,12 +72,12 @@ The following table lists the configurable parameters of the rook-operator chart | `monitoring.enabled` | Enable Prometheus integration, will also create necessary RBAC rules to allow Operator to create ServiceMonitors. Monitoring requires Prometheus to be pre-installed | `false` | | `monitoring.prometheusRule.annotations` | Annotations applied to PrometheusRule | `{}` | | `monitoring.prometheusRule.labels` | Labels applied to PrometheusRule | `{}` | -| `monitoring.rulesNamespaceOverride` | The namespace in which to create the prometheus rules, if different from the rook cluster namespace If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. | `nil` | +| `monitoring.rulesNamespaceOverride` | The namespace in which to create the prometheus rules, if different from the rook cluster namespace. If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. | `nil` | | `operatorNamespace` | Namespace of the main rook operator | `"rook-ceph"` | | `pspEnable` | Create & use PSP resources. Set this to the same value as the rook-ceph chart. | `false` | | `toolbox.affinity` | Toolbox affinity | `{}` | | `toolbox.enabled` | Enable Ceph debugging pod deployment. See [toolbox](../Troubleshooting/ceph-toolbox.md) | `false` | -| `toolbox.image` | Toolbox image | `"rook/ceph:VERSION"` | +| `toolbox.image` | Toolbox image, defaults to the image used by the Ceph cluster | `nil` | | `toolbox.priorityClassName` | Set the priority class for the toolbox if desired | `nil` | | `toolbox.resources` | Toolbox resources | `{"limits":{"cpu":"500m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"128Mi"}}` | | `toolbox.tolerations` | Toolbox tolerations | `[]` | diff --git a/deploy/charts/rook-ceph-cluster/templates/deployment.yaml b/deploy/charts/rook-ceph-cluster/templates/deployment.yaml index 7ec53919e..c176323a6 100644 --- a/deploy/charts/rook-ceph-cluster/templates/deployment.yaml +++ b/deploy/charts/rook-ceph-cluster/templates/deployment.yaml @@ -26,9 +26,74 @@ spec: {{- end }} containers: - name: rook-ceph-tools - image: {{ .Values.toolbox.image }} - command: ["/bin/bash"] - args: ["-m", "-c", "/usr/local/bin/toolbox.sh"] + image: {{ default .Values.cephClusterSpec.cephVersion.image .Values.toolbox.image }} + command: + - /bin/bash + - -c + - | + # Replicate the script from toolbox.sh inline so the ceph image + # can be run directly, instead of requiring the rook toolbox + CEPH_CONFIG="/etc/ceph/ceph.conf" + MON_CONFIG="/etc/rook/mon-endpoints" + KEYRING_FILE="/etc/ceph/keyring" + + # create a ceph config file in its default location so ceph/rados tools can be used + # without specifying any arguments + write_endpoints() { + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + cat < ${CEPH_CONFIG} + [global] + mon_host = ${mon_endpoints} + + [client.admin] + keyring = ${KEYRING_FILE} + EOF + } + + # watch the endpoints config file and update if the mon endpoints ever change + watch_endpoints() { + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do + real_path=$(realpath ${MON_CONFIG}) + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + + sleep 10 + done + } + + # read the secret from an env var (for backward compatibility), or from the secret file + ceph_secret=${ROOK_CEPH_SECRET} + if [[ "$ceph_secret" == "" ]]; then + ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring) + fi + + # create the keyring file + cat < ${KEYRING_FILE} + [${ROOK_CEPH_USERNAME}] + key = ${ROOK_CEPH_SECRET} + key = ${ceph_secret} + EOF + + # write the initial config file + write_endpoints + + # continuously update the mon endpoints if they fail over + watch_endpoints imagePullPolicy: IfNotPresent tty: true env: diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index ce9d66412..fbdba2884 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -24,8 +24,8 @@ configOverride: toolbox: # -- Enable Ceph debugging pod deployment. See [toolbox](../Troubleshooting/ceph-toolbox.md) enabled: false - # -- Toolbox image - image: rook/ceph:VERSION + # -- Toolbox image, defaults to the image used by the Ceph cluster + image: #quay.io/ceph/ceph:v17.2.3 # -- Toolbox tolerations tolerations: [] # -- Toolbox affinity @@ -47,7 +47,7 @@ monitoring: enabled: false # -- Whether to create the Prometheus rules for Ceph alerts createPrometheusRules: false - # -- The namespace in which to create the prometheus rules, if different from the rook cluster namespace + # -- The namespace in which to create the prometheus rules, if different from the rook cluster namespace. # If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. rulesNamespaceOverride: diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index 388d6fcdb..ed2f0df92 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -117,7 +117,6 @@ func (h *CephInstaller) configureRookCephClusterViaHelm(upgrade bool) error { values["configOverride"] = clusterCustomSettings values["toolbox"] = map[string]interface{}{ "enabled": true, - "image": "rook/ceph:" + h.settings.RookVersion, "resources": nil, } values["monitoring"] = map[string]interface{}{