From 3c6f3f79411b76a3584628e10c762642843e62d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 18 Feb 2021 19:16:56 +0100 Subject: [PATCH 1/2] ceph: lint toolbox.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply linter suggestions and remove used variables. Signed-off-by: Sébastien Han --- images/ceph/toolbox.sh | 49 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/images/ceph/toolbox.sh b/images/ceph/toolbox.sh index 160ae630b..ae2d92d2c 100755 --- a/images/ceph/toolbox.sh +++ b/images/ceph/toolbox.sh @@ -14,12 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARGS="$@" -if [ $# -eq 0 ]; then - ARGS=/bin/bash -fi - -ROOK_DIR="/var/lib/rook" CEPH_CONFIG="/etc/ceph/ceph.conf" MON_CONFIG="/etc/rook/mon-endpoints" KEYRING_FILE="/etc/ceph/keyring" @@ -27,14 +21,15 @@ KEYRING_FILE="/etc/ceph/keyring" # create a ceph config file in its default location so ceph/rados tools can be used # without specifying any arguments write_endpoints() { - endpoints=$(cat ${MON_CONFIG}) - - # filter out the mon names - # external cluster can have numbers or hyphens in mon names, handling them in regex - mon_endpoints=$(echo ${endpoints} | sed 's/[a-z0-9_-]\+=//g') - - DATE=$(date) - echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" cat < ${CEPH_CONFIG} [global] mon_host = ${mon_endpoints} @@ -46,19 +41,19 @@ EOF # watch the endpoints config file and update if the mon endpoints ever change watch_endpoints() { - # get the timestamp for the target of the soft link + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do real_path=$(realpath ${MON_CONFIG}) - initial_time=$(stat -c %Z ${real_path}) - while true; do - real_path=$(realpath ${MON_CONFIG}) - latest_time=$(stat -c %Z ${real_path}) - - if [[ "${latest_time}" != "${initial_time}" ]]; then - write_endpoints - initial_time=${latest_time} - fi - sleep 10 - done + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + sleep 10 + done } # create the keyring file @@ -73,4 +68,4 @@ write_endpoints # continuously update the mon endpoints if they fail over if [ "$1" != "--skip-watch" ]; then watch_endpoints -fi +fi \ No newline at end of file From c54acf449edc23018404cac49055443df2496af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 18 Feb 2021 19:22:16 +0100 Subject: [PATCH 2/2] ceph: ability to set cluster wide log level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From inside the toolbox, the ceph debug level can now be changed for all ceph logging subsystems, for instance, level 1 will be: ``` kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level 1 ``` To reset to default values just run: ``` kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level default ``` Signed-off-by: Sébastien Han --- Documentation/ceph-common-issues.md | 28 ++++++ images/ceph/Dockerfile | 2 +- images/ceph/Makefile | 1 + images/ceph/set-ceph-debug-level | 131 ++++++++++++++++++++++++++++ images/ceph/toolbox.sh | 3 +- 5 files changed, 163 insertions(+), 2 deletions(-) create mode 100755 images/ceph/set-ceph-debug-level diff --git a/Documentation/ceph-common-issues.md b/Documentation/ceph-common-issues.md index 2c1671a19..d1be6689d 100644 --- a/Documentation/ceph-common-issues.md +++ b/Documentation/ceph-common-issues.md @@ -23,6 +23,7 @@ If after trying the suggestions found on this page and the problem is not resolv * [Rook Agent modprobe exec format error](#rook-agent-modprobe-exec-format-error) * [Rook Agent rbd module missing error](#rook-agent-rbd-module-missing-error) * [Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7](#using-multiple-shared-filesystem-cephfs-is-attempted-on-a-kernel-version-older-than-47) +* [Set debug log level for all Ceph daemons](#set-debug-log-level-for-all-ceph-daemons) * [Activate log to file for a particular Ceph daemon](#activate-log-to-file-for-a-particular-ceph-daemon) * [Flex storage class versus Ceph CSI storage class](#flex-storage-class-versus-ceph-csi-storage-class) * [A worker node using RBD devices hangs up](#a-worker-node-using-rbd-devices-hangs-up) @@ -712,6 +713,33 @@ This is due to a mount flag added in the kernel version `4.7` which allows to ch For additional info on the kernel version requirement for multiple shared filesystems (CephFS), see [Filesystem - Kernel version requirement](ceph-filesystem.md#kernel-version-requirement). +## Set debug log level for all Ceph daemons + +You can set a given log level and apply it to all the Ceph daemons at the same time. +For this, make sure the toolbox pod is running, then determine the level you want (between 0 and 20). +You can find the list of all subsystems and their default values in [Ceph logging and debug official guide](https://docs.ceph.com/en/latest/rados/troubleshooting/log-and-debug/#ceph-subsystems). Be careful when increasing the level as it will produce very verbose logs. + +Assuming you want a log level of 1, you will run: + +```console +kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level 1 +``` + +Output: + +>```quote +> ceph config set global debug_context 1 +> ceph config set global debug_lockdep 1 +>... +>... +>``` + +Once you are done debugging, you can revert all the debug flag to their default value by running the following: + +```console +kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level default +``` + ## Activate log to file for a particular Ceph daemon They are cases where looking at Kubernetes logs is not enough for diverse reasons, but just to name a few: diff --git a/images/ceph/Dockerfile b/images/ceph/Dockerfile index 89b01b5ec..5e66b9bbb 100644 --- a/images/ceph/Dockerfile +++ b/images/ceph/Dockerfile @@ -22,7 +22,7 @@ ARG TINI_VERSION RUN curl --fail -sSL -o /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${ARCH} && \ chmod +x /tini -COPY rook rookflex toolbox.sh /usr/local/bin/ +COPY rook rookflex toolbox.sh set-ceph-debug-level /usr/local/bin/ COPY ceph-csi /etc/ceph-csi COPY ceph-monitoring /etc/ceph-monitoring COPY rook-external /etc/rook-external/ diff --git a/images/ceph/Makefile b/images/ceph/Makefile index f72ecf1e7..adc9265b4 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -44,6 +44,7 @@ do.build: generate-csv-ceph-templates @echo === container build $(CEPH_IMAGE) @cp Dockerfile $(TEMP) @cp toolbox.sh $(TEMP) + @cp set-ceph-debug-level $(TEMP) @cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP) @cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rookflex $(TEMP) @cp -r ../../cluster/examples/kubernetes/ceph/csi/template $(TEMP)/ceph-csi diff --git a/images/ceph/set-ceph-debug-level b/images/ceph/set-ceph-debug-level new file mode 100755 index 000000000..7fbbe66b4 --- /dev/null +++ b/images/ceph/set-ceph-debug-level @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ -z "$1" ]; then + echo "no debug level passed choose between 0 and 20" + exit 1 +fi + +CEPH_DEBUG_LEVEL=$1 +CEPH_DEBUG_FLAG=( + lockdep + context + crush + mds + mds_balancer + mds_locker + mds_log + mds_log_expire + mds_migrator + buffer + timer + filer + striper + objecter + rados + rbd + rbd_mirror + rbd_replay + journaler + objectcacher + client + osd + optracker + objclass + filestore + journal + ms + mon + monc + paxos + tp + auth + crypto + finisher + reserver + heartbeatmap + perfcounter + rgw + rgw_sync + civetweb + javaclient + asok + throttle + refs + compressor + bluestore + bluefs + bdev + kstore + rocksdb + leveldb + memdb + fuse + mgr + mgrc + dpdk + eventtrace +) + +############# +# FUNCTIONS # +############# +check() { + ok_to_run=1 + if [[ "$CEPH_DEBUG_LEVEL" =~ ^[0-9]+$ ]]; then + if [ "$CEPH_DEBUG_LEVEL" -ge 0 ] && [ "$CEPH_DEBUG_LEVEL" -le 20 ]; then + ok_to_run=0 + fi + elif [[ "$CEPH_DEBUG_LEVEL" == "default" ]]; then + ok_to_run=0 + fi +} + +exec_ceph_command() { + local debug_level=$1 + local action=set + if [[ "$debug_level" == "default" ]]; then + action="rm" + fi + + # exec command + for flag in "${CEPH_DEBUG_FLAG[@]}"; do + ARGS=("$action" global debug_"$flag") + if [[ "$debug_level" != "default" ]]; then + ARGS+=("$debug_level") + fi + # put stdout in /dev/null since increase debug log will overflow the terminal + echo "ceph config ${ARGS[*]}" + ceph config "${ARGS[@]}" &> /dev/null & pids+=($!) + + done + echo "waiting for all the new logging configuration to be applied, this can take a few seconds" + wait "${pids[@]}" +} + +######## +# MAIN # +######## +check +if [ "$ok_to_run" -eq 0 ]; then + exec_ceph_command "$CEPH_DEBUG_LEVEL" +else + echo "Wrong debug level $CEPH_DEBUG_LEVEL" + echo "MUST be integer between 0 and 20 or 'default' to reset all values" + exit 1 +fi diff --git a/images/ceph/toolbox.sh b/images/ceph/toolbox.sh index ae2d92d2c..78f631b3a 100755 --- a/images/ceph/toolbox.sh +++ b/images/ceph/toolbox.sh @@ -52,6 +52,7 @@ watch_endpoints() { write_endpoints initial_time=${latest_time} fi + sleep 10 done } @@ -68,4 +69,4 @@ write_endpoints # continuously update the mon endpoints if they fail over if [ "$1" != "--skip-watch" ]; then watch_endpoints -fi \ No newline at end of file +fi