diff --git a/Documentation/ceph-common-issues.md b/Documentation/ceph-common-issues.md index 3f5aa35cb..732d55545 100644 --- a/Documentation/ceph-common-issues.md +++ b/Documentation/ceph-common-issues.md @@ -23,6 +23,7 @@ If after trying the suggestions found on this page and the problem is not resolv * [Rook Agent modprobe exec format error](#rook-agent-modprobe-exec-format-error) * [Rook Agent rbd module missing error](#rook-agent-rbd-module-missing-error) * [Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7](#using-multiple-shared-filesystem-cephfs-is-attempted-on-a-kernel-version-older-than-47) +* [Set debug log level for all Ceph daemons](#set-debug-log-level-for-all-ceph-daemons) * [Activate log to file for a particular Ceph daemon](#activate-log-to-file-for-a-particular-ceph-daemon) * [Flex storage class versus Ceph CSI storage class](#flex-storage-class-versus-ceph-csi-storage-class) * [A worker node using RBD devices hangs up](#a-worker-node-using-rbd-devices-hangs-up) @@ -727,6 +728,33 @@ This is due to a mount flag added in the kernel version `4.7` which allows to ch For additional info on the kernel version requirement for multiple shared filesystems (CephFS), see [Filesystem - Kernel version requirement](ceph-filesystem.md#kernel-version-requirement). +## Set debug log level for all Ceph daemons + +You can set a given log level and apply it to all the Ceph daemons at the same time. +For this, make sure the toolbox pod is running, then determine the level you want (between 0 and 20). +You can find the list of all subsystems and their default values in [Ceph logging and debug official guide](https://docs.ceph.com/en/latest/rados/troubleshooting/log-and-debug/#ceph-subsystems). Be careful when increasing the level as it will produce very verbose logs. + +Assuming you want a log level of 1, you will run: + +```console +kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level 1 +``` + +Output: + +>```quote +> ceph config set global debug_context 1 +> ceph config set global debug_lockdep 1 +>... +>... +>``` + +Once you are done debugging, you can revert all the debug flag to their default value by running the following: + +```console +kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level default +``` + ## Activate log to file for a particular Ceph daemon They are cases where looking at Kubernetes logs is not enough for diverse reasons, but just to name a few: diff --git a/images/ceph/Dockerfile b/images/ceph/Dockerfile index 89b01b5ec..5e66b9bbb 100644 --- a/images/ceph/Dockerfile +++ b/images/ceph/Dockerfile @@ -22,7 +22,7 @@ ARG TINI_VERSION RUN curl --fail -sSL -o /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${ARCH} && \ chmod +x /tini -COPY rook rookflex toolbox.sh /usr/local/bin/ +COPY rook rookflex toolbox.sh set-ceph-debug-level /usr/local/bin/ COPY ceph-csi /etc/ceph-csi COPY ceph-monitoring /etc/ceph-monitoring COPY rook-external /etc/rook-external/ diff --git a/images/ceph/Makefile b/images/ceph/Makefile index f913bb847..8b07bc043 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -44,6 +44,7 @@ do.build: generate-csv-ceph-templates @echo === container build $(CEPH_IMAGE) @cp Dockerfile $(TEMP) @cp toolbox.sh $(TEMP) + @cp set-ceph-debug-level $(TEMP) @cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP) @cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rookflex $(TEMP) @cp -r ../../cluster/examples/kubernetes/ceph/csi/template $(TEMP)/ceph-csi diff --git a/images/ceph/set-ceph-debug-level b/images/ceph/set-ceph-debug-level new file mode 100755 index 000000000..7fbbe66b4 --- /dev/null +++ b/images/ceph/set-ceph-debug-level @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ -z "$1" ]; then + echo "no debug level passed choose between 0 and 20" + exit 1 +fi + +CEPH_DEBUG_LEVEL=$1 +CEPH_DEBUG_FLAG=( + lockdep + context + crush + mds + mds_balancer + mds_locker + mds_log + mds_log_expire + mds_migrator + buffer + timer + filer + striper + objecter + rados + rbd + rbd_mirror + rbd_replay + journaler + objectcacher + client + osd + optracker + objclass + filestore + journal + ms + mon + monc + paxos + tp + auth + crypto + finisher + reserver + heartbeatmap + perfcounter + rgw + rgw_sync + civetweb + javaclient + asok + throttle + refs + compressor + bluestore + bluefs + bdev + kstore + rocksdb + leveldb + memdb + fuse + mgr + mgrc + dpdk + eventtrace +) + +############# +# FUNCTIONS # +############# +check() { + ok_to_run=1 + if [[ "$CEPH_DEBUG_LEVEL" =~ ^[0-9]+$ ]]; then + if [ "$CEPH_DEBUG_LEVEL" -ge 0 ] && [ "$CEPH_DEBUG_LEVEL" -le 20 ]; then + ok_to_run=0 + fi + elif [[ "$CEPH_DEBUG_LEVEL" == "default" ]]; then + ok_to_run=0 + fi +} + +exec_ceph_command() { + local debug_level=$1 + local action=set + if [[ "$debug_level" == "default" ]]; then + action="rm" + fi + + # exec command + for flag in "${CEPH_DEBUG_FLAG[@]}"; do + ARGS=("$action" global debug_"$flag") + if [[ "$debug_level" != "default" ]]; then + ARGS+=("$debug_level") + fi + # put stdout in /dev/null since increase debug log will overflow the terminal + echo "ceph config ${ARGS[*]}" + ceph config "${ARGS[@]}" &> /dev/null & pids+=($!) + + done + echo "waiting for all the new logging configuration to be applied, this can take a few seconds" + wait "${pids[@]}" +} + +######## +# MAIN # +######## +check +if [ "$ok_to_run" -eq 0 ]; then + exec_ceph_command "$CEPH_DEBUG_LEVEL" +else + echo "Wrong debug level $CEPH_DEBUG_LEVEL" + echo "MUST be integer between 0 and 20 or 'default' to reset all values" + exit 1 +fi diff --git a/images/ceph/toolbox.sh b/images/ceph/toolbox.sh index 160ae630b..78f631b3a 100755 --- a/images/ceph/toolbox.sh +++ b/images/ceph/toolbox.sh @@ -14,12 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARGS="$@" -if [ $# -eq 0 ]; then - ARGS=/bin/bash -fi - -ROOK_DIR="/var/lib/rook" CEPH_CONFIG="/etc/ceph/ceph.conf" MON_CONFIG="/etc/rook/mon-endpoints" KEYRING_FILE="/etc/ceph/keyring" @@ -27,14 +21,15 @@ KEYRING_FILE="/etc/ceph/keyring" # create a ceph config file in its default location so ceph/rados tools can be used # without specifying any arguments write_endpoints() { - endpoints=$(cat ${MON_CONFIG}) - - # filter out the mon names - # external cluster can have numbers or hyphens in mon names, handling them in regex - mon_endpoints=$(echo ${endpoints} | sed 's/[a-z0-9_-]\+=//g') - - DATE=$(date) - echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" + endpoints=$(cat ${MON_CONFIG}) + + # filter out the mon names + # external cluster can have numbers or hyphens in mon names, handling them in regex + # shellcheck disable=SC2001 + mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g') + + DATE=$(date) + echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}" cat < ${CEPH_CONFIG} [global] mon_host = ${mon_endpoints} @@ -46,19 +41,20 @@ EOF # watch the endpoints config file and update if the mon endpoints ever change watch_endpoints() { - # get the timestamp for the target of the soft link + # get the timestamp for the target of the soft link + real_path=$(realpath ${MON_CONFIG}) + initial_time=$(stat -c %Z "${real_path}") + while true; do real_path=$(realpath ${MON_CONFIG}) - initial_time=$(stat -c %Z ${real_path}) - while true; do - real_path=$(realpath ${MON_CONFIG}) - latest_time=$(stat -c %Z ${real_path}) - - if [[ "${latest_time}" != "${initial_time}" ]]; then - write_endpoints - initial_time=${latest_time} - fi - sleep 10 - done + latest_time=$(stat -c %Z "${real_path}") + + if [[ "${latest_time}" != "${initial_time}" ]]; then + write_endpoints + initial_time=${latest_time} + fi + + sleep 10 + done } # create the keyring file