Merge pull request #7265 from leseb/toolbox-debug

ceph: ability to set cluster wide log level
2021-03-01 10:10:17 -07:00
parent bd9010e7fc c54acf449e
commit 4742400a8c
5 changed files with 183 additions and 27 deletions
@@ -23,6 +23,7 @@ If after trying the suggestions found on this page and the problem is not resolv
 * [Rook Agent modprobe exec format error](#rook-agent-modprobe-exec-format-error)
 * [Rook Agent rbd module missing error](#rook-agent-rbd-module-missing-error)
 * [Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7](#using-multiple-shared-filesystem-cephfs-is-attempted-on-a-kernel-version-older-than-47)
+* [Set debug log level for all Ceph daemons](#set-debug-log-level-for-all-ceph-daemons)
 * [Activate log to file for a particular Ceph daemon](#activate-log-to-file-for-a-particular-ceph-daemon)
 * [Flex storage class versus Ceph CSI storage class](#flex-storage-class-versus-ceph-csi-storage-class)
 * [A worker node using RBD devices hangs up](#a-worker-node-using-rbd-devices-hangs-up)
@@ -727,6 +728,33 @@ This is due to a mount flag added in the kernel version `4.7` which allows to ch

 For additional info on the kernel version requirement for multiple shared filesystems (CephFS), see [Filesystem - Kernel version requirement](ceph-filesystem.md#kernel-version-requirement).

+## Set debug log level for all Ceph daemons
+
+You can set a given log level and apply it to all the Ceph daemons at the same time.
+For this, make sure the toolbox pod is running, then determine the level you want (between 0 and 20).
+You can find the list of all subsystems and their default values in [Ceph logging and debug official guide](https://docs.ceph.com/en/latest/rados/troubleshooting/log-and-debug/#ceph-subsystems). Be careful when increasing the level as it will produce very verbose logs.
+
+Assuming you want a log level of 1, you will run:
+
+```console
+kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level 1
+```
+
+Output:
+
+>```quote
+> ceph config set global debug_context 1
+> ceph config set global debug_lockdep 1
+>...
+>...
+>```
+
+Once you are done debugging, you can revert all the debug flag to their default value by running the following:
+
+```console
+kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level default
+```
+
 ## Activate log to file for a particular Ceph daemon

 They are cases where looking at Kubernetes logs is not enough for diverse reasons, but just to name a few:
@@ -22,7 +22,7 @@ ARG TINI_VERSION
 RUN curl --fail -sSL -o /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${ARCH} && \
    chmod +x /tini

-COPY rook rookflex toolbox.sh /usr/local/bin/
+COPY rook rookflex toolbox.sh set-ceph-debug-level /usr/local/bin/
 COPY ceph-csi /etc/ceph-csi
 COPY ceph-monitoring /etc/ceph-monitoring
 COPY rook-external /etc/rook-external/
@@ -44,6 +44,7 @@ do.build: generate-csv-ceph-templates
 	@echo === container build $(CEPH_IMAGE)
 	@cp Dockerfile $(TEMP)
 	@cp toolbox.sh $(TEMP)
+	@cp set-ceph-debug-level $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rookflex $(TEMP)
 	@cp -r ../../cluster/examples/kubernetes/ceph/csi/template $(TEMP)/ceph-csi
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 The Rook Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+if [ -z "$1" ]; then
+  echo "no debug level passed choose between 0 and 20"
+  exit 1
+fi
+
+CEPH_DEBUG_LEVEL=$1
+CEPH_DEBUG_FLAG=(
+  lockdep
+  context
+  crush
+  mds
+  mds_balancer
+  mds_locker
+  mds_log
+  mds_log_expire
+  mds_migrator
+  buffer
+  timer
+  filer
+  striper
+  objecter
+  rados
+  rbd
+  rbd_mirror
+  rbd_replay
+  journaler
+  objectcacher
+  client
+  osd
+  optracker
+  objclass
+  filestore
+  journal
+  ms
+  mon
+  monc
+  paxos
+  tp
+  auth
+  crypto
+  finisher
+  reserver
+  heartbeatmap
+  perfcounter
+  rgw
+  rgw_sync
+  civetweb
+  javaclient
+  asok
+  throttle
+  refs
+  compressor
+  bluestore
+  bluefs
+  bdev
+  kstore
+  rocksdb
+  leveldb
+  memdb
+  fuse
+  mgr
+  mgrc
+  dpdk
+  eventtrace
+)
+
+#############
+# FUNCTIONS #
+#############
+check() {
+  ok_to_run=1
+  if [[ "$CEPH_DEBUG_LEVEL" =~ ^[0-9]+$ ]]; then
+    if [ "$CEPH_DEBUG_LEVEL" -ge 0 ] && [ "$CEPH_DEBUG_LEVEL" -le 20 ]; then
+      ok_to_run=0
+    fi
+    elif [[ "$CEPH_DEBUG_LEVEL" == "default" ]]; then
+    ok_to_run=0
+  fi
+}
+
+exec_ceph_command() {
+  local debug_level=$1
+  local action=set
+  if [[ "$debug_level" == "default" ]]; then
+    action="rm"
+  fi
+  
+  # exec command
+  for flag in "${CEPH_DEBUG_FLAG[@]}"; do
+    ARGS=("$action" global debug_"$flag")
+    if [[ "$debug_level" != "default" ]]; then
+      ARGS+=("$debug_level")
+    fi
+    # put stdout in /dev/null since increase debug log will overflow the terminal
+    echo "ceph config ${ARGS[*]}"
+    ceph config "${ARGS[@]}" &> /dev/null & pids+=($!)
+    
+  done
+  echo "waiting for all the new logging configuration to be applied, this can take a few seconds"
+  wait "${pids[@]}"
+}
+
+########
+# MAIN #
+########
+check
+if [ "$ok_to_run" -eq 0 ]; then
+  exec_ceph_command "$CEPH_DEBUG_LEVEL"
+else
+  echo "Wrong debug level $CEPH_DEBUG_LEVEL"
+  echo "MUST be integer between 0 and 20 or 'default' to reset all values"
+  exit 1
+fi
@@ -14,12 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-ARGS="$@"
-if [ $# -eq 0 ]; then
-    ARGS=/bin/bash
-fi
-
-ROOK_DIR="/var/lib/rook"
 CEPH_CONFIG="/etc/ceph/ceph.conf"
 MON_CONFIG="/etc/rook/mon-endpoints"
 KEYRING_FILE="/etc/ceph/keyring"
@@ -27,14 +21,15 @@ KEYRING_FILE="/etc/ceph/keyring"
 # create a ceph config file in its default location so ceph/rados tools can be used
 # without specifying any arguments
 write_endpoints() {
-    endpoints=$(cat ${MON_CONFIG})
-
-    # filter out the mon names
-    # external cluster can have numbers or hyphens in mon names, handling them in regex
-    mon_endpoints=$(echo ${endpoints} | sed 's/[a-z0-9_-]\+=//g')
-
-    DATE=$(date)
-    echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
+  endpoints=$(cat ${MON_CONFIG})
+  
+  # filter out the mon names
+  # external cluster can have numbers or hyphens in mon names, handling them in regex
+  # shellcheck disable=SC2001
+  mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
+  
+  DATE=$(date)
+  echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
    cat <<EOF > ${CEPH_CONFIG}
 [global]
 mon_host = ${mon_endpoints}
@@ -46,19 +41,20 @@ EOF

 # watch the endpoints config file and update if the mon endpoints ever change
 watch_endpoints() {
-    # get the timestamp for the target of the soft link
+  # get the timestamp for the target of the soft link
+  real_path=$(realpath ${MON_CONFIG})
+  initial_time=$(stat -c %Z "${real_path}")
+  while true; do
    real_path=$(realpath ${MON_CONFIG})
-    initial_time=$(stat -c %Z ${real_path})
-    while true; do
-       real_path=$(realpath ${MON_CONFIG})
-       latest_time=$(stat -c %Z ${real_path})
-
-       if [[ "${latest_time}" != "${initial_time}" ]]; then
-         write_endpoints
-         initial_time=${latest_time}
-       fi
-       sleep 10
-    done
+    latest_time=$(stat -c %Z "${real_path}")
+    
+    if [[ "${latest_time}" != "${initial_time}" ]]; then
+      write_endpoints
+      initial_time=${latest_time}
+    fi
+    
+    sleep 10
+  done
 }

 # create the keyring file