From b7edfee8249fbd62a3d9022d833d7415d07e8fa8 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sun, 1 Nov 2026 19:59:09 +0100 Subject: [PATCH] Consolidate deployed collectors and support reporter in repository --- _not_finished/bcachefs_time_stats.sh | 84 ---------------- bcachefs.sh | 2 +- bcachefs_device_time_stats.sh | 1 + bcachefs_fs_top.sh | 17 ++-- bcachefs_reconcile.sh | 113 ++++++++++++++++++++++ bcachefs_snapshot_delete_status.sh | 16 +++ bcachefs_time_stats.sh | 2 +- bcachefs_timestats.sh | 35 +++++++ kent_report.sh | 139 +++++++++++++++++++++++++++ 9 files changed, 313 insertions(+), 96 deletions(-) delete mode 100755 _not_finished/bcachefs_time_stats.sh create mode 120000 bcachefs_device_time_stats.sh create mode 100755 bcachefs_reconcile.sh create mode 100755 bcachefs_snapshot_delete_status.sh create mode 100755 bcachefs_timestats.sh create mode 100755 kent_report.sh diff --git a/_not_finished/bcachefs_time_stats.sh b/_not_finished/bcachefs_time_stats.sh deleted file mode 100755 index 7e45e69..0000000 --- a/_not_finished/bcachefs_time_stats.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -euo pipefail - -cd /sys/fs/bcachefs - -printf "%-36s %-43s %10s %12s %12s %12s %14s %14s %16s %16s\n" \ - "UUID" "STAT" "COUNT" \ - "DUR_MIN" "DUR_MAX" "DUR_TOTAL" \ - "MEAN_SINCE" "MEAN_RECENT" \ - "STDDEV_SINCE" "STDDEV_RECENT" - -for uuid in *; do - cd "$uuid" - - for stat in time_stats/* dev-*/*latency_stats* ; do - count="" - dur_min="" - dur_max="" - dur_total="" - - mean_since="" - mean_recent="" - stddev_since="" - stddev_recent="" - - in_duration=0 - - while IFS= read -r line; do - # trim leading whitespace - line="${line#"${line%%[![:space:]]*}"}" - - case "$line" in - count:*) - count="${line#count:}" - count="${count//[[:space:]]/}" - ;; - "duration of events") - in_duration=1 - ;; - "time between events") - in_duration=0 - ;; - min:*) - (( in_duration )) && dur_min="${line#min:}" - ;; - max:*) - (( in_duration )) && dur_max="${line#max:}" - ;; - total:*) - (( in_duration )) && dur_total="${line#total:}" - ;; - mean:*) - if (( in_duration )); then - rest="${line#mean:}" - read -r ms_val ms_unit mr_val mr_unit <<< "$rest" - mean_since="$ms_val $ms_unit" - mean_recent="$mr_val $mr_unit" - fi - ;; - stddev:*) - if (( in_duration )); then - rest="${line#stddev:}" - read -r ss_val ss_unit sr_val sr_unit <<< "$rest" - stddev_since="$ss_val $ss_unit" - stddev_recent="$sr_val $sr_unit" - fi - ;; - esac - done < "$stat" - - # trim leading whitespace - for v in dur_min dur_max dur_total mean_since mean_recent stddev_since stddev_recent; do - eval "$v=\"\${$v#\"\${$v%%[![:space:]]*}\"}\"" - done - - printf "%-36s %-43s %10s %12s %12s %12s %14s %14s %16s %16s\n" \ - "$uuid" "$stat" "$count" \ - "$dur_min" "$dur_max" "$dur_total" \ - "$mean_since" "$mean_recent" \ - "$stddev_since" "$stddev_recent" - done - - cd .. -done diff --git a/bcachefs.sh b/bcachefs.sh index ad4baec..63067e1 100755 --- a/bcachefs.sh +++ b/bcachefs.sh @@ -208,4 +208,4 @@ while IFS= read -r line; do #then # echo "Got line: $line" #fi -done < <(sudo bcachefs fs usage -a "$FSPATH") +done < <(bcachefs fs usage -f devices,rebalance_work,compression,btree "$FSPATH") diff --git a/bcachefs_device_time_stats.sh b/bcachefs_device_time_stats.sh new file mode 120000 index 0000000..d5b345b --- /dev/null +++ b/bcachefs_device_time_stats.sh @@ -0,0 +1 @@ +bcachefs_time_stats.sh \ No newline at end of file diff --git a/bcachefs_fs_top.sh b/bcachefs_fs_top.sh index e57d14a..053a58b 100755 --- a/bcachefs_fs_top.sh +++ b/bcachefs_fs_top.sh @@ -24,18 +24,15 @@ function h2b() } curl localhost:8000/fs_top 2>/dev/null | while IFS= read -r line; do - # data_read 0B/sec 55660544B 512118784B - if [[ "$line" =~ ^([a-zA-Z0-9_]+)[[:space:]]+([0-9.]+[BKMGTP]?)/sec[[:space:]]+([0-9.]+[BKMGTP]?)[[:space:]]+([0-9.]+[BKMGTP]?)$ ]]; then + if [[ "$line" =~ ^[[:space:]]*([a-zA-Z0-9_]+)[[:space:]]+([0-9]+)/s[[:space:]]+([0-9]+)[[:space:]]+([0-9]+)[[:space:]]*$ ]]; then stat="${BASH_REMATCH[1]}" - rate_raw="${BASH_REMATCH[2]}" - total_raw="${BASH_REMATCH[3]}" - mount_raw="${BASH_REMATCH[4]}" + rate="${BASH_REMATCH[2]}" + total="${BASH_REMATCH[3]}" + mount="${BASH_REMATCH[4]}" - rate=$(h2b "$rate_raw") - total=$(h2b "$total_raw") - mount=$(h2b "$mount_raw") - - printf "bcachefs_fs_top,stat=%s rate=%di,total=%di,mount=%di\n" \ + printf "bcachefs_fs_top,stat=%s,fspath=/mnt,uuid=3d3a0763-4dfe-41e6-93c1-8c791ec98176 rate=%si,total=%si,mount=%si\n" \ "$stat" "$rate" "$total" "$mount" + #else + # echo DOES NOT MATCH "$line" >&2 fi done diff --git a/bcachefs_reconcile.sh b/bcachefs_reconcile.sh new file mode 100755 index 0000000..6084d5e --- /dev/null +++ b/bcachefs_reconcile.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +set -euo pipefail +export LC_ALL=C + +# May be overridden by Telegraf's environment or with a positional argument. +FSPATH="${1:-${FSPATH:-/mnt}}" +UUID="${BCACHEFS_UUID:-3d3a0763-4dfe-41e6-93c1-8c791ec98176}" + +# Convert the rounded, binary human-readable values printed by bcachefs to +# bytes. The result is exact for the displayed value; bcachefs does not expose +# the additional precision that it rounded away (for example, in "4.73T"). +human_to_bytes() +{ + local value="$1" + local whole fraction unit multiplier scale bytes + + if [[ ! $value =~ ^([0-9]+)([.]([0-9]+))?([KMGTPEkmgtpe]?)(i?[Bb])?$ ]]; then + return 1 + fi + + whole="${BASH_REMATCH[1]}" + fraction="${BASH_REMATCH[3]}" + unit="${BASH_REMATCH[4]^^}" + + case "$unit" in + "") multiplier=1 ;; + K) multiplier=$((1024)) ;; + M) multiplier=$((1024 ** 2)) ;; + G) multiplier=$((1024 ** 3)) ;; + T) multiplier=$((1024 ** 4)) ;; + P) multiplier=$((1024 ** 5)) ;; + E) multiplier=$((1024 ** 6)) ;; + *) return 1 ;; + esac + + bytes=$((10#$whole * multiplier)) + if [[ -n $fraction ]]; then + scale=$((10 ** ${#fraction})) + # Round to the nearest byte instead of dropping the fractional byte. + bytes=$((bytes + (10#$fraction * multiplier + scale / 2) / scale)) + fi + + printf '%s\n' "$bytes" +} + +escape_tag() +{ + local value="$1" + value="${value//\\/\\\\}" + value="${value//,/\\,}" + value="${value// /\\ }" + value="${value//=/\\=}" + printf '%s' "$value" +} + +declare -A data_bytes=() +declare -A metadata_bytes=() +types=(replicas checksum erasure_code compression target high_priority pending stripes) +scan_pending="" +rows=0 + +# Capture everything before printing a metric: on command or parser failure, +# Telegraf gets no partial point that could look like a valid zero reading. +if ! status="$(bcachefs reconcile status "$FSPATH")"; then + echo "bcachefs reconcile status failed for $FSPATH" >&2 + exit 1 +fi + +while IFS= read -r line; do + if [[ $line =~ ^Scan[[:space:]]pending:[[:space:]]*([0-9]+)[[:space:]]*$ ]]; then + scan_pending="${BASH_REMATCH[1]}" + elif [[ $line =~ ^[[:space:]]*([[:alnum:]_]+):[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]*$ ]]; then + type="${BASH_REMATCH[1]}" + raw_data="${BASH_REMATCH[2]}" + raw_metadata="${BASH_REMATCH[3]}" + + if ! data_bytes["$type"]="$(human_to_bytes "$raw_data")" || + ! metadata_bytes["$type"]="$(human_to_bytes "$raw_metadata")"; then + echo "Unable to parse reconcile values for $type: $raw_data $raw_metadata" >&2 + exit 1 + fi + ((rows += 1)) + fi +done <<< "$status" + +if ((rows == 0)); then + echo "No reconcile values found for $FSPATH" >&2 + exit 1 +fi + +printf 'bcachefs_reconcile,fspath=%s,uuid=%s ' \ + "$(escape_tag "$FSPATH")" "$(escape_tag "$UUID")" + +first=1 +if [[ -n $scan_pending ]]; then + printf 'scan_pending=%s' "$scan_pending" + first=0 +fi + +for type in "${types[@]}"; do + # Current bcachefs versions print all types, including zero-valued ones. + # Skipping a missing type is safer than inventing a zero on format changes. + [[ -v data_bytes["$type"] ]] || continue + + ((first)) || printf ',' + printf '%s_data=%s,%s_metadata=%s' \ + "$type" "${data_bytes[$type]}" \ + "$type" "${metadata_bytes[$type]}" + first=0 +done + +printf '\n' diff --git a/bcachefs_snapshot_delete_status.sh b/bcachefs_snapshot_delete_status.sh new file mode 100755 index 0000000..4fc242a --- /dev/null +++ b/bcachefs_snapshot_delete_status.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +uuid=3d3a0763-4dfe-41e6-93c1-8c791ec98176 +fspath=/mnt + +file="/sys/fs/bcachefs/$uuid/snapshot_delete_status" + +while IFS= read -r line; do + if [[ "$line" =~ ^Progress:[[:space:]]([0-9]+)%.*,[[:space:]]done[[:space:]]([0-9]+)/([0-9]+) ]]; then + percent="${BASH_REMATCH[1]}" + done_nodes="${BASH_REMATCH[2]}" + total_nodes="${BASH_REMATCH[3]}" + + echo "bcachefs_snapshot_delete_status,uuid=$uuid,fspath=$fspath percent=$percent,done=$done_nodes,total=$total_nodes" + fi +done < "$file" diff --git a/bcachefs_time_stats.sh b/bcachefs_time_stats.sh index 2dd41f6..e697e3c 100755 --- a/bcachefs_time_stats.sh +++ b/bcachefs_time_stats.sh @@ -94,7 +94,7 @@ for uuid in *; do stddev_since_ns=$(to_ns "$stddev_since") stddev_recent_ns=$(to_ns "$stddev_recent") - printf "bcachefs_time_stats,uuid=%s,stat=%s " "$uuid" "$stat" + printf "bcachefs_time_stats,uuid=%s,fspath=/mnt,stat=%s " "$uuid" "$stat" printf "count=%si," "$count" printf "dur_min_ns=%si,dur_max_ns=%si,dur_total_ns=%si," \ "$dur_min_ns" "$dur_max_ns" "$dur_total_ns" diff --git a/bcachefs_timestats.sh b/bcachefs_timestats.sh new file mode 100755 index 0000000..2857e36 --- /dev/null +++ b/bcachefs_timestats.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euo pipefail + +bcachefs fs timestats --json --all | jq -r ' +to_entries[] as $fs +| $fs.value +| to_entries[] +| .key as $stat +| .value as $v +| "bcachefs_timestats,uuid=\($fs.key),stat=\($stat) " + + ( + [ + "count=\($v.count)i", + + "duration_min_ns=\($v.duration_ns.min)i", + "duration_max_ns=\($v.duration_ns.max)i", + "duration_total_ns=\($v.duration_ns.total)i", + "duration_mean_ns=\($v.duration_ns.mean)i", + "duration_stddev_ns=\($v.duration_ns.stddev)i", + + "duration_ewma_mean_ns=\($v.duration_ewma_ns.mean)i", + "duration_ewma_stddev_ns=\($v.duration_ewma_ns.stddev)i", + + "between_min_ns=\($v.between_ns.min)i", + "between_max_ns=\($v.between_ns.max)i", + "between_total_ns=\($v.between_ns.total)i", + "between_mean_ns=\($v.between_ns.mean)i", + "between_stddev_ns=\($v.between_ns.stddev)i", + + "between_ewma_mean_ns=\($v.between_ewma_ns.mean)i", + "between_ewma_stddev_ns=\($v.between_ewma_ns.stddev)i" + ] | join(",") + ) +' diff --git a/kent_report.sh b/kent_report.sh new file mode 100755 index 0000000..a859750 --- /dev/null +++ b/kent_report.sh @@ -0,0 +1,139 @@ +#!/bin/bash +#set -euo pipefail + +function paste_log() +{ + if [[ -z "$1" ]] ; then echo "Usage: $0 " ; exit 1 ; fi + if [[ ! -f "$1" ]] ; then echo "Usage: $0 " ; echo ; echo "File does not exist!" ; exit 2 ; fi + + POD_NAME="$(kubectl get pods | grep -Po 'brunner-ninja[^ ]+')" + if [[ -z "$POD_NAME" ]] + then + echo no pod + exit 1 + fi + + NAME="$(mktemp XXXXXXXX.log)" + kubectl cp "$1" "${POD_NAME}:/srv/http/pastes/$NAME" + echo https://brunner.ninja/pastes/$NAME +} + +( + +echo bcachefs support reporter for py1hon 1.0 +echo Date: $(date) +echo Kernel: $(uname -a) + +ver="$(pacman -Q bcachefs-dkms)" +echo "$ver" + +if [[ "$ver" =~ ^([^[:space:]]+)[[:space:]]+(.+)$ ]] ; then + ver="${BASH_REMATCH[2]}" + ver="${ver%-*}" + + # Convert Arch git-style version to git describe format + if [[ "$ver" =~ ^(.+)\.r([0-9]+)\.g([0-9a-fA-F]+)$ ]]; then + tag="${BASH_REMATCH[1]}" + rev="${BASH_REMATCH[2]}" + hash="${BASH_REMATCH[3]}" + ver="v${tag}-${rev}-g${hash}" + else + # Plain tag (no commits on top) + ver="v${ver}" + fi + + COMMIT_MESSAGE="$(git -C ~/bcachefs-tools-git/src/bcachefs-tools/ log --pretty=format:"%h %ad %s [%cn]" -n1 "$ver")" + if [[ $? -eq 0 ]] + then + echo Installed commit: "$COMMIT_MESSAGE" + echo + echo "Commits in master that are not installed on this system (yet? is this worth a reboot?):" + git -C ~/bcachefs-tools-git/src/bcachefs-tools log --pretty=format:"%h %ad %s [%cn]" "${ver}..master" -- ./libbcachefs/ + fi +else + echo "parsing arch package version failed: $ver" + exit 1 +fi + +echo +echo + +pacman -Q bcachefs-tools + +echo +echo +echo + +function print_cmd() +{ + if [[ -z "$INDEX_ONLY" ]] + then + echo + echo + echo + fi + echo "\$" "${@}" +} + +function run_cmd() +{ + print_cmd "${@}" + if [[ -z "$INDEX_ONLY" ]] + then + "${@}" + fi +} + +function do_work() +{ + run_cmd lsblk -e 7 -o NAME,KNAME,TYPE,SIZE,ROTA,SCHED,HCTL,TRAN,MODEL,SERIAL,WWN,FSTYPE,MOUNTPOINTS + run_cmd numactl --hardware + run_cmd lspci -nn + run_cmd iostat -xz -m -y 5 12 + run_cmd mpstat -P ALL 5 12 + run_cmd vmstat -w 5 12 + run_cmd iotop -b -o -t -qqq -d 5 -n 12 + run_cmd bcachefs reconcile status /mnt + run_cmd bcachefs fs timestats --once + run_cmd bcachefs fs timestats --once --sort mean-recent + run_cmd bcachefs fs timestats --once --sort dur-max + run_cmd bcachefs fs top --once /mnt + run_cmd bcachefs fs usage -h -a /mnt + run_cmd bcachefs fs top -h -n 60 -d 2 /mnt + run_cmd trace-cmd record -e bcachefs:copygc -e bcachefs:reconcile_phys -e bcachefs:data_update sleep 60 + run_cmd trace-cmd report + + for i in /sys/fs/bcachefs/*/* /sys/fs/bcachefs/*/internal/* /sys/fs/bcachefs/*/time_stats/* + do + if [[ -f "$i" ]] + then + run_cmd cat $i + fi + done + + print_cmd iostat -x -m -d 2 + if [[ -z "$INDEX_ONLY" ]] + then + timeout 1 sudo iostat -x -m -d 2 + fi + + run_cmd sudo dmesg + +# super slow since a few months (>5min) +# run_cmd sudo bcachefs fs usage -f devices,rebalance_work,compression,btree /mnt + + print_cmd sudo bcachefs show-super /dev/sda + if [[ -z "$INDEX_ONLY" ]] + then + cat /mnt/encryption_pw.txt | sudo bcachefs show-super /dev/sda + fi +} + +echo Dump-Index: The following commands will be printed in this order: +INDEX_ONLY=1 do_work + +do_work + +) | tee kent_report.log + +paste_log kent_report.log