Consolidate deployed collectors and support reporter in repository

_not_finished/bcachefs_time_stats.sh

@@ -1,84 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-cd /sys/fs/bcachefs
-
-printf "%-36s %-43s %10s %12s %12s %12s %14s %14s %16s %16s\n" \
-       "UUID" "STAT" "COUNT" \
-       "DUR_MIN" "DUR_MAX" "DUR_TOTAL" \
-       "MEAN_SINCE" "MEAN_RECENT" \
-       "STDDEV_SINCE" "STDDEV_RECENT"
-
-for uuid in *; do
-    cd "$uuid"
-
-    for stat in time_stats/* dev-*/*latency_stats* ; do
-        count=""
-        dur_min=""
-        dur_max=""
-        dur_total=""
-
-        mean_since=""
-        mean_recent=""
-        stddev_since=""
-        stddev_recent=""
-
-        in_duration=0
-
-        while IFS= read -r line; do
-            # trim leading whitespace
-            line="${line#"${line%%[![:space:]]*}"}"
-
-            case "$line" in
-                count:*)
-                    count="${line#count:}"
-                    count="${count//[[:space:]]/}"
-                    ;;
-                "duration of events")
-                    in_duration=1
-                    ;;
-                "time between events")
-                    in_duration=0
-                    ;;
-                min:*)
-                    (( in_duration )) && dur_min="${line#min:}"
-                    ;;
-                max:*)
-                    (( in_duration )) && dur_max="${line#max:}"
-                    ;;
-                total:*)
-                    (( in_duration )) && dur_total="${line#total:}"
-                    ;;
-                mean:*)
-                    if (( in_duration )); then
-                        rest="${line#mean:}"
-                        read -r ms_val ms_unit mr_val mr_unit <<< "$rest"
-                        mean_since="$ms_val $ms_unit"
-                        mean_recent="$mr_val $mr_unit"
-                    fi
-                    ;;
-                stddev:*)
-                    if (( in_duration )); then
-                        rest="${line#stddev:}"
-                        read -r ss_val ss_unit sr_val sr_unit <<< "$rest"
-                        stddev_since="$ss_val $ss_unit"
-                        stddev_recent="$sr_val $sr_unit"
-                    fi
-                    ;;
-            esac
-        done < "$stat"
-
-        # trim leading whitespace
-        for v in dur_min dur_max dur_total mean_since mean_recent stddev_since stddev_recent; do
-            eval "$v=\"\${$v#\"\${$v%%[![:space:]]*}\"}\""
-        done
-
-        printf "%-36s %-43s %10s %12s %12s %12s %14s %14s %16s %16s\n" \
-               "$uuid" "$stat" "$count" \
-               "$dur_min" "$dur_max" "$dur_total" \
-               "$mean_since" "$mean_recent" \
-               "$stddev_since" "$stddev_recent"
-    done
-
-    cd ..
-done

bcachefs.sh

@@ -208,4 +208,4 @@ while IFS= read -r line; do
     #then
     #    echo "Got line: $line"
     #fi
-done < <(sudo bcachefs fs usage -a "$FSPATH")
+done < <(bcachefs fs usage  -f devices,rebalance_work,compression,btree "$FSPATH")

bcachefs_device_time_stats.sh

@@ -0,0 +1 @@
+bcachefs_time_stats.sh

bcachefs_fs_top.sh

@@ -24,18 +24,15 @@ function h2b()
 }
 
 curl localhost:8000/fs_top 2>/dev/null | while IFS= read -r line; do
-    # data_read               0B/sec     55660544B    512118784B
-    if [[ "$line" =~ ^([a-zA-Z0-9_]+)[[:space:]]+([0-9.]+[BKMGTP]?)/sec[[:space:]]+([0-9.]+[BKMGTP]?)[[:space:]]+([0-9.]+[BKMGTP]?)$ ]]; then
+    if [[ "$line" =~ ^[[:space:]]*([a-zA-Z0-9_]+)[[:space:]]+([0-9]+)/s[[:space:]]+([0-9]+)[[:space:]]+([0-9]+)[[:space:]]*$ ]]; then
         stat="${BASH_REMATCH[1]}"
-        rate_raw="${BASH_REMATCH[2]}"
-        total_raw="${BASH_REMATCH[3]}"
-        mount_raw="${BASH_REMATCH[4]}"
+        rate="${BASH_REMATCH[2]}"
+        total="${BASH_REMATCH[3]}"
+        mount="${BASH_REMATCH[4]}"
 
-        rate=$(h2b "$rate_raw")
-        total=$(h2b "$total_raw")
-        mount=$(h2b "$mount_raw")
-
-        printf "bcachefs_fs_top,stat=%s rate=%di,total=%di,mount=%di\n" \
+        printf "bcachefs_fs_top,stat=%s,fspath=/mnt,uuid=3d3a0763-4dfe-41e6-93c1-8c791ec98176 rate=%si,total=%si,mount=%si\n" \
                "$stat" "$rate" "$total" "$mount"
+    #else
+    #    echo DOES NOT MATCH "$line" >&2
     fi
 done

bcachefs_reconcile.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+set -euo pipefail
+export LC_ALL=C
+
+# May be overridden by Telegraf's environment or with a positional argument.
+FSPATH="${1:-${FSPATH:-/mnt}}"
+UUID="${BCACHEFS_UUID:-3d3a0763-4dfe-41e6-93c1-8c791ec98176}"
+
+# Convert the rounded, binary human-readable values printed by bcachefs to
+# bytes.  The result is exact for the displayed value; bcachefs does not expose
+# the additional precision that it rounded away (for example, in "4.73T").
+human_to_bytes()
+{
+    local value="$1"
+    local whole fraction unit multiplier scale bytes
+
+    if [[ ! $value =~ ^([0-9]+)([.]([0-9]+))?([KMGTPEkmgtpe]?)(i?[Bb])?$ ]]; then
+        return 1
+    fi
+
+    whole="${BASH_REMATCH[1]}"
+    fraction="${BASH_REMATCH[3]}"
+    unit="${BASH_REMATCH[4]^^}"
+
+    case "$unit" in
+        "") multiplier=1 ;;
+        K)  multiplier=$((1024)) ;;
+        M)  multiplier=$((1024 ** 2)) ;;
+        G)  multiplier=$((1024 ** 3)) ;;
+        T)  multiplier=$((1024 ** 4)) ;;
+        P)  multiplier=$((1024 ** 5)) ;;
+        E)  multiplier=$((1024 ** 6)) ;;
+        *)  return 1 ;;
+    esac
+
+    bytes=$((10#$whole * multiplier))
+    if [[ -n $fraction ]]; then
+        scale=$((10 ** ${#fraction}))
+        # Round to the nearest byte instead of dropping the fractional byte.
+        bytes=$((bytes + (10#$fraction * multiplier + scale / 2) / scale))
+    fi
+
+    printf '%s\n' "$bytes"
+}
+
+escape_tag()
+{
+    local value="$1"
+    value="${value//\\/\\\\}"
+    value="${value//,/\\,}"
+    value="${value// /\\ }"
+    value="${value//=/\\=}"
+    printf '%s' "$value"
+}
+
+declare -A data_bytes=()
+declare -A metadata_bytes=()
+types=(replicas checksum erasure_code compression target high_priority pending stripes)
+scan_pending=""
+rows=0
+
+# Capture everything before printing a metric: on command or parser failure,
+# Telegraf gets no partial point that could look like a valid zero reading.
+if ! status="$(bcachefs reconcile status "$FSPATH")"; then
+    echo "bcachefs reconcile status failed for $FSPATH" >&2
+    exit 1
+fi
+
+while IFS= read -r line; do
+    if [[ $line =~ ^Scan[[:space:]]pending:[[:space:]]*([0-9]+)[[:space:]]*$ ]]; then
+        scan_pending="${BASH_REMATCH[1]}"
+    elif [[ $line =~ ^[[:space:]]*([[:alnum:]_]+):[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]*$ ]]; then
+        type="${BASH_REMATCH[1]}"
+        raw_data="${BASH_REMATCH[2]}"
+        raw_metadata="${BASH_REMATCH[3]}"
+
+        if ! data_bytes["$type"]="$(human_to_bytes "$raw_data")" ||
+           ! metadata_bytes["$type"]="$(human_to_bytes "$raw_metadata")"; then
+            echo "Unable to parse reconcile values for $type: $raw_data $raw_metadata" >&2
+            exit 1
+        fi
+        ((rows += 1))
+    fi
+done <<< "$status"
+
+if ((rows == 0)); then
+    echo "No reconcile values found for $FSPATH" >&2
+    exit 1
+fi
+
+printf 'bcachefs_reconcile,fspath=%s,uuid=%s ' \
+    "$(escape_tag "$FSPATH")" "$(escape_tag "$UUID")"
+
+first=1
+if [[ -n $scan_pending ]]; then
+    printf 'scan_pending=%s' "$scan_pending"
+    first=0
+fi
+
+for type in "${types[@]}"; do
+    # Current bcachefs versions print all types, including zero-valued ones.
+    # Skipping a missing type is safer than inventing a zero on format changes.
+    [[ -v data_bytes["$type"] ]] || continue
+
+    ((first)) || printf ','
+    printf '%s_data=%s,%s_metadata=%s' \
+        "$type" "${data_bytes[$type]}" \
+        "$type" "${metadata_bytes[$type]}"
+    first=0
+done
+
+printf '\n'

bcachefs_snapshot_delete_status.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+uuid=3d3a0763-4dfe-41e6-93c1-8c791ec98176
+fspath=/mnt
+
+file="/sys/fs/bcachefs/$uuid/snapshot_delete_status"
+
+while IFS= read -r line; do
+    if [[ "$line" =~ ^Progress:[[:space:]]([0-9]+)%.*,[[:space:]]done[[:space:]]([0-9]+)/([0-9]+) ]]; then
+        percent="${BASH_REMATCH[1]}"
+        done_nodes="${BASH_REMATCH[2]}"
+        total_nodes="${BASH_REMATCH[3]}"
+
+        echo "bcachefs_snapshot_delete_status,uuid=$uuid,fspath=$fspath percent=$percent,done=$done_nodes,total=$total_nodes"
+    fi
+done < "$file"

bcachefs_time_stats.sh

@@ -94,7 +94,7 @@ for uuid in *; do
         stddev_since_ns=$(to_ns "$stddev_since")
         stddev_recent_ns=$(to_ns "$stddev_recent")
 
-        printf "bcachefs_time_stats,uuid=%s,stat=%s " "$uuid" "$stat"
+        printf "bcachefs_time_stats,uuid=%s,fspath=/mnt,stat=%s " "$uuid" "$stat"
         printf "count=%si," "$count"
         printf "dur_min_ns=%si,dur_max_ns=%si,dur_total_ns=%si," \
                "$dur_min_ns" "$dur_max_ns" "$dur_total_ns"

bcachefs_timestats.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+bcachefs fs timestats --json --all | jq -r '
+to_entries[] as $fs
+| $fs.value
+| to_entries[]
+| .key as $stat
+| .value as $v
+| "bcachefs_timestats,uuid=\($fs.key),stat=\($stat) "
+  + (
+      [
+        "count=\($v.count)i",
+
+        "duration_min_ns=\($v.duration_ns.min)i",
+        "duration_max_ns=\($v.duration_ns.max)i",
+        "duration_total_ns=\($v.duration_ns.total)i",
+        "duration_mean_ns=\($v.duration_ns.mean)i",
+        "duration_stddev_ns=\($v.duration_ns.stddev)i",
+
+        "duration_ewma_mean_ns=\($v.duration_ewma_ns.mean)i",
+        "duration_ewma_stddev_ns=\($v.duration_ewma_ns.stddev)i",
+
+        "between_min_ns=\($v.between_ns.min)i",
+        "between_max_ns=\($v.between_ns.max)i",
+        "between_total_ns=\($v.between_ns.total)i",
+        "between_mean_ns=\($v.between_ns.mean)i",
+        "between_stddev_ns=\($v.between_ns.stddev)i",
+
+        "between_ewma_mean_ns=\($v.between_ewma_ns.mean)i",
+        "between_ewma_stddev_ns=\($v.between_ewma_ns.stddev)i"
+      ] | join(",")
+    )
+'

kent_report.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+#set -euo pipefail
+
+function paste_log()
+{
+    if [[ -z "$1" ]] ; then echo "Usage: $0 <filename>" ; exit 1 ; fi
+    if [[ ! -f "$1" ]] ; then echo "Usage: $0 <filename>" ; echo ; echo "File does not exist!" ; exit 2 ; fi
+
+    POD_NAME="$(kubectl get pods | grep -Po 'brunner-ninja[^ ]+')"
+    if [[ -z "$POD_NAME" ]]
+    then
+        echo no pod
+        exit 1
+    fi
+
+    NAME="$(mktemp XXXXXXXX.log)"
+    kubectl cp "$1" "${POD_NAME}:/srv/http/pastes/$NAME"
+    echo https://brunner.ninja/pastes/$NAME
+}
+
+(
+
+echo bcachefs support reporter for py1hon 1.0
+echo Date: $(date)
+echo Kernel: $(uname -a)
+
+ver="$(pacman -Q bcachefs-dkms)"
+echo "$ver"
+
+if [[ "$ver" =~ ^([^[:space:]]+)[[:space:]]+(.+)$ ]] ; then
+    ver="${BASH_REMATCH[2]}"
+    ver="${ver%-*}"
+
+    # Convert Arch git-style version to git describe format
+    if [[ "$ver" =~ ^(.+)\.r([0-9]+)\.g([0-9a-fA-F]+)$ ]]; then
+        tag="${BASH_REMATCH[1]}"
+        rev="${BASH_REMATCH[2]}"
+        hash="${BASH_REMATCH[3]}"
+        ver="v${tag}-${rev}-g${hash}"
+    else
+        # Plain tag (no commits on top)
+        ver="v${ver}"
+    fi
+
+    COMMIT_MESSAGE="$(git -C ~/bcachefs-tools-git/src/bcachefs-tools/ log --pretty=format:"%h %ad %s [%cn]" -n1 "$ver")"
+    if [[ $? -eq 0 ]]
+    then
+        echo Installed commit: "$COMMIT_MESSAGE"
+        echo
+        echo "Commits in master that are not installed on this system (yet? is this worth a reboot?):"
+        git -C ~/bcachefs-tools-git/src/bcachefs-tools log --pretty=format:"%h %ad %s [%cn]" "${ver}..master" -- ./libbcachefs/
+    fi
+else
+    echo "parsing arch package version failed: $ver"
+    exit 1
+fi
+
+echo
+echo
+
+pacman -Q bcachefs-tools
+
+echo
+echo
+echo
+
+function print_cmd()
+{
+    if [[ -z "$INDEX_ONLY" ]]
+    then
+        echo
+        echo
+        echo
+    fi
+    echo "\$" "${@}"
+}
+
+function run_cmd()
+{
+    print_cmd "${@}"
+    if [[ -z "$INDEX_ONLY" ]]
+    then
+        "${@}"
+    fi
+}
+
+function do_work()
+{
+    run_cmd lsblk -e 7 -o NAME,KNAME,TYPE,SIZE,ROTA,SCHED,HCTL,TRAN,MODEL,SERIAL,WWN,FSTYPE,MOUNTPOINTS
+    run_cmd numactl --hardware
+    run_cmd lspci -nn
+    run_cmd iostat -xz -m -y 5 12
+    run_cmd mpstat -P ALL 5 12
+    run_cmd vmstat -w 5 12
+    run_cmd iotop -b -o -t -qqq -d 5 -n 12
+    run_cmd bcachefs reconcile status /mnt
+    run_cmd bcachefs fs timestats --once
+    run_cmd bcachefs fs timestats --once --sort mean-recent
+    run_cmd bcachefs fs timestats --once --sort dur-max
+    run_cmd bcachefs fs top --once /mnt
+    run_cmd bcachefs fs usage -h -a /mnt
+    run_cmd bcachefs fs top -h -n 60 -d 2 /mnt
+    run_cmd trace-cmd record -e bcachefs:copygc -e bcachefs:reconcile_phys -e bcachefs:data_update sleep 60
+    run_cmd trace-cmd report
+
+    for i in /sys/fs/bcachefs/*/* /sys/fs/bcachefs/*/internal/* /sys/fs/bcachefs/*/time_stats/*
+    do
+        if [[ -f "$i" ]]
+        then
+            run_cmd cat $i
+        fi
+    done
+
+    print_cmd iostat -x -m -d 2
+    if [[ -z "$INDEX_ONLY" ]]
+    then
+        timeout 1 sudo iostat -x -m -d 2
+    fi
+
+    run_cmd sudo dmesg
+
+# super slow since a few months (>5min)
+#    run_cmd sudo bcachefs fs usage -f devices,rebalance_work,compression,btree /mnt
+
+    print_cmd sudo bcachefs show-super /dev/sda
+    if [[ -z "$INDEX_ONLY" ]]
+    then
+        cat /mnt/encryption_pw.txt | sudo bcachefs show-super /dev/sda
+    fi
+}
+
+echo Dump-Index: The following commands will be printed in this order:
+INDEX_ONLY=1 do_work
+
+do_work
+
+) | tee kent_report.log
+
+paste_log kent_report.log