Monitor near-limit hardware temperatures and record cooling mitigation

apps/prometheus/resources/hardware-temperature-alerts.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: hardware-temperature
+  namespace: prometheus
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: hardware-temperature
+      rules:
+        - alert: NodeHardwareNearTemperatureLimit
+          expr: |-
+            max by (instance,chip) (
+              (node_hwmon_temp_celsius{job="node-exporter"}
+                >= on(instance,chip,sensor)
+                (node_hwmon_temp_max_celsius{job="node-exporter"} - 5))
+              and on(instance,chip,sensor)
+              (node_hwmon_temp_max_celsius{job="node-exporter"} < 200)
+            )
+            * on(instance,chip) group_left(chip_name) node_hwmon_chip_names{job="node-exporter"}
+            * on(instance) group_left(nodename) node_uname_info{job="node-exporter"}
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.nodename }} hardware is near its temperature limit"
+            description: "{{ $labels.chip_name }} is {{ $value }} C, within 5 C of its reported maximum. Check cooling; thermal throttling can cause CPU, network and storage latency."

docs/alert-recovery.md

@@ -25,3 +25,12 @@ The node's small system disk also holds active container images, so deleting
 unused images alone may free nothing. Never remove Ceph monitor database files
 to make space. Ceph crash reports may be archived after reviewing the cause
 and verifying it has stopped; archival retains the reports for investigation.
+
+On 2026-11-01 odroid-h3-02 reported CPU temperature 103 C, NVMe composite
+temperature 84 C (internal sensor 97 C), and Ceph osd.5 slow operations.
+CPU turbo was temporarily disabled through
+`/sys/devices/system/cpu/intel_pstate/no_turbo` (previously `0`, now `1`).
+This runtime mitigation resets at reboot. After repairing cooling, restore
+with `printf '0\n' | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo`.
+There is no exposed fan/PWM control on this node. Hardware temperature rules
+now warn before the sensors' own reported maximum, deduplicated per chip.

tests/alert-recovery.test.yaml

@@ -2,8 +2,36 @@
 rule_files:
   - backup.rules.yaml
   - cpu.rules.yaml
+  - hardware.rules.yaml
 evaluation_interval: 1m
 tests:
+  - name: Temperature alert deduplicates chip sensors and ignores invalid limits
+    interval: 1m
+    input_series:
+      - series: 'node_hwmon_temp_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp1"}'
+        values: '103+0x20'
+      - series: 'node_hwmon_temp_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp2"}'
+        values: '103+0x20'
+      - series: 'node_hwmon_temp_max_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp1"}'
+        values: '105+0x20'
+      - series: 'node_hwmon_temp_max_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp2"}'
+        values: '105+0x20'
+      - series: 'node_hwmon_chip_names{job="node-exporter",instance="node:9100",chip="cpu",chip_name="coretemp"}'
+        values: '1+0x20'
+      - series: 'node_uname_info{job="node-exporter",instance="node:9100",nodename="test-node"}'
+        values: '1+0x20'
+      - series: 'node_hwmon_temp_celsius{job="node-exporter",instance="node:9100",chip="bad",sensor="temp1"}'
+        values: '65535+0x20'
+      - series: 'node_hwmon_temp_max_celsius{job="node-exporter",instance="node:9100",chip="bad",sensor="temp1"}'
+        values: '65535+0x20'
+      - series: 'node_hwmon_chip_names{job="node-exporter",instance="node:9100",chip="bad",chip_name="invalid"}'
+        values: '1+0x20'
+    promql_expr_test:
+      - expr: 'count(ALERTS{alertname="NodeHardwareNearTemperatureLimit",alertstate="firing"})'
+        eval_time: 20m
+        exp_samples:
+          - labels: '{}'
+            value: 1
   - name: Successful backups match CronJob info despite its extra schedule labels
     interval: 1m
     input_series:

tests/run-alert-recovery.py

@@ -16,6 +16,7 @@ with tempfile.TemporaryDirectory(prefix="homelab-alert-tests-") as directory:
     for name, source in (
         ("backup", "backup-alerts.yaml"),
         ("cpu", "cpu-capacity-alerts.yaml"),
+        ("hardware", "hardware-temperature-alerts.yaml"),
     ):
         resource = yaml.safe_load(
             (root / "apps/prometheus/resources" / source).read_text()