From 6d24ca79214a28b693a064de2ae53982663885d1 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sun, 1 Nov 2026 11:27:39 +0100 Subject: [PATCH] Monitor near-limit hardware temperatures and record cooling mitigation --- .../hardware-temperature-alerts.yaml | 28 +++++++++++++++++++ docs/alert-recovery.md | 9 ++++++ tests/alert-recovery.test.yaml | 28 +++++++++++++++++++ tests/run-alert-recovery.py | 1 + 4 files changed, 66 insertions(+) create mode 100644 apps/prometheus/resources/hardware-temperature-alerts.yaml diff --git a/apps/prometheus/resources/hardware-temperature-alerts.yaml b/apps/prometheus/resources/hardware-temperature-alerts.yaml new file mode 100644 index 0000000..1dc9630 --- /dev/null +++ b/apps/prometheus/resources/hardware-temperature-alerts.yaml @@ -0,0 +1,28 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: hardware-temperature + namespace: prometheus + labels: + release: prometheus +spec: + groups: + - name: hardware-temperature + rules: + - alert: NodeHardwareNearTemperatureLimit + expr: |- + max by (instance,chip) ( + (node_hwmon_temp_celsius{job="node-exporter"} + >= on(instance,chip,sensor) + (node_hwmon_temp_max_celsius{job="node-exporter"} - 5)) + and on(instance,chip,sensor) + (node_hwmon_temp_max_celsius{job="node-exporter"} < 200) + ) + * on(instance,chip) group_left(chip_name) node_hwmon_chip_names{job="node-exporter"} + * on(instance) group_left(nodename) node_uname_info{job="node-exporter"} + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.nodename }} hardware is near its temperature limit" + description: "{{ $labels.chip_name }} is {{ $value }} C, within 5 C of its reported maximum. Check cooling; thermal throttling can cause CPU, network and storage latency." diff --git a/docs/alert-recovery.md b/docs/alert-recovery.md index 6b853cb..1f14ee7 100644 --- a/docs/alert-recovery.md +++ b/docs/alert-recovery.md @@ -25,3 +25,12 @@ The node's small system disk also holds active container images, so deleting unused images alone may free nothing. Never remove Ceph monitor database files to make space. Ceph crash reports may be archived after reviewing the cause and verifying it has stopped; archival retains the reports for investigation. + +On 2026-11-01 odroid-h3-02 reported CPU temperature 103 C, NVMe composite +temperature 84 C (internal sensor 97 C), and Ceph osd.5 slow operations. +CPU turbo was temporarily disabled through +`/sys/devices/system/cpu/intel_pstate/no_turbo` (previously `0`, now `1`). +This runtime mitigation resets at reboot. After repairing cooling, restore +with `printf '0\n' | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo`. +There is no exposed fan/PWM control on this node. Hardware temperature rules +now warn before the sensors' own reported maximum, deduplicated per chip. diff --git a/tests/alert-recovery.test.yaml b/tests/alert-recovery.test.yaml index 8d79033..85bcda8 100644 --- a/tests/alert-recovery.test.yaml +++ b/tests/alert-recovery.test.yaml @@ -2,8 +2,36 @@ rule_files: - backup.rules.yaml - cpu.rules.yaml + - hardware.rules.yaml evaluation_interval: 1m tests: + - name: Temperature alert deduplicates chip sensors and ignores invalid limits + interval: 1m + input_series: + - series: 'node_hwmon_temp_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp1"}' + values: '103+0x20' + - series: 'node_hwmon_temp_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp2"}' + values: '103+0x20' + - series: 'node_hwmon_temp_max_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp1"}' + values: '105+0x20' + - series: 'node_hwmon_temp_max_celsius{job="node-exporter",instance="node:9100",chip="cpu",sensor="temp2"}' + values: '105+0x20' + - series: 'node_hwmon_chip_names{job="node-exporter",instance="node:9100",chip="cpu",chip_name="coretemp"}' + values: '1+0x20' + - series: 'node_uname_info{job="node-exporter",instance="node:9100",nodename="test-node"}' + values: '1+0x20' + - series: 'node_hwmon_temp_celsius{job="node-exporter",instance="node:9100",chip="bad",sensor="temp1"}' + values: '65535+0x20' + - series: 'node_hwmon_temp_max_celsius{job="node-exporter",instance="node:9100",chip="bad",sensor="temp1"}' + values: '65535+0x20' + - series: 'node_hwmon_chip_names{job="node-exporter",instance="node:9100",chip="bad",chip_name="invalid"}' + values: '1+0x20' + promql_expr_test: + - expr: 'count(ALERTS{alertname="NodeHardwareNearTemperatureLimit",alertstate="firing"})' + eval_time: 20m + exp_samples: + - labels: '{}' + value: 1 - name: Successful backups match CronJob info despite its extra schedule labels interval: 1m input_series: diff --git a/tests/run-alert-recovery.py b/tests/run-alert-recovery.py index 24e9263..db536fd 100644 --- a/tests/run-alert-recovery.py +++ b/tests/run-alert-recovery.py @@ -16,6 +16,7 @@ with tempfile.TemporaryDirectory(prefix="homelab-alert-tests-") as directory: for name, source in ( ("backup", "backup-alerts.yaml"), ("cpu", "cpu-capacity-alerts.yaml"), + ("hardware", "hardware-temperature-alerts.yaml"), ): resource = yaml.safe_load( (root / "apps/prometheus/resources" / source).read_text()