diff --git a/apps/immich-postgresql/cluster.yaml b/apps/immich-postgresql/cluster.yaml index dda0337..60f071a 100644 --- a/apps/immich-postgresql/cluster.yaml +++ b/apps/immich-postgresql/cluster.yaml @@ -23,4 +23,6 @@ spec: size: 50Gi storageClass: rook-ceph-block-ec monitoring: - enablePodMonitor: true + customQueriesConfigMap: + - name: immich-asset-metrics + key: custom-queries diff --git a/apps/immich-postgresql/custom-queries.yaml b/apps/immich-postgresql/custom-queries.yaml new file mode 100644 index 0000000..ab532fb --- /dev/null +++ b/apps/immich-postgresql/custom-queries.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: immich-asset-metrics + namespace: immich + labels: + cnpg.io/reload: "" +data: + custom-queries: | + immich_user_assets: + primary: true + target_databases: + - immich + query: | + SELECT user_guid, photos, videos, storage_bytes + FROM public.immich_monitoring_user_assets + metrics: + - user_guid: + usage: LABEL + description: Immich user GUID + - photos: + usage: GAUGE + description: Active photos owned by the user + - videos: + usage: GAUGE + description: Active videos owned by the user + - storage_bytes: + usage: GAUGE + description: Immich-reported storage used by the user in bytes diff --git a/apps/immich-postgresql/kustomization.yaml b/apps/immich-postgresql/kustomization.yaml index e35ee07..5fe9319 100644 --- a/apps/immich-postgresql/kustomization.yaml +++ b/apps/immich-postgresql/kustomization.yaml @@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cluster.yaml +- custom-queries.yaml - pod-monitor.yaml diff --git a/apps/immich-postgresql/metrics-view.sql b/apps/immich-postgresql/metrics-view.sql new file mode 100644 index 0000000..8724209 --- /dev/null +++ b/apps/immich-postgresql/metrics-view.sql @@ -0,0 +1,17 @@ +CREATE OR REPLACE VIEW public.immich_monitoring_user_assets AS +SELECT + u.id::text AS user_guid, + count(a.id) FILTER (WHERE a.type = 'IMAGE')::bigint AS photos, + count(a.id) FILTER (WHERE a.type = 'VIDEO')::bigint AS videos, + coalesce(u."quotaUsageInBytes", 0)::bigint AS storage_bytes +FROM public."user" AS u +LEFT JOIN public.asset AS a + ON a."ownerId" = u.id + AND a."deletedAt" IS NULL + AND a.status = 'active' +WHERE u."deletedAt" IS NULL + AND u.status = 'active' +GROUP BY u.id, u."quotaUsageInBytes"; + +GRANT USAGE ON SCHEMA public TO cnpg_metrics_exporter; +GRANT SELECT ON public.immich_monitoring_user_assets TO cnpg_metrics_exporter; diff --git a/apps/immich-postgresql/scripts/ensure-metrics-view.sh b/apps/immich-postgresql/scripts/ensure-metrics-view.sh new file mode 100644 index 0000000..55562e4 --- /dev/null +++ b/apps/immich-postgresql/scripts/ensure-metrics-view.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$(kubectl config current-context)" != "kubernetes-admin@kubernetes" ]]; then + echo "Refusing to modify Immich PostgreSQL outside the homelab context." >&2 + exit 1 +fi + +app_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +primary="$(kubectl --namespace immich get pods \ + --selector='cnpg.io/cluster=immich-postgresql,cnpg.io/instanceRole=primary' \ + --output=jsonpath='{.items[0].metadata.name}')" +if [[ -z "${primary}" ]]; then + echo "No Immich PostgreSQL primary is available." >&2 + exit 1 +fi + +kubectl --namespace immich exec -i "${primary}" --container postgres -- \ + psql --set=ON_ERROR_STOP=1 --username=postgres --dbname=immich \ + < "${app_dir}/metrics-view.sql" diff --git a/apps/immich/README.md b/apps/immich/README.md index 1ff8761..d7a2499 100644 --- a/apps/immich/README.md +++ b/apps/immich/README.md @@ -30,6 +30,17 @@ the source database's PostgreSQL 18.6 and pgvector extension versions. The shared Valkey instance is not copied wholesale: only keys identified as Immich-owned are imported, leaving all other applications' keys untouched. +## Monitoring + +The dashboard combines Immich's own metrics with container and CloudNativePG +metrics. CloudNativePG's custom query reads an aggregate-only database view to +show photos, videos, and Immich-reported storage by user GUID. It runs on the +primary only. The exporter can read this view but not the underlying asset or +user tables. After restoring the database from a backup, run +`bash apps/immich-postgresql/scripts/ensure-metrics-view.sh` from the GitOps +repository root to recreate the view and its grant if necessary. The storage +figure comes from Immich's `quotaUsageInBytes`, not a filesystem scan. + The old Podman stack and host PostgreSQL must remain available as rollback sources until the new application, database, monitoring, and routes have been verified. Never run both Immich servers against `/mnt/Bilder` concurrently. @@ -38,7 +49,7 @@ The migration dumps are retained with mode `0600` under `/home/feedc0de/immich-app/backups/k8s-migration-20261022/`. The source PostgreSQL database and shared Valkey keys were left intact. After the new instance begins accepting writes, a rollback must first stop the Kubernetes -server and ML pods with `./immich-storage-offline.sh`, then dump the *current* +server and ML pods with `bash apps/immich/scripts/storage-offline.sh`, then dump the *current* Kubernetes database and restore it to the preserved host PostgreSQL before starting Podman. The new database stores the media root as `/usr/src/app/upload`; the old Compose file mounts it at `/data`, so change that diff --git a/apps/immich/resources/dashboard.yaml b/apps/immich/resources/dashboard.yaml index 0596bc0..17f557a 100644 --- a/apps/immich/resources/dashboard.yaml +++ b/apps/immich/resources/dashboard.yaml @@ -14,15 +14,131 @@ data: "timezone": "browser", "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": {"from": "now-6h", "to": "now"}, "panels": [ + { + "id": 6, + "title": "Users", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4}, + "targets": [{"refId": "A", "expr": "max(immich_users_total{namespace=\"immich\",endpoint=\"metrics-api\"})", "instant": true}], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 7, + "title": "Photos", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4}, + "targets": [{"refId": "A", "expr": "sum(cnpg_immich_user_assets_photos{namespace=\"immich\"})", "instant": true}], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 8, + "title": "Videos", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4}, + "targets": [{"refId": "A", "expr": "sum(cnpg_immich_user_assets_videos{namespace=\"immich\"})", "instant": true}], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 9, + "title": "Immich-reported storage", + "description": "Sum of active users' quotaUsageInBytes; this is not the bcachefs volume's physical usage.", + "type": "stat", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 18, "y": 0, "w": 6, "h": 4}, + "targets": [{"refId": "A", "expr": "sum(cnpg_immich_user_assets_storage_bytes{namespace=\"immich\"})", "instant": true}], + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 10, + "title": "Top users by photos", + "type": "bargauge", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, + "targets": [{"refId": "A", "expr": "topk(10, cnpg_immich_user_assets_photos{namespace=\"immich\"})", "instant": true, "legendFormat": "{{user_guid}}"}], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 11, + "title": "Top users by videos", + "type": "bargauge", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, + "targets": [{"refId": "A", "expr": "topk(10, cnpg_immich_user_assets_videos{namespace=\"immich\"})", "instant": true, "legendFormat": "{{user_guid}}"}], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 12, + "title": "Top users by storage", + "type": "bargauge", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, + "targets": [{"refId": "A", "expr": "topk(10, cnpg_immich_user_assets_storage_bytes{namespace=\"immich\"})", "instant": true, "legendFormat": "{{user_guid}}"}], + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 13, + "title": "Library growth", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 7}, + "targets": [ + {"refId": "A", "expr": "sum(cnpg_immich_user_assets_photos{namespace=\"immich\"})", "legendFormat": "Photos"}, + {"refId": "B", "expr": "sum(cnpg_immich_user_assets_videos{namespace=\"immich\"})", "legendFormat": "Videos"} + ], + "fieldConfig": {"defaults": {"unit": "none", "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10}}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}} + }, + { + "id": 14, + "title": "Active background jobs", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 12, "y": 12, "w": 12, "h": 7}, + "targets": [ + {"refId": "A", "expr": "immich_queues_thumbnail_generation_active{namespace=\"immich\"}", "legendFormat": "Thumbnails"}, + {"refId": "B", "expr": "immich_queues_video_conversion_active{namespace=\"immich\"}", "legendFormat": "Video conversion"}, + {"refId": "C", "expr": "immich_queues_smart_search_active{namespace=\"immich\"}", "legendFormat": "Smart search"}, + {"refId": "D", "expr": "immich_queues_face_detection_active{namespace=\"immich\"}", "legendFormat": "Face detection"}, + {"refId": "E", "expr": "immich_queues_facial_recognition_active{namespace=\"immich\"}", "legendFormat": "Facial recognition"}, + {"refId": "F", "expr": "immich_queues_ocr_active{namespace=\"immich\"}", "legendFormat": "OCR"} + ], + "fieldConfig": {"defaults": {"unit": "none", "custom": {"drawStyle": "line", "lineWidth": 2}}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}} + }, + { + "id": 15, + "title": "Completed background jobs per minute", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, + "gridPos": {"x": 0, "y": 19, "w": 24, "h": 7}, + "targets": [ + {"refId": "A", "expr": "60 * rate(immich_jobs_asset_generate_thumbnails_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Thumbnails"}, + {"refId": "B", "expr": "60 * rate(immich_jobs_asset_encode_video_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Video conversion"}, + {"refId": "C", "expr": "60 * rate(immich_jobs_smart_search_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Smart search"}, + {"refId": "D", "expr": "60 * rate(immich_jobs_facial_recognition_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Facial recognition"} + ], + "fieldConfig": {"defaults": {"unit": "none", "custom": {"drawStyle": "line", "lineWidth": 2}}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}} + }, { "id": 1, "title": "CPU by pod and container", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, - "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, + "gridPos": {"x": 0, "y": 26, "w": 12, "h": 8}, "targets": [{"refId": "A", "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",container!=\"\",container!=\"POD\"}[5m]))", "legendFormat": "{{pod}} / {{container}}"}], "fieldConfig": {"defaults": {"unit": "cores", "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi", "sort": "desc"}} @@ -32,7 +148,7 @@ data: "title": "Memory working set by pod and container", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, - "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, + "gridPos": {"x": 12, "y": 26, "w": 12, "h": 8}, "targets": [{"refId": "A", "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",container!=\"\",container!=\"POD\"})", "legendFormat": "{{pod}} / {{container}}"}], "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi", "sort": "desc"}} @@ -42,7 +158,7 @@ data: "title": "Network throughput by pod", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, - "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, + "gridPos": {"x": 0, "y": 34, "w": 12, "h": 8}, "targets": [ {"refId": "A", "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",interface!=\"lo\"}[5m]))", "legendFormat": "{{pod}} receive"}, {"refId": "B", "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",interface!=\"lo\"}[5m]))", "legendFormat": "{{pod}} transmit"} @@ -55,7 +171,7 @@ data: "title": "Immich database size", "type": "stat", "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, - "gridPos": {"x": 12, "y": 8, "w": 6, "h": 4}, + "gridPos": {"x": 12, "y": 34, "w": 12, "h": 4}, "targets": [{"refId": "A", "expr": "max(cnpg_pg_database_size_bytes{namespace=\"immich\",datname=\"immich\"})", "instant": true, "legendFormat": "immich"}], "fieldConfig": {"defaults": {"unit": "bytes", "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}}, "overrides": []}, "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto"} @@ -65,7 +181,7 @@ data: "title": "PostgreSQL replication lag", "type": "timeseries", "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"}, - "gridPos": {"x": 12, "y": 12, "w": 12, "h": 4}, + "gridPos": {"x": 12, "y": 38, "w": 12, "h": 4}, "targets": [{"refId": "A", "expr": "max by (pod) (cnpg_pg_replication_lag{namespace=\"immich\"})", "legendFormat": "{{pod}}"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"drawStyle": "line", "lineWidth": 2}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}