Add Immich user and library metrics dashboard

apps/immich-postgresql/cluster.yaml

@@ -23,4 +23,6 @@ spec:
     size: 50Gi
     storageClass: rook-ceph-block-ec
   monitoring:
-    enablePodMonitor: true
+    customQueriesConfigMap:
+      - name: immich-asset-metrics
+        key: custom-queries

apps/immich-postgresql/custom-queries.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: immich-asset-metrics
+  namespace: immich
+  labels:
+    cnpg.io/reload: ""
+data:
+  custom-queries: |
+    immich_user_assets:
+      primary: true
+      target_databases:
+        - immich
+      query: |
+        SELECT user_guid, photos, videos, storage_bytes
+        FROM public.immich_monitoring_user_assets
+      metrics:
+        - user_guid:
+            usage: LABEL
+            description: Immich user GUID
+        - photos:
+            usage: GAUGE
+            description: Active photos owned by the user
+        - videos:
+            usage: GAUGE
+            description: Active videos owned by the user
+        - storage_bytes:
+            usage: GAUGE
+            description: Immich-reported storage used by the user in bytes

apps/immich-postgresql/kustomization.yaml

@@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - cluster.yaml
+- custom-queries.yaml
 - pod-monitor.yaml

apps/immich-postgresql/metrics-view.sql

@@ -0,0 +1,17 @@
+CREATE OR REPLACE VIEW public.immich_monitoring_user_assets AS
+SELECT
+  u.id::text AS user_guid,
+  count(a.id) FILTER (WHERE a.type = 'IMAGE')::bigint AS photos,
+  count(a.id) FILTER (WHERE a.type = 'VIDEO')::bigint AS videos,
+  coalesce(u."quotaUsageInBytes", 0)::bigint AS storage_bytes
+FROM public."user" AS u
+LEFT JOIN public.asset AS a
+  ON a."ownerId" = u.id
+  AND a."deletedAt" IS NULL
+  AND a.status = 'active'
+WHERE u."deletedAt" IS NULL
+  AND u.status = 'active'
+GROUP BY u.id, u."quotaUsageInBytes";
+
+GRANT USAGE ON SCHEMA public TO cnpg_metrics_exporter;
+GRANT SELECT ON public.immich_monitoring_user_assets TO cnpg_metrics_exporter;

apps/immich-postgresql/scripts/ensure-metrics-view.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$(kubectl config current-context)" != "kubernetes-admin@kubernetes" ]]; then
+  echo "Refusing to modify Immich PostgreSQL outside the homelab context." >&2
+  exit 1
+fi
+
+app_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+primary="$(kubectl --namespace immich get pods \
+  --selector='cnpg.io/cluster=immich-postgresql,cnpg.io/instanceRole=primary' \
+  --output=jsonpath='{.items[0].metadata.name}')"
+if [[ -z "${primary}" ]]; then
+  echo "No Immich PostgreSQL primary is available." >&2
+  exit 1
+fi
+
+kubectl --namespace immich exec -i "${primary}" --container postgres -- \
+  psql --set=ON_ERROR_STOP=1 --username=postgres --dbname=immich \
+  < "${app_dir}/metrics-view.sql"

apps/immich/README.md

@@ -30,6 +30,17 @@ the source database's PostgreSQL 18.6 and pgvector extension versions. The
 shared Valkey instance is not copied wholesale: only keys identified as
 Immich-owned are imported, leaving all other applications' keys untouched.
 
+## Monitoring
+
+The dashboard combines Immich's own metrics with container and CloudNativePG
+metrics. CloudNativePG's custom query reads an aggregate-only database view to
+show photos, videos, and Immich-reported storage by user GUID. It runs on the
+primary only. The exporter can read this view but not the underlying asset or
+user tables. After restoring the database from a backup, run
+`bash apps/immich-postgresql/scripts/ensure-metrics-view.sh` from the GitOps
+repository root to recreate the view and its grant if necessary. The storage
+figure comes from Immich's `quotaUsageInBytes`, not a filesystem scan.
+
 The old Podman stack and host PostgreSQL must remain available as rollback
 sources until the new application, database, monitoring, and routes have been
 verified. Never run both Immich servers against `/mnt/Bilder` concurrently.
@@ -38,7 +49,7 @@ The migration dumps are retained with mode `0600` under
 `/home/feedc0de/immich-app/backups/k8s-migration-20261022/`. The source
 PostgreSQL database and shared Valkey keys were left intact. After the new
 instance begins accepting writes, a rollback must first stop the Kubernetes
-server and ML pods with `./immich-storage-offline.sh`, then dump the *current*
+server and ML pods with `bash apps/immich/scripts/storage-offline.sh`, then dump the *current*
 Kubernetes database and restore it to the preserved host PostgreSQL before
 starting Podman. The new database stores the media root as
 `/usr/src/app/upload`; the old Compose file mounts it at `/data`, so change that

apps/immich/resources/dashboard.yaml

@@ -14,15 +14,131 @@ data:
       "timezone": "browser",
       "schemaVersion": 39,
       "version": 1,
-      "refresh": "30s",
+      "refresh": "1m",
       "time": {"from": "now-6h", "to": "now"},
       "panels": [
+        {
+          "id": 6,
+          "title": "Users",
+          "type": "stat",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
+          "targets": [{"refId": "A", "expr": "max(immich_users_total{namespace=\"immich\",endpoint=\"metrics-api\"})", "instant": true}],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
+          "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 7,
+          "title": "Photos",
+          "type": "stat",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
+          "targets": [{"refId": "A", "expr": "sum(cnpg_immich_user_assets_photos{namespace=\"immich\"})", "instant": true}],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
+          "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 8,
+          "title": "Videos",
+          "type": "stat",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4},
+          "targets": [{"refId": "A", "expr": "sum(cnpg_immich_user_assets_videos{namespace=\"immich\"})", "instant": true}],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
+          "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 9,
+          "title": "Immich-reported storage",
+          "description": "Sum of active users' quotaUsageInBytes; this is not the bcachefs volume's physical usage.",
+          "type": "stat",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 18, "y": 0, "w": 6, "h": 4},
+          "targets": [{"refId": "A", "expr": "sum(cnpg_immich_user_assets_storage_bytes{namespace=\"immich\"})", "instant": true}],
+          "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
+          "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 10,
+          "title": "Top users by photos",
+          "type": "bargauge",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8},
+          "targets": [{"refId": "A", "expr": "topk(10, cnpg_immich_user_assets_photos{namespace=\"immich\"})", "instant": true, "legendFormat": "{{user_guid}}"}],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
+          "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 11,
+          "title": "Top users by videos",
+          "type": "bargauge",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8},
+          "targets": [{"refId": "A", "expr": "topk(10, cnpg_immich_user_assets_videos{namespace=\"immich\"})", "instant": true, "legendFormat": "{{user_guid}}"}],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
+          "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 12,
+          "title": "Top users by storage",
+          "type": "bargauge",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8},
+          "targets": [{"refId": "A", "expr": "topk(10, cnpg_immich_user_assets_storage_bytes{namespace=\"immich\"})", "instant": true, "legendFormat": "{{user_guid}}"}],
+          "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
+          "options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+        },
+        {
+          "id": 13,
+          "title": "Library growth",
+          "type": "timeseries",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 0, "y": 12, "w": 12, "h": 7},
+          "targets": [
+            {"refId": "A", "expr": "sum(cnpg_immich_user_assets_photos{namespace=\"immich\"})", "legendFormat": "Photos"},
+            {"refId": "B", "expr": "sum(cnpg_immich_user_assets_videos{namespace=\"immich\"})", "legendFormat": "Videos"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "none", "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10}}, "overrides": []},
+          "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}
+        },
+        {
+          "id": 14,
+          "title": "Active background jobs",
+          "type": "timeseries",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 12, "y": 12, "w": 12, "h": 7},
+          "targets": [
+            {"refId": "A", "expr": "immich_queues_thumbnail_generation_active{namespace=\"immich\"}", "legendFormat": "Thumbnails"},
+            {"refId": "B", "expr": "immich_queues_video_conversion_active{namespace=\"immich\"}", "legendFormat": "Video conversion"},
+            {"refId": "C", "expr": "immich_queues_smart_search_active{namespace=\"immich\"}", "legendFormat": "Smart search"},
+            {"refId": "D", "expr": "immich_queues_face_detection_active{namespace=\"immich\"}", "legendFormat": "Face detection"},
+            {"refId": "E", "expr": "immich_queues_facial_recognition_active{namespace=\"immich\"}", "legendFormat": "Facial recognition"},
+            {"refId": "F", "expr": "immich_queues_ocr_active{namespace=\"immich\"}", "legendFormat": "OCR"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "none", "custom": {"drawStyle": "line", "lineWidth": 2}}, "overrides": []},
+          "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}
+        },
+        {
+          "id": 15,
+          "title": "Completed background jobs per minute",
+          "type": "timeseries",
+          "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
+          "gridPos": {"x": 0, "y": 19, "w": 24, "h": 7},
+          "targets": [
+            {"refId": "A", "expr": "60 * rate(immich_jobs_asset_generate_thumbnails_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Thumbnails"},
+            {"refId": "B", "expr": "60 * rate(immich_jobs_asset_encode_video_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Video conversion"},
+            {"refId": "C", "expr": "60 * rate(immich_jobs_smart_search_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Smart search"},
+            {"refId": "D", "expr": "60 * rate(immich_jobs_facial_recognition_success_total{namespace=\"immich\"}[5m])", "legendFormat": "Facial recognition"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "none", "custom": {"drawStyle": "line", "lineWidth": 2}}, "overrides": []},
+          "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}
+        },
         {
           "id": 1,
           "title": "CPU by pod and container",
           "type": "timeseries",
           "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
-          "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8},
+          "gridPos": {"x": 0, "y": 26, "w": 12, "h": 8},
           "targets": [{"refId": "A", "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",container!=\"\",container!=\"POD\"}[5m]))", "legendFormat": "{{pod}} / {{container}}"}],
           "fieldConfig": {"defaults": {"unit": "cores", "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10}}, "overrides": []},
           "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi", "sort": "desc"}}
@@ -32,7 +148,7 @@ data:
           "title": "Memory working set by pod and container",
           "type": "timeseries",
           "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
-          "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8},
+          "gridPos": {"x": 12, "y": 26, "w": 12, "h": 8},
           "targets": [{"refId": "A", "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",container!=\"\",container!=\"POD\"})", "legendFormat": "{{pod}} / {{container}}"}],
           "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 10}}, "overrides": []},
           "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi", "sort": "desc"}}
@@ -42,7 +158,7 @@ data:
           "title": "Network throughput by pod",
           "type": "timeseries",
           "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
-          "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
+          "gridPos": {"x": 0, "y": 34, "w": 12, "h": 8},
           "targets": [
             {"refId": "A", "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",interface!=\"lo\"}[5m]))", "legendFormat": "{{pod}} receive"},
             {"refId": "B", "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"immich\",pod=~\"immich-(server|machine-learning|valkey|postgresql-[0-9]+).*\",interface!=\"lo\"}[5m]))", "legendFormat": "{{pod}} transmit"}
@@ -55,7 +171,7 @@ data:
           "title": "Immich database size",
           "type": "stat",
           "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
-          "gridPos": {"x": 12, "y": 8, "w": 6, "h": 4},
+          "gridPos": {"x": 12, "y": 34, "w": 12, "h": 4},
           "targets": [{"refId": "A", "expr": "max(cnpg_pg_database_size_bytes{namespace=\"immich\",datname=\"immich\"})", "instant": true, "legendFormat": "immich"}],
           "fieldConfig": {"defaults": {"unit": "bytes", "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}}, "overrides": []},
           "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto"}
@@ -65,7 +181,7 @@ data:
           "title": "PostgreSQL replication lag",
           "type": "timeseries",
           "datasource": {"type": "prometheus", "uid": "aevstx976yku8a"},
-          "gridPos": {"x": 12, "y": 12, "w": 12, "h": 4},
+          "gridPos": {"x": 12, "y": 38, "w": 12, "h": 4},
           "targets": [{"refId": "A", "expr": "max by (pod) (cnpg_pg_replication_lag{namespace=\"immich\"})", "legendFormat": "{{pod}}"}],
           "fieldConfig": {"defaults": {"unit": "s", "custom": {"drawStyle": "line", "lineWidth": 2}}, "overrides": []},
           "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}