Keep Immich mount safeguards in GitOps

apps/immich/README.md

@@ -12,11 +12,13 @@ the photo library. The server has a read-only startup check for the bcachefs
 filesystem magic and expected `upload` directory, in addition to a node label
 that prevents scheduling until the host helper confirms the mount.
 
-Before unmounting `/mnt`, run `./immich-storage-offline.sh` from the workspace
-root and wait for it to report that the server and machine-learning Pods have
-stopped. Do not unmount if that helper fails. After mounting `/mnt` again, run
-`./immich-storage-online.sh`; it validates bcachefs and the Immich upload
-directory, forces fresh Pod mounts, and then restores the readiness label.
+Before unmounting `/mnt`, run `bash apps/immich/scripts/storage-offline.sh`
+from the `homelab-gitops` repository root and wait for it to report that the
+server and machine-learning Pods have stopped. Do not unmount if that helper
+fails. After mounting `/mnt` again, run
+`bash apps/immich/scripts/storage-online.sh`; it validates bcachefs and the
+Immich upload directory, forces fresh Pod mounts, and then restores the
+readiness label.
 These helpers assume the default context `kubernetes-admin@kubernetes` and
 refuse other contexts. No systemd hook controlling `/mnt` was found during
 migration setup, so a future systemd-based mount change must explicitly order

apps/immich/scripts/storage-offline.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+readonly expected_context="kubernetes-admin@kubernetes"
+readonly node="arschrock"
+readonly ready_label="immich.brunner.ninja/library-ready"
+readonly selector='app.kubernetes.io/instance=immich,app.kubernetes.io/name in (server,machine-learning)'
+
+current_context="$(kubectl config current-context)"
+if [[ "${current_context}" != "${expected_context}" ]]; then
+  echo "Refusing to modify ${node}: context is ${current_context}, expected ${expected_context}." >&2
+  exit 1
+fi
+
+kubectl get node "${node}" >/dev/null
+if ! kubectl label node "${node}" "${ready_label}-" 2>/dev/null; then
+  labels="$(kubectl get node "${node}" --output=jsonpath='{.metadata.labels}')"
+  if [[ "${labels}" == *"${ready_label}"* ]]; then
+    echo "Could not remove the Immich library-ready label from ${node}; do not unmount /mnt." >&2
+    exit 1
+  fi
+fi
+
+mapfile -t pods < <(
+  kubectl --namespace immich get pods \
+    --selector="${selector}" \
+    --field-selector="spec.nodeName=${node}" \
+    --output=name 2>/dev/null || true
+)
+if ((${#pods[@]})); then
+  kubectl --namespace immich delete "${pods[@]}" --wait=false
+  for pod in "${pods[@]}"; do
+    kubectl --namespace immich wait --for=delete "${pod}" --timeout=5m
+  done
+fi
+
+remaining_pods="$(kubectl --namespace immich get pods \
+  --selector="${selector}" \
+  --field-selector="spec.nodeName=${node}" \
+  --output=name 2>/dev/null || true)"
+if [[ -n "${remaining_pods}" ]]; then
+  echo "Immich Pods remain on ${node}; do not unmount /mnt." >&2
+  exit 1
+fi
+
+echo "Immich server and machine-learning Pods are stopped on ${node}; /mnt can be unmounted safely."

apps/immich/scripts/storage-online.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+readonly expected_context="kubernetes-admin@kubernetes"
+readonly node="arschrock"
+readonly library_path="/mnt/Bilder"
+readonly ready_label="immich.brunner.ninja/library-ready"
+readonly selector='app.kubernetes.io/instance=immich,app.kubernetes.io/name in (server,machine-learning)'
+
+current_context="$(kubectl config current-context)"
+if [[ "${current_context}" != "${expected_context}" ]]; then
+  echo "Refusing to modify ${node}: context is ${current_context}, expected ${expected_context}." >&2
+  exit 1
+fi
+
+if ! mountpoint -q /mnt; then
+  echo "/mnt is not mounted; Immich remains storage-offline." >&2
+  exit 1
+fi
+if [[ "$(findmnt -T "${library_path}" -n -o FSTYPE)" != bcachefs ]]; then
+  echo "${library_path} is not backed by bcachefs; Immich remains storage-offline." >&2
+  exit 1
+fi
+if [[ "$(stat -f -c '%t' "${library_path}")" != ca451a4e ]]; then
+  echo "${library_path} has an unexpected filesystem; Immich remains storage-offline." >&2
+  exit 1
+fi
+if [[ ! -d "${library_path}/upload" ]]; then
+  echo "${library_path}/upload is missing; Immich remains storage-offline." >&2
+  exit 1
+fi
+
+kubectl get node "${node}" >/dev/null
+if ! kubectl label node "${node}" "${ready_label}-" 2>/dev/null; then
+  labels="$(kubectl get node "${node}" --output=jsonpath='{.metadata.labels}')"
+  if [[ "${labels}" == *"${ready_label}"* ]]; then
+    echo "Could not clear a stale Immich readiness label on ${node}." >&2
+    exit 1
+  fi
+fi
+
+# Force a fresh hostPath/local-PV mount after a mount cycle or reboot.
+mapfile -t pods < <(
+  kubectl --namespace immich get pods \
+    --selector="${selector}" \
+    --field-selector="spec.nodeName=${node}" \
+    --output=name 2>/dev/null || true
+)
+if ((${#pods[@]})); then
+  kubectl --namespace immich delete "${pods[@]}" --wait=false
+  for pod in "${pods[@]}"; do
+    kubectl --namespace immich wait --for=delete "${pod}" --timeout=5m
+  done
+fi
+
+kubectl label node "${node}" "${ready_label}=true" --overwrite
+echo "${node} is Immich-library-ready; its Immich workloads may now start."