diff --git a/apps/immich/README.md b/apps/immich/README.md index 34bb6a5..ea1cf92 100644 --- a/apps/immich/README.md +++ b/apps/immich/README.md @@ -12,11 +12,13 @@ the photo library. The server has a read-only startup check for the bcachefs filesystem magic and expected `upload` directory, in addition to a node label that prevents scheduling until the host helper confirms the mount. -Before unmounting `/mnt`, run `./immich-storage-offline.sh` from the workspace -root and wait for it to report that the server and machine-learning Pods have -stopped. Do not unmount if that helper fails. After mounting `/mnt` again, run -`./immich-storage-online.sh`; it validates bcachefs and the Immich upload -directory, forces fresh Pod mounts, and then restores the readiness label. +Before unmounting `/mnt`, run `bash apps/immich/scripts/storage-offline.sh` +from the `homelab-gitops` repository root and wait for it to report that the +server and machine-learning Pods have stopped. Do not unmount if that helper +fails. After mounting `/mnt` again, run +`bash apps/immich/scripts/storage-online.sh`; it validates bcachefs and the +Immich upload directory, forces fresh Pod mounts, and then restores the +readiness label. These helpers assume the default context `kubernetes-admin@kubernetes` and refuse other contexts. No systemd hook controlling `/mnt` was found during migration setup, so a future systemd-based mount change must explicitly order diff --git a/apps/immich/scripts/storage-offline.sh b/apps/immich/scripts/storage-offline.sh new file mode 100644 index 0000000..fe39aaa --- /dev/null +++ b/apps/immich/scripts/storage-offline.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly expected_context="kubernetes-admin@kubernetes" +readonly node="arschrock" +readonly ready_label="immich.brunner.ninja/library-ready" +readonly selector='app.kubernetes.io/instance=immich,app.kubernetes.io/name in (server,machine-learning)' + +current_context="$(kubectl config current-context)" +if [[ "${current_context}" != "${expected_context}" ]]; then + echo "Refusing to modify ${node}: context is ${current_context}, expected ${expected_context}." >&2 + exit 1 +fi + +kubectl get node "${node}" >/dev/null +if ! kubectl label node "${node}" "${ready_label}-" 2>/dev/null; then + labels="$(kubectl get node "${node}" --output=jsonpath='{.metadata.labels}')" + if [[ "${labels}" == *"${ready_label}"* ]]; then + echo "Could not remove the Immich library-ready label from ${node}; do not unmount /mnt." >&2 + exit 1 + fi +fi + +mapfile -t pods < <( + kubectl --namespace immich get pods \ + --selector="${selector}" \ + --field-selector="spec.nodeName=${node}" \ + --output=name 2>/dev/null || true +) +if ((${#pods[@]})); then + kubectl --namespace immich delete "${pods[@]}" --wait=false + for pod in "${pods[@]}"; do + kubectl --namespace immich wait --for=delete "${pod}" --timeout=5m + done +fi + +remaining_pods="$(kubectl --namespace immich get pods \ + --selector="${selector}" \ + --field-selector="spec.nodeName=${node}" \ + --output=name 2>/dev/null || true)" +if [[ -n "${remaining_pods}" ]]; then + echo "Immich Pods remain on ${node}; do not unmount /mnt." >&2 + exit 1 +fi + +echo "Immich server and machine-learning Pods are stopped on ${node}; /mnt can be unmounted safely." diff --git a/apps/immich/scripts/storage-online.sh b/apps/immich/scripts/storage-online.sh new file mode 100644 index 0000000..0bd3c9e --- /dev/null +++ b/apps/immich/scripts/storage-online.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly expected_context="kubernetes-admin@kubernetes" +readonly node="arschrock" +readonly library_path="/mnt/Bilder" +readonly ready_label="immich.brunner.ninja/library-ready" +readonly selector='app.kubernetes.io/instance=immich,app.kubernetes.io/name in (server,machine-learning)' + +current_context="$(kubectl config current-context)" +if [[ "${current_context}" != "${expected_context}" ]]; then + echo "Refusing to modify ${node}: context is ${current_context}, expected ${expected_context}." >&2 + exit 1 +fi + +if ! mountpoint -q /mnt; then + echo "/mnt is not mounted; Immich remains storage-offline." >&2 + exit 1 +fi +if [[ "$(findmnt -T "${library_path}" -n -o FSTYPE)" != bcachefs ]]; then + echo "${library_path} is not backed by bcachefs; Immich remains storage-offline." >&2 + exit 1 +fi +if [[ "$(stat -f -c '%t' "${library_path}")" != ca451a4e ]]; then + echo "${library_path} has an unexpected filesystem; Immich remains storage-offline." >&2 + exit 1 +fi +if [[ ! -d "${library_path}/upload" ]]; then + echo "${library_path}/upload is missing; Immich remains storage-offline." >&2 + exit 1 +fi + +kubectl get node "${node}" >/dev/null +if ! kubectl label node "${node}" "${ready_label}-" 2>/dev/null; then + labels="$(kubectl get node "${node}" --output=jsonpath='{.metadata.labels}')" + if [[ "${labels}" == *"${ready_label}"* ]]; then + echo "Could not clear a stale Immich readiness label on ${node}." >&2 + exit 1 + fi +fi + +# Force a fresh hostPath/local-PV mount after a mount cycle or reboot. +mapfile -t pods < <( + kubectl --namespace immich get pods \ + --selector="${selector}" \ + --field-selector="spec.nodeName=${node}" \ + --output=name 2>/dev/null || true +) +if ((${#pods[@]})); then + kubectl --namespace immich delete "${pods[@]}" --wait=false + for pod in "${pods[@]}"; do + kubectl --namespace immich wait --for=delete "${pod}" --timeout=5m + done +fi + +kubectl label node "${node}" "${ready_label}=true" --overwrite +echo "${node} is Immich-library-ready; its Immich workloads may now start."