From 3db0328124290ffbf0ec9bac6aec7b08e99fc00e Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Mon, 5 Oct 2026 09:24:50 +0200 Subject: [PATCH] Move Paperless to PostgreSQL and block-backed runtime storage --- README.md | 14 +++++++------- app.yaml | 17 ++++++++++++----- install.sh | 12 ++++++++++++ postgresql-values.yaml | 34 ++++++++++++++++++++++++++++++++++ storage.yaml | 12 ++++++++++++ 5 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 postgresql-values.yaml diff --git a/README.md b/README.md index 05bdc89..c9420c4 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ # Paperless-ngx on the homelab -One Paperless-ngx instance, SQLite, and Valkey in the `paperless-ngx` namespace. Documents, database, index, and inbox are on one CephFS PVC. The app is served at through Traefik. Back up the PVC and the `paperless-secret-key` Secret together. +One Paperless-ngx instance, PostgreSQL, and Valkey in the `paperless-ngx` namespace. Documents, scanner staging, and the incoming folder use the retained CephFS PVC. PostgreSQL and Paperless's search index use separate Ceph block PVCs. The app is served at through Traefik. Back up the three PVCs and the `paperless-secret-key` and `paperless-postgres` Secrets together. ## Install -Run `./install.sh` from any directory. It creates the namespace and random, persistent application and OIDC secrets on first install, creates the Authentik applications, checks the manifests with a server dry run, applies them, then waits for the deployments. Open the URL and sign in with the **brunner.ninja Auth** button. Drop scanned files in the web UI or into `/usr/src/paperless/consume` on the PVC. +Run `./install.sh` from any directory. It creates the namespace and random, persistent application, PostgreSQL, and OIDC secrets on first install, creates the Authentik applications, installs the pinned PostgreSQL Helm chart with the official PostgreSQL image, checks the manifests with a server dry run, applies them, then waits for the deployments. Open the URL and sign in with the **brunner.ninja Auth** button. Drop scanned files in the web UI or into `/usr/src/paperless/consume` on the PVC. An existing SQLite installation needs a database migration before switching `app.yaml` to PostgreSQL; the homelab instance was migrated on 2026-10-05. -Change settings in `app.yaml`, then run `./test.sh` and `./install.sh`. The Paperless image is pinned there. The CephFS inbox uses polling because network filesystems may not deliver file notifications. +Change Paperless settings in `app.yaml` and PostgreSQL settings in `postgresql-values.yaml`, then run `./test.sh` and `./install.sh`. Both images and the chart version are pinned. The CephFS inbox uses polling because network filesystems may not deliver file notifications. -Paperless prefers the 24-core `arschrock` node for OCR. This is a scheduler preference, so it can run on an Odroid when `arschrock` is unavailable. OCR is capped at four concurrent pages and the pod has a 4 GiB memory limit; using the host's default 24 OCR workers caused an out-of-memory restart while editing a large PDF. The control-plane taint is tolerated only by Paperless, not Valkey. A node failure may briefly interrupt OCR while Kubernetes reschedules the single Paperless pod. +Paperless prefers the 24-core `arschrock` node for OCR. This is a scheduler preference, so it can run on an Odroid when `arschrock` is unavailable. It has 24 task workers with one OCR thread each and no Kubernetes memory limit. This can be heavy on an Odroid during failover; reduce `PAPERLESS_TASK_WORKERS` in `app.yaml` if that happens. The control-plane taint is tolerated only by Paperless, not Valkey. A node failure may briefly interrupt OCR while Kubernetes reschedules the single Paperless pod. -The 20 Gi PVC uses `rook-cephfs-ec`, whose CephFS data pool is erasure coded with three data and two coding chunks across hosts. CephFS metadata uses three replicas. The claim can be expanded if high-resolution scans outgrow it; erasure coding does not replace backups. +The 20 Gi CephFS PVC uses `rook-cephfs-ec`, whose data pool is erasure coded with three data and two coding chunks across hosts. CephFS metadata uses three replicas. PostgreSQL and Paperless's runtime data use `rook-ceph-block-ec`. The claims can be expanded if scans or the index outgrow them; erasure coding does not replace backups. ## Scanner @@ -24,7 +24,7 @@ To merge separate Paperless documents imported from JPG scans, enable **Try to i ## CI/CD -`./test.sh` checks syntax, the deployment manifest, and the scanner PDF handoff. `./deploy.sh` updates the existing workloads, services, ConfigMap, and ingress using `app.yaml`. The GitLab and Gitea workflows call these same scripts. Bootstrap once with `./create-ci-kubeconfig.sh`, then save its single output line as the protected CI variable / Gitea Actions secret `KUBE_CONFIG_BASE64`. Do not commit it. The deployer has named-object read and patch permissions and cannot read Secrets. Run `./install.sh` manually for first installation or changes to `storage.yaml` or Authentik setup. +`./test.sh` checks syntax, the deployment manifest, and the scanner PDF handoff. `./deploy.sh` updates the existing workloads, services, ConfigMap, and ingress using `app.yaml`. The GitLab and Gitea workflows call these same scripts. Bootstrap once with `./create-ci-kubeconfig.sh`, then save its single output line as the protected CI variable / Gitea Actions secret `KUBE_CONFIG_BASE64`. Do not commit it. The deployer has named-object read and patch permissions and cannot read Secrets. Run `./install.sh` manually for first installation or changes to `storage.yaml`, `postgresql-values.yaml`, or Authentik setup. The Git remote is `gitea@brunner.ninja:feedc0de/paperless-ngx-deployment.git`. No credentials or generated kubeconfig are committed. To publish changes: `git add . && git commit -m 'Deploy Paperless-ngx' && git push origin main`. @@ -38,4 +38,4 @@ The first-visit create-user form is Paperless's bootstrap screen. Since the init ## Recovery -The PVC has a `Retain` storage class. Do not delete it on uninstall. Restore the PVC and the application Secret together. SQLite requires a consistent backup; stop the Paperless deployment or use its document exporter before copying the database. +The PVCs have retained storage. Do not delete them on uninstall. Restore the PostgreSQL PVC, document PVC, runtime PVC, and both application and database Secrets together. The former SQLite database remains on the CephFS `data` directory as a migration fallback; it is no longer used by Paperless. For a consistent backup, use PostgreSQL's backup tools and copy the document storage at a coordinated point in time, or use Paperless's document exporter. diff --git a/app.yaml b/app.yaml index f20f240..8b08f49 100644 --- a/app.yaml +++ b/app.yaml @@ -27,7 +27,6 @@ spec: periodSeconds: 20 resources: requests: {cpu: 25m, memory: 64Mi} - limits: {memory: 256Mi} --- apiVersion: v1 kind: Service @@ -75,9 +74,17 @@ spec: env: - {name: PAPERLESS_URL, value: "https://paper.brunner.ninja"} - {name: PAPERLESS_REDIS, value: "redis://valkey:6379"} + - {name: PAPERLESS_DBENGINE, value: postgresql} + - {name: PAPERLESS_DBHOST, value: postgresql} + - {name: PAPERLESS_DBNAME, value: paperless} + - {name: PAPERLESS_DBUSER, value: paperless} + - name: PAPERLESS_DBPASS + valueFrom: + secretKeyRef: {name: paperless-postgres, key: USERDB_PASSWORD} - {name: PAPERLESS_TIME_ZONE, value: "Europe/Vienna"} - {name: PAPERLESS_OCR_LANGUAGE, value: "deu+eng"} - - {name: PAPERLESS_THREADS_PER_WORKER, value: "4"} + - {name: PAPERLESS_TASK_WORKERS, value: "24"} + - {name: PAPERLESS_THREADS_PER_WORKER, value: "1"} - {name: PAPERLESS_CONSUMER_POLLING_INTERVAL, value: "30"} - {name: PAPERLESS_APPS, value: "allauth.socialaccount.providers.openid_connect"} - {name: PAPERLESS_SOCIAL_AUTO_SIGNUP, value: "true"} @@ -90,7 +97,7 @@ spec: valueFrom: secretKeyRef: {name: paperless-secret-key, key: secret-key} volumeMounts: - - {name: data, mountPath: /usr/src/paperless/data, subPath: data} + - {name: runtime, mountPath: /usr/src/paperless/data} - {name: data, mountPath: /usr/src/paperless/media, subPath: media} - {name: data, mountPath: /usr/src/paperless/consume, subPath: consume} - {name: data, mountPath: /usr/src/paperless/export, subPath: export} @@ -106,8 +113,9 @@ spec: periodSeconds: 30 resources: requests: {cpu: 100m, memory: 1Gi} - limits: {memory: 4Gi} volumes: + - name: runtime + persistentVolumeClaim: {claimName: paperless-runtime} - name: data persistentVolumeClaim: {claimName: paperless-data} --- @@ -252,7 +260,6 @@ spec: periodSeconds: 30 resources: requests: {cpu: 100m, memory: 256Mi} - limits: {memory: 2Gi} volumes: - name: data persistentVolumeClaim: {claimName: paperless-data} diff --git a/install.sh b/install.sh index 4c5c0f8..c70fbff 100755 --- a/install.sh +++ b/install.sh @@ -7,8 +7,20 @@ if ! kubectl -n paperless-ngx get secret paperless-secret-key >/dev/null 2>&1; t key=$(openssl rand -base64 64) kubectl -n paperless-ngx create secret generic paperless-secret-key --from-literal="secret-key=${key}" fi +if ! kubectl -n paperless-ngx get secret paperless-postgres >/dev/null 2>&1; then + postgres_admin_password=$(openssl rand -hex 32) + postgres_app_password=$(openssl rand -hex 32) + kubectl -n paperless-ngx create secret generic paperless-postgres \ + --from-literal=POSTGRES_USER=postgres \ + --from-literal="POSTGRES_PASSWORD=${postgres_admin_password}" \ + --from-literal=USERDB_NAME=paperless \ + --from-literal=USERDB_USER=paperless \ + --from-literal="USERDB_PASSWORD=${postgres_app_password}" + unset postgres_admin_password postgres_app_password +fi ./setup-authentik.sh ./test.sh +helm upgrade --install postgresql postgres --repo https://groundhog2k.github.io/helm-charts/ --version 1.6.7 -n paperless-ngx -f postgresql-values.yaml --wait --timeout 10m kubectl apply --dry-run=server -f storage.yaml -f app.yaml apply_output=$(kubectl apply -f storage.yaml -f app.yaml) printf '%s\n' "$apply_output" diff --git a/postgresql-values.yaml b/postgresql-values.yaml new file mode 100644 index 0000000..6370540 --- /dev/null +++ b/postgresql-values.yaml @@ -0,0 +1,34 @@ +image: + registry: docker.io + repository: postgres + tag: 17.10-bookworm +settings: + existingSecret: paperless-postgres + superuser: + secretKey: POSTGRES_USER + superuserPassword: + secretKey: POSTGRES_PASSWORD +userDatabase: + existingSecret: paperless-postgres + name: + secretKey: USERDB_NAME + user: + secretKey: USERDB_USER + password: + secretKey: USERDB_PASSWORD +storage: + requestedSize: 8Gi + className: rook-ceph-block-ec + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain +resources: + requests: + cpu: 100m + memory: 256Mi +securityContext: + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false diff --git a/storage.yaml b/storage.yaml index 401b6ed..51129fb 100644 --- a/storage.yaml +++ b/storage.yaml @@ -9,3 +9,15 @@ spec: resources: requests: storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: paperless-runtime + namespace: paperless-ngx +spec: + accessModes: [ReadWriteOnce] + storageClassName: rook-ceph-block-ec + resources: + requests: + storage: 4Gi