From fbafa72cba3c53c76718fc46aeb9300170df54e6 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sun, 4 Oct 2026 19:17:01 +0200 Subject: [PATCH] Add authenticated scanner UI and Paperless handoff --- .gitignore | 1 + .gitlab-ci.yml | 4 +- README.md | 12 ++++- app.yaml | 121 +++++++++++++++++++++++++++++++++++++++++++++ authentik-setup.py | 35 ++++++++++++- ci-deployer.yaml | 10 ++-- deploy.sh | 8 ++- install.sh | 7 ++- test.sh | 24 ++++++++- 9 files changed, 211 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 2eed864..922315b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ kubeconfig *.secret +__pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a976d78..794a3f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ test: stage: test image: alpine/k8s:1.33.0 script: - - apk add --no-cache bash python3 py3-yaml + - apk add --no-cache bash python3 py3-yaml nodejs - ./test.sh deploy: @@ -13,7 +13,7 @@ deploy: rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' script: - - apk add --no-cache bash python3 py3-yaml + - apk add --no-cache bash python3 py3-yaml nodejs - umask 077 - echo "$KUBE_CONFIG_BASE64" | base64 -d > kubeconfig - export KUBECONFIG="$CI_PROJECT_DIR/kubeconfig" diff --git a/README.md b/README.md index 4ddd789..4d65efb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ One Paperless-ngx instance, SQLite, and Valkey in the `paperless-ngx` namespace. ## Install -Run `./install.sh` from any directory. It creates the namespace and random, persistent application and OIDC secrets on first install, creates the Authentik OIDC application, checks the manifests with a server dry run, applies them, then waits for both deployments. Open the URL and sign in with the **authentik** button. Drop scanned files in the web UI or into `/usr/src/paperless/consume` on the PVC. +Run `./install.sh` from any directory. It creates the namespace and random, persistent application and OIDC secrets on first install, creates the Authentik applications, checks the manifests with a server dry run, applies them, then waits for the deployments. Open the URL and sign in with the **brunner.ninja Auth** button. Drop scanned files in the web UI or into `/usr/src/paperless/consume` on the PVC. Change settings in `app.yaml`, then run `./test.sh` and `./install.sh`. The Paperless image is pinned there. The CephFS inbox uses polling because network filesystems may not deliver file notifications. @@ -12,9 +12,17 @@ Paperless prefers the 24-core `arschrock` node for OCR. This is a scheduler pref The 20 Gi PVC uses `rook-cephfs-ec`, whose CephFS data pool is erasure coded with three data and two coding chunks across hosts. CephFS metadata uses three replicas. The claim can be expanded if high-resolution scans outgrow it; erasure coding does not replace backups. +## Scanner + +The [ScanservJS](https://sbs20.github.io/scanservjs/) web UI is at . Point its DNS CNAME at the same target as `paper.brunner.ninja`. Traefik requires Authentik login and access to the **Paperless Users** group. The scanner's eSCL address is set once as `AIRSCAN_DEVICES` in `app.yaml`; update it there if the printer gets a new IP. The UI pulls scans directly from the HP OfficeJet Pro 8020 at `192.168.4.189:8080`, without HP cloud services or a running laptop. + +Load the automatic document feeder, choose the HP scanner, its feeder source, **Auto** batch mode, and a **PDF** output format, then press **Scan**. The feeder batch becomes one multipage PDF. Review the staged result in the Files view, then use **Send to Paperless**. This moves the PDF into the existing Paperless consume directory; Paperless polls it every 30 seconds and handles OCR. The scan remains in staging until you send or delete it. ScanservJS does not offer reliable individual-page deletion or retry before sending: rescan the batch, or edit/split the PDF after import in Paperless. The OfficeJet Pro 8020's feeder is single-sided. This workflow starts scans from the web UI rather than a button on the printer panel. + +ScanservJS stores staged scans in the `scanner-staging` directory on the same retained CephFS claim. The deployment uses the pinned upstream image; its `config.local.js` action is in `app.yaml` and has a local handoff test in `test.sh`. The Authentik proxy application is created by `setup-authentik.sh` along with Paperless's OIDC application. + ## CI/CD -`./test.sh` checks syntax and the deployment manifest. `./deploy.sh` updates the existing workloads, services, and ingress using `app.yaml`. The GitLab and Gitea workflows call these same scripts. Bootstrap once with `./create-ci-kubeconfig.sh`, then save its single output line as the protected CI variable / Gitea Actions secret `KUBE_CONFIG_BASE64`. Do not commit it. The deployer has named-object read and patch permissions and cannot read Secrets. Run `./install.sh` manually for first installation or changes to `storage.yaml`. +`./test.sh` checks syntax, the deployment manifest, and the scanner PDF handoff. `./deploy.sh` updates the existing workloads, services, ConfigMap, and ingress using `app.yaml`. The GitLab and Gitea workflows call these same scripts. Bootstrap once with `./create-ci-kubeconfig.sh`, then save its single output line as the protected CI variable / Gitea Actions secret `KUBE_CONFIG_BASE64`. Do not commit it. The deployer has named-object read and patch permissions and cannot read Secrets. Run `./install.sh` manually for first installation or changes to `storage.yaml` or Authentik setup. The Git remote is `gitea@brunner.ninja:feedc0de/paperless-ngx-deployment.git`. No credentials or generated kubeconfig are committed. To publish changes: `git add . && git commit -m 'Deploy Paperless-ngx' && git push origin main`. diff --git a/app.yaml b/app.yaml index c200d7d..64ed22d 100644 --- a/app.yaml +++ b/app.yaml @@ -139,3 +139,124 @@ spec: service: name: paperless port: {name: http} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: scanner-config + namespace: paperless-ngx +data: + config.local.js: | + const fs = require('node:fs/promises'); + const path = require('node:path'); + const crypto = require('node:crypto'); + + const output = process.env.SCAN_OUTPUT_DIR || '/var/lib/scanservjs/output'; + const consume = process.env.PAPERLESS_CONSUME_DIR || '/paperless-consume'; + + module.exports = { + actions: [{ + name: 'Send to Paperless', + async execute(fileInfo) { + const source = await fs.realpath(fileInfo.fullname); + if (!source.startsWith(output + '/') || path.extname(source).toLowerCase() !== '.pdf') { + throw new Error('Only staged PDF files can be sent to Paperless'); + } + const name = `${Date.now()}-${crypto.randomBytes(6).toString('hex')}-${path.basename(source)}`; + const destination = path.join(consume, name); + const pending = destination + '.part'; + try { + await fs.copyFile(source, pending); + await fs.rename(pending, destination); + await fs.unlink(source); + } catch (error) { + await fs.rm(pending, {force: true}); + throw error; + } + }, + }], + }; +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: scanner + namespace: paperless-ngx +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: {app: scanner} + template: + metadata: + labels: {app: scanner} + spec: + automountServiceAccountToken: false + securityContext: + supplementalGroups: [1000] + initContainers: + - name: prepare-output + image: busybox:1.37.0 + command: [sh, -c, 'mkdir -p /volume/scanner-staging && chown 102:100 /volume/scanner-staging && chmod 775 /volume/scanner-staging && chmod g+w /volume/consume'] + volumeMounts: + - {name: data, mountPath: /volume} + containers: + - name: scanner + image: sbs20/scanservjs:3.3.0 + command: [sh, -c, "sed -i '/^\\[options\\]/a discovery = disable' /etc/sane.d/airscan.conf && exec /entrypoint.sh"] + env: + - {name: AIRSCAN_DEVICES, value: '"HP OfficeJet Pro 8020" = http://192.168.4.189:8080/eSCL, eSCL'} + ports: [{name: http, containerPort: 8080}] + volumeMounts: + - {name: data, mountPath: /var/lib/scanservjs/output, subPath: scanner-staging} + - {name: data, mountPath: /paperless-consume, subPath: consume} + - {name: config, mountPath: /etc/scanservjs/config.local.js, subPath: config.local.js, readOnly: true} + startupProbe: + httpGet: {path: /, port: http} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /, port: http} + periodSeconds: 10 + livenessProbe: + httpGet: {path: /, port: http} + periodSeconds: 30 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 2Gi} + volumes: + - name: data + persistentVolumeClaim: {claimName: paperless-data} + - name: config + configMap: {name: scanner-config} +--- +apiVersion: v1 +kind: Service +metadata: + name: scanner + namespace: paperless-ngx +spec: + selector: {app: scanner} + ports: [{name: http, port: 8080, targetPort: http}] +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: scanner + namespace: paperless-ngx + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: default-authentik@kubernetescrd +spec: + ingressClassName: traefik + rules: + - host: scanner.brunner.ninja + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: scanner + port: {name: http} diff --git a/authentik-setup.py b/authentik-setup.py index a394269..a0c07ef 100644 --- a/authentik-setup.py +++ b/authentik-setup.py @@ -2,6 +2,8 @@ from django.db import transaction from authentik.core.models import Application, Group, User from authentik.policies.models import PolicyBinding from authentik.providers.oauth2.models import OAuth2Provider +from authentik.providers.proxy.models import ProxyProvider +from authentik.outposts.models import Outpost with transaction.atomic(): template = OAuth2Provider.objects.get(name="Argo CD") @@ -57,4 +59,35 @@ with transaction.atomic(): binding.enabled = True binding.save(update_fields=["enabled"]) -print("PAPERLESS_AUTHENTIK_READY") + proxy_template = ProxyProvider.objects.get(name="Provider for 30-birthday-admin") + scanner_provider, _ = ProxyProvider.objects.get_or_create( + name="Paperless scanner", + defaults={ + "authorization_flow": proxy_template.authorization_flow, + "invalidation_flow": proxy_template.invalidation_flow, + "mode": "forward_single", + "external_host": "https://scanner.brunner.ninja/", + }, + ) + scanner_provider.authorization_flow = proxy_template.authorization_flow + scanner_provider.invalidation_flow = proxy_template.invalidation_flow + scanner_provider.mode = "forward_single" + scanner_provider.external_host = "https://scanner.brunner.ninja/" + scanner_provider.save() + scanner_app, _ = Application.objects.get_or_create( + slug="paperless-scanner", + defaults={"name": "Paperless scanner", "provider": scanner_provider}, + ) + scanner_app.name = "Paperless scanner" + scanner_app.provider = scanner_provider + scanner_app.meta_launch_url = "https://scanner.brunner.ninja/" + scanner_app.save() + scanner_binding, _ = PolicyBinding.objects.get_or_create( + target=scanner_app, group=group, defaults={"order": 0, "enabled": True} + ) + if not scanner_binding.enabled: + scanner_binding.enabled = True + scanner_binding.save(update_fields=["enabled"]) + Outpost.objects.get(name="authentik Embedded Outpost").providers.add(scanner_provider) + +print("PAPERLESS_AUTHENTIK_AND_SCANNER_READY") diff --git a/ci-deployer.yaml b/ci-deployer.yaml index 7f01171..265f400 100644 --- a/ci-deployer.yaml +++ b/ci-deployer.yaml @@ -12,15 +12,19 @@ metadata: rules: - apiGroups: [apps] resources: [deployments] - resourceNames: [paperless, valkey] + resourceNames: [paperless, valkey, scanner] verbs: [get, patch, watch] - apiGroups: [""] resources: [services] - resourceNames: [paperless, valkey] + resourceNames: [paperless, valkey, scanner] verbs: [get, patch] - apiGroups: [networking.k8s.io] resources: [ingresses] - resourceNames: [paperless] + resourceNames: [paperless, scanner] + verbs: [get, patch] + - apiGroups: [""] + resources: [configmaps] + resourceNames: [scanner-config] verbs: [get, patch] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy.sh b/deploy.sh index efdee5d..cdce27f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" +[[ $(kubectl config current-context) == kubernetes-admin@kubernetes ]] || { echo 'Wrong Kubernetes context' >&2; exit 1; } ./test.sh -kubectl apply -f app.yaml +apply_output=$(kubectl apply -f app.yaml) +printf '%s\n' "$apply_output" +if [[ $apply_output == *'configmap/scanner-config configured'* ]]; then + kubectl -n paperless-ngx rollout restart deployment/scanner +fi kubectl -n paperless-ngx rollout status deployment/valkey --timeout=5m || { kubectl -n paperless-ngx describe deployment valkey; exit 1; } kubectl -n paperless-ngx rollout status deployment/paperless --timeout=15m || { kubectl -n paperless-ngx describe deployment paperless; exit 1; } +kubectl -n paperless-ngx rollout status deployment/scanner --timeout=10m || { kubectl -n paperless-ngx describe deployment scanner; kubectl -n paperless-ngx logs deployment/scanner --tail=100; exit 1; } diff --git a/install.sh b/install.sh index c093793..4c5c0f8 100755 --- a/install.sh +++ b/install.sh @@ -10,7 +10,12 @@ fi ./setup-authentik.sh ./test.sh kubectl apply --dry-run=server -f storage.yaml -f app.yaml -kubectl apply -f storage.yaml -f app.yaml +apply_output=$(kubectl apply -f storage.yaml -f app.yaml) +printf '%s\n' "$apply_output" +if [[ $apply_output == *'configmap/scanner-config configured'* ]]; then + kubectl -n paperless-ngx rollout restart deployment/scanner +fi kubectl -n paperless-ngx rollout status deployment/valkey --timeout=5m kubectl -n paperless-ngx rollout status deployment/paperless --timeout=15m +kubectl -n paperless-ngx rollout status deployment/scanner --timeout=10m kubectl -n paperless-ngx exec deployment/paperless -- python manage.py shell -c 'from django.contrib.auth.models import Group; [Group.objects.get_or_create(name=name) for name in ("Paperless Users", "Paperless Admins")]' diff --git a/test.sh b/test.sh index 6527ebe..42000bd 100755 --- a/test.sh +++ b/test.sh @@ -3,6 +3,10 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" bash -n install.sh deploy.sh create-ci-kubeconfig.sh setup-authentik.sh test.sh python3 - <<'PY' +import os +import subprocess +import tempfile +from pathlib import Path import yaml documents = [] @@ -12,5 +16,23 @@ for path in ("storage.yaml", "app.yaml", "ci-deployer.yaml"): images = [container["image"] for doc in documents if doc["kind"] == "Deployment" for container in doc["spec"]["template"]["spec"]["containers"]] assert images and all(":" in image and not image.endswith(":latest") for image in images) -print("Manifest valid; deployment images are pinned.") +config = next(doc["data"]["config.local.js"] for doc in documents + if doc["kind"] == "ConfigMap" and doc["metadata"]["name"] == "scanner-config") +with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + config_path = root / "config.local.js" + config_path.write_text(config, encoding="utf-8") + output = root / "staged" + consume = root / "consume" + output.mkdir() + consume.mkdir() + source = output / "test.pdf" + source.write_bytes(b"%PDF-1.4\nscanner handoff test\n") + env = {**os.environ, "SCAN_OUTPUT_DIR": str(output), "PAPERLESS_CONSUME_DIR": str(consume)} + subprocess.run(["node", "--check", str(config_path)], check=True) + subprocess.run(["node", "-e", "require(process.argv[1]).actions[0].execute({fullname: process.argv[2]}).catch(e => {console.error(e); process.exit(1)})", str(config_path), str(source)], check=True, env=env) + delivered = list(consume.glob("*.pdf")) + assert len(delivered) == 1 and delivered[0].read_bytes().startswith(b"%PDF-1.4") + assert not source.exists() and not list(consume.glob("*.part")) +print("Manifests valid; images pinned; scanner PDF handoff works.") PY