From 49342eaef6957726c608f9ff6ed7eeae3292f1a7 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Tue, 4 Aug 2026 20:59:49 +0200 Subject: [PATCH] bug fixes and depliyment --- .gitea/workflows/container.yml | 40 +++++++++++++++++ README.md | 24 ++++++++-- ci-deployer.yaml | 47 +++++++++++++++++++ create-ci-kubeconfig.sh | 25 +++++++++++ immich-sync.yaml | 10 ++--- src/SyncSession.cpp | 82 +++++++++++++++++++++++++++++++--- src/SyncSession.h | 4 ++ web/styles.css | 2 +- 8 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 ci-deployer.yaml create mode 100755 create-ci-kubeconfig.sh diff --git a/.gitea/workflows/container.yml b/.gitea/workflows/container.yml index f4b2e36..bf2adba 100644 --- a/.gitea/workflows/container.yml +++ b/.gitea/workflows/container.yml @@ -45,3 +45,43 @@ jobs: run: | docker tag "${IMAGE}:${COMMIT_SHA}" "${IMAGE}:latest" docker push "${IMAGE}:latest" + + deploy: + name: Deploy to Kubernetes + if: gitea.ref == 'refs/heads/main' + needs: publish + runs-on: ubuntu-latest + + steps: + - name: Install kubectl + env: + KUBECTL_VERSION: v1.36.2 + run: | + curl --fail --silent --show-error --location \ + --output "${RUNNER_TEMP}/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + curl --fail --silent --show-error --location \ + --output "${RUNNER_TEMP}/kubectl.sha256" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" + printf '%s %s\n' "$(cat "${RUNNER_TEMP}/kubectl.sha256")" "${RUNNER_TEMP}/kubectl" | sha256sum --check + chmod 0700 "${RUNNER_TEMP}/kubectl" + + - name: Configure Kubernetes access + env: + KUBE_CONFIG_BASE64: ${{ secrets.KUBE_CONFIG_BASE64 }} + run: | + test -n "${KUBE_CONFIG_BASE64}" + printf '%s' "${KUBE_CONFIG_BASE64}" | base64 --decode > "${RUNNER_TEMP}/kubeconfig" + chmod 0600 "${RUNNER_TEMP}/kubeconfig" + + - name: Deploy commit image + env: + COMMIT_SHA: ${{ gitea.sha }} + run: | + export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" + "${RUNNER_TEMP}/kubectl" --namespace default set image \ + deployment/immich-sync \ + "immich-sync=${IMAGE}:${COMMIT_SHA}" + "${RUNNER_TEMP}/kubectl" --namespace default rollout status \ + deployment/immich-sync \ + --timeout=5m diff --git a/README.md b/README.md index 88fbdb8..2ba7077 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Immich Share Sync -An ephemeral Qt 6 web service that compares two Immich public shares by SHA-1 checksum and streams missing originals in either direction. It never deletes assets and never writes transfer data to disk. +An ephemeral Qt 6 web service that compares two Immich public shares by SHA-1 checksum and copies missing originals in either direction. It never deletes assets or keeps recovery state. ## How it works @@ -9,7 +9,7 @@ An ephemeral Qt 6 web service that compares two Immich public shares by SHA-1 ch - Each short-lived WebSocket owns exactly one job, one `SyncSession`, two `ImmichClient` instances, and their `QNetworkAccessManager` objects. - Link inspection tries Immich's share key and custom-slug authentication forms. Current password-protected shares use `POST /api/shared-links/login`; Qt's cookie jar retains the short-lived share cookie for that socket session. - The service reads `QJsonValue`/`QJsonObject` responses and compares the base64-encoded SHA-1 checksums in Immich's shared-link response. -- Each missing original is a sequential `QNetworkReply` used as the `QIODevice` body of the destination multipart request. A 1 MiB source read buffer provides backpressure; there is no temporary file or whole-asset buffer. +- Each missing original is downloaded into a `QTemporaryFile`, rewound, and used as the seekable `QIODevice` body of the destination multipart request. Only one asset per job is staged, and the file is removed after success, failure, or WebSocket abort. - The browser sends one complete job request immediately after connecting. The service closes the WebSocket when that inspection or synchronization finishes. - Closing the WebSocket early destroys the session and aborts every active network reply. No inspection ID, plan, credentials, or recovery state survives the connection. @@ -60,7 +60,7 @@ To synchronize, open a new WebSocket and send both shares again along with one o } ``` -The synchronization job repeats inspection so its checksum plan and permissions reflect current Immich state. Status messages include the inspection events followed by `sync-status`, `asset-status`, and throttled `asset-progress` events. The service closes the socket after completion; close it from the client to abort and discard the whole session immediately. +The synchronization job repeats inspection so its checksum plan and permissions reflect current Immich state. Status messages include the inspection events followed by `sync-status`, `asset-status`, and throttled `asset-progress` events. After upload reaches 100%, a `processing` asset stage makes it clear that the destination Immich is still handling the request. The service closes the socket after completion; close it from the client to abort, delete the temporary asset, and discard the whole session immediately. ## Container and Kubernetes @@ -70,7 +70,11 @@ docker run --rm -p 8090:8090 registry.brunner.ninja/feedc0de/immich-sync:latest ./install.sh ``` -The Kubernetes manifest follows the neighboring `brunner-ninja` and `visual-studio-code` layout. It assumes the image name and `immich-sync.brunner.ninja` hostname shown in the manifest. It enables the existing Authentik Traefik middleware because accepting arbitrary server URLs creates an SSRF/bandwidth-abuse surface; remove that annotation only if intentionally exposing the service publicly. +The Kubernetes manifest follows the neighboring `brunner-ninja` and `visual-studio-code` layout. It pulls the public +`registry.brunner.ninja/feedc0de/immich-sync:latest` image on every pod start, so no Kubernetes image-pull secret is +required. It assumes the `immich-sync.brunner.ninja` hostname shown in the manifest. The Ingress is intentionally +public and has no authentication middleware. Because the +service can make outbound requests to user-supplied URLs, operators should monitor it for SSRF and bandwidth abuse. ## Continuous delivery @@ -86,10 +90,22 @@ The workflow expects these Gitea Actions repository secrets: - `QUAY_USERNAME`: the complete Quay robot account name, including the `feedc0de+` prefix. - `QUAY_TOKEN`: the robot account token. +- `KUBE_CONFIG_BASE64`: a kubeconfig for the restricted `immich-sync-deployer` service account, base64-encoded on one line. Give the robot account `Write` permission only on the `feedc0de/immich-sync` repository. Public image pulls do not use these credentials and do not require a Gitea secret. +On successful `main` builds, the deploy job patches only the `default/immich-sync` Deployment to the immutable commit +image and waits up to five minutes for rollout. Apply `ci-deployer.yaml` once, generate a kubeconfig for its token, and +store it as `KUBE_CONFIG_BASE64`. The role cannot access secrets, pods, or any other Deployment. + +```sh +kubectl apply -f ci-deployer.yaml +./create-ci-kubeconfig.sh +``` + +Copy the single output line from `create-ci-kubeconfig.sh` into the Gitea repository secret. Treat it as a password. + ## Current scope - Assets without a checksum are ignored by planning. diff --git a/ci-deployer.yaml b/ci-deployer.yaml new file mode 100644 index 0000000..726c209 --- /dev/null +++ b/ci-deployer.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: immich-sync-deployer + namespace: default +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: immich-sync-deployer + namespace: default +rules: +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - immich-sync + verbs: + - get + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: immich-sync-deployer + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: immich-sync-deployer +subjects: +- kind: ServiceAccount + name: immich-sync-deployer + namespace: default +--- +apiVersion: v1 +kind: Secret +metadata: + name: immich-sync-deployer-token + namespace: default + annotations: + kubernetes.io/service-account.name: immich-sync-deployer +type: kubernetes.io/service-account-token diff --git a/create-ci-kubeconfig.sh b/create-ci-kubeconfig.sh new file mode 100755 index 0000000..09c3593 --- /dev/null +++ b/create-ci-kubeconfig.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +namespace=default +secret=immich-sync-deployer-token +workdir=$(mktemp --directory) +trap 'rm -rf "${workdir}"' EXIT + +server=$(kubectl config view --minify --output 'jsonpath={.clusters[0].cluster.server}') +kubectl --namespace "${namespace}" get secret "${secret}" --output 'jsonpath={.data.ca\.crt}' \ + | base64 --decode > "${workdir}/ca.crt" +token=$(kubectl --namespace "${namespace}" get secret "${secret}" --output 'jsonpath={.data.token}' \ + | base64 --decode) + +export KUBECONFIG="${workdir}/config" +kubectl config set-cluster cluster --server "${server}" --certificate-authority "${workdir}/ca.crt" --embed-certs=true \ + > /dev/null +kubectl config set-credentials immich-sync-deployer --token "${token}" > /dev/null +kubectl config set-context immich-sync --cluster cluster --user immich-sync-deployer --namespace "${namespace}" \ + > /dev/null +kubectl config use-context immich-sync > /dev/null + +base64 --wrap=0 "${KUBECONFIG}" +printf '\n' diff --git a/immich-sync.yaml b/immich-sync.yaml index 40614d8..6a2014d 100644 --- a/immich-sync.yaml +++ b/immich-sync.yaml @@ -45,9 +45,11 @@ spec: requests: cpu: 10m memory: 32Mi + ephemeral-storage: 64Mi limits: cpu: "1" memory: 128Mi + ephemeral-storage: 20Gi securityContext: allowPrivilegeEscalation: false capabilities: @@ -63,10 +65,7 @@ spec: volumes: - name: tmp emptyDir: - medium: Memory - sizeLimit: 16Mi - imagePullSecrets: - - name: quay-pull-secret + sizeLimit: 20Gi --- apiVersion: v1 kind: Service @@ -90,8 +89,6 @@ metadata: annotations: traefik.ingress.kubernetes.io/router.entrypoints: websecure traefik.ingress.kubernetes.io/router.tls.certresolver: letsencrypt - # This service can make outbound requests to user-supplied URLs. Keep authentication enabled. - traefik.ingress.kubernetes.io/router.middlewares: "default-authentik@kubernetescrd" spec: ingressClassName: traefik rules: @@ -105,4 +102,3 @@ spec: name: immich-sync port: name: http - diff --git a/src/SyncSession.cpp b/src/SyncSession.cpp index a2a8710..976ee18 100644 --- a/src/SyncSession.cpp +++ b/src/SyncSession.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -314,6 +315,7 @@ void SyncSession::startSync(const QString &direction) m_completed = 0; m_failed = 0; m_active = true; + qInfo().noquote() << u"Synchronization started: %1, %2 asset(s)"_s.arg(direction).arg(m_total); send({{u"type"_s, u"sync-status"_s}, {u"stage"_s, u"started"_s}, {u"direction"_s, direction}, @@ -344,6 +346,10 @@ void SyncSession::startNextTransfer() const TransferSpec &transfer = m_queue.front(); const quint64 serial = ++m_transferSerial; + qInfo().noquote() << u"Transfer %1/%2 started: %3 (%4)"_s + .arg(m_completed + m_failed + 1) + .arg(m_total) + .arg(transfer.asset.fileName, transfer.direction); send({{u"type"_s, u"asset-status"_s}, {u"stage"_s, u"downloading"_s}, {u"direction"_s, transfer.direction}, @@ -353,8 +359,14 @@ void SyncSession::startNextTransfer() {u"message"_s, u"Downloading original"_s}}); m_progressTimer.restart(); + m_temporaryFile = new QTemporaryFile{u"/tmp/immich-sync-XXXXXX"_s, this}; + if (!m_temporaryFile->open()) + { + finishTransfer(serial, false, u"Unable to create a temporary transfer file"_s); + return; + } m_download = transfer.source->download(transfer.asset); - connect(m_download, &QNetworkReply::metaDataChanged, this, [this, serial] { beginUpload(serial); }); + connect(m_download, &QNetworkReply::readyRead, this, [this, serial] { storeDownloadedData(serial); }); connect(m_download, &QNetworkReply::downloadProgress, this, [this, serial](qint64 received, qint64 total) { if (serial == m_transferSerial) { @@ -366,19 +378,50 @@ void SyncSession::startNextTransfer() { return; } + if (!storeDownloadedData(serial)) + { + return; + } const int status = m_download->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (m_download->error() != QNetworkReply::NoError || status < 200 || status >= 300) { finishTransfer(serial, false, u"Download failed: %1"_s.arg(m_download->errorString())); return; } + if (!m_temporaryFile->flush() || !m_temporaryFile->seek(0)) + { + finishTransfer(serial, false, u"Unable to rewind the temporary transfer file"_s); + return; + } beginUpload(serial); }); } +bool SyncSession::storeDownloadedData(quint64 serial) +{ + if (serial != m_transferSerial || !m_download || !m_temporaryFile) + { + return false; + } + while (m_download->bytesAvailable() > 0) + { + const QByteArray chunk = m_download->read(1024 * 1024); + if (chunk.isEmpty()) + { + break; + } + if (m_temporaryFile->write(chunk) != chunk.size()) + { + finishTransfer(serial, false, u"Unable to write the temporary transfer file"_s); + return false; + } + } + return true; +} + void SyncSession::beginUpload(quint64 serial) { - if (serial != m_transferSerial || !m_download || m_upload) + if (serial != m_transferSerial || !m_download || !m_temporaryFile || m_upload) { return; } @@ -387,22 +430,36 @@ void SyncSession::beginUpload(quint64 serial) { return; } - const TransferSpec &transfer = m_queue.front(); + m_download->deleteLater(); + m_download = nullptr; m_multipart = new QHttpMultiPart{QHttpMultiPart::FormDataType}; - m_upload = transfer.destination->upload(transfer.asset, m_download, m_multipart); + m_upload = transfer.destination->upload(transfer.asset, m_temporaryFile, m_multipart); m_multipart->setParent(m_upload); + m_uploadCompleteNotified = false; send({{u"type"_s, u"asset-status"_s}, {u"stage"_s, u"uploading"_s}, {u"direction"_s, transfer.direction}, {u"fileName"_s, transfer.asset.fileName}, {u"current"_s, static_cast(m_completed + m_failed + 1)}, {u"total"_s, static_cast(m_total)}, - {u"message"_s, u"Streaming into destination"_s}}); + {u"message"_s, u"Uploading original to destination"_s}}); connect(m_upload, &QNetworkReply::uploadProgress, this, [this, serial](qint64 sent, qint64 total) { if (serial == m_transferSerial) { sendProgress(u"uploading"_s, sent, total); + if (total > 0 && sent >= total && !m_uploadCompleteNotified && !m_queue.isEmpty()) + { + m_uploadCompleteNotified = true; + const TransferSpec ¤tTransfer = m_queue.front(); + send({{u"type"_s, u"asset-status"_s}, + {u"stage"_s, u"processing"_s}, + {u"direction"_s, currentTransfer.direction}, + {u"fileName"_s, currentTransfer.asset.fileName}, + {u"current"_s, static_cast(m_completed + m_failed + 1)}, + {u"total"_s, static_cast(m_total)}, + {u"message"_s, u"Destination Immich is processing the asset"_s}}); + } } }); connect(m_upload, &QNetworkReply::finished, this, [this, serial] { @@ -433,6 +490,8 @@ void SyncSession::finishTransfer(quint64 serial, bool success, const QString &me } ++m_transferSerial; const TransferSpec transfer = m_queue.takeFirst(); + qInfo().noquote() << u"Transfer finished: %1 (%2): %3"_s.arg(transfer.asset.fileName, + success ? u"success"_s : u"failed"_s, message); if (success) { ++m_completed; @@ -470,6 +529,12 @@ void SyncSession::finishTransfer(quint64 serial, bool success, const QString &me m_upload = nullptr; m_download = nullptr; m_multipart = nullptr; + m_uploadCompleteNotified = false; + if (m_temporaryFile) + { + m_temporaryFile->deleteLater(); + m_temporaryFile = nullptr; + } startNextTransfer(); } @@ -514,8 +579,15 @@ void SyncSession::abort() m_upload = nullptr; m_download = nullptr; m_multipart = nullptr; + m_uploadCompleteNotified = false; + if (m_temporaryFile) + { + m_temporaryFile->deleteLater(); + m_temporaryFile = nullptr; + } if (wasActive) { + qInfo() << "Synchronization aborted because its WebSocket disconnected"; send({{u"type"_s, u"sync-status"_s}, {u"stage"_s, u"aborted"_s}, {u"message"_s, u"Synchronization and network transfers were aborted"_s}}); diff --git a/src/SyncSession.h b/src/SyncSession.h index 30c901f..f7e48cf 100644 --- a/src/SyncSession.h +++ b/src/SyncSession.h @@ -10,6 +10,7 @@ class QHttpMultiPart; class QNetworkReply; +class QTemporaryFile; class QWebSocket; class SyncSession final : public QObject { @@ -44,6 +45,7 @@ class SyncSession final : public QObject { void sendInspection(); void startSync(const QString &direction); void startNextTransfer(); + bool storeDownloadedData(quint64 serial); void beginUpload(quint64 serial); void finishTransfer(quint64 serial, bool success, const QString &message, const QString &result = {}); void finishJob(const QString &reason); @@ -77,5 +79,7 @@ class SyncSession final : public QObject { QNetworkReply *m_download = nullptr; QNetworkReply *m_upload = nullptr; QHttpMultiPart *m_multipart = nullptr; + QTemporaryFile *m_temporaryFile = nullptr; + bool m_uploadCompleteNotified = false; QElapsedTimer m_progressTimer; }; diff --git a/web/styles.css b/web/styles.css index fbae6f2..4ae870e 100644 --- a/web/styles.css +++ b/web/styles.css @@ -96,7 +96,7 @@ button:disabled { opacity: .35; cursor: not-allowed; } .events li::before { content: ''; grid-column: 2; width: 7px; height: 7px; margin-top: 5px; border-radius: 50%; background: var(--blue); } .events time { grid-column: 1; grid-row: 1; color: #64748b; font-variant-numeric: tabular-nums; } .events div { grid-column: 3; display: flex; flex-direction: column; gap: 3px; }.events span { color: var(--muted); } -.events .success::before, .events .complete::before { background: var(--green); }.events .error::before, .events .failed::before, .events .aborted::before { background: var(--red); }.events .uploading::before { background: var(--violet); } +.events .success::before, .events .complete::before { background: var(--green); }.events .error::before, .events .failed::before, .events .aborted::before { background: var(--red); }.events .uploading::before { background: var(--violet); }.events .processing::before { background: var(--amber); } footer { display: flex; justify-content: center; flex-wrap: wrap; gap: 10px 22px; padding: 25px 0 0; color: #64748b; text-transform: uppercase; letter-spacing: .08em; font-size: .62rem; font-weight: 800; } footer span::before { content: '✓'; color: var(--green); margin-right: 7px; }