diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..b3a4147 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,143 @@ +name: Validate and deploy asciinema + +on: + push: + pull_request: + +env: + KUBECTL_VERSION: v1.36.3 + KUBERNETES_API: https://host.containers.internal:6443 + KUBERNETES_TLS_SERVER_NAME: 192.168.0.2 + NAMESPACE: asciinema + +jobs: + validate: + name: Validate manifests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Validate scripts and Kubernetes resources + run: bash test.sh + + deploy: + name: Deploy to Kubernetes + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Install kubectl + run: | + curl --fail --silent --show-error --location \ + --output "${RUNNER_TEMP}/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + curl --fail --silent --show-error --location \ + --output "${RUNNER_TEMP}/kubectl.sha256" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" + printf '%s %s\n' "$(cat "${RUNNER_TEMP}/kubectl.sha256")" "${RUNNER_TEMP}/kubectl" | sha256sum --check + chmod 0700 "${RUNNER_TEMP}/kubectl" + + - name: Configure Kubernetes access + env: + KUBE_CONFIG_BASE64: ${{ secrets.KUBE_CONFIG_BASE64 }} + run: | + test -n "${KUBE_CONFIG_BASE64}" + printf '%s' "${KUBE_CONFIG_BASE64}" | base64 --decode > "${RUNNER_TEMP}/kubeconfig" + chmod 0600 "${RUNNER_TEMP}/kubeconfig" + export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" + "${RUNNER_TEMP}/kubectl" config set-cluster cluster \ + --server="${KUBERNETES_API}" \ + --tls-server-name="${KUBERNETES_TLS_SERVER_NAME}" + + - name: Validate desired state on the server + run: | + export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" + "${RUNNER_TEMP}/kubectl" apply \ + --dry-run=server \ + --validate=false \ + --filename infrastructure.yaml + "${RUNNER_TEMP}/kubectl" apply \ + --dry-run=server \ + --validate=false \ + --filename asciinema.yaml + + - name: Apply and verify + run: | + set -euo pipefail + export KUBECONFIG="${RUNNER_TEMP}/kubeconfig" + + "${RUNNER_TEMP}/kubectl" apply \ + --validate=false \ + --filename infrastructure.yaml + + for attempt in {1..120}; do + IFS='|' read -r generation observed desired current ready updated <<< "$( + "${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" \ + get statefulset/postgresql \ + --output=jsonpath='{.metadata.generation}|{.status.observedGeneration}|{.spec.replicas}|{.status.currentReplicas}|{.status.readyReplicas}|{.status.updatedReplicas}' + )" + + echo "PostgreSQL ${attempt}/120: generation ${observed}/${generation}, replicas ${updated}/${desired} updated, ${current}/${desired} current, ${ready}/${desired} ready" + if [[ "${observed}" == "${generation}" \ + && "${updated}" == "${desired}" \ + && "${current}" == "${desired}" \ + && "${ready}" == "${desired}" ]]; then + break + fi + + if [[ "${attempt}" == 120 ]]; then + echo "PostgreSQL rollout did not complete within ten minutes" >&2 + exit 1 + fi + sleep 5 + done + + "${RUNNER_TEMP}/kubectl" apply \ + --validate=false \ + --filename asciinema.yaml + # ConfigMap and Secret-backed environment variables are read only at + # process start, so every successful deployment recreates the sole pod. + "${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" \ + rollout restart deployment/asciinema + + for attempt in {1..120}; do + IFS='|' read -r generation observed desired updated ready available unavailable image <<< "$( + "${RUNNER_TEMP}/kubectl" --namespace "${NAMESPACE}" \ + get deployment/asciinema \ + --output=jsonpath='{.metadata.generation}|{.status.observedGeneration}|{.spec.replicas}|{.status.updatedReplicas}|{.status.readyReplicas}|{.status.availableReplicas}|{.status.unavailableReplicas}|{.spec.template.spec.containers[0].image}' + )" + + echo "asciinema ${attempt}/120: generation ${observed}/${generation}, replicas ${updated}/${desired} updated, ${ready}/${desired} ready, ${available}/${desired} available, image ${image}" + if [[ "${observed}" == "${generation}" \ + && "${updated}" == "${desired}" \ + && "${ready}" == "${desired}" \ + && "${available}" == "${desired}" \ + && ( -z "${unavailable}" || "${unavailable}" == "0" ) ]]; then + break + fi + + if [[ "${attempt}" == 120 ]]; then + echo "asciinema rollout did not complete within ten minutes" >&2 + exit 1 + fi + sleep 5 + done + + for attempt in {1..30}; do + if curl --fail --silent --show-error --max-time 15 \ + https://asciinema.brunner.ninja/ >/dev/null; then + echo "asciinema deployment is healthy" + exit 0 + fi + echo "Public health check ${attempt}/30 failed; retrying" + sleep 5 + done + + echo "Public health check did not pass" >&2 + exit 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c733b9d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Never commit generated cluster credentials or private environment files. +kubeconfig +*-kubeconfig +*.secret.yaml +.env diff --git a/README.md b/README.md index a0bea77..bba02e9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,125 @@ -# asciinema-deployment +# asciinema on Kubernetes -Deployment files for asciinema in my cluster \ No newline at end of file +Production deployment for `https://asciinema.brunner.ninja` in the `asciinema` +namespace. + +## Durable data and dependencies + +asciinema has two durable data sets: + +- A dedicated PostgreSQL 17 StatefulSet stores users, recording metadata, + searchable terminal text, and stream state. Its 8 Gi RBD claim uses the + retained `rook-ceph-block-ec` StorageClass. +- The private Rook Ceph RGW bucket `asciinema` stores uploaded asciicast files. + Downloads use expiring pre-signed URLs through `https://s3.brunner.ninja`. + +The public RGW ingress rejects ExAws signatures for some mutating methods, so +`custom.exs` uses the cluster-local RGW Service for server-side writes, moves, +and deletes while generating browser-visible pre-signed GET URLs with the +public endpoint. This mirrors the internal/public endpoint split used by other +S3-backed applications in the cluster. + +The application needs neither Redis nor Valkey. Its cache and temporary upload +workspace are bounded `emptyDir` volumes and may be discarded when the pod +moves. PostgreSQL's dedicated PVC and the RGW pool are the durable storage and +both must be backed up. + +A PVC mounted at `/var/lib/asciinema` would also work instead of S3, but it +would couple the pod to a filesystem volume and make future replicas and file +serving less flexible. Since this cluster already operates RGW, S3 is the +simpler production fit. + +## Install + +Run with the administrator kubeconfig for this cluster: + +```sh +./install.sh +``` + +The script creates and preserves random database, Phoenix signing, and Erlang +cookie values in `asciinema/asciinema-secrets`. It provisions a dedicated Rook +object user, copies only that user's generated credentials into the application +namespace, initializes the dedicated PostgreSQL instance and RGW bucket, then +applies and verifies the application. + +No secret values are committed or printed. Re-running the script preserves the +application secrets and reconciles the database password and Rook credentials. + +`asciinema.brunner.ninja` is a CNAME to `brunner.ninja`, and the existing +wildcard certificate covers the hostname. + +## First login and hardening + +Sign-ups are initially enabled so the first account can be created; upstream +makes that first account an administrator. Login links are sent through the +existing `smtp-relay.gmail.com` relay. If mail delivery is unavailable, find +the generated one-time URL in the application logs: + +```sh +kubectl logs deployment/asciinema -n asciinema | grep 'url from email' +``` + +After creating the account, set `SIGN_UP_DISABLED` to `"true"` in +`asciinema.yaml`, apply it, and wait for the rollout. CLI uploads already +require authentication, and new recordings and streams default to unlisted. +The login-gated admin panel is available at `/admin`; the unauthenticated +dedicated admin port is not exposed by the Service. + +Configure asciinema CLI 3.x with: + +```sh +export ASCIINEMA_SERVER_URL=https://asciinema.brunner.ninja +asciinema auth +``` + +For asciinema CLI 2.x, use `ASCIINEMA_API_URL` instead. + +## Validate + +```sh +./test.sh +``` + +The test checks the scripts, enforces the pinned server image and storage +architecture, and validates all Kubernetes resources with kubeconform. + +## Gitea Actions deployment + +The workflow in `.gitea/workflows/deploy.yml` validates every push and pull +request. A successful push to `main` performs server-side dry runs, reconciles +the dedicated PostgreSQL Service and StatefulSet plus the asciinema ConfigMaps, +Deployment, Service, and Ingress, waits for both workloads, and checks the +public HTTPS endpoint. + +CI deliberately does not run `install.sh` or apply `namespace.yaml`, +`bootstrap.yaml`, or `ceph-object-user.yaml`. Those administrator-only resources +create credentials, storage, the bucket, and the initial workload. CI cannot +read Secrets, create Jobs, change PVCs, modify the Namespace, or access Rook. +If a managed runtime resource is deleted, CI fails instead of recreating it; +restore it with `./install.sh`. + +Bootstrap the restricted deployment identity once: + +```sh +./create-ci-kubeconfig.sh +``` + +For remote cluster access through the usual SSH host: + +```sh +KUBECTL_SSH_HOST=arschrock ./create-ci-kubeconfig.sh +``` + +Store the script's single base64 output line as the Gitea repository Actions +secret `KUBE_CONFIG_BASE64`. Protect `main`, require the validation job before +merging, and push a harmless change to verify the first deployment. + +Delete `asciinema/asciinema-deployer-token` and the Gitea Actions secret to +revoke CI access. Running `create-ci-kubeconfig.sh` again recreates the token. + +The Role is intentionally name-scoped, but Kubernetes RBAC cannot constrain +individual fields of a Deployment or StatefulSet patch. A malicious change +merged into `main` could alter the pod specifications and expose Secrets already +referenced by those workloads. Branch protection and review remain part of the +security boundary; admission policy would be needed for field-level controls. diff --git a/asciinema.yaml b/asciinema.yaml new file mode 100644 index 0000000..223f879 --- /dev/null +++ b/asciinema.yaml @@ -0,0 +1,257 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: asciinema + namespace: asciinema + labels: + app.kubernetes.io/name: asciinema +data: + URL_HOST: asciinema.brunner.ninja + URL_SCHEME: https + TZ: Europe/Vienna + DB_POOL_SIZE: "5" + S3_BUCKET: asciinema + # Mutating requests bypass the ingress because RGW rejects some ExAws + # DELETE/HEAD signatures after they pass through the public proxy. + S3_ENDPOINT: http://rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local + # The custom adapter uses this endpoint only when producing browser-visible + # pre-signed GET URLs. + S3_PUBLIC_ENDPOINT: https://s3.brunner.ninja + S3_REGION: keller + SMTP_HOST: smtp-relay.gmail.com + SMTP_PORT: "587" + SMTP_TLS: if_available + MAIL_FROM_ADDRESS: asciinema@brunner.ninja + MAIL_REPLY_TO_ADDRESS: daniel@brunner.ninja + CONTACT_EMAIL_ADDRESS: daniel@brunner.ninja + ADMIN_PANEL_ON_MAIN_ENDPOINT: "true" + # The image defaults this to true. Keep the unauthenticated dedicated admin + # listener on loopback; administrators use the authenticated /admin route. + ADMIN_BIND_ALL: "false" + UPLOAD_AUTH_REQUIRED: "true" + UNREGISTERED_UPLOAD_COUNT_LIMIT: "0" + DEFAULT_RECORDING_VISIBILITY: unlisted + DEFAULT_STREAM_VISIBILITY: unlisted + # Change this to "true" immediately after creating the first account if the + # instance should remain single-user. + SIGN_UP_DISABLED: "false" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: asciinema-custom-config + namespace: asciinema + labels: + app.kubernetes.io/name: asciinema +data: + custom.exs: | + import Config + + # asciinema has one S3 endpoint setting for both internal operations and + # browser-facing pre-signed URLs. Ceph RGW in this cluster needs separate + # endpoints, so delegate all mutations to the upstream adapter and only + # customize URL generation. + defmodule Asciinema.FileStore.SplitEndpointS3 do + @behaviour Asciinema.FileStore + + alias ExAws.{Config, S3} + + @impl true + def uri(path) do + endpoint = System.fetch_env!("S3_PUBLIC_ENDPOINT") |> URI.parse() + + config = + :s3 + |> Config.new() + |> Map.put(:scheme, "#{endpoint.scheme}://") + |> Map.put(:host, endpoint.host) + |> Map.put(:port, endpoint.port) + + {:ok, url} = + S3.presigned_url(config, :get, bucket(), "uploads/" <> path) + + url + end + + @impl true + defdelegate put_file(dst_path, src_local_path, content_type), + to: Asciinema.FileStore.S3 + + @impl true + defdelegate move_file(from_path, to_path), to: Asciinema.FileStore.S3 + + @impl true + defdelegate delete_file(path), to: Asciinema.FileStore.S3 + + defp bucket, do: System.fetch_env!("S3_BUCKET") + end + + config :asciinema, Asciinema.FileStore, + adapter: Asciinema.FileStore.SplitEndpointS3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: asciinema + namespace: asciinema + labels: + app.kubernetes.io/name: asciinema +spec: + replicas: 1 + # Live stream state is process-local. A Recreate update avoids briefly + # routing a stream's WebSockets to two independent server processes. + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: asciinema + template: + metadata: + labels: + app.kubernetes.io/name: asciinema + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 10000 + runAsGroup: 0 + fsGroup: 0 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: asciinema + image: ghcr.io/asciinema/asciinema-server:20260626@sha256:0474b2186be46b23f5e16dbd4319727dbb68f27a303705fbe632821bf524b313 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: asciinema + env: + - name: HOME + value: /tmp + - name: CACHE_PATH + value: /var/cache/asciinema + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: database-url + - name: SECRET_KEY_BASE + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: secret-key-base + - name: RELEASE_COOKIE + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: release-cookie + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: s3-access-key-id + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: s3-secret-access-key + ports: + - name: http + containerPort: 4000 + protocol: TCP + startupProbe: + httpGet: + path: / + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 + readinessProbe: + httpGet: + path: / + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 50m + memory: 256Mi + ephemeral-storage: 128Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 3Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: cache + mountPath: /var/cache/asciinema + - name: custom-config + mountPath: /opt/app/etc/custom.exs + subPath: custom.exs + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: cache + emptyDir: + sizeLimit: 2Gi + - name: custom-config + configMap: + name: asciinema-custom-config + - name: tmp + emptyDir: + sizeLimit: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: asciinema + namespace: asciinema + labels: + app.kubernetes.io/name: asciinema +spec: + selector: + app.kubernetes.io/name: asciinema + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: asciinema + namespace: asciinema + labels: + app.kubernetes.io/name: asciinema + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure +spec: + ingressClassName: traefik + rules: + - host: asciinema.brunner.ninja + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: asciinema + port: + name: http diff --git a/bootstrap.yaml b/bootstrap.yaml new file mode 100644 index 0000000..db17173 --- /dev/null +++ b/bootstrap.yaml @@ -0,0 +1,61 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: asciinema-bootstrap + namespace: asciinema + labels: + app.kubernetes.io/name: asciinema + app.kubernetes.io/component: bootstrap +spec: + backoffLimit: 5 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app.kubernetes.io/name: asciinema + app.kubernetes.io/component: bootstrap + spec: + restartPolicy: OnFailure + automountServiceAccountToken: false + containers: + - name: bucket + image: amazon/aws-cli:2.28.22 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -ec + - >- + aws --endpoint-url "$S3_ENDPOINT" s3api head-bucket --bucket asciinema >/dev/null 2>&1 || + aws --endpoint-url "$S3_ENDPOINT" s3api create-bucket --bucket asciinema >/dev/null + env: + - name: S3_ENDPOINT + value: http://rook-ceph-rgw-my-store.rook-ceph.svc.cluster.local + - name: AWS_DEFAULT_REGION + value: keller + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: s3-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: s3-secret-access-key + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 250m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault diff --git a/ceph-object-user.yaml b/ceph-object-user.yaml new file mode 100644 index 0000000..75edd39 --- /dev/null +++ b/ceph-object-user.yaml @@ -0,0 +1,10 @@ +# This is an infrastructure prerequisite. Rook creates the matching credentials +# Secret in rook-ceph; install.sh copies only this user's keys to default. +apiVersion: ceph.rook.io/v1 +kind: CephObjectStoreUser +metadata: + name: asciinema + namespace: rook-ceph +spec: + store: my-store + displayName: asciinema recordings diff --git a/ci-deployer.yaml b/ci-deployer.yaml new file mode 100644 index 0000000..d1fd8df --- /dev/null +++ b/ci-deployer.yaml @@ -0,0 +1,91 @@ +# Bootstrap once with an administrator identity. CI can update only the named +# existing runtime resources in the asciinema namespace. It cannot access +# Secrets, Jobs, the Namespace, PVCs, or the Rook object user and bucket. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: asciinema-deployer + namespace: asciinema +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: asciinema-deployer + namespace: asciinema +rules: + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - asciinema + - asciinema-custom-config + verbs: + - get + - patch + - update + - apiGroups: + - "" + resources: + - services + resourceNames: + - asciinema + - postgresql + verbs: + - get + - patch + - update + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - asciinema + verbs: + - get + - patch + - update + - apiGroups: + - apps + resources: + - statefulsets + resourceNames: + - postgresql + verbs: + - get + - patch + - update + - apiGroups: + - networking.k8s.io + resources: + - ingresses + resourceNames: + - asciinema + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: asciinema-deployer + namespace: asciinema +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: asciinema-deployer +subjects: + - kind: ServiceAccount + name: asciinema-deployer + namespace: asciinema +--- +apiVersion: v1 +kind: Secret +metadata: + name: asciinema-deployer-token + namespace: asciinema + annotations: + kubernetes.io/service-account.name: asciinema-deployer +type: kubernetes.io/service-account-token diff --git a/create-ci-kubeconfig.sh b/create-ci-kubeconfig.sh new file mode 100755 index 0000000..6c20311 --- /dev/null +++ b/create-ci-kubeconfig.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -euo pipefail + +project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +namespace=${NAMESPACE:-asciinema} +service_account=${SERVICE_ACCOUNT:-asciinema-deployer} +secret=${TOKEN_SECRET:-asciinema-deployer-token} +ssh_host=${KUBECTL_SSH_HOST:-} +ssh_config=${SSH_CONFIG:-} + +cluster_kubectl() { + if [[ -n "${ssh_host}" ]]; then + if [[ -n "${ssh_config}" ]]; then + ssh -F "${ssh_config}" "${ssh_host}" kubectl "$@" + else + ssh "${ssh_host}" kubectl "$@" + fi + else + kubectl "$@" + fi +} + +if [[ -n "${ssh_host}" ]]; then + cluster_kubectl apply --filename=- < "${project_dir}/ci-deployer.yaml" >&2 +else + cluster_kubectl apply --filename "${project_dir}/ci-deployer.yaml" >&2 +fi + +for attempt in {1..30}; do + if cluster_kubectl --namespace "${namespace}" get secret "${secret}" \ + --output=jsonpath='{.data.token}' 2>/dev/null | grep -q .; then + break + fi + if [[ "${attempt}" == 30 ]]; then + echo "Timed out waiting for the service-account token" >&2 + exit 1 + fi + sleep 1 +done + +workdir=$(mktemp --directory) +trap 'rm -rf "${workdir}"' EXIT HUP INT TERM + +server=$(cluster_kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.server}') +tls_server_name=$(cluster_kubectl config view --minify --output=jsonpath='{.clusters[0].cluster.tls-server-name}') +cluster_kubectl --namespace "${namespace}" get secret "${secret}" \ + --output=jsonpath='{.data.ca\.crt}' | base64 --decode > "${workdir}/ca.crt" +token=$(cluster_kubectl --namespace "${namespace}" get secret "${secret}" \ + --output=jsonpath='{.data.token}' | base64 --decode) + +export KUBECONFIG="${workdir}/config" +cluster_options=( + --server="${server}" + --certificate-authority="${workdir}/ca.crt" + --embed-certs=true +) +if [[ -n "${tls_server_name}" ]]; then + cluster_options+=(--tls-server-name="${tls_server_name}") +fi +kubectl config set-cluster cluster "${cluster_options[@]}" >/dev/null +kubectl config set-credentials "${service_account}" --token="${token}" >/dev/null +kubectl config set-context asciinema \ + --cluster=cluster \ + --user="${service_account}" \ + --namespace="${namespace}" >/dev/null +kubectl config use-context asciinema >/dev/null + +echo "Store the following single line as the Gitea Actions secret KUBE_CONFIG_BASE64." >&2 +echo "Treat it as a password; do not commit it." >&2 +base64 --wrap=0 "${KUBECONFIG}" +printf '\n' diff --git a/infrastructure.yaml b/infrastructure.yaml new file mode 100644 index 0000000..1f425d8 --- /dev/null +++ b/infrastructure.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgresql + namespace: asciinema + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: asciinema +spec: + clusterIP: None + selector: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: asciinema + ports: + - name: postgresql + port: 5432 + targetPort: postgresql + protocol: TCP +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgresql + namespace: asciinema + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: asciinema +spec: + serviceName: postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: asciinema + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: asciinema + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 120 + securityContext: + runAsNonRoot: true + runAsUser: 70 + runAsGroup: 70 + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: postgresql + image: postgres:17.6-alpine@sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94 + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_DB + value: asciinema + - name: POSTGRES_USER + value: asciinema + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: asciinema-secrets + key: database-password + - name: POSTGRES_INITDB_ARGS + value: --auth-host=scram-sha-256 + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgresql + containerPort: 5432 + protocol: TCP + startupProbe: + exec: + command: + - pg_isready + - --username=asciinema + - --dbname=asciinema + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 + readinessProbe: + exec: + command: + - pg_isready + - --username=asciinema + - --dbname=asciinema + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + exec: + command: + - pg_isready + - --username=asciinema + - --dbname=asciinema + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + resources: + requests: + cpu: 50m + memory: 128Mi + ephemeral-storage: 64Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 1Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: run + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + volumes: + - name: run + emptyDir: + medium: Memory + sizeLimit: 16Mi + - name: tmp + emptyDir: + sizeLimit: 64Mi + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/part-of: asciinema + spec: + accessModes: + - ReadWriteOnce + storageClassName: rook-ceph-block-ec + resources: + requests: + storage: 8Gi diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..f69db5c --- /dev/null +++ b/install.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +set -euo pipefail + +namespace=asciinema +rook_namespace=rook-ceph +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +secret_name=asciinema-secrets +rook_secret=rook-ceph-object-user-my-store-asciinema +temp_dir=$(mktemp --directory) +trap 'rm -rf "${temp_dir}"' EXIT HUP INT TERM +chmod 0700 "${temp_dir}" + +for command in kubectl jq openssl base64; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +kubectl apply --filename "${script_dir}/namespace.yaml" +kubectl apply --filename "${script_dir}/ceph-object-user.yaml" + +echo "Waiting for Rook to provision the asciinema S3 credentials..." +for _ in $(seq 1 60); do + if kubectl get secret "${rook_secret}" --namespace "${rook_namespace}" >/dev/null 2>&1; then + break + fi + sleep 2 +done +kubectl get secret "${rook_secret}" --namespace "${rook_namespace}" >/dev/null + +read_secret_file() { + local source_namespace=$1 source_secret=$2 source_key=$3 destination=$4 + kubectl get secret "${source_secret}" --namespace "${source_namespace}" --output=json \ + | jq --raw-output --join-output --arg key "${source_key}" '.data[$key]' \ + | base64 --decode > "${destination}" + test -s "${destination}" +} + +if kubectl get secret "${secret_name}" --namespace "${namespace}" >/dev/null 2>&1; then + read_secret_file "${namespace}" "${secret_name}" database-password "${temp_dir}/database-password" + read_secret_file "${namespace}" "${secret_name}" secret-key-base "${temp_dir}/secret-key-base" + read_secret_file "${namespace}" "${secret_name}" release-cookie "${temp_dir}/release-cookie" +else + openssl rand -hex 32 | tr -d '\r\n' > "${temp_dir}/database-password" + openssl rand -hex 64 | tr -d '\r\n' > "${temp_dir}/secret-key-base" + openssl rand -hex 32 | tr -d '\r\n' > "${temp_dir}/release-cookie" +fi + +# Environment variables preserve newlines stored in Secret values, whereas +# command substitution below strips them. Normalize values created by older +# versions of this installer so the PostgreSQL container and DATABASE_URL see +# the exact same password. +for key in database-password secret-key-base release-cookie; do + value=$(<"${temp_dir}/${key}") + printf '%s' "${value}" > "${temp_dir}/${key}" +done + +# Rook owns and may rotate these credentials. Refresh the namespaced copy on +# every installation while preserving the database and application secrets. +read_secret_file "${rook_namespace}" "${rook_secret}" AccessKey "${temp_dir}/s3-access-key-id" +read_secret_file "${rook_namespace}" "${rook_secret}" SecretKey "${temp_dir}/s3-secret-access-key" +database_password=$(<"${temp_dir}/database-password") +printf 'postgresql://asciinema:%s@postgresql.asciinema.svc.cluster.local:5432/asciinema?pool_size=5' \ + "${database_password}" > "${temp_dir}/database-url" + +kubectl create secret generic "${secret_name}" \ + --namespace "${namespace}" \ + --from-file="database-password=${temp_dir}/database-password" \ + --from-file="database-url=${temp_dir}/database-url" \ + --from-file="secret-key-base=${temp_dir}/secret-key-base" \ + --from-file="release-cookie=${temp_dir}/release-cookie" \ + --from-file="s3-access-key-id=${temp_dir}/s3-access-key-id" \ + --from-file="s3-secret-access-key=${temp_dir}/s3-secret-access-key" \ + --dry-run=client \ + --output=yaml | kubectl apply --filename=- + +kubectl apply --dry-run=server --filename "${script_dir}/infrastructure.yaml" +kubectl apply --filename "${script_dir}/infrastructure.yaml" +kubectl rollout status statefulset/postgresql --namespace "${namespace}" --timeout=10m + +# Jobs are immutable and bucket bootstrap must be repeatable for restores. +# Only this exact, disposable Job is replaced. +kubectl delete job asciinema-bootstrap --namespace "${namespace}" --ignore-not-found +kubectl apply --filename "${script_dir}/bootstrap.yaml" +if ! kubectl wait --for=condition=complete job/asciinema-bootstrap \ + --namespace "${namespace}" --timeout=5m; then + kubectl logs job/asciinema-bootstrap --namespace "${namespace}" --all-containers=true || true + exit 1 +fi + +kubectl apply --dry-run=server --filename "${script_dir}/asciinema.yaml" +kubectl apply --filename "${script_dir}/asciinema.yaml" +# envFrom values and Secret-backed environment variables are read only when a +# pod starts. Recreate the sole replica so config changes and rotated RGW keys +# take effect on every successful installer run. +kubectl rollout restart deployment/asciinema --namespace "${namespace}" +kubectl rollout status deployment/asciinema --namespace "${namespace}" --timeout=10m + +echo +echo "asciinema is running for https://asciinema.brunner.ninja." diff --git a/namespace.yaml b/namespace.yaml new file mode 100644 index 0000000..141d07d --- /dev/null +++ b/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: asciinema + labels: + app.kubernetes.io/name: asciinema diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..fc1d85d --- /dev/null +++ b/test.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euo pipefail + +project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + +bash -n "${project_dir}/install.sh" +bash -n "${project_dir}/create-ci-kubeconfig.sh" +bash -n "${project_dir}/test.sh" + +image=$(sed -n 's|^[[:space:]]*image: \(ghcr.io/asciinema/asciinema-server[^[:space:]]*\)$|\1|p' \ + "${project_dir}/asciinema.yaml") +if [[ -z "${image}" || "${image}" == *:latest || "${image}" != *@sha256:* ]]; then + echo "The asciinema server image must use a non-latest tag pinned by digest." >&2 + exit 1 +fi + +postgres_image=$(sed -n 's|^[[:space:]]*image: \(postgres:[^[:space:]]*\)$|\1|p' \ + "${project_dir}/infrastructure.yaml") +if [[ -z "${postgres_image}" || "${postgres_image}" == *:latest || "${postgres_image}" != *@sha256:* ]]; then + echo "The PostgreSQL image must use a non-latest tag pinned by digest." >&2 + exit 1 +fi + +if ! grep -Fq 'storageClassName: rook-ceph-block-ec' "${project_dir}/infrastructure.yaml"; then + echo "The dedicated PostgreSQL claim must use retained Ceph block storage." >&2 + exit 1 +fi + +if grep -Eqi 'redis|valkey' \ + "${project_dir}/asciinema.yaml" "${project_dir}/bootstrap.yaml"; then + echo "asciinema should not use Redis/Valkey." >&2 + exit 1 +fi + +validate_file() { + local manifest=$1 + shift + + if command -v kubeconform >/dev/null 2>&1; then + kubeconform -strict -summary "$@" "${manifest}" + elif command -v docker >/dev/null 2>&1; then + docker run --rm --interactive \ + ghcr.io/yannh/kubeconform:v0.7.0@sha256:85dbef6b4b312b99133decc9c6fc9495e9fc5f92293d4ff3b7e1b30f5611823c \ + -strict -summary "$@" < "${manifest}" + else + echo "Required command not found: install kubeconform or Docker" >&2 + exit 1 + fi +} + +validate_file "${project_dir}/asciinema.yaml" +validate_file "${project_dir}/bootstrap.yaml" +validate_file "${project_dir}/infrastructure.yaml" +validate_file "${project_dir}/namespace.yaml" +validate_file "${project_dir}/ceph-object-user.yaml" -ignore-missing-schemas +validate_file "${project_dir}/ci-deployer.yaml" + +echo "asciinema scripts and Kubernetes resources are valid for ${image}."