From d1978abb0569a1a20f9c56d668bb3733a0b3eef6 Mon Sep 17 00:00:00 2001 From: 0xFEEDC0DE64 Date: Sat, 17 Oct 2026 13:50:19 +0200 Subject: [PATCH] Deploy achievement-safe Factorio 2.0.77 with strict placement, monitoring and limited CD --- .gitea/workflows/deploy.yml | 53 +++++++ .gitignore | 4 + README.md | 112 +++++++++++++++ bootstrap.sh | 20 +++ ci-deployer.yaml | 144 +++++++++++++++++++ create-ci-kubeconfig.sh | 11 ++ dashboard.yaml | 274 ++++++++++++++++++++++++++++++++++++ factorio.yaml | 261 ++++++++++++++++++++++++++++++++++ install.sh | 20 +++ kustomization.yaml | 32 +++++ monitoring.yaml | 83 +++++++++++ namespace.yaml | 4 + placement-rbac.yaml | 62 ++++++++ placement.yaml | 68 +++++++++ render.sh | 5 + scripts/collect.sh | 27 ++++ scripts/collector-health.sh | 4 + scripts/health.sh | 8 ++ scripts/placement.sh | 34 +++++ scripts/prepare-deploy.py | 23 +++ scripts/start.sh | 19 +++ server-adminlist.json | 1 + server-settings.json | 26 ++++ set-primary-available.sh | 9 ++ test.sh | 11 ++ tests/smoke.py | 43 ++++++ tests/test_collector.py | 36 +++++ tests/test_placement.py | 46 ++++++ tests/validate.py | 19 +++ 29 files changed, 1459 insertions(+) create mode 100644 .gitea/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100755 bootstrap.sh create mode 100644 ci-deployer.yaml create mode 100755 create-ci-kubeconfig.sh create mode 100644 dashboard.yaml create mode 100644 factorio.yaml create mode 100755 install.sh create mode 100644 kustomization.yaml create mode 100644 monitoring.yaml create mode 100644 namespace.yaml create mode 100644 placement-rbac.yaml create mode 100644 placement.yaml create mode 100755 render.sh create mode 100755 scripts/collect.sh create mode 100755 scripts/collector-health.sh create mode 100755 scripts/health.sh create mode 100755 scripts/placement.sh create mode 100644 scripts/prepare-deploy.py create mode 100755 scripts/start.sh create mode 100644 server-adminlist.json create mode 100644 server-settings.json create mode 100755 set-primary-available.sh create mode 100755 test.sh create mode 100644 tests/smoke.py create mode 100644 tests/test_collector.py create mode 100644 tests/test_placement.py create mode 100644 tests/validate.py diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..5ebf48d --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,53 @@ +name: Test and deploy Factorio +on: + push: + pull_request: +concurrency: + group: factorio-${{ gitea.ref }} + cancel-in-progress: false +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - name: Install tools + run: | + python3 -m pip install --user 'PyYAML==6.0.3' + curl -fsSL https://dl.k8s.io/release/v1.35.3/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl + chmod 0755 /usr/local/bin/kubectl + - name: Test policy and boot the actual game configuration + env: + FACTORIO_SMOKE_TEST: 'true' + run: ./test.sh + - name: Validate Kubernetes schemas + run: | + ./render.sh | docker run --rm -i ghcr.io/yannh/kubeconform:v0.7.0 -strict -ignore-missing-schemas -summary + for file in namespace.yaml placement-rbac.yaml ci-deployer.yaml; do + printf '%s\n' '---' + cat "$file" + done | docker run --rm -i ghcr.io/yannh/kubeconform:v0.7.0 -strict -summary + deploy: + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' + needs: [validate] + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - name: Install tools + run: | + python3 -m pip install --user 'PyYAML==6.0.3' + curl -fsSL https://dl.k8s.io/release/v1.35.3/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl + chmod 0755 /usr/local/bin/kubectl + - name: Deploy using the dedicated Factorio service account + env: + KUBE_CONFIG_BASE64: ${{ secrets.KUBE_CONFIG_BASE64 }} + FACTORIO_CI: 'true' + run: | + set -eu + export KUBECONFIG="${RUNNER_TEMP}/factorio-kubeconfig" + trap 'rm -f "$KUBECONFIG"' EXIT + umask 077 + printf '%s' "$KUBE_CONFIG_BASE64" | base64 -d > "$KUBECONFIG" + kubectl config set-cluster cluster --server=https://host.containers.internal:6443 --tls-server-name=192.168.0.2 + ./install.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c7e599f --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +*.kubeconfig +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..f5e7f4e --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# Factorio team server + +Achievement-safe Factorio 2.0.77 on Kubernetes. Public upstream images, a fresh +Ceph RBD world, Prometheus metrics, and the provisioned Grafana dashboard: +https://grafana.brunner.ninja/d/factorio-server + +Connect on the LAN to `192.168.0.17:34197` (UDP). The existing WAN UDP +forwarding on `newgw` is retained. +The world uses the base game, without Space Age or gameplay mods. No Lua console +commands are issued. The player collector only uses `/players online`, which +preserves achievements. Production, science, UPS and other Lua-only statistics +are intentionally excluded. The server pauses when empty; autosaves occur every +five minutes with ten rotating slots. Public server-list registration is off. + +## Placement and maintenance + +`factorio-placement` reconciles once per minute from an Odroid. Required node +affinity pins the server to **arschrock**, even if that node has insufficient +resources: it waits instead of quietly spilling onto an Odroid. After the +node is NotReady/Unknown for five minutes, or its Node object is deleted, the +reconciler restricts the replacement to odroid-h3-00 through odroid-h3-05. +After arschrock has been Ready for five minutes it moves the server back. +Each move interrupts play and follows the same Recreate/save procedure. +Transient API errors never count as a missing node. + +To mark arschrock unavailable **for Factorio only**: + +```sh +./set-primary-available.sh unavailable +./set-primary-available.sh available +``` + +This toggles `factorio.brunner.ninja/unavailable=true` on the Node; it does not +forge global Node Ready status. Cordoning arschrock also enables fallback. +Kubernetes' native NotReady/Unreachable conditions are honored. Readiness and +startup probes check authenticated RCON, the save directory and its writability; +liveness allows five minutes for slow saves before restarting a hung game. + +**Current infrastructure limitation:** arschrock is the sole API server, +scheduler and etcd host. A complete outage cannot trigger rescheduling until +the control plane is reachable. No control-plane configuration is changed here. +Ceph must also be healthy and able to detach/fence the old RBD attachment before +a new writer can start. This project never force-deletes pods or attachments to +bypass storage safety. Future control-plane HA needs no placement-policy change. + +## Deploy and upgrade + +Requirements: kubectl, Python 3 with PyYAML, and the configured homelab context. +`bootstrap.sh` is the one-time administrator path: namespace, limited RBAC, +random RCON secret, PVC and initial resources. Run `install.sh` thereafter. +Both work from any directory, validate on the API before applying, and wait for +rollout. Routine installs leave PVCs and reconciler-owned placement untouched. + +Change the server image in **factorio.yaml** and commit to **main**. Kustomize +copies that one image value to the player collector. The pipeline validates +manifests, tests failover policy, starts the actual server with the actual +rendered scripts/settings, tests probes and save/restart persistence, then +applies only the pre-created named resources. No custom game image is built. +The dashboard and scripts are deployed in the same pipeline. Configuration +checksums trigger a restart when mounted game scripts/settings change. + +Recreate prevents simultaneous old/new servers. Shutdown requests a save and +allows 180 seconds. Startup copies existing save ZIPs into `upgrade-backups/` +on the same PVC before the new binary loads them, retaining five startup +snapshots. This protects against upgrade conversion but is **not** an off-volume +backup. No automatic downgrade is attempted: older game binaries may not load a +newer save. To roll back, stop the server and restore an appropriate snapshot, +then deploy the matching older image. + +The original default/factorio-pvc world and its backing PV were explicitly +retired on 2026-10-17. Fresh data lives in factorio/factorio-data (16 GiB). +Routine install/CD never deletes or recreates this storage. + +## CI access + +Create `KUBE_CONFIG_BASE64` from `./create-ci-kubeconfig.sh` and upload its output +directly as a Gitea Actions repository secret; never commit it. The credential +belongs to `system:serviceaccount:factorio:factorio-deployer`, not an administrator. +It can get/patch/update/watch only the named Deployment, Services, ConfigMaps, +CronJob and monitoring resources, read pod/event diagnostics in the isolated +factorio namespace, and update only `default/factorio-dashboard`. It cannot read +or list Secrets, modify PVCs, delete resources, modify nodes or grant RBAC. +Bootstrap creates these objects because resourceNames cannot constrain create. + +The placement reconciler has its own service account: get only Node arschrock, +and get/patch only Deployment factorio. It has no node mutation or Secret access. + +The runner currently reaches the API at host.containers.internal:6443 with TLS +server name 192.168.0.2, matching the other homelab pipelines. Update that CI +endpoint when the control plane moves. Local commands use the existing context +without kubeconfig or context overrides. + +## Monitoring and tests + +The Grafana dashboard shows RAM, CPU, player count, RCON responsiveness, disk +space, network throughput, restarts, active host, and scrape latency. Metrics +are a textfile collector served by upstream node-exporter. Failed RCON queries +report server_up=0 and omit player counts rather than fabricate zero players. +Prometheus alerts cover unavailable server/exporter, stale player collection, +restarts, nearly full save storage and stalled placement reconciliation. +Neither Service exposes RCON. Its random password is stored only in a Secret +and the server's private config file; upstream shell tracing is suppressed. + +```sh +./test.sh +FACTORIO_SMOKE_TEST=true ./test.sh # requires Docker +./set-primary-available.sh unavailable # deliberate live failover exercise +./set-primary-available.sh available +``` + +Tests consume the rendered deployment; they do not maintain a second copy of +its image, environment, scripts or server settings. diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..e439879 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# One-time administrator bootstrap. Routine deployments use install.sh and limited RBAC. +set -euo pipefail +root=$(cd -- "$(dirname -- "$0")" && pwd) +[[ $(kubectl config current-context) == kubernetes-admin@kubernetes ]] || { echo 'Wrong Kubernetes context' >&2; exit 1; } +kubectl apply --dry-run=server -f "$root/namespace.yaml" +kubectl apply -f "$root/namespace.yaml" +for file in placement-rbac.yaml ci-deployer.yaml; do + kubectl apply --dry-run=server -f "$root/$file" + kubectl apply -f "$root/$file" +done +if ! kubectl -n factorio get secret factorio-runtime >/dev/null 2>&1; then + python3 - <<'PY' | kubectl create -f - +import base64,json,secrets +print(json.dumps({'apiVersion':'v1','kind':'Secret','metadata':{'name':'factorio-runtime','namespace':'factorio'},'type':'Opaque','data':{'rcon-password':base64.b64encode(secrets.token_urlsafe(32).encode()).decode()}})) +PY +fi +"$root/render.sh" | kubectl apply --dry-run=server -f - +"$root/render.sh" | kubectl apply -f - +"$root/install.sh" diff --git a/ci-deployer.yaml b/ci-deployer.yaml new file mode 100644 index 0000000..691d96d --- /dev/null +++ b/ci-deployer.yaml @@ -0,0 +1,144 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: factorio-deployer + namespace: factorio +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: factorio-deployer + namespace: factorio +rules: +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - factorio + verbs: + - get + - patch + - update + - watch +- apiGroups: + - '' + resources: + - services + resourceNames: + - factorio + - factorio-metrics + verbs: + - get + - patch + - update + - watch +- apiGroups: + - '' + resources: + - configmaps + resourceNames: + - factorio-scripts + - factorio-settings + verbs: + - get + - patch + - update + - watch +- apiGroups: + - batch + resources: + - cronjobs + resourceNames: + - factorio-placement + verbs: + - get + - patch + - update + - watch +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + resourceNames: + - factorio + verbs: + - get + - patch + - update + - watch +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + resourceNames: + - factorio + verbs: + - get + - patch + - update + - watch +- apiGroups: + - '' + resources: + - pods + - events + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: factorio-deployer + namespace: factorio +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: factorio-deployer +subjects: +- kind: ServiceAccount + name: factorio-deployer + namespace: factorio +--- +apiVersion: v1 +kind: Secret +metadata: + name: factorio-deployer-token + namespace: factorio + annotations: + kubernetes.io/service-account.name: factorio-deployer +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: factorio-dashboard-deployer + namespace: default +rules: +- apiGroups: + - '' + resources: + - configmaps + resourceNames: + - factorio-dashboard + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: factorio-dashboard-deployer + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: factorio-dashboard-deployer +subjects: +- kind: ServiceAccount + name: factorio-deployer + namespace: factorio diff --git a/create-ci-kubeconfig.sh b/create-ci-kubeconfig.sh new file mode 100755 index 0000000..8ff134f --- /dev/null +++ b/create-ci-kubeconfig.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +[[ $(kubectl config current-context) == kubernetes-admin@kubernetes ]] || { echo 'Wrong Kubernetes context' >&2; exit 1; } +python3 - <<'PY' +import base64,json,subprocess +get=lambda *args: json.loads(subprocess.check_output(['kubectl',*args])) +cluster=get('config','view','--minify','-o','json')['clusters'][0]['cluster'] +secret=get('-n','factorio','get','secret','factorio-deployer-token','-o','json')['data'] +config={'apiVersion':'v1','kind':'Config','clusters':[{'name':'cluster','cluster':{'server':cluster['server'],'certificate-authority-data':secret['ca.crt']}}], 'users':[{'name':'factorio-deployer','user':{'token':base64.b64decode(secret['token']).decode()}}], 'contexts':[{'name':'factorio-ci','context':{'cluster':'cluster','user':'factorio-deployer','namespace':'factorio'}}], 'current-context':'factorio-ci'} +print(base64.b64encode(json.dumps(config).encode()).decode()) +PY diff --git a/dashboard.yaml b/dashboard.yaml new file mode 100644 index 0000000..c762cf8 --- /dev/null +++ b/dashboard.yaml @@ -0,0 +1,274 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: factorio-dashboard + namespace: default + labels: + grafana_dashboard: '1' + app.kubernetes.io/name: factorio +data: + factorio.json: "{\n \"uid\": \"factorio-server\",\n \"title\": \"Factorio \\u2022\ + \ Team Server\",\n \"description\": \"Achievement-safe Factorio monitoring: normal\ + \ RCON player queries plus Kubernetes resource metrics. No mods or Lua commands.\ + \ Missing player data means a failed query, not zero players.\",\n \"tags\":\ + \ [\n \"factorio\",\n \"kubernetes\",\n \"gameserver\"\n ],\n \"schemaVersion\"\ + : 39,\n \"version\": 1,\n \"refresh\": \"30s\",\n \"timezone\": \"browser\"\ + ,\n \"time\": {\n \"from\": \"now-3h\",\n \"to\": \"now\"\n },\n \"panels\"\ + : [\n {\n \"id\": 1,\n \"type\": \"row\",\n \"title\": \"Factory\ + \ control room\",\n \"collapsed\": false,\n \"gridPos\": {\n \ + \ \"x\": 0,\n \"y\": 0,\n \"w\": 24,\n \"h\": 1\n }\n\ + \ },\n {\n \"id\": 2,\n \"type\": \"stat\",\n \"title\":\ + \ \"Server ready\",\n \"description\": \"\",\n \"datasource\": {\n \ + \ \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n \ + \ },\n \"gridPos\": {\n \"x\": 0,\n \"y\": 1,\n \"\ + w\": 4,\n \"h\": 5\n },\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"unit\": \"short\",\n \"min\": 0,\n \"color\"\ + : {\n \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10,\n \"spanNulls\": false\n }\n\ + \ },\n \"overrides\": []\n },\n \"targets\": [\n \ + \ {\n \"refId\": \"A\",\n \"expr\": \"max(kube_deployment_status_replicas_available{namespace=\\\ + \"factorio\\\",deployment=\\\"factorio\\\"})\",\n \"legendFormat\": \"\ + Ready\",\n \"range\": true\n }\n ],\n \"options\": {\n\ + \ \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\ + \n ],\n \"values\": false\n },\n \"graphMode\"\ + : \"area\",\n \"colorMode\": \"value\"\n }\n },\n {\n \"\ + id\": 3,\n \"type\": \"stat\",\n \"title\": \"Players online\",\n \ + \ \"description\": \"\",\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"${DS_PROMETHEUS}\"\n },\n \"gridPos\": {\n \ + \ \"x\": 4,\n \"y\": 1,\n \"w\": 4,\n \"h\": 5\n \ + \ },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\":\ + \ \"short\",\n \"min\": 0,\n \"color\": {\n \"mode\"\ + : \"palette-classic\"\n },\n \"custom\": {\n \"drawStyle\"\ + : \"line\",\n \"lineWidth\": 2,\n \"fillOpacity\": 10,\n\ + \ \"spanNulls\": false\n }\n },\n \"overrides\"\ + : []\n },\n \"targets\": [\n {\n \"refId\": \"A\",\n\ + \ \"expr\": \"max(factorio_players_online{namespace=\\\"factorio\\\"\ + }) and on() (max(factorio_server_up{namespace=\\\"factorio\\\"}) == 1)\",\n \ + \ \"legendFormat\": \"Players\",\n \"range\": true\n }\n\ + \ ],\n \"options\": {\n \"reduceOptions\": {\n \"calcs\"\ + : [\n \"lastNotNull\"\n ],\n \"values\": false\n\ + \ },\n \"graphMode\": \"area\",\n \"colorMode\": \"value\"\ + \n }\n },\n {\n \"id\": 4,\n \"type\": \"stat\",\n \"\ + title\": \"Server RAM\",\n \"description\": \"\",\n \"datasource\":\ + \ {\n \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n\ + \ },\n \"gridPos\": {\n \"x\": 8,\n \"y\": 1,\n \ + \ \"w\": 4,\n \"h\": 5\n },\n \"fieldConfig\": {\n \"\ + defaults\": {\n \"unit\": \"bytes\",\n \"min\": 0,\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\"\ + : 2,\n \"fillOpacity\": 10,\n \"spanNulls\": false\n \ + \ }\n },\n \"overrides\": []\n },\n \"targets\"\ + : [\n {\n \"refId\": \"A\",\n \"expr\": \"sum(container_memory_working_set_bytes{namespace=\\\ + \"factorio\\\",container=\\\"factorio\\\"})\",\n \"legendFormat\": \"\ + Working set\",\n \"range\": true\n }\n ],\n \"options\"\ + : {\n \"reduceOptions\": {\n \"calcs\": [\n \"lastNotNull\"\ + \n ],\n \"values\": false\n },\n \"graphMode\"\ + : \"area\",\n \"colorMode\": \"value\"\n }\n },\n {\n \"\ + id\": 5,\n \"type\": \"stat\",\n \"title\": \"CPU cores\",\n \"\ + description\": \"\",\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"${DS_PROMETHEUS}\"\n },\n \"gridPos\": {\n \ + \ \"x\": 12,\n \"y\": 1,\n \"w\": 4,\n \"h\": 5\n \ + \ },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\"\ + : \"short\",\n \"min\": 0,\n \"color\": {\n \"mode\"\ + : \"palette-classic\"\n },\n \"custom\": {\n \"drawStyle\"\ + : \"line\",\n \"lineWidth\": 2,\n \"fillOpacity\": 10,\n\ + \ \"spanNulls\": false\n }\n },\n \"overrides\"\ + : []\n },\n \"targets\": [\n {\n \"refId\": \"A\",\n\ + \ \"expr\": \"sum(rate(container_cpu_usage_seconds_total{namespace=\\\ + \"factorio\\\",container=\\\"factorio\\\"}[$__rate_interval]))\",\n \"\ + legendFormat\": \"CPU\",\n \"range\": true\n }\n ],\n \ + \ \"options\": {\n \"reduceOptions\": {\n \"calcs\": [\n \ + \ \"lastNotNull\"\n ],\n \"values\": false\n \ + \ },\n \"graphMode\": \"area\",\n \"colorMode\": \"value\"\n \ + \ }\n },\n {\n \"id\": 17,\n \"type\": \"stat\",\n \"title\"\ + : \"Game responding\",\n \"description\": \"\",\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n \ + \ },\n \"gridPos\": {\n \"x\": 16,\n \"y\": 1,\n \ + \ \"w\": 4,\n \"h\": 5\n },\n \"fieldConfig\": {\n \"\ + defaults\": {\n \"unit\": \"short\",\n \"min\": 0,\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\"\ + : 2,\n \"fillOpacity\": 10,\n \"spanNulls\": false\n \ + \ }\n },\n \"overrides\": []\n },\n \"targets\"\ + : [\n {\n \"refId\": \"A\",\n \"expr\": \"max(factorio_server_up{namespace=\\\ + \"factorio\\\"})\",\n \"legendFormat\": \"Game responding\",\n \ + \ \"range\": true\n }\n ],\n \"options\": {\n \"reduceOptions\"\ + : {\n \"calcs\": [\n \"lastNotNull\"\n ],\n \ + \ \"values\": false\n },\n \"graphMode\": \"area\",\n \ + \ \"colorMode\": \"value\"\n }\n },\n {\n \"id\": 18,\n \ + \ \"type\": \"stat\",\n \"title\": \"Save disk free\",\n \"description\"\ + : \"\",\n \"datasource\": {\n \"type\": \"prometheus\",\n \"\ + uid\": \"${DS_PROMETHEUS}\"\n },\n \"gridPos\": {\n \"x\": 20,\n\ + \ \"y\": 1,\n \"w\": 4,\n \"h\": 5\n },\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"unit\": \"bytes\",\n \"min\"\ + : 0,\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineWidth\": 2,\n \"fillOpacity\": 10,\n \"\ + spanNulls\": false\n }\n },\n \"overrides\": []\n \ + \ },\n \"targets\": [\n {\n \"refId\": \"A\",\n \ + \ \"expr\": \"kubelet_volume_stats_available_bytes{namespace=\\\"factorio\\\"\ + ,persistentvolumeclaim=\\\"factorio-data\\\"}\",\n \"legendFormat\":\ + \ \"Save disk free\",\n \"range\": true\n }\n ],\n \"\ + options\": {\n \"reduceOptions\": {\n \"calcs\": [\n \ + \ \"lastNotNull\"\n ],\n \"values\": false\n },\n \ + \ \"graphMode\": \"area\",\n \"colorMode\": \"value\"\n }\n\ + \ },\n {\n \"id\": 8,\n \"type\": \"row\",\n \"title\": \"\ + Server resources & availability\",\n \"collapsed\": false,\n \"gridPos\"\ + : {\n \"x\": 0,\n \"y\": 6,\n \"w\": 24,\n \"h\":\ + \ 1\n }\n },\n {\n \"id\": 9,\n \"type\": \"timeseries\"\ + ,\n \"title\": \"RAM \\u2014 game and monitoring\",\n \"description\"\ + : \"\",\n \"datasource\": {\n \"type\": \"prometheus\",\n \"\ + uid\": \"${DS_PROMETHEUS}\"\n },\n \"gridPos\": {\n \"x\": 0,\n\ + \ \"y\": 7,\n \"w\": 12,\n \"h\": 8\n },\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"unit\": \"bytes\",\n \"min\"\ + : 0,\n \"color\": {\n \"mode\": \"palette-classic\"\n \ + \ },\n \"custom\": {\n \"drawStyle\": \"line\",\n \ + \ \"lineWidth\": 2,\n \"fillOpacity\": 10,\n \"\ + spanNulls\": false\n }\n },\n \"overrides\": []\n \ + \ },\n \"targets\": [\n {\n \"refId\": \"A\",\n \ + \ \"expr\": \"sum by (container) (container_memory_working_set_bytes{namespace=\\\ + \"factorio\\\",pod=~\\\"factorio-[a-z0-9]+-[a-z0-9]+\\\",container!=\\\"\\\",container!=\\\ + \"POD\\\"})\",\n \"legendFormat\": \"{{container}}\",\n \"range\"\ + : true\n }\n ],\n \"options\": {\n \"legend\": {\n \ + \ \"displayMode\": \"table\",\n \"placement\": \"bottom\",\n \ + \ \"calcs\": [\n \"lastNotNull\",\n \"max\"\n \ + \ ]\n },\n \"tooltip\": {\n \"mode\": \"multi\",\n\ + \ \"sort\": \"desc\"\n }\n }\n },\n {\n \"id\"\ + : 10,\n \"type\": \"timeseries\",\n \"title\": \"CPU \\u2014 game and\ + \ monitoring\",\n \"description\": \"\",\n \"datasource\": {\n \ + \ \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n },\n\ + \ \"gridPos\": {\n \"x\": 12,\n \"y\": 7,\n \"w\": 12,\n\ + \ \"h\": 8\n },\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"unit\": \"short\",\n \"min\": 0,\n \"color\": {\n\ + \ \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10,\n \"spanNulls\": false\n }\n\ + \ },\n \"overrides\": []\n },\n \"targets\": [\n \ + \ {\n \"refId\": \"A\",\n \"expr\": \"sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\\\ + \"factorio\\\",pod=~\\\"factorio-[a-z0-9]+-[a-z0-9]+\\\",container!=\\\"\\\",container!=\\\ + \"POD\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{container}}\"\ + ,\n \"range\": true\n }\n ],\n \"options\": {\n \ + \ \"legend\": {\n \"displayMode\": \"table\",\n \"placement\"\ + : \"bottom\",\n \"calcs\": [\n \"lastNotNull\",\n \ + \ \"max\"\n ]\n },\n \"tooltip\": {\n \"mode\"\ + : \"multi\",\n \"sort\": \"desc\"\n }\n }\n },\n {\n\ + \ \"id\": 11,\n \"type\": \"timeseries\",\n \"title\": \"Players\ + \ online\",\n \"description\": \"\",\n \"datasource\": {\n \"\ + type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n },\n \ + \ \"gridPos\": {\n \"x\": 0,\n \"y\": 15,\n \"w\": 12,\n\ + \ \"h\": 8\n },\n \"fieldConfig\": {\n \"defaults\": {\n\ + \ \"unit\": \"short\",\n \"min\": 0,\n \"color\": {\n\ + \ \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10,\n \"spanNulls\": false\n }\n\ + \ },\n \"overrides\": []\n },\n \"targets\": [\n \ + \ {\n \"refId\": \"A\",\n \"expr\": \"max(factorio_players_online{namespace=\\\ + \"factorio\\\"}) and on() (max(factorio_server_up{namespace=\\\"factorio\\\"})\ + \ == 1)\",\n \"legendFormat\": \"Players\",\n \"range\": true\n\ + \ }\n ],\n \"options\": {\n \"legend\": {\n \"\ + displayMode\": \"table\",\n \"placement\": \"bottom\",\n \"\ + calcs\": [\n \"lastNotNull\",\n \"max\"\n ]\n \ + \ },\n \"tooltip\": {\n \"mode\": \"multi\",\n \ + \ \"sort\": \"desc\"\n }\n }\n },\n {\n \"id\": 12,\n \ + \ \"type\": \"timeseries\",\n \"title\": \"World storage\",\n \"\ + description\": \"\",\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"${DS_PROMETHEUS}\"\n },\n \"gridPos\": {\n \ + \ \"x\": 12,\n \"y\": 15,\n \"w\": 12,\n \"h\": 8\n \ + \ },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\"\ + : \"bytes\",\n \"min\": 0,\n \"color\": {\n \"mode\"\ + : \"palette-classic\"\n },\n \"custom\": {\n \"drawStyle\"\ + : \"line\",\n \"lineWidth\": 2,\n \"fillOpacity\": 10,\n\ + \ \"spanNulls\": false\n }\n },\n \"overrides\"\ + : []\n },\n \"targets\": [\n {\n \"refId\": \"A\",\n\ + \ \"expr\": \"kubelet_volume_stats_used_bytes{namespace=\\\"factorio\\\ + \",persistentvolumeclaim=\\\"factorio-data\\\"}\",\n \"legendFormat\"\ + : \"Used\",\n \"range\": true\n },\n {\n \"refId\"\ + : \"B\",\n \"expr\": \"kubelet_volume_stats_capacity_bytes{namespace=\\\ + \"factorio\\\",persistentvolumeclaim=\\\"factorio-data\\\"}\",\n \"legendFormat\"\ + : \"Capacity\",\n \"range\": true\n }\n ],\n \"options\"\ + : {\n \"legend\": {\n \"displayMode\": \"table\",\n \"\ + placement\": \"bottom\",\n \"calcs\": [\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n },\n \"tooltip\": {\n \ + \ \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n }\n\ + \ },\n {\n \"id\": 13,\n \"type\": \"timeseries\",\n \"title\"\ + : \"Network traffic\",\n \"description\": \"\",\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n \ + \ },\n \"gridPos\": {\n \"x\": 0,\n \"y\": 23,\n \ + \ \"w\": 12,\n \"h\": 8\n },\n \"fieldConfig\": {\n \"\ + defaults\": {\n \"unit\": \"Bps\",\n \"min\": 0,\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\"\ + : 2,\n \"fillOpacity\": 10,\n \"spanNulls\": false\n \ + \ }\n },\n \"overrides\": []\n },\n \"targets\"\ + : [\n {\n \"refId\": \"A\",\n \"expr\": \"sum(rate(container_network_receive_bytes_total{namespace=\\\ + \"factorio\\\",pod=~\\\"factorio-[a-z0-9]+-[a-z0-9]+\\\"}[$__rate_interval]))\"\ + ,\n \"legendFormat\": \"Received\",\n \"range\": true\n \ + \ },\n {\n \"refId\": \"B\",\n \"expr\": \"sum(rate(container_network_transmit_bytes_total{namespace=\\\ + \"factorio\\\",pod=~\\\"factorio-[a-z0-9]+-[a-z0-9]+\\\"}[$__rate_interval]))\"\ + ,\n \"legendFormat\": \"Sent\",\n \"range\": true\n }\n\ + \ ],\n \"options\": {\n \"legend\": {\n \"displayMode\"\ + : \"table\",\n \"placement\": \"bottom\",\n \"calcs\": [\n \ + \ \"lastNotNull\",\n \"max\"\n ]\n },\n \ + \ \"tooltip\": {\n \"mode\": \"multi\",\n \"sort\": \"\ + desc\"\n }\n }\n },\n {\n \"id\": 14,\n \"type\":\ + \ \"timeseries\",\n \"title\": \"Container restarts\",\n \"description\"\ + : \"\",\n \"datasource\": {\n \"type\": \"prometheus\",\n \"\ + uid\": \"${DS_PROMETHEUS}\"\n },\n \"gridPos\": {\n \"x\": 12,\n\ + \ \"y\": 23,\n \"w\": 12,\n \"h\": 8\n },\n \"\ + fieldConfig\": {\n \"defaults\": {\n \"unit\": \"short\",\n \ + \ \"min\": 0,\n \"color\": {\n \"mode\": \"palette-classic\"\ + \n },\n \"custom\": {\n \"drawStyle\": \"line\",\n\ + \ \"lineWidth\": 2,\n \"fillOpacity\": 10,\n \ + \ \"spanNulls\": false\n }\n },\n \"overrides\": []\n \ + \ },\n \"targets\": [\n {\n \"refId\": \"A\",\n \ + \ \"expr\": \"sum by (container) (kube_pod_container_status_restarts_total{namespace=\\\ + \"factorio\\\",pod=~\\\"factorio-[a-z0-9]+-[a-z0-9]+\\\"})\",\n \"legendFormat\"\ + : \"{{container}}\",\n \"range\": true\n }\n ],\n \"\ + options\": {\n \"legend\": {\n \"displayMode\": \"table\",\n \ + \ \"placement\": \"bottom\",\n \"calcs\": [\n \"lastNotNull\"\ + ,\n \"max\"\n ]\n },\n \"tooltip\": {\n \ + \ \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n }\n\ + \ },\n {\n \"id\": 15,\n \"type\": \"timeseries\",\n \"title\"\ + : \"Host placement\",\n \"description\": \"\",\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n \ + \ },\n \"gridPos\": {\n \"x\": 0,\n \"y\": 31,\n \ + \ \"w\": 12,\n \"h\": 8\n },\n \"fieldConfig\": {\n \"\ + defaults\": {\n \"unit\": \"short\",\n \"min\": 0,\n \ + \ \"color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\"\ + : 2,\n \"fillOpacity\": 10,\n \"spanNulls\": false\n \ + \ }\n },\n \"overrides\": []\n },\n \"targets\"\ + : [\n {\n \"refId\": \"A\",\n \"expr\": \"max by (node)\ + \ (kube_pod_info{namespace=\\\"factorio\\\",pod=~\\\"factorio-[a-z0-9]+-[a-z0-9]+\\\ + \"} * on(namespace,pod) group_left() (kube_pod_status_phase{namespace=\\\"factorio\\\ + \",phase=\\\"Running\\\"} == 1))\",\n \"legendFormat\": \"{{node}}\"\ + ,\n \"range\": true\n }\n ],\n \"options\": {\n \ + \ \"legend\": {\n \"displayMode\": \"table\",\n \"placement\"\ + : \"bottom\",\n \"calcs\": [\n \"lastNotNull\",\n \ + \ \"max\"\n ]\n },\n \"tooltip\": {\n \"mode\"\ + : \"multi\",\n \"sort\": \"desc\"\n }\n }\n },\n {\n\ + \ \"id\": 16,\n \"type\": \"timeseries\",\n \"title\": \"Monitoring\ + \ response time\",\n \"description\": \"\",\n \"datasource\": {\n \ + \ \"type\": \"prometheus\",\n \"uid\": \"${DS_PROMETHEUS}\"\n \ + \ },\n \"gridPos\": {\n \"x\": 12,\n \"y\": 31,\n \"\ + w\": 12,\n \"h\": 8\n },\n \"fieldConfig\": {\n \"defaults\"\ + : {\n \"unit\": \"s\",\n \"min\": 0,\n \"color\": {\n\ + \ \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \ + \ \"fillOpacity\": 10,\n \"spanNulls\": false\n }\n\ + \ },\n \"overrides\": []\n },\n \"targets\": [\n \ + \ {\n \"refId\": \"A\",\n \"expr\": \"scrape_duration_seconds{namespace=\\\ + \"factorio\\\",service=\\\"factorio-metrics\\\"}\",\n \"legendFormat\"\ + : \"Exporter scrape\",\n \"range\": true\n }\n ],\n \ + \ \"options\": {\n \"legend\": {\n \"displayMode\": \"table\"\ + ,\n \"placement\": \"bottom\",\n \"calcs\": [\n \"\ + lastNotNull\",\n \"max\"\n ]\n },\n \"tooltip\"\ + : {\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n\ + \ }\n }\n ],\n \"templating\": {\n \"list\": [\n {\n \ + \ \"name\": \"DS_PROMETHEUS\",\n \"type\": \"datasource\",\n \"\ + query\": \"prometheus\",\n \"label\": \"Prometheus\",\n \"current\"\ + : {\n \"text\": \"Prometheus\",\n \"value\": \"aevstx976yku8a\"\ + \n },\n \"refresh\": 1\n }\n ]\n },\n \"annotations\"\ + : {\n \"list\": []\n }\n}" diff --git a/factorio.yaml b/factorio.yaml new file mode 100644 index 0000000..c6bd53f --- /dev/null +++ b/factorio.yaml @@ -0,0 +1,261 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: factorio + namespace: factorio + labels: &id001 + app.kubernetes.io/name: factorio +spec: + replicas: 1 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 1800 + strategy: + type: Recreate + selector: + matchLabels: *id001 + template: + metadata: + labels: *id001 + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 180 + securityContext: + seccompProfile: + type: RuntimeDefault + fsGroup: 845 + fsGroupChangePolicy: OnRootMismatch + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - arschrock + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 300 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 300 + containers: + - name: factorio + image: docker.io/factoriotools/factorio:2.0.77 + command: + - /scripts/start.sh + env: + - name: TZ + value: Europe/Vienna + - name: DLC_SPACE_AGE + value: 'false' + ports: + - name: game-udp + containerPort: 34197 + protocol: UDP + - name: rcon + containerPort: 27015 + startupProbe: + exec: &id002 + command: + - /scripts/health.sh + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 80 + readinessProbe: + exec: *id002 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 4 + livenessProbe: + exec: *id002 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 10 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - timeout 90 rcon /server-save || true + resources: + requests: + cpu: '1' + memory: 2Gi + ephemeral-storage: 256Mi + limits: + memory: 12Gi + ephemeral-storage: 2Gi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - name: data + mountPath: /factorio + - name: scripts + mountPath: /scripts + readOnly: true + - name: settings + mountPath: /settings + readOnly: true + - name: credentials + mountPath: /credentials + readOnly: true + - name: player-collector + image: factorio-image-from-server + command: + - /scripts/collect.sh + env: + - name: CONFIG + value: /factorio/config + startupProbe: + exec: &id003 + command: + - /scripts/collector-health.sh + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 12 + readinessProbe: &id004 + exec: *id003 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: *id004 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + securityContext: &id007 + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 845 + runAsGroup: 845 + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: rcon-collector + mountPath: /factorio/config + readOnly: true + - name: metrics + mountPath: /metrics + - name: exporter + image: docker.io/prom/node-exporter:v1.10.2 + args: + - --web.listen-address=:9772 + - --collector.disable-defaults + - --collector.textfile + - --collector.textfile.directory=/metrics + ports: + - name: metrics + containerPort: 9772 + startupProbe: + httpGet: &id005 + path: / + port: metrics + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 12 + readinessProbe: &id006 + httpGet: *id005 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: *id006 + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + memory: 128Mi + securityContext: *id007 + volumeMounts: + - name: metrics + mountPath: /metrics + readOnly: true + volumes: + - name: data + persistentVolumeClaim: + claimName: factorio-data + - name: scripts + configMap: + name: factorio-scripts + defaultMode: 365 + - name: settings + configMap: + name: factorio-settings + - name: credentials + secret: + secretName: factorio-runtime + defaultMode: 256 + - name: metrics + emptyDir: + sizeLimit: 16Mi + - name: rcon-collector + secret: + secretName: factorio-runtime + defaultMode: 288 + items: + - key: rcon-password + path: rconpw +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: factorio-data + namespace: factorio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 16Gi + storageClassName: rook-ceph-block +--- +apiVersion: v1 +kind: Service +metadata: + name: factorio + namespace: factorio + labels: &id001 + app.kubernetes.io/name: factorio + annotations: + mikrotik.brunner.ninja/router: newgw + mikrotik.brunner.ninja/forwardings: '[{"name":"game-udp","servicePort":"game-udp","dstAddressList":"WANs"}]' +spec: + type: LoadBalancer + loadBalancerIP: 192.168.0.17 + ports: + - name: game-udp + port: 34197 + targetPort: game-udp + protocol: UDP + selector: *id001 +--- +apiVersion: v1 +kind: Service +metadata: + name: factorio-metrics + namespace: factorio + labels: + app.kubernetes.io/name: factorio + app.kubernetes.io/component: metrics +spec: + ports: + - name: metrics + port: 9772 + targetPort: metrics + selector: + app.kubernetes.io/name: factorio diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..d4ac6de --- /dev/null +++ b/install.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd -- "$(dirname -- "$0")" && pwd) +if [[ ${FACTORIO_CI:-false} != true ]]; then + [[ $(kubectl config current-context) == kubernetes-admin@kubernetes ]] || { echo 'Wrong Kubernetes context' >&2; exit 1; } +fi +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +"$root/render.sh" > "$work/rendered.yaml" +# The reconciler owns live placement. An image/config update must not reset failover. +kubectl -n factorio get deployment factorio --ignore-not-found -o json > "$work/live.json" +python3 "$root/scripts/prepare-deploy.py" "$work/rendered.yaml" "$work/live.json" > "$work/apply.yaml" +kubectl apply --dry-run=server -f "$work/apply.yaml" +kubectl apply -f "$work/apply.yaml" +if ! kubectl -n factorio rollout status deployment/factorio --timeout=1800s; then + kubectl -n factorio get deployment factorio -o yaml + kubectl -n factorio get pods -o wide + kubectl -n factorio get events --sort-by=.lastTimestamp + exit 1 +fi diff --git a/kustomization.yaml b/kustomization.yaml new file mode 100644 index 0000000..b9ecfee --- /dev/null +++ b/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: factorio +resources: +- factorio.yaml +- monitoring.yaml +- placement.yaml +configMapGenerator: +- name: factorio-scripts + files: + - scripts/start.sh + - scripts/health.sh + - scripts/placement.sh + - scripts/collect.sh + - scripts/collector-health.sh +- name: factorio-settings + files: + - server-settings.json + - server-adminlist.json +generatorOptions: + disableNameSuffixHash: true +replacements: +- source: + kind: Deployment + name: factorio + fieldPath: spec.template.spec.containers.[name=factorio].image + targets: + - select: + kind: Deployment + name: factorio + fieldPaths: + - spec.template.spec.containers.[name=player-collector].image diff --git a/monitoring.yaml b/monitoring.yaml new file mode 100644 index 0000000..929e134 --- /dev/null +++ b/monitoring.yaml @@ -0,0 +1,83 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: factorio + namespace: factorio + labels: + app.kubernetes.io/name: factorio +spec: + selector: + matchLabels: + app.kubernetes.io/name: factorio + app.kubernetes.io/component: metrics + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: factorio + namespace: factorio + labels: + app.kubernetes.io/name: factorio +spec: + groups: + - name: factorio + rules: + - alert: FactorioUnavailable + expr: kube_deployment_status_replicas_available{namespace="factorio",deployment="factorio"} + < 1 or absent(kube_deployment_status_replicas_available{namespace="factorio",deployment="factorio"}) + for: 5m + labels: + severity: warning + annotations: + summary: Factorio has no ready server + - alert: FactorioMetricsUnavailable + expr: up{namespace="factorio",service="factorio-metrics"} == 0 or absent(up{namespace="factorio",service="factorio-metrics"}) + for: 5m + labels: + severity: warning + annotations: + summary: Factorio metrics collection is failing + - alert: FactorioRestarting + expr: sum by (container) (increase(kube_pod_container_status_restarts_total{namespace="factorio"}[15m])) + > 2 + for: 5m + labels: + severity: warning + annotations: + summary: Factorio container is repeatedly restarting + - alert: FactorioStorageAlmostFull + expr: kubelet_volume_stats_available_bytes{namespace="factorio",persistentvolumeclaim="factorio-data"} + / kubelet_volume_stats_capacity_bytes{namespace="factorio",persistentvolumeclaim="factorio-data"} + < 0.15 + for: 10m + labels: + severity: warning + annotations: + summary: Factorio save storage has less than 15% free + - alert: FactorioPlacementReconcilerStale + expr: time() - kube_cronjob_status_last_successful_time{namespace="factorio",cronjob="factorio-placement"} + > 300 + for: 5m + labels: + severity: warning + annotations: + summary: Factorio placement reconciliation has stopped succeeding + - alert: FactorioGameNotResponding + expr: factorio_server_up{namespace="factorio"} == 0 or absent(factorio_server_up{namespace="factorio"}) + for: 3m + labels: + severity: warning + annotations: + summary: Factorio is not answering authenticated player queries + - alert: FactorioPlayerCollectorStale + expr: time() - factorio_collector_last_poll_timestamp_seconds{namespace="factorio"} + > 60 + for: 2m + labels: + severity: warning + annotations: + summary: Factorio player collector has stopped updating diff --git a/namespace.yaml b/namespace.yaml new file mode 100644 index 0000000..2e65976 --- /dev/null +++ b/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: factorio diff --git a/placement-rbac.yaml b/placement-rbac.yaml new file mode 100644 index 0000000..2a85770 --- /dev/null +++ b/placement-rbac.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: factorio-placement + namespace: factorio +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: factorio-placement + namespace: factorio +rules: +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - factorio + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: factorio-placement + namespace: factorio +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: factorio-placement +subjects: +- kind: ServiceAccount + name: factorio-placement + namespace: factorio +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: factorio-placement-node-reader +rules: +- apiGroups: + - '' + resources: + - nodes + resourceNames: + - arschrock + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: factorio-placement-node-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: factorio-placement-node-reader +subjects: +- kind: ServiceAccount + name: factorio-placement + namespace: factorio diff --git a/placement.yaml b/placement.yaml new file mode 100644 index 0000000..a0e0b61 --- /dev/null +++ b/placement.yaml @@ -0,0 +1,68 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: factorio-placement + namespace: factorio +spec: + schedule: '* * * * *' + concurrencyPolicy: Forbid + startingDeadlineSeconds: 45 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 50 + template: + spec: + serviceAccountName: factorio-placement + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - odroid-h3-00 + - odroid-h3-01 + - odroid-h3-02 + - odroid-h3-03 + - odroid-h3-04 + - odroid-h3-05 + containers: + - name: placement + image: docker.io/alpine/k8s:1.35.3 + command: + - /bin/sh + - /scripts/placement.sh + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: HOME + value: /tmp + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: scripts + configMap: + name: factorio-scripts + - name: tmp + emptyDir: {} diff --git a/render.sh b/render.sh new file mode 100755 index 0000000..1eab666 --- /dev/null +++ b/render.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +kubectl kustomize "$root" +cat "$root/dashboard.yaml" diff --git a/scripts/collect.sh b/scripts/collect.sh new file mode 100755 index 0000000..84338ea --- /dev/null +++ b/scripts/collect.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Only ordinary console commands: no /c, /sc, Lua, mods, or achievement changes. +set -eu +metrics_dir=${METRICS_DIR:-/metrics} +mkdir -p "$metrics_dir" +while :; do + now=$(date +%s) + up=0 + players='' + if response=$(timeout 8 rcon /players online 2>/dev/null); then + players=$(printf '%s\n' "$response" | sed -n 's/^Online players (\([0-9][0-9]*\)):.*/\1/p' | head -1) + [ -z "$players" ] || up=1 + fi + { + echo '# HELP factorio_server_up Authenticated non-Lua RCON player query succeeded.' + echo '# TYPE factorio_server_up gauge' + echo "factorio_server_up $up" + echo '# HELP factorio_players_online Connected players; omitted if the query fails.' + echo '# TYPE factorio_players_online gauge' + [ -z "$players" ] || echo "factorio_players_online $players" + echo '# HELP factorio_collector_last_poll_timestamp_seconds Last completed polling attempt.' + echo '# TYPE factorio_collector_last_poll_timestamp_seconds gauge' + echo "factorio_collector_last_poll_timestamp_seconds $now" + } > "$metrics_dir/factorio.prom.tmp" + mv "$metrics_dir/factorio.prom.tmp" "$metrics_dir/factorio.prom" + sleep 15 +done diff --git a/scripts/collector-health.sh b/scripts/collector-health.sh new file mode 100755 index 0000000..a4a71e2 --- /dev/null +++ b/scripts/collector-health.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +test -s /metrics/factorio.prom +[ "$(( $(date +%s) - $(stat -c %Y /metrics/factorio.prom) ))" -lt 60 ] diff --git a/scripts/health.sh b/scripts/health.sh new file mode 100755 index 0000000..b36c29f --- /dev/null +++ b/scripts/health.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Exercise the game loop over authenticated RCON, and check that its data is writable. +test -s /factorio/config/rconpw +test -d /factorio/saves +test -w /factorio/saves +response=$(rcon /players online) +printf '%s\n' "$response" | grep -q 'Online players' diff --git a/scripts/placement.sh b/scripts/placement.sh new file mode 100755 index 0000000..c0bfdb9 --- /dev/null +++ b/scripts/placement.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Read-only node health; patch only the named Factorio Deployment. +set -eu +node=$(kubectl get node arschrock --ignore-not-found -o json) +deployment=$(kubectl -n factorio get deployment factorio -o json) +current=$(printf '%s' "$deployment" | jq -c '.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values') +if [ -z "$node" ]; then + mode=odroids +else + mode=$(printf '%s' "$node" | jq -r ' + if .metadata.labels["factorio.brunner.ninja/unavailable"] == "true" or .spec.unschedulable == true then "odroids" + else ([.status.conditions[] | select(.type == "Ready")][0]) as $ready | + if $ready == null then "hold" + elif (now - ($ready.lastTransitionTime | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601)) < 300 then "hold" + elif $ready.status == "True" then "primary" else "odroids" end + end') +fi +case "$mode" in + primary) desired='["arschrock"]' ;; + odroids) desired='["odroid-h3-00","odroid-h3-01","odroid-h3-02","odroid-h3-03","odroid-h3-04","odroid-h3-05"]' ;; + hold) echo 'Node health transition less than five minutes ago; holding placement'; exit 0 ;; + *) exit 1 ;; +esac +if [ "$current" = "$desired" ]; then + echo "Placement unchanged: $mode" + exit 0 +fi +# Optimistic concurrency: never overwrite a concurrent deployment or maintenance action. +patch=$(printf '%s' "$deployment" | jq -c --argjson nodes "$desired" '[ + {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion}, + {op:"replace",path:"/spec/template/spec/affinity/nodeAffinity/requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms/0/matchExpressions/0/values",value:$nodes} +]') +kubectl -n factorio patch deployment factorio --type=json -p "$patch" +echo "Placement changed to $mode" diff --git a/scripts/prepare-deploy.py b/scripts/prepare-deploy.py new file mode 100644 index 0000000..a1be088 --- /dev/null +++ b/scripts/prepare-deploy.py @@ -0,0 +1,23 @@ +"""Preserve reconciler-owned placement; roll when mounted configuration changes.""" +import hashlib +import json +import sys +from pathlib import Path +import yaml + +docs = list(yaml.safe_load_all(Path(sys.argv[1]).read_text())) +live_text = Path(sys.argv[2]).read_text().strip() +live = json.loads(live_text) if live_text else None +config = {d['metadata']['name']: d.get('data', {}) for d in docs if d['kind'] == 'ConfigMap' and d['metadata']['namespace'] == 'factorio'} +checksum = hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest() +for doc in docs: + if doc['kind'] == 'Deployment' and doc['metadata']['name'] == 'factorio': + template = doc['spec']['template'] + template['metadata'].setdefault('annotations', {})['factorio.brunner.ninja/config-revision'] = checksum + if live: + template['spec']['affinity'] = live['spec']['template']['spec']['affinity'] + # The fresh PVC is an administrator bootstrap resource. CI never owns storage. + if doc['kind'] == 'PersistentVolumeClaim': + continue + print('---') + print(yaml.safe_dump(doc, sort_keys=False), end='') diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..0a1044c --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +mkdir -p /factorio/config /factorio/upgrade-backups +# A startup snapshot preserves the pre-upgrade saves. Keep the last five starts. +if compgen -G '/factorio/saves/*.zip' >/dev/null; then + backup="/factorio/upgrade-backups/$(date -u +%Y%m%dT%H%M%S)" + mkdir -p "$backup" + cp -a /factorio/saves/*.zip "$backup/" + mapfile -t old < <(find /factorio/upgrade-backups -mindepth 1 -maxdepth 1 -type d | sort -r | tail -n +6) + for path in "${old[@]}"; do rm -rf -- "$path"; done +fi +cp /settings/server-settings.json /factorio/config/server-settings.json +cp /settings/server-adminlist.json /factorio/config/server-adminlist.json +cp /credentials/rcon-password /factorio/config/rconpw +chmod 600 /factorio/config/rconpw +# Upstream uses bash xtrace, which would otherwise log the RCON password. +exec 3>/dev/null +export BASH_XTRACEFD=3 +exec /docker-entrypoint.sh diff --git a/server-adminlist.json b/server-adminlist.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/server-adminlist.json @@ -0,0 +1 @@ +[] diff --git a/server-settings.json b/server-settings.json new file mode 100644 index 0000000..936f040 --- /dev/null +++ b/server-settings.json @@ -0,0 +1,26 @@ +{ + "name": "Brunner Factorio", + "description": "Team factory \u2014 Factorio 2.0", + "tags": [ + "team", + "brunner" + ], + "max_players": 0, + "visibility": { + "public": false, + "lan": true + }, + "require_user_verification": true, + "max_upload_in_kilobytes_per_second": 0, + "max_upload_slots": 5, + "minimum_latency_in_ticks": 0, + "ignore_player_limit_for_returning_players": false, + "allow_commands": "admins-only", + "autosave_interval": 5, + "autosave_slots": 10, + "afk_autokick_interval": 0, + "auto_pause": true, + "only_admins_can_pause_the_game": true, + "autosave_only_on_server": true, + "non_blocking_saving": true +} diff --git a/set-primary-available.sh b/set-primary-available.sh new file mode 100755 index 0000000..928ad30 --- /dev/null +++ b/set-primary-available.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +[ "$(kubectl config current-context)" = kubernetes-admin@kubernetes ] || { echo 'Wrong Kubernetes context' >&2; exit 1; } +case "${1:-}" in + unavailable) kubectl label node arschrock factorio.brunner.ninja/unavailable=true --overwrite ;; + available) kubectl label node arschrock factorio.brunner.ninja/unavailable- ;; + *) echo "Usage: $0 available|unavailable" >&2; exit 2 ;; +esac +echo 'The placement reconciler reacts within one minute. Recovery also requires Ready for five minutes.' diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..28a3bb2 --- /dev/null +++ b/test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd -- "$(dirname -- "$0")" && pwd) +cd "$root" +for script in ./*.sh scripts/*.sh; do bash -n "$script"; done +python3 -m unittest discover -s tests -v +./render.sh > /tmp/factorio-rendered.yaml +python3 tests/validate.py /tmp/factorio-rendered.yaml +if [[ ${FACTORIO_SMOKE_TEST:-false} == true ]]; then + python3 tests/smoke.py /tmp/factorio-rendered.yaml +fi diff --git a/tests/smoke.py b/tests/smoke.py new file mode 100644 index 0000000..02dc17b --- /dev/null +++ b/tests/smoke.py @@ -0,0 +1,43 @@ +"""Boot the actual server image with the actual rendered configuration, then probe/save/restart it.""" +import json,os,secrets,subprocess,sys,tempfile,time +from pathlib import Path +import yaml + +docs=list(yaml.safe_load_all(Path(sys.argv[1]).read_text())) +d=next(d for d in docs if d['kind']=='Deployment') +c=next(c for c in d['spec']['template']['spec']['containers'] if c['name']=='factorio') +cms={d['metadata']['name']:d['data'] for d in docs if d['kind']=='ConfigMap'} +run=lambda *a,**kw: subprocess.run(a,check=True,**kw) +with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + for name in ['scripts','settings','credentials','data']:(root/name).mkdir() + for folder,name in [('scripts','factorio-scripts'),('settings','factorio-settings')]: + for key,value in cms[name].items(): + file=root/folder/key; file.write_text(value); file.chmod(0o755 if key.endswith('.sh') else 0o644) + (root/'credentials/rcon-password').write_text(secrets.token_urlsafe(32)) + args=['docker','run','--detach','--name','factorio-smoke-'+secrets.token_hex(5)] + for env in c['env']:args+=['--env',env['name']+'='+env['value']] + for mount in c['volumeMounts']: + folder={'data':'data','scripts':'scripts','settings':'settings','credentials':'credentials'}[mount['name']] + args+=['--volume',f'{root/folder}:{mount["mountPath"]}'+(':ro' if mount.get('readOnly') else '')] + args+=['--entrypoint',c['command'][0],c['image'],*c['command'][1:]] + container=subprocess.check_output(args,text=True).strip() + def ready(): + command=c['readinessProbe']['exec']['command'] + for _ in range(90): + if subprocess.run(['docker','exec',container,*command],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=15).returncode==0:return + time.sleep(2) + raise RuntimeError('Server did not pass the production readiness probe') + try: + ready() + run('docker','exec',container,'rcon','/server-save') + run('docker','stop','--time','180',container,stdout=subprocess.DEVNULL) + assert list((root/'data/saves').glob('*.zip')), 'No persistent save' + run('docker','start',container,stdout=subprocess.DEVNULL) + ready() + assert list((root/'data/upgrade-backups').glob('*/*.zip')), 'No pre-upgrade backup' + print('Actual image passed RCON readiness, persistent save, restart, and upgrade-backup checks') + finally: + run('docker','rm','--force',container,stdout=subprocess.DEVNULL) + # The upstream image owns its world files; cleanup through the same image. + run('docker','run','--rm','--volume',f'{root}/data:/cleanup','--entrypoint','sh',c['image'],'-c','rm -rf /cleanup/*',stdout=subprocess.DEVNULL) diff --git a/tests/test_collector.py b/tests/test_collector.py new file mode 100644 index 0000000..50b6020 --- /dev/null +++ b/tests/test_collector.py @@ -0,0 +1,36 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest + +SCRIPT=Path(__file__).resolve().parents[1]/'scripts/collect.sh' + +class CollectorTests(unittest.TestCase): + def collect(self,response,code=0): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp) + fake=root/'rcon' + fake.write_text('#!/bin/sh\n[ "$*" = "/players online" ] || exit 88\nprintf "%s\\n" "$RESPONSE"\nexit "$CODE"\n') + fake.chmod(0o755) + proc=subprocess.Popen(['sh',str(SCRIPT)],env={**os.environ,'PATH':tmp+':'+os.environ['PATH'],'METRICS_DIR':tmp,'RESPONSE':response,'CODE':str(code)},start_new_session=True) + try: + output=root/'factorio.prom' + for _ in range(100): + if output.exists():return output.read_text() + time.sleep(.01) + self.fail('Collector did not atomically publish metrics') + finally: + import signal + os.killpg(proc.pid,signal.SIGTERM);proc.wait() + def test_empty_world_reports_zero(self): + metrics=self.collect('Online players (0):') + self.assertIn('factorio_server_up 1',metrics);self.assertIn('factorio_players_online 0',metrics) + def test_players_reported_without_lua(self): + self.assertIn('factorio_players_online 12',self.collect('Online players (12):\n Alice (online)')) + def test_failed_rcon_is_not_zero_players(self): + metrics=self.collect('',1) + self.assertIn('factorio_server_up 0',metrics);self.assertNotIn('\nfactorio_players_online ',metrics) + def test_unexpected_response_does_not_report_healthy(self): + self.assertIn('factorio_server_up 0',self.collect('Authentication failed')) diff --git a/tests/test_placement.py b/tests/test_placement.py new file mode 100644 index 0000000..c1215bd --- /dev/null +++ b/tests/test_placement.py @@ -0,0 +1,46 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / 'scripts/placement.sh' + +class PlacementTests(unittest.TestCase): + def run_case(self, status='True', age=600, missing=False, unavailable=False, cordoned=False, fallback=False, error=False): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + now = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time()-age)) + node = {'metadata': {'labels': {'factorio.brunner.ninja/unavailable': str(unavailable).lower()}}, 'spec': {'unschedulable': cordoned}, 'status': {'conditions':[{'type':'Ready','status':status,'lastTransitionTime':now}]}} + nodes = ['odroid-h3-'+str(i).zfill(2) for i in range(6)] if fallback else ['arschrock'] + deployment={'metadata':{'resourceVersion':'123'},'spec':{'template':{'spec':{'affinity':{'nodeAffinity':{'requiredDuringSchedulingIgnoredDuringExecution':{'nodeSelectorTerms':[{'matchExpressions':[{'values':nodes}]}]}}}}}}} + (root/'node').write_text('' if missing else json.dumps(node)) + (root/'deployment').write_text(json.dumps(deployment)) + fake=root/'kubectl' + fake.write_text('#!/bin/sh\ncase "$*" in\n *"get node"*) '+('exit 1' if error else 'cat "$FIXTURES/node"')+' ;;\n *"get deployment"*) cat "$FIXTURES/deployment" ;;\n *"patch deployment"*) printf "%s\\n" "$*" > "$FIXTURES/patch" ;;\n *) exit 99 ;;\nesac\n') + fake.chmod(0o755) + result=subprocess.run(['sh',str(SCRIPT)],env={**os.environ,'PATH':tmp+':'+os.environ['PATH'],'FIXTURES':tmp},capture_output=True,text=True) + return result.returncode,(root/'patch').read_text() if (root/'patch').exists() else '' + + def test_healthy_primary_stays(self): + self.assertEqual(self.run_case(),(0,'')) + def test_notready_fails_over_after_grace(self): + code,patch=self.run_case(status='False'); self.assertEqual(code,0); self.assertIn('odroid-h3-05',patch) + def test_unreachable_fails_over(self): + self.assertIn('odroid-h3-00',self.run_case(status='Unknown')[1]) + def test_missing_primary_fails_over(self): + self.assertIn('odroid-h3-00',self.run_case(missing=True)[1]) + def test_transient_failure_does_not_move_world(self): + self.assertEqual(self.run_case(status='False',age=30),(0,'')) + def test_recovery_waits_for_stability(self): + self.assertEqual(self.run_case(age=30,fallback=True),(0,'')) + def test_stable_recovery_returns_to_primary(self): + self.assertIn('arschrock',self.run_case(fallback=True)[1]) + def test_maintenance_moves_immediately(self): + self.assertIn('odroid-h3-00',self.run_case(unavailable=True,age=5)[1]) + def test_cordon_moves_immediately(self): + self.assertIn('odroid-h3-00',self.run_case(cordoned=True)[1]) + def test_api_error_is_not_a_missing_node(self): + code,patch=self.run_case(error=True); self.assertNotEqual(code,0); self.assertEqual(patch,'') diff --git a/tests/validate.py b/tests/validate.py new file mode 100644 index 0000000..e088244 --- /dev/null +++ b/tests/validate.py @@ -0,0 +1,19 @@ +import json,sys,yaml +from pathlib import Path +resources=list(yaml.safe_load_all(Path(sys.argv[1]).read_text())) +assert len({(d['kind'],d['metadata'].get('namespace'),d['metadata']['name']) for d in resources})==len(resources) +d=next(d for d in resources if d['kind']=='Deployment') +assert d['spec']['strategy']['type']=='Recreate' +assert d['spec']['replicas']==1 +spec=d['spec']['template']['spec'] +assert not spec['automountServiceAccountToken'] +for c in spec['containers']: + assert all(p in c for p in ('startupProbe','readinessProbe','livenessProbe','resources')), c['name'] + assert ':' in c['image'] and not c['image'].endswith(':latest') +assert spec['affinity']['nodeAffinity']['requiredDuringSchedulingIgnoredDuringExecution'] +for service in (d for d in resources if d['kind']=='Service'): + assert all(p['port']!=27015 for p in service['spec']['ports']) +for cm in (d for d in resources if d['kind']=='ConfigMap'): + for name,value in cm.get('data',{}).items(): + if name.endswith('.json'): json.loads(value) +print(f'Validated {len(resources)} rendered resources')