diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index f545e79..16fbc61 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -24,7 +24,7 @@ jobs: - name: Validate Kubernetes schemas run: | ./render.sh | docker run --rm -i ghcr.io/yannh/kubeconform:v0.7.0 -strict -ignore-missing-schemas -summary - for file in namespace.yaml placement-rbac.yaml ci-deployer.yaml; do + for file in namespace.yaml ci-deployer.yaml; do printf '%s\n' '---' cat "$file" done | docker run --rm -i ghcr.io/yannh/kubeconform:v0.7.0 -strict -summary diff --git a/README.md b/README.md index 1c8baf6..d809481 100644 --- a/README.md +++ b/README.md @@ -15,34 +15,23 @@ five minutes with ten rotating slots. Public server-list registration is off. ## Placement and maintenance -`factorio-placement` reconciles once per minute from an Odroid. Required node -affinity pins the server to **arschrock**, even if that node has insufficient -resources: it waits instead of quietly spilling onto an Odroid. After the -node is NotReady/Unknown for five minutes, or its Node object is deleted, the -reconciler restricts the replacement to odroid-h3-00 through odroid-h3-05. -After arschrock has been Ready for five minutes it moves the server back. -Each move interrupts play and follows the same Recreate/save procedure. -Transient API errors never count as a missing node. +Factorio uses the same scheduling model as Garry's Mod: node affinity gives +**arschrock** the highest preference, while the scheduler may use an Odroid if +arschrock is unavailable, cordoned, or cannot satisfy the Pod's resource +request. There is no custom placement controller or periodic placement Pod. +Kubernetes does not automatically move an already-running Pod back merely +because arschrock later becomes preferable again. -To mark arschrock unavailable **for Factorio only**: - -```sh -./set-primary-available.sh unavailable -./set-primary-available.sh available -``` - -This toggles `factorio.brunner.ninja/unavailable=true` on the Node; it does not -forge global Node Ready status. Cordoning arschrock also enables fallback. -Kubernetes' native NotReady/Unreachable conditions are honored. Readiness and -startup probes check authenticated RCON, the save directory and its writability; -liveness allows five minutes for slow saves before restarting a hung game. +Readiness and startup probes check authenticated RCON, the save directory and +its writability; liveness allows five minutes for slow saves before restarting +a hung game. **Current infrastructure limitation:** arschrock is the sole API server, scheduler and etcd host. A complete outage cannot trigger rescheduling until the control plane is reachable. No control-plane configuration is changed here. Ceph must also be healthy and able to detach/fence the old RBD attachment before a new writer can start. This project never force-deletes pods or attachments to -bypass storage safety. Future control-plane HA needs no placement-policy change. +bypass storage safety. ## Deploy and upgrade @@ -50,11 +39,11 @@ Requirements: kubectl, Python 3 with PyYAML, and the configured homelab context. `bootstrap.sh` is the one-time administrator path: namespace, limited RBAC, random RCON secret, PVC and initial resources. Run `install.sh` thereafter. Both work from any directory, validate on the API before applying, and wait for -rollout. Routine installs leave PVCs and reconciler-owned placement untouched. +rollout. Routine installs leave PVCs untouched. Change the server image in **factorio.yaml** and commit to **main**. Kustomize copies that one image value to the player collector. The pipeline validates -manifests, tests failover policy, starts the actual server with the actual +manifests, validates the arschrock scheduling preference, starts the actual server with the actual rendered scripts/settings, tests probes and save/restart persistence, then applies only the pre-created named resources. No custom game image is built. The dashboard and scripts are deployed in the same pipeline. Configuration @@ -77,15 +66,12 @@ Routine install/CD never deletes or recreates this storage. Create `KUBE_CONFIG_BASE64` from `./create-ci-kubeconfig.sh` and upload its output directly as a Gitea Actions repository secret; never commit it. The credential belongs to `system:serviceaccount:factorio:factorio-deployer`, not an administrator. -It can get/patch/update/watch only the named Deployment, Services, ConfigMaps, -CronJob and monitoring resources, read pod/event diagnostics in the isolated +It can get/patch/update/watch only the named Deployment, Services, ConfigMaps +and monitoring resources, read pod/event diagnostics in the isolated factorio namespace, and update only `default/factorio-dashboard`. It cannot read or list Secrets, modify PVCs, delete resources, modify nodes or grant RBAC. Bootstrap creates these objects because resourceNames cannot constrain create. -The placement reconciler has its own service account: get only Node arschrock, -and get/patch only Deployment factorio. It has no node mutation or Secret access. - The runner currently reaches the API at host.containers.internal:6443 with TLS server name 192.168.0.2, matching the other homelab pipelines. Update that CI endpoint when the control plane moves. Local commands use the existing context @@ -98,15 +84,13 @@ space, network throughput, restarts, active host, and scrape latency. Metrics are a textfile collector served by upstream node-exporter. Failed RCON queries report server_up=0 and omit player counts rather than fabricate zero players. Prometheus alerts cover unavailable server/exporter, stale player collection, -restarts, nearly full save storage and stalled placement reconciliation. +restarts and nearly full save storage. Neither Service exposes RCON. Its random password is stored only in a Secret and the server's private config file; upstream shell tracing is suppressed. ```sh ./test.sh FACTORIO_SMOKE_TEST=true ./test.sh # requires Docker -./set-primary-available.sh unavailable # deliberate live failover exercise -./set-primary-available.sh available ``` Tests consume the rendered deployment; they do not maintain a second copy of diff --git a/bootstrap.sh b/bootstrap.sh index e439879..1bd761e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -5,7 +5,7 @@ root=$(cd -- "$(dirname -- "$0")" && pwd) [[ $(kubectl config current-context) == kubernetes-admin@kubernetes ]] || { echo 'Wrong Kubernetes context' >&2; exit 1; } kubectl apply --dry-run=server -f "$root/namespace.yaml" kubectl apply -f "$root/namespace.yaml" -for file in placement-rbac.yaml ci-deployer.yaml; do +for file in ci-deployer.yaml; do kubectl apply --dry-run=server -f "$root/$file" kubectl apply -f "$root/$file" done diff --git a/ci-deployer.yaml b/ci-deployer.yaml index 691d96d..f9776cc 100644 --- a/ci-deployer.yaml +++ b/ci-deployer.yaml @@ -46,17 +46,6 @@ rules: - patch - update - watch -- apiGroups: - - batch - resources: - - cronjobs - resourceNames: - - factorio-placement - verbs: - - get - - patch - - update - - watch - apiGroups: - monitoring.coreos.com resources: diff --git a/factorio.yaml b/factorio.yaml index 46ebab9..fd08068 100644 --- a/factorio.yaml +++ b/factorio.yaml @@ -26,9 +26,10 @@ spec: fsGroupChangePolicy: OnRootMismatch affinity: nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: - key: kubernetes.io/hostname operator: In values: diff --git a/install.sh b/install.sh index d4ac6de..1563968 100755 --- a/install.sh +++ b/install.sh @@ -7,9 +7,7 @@ fi work=$(mktemp -d) trap 'rm -rf "$work"' EXIT "$root/render.sh" > "$work/rendered.yaml" -# The reconciler owns live placement. An image/config update must not reset failover. -kubectl -n factorio get deployment factorio --ignore-not-found -o json > "$work/live.json" -python3 "$root/scripts/prepare-deploy.py" "$work/rendered.yaml" "$work/live.json" > "$work/apply.yaml" +python3 "$root/scripts/prepare-deploy.py" "$work/rendered.yaml" > "$work/apply.yaml" kubectl apply --dry-run=server -f "$work/apply.yaml" kubectl apply -f "$work/apply.yaml" if ! kubectl -n factorio rollout status deployment/factorio --timeout=1800s; then diff --git a/kustomization.yaml b/kustomization.yaml index b9ecfee..7d8d9e0 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -4,13 +4,11 @@ namespace: factorio resources: - factorio.yaml - monitoring.yaml -- placement.yaml configMapGenerator: - name: factorio-scripts files: - scripts/start.sh - scripts/health.sh - - scripts/placement.sh - scripts/collect.sh - scripts/collector-health.sh - name: factorio-settings diff --git a/monitoring.yaml b/monitoring.yaml index 929e134..da57c8d 100644 --- a/monitoring.yaml +++ b/monitoring.yaml @@ -58,14 +58,6 @@ spec: severity: warning annotations: summary: Factorio save storage has less than 15% free - - alert: FactorioPlacementReconcilerStale - expr: time() - kube_cronjob_status_last_successful_time{namespace="factorio",cronjob="factorio-placement"} - > 300 - for: 5m - labels: - severity: warning - annotations: - summary: Factorio placement reconciliation has stopped succeeding - alert: FactorioGameNotResponding expr: factorio_server_up{namespace="factorio"} == 0 or absent(factorio_server_up{namespace="factorio"}) for: 3m diff --git a/placement-rbac.yaml b/placement-rbac.yaml deleted file mode 100644 index 2a85770..0000000 --- a/placement-rbac.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: factorio-placement - namespace: factorio ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: factorio-placement - namespace: factorio -rules: -- apiGroups: - - apps - resources: - - deployments - resourceNames: - - factorio - verbs: - - get - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: factorio-placement - namespace: factorio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: factorio-placement -subjects: -- kind: ServiceAccount - name: factorio-placement - namespace: factorio ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: factorio-placement-node-reader -rules: -- apiGroups: - - '' - resources: - - nodes - resourceNames: - - arschrock - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: factorio-placement-node-reader -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: factorio-placement-node-reader -subjects: -- kind: ServiceAccount - name: factorio-placement - namespace: factorio diff --git a/placement.yaml b/placement.yaml deleted file mode 100644 index a0e0b61..0000000 --- a/placement.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: factorio-placement - namespace: factorio -spec: - schedule: '* * * * *' - concurrencyPolicy: Forbid - startingDeadlineSeconds: 45 - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 2 - jobTemplate: - spec: - backoffLimit: 1 - activeDeadlineSeconds: 50 - template: - spec: - serviceAccountName: factorio-placement - restartPolicy: Never - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - odroid-h3-00 - - odroid-h3-01 - - odroid-h3-02 - - odroid-h3-03 - - odroid-h3-04 - - odroid-h3-05 - containers: - - name: placement - image: docker.io/alpine/k8s:1.35.3 - command: - - /bin/sh - - /scripts/placement.sh - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - memory: 128Mi - securityContext: - runAsNonRoot: true - runAsUser: 65534 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - env: - - name: HOME - value: /tmp - volumeMounts: - - name: scripts - mountPath: /scripts - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: scripts - configMap: - name: factorio-scripts - - name: tmp - emptyDir: {} diff --git a/scripts/placement.sh b/scripts/placement.sh deleted file mode 100755 index c0bfdb9..0000000 --- a/scripts/placement.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# Read-only node health; patch only the named Factorio Deployment. -set -eu -node=$(kubectl get node arschrock --ignore-not-found -o json) -deployment=$(kubectl -n factorio get deployment factorio -o json) -current=$(printf '%s' "$deployment" | jq -c '.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values') -if [ -z "$node" ]; then - mode=odroids -else - mode=$(printf '%s' "$node" | jq -r ' - if .metadata.labels["factorio.brunner.ninja/unavailable"] == "true" or .spec.unschedulable == true then "odroids" - else ([.status.conditions[] | select(.type == "Ready")][0]) as $ready | - if $ready == null then "hold" - elif (now - ($ready.lastTransitionTime | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601)) < 300 then "hold" - elif $ready.status == "True" then "primary" else "odroids" end - end') -fi -case "$mode" in - primary) desired='["arschrock"]' ;; - odroids) desired='["odroid-h3-00","odroid-h3-01","odroid-h3-02","odroid-h3-03","odroid-h3-04","odroid-h3-05"]' ;; - hold) echo 'Node health transition less than five minutes ago; holding placement'; exit 0 ;; - *) exit 1 ;; -esac -if [ "$current" = "$desired" ]; then - echo "Placement unchanged: $mode" - exit 0 -fi -# Optimistic concurrency: never overwrite a concurrent deployment or maintenance action. -patch=$(printf '%s' "$deployment" | jq -c --argjson nodes "$desired" '[ - {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion}, - {op:"replace",path:"/spec/template/spec/affinity/nodeAffinity/requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms/0/matchExpressions/0/values",value:$nodes} -]') -kubectl -n factorio patch deployment factorio --type=json -p "$patch" -echo "Placement changed to $mode" diff --git a/scripts/prepare-deploy.py b/scripts/prepare-deploy.py index a1be088..2e0a7e4 100644 --- a/scripts/prepare-deploy.py +++ b/scripts/prepare-deploy.py @@ -1,4 +1,4 @@ -"""Preserve reconciler-owned placement; roll when mounted configuration changes.""" +"""Roll the server when mounted configuration changes.""" import hashlib import json import sys @@ -6,16 +6,12 @@ from pathlib import Path import yaml docs = list(yaml.safe_load_all(Path(sys.argv[1]).read_text())) -live_text = Path(sys.argv[2]).read_text().strip() -live = json.loads(live_text) if live_text else None config = {d['metadata']['name']: d.get('data', {}) for d in docs if d['kind'] == 'ConfigMap' and d['metadata']['namespace'] == 'factorio'} checksum = hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest() for doc in docs: if doc['kind'] == 'Deployment' and doc['metadata']['name'] == 'factorio': template = doc['spec']['template'] template['metadata'].setdefault('annotations', {})['factorio.brunner.ninja/config-revision'] = checksum - if live: - template['spec']['affinity'] = live['spec']['template']['spec']['affinity'] # The fresh PVC is an administrator bootstrap resource. CI never owns storage. if doc['kind'] == 'PersistentVolumeClaim': continue diff --git a/set-primary-available.sh b/set-primary-available.sh deleted file mode 100755 index 928ad30..0000000 --- a/set-primary-available.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -eu -[ "$(kubectl config current-context)" = kubernetes-admin@kubernetes ] || { echo 'Wrong Kubernetes context' >&2; exit 1; } -case "${1:-}" in - unavailable) kubectl label node arschrock factorio.brunner.ninja/unavailable=true --overwrite ;; - available) kubectl label node arschrock factorio.brunner.ninja/unavailable- ;; - *) echo "Usage: $0 available|unavailable" >&2; exit 2 ;; -esac -echo 'The placement reconciler reacts within one minute. Recovery also requires Ready for five minutes.' diff --git a/tests/test_placement.py b/tests/test_placement.py deleted file mode 100644 index c1215bd..0000000 --- a/tests/test_placement.py +++ /dev/null @@ -1,46 +0,0 @@ -import json -import os -from pathlib import Path -import subprocess -import tempfile -import time -import unittest - -SCRIPT = Path(__file__).resolve().parents[1] / 'scripts/placement.sh' - -class PlacementTests(unittest.TestCase): - def run_case(self, status='True', age=600, missing=False, unavailable=False, cordoned=False, fallback=False, error=False): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - now = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time()-age)) - node = {'metadata': {'labels': {'factorio.brunner.ninja/unavailable': str(unavailable).lower()}}, 'spec': {'unschedulable': cordoned}, 'status': {'conditions':[{'type':'Ready','status':status,'lastTransitionTime':now}]}} - nodes = ['odroid-h3-'+str(i).zfill(2) for i in range(6)] if fallback else ['arschrock'] - deployment={'metadata':{'resourceVersion':'123'},'spec':{'template':{'spec':{'affinity':{'nodeAffinity':{'requiredDuringSchedulingIgnoredDuringExecution':{'nodeSelectorTerms':[{'matchExpressions':[{'values':nodes}]}]}}}}}}} - (root/'node').write_text('' if missing else json.dumps(node)) - (root/'deployment').write_text(json.dumps(deployment)) - fake=root/'kubectl' - fake.write_text('#!/bin/sh\ncase "$*" in\n *"get node"*) '+('exit 1' if error else 'cat "$FIXTURES/node"')+' ;;\n *"get deployment"*) cat "$FIXTURES/deployment" ;;\n *"patch deployment"*) printf "%s\\n" "$*" > "$FIXTURES/patch" ;;\n *) exit 99 ;;\nesac\n') - fake.chmod(0o755) - result=subprocess.run(['sh',str(SCRIPT)],env={**os.environ,'PATH':tmp+':'+os.environ['PATH'],'FIXTURES':tmp},capture_output=True,text=True) - return result.returncode,(root/'patch').read_text() if (root/'patch').exists() else '' - - def test_healthy_primary_stays(self): - self.assertEqual(self.run_case(),(0,'')) - def test_notready_fails_over_after_grace(self): - code,patch=self.run_case(status='False'); self.assertEqual(code,0); self.assertIn('odroid-h3-05',patch) - def test_unreachable_fails_over(self): - self.assertIn('odroid-h3-00',self.run_case(status='Unknown')[1]) - def test_missing_primary_fails_over(self): - self.assertIn('odroid-h3-00',self.run_case(missing=True)[1]) - def test_transient_failure_does_not_move_world(self): - self.assertEqual(self.run_case(status='False',age=30),(0,'')) - def test_recovery_waits_for_stability(self): - self.assertEqual(self.run_case(age=30,fallback=True),(0,'')) - def test_stable_recovery_returns_to_primary(self): - self.assertIn('arschrock',self.run_case(fallback=True)[1]) - def test_maintenance_moves_immediately(self): - self.assertIn('odroid-h3-00',self.run_case(unavailable=True,age=5)[1]) - def test_cordon_moves_immediately(self): - self.assertIn('odroid-h3-00',self.run_case(cordoned=True)[1]) - def test_api_error_is_not_a_missing_node(self): - code,patch=self.run_case(error=True); self.assertNotEqual(code,0); self.assertEqual(patch,'') diff --git a/tests/validate.py b/tests/validate.py index e088244..dbd7a7f 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -10,7 +10,8 @@ assert not spec['automountServiceAccountToken'] for c in spec['containers']: assert all(p in c for p in ('startupProbe','readinessProbe','livenessProbe','resources')), c['name'] assert ':' in c['image'] and not c['image'].endswith(':latest') -assert spec['affinity']['nodeAffinity']['requiredDuringSchedulingIgnoredDuringExecution'] +preference=spec['affinity']['nodeAffinity']['preferredDuringSchedulingIgnoredDuringExecution'] +assert preference == [{'weight': 100, 'preference': {'matchExpressions': [{'key': 'kubernetes.io/hostname', 'operator': 'In', 'values': ['arschrock']}]}}] for service in (d for d in resources if d['kind']=='Service'): assert all(p['port']!=27015 for p in service['spec']['ports']) for cm in (d for d in resources if d['kind']=='ConfigMap'):