Deploy achievement-safe Factorio 2.0.77 with strict placement, monitoring and limited CD

tests/smoke.py

@@ -0,0 +1,43 @@
+"""Boot the actual server image with the actual rendered configuration, then probe/save/restart it."""
+import json,os,secrets,subprocess,sys,tempfile,time
+from pathlib import Path
+import yaml
+
+docs=list(yaml.safe_load_all(Path(sys.argv[1]).read_text()))
+d=next(d for d in docs if d['kind']=='Deployment')
+c=next(c for c in d['spec']['template']['spec']['containers'] if c['name']=='factorio')
+cms={d['metadata']['name']:d['data'] for d in docs if d['kind']=='ConfigMap'}
+run=lambda *a,**kw: subprocess.run(a,check=True,**kw)
+with tempfile.TemporaryDirectory() as directory:
+    root=Path(directory)
+    for name in ['scripts','settings','credentials','data']:(root/name).mkdir()
+    for folder,name in [('scripts','factorio-scripts'),('settings','factorio-settings')]:
+        for key,value in cms[name].items():
+            file=root/folder/key; file.write_text(value); file.chmod(0o755 if key.endswith('.sh') else 0o644)
+    (root/'credentials/rcon-password').write_text(secrets.token_urlsafe(32))
+    args=['docker','run','--detach','--name','factorio-smoke-'+secrets.token_hex(5)]
+    for env in c['env']:args+=['--env',env['name']+'='+env['value']]
+    for mount in c['volumeMounts']:
+        folder={'data':'data','scripts':'scripts','settings':'settings','credentials':'credentials'}[mount['name']]
+        args+=['--volume',f'{root/folder}:{mount["mountPath"]}'+(':ro' if mount.get('readOnly') else '')]
+    args+=['--entrypoint',c['command'][0],c['image'],*c['command'][1:]]
+    container=subprocess.check_output(args,text=True).strip()
+    def ready():
+        command=c['readinessProbe']['exec']['command']
+        for _ in range(90):
+            if subprocess.run(['docker','exec',container,*command],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=15).returncode==0:return
+            time.sleep(2)
+        raise RuntimeError('Server did not pass the production readiness probe')
+    try:
+        ready()
+        run('docker','exec',container,'rcon','/server-save')
+        run('docker','stop','--time','180',container,stdout=subprocess.DEVNULL)
+        assert list((root/'data/saves').glob('*.zip')), 'No persistent save'
+        run('docker','start',container,stdout=subprocess.DEVNULL)
+        ready()
+        assert list((root/'data/upgrade-backups').glob('*/*.zip')), 'No pre-upgrade backup'
+        print('Actual image passed RCON readiness, persistent save, restart, and upgrade-backup checks')
+    finally:
+        run('docker','rm','--force',container,stdout=subprocess.DEVNULL)
+        # The upstream image owns its world files; cleanup through the same image.
+        run('docker','run','--rm','--volume',f'{root}/data:/cleanup','--entrypoint','sh',c['image'],'-c','rm -rf /cleanup/*',stdout=subprocess.DEVNULL)

tests/test_collector.py

@@ -0,0 +1,36 @@
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+import unittest
+
+SCRIPT=Path(__file__).resolve().parents[1]/'scripts/collect.sh'
+
+class CollectorTests(unittest.TestCase):
+    def collect(self,response,code=0):
+        with tempfile.TemporaryDirectory() as tmp:
+            root=Path(tmp)
+            fake=root/'rcon'
+            fake.write_text('#!/bin/sh\n[ "$*" = "/players online" ] || exit 88\nprintf "%s\\n" "$RESPONSE"\nexit "$CODE"\n')
+            fake.chmod(0o755)
+            proc=subprocess.Popen(['sh',str(SCRIPT)],env={**os.environ,'PATH':tmp+':'+os.environ['PATH'],'METRICS_DIR':tmp,'RESPONSE':response,'CODE':str(code)},start_new_session=True)
+            try:
+                output=root/'factorio.prom'
+                for _ in range(100):
+                    if output.exists():return output.read_text()
+                    time.sleep(.01)
+                self.fail('Collector did not atomically publish metrics')
+            finally:
+                import signal
+                os.killpg(proc.pid,signal.SIGTERM);proc.wait()
+    def test_empty_world_reports_zero(self):
+        metrics=self.collect('Online players (0):')
+        self.assertIn('factorio_server_up 1',metrics);self.assertIn('factorio_players_online 0',metrics)
+    def test_players_reported_without_lua(self):
+        self.assertIn('factorio_players_online 12',self.collect('Online players (12):\n  Alice (online)'))
+    def test_failed_rcon_is_not_zero_players(self):
+        metrics=self.collect('',1)
+        self.assertIn('factorio_server_up 0',metrics);self.assertNotIn('\nfactorio_players_online ',metrics)
+    def test_unexpected_response_does_not_report_healthy(self):
+        self.assertIn('factorio_server_up 0',self.collect('Authentication failed'))

tests/test_placement.py

@@ -0,0 +1,46 @@
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+import unittest
+
+SCRIPT = Path(__file__).resolve().parents[1] / 'scripts/placement.sh'
+
+class PlacementTests(unittest.TestCase):
+    def run_case(self, status='True', age=600, missing=False, unavailable=False, cordoned=False, fallback=False, error=False):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            now = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time()-age))
+            node = {'metadata': {'labels': {'factorio.brunner.ninja/unavailable': str(unavailable).lower()}}, 'spec': {'unschedulable': cordoned}, 'status': {'conditions':[{'type':'Ready','status':status,'lastTransitionTime':now}]}}
+            nodes = ['odroid-h3-'+str(i).zfill(2) for i in range(6)] if fallback else ['arschrock']
+            deployment={'metadata':{'resourceVersion':'123'},'spec':{'template':{'spec':{'affinity':{'nodeAffinity':{'requiredDuringSchedulingIgnoredDuringExecution':{'nodeSelectorTerms':[{'matchExpressions':[{'values':nodes}]}]}}}}}}}
+            (root/'node').write_text('' if missing else json.dumps(node))
+            (root/'deployment').write_text(json.dumps(deployment))
+            fake=root/'kubectl'
+            fake.write_text('#!/bin/sh\ncase "$*" in\n *"get node"*) '+('exit 1' if error else 'cat "$FIXTURES/node"')+' ;;\n *"get deployment"*) cat "$FIXTURES/deployment" ;;\n *"patch deployment"*) printf "%s\\n" "$*" > "$FIXTURES/patch" ;;\n *) exit 99 ;;\nesac\n')
+            fake.chmod(0o755)
+            result=subprocess.run(['sh',str(SCRIPT)],env={**os.environ,'PATH':tmp+':'+os.environ['PATH'],'FIXTURES':tmp},capture_output=True,text=True)
+            return result.returncode,(root/'patch').read_text() if (root/'patch').exists() else ''
+
+    def test_healthy_primary_stays(self):
+        self.assertEqual(self.run_case(),(0,''))
+    def test_notready_fails_over_after_grace(self):
+        code,patch=self.run_case(status='False'); self.assertEqual(code,0); self.assertIn('odroid-h3-05',patch)
+    def test_unreachable_fails_over(self):
+        self.assertIn('odroid-h3-00',self.run_case(status='Unknown')[1])
+    def test_missing_primary_fails_over(self):
+        self.assertIn('odroid-h3-00',self.run_case(missing=True)[1])
+    def test_transient_failure_does_not_move_world(self):
+        self.assertEqual(self.run_case(status='False',age=30),(0,''))
+    def test_recovery_waits_for_stability(self):
+        self.assertEqual(self.run_case(age=30,fallback=True),(0,''))
+    def test_stable_recovery_returns_to_primary(self):
+        self.assertIn('arschrock',self.run_case(fallback=True)[1])
+    def test_maintenance_moves_immediately(self):
+        self.assertIn('odroid-h3-00',self.run_case(unavailable=True,age=5)[1])
+    def test_cordon_moves_immediately(self):
+        self.assertIn('odroid-h3-00',self.run_case(cordoned=True)[1])
+    def test_api_error_is_not_a_missing_node(self):
+        code,patch=self.run_case(error=True); self.assertNotEqual(code,0); self.assertEqual(patch,'')

tests/validate.py

@@ -0,0 +1,19 @@
+import json,sys,yaml
+from pathlib import Path
+resources=list(yaml.safe_load_all(Path(sys.argv[1]).read_text()))
+assert len({(d['kind'],d['metadata'].get('namespace'),d['metadata']['name']) for d in resources})==len(resources)
+d=next(d for d in resources if d['kind']=='Deployment')
+assert d['spec']['strategy']['type']=='Recreate'
+assert d['spec']['replicas']==1
+spec=d['spec']['template']['spec']
+assert not spec['automountServiceAccountToken']
+for c in spec['containers']:
+    assert all(p in c for p in ('startupProbe','readinessProbe','livenessProbe','resources')), c['name']
+    assert ':' in c['image'] and not c['image'].endswith(':latest')
+assert spec['affinity']['nodeAffinity']['requiredDuringSchedulingIgnoredDuringExecution']
+for service in (d for d in resources if d['kind']=='Service'):
+    assert all(p['port']!=27015 for p in service['spec']['ports'])
+for cm in (d for d in resources if d['kind']=='ConfigMap'):
+    for name,value in cm.get('data',{}).items():
+        if name.endswith('.json'): json.loads(value)
+print(f'Validated {len(resources)} rendered resources')