#!/usr/bin/env -S uv run --script --quiet # /// script # requires-python = ">=3.12" # dependencies = ["pyyaml"] # /// import argparse import json import os import selectors import subprocess import tempfile import time from pathlib import Path from uuid import uuid4 import yaml def main(): parser = argparse.ArgumentParser( description="Isolated, repeated Postgres outages against a published server image" ) parser.add_argument("--kubeconfig", required=True, type=Path) parser.add_argument("--namespace", default="prefect") parser.add_argument("--image", required=True) args = parser.parse_args() scripts = Path(__file__).resolve().parent pod = f"prefect-reconnect-{uuid4().hex[:8]}" kubectl = [ "kubectl", "--kubeconfig", str(args.kubeconfig.resolve()), "-n", args.namespace, ] def run(*command, **kwargs): return subprocess.run( command, check=True, text=True, capture_output=True, timeout=180, **kwargs ).stdout.strip() def execute(container, *command): return run(*kubectl, "exec", pod, "-c", container, "--", *command) def sql(statement): return execute( "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres", "-Atc", statement, ) def sessions(): return set( sql( "SELECT pid FROM pg_stat_activity WHERE datname='reconnect';" ).splitlines() ) def snapshot(): data = json.loads(run(*kubectl, "get", "pod", pod, "-o", "json")) return { c["name"]: (c["containerID"], c["restartCount"], c["imageID"]) for c in data["status"]["containerStatuses"] } def api(path, data): return json.loads( execute( "server", "curl", "-fsS", "--max-time", "15", "-X", "POST", f"http://127.0.0.1:4200/api/{path}", "-H", "Content-Type: application/json", "-d", json.dumps(data), ) ) manifest = yaml.safe_load((scripts / "chaos/postgres-reconnect.yaml").read_text()) manifest["metadata"].update(name=pod, namespace=args.namespace) manifest["spec"]["containers"][1]["image"] = args.image forward = None created = False try: run(*kubectl, "create", "-f", "-", input=json.dumps(manifest)) created = True run(*kubectl, "wait", "--for=condition=Ready", f"pod/{pod}", "--timeout=120s") before = snapshot() print(f"Image: {before['server'][2]}", flush=True) marker = api("flows/", {"name": f"reconnect-marker-{uuid4().hex}"}) baseline = sessions() assert len(baseline) >= 2, baseline for cycle in (1, 2): old = sessions() sql("ALTER DATABASE reconnect ALLOW_CONNECTIONS false;") try: killed = sql( "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='reconnect';" ) assert killed.splitlines() == ["t"] * len(old), killed responses = execute( "server", "sh", "-c", 'for i in 1 2 3 4 5 6 7; do curl -sS --max-time 12 -o /dev/null -w "%{http_code}\\n" -X POST http://127.0.0.1:4200/api/flows/count -H "Content-Type: application/json" -d "{}" & done; wait', ) assert len(responses.splitlines()) == 7, responses assert all( code in {"500", "503", "000"} for code in responses.splitlines() ), responses assert not sessions(), "database accepted connections during outage" finally: sql("ALTER DATABASE reconnect ALLOW_CONNECTIONS true;") started = time.monotonic() while True: restored = sessions() if len(restored) == len(baseline): break assert time.monotonic() - started < 30, ( f"pool not restored: {restored} (expected {len(baseline)})" ) time.sleep(0.25) assert not restored & old, "old database sessions survived" assert api("flows/count", {}) == 1 assert api("flows/filter", {})[0]["id"] == marker["id"] assert snapshot() == before, "container restarted during recovery" print( f"Cycle {cycle}: {len(old)} sessions terminated; all {len(restored)} restored in {time.monotonic() - started:.2f}s; data intact; no restarts", flush=True, ) forward = subprocess.Popen( [*kubectl, "port-forward", f"pod/{pod}", ":4200", "--address=127.0.0.1"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) with selectors.DefaultSelector() as selector: selector.register(forward.stdout, selectors.EVENT_READ) assert selector.select(timeout=30), "port-forward did not start" line = forward.stdout.readline() assert line.startswith("Forwarding from 127.0.0.1:"), line port = line.split(":")[1].split()[0] with tempfile.TemporaryDirectory(prefix="prefect-chaos-client-") as client_home: env = { key: value for key, value in os.environ.items() if not key.startswith("PREFECT_") } env.update( PREFECT_API_URL=f"http://127.0.0.1:{port}/api", PREFECT_HOME=client_home ) result = subprocess.run( [str(scripts / "test-worker-execution")], env=env, text=True, capture_output=True, timeout=180, check=False, ) if result.returncode: raise RuntimeError(result.stdout + result.stderr) print(result.stdout, flush=True) assert snapshot() == before, "container restarted during worker test" print( "PASS: repeated connection loss, full pool recovery, persisted data, and real worker execution", flush=True, ) except BaseException: if created: print(run(*kubectl, "logs", pod, "-c", "server", "--tail=60"), flush=True) raise finally: if forward: forward.terminate() forward.wait(timeout=10) if created: run(*kubectl, "delete", "pod", pod, "--wait=true", "--timeout=60s") if __name__ == "__main__": main()