Something went wrong. Try again.
prefect server in zig
Something went wrong. Try again.
6.9 kB · 194 lines
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195#!/usr/bin/env -S uv run --script --quiet# /// script# requires-python = ">=3.12"# dependencies = ["pyyaml"]# ///import argparseimport jsonimport osimport selectorsimport subprocessimport tempfileimport timefrom pathlib import Pathfrom uuid import uuid4
import yaml
def main(): parser = argparse.ArgumentParser( description="Isolated, repeated Postgres outages against a published server image" ) parser.add_argument("--kubeconfig", required=True, type=Path) parser.add_argument("--namespace", default="prefect") parser.add_argument("--image", required=True) args = parser.parse_args() scripts = Path(__file__).resolve().parent pod = f"prefect-reconnect-{uuid4().hex[:8]}" kubectl = [ "kubectl", "--kubeconfig", str(args.kubeconfig.resolve()), "-n", args.namespace, ]
def run(*command, **kwargs): return subprocess.run( command, check=True, text=True, capture_output=True, timeout=180, **kwargs ).stdout.strip()
def execute(container, *command): return run(*kubectl, "exec", pod, "-c", container, "--", *command)
def sql(statement): return execute( "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "postgres", "-Atc", statement, )
def sessions(): return set( sql( "SELECT pid FROM pg_stat_activity WHERE datname='reconnect';" ).splitlines() )
def snapshot(): data = json.loads(run(*kubectl, "get", "pod", pod, "-o", "json")) return { c["name"]: (c["containerID"], c["restartCount"], c["imageID"]) for c in data["status"]["containerStatuses"] }
def api(path, data): return json.loads( execute( "server", "curl", "-fsS", "--max-time", "15", "-X", "POST", f"http://127.0.0.1:4200/api/{path}", "-H", "Content-Type: application/json", "-d", json.dumps(data), ) )
manifest = yaml.safe_load((scripts / "chaos/postgres-reconnect.yaml").read_text()) manifest["metadata"].update(name=pod, namespace=args.namespace) manifest["spec"]["containers"][1]["image"] = args.image forward = None created = False try: run(*kubectl, "create", "-f", "-", input=json.dumps(manifest)) created = True run(*kubectl, "wait", "--for=condition=Ready", f"pod/{pod}", "--timeout=120s") before = snapshot() print(f"Image: {before['server'][2]}", flush=True) marker = api("flows/", {"name": f"reconnect-marker-{uuid4().hex}"}) baseline = sessions() assert len(baseline) >= 2, baseline for cycle in (1, 2): old = sessions() sql("ALTER DATABASE reconnect ALLOW_CONNECTIONS false;") try: killed = sql( "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='reconnect';" ) assert killed.splitlines() == ["t"] * len(old), killed responses = execute( "server", "sh", "-c", 'for i in 1 2 3 4 5 6 7; do curl -sS --max-time 12 -o /dev/null -w "%{http_code}\\n" -X POST http://127.0.0.1:4200/api/flows/count -H "Content-Type: application/json" -d "{}" & done; wait', ) assert len(responses.splitlines()) == 7, responses assert all( code in {"500", "503", "000"} for code in responses.splitlines() ), responses assert not sessions(), "database accepted connections during outage" finally: sql("ALTER DATABASE reconnect ALLOW_CONNECTIONS true;") started = time.monotonic() while True: restored = sessions() if len(restored) == len(baseline): break assert time.monotonic() - started < 30, ( f"pool not restored: {restored} (expected {len(baseline)})" ) time.sleep(0.25) assert not restored & old, "old database sessions survived" assert api("flows/count", {}) == 1 assert api("flows/filter", {})[0]["id"] == marker["id"] assert snapshot() == before, "container restarted during recovery" print( f"Cycle {cycle}: {len(old)} sessions terminated; all {len(restored)} restored in {time.monotonic() - started:.2f}s; data intact; no restarts", flush=True, )
forward = subprocess.Popen( [*kubectl, "port-forward", f"pod/{pod}", ":4200", "--address=127.0.0.1"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) with selectors.DefaultSelector() as selector: selector.register(forward.stdout, selectors.EVENT_READ) assert selector.select(timeout=30), "port-forward did not start" line = forward.stdout.readline() assert line.startswith("Forwarding from 127.0.0.1:"), line port = line.split(":")[1].split()[0] with tempfile.TemporaryDirectory(prefix="prefect-chaos-client-") as client_home: env = { key: value for key, value in os.environ.items() if not key.startswith("PREFECT_") } env.update( PREFECT_API_URL=f"http://127.0.0.1:{port}/api", PREFECT_HOME=client_home ) result = subprocess.run( [str(scripts / "test-worker-execution")], env=env, text=True, capture_output=True, timeout=180, check=False, ) if result.returncode: raise RuntimeError(result.stdout + result.stderr) print(result.stdout, flush=True) assert snapshot() == before, "container restarted during worker test" print( "PASS: repeated connection loss, full pool recovery, persisted data, and real worker execution", flush=True, ) except BaseException: if created: print(run(*kubectl, "logs", pod, "-c", "server", "--tail=60"), flush=True) raise finally: if forward: forward.terminate() forward.wait(timeout=10) if created: run(*kubectl, "delete", "pod", pod, "--wait=true", "--timeout=60s")
if __name__ == "__main__": main()