Something went wrong. Try again.
prefect server in zig
Something went wrong. Try again.
105 kB · 2,595 lines
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596#!/usr/bin/env -S uv run --script --quiet# /// script# requires-python = ">=3.12"# dependencies = ["httpx", "rich", "websockets"]# ///"""Functional test suite for prefect-server API.
Tests API correctness by exercising all endpoints with expected request/response patterns.Includes scheduler integration tests (which have intentional delays to verify background services).
For performance benchmarking, use ./scripts/benchmark instead.
Usage: ./scripts/test-api-sequence # human-readable output ./scripts/test-api-sequence --json # machine-readable for CI ./scripts/test-api-sequence --quiet # minimal output"""
import json as json_libimport osimport sysimport timeimport uuidfrom dataclasses import dataclass, fieldfrom typing import Callable
import httpxfrom rich.console import Consolefrom rich.panel import Panelfrom rich.table import Table
console = Console()BASE_URL = os.environ.get("PREFECT_API_URL", "http://localhost:4200/api")QUIET = "--json" in sys.argv or "--quiet" in sys.argv
@dataclassclass TestResult: name: str passed: bool duration_ms: float requests: int = 0 error: str | None = None
class CountingClient(httpx.Client): """HTTP client that counts requests."""
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.request_count = 0
def request(self, *args, **kwargs): self.request_count += 1 return super().request(*args, **kwargs)
def validate_response(data: dict, required_fields: list[str], field_types: dict | None = None) -> bool: """Validate response contains required fields with expected types.""" for field in required_fields: if field not in data: if not QUIET: console.print(f"[red]VALIDATION[/red]: missing field '{field}'") return False if field_types: for field, expected_type in field_types.items(): if field in data and data[field] is not None: if not isinstance(data[field], expected_type): if not QUIET: console.print(f"[red]VALIDATION[/red]: field '{field}' expected {expected_type.__name__}, got {type(data[field]).__name__}") return False return True
def run_test(name: str, test_fn: Callable[[CountingClient], bool]) -> TestResult: """Run a test function with timing and request counting.""" if not QUIET: console.print(Panel(f"testing {name}", style="blue"))
client = CountingClient(base_url=BASE_URL, timeout=10) start = time.perf_counter()
try: passed = test_fn(client) duration_ms = (time.perf_counter() - start) * 1000 return TestResult( name=name, passed=passed, duration_ms=duration_ms, requests=client.request_count, ) except Exception as e: duration_ms = (time.perf_counter() - start) * 1000 return TestResult( name=name, passed=False, duration_ms=duration_ms, requests=client.request_count, error=str(e), ) finally: client.close()
# ---------- test functions ----------
def test_admin(client: CountingClient) -> bool: """Test admin/health endpoints.""" # health if not QUIET: console.print("[bold]GET /health[/bold]") resp = client.get("/health") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False if not QUIET: console.print(f" {resp.text}")
# version if not QUIET: console.print("[bold]GET /admin/version[/bold]") resp = client.get("/admin/version") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False if not QUIET: console.print(f" {resp.json()}")
# csrf-token if not QUIET: console.print("[bold]GET /csrf-token[/bold]") resp = client.get("/csrf-token", params={"client": "test-client"}) if resp.status_code == 200: if not QUIET: console.print(f" token received") elif resp.status_code == 422: if not QUIET: console.print(f" csrf protection disabled (ok)") else: if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False
# hello — python returns "👋" resp = client.get("/hello") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: /hello {resp.status_code}") return False
# ready — DB connectivity check; 200 {"message":"OK"} when up resp = client.get("/ready") if resp.status_code != 200 or resp.json().get("message") != "OK": if not QUIET: console.print(f"[red]FAIL[/red]: /ready {resp.status_code} {resp.text!r}") return False
# admin/settings — minimum shape: {"server": {"analytics_enabled": bool}} resp = client.get("/admin/settings") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: /admin/settings {resp.status_code}") return False settings_body = resp.json() if not isinstance(settings_body.get("server"), dict): if not QUIET: console.print(f"[red]FAIL[/red]: /admin/settings missing 'server' object: {settings_body!r}") return False
return True
def test_flow_run(client: CountingClient, should_fail: bool = False) -> bool: """Test flow run lifecycle.""" suffix = "fail" if should_fail else "success" if not QUIET: console.print(f"server: {BASE_URL}\n")
# create flow if not QUIET: console.print("[bold]1. POST /flows/[/bold]") resp = client.post("/flows/", json={"name": f"bench-flow-{suffix}"}) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code} {resp.text}") return False flow = resp.json() if not validate_response(flow, ["id", "name", "created", "updated"], {"id": str, "name": str}): return False if not QUIET: console.print(f" flow_id: {flow.get('id')}")
# create flow run if not QUIET: console.print("\n[bold]2. POST /flow_runs/[/bold]") resp = client.post("/flow_runs/", json={ "flow_id": flow["id"], "name": f"run-{uuid.uuid4().hex[:8]}", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code} {resp.text}") return False flow_run = resp.json() if not validate_response(flow_run, ["id", "name", "flow_id", "state_type"], {"id": str, "name": str}): return False flow_run_id = flow_run.get("id") if not QUIET: console.print(f" flow_run_id: {flow_run_id}")
# read flow run if not QUIET: console.print("\n[bold]3. GET /flow_runs/{id}[/bold]") resp = client.get(f"/flow_runs/{flow_run_id}") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False if not QUIET: console.print(f" state: {resp.json().get('state_type')}")
# set RUNNING if not QUIET: console.print("\n[bold]4. POST /flow_runs/{id}/set_state (RUNNING)[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": False, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False if not QUIET: console.print(f" status: {resp.json().get('status')}")
# set final state final_type = "FAILED" if should_fail else "COMPLETED" final_name = "Failed" if should_fail else "Completed" if not QUIET: console.print(f"\n[bold]5. POST /flow_runs/{{id}}/set_state ({final_type})[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": final_type, "name": final_name}, "force": False, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False if not QUIET: console.print(f" status: {resp.json().get('status')}")
# verify if not QUIET: console.print("\n[bold]6. GET /flow_runs/{id} (verify)[/bold]") resp = client.get(f"/flow_runs/{flow_run_id}") if resp.status_code != 200: return False actual_type = resp.json().get("state_type") if actual_type != final_type: if not QUIET: console.print(f"[red]FAIL[/red]: expected {final_type}, got {actual_type}") return False if not QUIET: console.print(f" [green]state: {actual_type} (correct)[/green]")
return True
def test_flow_with_task_runs(client: CountingClient) -> bool: """Test flow execution with nested task runs - simulates real flow execution."""
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup: create flow log("[bold]setup: create flow[/bold]") resp = client.post("/flows/", json={"name": f"flow-with-tasks-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id") log(f" flow_id: {flow_id}")
# create flow run log("[bold]1. create flow run[/bold]") resp = client.post("/flow_runs/", json={ "flow_id": flow_id, "name": f"run-with-tasks-{uuid.uuid4().hex[:8]}", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run_id = resp.json().get("id") log(f" flow_run_id: {flow_run_id}")
# start flow run log("[bold]2. start flow run (RUNNING)[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) if resp.status_code not in (200, 201) or resp.json().get("status") != "ACCEPT": return fail(f"start flow run failed") log(f" flow run started")
# create and execute task 1 log("[bold]3. execute task 1: extract[/bold]") resp = client.post("/task_runs/", json={ "flow_run_id": flow_run_id, "task_key": "extract_data", "dynamic_key": "extract-0", "name": "extract_data-0", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"create task 1 {resp.status_code}") task1_id = resp.json().get("id")
# run task 1 resp = client.post(f"/task_runs/{task1_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) if resp.status_code not in (200, 201): return fail(f"task 1 RUNNING {resp.status_code}")
resp = client.post(f"/task_runs/{task1_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.status_code not in (200, 201): return fail(f"task 1 COMPLETED {resp.status_code}") log(f" task 1 completed")
# create and execute task 2 (depends on task 1) log("[bold]4. execute task 2: transform[/bold]") resp = client.post("/task_runs/", json={ "flow_run_id": flow_run_id, "task_key": "transform_data", "dynamic_key": "transform-0", "name": "transform_data-0", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"create task 2 {resp.status_code}") task2_id = resp.json().get("id")
resp = client.post(f"/task_runs/{task2_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) resp = client.post(f"/task_runs/{task2_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.status_code not in (200, 201): return fail(f"task 2 COMPLETED {resp.status_code}") log(f" task 2 completed")
# create and execute task 3 log("[bold]5. execute task 3: load[/bold]") resp = client.post("/task_runs/", json={ "flow_run_id": flow_run_id, "task_key": "load_data", "dynamic_key": "load-0", "name": "load_data-0", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"create task 3 {resp.status_code}") task3_id = resp.json().get("id")
resp = client.post(f"/task_runs/{task3_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) resp = client.post(f"/task_runs/{task3_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.status_code not in (200, 201): return fail(f"task 3 COMPLETED {resp.status_code}") log(f" task 3 completed")
# complete flow run log("[bold]6. complete flow run[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.status_code not in (200, 201) or resp.json().get("status") != "ACCEPT": return fail(f"complete flow run failed") log(f" [green]flow run completed[/green]")
# verify task runs can be read individually and are completed log("[bold]7. verify task runs completed[/bold]") for task_id, task_name in [(task1_id, "extract"), (task2_id, "transform"), (task3_id, "load")]: resp = client.get(f"/task_runs/{task_id}") if resp.status_code != 200: return fail(f"read task {task_name} {resp.status_code}") task_run = resp.json() if task_run.get("flow_run_id") != flow_run_id: return fail(f"task {task_name} not linked to flow run") if task_run.get("state_type") != "COMPLETED": return fail(f"task {task_name} not COMPLETED") log(f" [green]all 3 tasks completed and linked to flow run[/green]")
return True
def test_task_run(client: CountingClient) -> bool: """Test task run lifecycle.""" # create if not QUIET: console.print("[bold]POST /task_runs/[/bold]") resp = client.post("/task_runs/", json={ "task_key": "bench-task", "dynamic_key": f"key-{uuid.uuid4().hex[:8]}", "name": f"task-{uuid.uuid4().hex[:8]}", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False task_run_id = resp.json().get("id") if not QUIET: console.print(f" task_run_id: {task_run_id}")
# read resp = client.get(f"/task_runs/{task_run_id}") if resp.status_code != 200: return False
# RUNNING resp = client.post(f"/task_runs/{task_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": False, }) if resp.status_code not in (200, 201): return False if not QUIET: console.print(f" -> RUNNING: {resp.json().get('status')}")
# COMPLETED resp = client.post(f"/task_runs/{task_run_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, "force": False, }) if resp.status_code not in (200, 201): return False if not QUIET: console.print(f" -> COMPLETED: {resp.json().get('status')}")
return True
def test_filters(client: CountingClient) -> bool: """Test filter endpoints.""" for endpoint, label in [ ("/flows/filter", "flows"), ("/flow_runs/filter", "flow_runs"), ("/task_runs/filter", "task_runs"), ]: resp = client.post(endpoint, json={}) if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: {endpoint} {resp.status_code}") return False if not QUIET: console.print(f" {label}: {len(resp.json())} items")
return True
def test_logs(client: CountingClient) -> bool: """Test logs endpoint.""" from datetime import datetime, timezone
logs = [ {"level": 20, "message": "test log 1", "name": "test", "timestamp": datetime.now(timezone.utc).isoformat()}, {"level": 30, "message": "test log 2", "name": "test", "timestamp": datetime.now(timezone.utc).isoformat()}, ] resp = client.post("/logs/", json=logs) if resp.status_code not in (200, 201, 204): if not QUIET: console.print(f"[red]FAIL[/red]: {resp.status_code}") return False if not QUIET: console.print(f" {len(logs)} logs sent") return True
def test_variables(client: CountingClient) -> bool: """Test variables API (CRUD).""" var_name = f"bench-var-{uuid.uuid4().hex[:8]}"
# create if not QUIET: console.print("[bold]POST /variables/[/bold]") resp = client.post("/variables/", json={ "name": var_name, "value": {"nested": "object", "count": 42}, "tags": ["benchmark", "test"], }) if resp.status_code != 201: if not QUIET: console.print(f"[red]FAIL[/red]: create {resp.status_code}") return False variable = resp.json() if not validate_response(variable, ["id", "name", "value", "tags", "created", "updated"], {"id": str, "name": str, "tags": list}): return False var_id = variable.get("id") if not QUIET: console.print(f" created: {var_id}")
# get by name resp = client.get(f"/variables/name/{var_name}") if resp.status_code != 200: return False if not QUIET: console.print(f" get by name: ok")
# get by id resp = client.get(f"/variables/{var_id}") if resp.status_code != 200: return False
# update by name resp = client.patch(f"/variables/name/{var_name}", json={"value": "updated"}) if resp.status_code != 204: return False if not QUIET: console.print(f" updated by name")
# filter resp = client.post("/variables/filter", json={"limit": 10}) if resp.status_code != 200: return False if not QUIET: console.print(f" filter: {len(resp.json())} items")
# count resp = client.post("/variables/count", json={}) if resp.status_code != 200: return False if not QUIET: console.print(f" count: {resp.text}")
# duplicate name should fail resp = client.post("/variables/", json={"name": var_name, "value": "dupe"}) if resp.status_code != 409: if not QUIET: console.print(f"[red]FAIL[/red]: duplicate should return 409, got {resp.status_code}") return False if not QUIET: console.print(f" duplicate rejected: 409")
# delete resp = client.delete(f"/variables/name/{var_name}") if resp.status_code != 204: return False if not QUIET: console.print(f" deleted")
return True
def test_artifacts(client: CountingClient) -> bool: """Test artifacts API (CRUD + filter + latest-by-key).""" key1 = f"art-{uuid.uuid4().hex[:8]}" key2 = f"art-{uuid.uuid4().hex[:8]}"
# 1. create with key if not QUIET: console.print("[bold]POST /artifacts/ (with key)[/bold]") resp = client.post("/artifacts/", json={ "key": key1, "type": "markdown", "data": {"body": "first version"}, "description": "v1", "metadata_": {"source": "smoke"}, }) if resp.status_code != 201: if not QUIET: console.print(f"[red]FAIL[/red]: create artifact {resp.status_code}: {resp.text}") return False artifact = resp.json() if not validate_response(artifact, ["id", "key", "type", "data", "created"], {"id": str, "key": str}): return False art_id = artifact["id"] if not QUIET: console.print(f" created: {art_id}")
# 2. get by id resp = client.get(f"/artifacts/{art_id}") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: get by id {resp.status_code}") return False
# 3. get latest by key resp = client.get(f"/artifacts/{key1}/latest") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: get latest {resp.status_code}: {resp.text}") return False latest = resp.json() if latest.get("latest_id") != art_id: if not QUIET: console.print(f"[red]FAIL[/red]: latest_id mismatch: got {latest.get('latest_id')} want {art_id}") return False if not QUIET: console.print(f" latest_id = {latest['latest_id']}")
# 4. create a SECOND artifact with the SAME key — collection latest should advance resp = client.post("/artifacts/", json={ "key": key1, "type": "markdown", "data": {"body": "second version"}, }) if resp.status_code != 201: return False art_id_v2 = resp.json()["id"] resp = client.get(f"/artifacts/{key1}/latest") if resp.status_code != 200 or resp.json().get("latest_id") != art_id_v2: if not QUIET: console.print(f"[red]FAIL[/red]: latest didn't advance to v2") return False if not QUIET: console.print(f" latest advanced to v2: {art_id_v2}")
# 5. create a keyless artifact — should NOT appear in artifact_collection resp = client.post("/artifacts/", json={ "type": "result", "data": {"value": 123}, }) if resp.status_code != 201: return False keyless_id = resp.json()["id"] if not QUIET: console.print(f" keyless: {keyless_id}")
# 6. update by id resp = client.patch(f"/artifacts/{art_id}", json={"description": "v1 updated"}) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: patch {resp.status_code}") return False
# 7. filter — at least 3 artifacts exist for this run resp = client.post("/artifacts/filter", json={"limit": 100}) if resp.status_code != 200: return False arts = resp.json() if len(arts) < 3: if not QUIET: console.print(f"[red]FAIL[/red]: filter returned {len(arts)} (want >= 3)") return False
# 8. filter by key.any_ resp = client.post("/artifacts/filter", json={ "limit": 10, "artifacts": {"key": {"any_": [key1]}}, }) if resp.status_code != 200: return False by_key = resp.json() if len(by_key) != 2: if not QUIET: console.print(f"[red]FAIL[/red]: key.any_ filter returned {len(by_key)} (want 2)") return False
# 9. filter by key.exists_ = False (keyless) resp = client.post("/artifacts/filter", json={ "limit": 100, "artifacts": {"key": {"exists_": False}}, }) if resp.status_code != 200: return False keyless = resp.json() if not any(a["id"] == keyless_id for a in keyless): if not QUIET: console.print(f"[red]FAIL[/red]: keyless artifact not found via exists_=False") return False
# 10. latest filter resp = client.post("/artifacts/latest/filter", json={"limit": 100}) if resp.status_code != 200: return False latest_list = resp.json() if not any(c["key"] == key1 for c in latest_list): if not QUIET: console.print(f"[red]FAIL[/red]: key {key1} not in latest filter") return False
# 11. count resp = client.post("/artifacts/count", json={}) if resp.status_code != 200: return False total_count = int(resp.text) if total_count < 3: if not QUIET: console.print(f"[red]FAIL[/red]: count {total_count} too low") return False
# 12. count latest resp = client.post("/artifacts/latest/count", json={}) if resp.status_code != 200: return False if not QUIET: console.print(f" counts: total={total_count} latest={resp.text}")
# 13. delete keyless resp = client.delete(f"/artifacts/{keyless_id}") if resp.status_code != 204: return False
# 14. 404 after delete resp = client.get(f"/artifacts/{keyless_id}") if resp.status_code != 404: if not QUIET: console.print(f"[red]FAIL[/red]: deleted artifact still readable") return False
# 15. latest by unknown key → 404 resp = client.get(f"/artifacts/{key2}/latest") if resp.status_code != 404: if not QUIET: console.print(f"[red]FAIL[/red]: unknown key should 404, got {resp.status_code}") return False
if not QUIET: console.print(f" all artifact endpoints exercised") return True
def test_blocks(client: CountingClient) -> bool: """Test blocks API (types, schemas, documents).""" slug = f"bench-block-{uuid.uuid4().hex[:8]}"
# create block type if not QUIET: console.print("[bold]block_types[/bold]") resp = client.post("/block_types/", json={ "name": f"Bench Block {slug}", "slug": slug, "description": "benchmark block type", }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create block_type {resp.status_code}") return False block_type = resp.json() if not validate_response(block_type, ["id", "name", "slug"], {"id": str, "name": str, "slug": str}): return False block_type_id = block_type.get("id") if not QUIET: console.print(f" created: {block_type_id}")
# get by slug resp = client.get(f"/block_types/slug/{slug}") if resp.status_code != 200: return False
# create schema if not QUIET: console.print("[bold]block_schemas[/bold]") resp = client.post("/block_schemas/", json={ "block_type_id": block_type_id, "fields": {"properties": {"value": {"type": "string"}}}, "capabilities": ["test"], "version": "1.0.0", }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create block_schema {resp.status_code}") return False block_schema = resp.json() block_schema_id = block_schema.get("id") checksum = block_schema.get("checksum") if not QUIET: console.print(f" created: {block_schema_id}")
# get by checksum resp = client.get(f"/block_schemas/checksum/{checksum}") if resp.status_code != 200: return False
# create document if not QUIET: console.print("[bold]block_documents[/bold]") doc_name = f"bench-doc-{uuid.uuid4().hex[:8]}" resp = client.post("/block_documents/", json={ "name": doc_name, "block_type_id": block_type_id, "block_schema_id": block_schema_id, "data": {"value": "secret-value"}, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create block_document {resp.status_code}") return False block_doc = resp.json() block_doc_id = block_doc.get("id") if not QUIET: console.print(f" created: {block_doc_id}")
# get by id resp = client.get(f"/block_documents/{block_doc_id}") if resp.status_code != 200: return False
# get by slug/name resp = client.get(f"/block_types/slug/{slug}/block_documents/name/{doc_name}") if resp.status_code != 200: return False
# update resp = client.patch(f"/block_documents/{block_doc_id}", json={"data": {"value": "updated"}}) if resp.status_code != 204: return False if not QUIET: console.print(f" updated")
# filters for endpoint in ["/block_types/filter", "/block_schemas/filter", "/block_documents/filter"]: resp = client.post(endpoint, json={}) if resp.status_code != 200: return False
# delete resp = client.delete(f"/block_documents/{block_doc_id}") if resp.status_code != 204: return False if not QUIET: console.print(f" deleted")
return True
def test_work_pools(client: CountingClient) -> bool: """Test work pools API (pools, queues, workers).""" pool_name = f"test-pool-{uuid.uuid4().hex[:8]}"
# create work pool if not QUIET: console.print("[bold]work_pools[/bold]") resp = client.post("/work_pools/", json={ "name": pool_name, "type": "process", "description": "test work pool", }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create work_pool {resp.status_code} {resp.text}") return False pool = resp.json() if not validate_response(pool, ["id", "name", "type", "status", "default_queue_id"], {"id": str, "name": str}): return False if not QUIET: console.print(f" created: {pool.get('id')}")
# check default queue was created if not pool.get("default_queue_id"): if not QUIET: console.print("[red]FAIL[/red]: no default_queue_id") return False
# get by name resp = client.get(f"/work_pools/{pool_name}") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: get work_pool {resp.status_code}") return False
# update resp = client.patch(f"/work_pools/{pool_name}", json={"description": "updated"}) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: update work_pool {resp.status_code}") return False if not QUIET: console.print(" updated")
# filter resp = client.post("/work_pools/filter", json={}) if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: filter work_pools {resp.status_code}") return False pools = resp.json() if not isinstance(pools, list): return False
# create queue if not QUIET: console.print("[bold]work_queues[/bold]") queue_name = f"test-queue-{uuid.uuid4().hex[:8]}" resp = client.post(f"/work_pools/{pool_name}/queues/", json={ "name": queue_name, "description": "test queue", "priority": 5, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create work_queue {resp.status_code} {resp.text}") return False queue = resp.json() if not validate_response(queue, ["id", "name", "priority", "work_pool_id"], {"id": str, "priority": int}): return False if not QUIET: console.print(f" created: {queue.get('id')}")
# get queue resp = client.get(f"/work_pools/{pool_name}/queues/{queue_name}") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: get work_queue {resp.status_code}") return False
# filter queues resp = client.post(f"/work_pools/{pool_name}/queues/filter", json={}) if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: filter work_queues {resp.status_code}") return False queues = resp.json() if not isinstance(queues, list) or len(queues) < 2: # default + our queue if not QUIET: console.print(f"[red]FAIL[/red]: expected at least 2 queues, got {len(queues) if isinstance(queues, list) else 'not a list'}") return False
# worker heartbeat if not QUIET: console.print("[bold]workers[/bold]") resp = client.post(f"/work_pools/{pool_name}/workers/heartbeat", json={ "name": "test-worker-1", "heartbeat_interval_seconds": 30, }) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: worker heartbeat {resp.status_code} {resp.text}") return False if not QUIET: console.print(" heartbeat sent")
# check pool status is now READY resp = client.get(f"/work_pools/{pool_name}") if resp.status_code != 200: return False pool = resp.json() if pool.get("status") != "READY": if not QUIET: console.print(f"[red]FAIL[/red]: expected pool status READY, got {pool.get('status')}") return False if not QUIET: console.print(" pool status: READY")
# filter workers resp = client.post(f"/work_pools/{pool_name}/workers/filter", json={}) if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: filter workers {resp.status_code}") return False workers = resp.json() if not isinstance(workers, list) or len(workers) < 1: return False
# delete queue (not default) resp = client.delete(f"/work_pools/{pool_name}/queues/{queue_name}") if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: delete work_queue {resp.status_code}") return False if not QUIET: console.print(" deleted queue")
# delete pool resp = client.delete(f"/work_pools/{pool_name}") if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: delete work_pool {resp.status_code}") return False if not QUIET: console.print(" deleted pool")
return True
def test_deployments(client: CountingClient) -> bool: """Test deployments API (deployments, schedules, create_flow_run).""" # create a flow first if not QUIET: console.print("[bold]setup: create flow[/bold]") resp = client.post("/flows/", json={"name": f"deploy-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create flow {resp.status_code}") return False flow = resp.json() flow_id = flow.get("id") flow_name = flow.get("name")
# create deployment deployment_name = f"test-deployment-{uuid.uuid4().hex[:8]}" if not QUIET: console.print("[bold]deployments[/bold]") resp = client.post("/deployments/", json={ "name": deployment_name, "flow_id": flow_id, "description": "test deployment", "tags": ["test", "benchmark"], "parameters": {"key": "value"}, "schedules": [ {"schedule": {"interval": 3600}, "active": True}, ], }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create deployment {resp.status_code} {resp.text}") return False deployment = resp.json() if not validate_response(deployment, ["id", "name", "flow_id", "status", "schedules"], {"id": str, "name": str}): return False deployment_id = deployment.get("id") if not QUIET: console.print(f" created: {deployment_id}")
# verify schedules were created schedules = deployment.get("schedules", []) if not isinstance(schedules, list) or len(schedules) != 1: if not QUIET: console.print(f"[red]FAIL[/red]: expected 1 schedule, got {len(schedules) if isinstance(schedules, list) else 'not a list'}") return False if not QUIET: console.print(f" schedules: {len(schedules)}")
# get by id resp = client.get(f"/deployments/{deployment_id}") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: get deployment {resp.status_code}") return False
# get by name resp = client.get(f"/deployments/name/{flow_name}/{deployment_name}") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: get deployment by name {resp.status_code}") return False if not QUIET: console.print(" get by name: ok")
# update resp = client.patch(f"/deployments/{deployment_id}", json={"description": "updated"}) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: update deployment {resp.status_code}") return False if not QUIET: console.print(" updated")
# filter resp = client.post("/deployments/filter", json={"limit": 10}) if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: filter deployments {resp.status_code}") return False if not QUIET: console.print(f" filter: {len(resp.json())} items")
# count resp = client.post("/deployments/count", json={}) if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: count deployments {resp.status_code}") return False if not QUIET: console.print(f" count: {resp.text}")
# pause resp = client.post(f"/deployments/{deployment_id}/pause_deployment", json={}) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: pause deployment {resp.status_code}") return False if not QUIET: console.print(" paused")
# resume resp = client.post(f"/deployments/{deployment_id}/resume_deployment", json={}) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: resume deployment {resp.status_code}") return False if not QUIET: console.print(" resumed")
# create flow run from deployment if not QUIET: console.print("[bold]create_flow_run[/bold]") resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={}) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create flow run {resp.status_code} {resp.text}") return False flow_run = resp.json() if not validate_response(flow_run, ["id", "flow_id", "deployment_id"], {"id": str}): return False if flow_run.get("deployment_id") != deployment_id: if not QUIET: console.print(f"[red]FAIL[/red]: deployment_id mismatch") return False if not QUIET: console.print(f" created flow run: {flow_run.get('id')}")
# schedules - list if not QUIET: console.print("[bold]deployment_schedules[/bold]") resp = client.get(f"/deployments/{deployment_id}/schedules") if resp.status_code != 200: if not QUIET: console.print(f"[red]FAIL[/red]: list schedules {resp.status_code}") return False schedules = resp.json() schedule_id = schedules[0].get("id") if schedules else None if not QUIET: console.print(f" list: {len(schedules)} schedules")
# schedules - create resp = client.post(f"/deployments/{deployment_id}/schedules", json={ "schedule": {"cron": "0 0 * * *"}, "active": False, }) if resp.status_code not in (200, 201): if not QUIET: console.print(f"[red]FAIL[/red]: create schedule {resp.status_code}") return False created_schedules = resp.json() if not isinstance(created_schedules, list) or len(created_schedules) != 1: if not QUIET: console.print(f"[red]FAIL[/red]: expected 1 created schedule") return False new_schedule_id = created_schedules[0].get("id") if not QUIET: console.print(f" created schedule: {new_schedule_id}")
# schedules - update resp = client.patch(f"/deployments/{deployment_id}/schedules/{new_schedule_id}", json={"active": True}) if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: update schedule {resp.status_code}") return False if not QUIET: console.print(" updated schedule")
# schedules - delete resp = client.delete(f"/deployments/{deployment_id}/schedules/{new_schedule_id}") if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: delete schedule {resp.status_code}") return False if not QUIET: console.print(" deleted schedule")
# delete deployment resp = client.delete(f"/deployments/{deployment_id}") if resp.status_code != 204: if not QUIET: console.print(f"[red]FAIL[/red]: delete deployment {resp.status_code}") return False if not QUIET: console.print(" deleted deployment")
return True
def test_scheduler_idempotency(client: CountingClient) -> bool: """Test that scheduler is idempotent - running twice doesn't create duplicates.""" import time as time_mod
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
def wait_for_runs(deployment_id: str, min_count: int, max_wait: float = 10.0) -> int: """Poll until we have at least min_count runs, or timeout (client-side filter).""" start = time_mod.time() while time_mod.time() - start < max_wait: resp = client.post("/flow_runs/filter", json={"limit": 100}) if resp.status_code == 200: all_runs = resp.json() matching = [r for r in all_runs if r.get("deployment_id") == deployment_id] if len(matching) >= min_count: return len(matching) time_mod.sleep(0.5) return 0
# setup: create flow, work pool, deployment with interval schedule log("[bold]setup[/bold]") resp = client.post("/flows/", json={"name": f"idem-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
pool_name = f"idem-pool-{uuid.uuid4().hex[:8]}" resp = client.post("/work_pools/", json={"name": pool_name, "type": "process"}) if resp.status_code not in (200, 201): return fail(f"create work_pool {resp.status_code}") log(f" pool: {pool_name}")
# create deployment with interval schedule (every hour) resp = client.post("/deployments/", json={ "name": f"idem-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "work_pool_name": pool_name, "schedules": [{"schedule": {"interval": 3600}, "active": True}], # every hour }) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code}") deployment = resp.json() deployment_id = deployment.get("id") log(f" deployment: {deployment_id}")
# poll for scheduler to create runs and wait for tick to complete # we need to wait for count to stabilize, not just see the first run log("[bold]waiting for scheduler tick to complete...[/bold]") count_after_first = wait_for_runs(deployment_id, min_count=1) # wait for count to stabilize (scheduler tick may still be inserting) for _ in range(10): time_mod.sleep(0.2) resp = client.post("/flow_runs/filter", json={"limit": 100}) if resp.status_code == 200: matching = [r for r in resp.json() if r.get("deployment_id") == deployment_id] if len(matching) == count_after_first: break # count stable count_after_first = len(matching) log(f" runs after first tick: {count_after_first}")
if count_after_first == 0: return fail("scheduler did not create any runs (timeout)")
# verify idempotency: poll for a few seconds to confirm count stays stable # if scheduler creates duplicates, count would increase during this window log("[bold]verifying idempotency (polling to confirm count stable)...[/bold]") stable_checks = 0 for _ in range(6): # 6 checks over ~3 seconds time_mod.sleep(0.5) resp = client.post("/flow_runs/filter", json={"limit": 100}) if resp.status_code != 200: continue all_runs = resp.json() matching = [r for r in all_runs if r.get("deployment_id") == deployment_id] current_count = len(matching) if current_count == count_after_first: stable_checks += 1 elif current_count > count_after_first: return fail(f"idempotency failed: {count_after_first} -> {current_count} runs")
log(f" stable count checks: {stable_checks}/6") log(f" [green]idempotency verified: count unchanged at {count_after_first}[/green]")
# cleanup client.delete(f"/deployments/{deployment_id}") client.delete(f"/work_pools/{pool_name}") log(" cleanup: ok")
return True
def test_parameter_merging(client: CountingClient) -> bool: """Test that schedule parameters override deployment parameters.""" import time as time_mod
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup log("[bold]setup[/bold]") resp = client.post("/flows/", json={"name": f"params-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
pool_name = f"params-pool-{uuid.uuid4().hex[:8]}" resp = client.post("/work_pools/", json={"name": pool_name, "type": "process"}) if resp.status_code not in (200, 201): return fail(f"create work_pool {resp.status_code}") log(f" pool: {pool_name}")
# create deployment with base parameters # schedule has override parameter resp = client.post("/deployments/", json={ "name": f"params-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "work_pool_name": pool_name, "parameters": {"base_key": "base_value", "override_key": "deployment_value"}, "schedules": [{ "schedule": {"interval": 3600}, "active": True, "parameters": {"override_key": "schedule_value", "schedule_key": "schedule_only"}, }], }) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code}") deployment = resp.json() deployment_id = deployment.get("id") log(f" deployment: {deployment_id}") log(f" deployment params: {deployment.get('parameters')}")
# poll for scheduler to create runs (filter client-side since server filter doesn't support deployment_id yet) log("[bold]polling for scheduler to create runs...[/bold]") runs = [] start = time_mod.time() max_wait = 10.0 while time_mod.time() - start < max_wait: resp = client.post("/flow_runs/filter", json={"limit": 100}) if resp.status_code == 200: all_runs = resp.json() runs = [r for r in all_runs if r.get("deployment_id") == deployment_id] if len(runs) > 0: break time_mod.sleep(0.5) log(f" found {len(runs)} runs for deployment in {time_mod.time() - start:.1f}s")
if len(runs) == 0: return fail("scheduler did not create any runs (timeout)")
# check merged parameters on first run run_params = runs[0].get("parameters", {}) if isinstance(run_params, str): import json as json_mod run_params = json_mod.loads(run_params) log(f" run params: {run_params}")
# verify merging: # - base_key should be from deployment # - override_key should be from schedule (override) # - schedule_key should be from schedule (new key) if run_params.get("base_key") != "base_value": return fail(f"base_key not preserved: {run_params.get('base_key')}") if run_params.get("override_key") != "schedule_value": return fail(f"override_key not overridden: {run_params.get('override_key')}") if run_params.get("schedule_key") != "schedule_only": return fail(f"schedule_key not added: {run_params.get('schedule_key')}")
log(" [green]parameter merging verified[/green]")
# cleanup client.delete(f"/deployments/{deployment_id}") client.delete(f"/work_pools/{pool_name}") log(" cleanup: ok")
return True
def test_parameter_schema_enforcement(client: CountingClient) -> bool: """Runs with schema-violating parameters are rejected at creation (parity with python)."""
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
log("[bold]setup[/bold]") resp = client.post("/flows/", json={"name": f"schema-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
schema = { "title": "Parameters", "type": "object", "properties": { "payload": {"$ref": "#/definitions/Payload", "position": 0, "title": "payload"}, "count": {"position": 1, "title": "count", "type": "integer"}, }, "required": ["payload", "count"], "definitions": { "Payload": { "title": "Payload", "type": "object", "properties": { "title": {"title": "Title", "type": "string"}, "body": {"title": "Body", "type": "string"}, }, "required": ["title", "body"], }, }, }
resp = client.post("/deployments/", json={ "name": f"schema-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "parameter_openapi_schema": schema, "enforce_parameter_schema": True, }) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code} {resp.text}") deployment_id = resp.json().get("id") log(f" deployment: {deployment_id}")
# missing required parameters -> rejected at creation, not deferred to the worker resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={}) if resp.status_code != 409: return fail(f"missing params: expected 409, got {resp.status_code} {resp.text}") if "is a required property" not in resp.json().get("detail", ""): return fail(f"missing params detail: {resp.text}") log(" missing required -> 409")
# nested required violation names the field resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": {"payload": {"title": "t"}, "count": 1}, }) if resp.status_code != 409: return fail(f"nested missing: expected 409, got {resp.status_code} {resp.text}") detail = resp.json().get("detail", "") if detail != "Error creating flow run: Validation failed for field 'payload'. Failure reason: 'body' is a required property": return fail(f"nested missing detail mismatch: {detail}") log(" nested missing required -> 409 with python-parity detail")
# type violation resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": {"payload": {"title": "t", "body": "b"}, "count": "three"}, }) if resp.status_code != 409 or "is not of type 'integer'" not in resp.json().get("detail", ""): return fail(f"type violation: {resp.status_code} {resp.text}") log(" type violation -> 409")
# valid parameters pass resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": {"payload": {"title": "t", "body": "b"}, "count": 3}, }) if resp.status_code not in (200, 201): return fail(f"valid params: expected 201, got {resp.status_code} {resp.text}") log(" valid params -> 201")
# per-run enforcement override skips validation resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "enforce_parameter_schema": False, }) if resp.status_code not in (200, 201): return fail(f"per-run override: expected 201, got {resp.status_code} {resp.text}") log(" per-run enforce=false -> 201")
# schema-less create gets python's permissive empty-object default, so # enforcement is a no-op there (parity with DeploymentCreate default_factory) resp = client.post("/deployments/", json={ "name": f"schema-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "enforce_parameter_schema": True, }) if resp.status_code not in (200, 201): return fail(f"create schemaless deployment {resp.status_code}") schemaless_id = resp.json().get("id") resp = client.post(f"/deployments/{schemaless_id}/create_flow_run", json={"parameters": {"anything": "goes"}}) if resp.status_code not in (200, 201): return fail(f"schemaless deployment: expected 201, got {resp.status_code} {resp.text}") log(" schema-less deployment defaults to permissive schema -> 201")
# enforcement defaulted off -> anything goes resp = client.post("/deployments/", json={ "name": f"schema-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "parameter_openapi_schema": schema, "enforce_parameter_schema": False, }) if resp.status_code not in (200, 201): return fail(f"create lax deployment {resp.status_code}") lax_id = resp.json().get("id") resp = client.post(f"/deployments/{lax_id}/create_flow_run", json={}) if resp.status_code not in (200, 201): return fail(f"lax deployment: expected 201, got {resp.status_code} {resp.text}") log(" enforce=false deployment -> 201")
log(" [green]parameter schema enforcement verified[/green]")
# cleanup for did in (deployment_id, schemaless_id, lax_id): client.delete(f"/deployments/{did}") log(" cleanup: ok")
return True
def test_parameter_hydration(client: CountingClient) -> bool: """__prefect_kind blobs hydrate before validation (parity with python)."""
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
log("[bold]setup[/bold]") resp = client.post("/flows/", json={"name": f"hydrate-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
schema = { "title": "Parameters", "type": "object", "properties": { "config": {"title": "config", "type": "object"}, "label": {"title": "label", "type": "string"}, }, "required": ["config", "label"], } resp = client.post("/deployments/", json={ "name": f"hydrate-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "parameter_openapi_schema": schema, "enforce_parameter_schema": True, }) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code} {resp.text}") deployment_id = resp.json().get("id")
# json kind hydrates into an object that then passes validation resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": { "config": {"__prefect_kind": "json", "value": "{\"retries\": 2}"}, "label": {"__prefect_kind": "none", "value": "ok"}, }, }) if resp.status_code not in (200, 201): return fail(f"json kind: expected 201, got {resp.status_code} {resp.text}") run = resp.json() if run.get("parameters") != {"config": {"retries": 2}, "label": "ok"}: return fail(f"json kind stored raw params: {run.get('parameters')}") log(" json + none kinds hydrate and validate -> 201")
# malformed json payload -> 400 hydration error, not a schema 409 resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": { "config": {"__prefect_kind": "json", "value": "{nope"}, "label": "x", }, }) if resp.status_code != 400 or "Error hydrating flow run parameters" not in resp.json().get("detail", ""): return fail(f"invalid json kind: {resp.status_code} {resp.text}") log(" invalid json kind -> 400 hydration error")
# hydrated-but-invalid still hits schema validation (string where object due) resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": { "config": {"__prefect_kind": "json", "value": "\"just a string\""}, "label": "x", }, }) if resp.status_code != 409: return fail(f"hydrated invalid: expected 409, got {resp.status_code} {resp.text}") log(" hydrated value validates against schema -> 409")
# jinja renders against an empty context (undefined -> "") resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": { "config": {}, "label": {"__prefect_kind": "jinja", "template": "run-{{ flow_run.name }}"}, }, }) if resp.status_code not in (200, 201): return fail(f"jinja kind: {resp.status_code} {resp.text}") if resp.json().get("parameters", {}).get("label") != "run-": return fail(f"jinja rendering: {resp.json().get('parameters')}") log(" jinja kind renders empty-context -> 201")
# workspace_variable resolves through the variables table var_name = f"hydrate-var-{uuid.uuid4().hex[:8]}" resp = client.post("/variables/", json={"name": var_name, "value": "from-variable"}) if resp.status_code not in (200, 201): return fail(f"create variable {resp.status_code} {resp.text}") resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": { "config": {}, "label": {"__prefect_kind": "workspace_variable", "variable_name": var_name}, }, }) if resp.status_code not in (200, 201): return fail(f"workspace_variable: {resp.status_code} {resp.text}") if resp.json().get("parameters", {}).get("label") != "from-variable": return fail(f"workspace_variable value: {resp.json().get('parameters')}") log(" workspace_variable resolves -> 201")
# missing variable -> 400 with python's message resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "parameters": { "config": {}, "label": {"__prefect_kind": "workspace_variable", "variable_name": "no-such-variable"}, }, }) detail = resp.json().get("detail", "") if resp.status_code != 400 or "Variable 'no-such-variable' not found in workspace." not in detail: return fail(f"missing variable: {resp.status_code} {resp.text}") log(" missing workspace_variable -> 400")
log(" [green]parameter hydration verified[/green]")
# cleanup client.delete(f"/deployments/{deployment_id}") client.delete(f"/variables/name/{var_name}") return True
def test_get_scheduled_flow_runs(client: CountingClient) -> bool: """Test get_scheduled_flow_runs endpoint (worker polling).""" from datetime import datetime, timezone
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup log("[bold]setup: create flow[/bold]") resp = client.post("/flows/", json={"name": f"schedule-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
pool_name = f"schedule-pool-{uuid.uuid4().hex[:8]}" resp = client.post("/work_pools/", json={"name": pool_name, "type": "process"}) if resp.status_code not in (200, 201): return fail(f"create work_pool {resp.status_code}") pool = resp.json() pool_id, default_queue_id = pool.get("id"), pool.get("default_queue_id") log(f" pool: {pool_id}") if pool.get("status") != "NOT_READY": return fail(f"expected pool NOT_READY, got {pool.get('status')}")
resp = client.post("/deployments/", json={"name": f"schedule-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "work_pool_name": pool_name}) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code}") deployment = resp.json() deployment_id = deployment.get("id") log(f" deployment: {deployment_id}") if deployment.get("status") != "NOT_READY": return fail(f"expected deployment NOT_READY, got {deployment.get('status')}")
# create scheduled flow run log("[bold]create scheduled flow run[/bold]") resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={"state": {"type": "SCHEDULED", "name": "Scheduled"}}) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run = resp.json() flow_run_id = flow_run.get("id") log(f" flow_run: {flow_run_id}") log(f" state: {flow_run.get('state_type')}") if flow_run.get("state_type") != "SCHEDULED": return fail(f"expected SCHEDULED, got {flow_run.get('state_type')}")
# test polling log("[bold]get_scheduled_flow_runs[/bold]") resp = client.post(f"/work_pools/{pool_name}/get_scheduled_flow_runs", json={}) if resp.status_code != 200: return fail(f"get_scheduled_flow_runs {resp.status_code}") scheduled_runs = resp.json() if not isinstance(scheduled_runs, list): return fail(f"expected list, got {type(scheduled_runs)}") log(f" returned {len(scheduled_runs)} runs")
# verify our run is in results found = any(item.get("flow_run", {}).get("id") == flow_run_id and item.get("work_pool_id") == pool_id and item.get("work_queue_id") == default_queue_id for item in scheduled_runs) if not found: return fail("scheduled flow run not found in results") log(" flow run found in results")
# verify status changes resp = client.get(f"/work_pools/{pool_name}") if resp.status_code != 200 or resp.json().get("status") != "READY": return fail(f"expected pool READY after polling") log(" pool status: READY")
resp = client.get(f"/deployments/{deployment_id}") if resp.status_code != 200 or resp.json().get("status") != "READY": return fail(f"expected deployment READY after polling") log(" deployment status: READY")
# test filters resp = client.post(f"/work_pools/{pool_name}/get_scheduled_flow_runs", json={"work_queue_names": ["default"]}) if resp.status_code != 200: return fail(f"filter test {resp.status_code}") log(" filtered by queue: ok")
resp = client.post(f"/work_pools/{pool_name}/get_scheduled_flow_runs", json={"scheduled_before": datetime.now(timezone.utc).isoformat()}) if resp.status_code != 200: return fail(f"scheduled_before test {resp.status_code}") log(f" scheduled_before filter: {len(resp.json())} runs")
# cleanup client.delete(f"/deployments/{deployment_id}") client.delete(f"/work_pools/{pool_name}") log(" cleanup: ok") return True
def test_retry_failed_flows(client: CountingClient) -> bool: """Test RetryFailedFlows rule - retry cycle when configured."""
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup log("[bold]setup[/bold]") resp = client.post("/flows/", json={"name": f"retry-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
# create flow run with retry configuration via empirical_policy log("[bold]create flow run with retries=2, retry_delay=1[/bold]") resp = client.post("/flow_runs/", json={ "flow_id": flow_id, "name": f"retry-run-{uuid.uuid4().hex[:8]}", "state": {"type": "PENDING", "name": "Pending"}, "empirical_policy": {"retries": 2, "retry_delay": 1}, }) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run = resp.json() flow_run_id = flow_run.get("id") log(f" flow_run_id: {flow_run_id}")
# run_count starts at 0, increments when entering RUNNING # transition to RUNNING (run_count becomes 1) log("[bold]PENDING → RUNNING (first attempt)[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) if resp.status_code not in (200, 201): return fail(f"set RUNNING {resp.status_code}") if resp.json().get("status") != "ACCEPT": return fail(f"expected ACCEPT for RUNNING, got {resp.json().get('status')}")
# verify run_count resp = client.get(f"/flow_runs/{flow_run_id}") if resp.status_code != 200: return fail(f"get flow run {resp.status_code}") run_count = resp.json().get("run_count", 0) log(f" run_count after RUNNING: {run_count}")
# fail - with retries available, should REJECT and schedule retry log("[bold]RUNNING → FAILED (expect REJECT + AwaitingRetry)[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "FAILED", "name": "Failed", "message": "Test failure"}, }) if resp.status_code not in (200, 201): return fail(f"set FAILED {resp.status_code}") result = resp.json() status = result.get("status") log(f" status: {status}")
# with retries, should get REJECT and the run should be in AwaitingRetry state if status == "REJECT": log(f" [green]REJECT (retries available)[/green]") # verify state is now AwaitingRetry (SCHEDULED) resp = client.get(f"/flow_runs/{flow_run_id}") if resp.status_code != 200: return fail(f"get flow run {resp.status_code}") actual_state = resp.json().get("state_type") actual_name = resp.json().get("state_name") log(f" state: {actual_state}/{actual_name}") if actual_state != "SCHEDULED": return fail(f"expected SCHEDULED (AwaitingRetry), got {actual_state}") log(f" [green]state: SCHEDULED/AwaitingRetry (correct)[/green]") elif status == "ACCEPT": # this could happen if run_count > retries - check if that's the case log(f" ACCEPT (retries may be exhausted)") return True else: return fail(f"unexpected status: {status}")
# complete the retry cycle: SCHEDULED → PENDING → RUNNING → COMPLETED log("[bold]retry cycle: SCHEDULED → PENDING → RUNNING → COMPLETED[/bold]")
# SCHEDULED → PENDING resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"retry PENDING {resp.status_code}") log(f" → PENDING: {resp.json().get('status')}")
# PENDING → RUNNING (run_count increments) resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) if resp.status_code not in (200, 201): return fail(f"retry RUNNING {resp.status_code}") log(f" → RUNNING: {resp.json().get('status')}")
# verify run_count incremented resp = client.get(f"/flow_runs/{flow_run_id}") run_count = resp.json().get("run_count", 0) log(f" run_count: {run_count}")
# complete successfully resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.status_code not in (200, 201): return fail(f"COMPLETED {resp.status_code}") if resp.json().get("status") != "ACCEPT": return fail(f"expected ACCEPT for COMPLETED, got {resp.json().get('status')}") log(f" [green]→ COMPLETED: ACCEPT[/green]")
return True
def test_cancellation_flow(client: CountingClient) -> bool: """Test cancellation states (CANCELLING, CANCELLED)."""
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup log("[bold]setup[/bold]") resp = client.post("/flows/", json={"name": f"cancel-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
resp = client.post("/flow_runs/", json={ "flow_id": flow_id, "name": f"cancel-run-{uuid.uuid4().hex[:8]}", "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run_id = resp.json().get("id") log(f" flow_run_id: {flow_run_id}")
# start the run log("[bold]PENDING → RUNNING[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) if resp.status_code not in (200, 201) or resp.json().get("status") != "ACCEPT": return fail(f"RUNNING transition failed") log(f" status: ACCEPT")
# cancel while running - first CANCELLING log("[bold]RUNNING → CANCELLING[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "CANCELLING", "name": "Cancelling"}, }) if resp.status_code not in (200, 201): return fail(f"CANCELLING {resp.status_code}") if resp.json().get("status") != "ACCEPT": return fail(f"expected ACCEPT for CANCELLING, got {resp.json().get('status')}") log(f" [green]status: ACCEPT[/green]")
# verify state resp = client.get(f"/flow_runs/{flow_run_id}") if resp.json().get("state_type") != "CANCELLING": return fail(f"expected CANCELLING, got {resp.json().get('state_type')}")
# complete cancellation log("[bold]CANCELLING → CANCELLED[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "CANCELLED", "name": "Cancelled"}, }) if resp.status_code not in (200, 201): return fail(f"CANCELLED {resp.status_code}") if resp.json().get("status") != "ACCEPT": return fail(f"expected ACCEPT for CANCELLED, got {resp.json().get('status')}") log(f" [green]status: ACCEPT[/green]")
# verify final state resp = client.get(f"/flow_runs/{flow_run_id}") if resp.json().get("state_type") != "CANCELLED": return fail(f"expected CANCELLED, got {resp.json().get('state_type')}") log(f" [green]final state: CANCELLED[/green]")
# CANCELLED is terminal - verify can't go back to PENDING. python's # PreventPendingTransitions aborts, and so does this server. log("[bold]CANCELLED → PENDING (expect ABORT)[/bold]") resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"PENDING attempt {resp.status_code}") if resp.json().get("status") != "ABORT": return fail(f"expected ABORT for CANCELLED→PENDING, got {resp.json().get('status')}") log(f" [green]CANCELLED → PENDING: ABORT (correct)[/green]")
return True
def test_serve_pattern(client: CountingClient) -> bool: """Test Runner/.serve() pattern: poll /deployments/get_scheduled_flow_runs, execute locally.
This simulates what happens when you call flow.serve() - a Runner process polls the deployments endpoint for scheduled runs and executes them in subprocesses. """
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup: create flow and deployment (no work pool - serve doesn't use workers) log("[bold]setup: flow + deployment (no work pool)[/bold]") resp = client.post("/flows/", json={"name": f"serve-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
resp = client.post("/deployments/", json={ "name": f"serve-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, # no work_pool_name - this is the serve pattern }) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code}") deployment = resp.json() deployment_id = deployment.get("id") log(f" deployment: {deployment_id}")
# create a scheduled flow run log("[bold]1. create scheduled flow run[/bold]") resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "state": {"type": "SCHEDULED", "name": "Scheduled"}, }) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run = resp.json() flow_run_id = flow_run.get("id") log(f" flow_run: {flow_run_id}")
# Runner polls: POST /deployments/get_scheduled_flow_runs log("[bold]2. Runner polls /deployments/get_scheduled_flow_runs[/bold]") from datetime import datetime, timezone resp = client.post("/deployments/get_scheduled_flow_runs", json={ "deployment_ids": [deployment_id], "scheduled_before": datetime.now(timezone.utc).isoformat(), "limit": 10, }) if resp.status_code != 200: return fail(f"poll {resp.status_code}") scheduled = resp.json() found = any(r.get("id") == flow_run_id for r in scheduled) if not found: return fail("run not found in scheduled runs") log(f" found {len(scheduled)} scheduled run(s)")
# Runner executes: SCHEDULED → PENDING → RUNNING → COMPLETED log("[bold]3. Runner executes flow locally[/bold]")
# claim the run resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "PENDING", "name": "Pending"}, }) if resp.status_code not in (200, 201): return fail(f"PENDING {resp.status_code}") log(f" → PENDING: {resp.json().get('status')}")
# execute resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) if resp.status_code not in (200, 201): return fail(f"RUNNING {resp.status_code}") log(f" → RUNNING: {resp.json().get('status')}")
# complete resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.status_code not in (200, 201): return fail(f"COMPLETED {resp.status_code}") if resp.json().get("status") != "ACCEPT": return fail(f"expected ACCEPT, got {resp.json().get('status')}") log(f" [green]→ COMPLETED: ACCEPT[/green]")
# cleanup client.delete(f"/deployments/{deployment_id}") log(" cleanup: ok")
return True
def test_worker_pattern(client: CountingClient) -> bool: """Test Worker pattern: poll /work_pools/{name}/get_scheduled_flow_runs, dispatch to infra.
This simulates what a Prefect Worker does - it polls a work pool for scheduled runs and dispatches them to infrastructure (k8s, docker, etc). The worker doesn't execute the flow itself; infrastructure does. """
def fail(msg: str) -> bool: if not QUIET: console.print(f"[red]FAIL[/red]: {msg}") return False
def log(msg: str) -> None: if not QUIET: console.print(msg)
# setup: flow + work pool + deployment with work_pool_name log("[bold]setup: flow + work pool + deployment[/bold]") resp = client.post("/flows/", json={"name": f"worker-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
pool_name = f"worker-pool-{uuid.uuid4().hex[:8]}" resp = client.post("/work_pools/", json={"name": pool_name, "type": "kubernetes"}) if resp.status_code not in (200, 201): return fail(f"create work_pool {resp.status_code}") log(f" pool: {pool_name}")
resp = client.post("/deployments/", json={ "name": f"worker-deploy-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "work_pool_name": pool_name, # KEY: uses work pool }) if resp.status_code not in (200, 201): return fail(f"create deployment {resp.status_code}") deployment_id = resp.json().get("id") log(f" deployment: {deployment_id}")
# worker registers via heartbeat log("[bold]1. Worker sends heartbeat[/bold]") resp = client.post(f"/work_pools/{pool_name}/workers/heartbeat", json={ "name": "k8s-worker-1", "heartbeat_interval_seconds": 30, }) if resp.status_code != 204: return fail(f"heartbeat {resp.status_code}") log(f" heartbeat sent")
# create a scheduled flow run log("[bold]2. create scheduled flow run[/bold]") resp = client.post(f"/deployments/{deployment_id}/create_flow_run", json={ "state": {"type": "SCHEDULED", "name": "Scheduled"}, }) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run_id = resp.json().get("id") log(f" flow_run: {flow_run_id}")
# Worker polls: POST /work_pools/{name}/get_scheduled_flow_runs log("[bold]3. Worker polls /work_pools/{name}/get_scheduled_flow_runs[/bold]") resp = client.post(f"/work_pools/{pool_name}/get_scheduled_flow_runs", json={}) if resp.status_code != 200: return fail(f"poll {resp.status_code}") scheduled = resp.json() found = any(r.get("flow_run", {}).get("id") == flow_run_id for r in scheduled) if not found: return fail("run not found in work pool queue") log(f" found run in work pool queue")
# Worker dispatches to infrastructure (simulated - infra reports back state) log("[bold]4. Infrastructure executes and reports state[/bold]")
# infra claims resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "PENDING", "name": "Pending"}, }) log(f" → PENDING: {resp.json().get('status')}")
# infra runs resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, }) log(f" → RUNNING: {resp.json().get('status')}")
# infra completes resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, }) if resp.json().get("status") != "ACCEPT": return fail(f"expected ACCEPT, got {resp.json().get('status')}") log(f" [green]→ COMPLETED: ACCEPT[/green]")
# cleanup client.delete(f"/deployments/{deployment_id}") client.delete(f"/work_pools/{pool_name}") log(" cleanup: ok")
return True
def test_log_broadcaster(client: CountingClient) -> bool: """Test that posting a log triggers a /api/logs/out WebSocket delivery.
Client signal: the prefect UI connects to /api/logs/out, sends a LogFilter, and expects every matching log to arrive as a `{"type":"log","log":{...}}` message. """ import asyncio import json as _json fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log_msg = lambda msg: QUIET or console.print(msg)
resp = client.post("/flows/", json={"name": f"logb-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json()["id"]
resp = client.post("/flow_runs/", json={"flow_id": flow_id, "name": f"logb-run-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow_run {resp.status_code}") flow_run_id = resp.json()["id"] log_msg(f" flow_run {flow_run_id[:8]}")
# base URL for ws ws_url = BASE_URL.replace("http://", "ws://").replace("https://", "wss://").rsplit("/api", 1)[0] + "/api/logs/out"
async def run() -> bool: import websockets try: async with websockets.connect(ws_url) as ws: # handshake await ws.send(_json.dumps({"type": "auth"})) resp_raw = await asyncio.wait_for(ws.recv(), timeout=5) ok = _json.loads(resp_raw) if ok.get("type") != "auth_success": return fail(f"auth_success expected, got {ok}") # filter await ws.send(_json.dumps({ "type": "filter", "filter": {"flow_run_id": {"any_": [flow_run_id]}}, })) # post a log via httpx (sync, in a thread so we don't block the loop) from datetime import datetime, timezone ts = datetime.now(timezone.utc).isoformat() payload = [{ "name": "smoke.logger", "level": 20, "message": "hello from smoke", "timestamp": ts, "flow_run_id": flow_run_id, }] def _post(): return client.post("/logs/", json=payload) post_resp = await asyncio.to_thread(_post) if post_resp.status_code not in (200, 201): return fail(f"POST /logs/ {post_resp.status_code}")
# wait for the log to arrive received = await asyncio.wait_for(ws.recv(), timeout=10) envelope = _json.loads(received) if envelope.get("type") != "log": return fail(f"unexpected envelope: {envelope}") log_obj = envelope.get("log") or {} if log_obj.get("flow_run_id") != flow_run_id or log_obj.get("message") != "hello from smoke": return fail(f"unexpected log payload: {log_obj}") log_msg(f" [green]received log via WS[/green]") return True except asyncio.TimeoutError: return fail("timed out waiting for log delivery") except Exception as e: return fail(f"ws error: {type(e).__name__}: {e}")
return asyncio.run(run())
def test_db_vacuum(client: CountingClient) -> bool: """Test that db_vacuum deletes terminal flow_runs past retention.
Client signal: a flow_run that's been in a terminal state with `end_time` older than the configured retention should disappear (GET → 404) after one vacuum tick.
The smoke harness sets retention to 10s and the loop to 3s (see compose.yml), so a freshly-completed run is in safe-zone for now, but we manufacture an old one by backdating the COMPLETED state transition via a custom set_state with a far-past timestamp on the end_time field… actually, since the server stamps end_time itself, we instead create a run, complete it, then poll for `count` of terminal runs to drop after retention elapses. """ import time as time_mod fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
resp = client.post("/flows/", json={"name": f"vac-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json()["id"]
resp = client.post("/flow_runs/", json={"flow_id": flow_id, "name": f"vac-run-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow_run {resp.status_code}") run_id = resp.json()["id"]
# complete it (RUNNING → COMPLETED so end_time is stamped now) client.post(f"/flow_runs/{run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": True, }) resp = client.post(f"/flow_runs/{run_id}/set_state", json={ "state": {"type": "COMPLETED", "name": "Completed"}, "force": True, }) if resp.status_code not in (200, 201): return fail(f"complete {resp.status_code}") log(f" completed run {run_id[:8]}, waiting for retention (60s) + vacuum tick (5s)")
# Wait for retention to elapse + one vacuum tick. Retention is 60s, # loop is 5s, so 65–70s expected. Give it 90s. deadline = time_mod.time() + 90 while time_mod.time() < deadline: time_mod.sleep(3) resp = client.get(f"/flow_runs/{run_id}") if resp.status_code == 404: log(f" [green]flow_run vacuumed[/green]") return True
return fail(f"flow_run still present after 90s")
def test_foreman(client: CountingClient) -> bool: """Test that foreman marks stale workers OFFLINE and propagates to work_pool NOT_READY.
Client signal: a worker that stops heartbeating beyond `heartbeat_interval_seconds * multiplier` flips to OFFLINE; once a work_pool has zero ONLINE workers, the pool itself flips to NOT_READY.
Requires foreman with a short loop (set via env in the docker compose smoke run). """ import time as time_mod fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
pool_name = f"foreman-pool-{uuid.uuid4().hex[:8]}" resp = client.post("/work_pools/", json={"name": pool_name, "type": "process"}) if resp.status_code not in (200, 201): return fail(f"create work_pool {resp.status_code}")
# heartbeat once with a tiny interval so the worker is immediately stale. worker_name = f"foreman-worker-{uuid.uuid4().hex[:8]}" resp = client.post(f"/work_pools/{pool_name}/workers/heartbeat", json={ "name": worker_name, "heartbeat_interval_seconds": 1, }) if resp.status_code not in (200, 201, 204): return fail(f"heartbeat {resp.status_code}") log(f" worker {worker_name} heartbeat'd with interval=1s")
# pool should be READY now (the heartbeat handler sets that) resp = client.get(f"/work_pools/{pool_name}") if resp.status_code != 200: return fail(f"get pool {resp.status_code}") initial_status = resp.json().get("status") log(f" pool initial status: {initial_status}")
# poll for worker OFFLINE + pool NOT_READY. Multiplier default is 3, # heartbeat_interval=1 ⇒ stale after 3s, foreman loop is 5s in test env. deadline = time_mod.time() + 25 saw_offline = False saw_not_ready = False while time_mod.time() < deadline: time_mod.sleep(1) resp = client.post(f"/work_pools/{pool_name}/workers/filter", json={}) if resp.status_code == 200: workers = resp.json() if workers and workers[0].get("status") == "OFFLINE": saw_offline = True resp = client.get(f"/work_pools/{pool_name}") if resp.status_code == 200 and resp.json().get("status") == "NOT_READY": saw_not_ready = True if saw_offline and saw_not_ready: log(f" [green]worker OFFLINE, pool NOT_READY[/green]") break else: return fail(f"timeout: worker_offline={saw_offline} pool_not_ready={saw_not_ready}")
# foreman should have emitted status-change events for both transitions time_mod.sleep(2) resp = client.post("/events/filter", json={ "filter": {"event": {"name": ["prefect.worker.offline", "prefect.work-pool.not-ready"]}}, "limit": 100, }) if resp.status_code != 200: return fail(f"events filter {resp.status_code}") body = resp.json() events = body.get("events", []) if isinstance(body, dict) else (body if isinstance(body, list) else []) names = {e.get("event") for e in events} if "prefect.worker.offline" not in names or "prefect.work-pool.not-ready" not in names: return fail(f"missing status events; saw {names}") log(f" [green]status events emitted: {sorted(names)}[/green]") return True
def test_pause_expirations(client: CountingClient) -> bool: """Test that pause_expirations flips PAUSED flow_runs with elapsed pause_timeout to FAILED.
Client signal: a paused flow run whose `state_details.pause_timeout` lies in the past should become FAILED within one service tick (~5s), with `state.message = "The flow was paused and never resumed."`. """ from datetime import datetime, timezone, timedelta import time as time_mod fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
resp = client.post("/flows/", json={"name": f"pe-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json()["id"]
resp = client.post("/flow_runs/", json={"flow_id": flow_id, "name": f"pe-run-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run_id = resp.json()["id"]
# pause with timeout already in the past past = (datetime.now(timezone.utc) - timedelta(seconds=60)).isoformat() resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": { "type": "PAUSED", "name": "Paused", "state_details": {"pause_timeout": past}, }, "force": True, }) if resp.status_code not in (200, 201): return fail(f"pause {resp.status_code}: {resp.text}") log(f" paused with pause_timeout in the past")
# poll for FAILED deadline = time_mod.time() + 20 while time_mod.time() < deadline: time_mod.sleep(1) resp = client.get(f"/flow_runs/{flow_run_id}") if resp.status_code == 200 and resp.json().get("state_type") == "FAILED": log(f" [green]flipped to FAILED after pause expired[/green]") return True state = resp.json().get("state_type", "?") if resp.status_code == 200 else "?" return fail(f"flow run still {state} after 20s")
def test_cancellation_cleanup(client: CountingClient) -> bool: """Test that cancellation_cleanup cancels child task_runs of CANCELLED flow_runs.
Client signal: after a flow_run is cancelled, any of its task_runs in non-terminal states should flip to CANCELLED within ~one service tick. """ import time as time_mod fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
# 1. Create a flow + run, attach two RUNNING task_runs. resp = client.post("/flows/", json={"name": f"cc-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json()["id"]
resp = client.post("/flow_runs/", json={"flow_id": flow_id, "name": f"cc-run-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run_id = resp.json()["id"]
task_ids = [] for i in range(2): resp = client.post("/task_runs/", json={ "flow_run_id": flow_run_id, "name": f"cc-task-{i}", "task_key": f"cc-task-key-{i}", "dynamic_key": str(i), }) if resp.status_code not in (200, 201): return fail(f"create task_run {i}: {resp.status_code}") task_id = resp.json()["id"] task_ids.append(task_id) # set RUNNING so they're non-terminal client.post(f"/task_runs/{task_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": True, })
# 2. Take the flow_run through RUNNING then CANCELLED, with end_time # so the cleanup service picks it up. client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": True, }) resp = client.post(f"/flow_runs/{flow_run_id}/set_state", json={ "state": {"type": "CANCELLED", "name": "Cancelled"}, "force": True, }) if resp.status_code not in (200, 201): return fail(f"set CANCELLED: {resp.status_code}") log(f" flow_run {flow_run_id[:8]} CANCELLED")
# 3. Poll for child task_runs to be cancelled by the service. deadline = time_mod.time() + 30 while time_mod.time() < deadline: time_mod.sleep(1) all_cancelled = True for tid in task_ids: resp = client.get(f"/task_runs/{tid}") if resp.status_code != 200: all_cancelled = False; break if resp.json().get("state_type") != "CANCELLED": all_cancelled = False break if all_cancelled: log(f" [green]both child task_runs cancelled[/green]") return True
# one more diagnostic pass for the log states = [] for tid in task_ids: resp = client.get(f"/task_runs/{tid}") states.append(resp.json().get("state_type") if resp.status_code == 200 else "?") return fail(f"task_runs not cancelled within 30s, states={states}")
def test_subflow_cancellation(client: CountingClient) -> bool: """Test that cancellation propagates from a parent flow_run to its subflow flow_runs (the ones with `parent_task_run_id` set).
Client signal: cancel a parent flow_run that has a task_run with a subflow flow_run hanging off it; within one service tick the subflow should flip to CANCELLED (no deployment) or CANCELLING (with one). """ import time as time_mod fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
resp = client.post("/flows/", json={"name": f"sc-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json()["id"]
# parent flow_run + a task_run on it resp = client.post("/flow_runs/", json={"flow_id": flow_id, "name": f"sc-parent-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create parent run {resp.status_code}") parent_run_id = resp.json()["id"]
resp = client.post("/task_runs/", json={ "flow_run_id": parent_run_id, "name": "sc-parent-task", "task_key": "sc-parent-task-key", "dynamic_key": "0", }) if resp.status_code not in (200, 201): return fail(f"create parent task_run {resp.status_code}") parent_task_run_id = resp.json()["id"]
# subflow flow_run linked via parent_task_run_id resp = client.post("/flow_runs/", json={ "flow_id": flow_id, "name": f"sc-subflow-{uuid.uuid4().hex[:8]}", "parent_task_run_id": parent_task_run_id, }) if resp.status_code not in (200, 201): return fail(f"create subflow {resp.status_code}: {resp.text}") subflow_run_id = resp.json()["id"] if resp.json().get("parent_task_run_id") != parent_task_run_id: return fail(f"parent_task_run_id not round-tripped: got {resp.json().get('parent_task_run_id')}") log(f" subflow {subflow_run_id[:8]} hangs off parent_task_run_id {parent_task_run_id[:8]}")
# take the subflow non-terminal so the cancellation cascade has work to do client.post(f"/flow_runs/{subflow_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": True, })
# cancel the PARENT client.post(f"/flow_runs/{parent_run_id}/set_state", json={ "state": {"type": "RUNNING", "name": "Running"}, "force": True, }) resp = client.post(f"/flow_runs/{parent_run_id}/set_state", json={ "state": {"type": "CANCELLED", "name": "Cancelled"}, "force": True, }) if resp.status_code not in (200, 201): return fail(f"set parent CANCELLED: {resp.status_code}") log(f" parent {parent_run_id[:8]} CANCELLED")
# poll for the subflow to cascade. Subflow has no deployment → expect CANCELLED. deadline = time_mod.time() + 40 while time_mod.time() < deadline: time_mod.sleep(1) r = client.get(f"/flow_runs/{subflow_run_id}") if r.status_code == 200 and r.json().get("state_type") == "CANCELLED": log(f" [green]subflow cascaded to CANCELLED[/green]") return True
last = client.get(f"/flow_runs/{subflow_run_id}").json().get("state_type", "?") return fail(f"subflow stuck at {last} after 40s")
def test_late_runs(client: CountingClient) -> bool: """Test that late_runs service marks overdue scheduled runs as Late.""" from datetime import datetime, timezone, timedelta import time as time_mod fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
resp = client.post("/flows/", json={"name": f"late-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
# create flow run with past scheduled time (60s ago, threshold is 15s) past_time = (datetime.now(timezone.utc) - timedelta(seconds=60)).isoformat() resp = client.post("/flow_runs/", json={ "flow_id": flow_id, "name": f"late-run-{uuid.uuid4().hex[:8]}", "state": {"type": "SCHEDULED", "name": "Scheduled"}, "next_scheduled_start_time": past_time}) if resp.status_code not in (200, 201): return fail(f"create flow run {resp.status_code}") flow_run_id = resp.json().get("id")
# wait for late_runs service (runs every 5s) for i in range(15): time_mod.sleep(1) resp = client.get(f"/flow_runs/{flow_run_id}") if resp.status_code == 200 and resp.json().get("state_name") == "Late": log(f" [green]state changed to Late after {i+1}s[/green]") return True return fail("late_runs service did not mark run as Late within timeout")
def test_work_queue_priority(client: CountingClient) -> bool: """Test work queue priority ordering in get_scheduled_flow_runs.""" fail = lambda msg: (QUIET or console.print(f"[red]FAIL[/red]: {msg}"), False)[1] log = lambda msg: QUIET or console.print(msg)
resp = client.post("/flows/", json={"name": f"priority-flow-{uuid.uuid4().hex[:8]}"}) if resp.status_code not in (200, 201): return fail(f"create flow {resp.status_code}") flow_id = resp.json().get("id")
pool_name = f"priority-pool-{uuid.uuid4().hex[:8]}" resp = client.post("/work_pools/", json={"name": pool_name, "type": "process"}) if resp.status_code not in (200, 201): return fail(f"create pool {resp.status_code}")
# create high and low priority queues for name, pri in [("high-priority", 1), ("low-priority", 100)]: resp = client.post(f"/work_pools/{pool_name}/queues/", json={"name": name, "priority": pri}) if resp.status_code not in (200, 201): return fail(f"create {name} queue {resp.status_code}")
# create deployments for each queue and flow runs in reverse priority order deployments, flow_runs = {}, {} for q in ["high-priority", "low-priority", "default"]: resp = client.post("/deployments/", json={"name": f"d-{q}-{uuid.uuid4().hex[:8]}", "flow_id": flow_id, "work_pool_name": pool_name, "work_queue_name": q}) if resp.status_code not in (200, 201): return fail(f"create deployment {q} {resp.status_code}") deployments[q] = resp.json().get("id") for q in ["low-priority", "default", "high-priority"]: resp = client.post(f"/deployments/{deployments[q]}/create_flow_run", json={"state": {"type": "SCHEDULED", "name": "Scheduled"}}) if resp.status_code not in (200, 201): return fail(f"create run for {q} {resp.status_code}") flow_runs[q] = resp.json().get("id")
# poll and verify priority ordering resp = client.post(f"/work_pools/{pool_name}/get_scheduled_flow_runs", json={}) if resp.status_code != 200: return fail(f"poll {resp.status_code}") scheduled = resp.json() if len(scheduled) < 3: return fail(f"expected 3 runs, got {len(scheduled)}")
run_ids = [r.get("flow_run", {}).get("id") for r in scheduled] pos = {q: run_ids.index(flow_runs[q]) if flow_runs[q] in run_ids else -1 for q in flow_runs} if -1 in pos.values(): return fail("not all runs found") if pos["high-priority"] > pos["low-priority"]: return fail("high should come before low") log(f" [green]priority ordering verified (high={pos['high-priority']}, low={pos['low-priority']})[/green]")
for d in deployments.values(): client.delete(f"/deployments/{d}") client.delete(f"/work_pools/{pool_name}") return True
def main(): json_output = "--json" in sys.argv
if not QUIET: console.print("\n[bold cyan]prefect-server API test[/bold cyan]\n")
results: list[TestResult] = []
# run all tests # core API tests (fast, no sleeps) results.append(run_test("admin", test_admin)) results.append(run_test("flow_run (success)", lambda c: test_flow_run(c, should_fail=False))) results.append(run_test("flow_run (failure)", lambda c: test_flow_run(c, should_fail=True))) results.append(run_test("flow_with_task_runs", test_flow_with_task_runs)) results.append(run_test("task_run", test_task_run)) results.append(run_test("filters", test_filters)) results.append(run_test("logs", test_logs)) results.append(run_test("variables", test_variables)) results.append(run_test("artifacts", test_artifacts)) results.append(run_test("blocks", test_blocks)) results.append(run_test("work_pools", test_work_pools)) results.append(run_test("deployments", test_deployments))
# execution patterns (two distinct models) results.append(run_test("serve_pattern", test_serve_pattern)) # Runner: polls deployments results.append(run_test("worker_pattern", test_worker_pattern)) # Worker: polls work pools results.append(run_test("work_queue_priority", test_work_queue_priority)) results.append(run_test("retry_failed_flows", test_retry_failed_flows)) results.append(run_test("cancellation_flow", test_cancellation_flow)) results.append(run_test("cancellation_cleanup", test_cancellation_cleanup)) # background service cancels child task_runs results.append(run_test("subflow_cancellation", test_subflow_cancellation)) # cascade from parent flow_run to subflows results.append(run_test("pause_expirations", test_pause_expirations)) # background service fails expired pauses results.append(run_test("foreman", test_foreman)) # background service marks stale workers / pools NOT_READY results.append(run_test("log_broadcaster", test_log_broadcaster)) # webserver service streams logs over WS results.append(run_test("late_runs", test_late_runs)) # background service marks overdue runs
# scheduler integration tests (require sleeps for background service) results.append(run_test("scheduler_idempotency", test_scheduler_idempotency)) results.append(run_test("parameter_merging", test_parameter_merging)) results.append(run_test("parameter_schema_enforcement", test_parameter_schema_enforcement)) results.append(run_test("parameter_hydration", test_parameter_hydration)) # db_vacuum runs last so its 60s retention doesn't race with the # cancellation_cleanup test (whose 30s poll window would otherwise # see its own flow_run vacuumed mid-test). results.append(run_test("db_vacuum", test_db_vacuum))
total_duration = sum(r.duration_ms for r in results) total_requests = sum(r.requests for r in results) all_passed = all(r.passed for r in results)
if json_output: # machine-readable output for benchmark script output = { "passed": all_passed, "total_duration_ms": total_duration, "total_requests": total_requests, "sections": [ { "name": r.name, "passed": r.passed, "duration_ms": r.duration_ms, "requests": r.requests, "error": r.error, } for r in results ], } print(json_lib.dumps(output)) else: # human-readable output console.print("\n" + "=" * 60)
table = Table(title="test results") table.add_column("section", style="cyan") table.add_column("time", justify="right") table.add_column("reqs", justify="right") table.add_column("status", justify="center")
for r in results: status = "[green]✓[/green]" if r.passed else "[red]✗[/red]" table.add_row(r.name, f"{r.duration_ms:.1f}ms", str(r.requests), status)
table.add_row("", "", "", "", style="dim") table.add_row("[bold]total[/bold]", f"[bold]{total_duration:.1f}ms[/bold]", f"[bold]{total_requests}[/bold]", "")
console.print(table)
if all_passed: console.print("\n[bold green]all tests passed[/bold green]") else: console.print("\n[bold red]some tests failed[/bold red]")
sys.exit(0 if all_passed else 1)
if __name__ == "__main__": main()