Something went wrong. Try again.
prefect server in zig
Something went wrong. Try again.
8.0 kB · 228 lines
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229#!/usr/bin/env -S uv run --script --quiet# /// script# requires-python = ">=3.12"# dependencies = ["prefect>=3.0", "httpx"]# ///"""End-to-end worker test for prefect-server.
Exercises the real worker path: a `prefect worker start --run-once` subprocesspolls a process-type work pool, picks up a flow run, executes the flow in asubprocess, and reports the result back to the server.
Tests against the canonical python `prefect.workers.process.ProcessWorker` —this verifies that prefect-server's worker-facing endpoints compose correctlyagainst the upstream client implementation:- POST /work_pools/{name}/workers/heartbeat- POST /work_pools/{name}/get_scheduled_flow_runs- POST /flow_runs/{id}/set_state- POST /flows/, POST /deployments/
Requires:- prefect-server running and reachable at PREFECT_API_URL- uv on PATH to launch this script"""
import osimport shutilimport subprocessimport sysimport tempfileimport timeimport uuidfrom pathlib import Pathfrom textwrap import dedent
import httpx
API_URL = os.environ.get("PREFECT_API_URL", "http://localhost:4200/api")WORKER_TIMEOUT_SEC = 120
def api(method: str, path: str, **kwargs) -> httpx.Response: return httpx.request(method, f"{API_URL}{path}", timeout=30, **kwargs)
def wait_for(predicate, *, timeout: float, message: str, interval: float = 1.0): """Poll until predicate is truthy or timeout. Returns predicate's last result.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: result = predicate() if result: return result time.sleep(interval) raise AssertionError(f"timeout after {timeout}s: {message}")
def write_flow_file(tmp_dir: Path, marker_file: Path) -> Path: """Write a self-contained flow script that touches a marker file on execution.
Returning a marker proves the worker actually executed the flow body — not just transitioned states. """ flow_file = tmp_dir / "flow.py" flow_file.write_text( dedent(f""" from prefect import flow import pathlib
@flow(log_prints=True) def hello(marker_path: str = {str(marker_file)!r}): pathlib.Path(marker_path).write_text("executed") print(f"flow ran, wrote {{marker_path}}") return "done" """) ) return flow_file
def test_worker_executes_deployment() -> bool: suffix = uuid.uuid4().hex[:8] tmp_dir = Path(tempfile.mkdtemp(prefix=f"prefect-worker-test-{suffix}-")) marker_file = tmp_dir / "executed.marker" pool_name = f"worker-test-pool-{suffix}" worker_name = f"worker-test-{suffix}" deployment_name = f"worker-test-deployment-{suffix}"
print(f"\n=== test_worker_executes_deployment ===") print(f" tmp_dir: {tmp_dir}") print(f" pool: {pool_name}")
write_flow_file(tmp_dir, marker_file)
# Stand up: work pool → deployment → flow run, then run a worker that # drains it. pool_created = False deployment_id = None flow_run_id = None try: # work pool (process type) resp = api("POST", "/work_pools/", json={"name": pool_name, "type": "process"}) assert resp.status_code in (200, 201), f"create pool: {resp.status_code} {resp.text}" pool_created = True
# deployment from local source — uses prefect client so the server sees # the same deployment shape a real user would submit. deploy_proc = subprocess.run( [ sys.executable, "-c", dedent(f""" from prefect import flow flow_obj = flow.from_source( source={str(tmp_dir)!r}, entrypoint="flow.py:hello", ) deployment_id = flow_obj.deploy( name={deployment_name!r}, work_pool_name={pool_name!r}, ) print(deployment_id) """), ], env={**os.environ, "PREFECT_API_URL": API_URL}, capture_output=True, text=True, timeout=60, ) if deploy_proc.returncode != 0: print(f" deploy stderr: {deploy_proc.stderr[-1000:]}") assert False, "deploy subprocess failed" deployment_id = deploy_proc.stdout.strip().split("\n")[-1].strip() print(f" deployment: {deployment_id}")
# create a flow run on the deployment resp = api( "POST", f"/deployments/{deployment_id}/create_flow_run", json={"name": f"worker-test-run-{suffix}"}, ) assert resp.status_code in (200, 201), f"create flow run: {resp.status_code} {resp.text}" flow_run_id = resp.json()["id"] print(f" flow run: {flow_run_id}")
# run the worker — `--run-once` drains ready runs and exits, so the # test owns no daemon lifetime. print(f" starting worker (--run-once)...") worker_proc = subprocess.run( [ sys.executable, "-m", "prefect", "worker", "start", "-p", pool_name, "-t", "process", "-n", worker_name, "--run-once", ], env={**os.environ, "PREFECT_API_URL": API_URL}, capture_output=True, text=True, timeout=WORKER_TIMEOUT_SEC, ) print(f" worker exit code: {worker_proc.returncode}") if worker_proc.returncode != 0: print(f" worker stderr: {worker_proc.stderr[-2000:]}")
# poll until the flow run reaches a terminal state def get_final_state(): r = api("GET", f"/flow_runs/{flow_run_id}") if r.status_code != 200: return None state = r.json().get("state", {}) t = state.get("type") if t in ("COMPLETED", "FAILED", "CRASHED", "CANCELLED"): return state return None
final = wait_for( get_final_state, timeout=30, message=f"flow_run {flow_run_id} did not reach terminal state", ) assert final["type"] == "COMPLETED", f"flow run not Completed: {final}" print(f" ✓ flow run reached state: {final['type']}")
assert marker_file.exists(), f"flow body did not execute (no marker at {marker_file})" print(f" ✓ flow body executed (marker present)")
# verify the worker registered + heartbeated resp = api("POST", f"/work_pools/{pool_name}/workers/filter", json={}) assert resp.status_code == 200, f"worker filter: {resp.status_code}" workers = resp.json() assert any(w.get("name") == worker_name for w in workers), ( f"worker {worker_name!r} not registered (saw {[w.get('name') for w in workers]})" ) print(f" ✓ worker {worker_name!r} registered with pool")
print(" ✓ worker integration test passed") return True
finally: # cleanup — the worker subprocess already exited because --run-once if deployment_id is not None: try: api("DELETE", f"/deployments/{deployment_id}") except Exception: pass if pool_created: try: api("DELETE", f"/work_pools/{pool_name}") except Exception: pass shutil.rmtree(tmp_dir, ignore_errors=True)
def main() -> int: print(f"API: {API_URL}") print(f"tmp dir convention: /tmp/prefect-worker-test-*") try: test_worker_executes_deployment() except AssertionError as e: print(f"\n✗ FAILED: {e}") return 1 except Exception as e: print(f"\n✗ UNEXPECTED ERROR: {type(e).__name__}: {e}") return 1 print("\n=== all worker tests passed ===") return 0
if __name__ == "__main__": sys.exit(main())