#!/usr/bin/env -S uv run --script --quiet # /// script # requires-python = ">=3.12" # dependencies = ["prefect>=3.0", "httpx"] # /// """ End-to-end worker test for prefect-server. Exercises the real worker path: a `prefect worker start --run-once` subprocess polls a process-type work pool, picks up a flow run, executes the flow in a subprocess, and reports the result back to the server. Tests against the canonical python `prefect.workers.process.ProcessWorker` — this verifies that prefect-server's worker-facing endpoints compose correctly against the upstream client implementation: - POST /work_pools/{name}/workers/heartbeat - POST /work_pools/{name}/get_scheduled_flow_runs - POST /flow_runs/{id}/set_state - POST /flows/, POST /deployments/ Requires: - prefect-server running and reachable at PREFECT_API_URL - uv on PATH to launch this script """ import os import shutil import subprocess import sys import tempfile import time import uuid from pathlib import Path from textwrap import dedent import httpx API_URL = os.environ.get("PREFECT_API_URL", "http://localhost:4200/api") WORKER_TIMEOUT_SEC = 120 def api(method: str, path: str, **kwargs) -> httpx.Response: return httpx.request(method, f"{API_URL}{path}", timeout=30, **kwargs) def wait_for(predicate, *, timeout: float, message: str, interval: float = 1.0): """Poll until predicate is truthy or timeout. Returns predicate's last result.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: result = predicate() if result: return result time.sleep(interval) raise AssertionError(f"timeout after {timeout}s: {message}") def write_flow_file(tmp_dir: Path, marker_file: Path) -> Path: """Write a self-contained flow script that touches a marker file on execution. Returning a marker proves the worker actually executed the flow body — not just transitioned states. """ flow_file = tmp_dir / "flow.py" flow_file.write_text( dedent(f""" from prefect import flow import pathlib @flow(log_prints=True) def hello(marker_path: str = {str(marker_file)!r}): pathlib.Path(marker_path).write_text("executed") print(f"flow ran, wrote {{marker_path}}") return "done" """) ) return flow_file def test_worker_executes_deployment() -> bool: suffix = uuid.uuid4().hex[:8] tmp_dir = Path(tempfile.mkdtemp(prefix=f"prefect-worker-test-{suffix}-")) marker_file = tmp_dir / "executed.marker" pool_name = f"worker-test-pool-{suffix}" worker_name = f"worker-test-{suffix}" deployment_name = f"worker-test-deployment-{suffix}" print(f"\n=== test_worker_executes_deployment ===") print(f" tmp_dir: {tmp_dir}") print(f" pool: {pool_name}") write_flow_file(tmp_dir, marker_file) # Stand up: work pool → deployment → flow run, then run a worker that # drains it. pool_created = False deployment_id = None flow_run_id = None try: # work pool (process type) resp = api("POST", "/work_pools/", json={"name": pool_name, "type": "process"}) assert resp.status_code in (200, 201), f"create pool: {resp.status_code} {resp.text}" pool_created = True # deployment from local source — uses prefect client so the server sees # the same deployment shape a real user would submit. deploy_proc = subprocess.run( [ sys.executable, "-c", dedent(f""" from prefect import flow flow_obj = flow.from_source( source={str(tmp_dir)!r}, entrypoint="flow.py:hello", ) deployment_id = flow_obj.deploy( name={deployment_name!r}, work_pool_name={pool_name!r}, ) print(deployment_id) """), ], env={**os.environ, "PREFECT_API_URL": API_URL}, capture_output=True, text=True, timeout=60, ) if deploy_proc.returncode != 0: print(f" deploy stderr: {deploy_proc.stderr[-1000:]}") assert False, "deploy subprocess failed" deployment_id = deploy_proc.stdout.strip().split("\n")[-1].strip() print(f" deployment: {deployment_id}") # create a flow run on the deployment resp = api( "POST", f"/deployments/{deployment_id}/create_flow_run", json={"name": f"worker-test-run-{suffix}"}, ) assert resp.status_code in (200, 201), f"create flow run: {resp.status_code} {resp.text}" flow_run_id = resp.json()["id"] print(f" flow run: {flow_run_id}") # run the worker — `--run-once` drains ready runs and exits, so the # test owns no daemon lifetime. print(f" starting worker (--run-once)...") worker_proc = subprocess.run( [ sys.executable, "-m", "prefect", "worker", "start", "-p", pool_name, "-t", "process", "-n", worker_name, "--run-once", ], env={**os.environ, "PREFECT_API_URL": API_URL}, capture_output=True, text=True, timeout=WORKER_TIMEOUT_SEC, ) print(f" worker exit code: {worker_proc.returncode}") if worker_proc.returncode != 0: print(f" worker stderr: {worker_proc.stderr[-2000:]}") # poll until the flow run reaches a terminal state def get_final_state(): r = api("GET", f"/flow_runs/{flow_run_id}") if r.status_code != 200: return None state = r.json().get("state", {}) t = state.get("type") if t in ("COMPLETED", "FAILED", "CRASHED", "CANCELLED"): return state return None final = wait_for( get_final_state, timeout=30, message=f"flow_run {flow_run_id} did not reach terminal state", ) assert final["type"] == "COMPLETED", f"flow run not Completed: {final}" print(f" ✓ flow run reached state: {final['type']}") assert marker_file.exists(), f"flow body did not execute (no marker at {marker_file})" print(f" ✓ flow body executed (marker present)") # verify the worker registered + heartbeated resp = api("POST", f"/work_pools/{pool_name}/workers/filter", json={}) assert resp.status_code == 200, f"worker filter: {resp.status_code}" workers = resp.json() assert any(w.get("name") == worker_name for w in workers), ( f"worker {worker_name!r} not registered (saw {[w.get('name') for w in workers]})" ) print(f" ✓ worker {worker_name!r} registered with pool") print(" ✓ worker integration test passed") return True finally: # cleanup — the worker subprocess already exited because --run-once if deployment_id is not None: try: api("DELETE", f"/deployments/{deployment_id}") except Exception: pass if pool_created: try: api("DELETE", f"/work_pools/{pool_name}") except Exception: pass shutil.rmtree(tmp_dir, ignore_errors=True) def main() -> int: print(f"API: {API_URL}") print(f"tmp dir convention: /tmp/prefect-worker-test-*") try: test_worker_executes_deployment() except AssertionError as e: print(f"\n✗ FAILED: {e}") return 1 except Exception as e: print(f"\n✗ UNEXPECTED ERROR: {type(e).__name__}: {e}") return 1 print("\n=== all worker tests passed ===") return 0 if __name__ == "__main__": sys.exit(main())