diff --git a/test/perf/albedo_perf_probe.erl b/test/perf/albedo_perf_probe.erl index acd6dd80..e103fa3d 100644 --- a/test/perf/albedo_perf_probe.erl +++ b/test/perf/albedo_perf_probe.erl @@ -1,6 +1,6 @@ %% Loaded only by the external performance harness through the inspect node. -module(albedo_perf_probe). --export([start/0, snapshot/0, peak/0, stop/0]). +-export([start/0, snapshot/0, peak/0, stop/0, kernels/0]). start() -> Pid = spawn(fun() -> sampler(0) end), @@ -26,11 +26,16 @@ snapshot() -> Sessions = [{Id, P} || P <- processes(), {albedo_session, Id} <- [proc_lib:get_label(P)]], Owned = maps:from_list([{Id, owned(P)} || {Id, P} <- Sessions]), + json:encode(#{beam => maps:from_list(erlang:memory()), sessions => Owned, kernels => held()}). + +%% Held session kernels by session id, without a GC. +kernels() -> json:encode(held()). + +held() -> [Name] = [N || N <- registered(), lists:prefix("albedo_registry", atom_to_list(N))], {ok, Runtime} = 'albedo@daemon@registry':host('gleam@erlang@process':named_subject(Name)), - Kernels = maps:from_list([{Id, Pid} || {Id, Kernel} <- 'albedo@harness@runtime':held_kernels(Runtime), - {ok, Pid} <- ['albedo@harness@runtime':kernel_pid(Kernel)]]), - json:encode(#{beam => maps:from_list(erlang:memory()), sessions => Owned, kernels => Kernels}). + maps:from_list([{Id, Pid} || {Id, Kernel} <- 'albedo@harness@runtime':held_kernels(Runtime), + {ok, Pid} <- ['albedo@harness@runtime':kernel_pid(Kernel)]]). owned(Owner) -> Pids = [P || P <- processes(), owns(Owner, P)], diff --git a/test/perf/budgets.toml b/test/perf/budgets.toml index f53c636f..d60d9d36 100644 --- a/test/perf/budgets.toml +++ b/test/perf/budgets.toml @@ -31,6 +31,18 @@ floor = 0 kind = "bytes" ceiling = 4096 floor = 0 +[metrics.kernel_rest_uss] +kind = "memory" +ceiling = 100000000 +floor = 1048576 +[metrics.kernel_cell_slope] +kind = "memory" +ceiling = 1048576 +floor = 16384 +[metrics.kernel_pool_pss] +kind = "memory" +ceiling = 200000000 +floor = 1048576 [metrics.attach_bytes] kind = "bytes" ceiling = 1000000 diff --git a/test/perf/measurement.py b/test/perf/measurement.py index 8eb5ca80..ac407902 100644 --- a/test/perf/measurement.py +++ b/test/perf/measurement.py @@ -89,6 +89,10 @@ class Measurement: ] = by_pid[pid]["uss"] return snapshot + def kernels(self): + """Held session kernels by session id, and the tree's processes.""" + return self.probe.call_json("kernels"), tree(self.app) + def peaks(self): beam = self.probe.call_json("peak")["total"] with self.lock: diff --git a/test/perf/record_container.py b/test/perf/record_container.py index 256cc825..327dcb81 100644 --- a/test/perf/record_container.py +++ b/test/perf/record_container.py @@ -255,8 +255,8 @@ def launch(args): "--provider", "bonsai", ] - if args.scenario: - runner += ["--scenario", args.scenario] + for scenario in args.scenario or []: + runner += ["--scenario", scenario] command += [ "--entrypoint", clean_entry[0], @@ -399,7 +399,9 @@ def main(): parser.add_argument("--pids", type=int, default=4096) parser.add_argument("--timeout", type=int, default=3600) parser.add_argument( - "--scenario", help="one diagnostic scenario; omit to record every scenario" + "--scenario", + action="append", + help="record only this scenario (repeatable); omit to record every scenario", ) parser.add_argument("--inside", nargs=2, help=argparse.SUPPRESS) parser.add_argument("--build", type=Path, help=argparse.SUPPRESS) diff --git a/test/perf/run.py b/test/perf/run.py index c898e2ec..56eaa9b1 100644 --- a/test/perf/run.py +++ b/test/perf/run.py @@ -11,7 +11,14 @@ import subprocess import sys from compare import compare, load -from scenarios import NAMES, WARMUP_TURNS, harness, run_scenario +from scenarios import ( + KERNEL_WARMUP_CELLS, + NAMES, + WARMUP_TURNS, + harness, + pool_processes, + run_scenario, +) from transport import Pacer from safety import SecretScanner import resources @@ -84,6 +91,55 @@ def warmup_retained(runs): } +def held_kernel(point, session): + """The process of the kernel a session holds at a checkpoint.""" + pid = point["kernels"][session] + return next(item for item in point["processes"] if item["pid"] == pid) + + +def kernel_uss(run, label, count): + point = next( + p for p in run["checkpoints"] if (p["label"], p["count"]) == (label, count) + ) + session = next(iter(point["kernels"])) + return held_kernel(point, session)["uss"] + + +def pool_report(runs): + """Pool cost when warm, and one session kernel started cold vs adopted; not gated.""" + + def at(run, label): + return next(p for p in run["checkpoints"] if p["label"] == label) + + def kernel(run, label, which): + item = held_kernel(at(run, label), run["pool_sessions"][which]) + return item + + return { + "warm_pss": summary( + [ + sum( + p["pss"] + for p in pool_processes( + at(run, "warm")["kernels"], at(run, "warm")["processes"] + ) + ) + for run in runs + ] + ), + "first_kernel": { + "kinds": [kernel(run, "cold", "first")["kind"] for run in runs], + "uss": summary([kernel(run, "cold", "first")["uss"] for run in runs]), + "pss": summary([kernel(run, "cold", "first")["pss"] for run in runs]), + }, + "adopted_kernel": { + "kinds": [kernel(run, "adopted", "second")["kind"] for run in runs], + "uss": summary([kernel(run, "adopted", "second")["uss"] for run in runs]), + "pss": summary([kernel(run, "adopted", "second")["pss"] for run in runs]), + }, + } + + def derived(runs): metrics = {} @@ -157,6 +213,34 @@ def derived(runs): ], ) add("attach_bytes", [run["wire"]["attach_received"] for run in runs["long"]]) + if runs["kernel"][0]["checkpoints"]: + add("kernel_rest_uss", [kernel_uss(run, "turn", 1) for run in runs["kernel"]]) + add( + "kernel_cell_slope", + [ + slope( + [ + (point["count"], kernel_uss(run, "turn", point["count"])) + for point in run["checkpoints"] + if point["label"] == "turn" + and point["count"] > 1 + KERNEL_WARMUP_CELLS + ] + ) + for run in runs["kernel"] + ], + ) + add( + "kernel_pool_pss", + [ + sum( + item["pss"] + for item in pool_processes(point["kernels"], point["processes"]) + ) + for run in runs["pool"] + for point in run["checkpoints"] + if point["label"] == "rest" + ], + ) return metrics @@ -255,6 +339,8 @@ def main(): result["scenarios"][name]["warmup_retained"] = warmup_retained( raw[name] ) + if name == "pool" and raw[name][0]["checkpoints"]: + result["scenarios"][name]["pool"] = pool_report(raw[name]) args.out.parent.mkdir(parents=True, exist_ok=True) scanner.write_json(args.out, result) if not args.scenario: diff --git a/test/perf/scenarios.py b/test/perf/scenarios.py index 55b3baf9..de2cdadd 100644 --- a/test/perf/scenarios.py +++ b/test/perf/scenarios.py @@ -18,7 +18,7 @@ from harness import Albedo, Daemon, operation_id # noqa: E402 from measurement import Measurement # noqa: E402 from transport import Recordings, TCPProxy # noqa: E402 -NAMES = ("boot", "chat", "coding", "long", "fan-out", "agents") +NAMES = ("boot", "chat", "coding", "long", "fan-out", "agents", "kernel", "pool") # long fills one-time caches (the SQLite writer page cache) for this many turns # before the gated per-turn slope starts; LONG_TURNS - WARMUP_TURNS are gated. WARMUP_TURNS = 20 @@ -35,6 +35,53 @@ CODES = ( "print('coding complete')", ) +# kernel: one trivial cell (the kernel at rest), KERNEL_WARMUP_CELLS cells of +# ordinary work, then KERNEL_CELLS more whose retained kernel USS is gated. +KERNEL_WARMUP_CELLS = 5 +KERNEL_CELLS = 30 +KERNEL_WORK = ( + "values = [i * i for i in range(1000)]\nprint(sum(values))", + "import statistics\nprint(statistics.mean(range(1000)))", + "job = run('python3', '-c', 'print(7)'); print(await job.text())", + "print(files.read('calc.py'))", + "note = {'items': list(range(10))}\nprint(len(note['items']))", +) +# pool runs the product's kernel pool with its default sizes; only its timers +# (2 min idle release, 1 min spare cool-down) run this many times faster. +POOL_TIME_SCALE = "20" +POOL_WAIT_SECONDS = 60 + + +def cell(code): + return ( + "Execute exactly this Python cell once using your python tool, then reply " + "done. Do not do anything else:\n```python\n" + code + "\n```" + ) + + +def await_pool(meter, condition, what): + """Wait for the kernel pool to reach a state the product promises.""" + deadline = time.monotonic() + POOL_WAIT_SECONDS + while True: + held, processes = meter.kernels() + if condition(held, processes): + return + if time.monotonic() > deadline: + raise AssertionError( + f"kernel pool never reached: {what}: {held} {processes}" + ) + time.sleep(0.2) + + +def pool_processes(held, processes): + """Zygote, spares and forks that no session holds.""" + kernels = set(held.values()) + return [ + item + for item in processes + if item["kind"] in ("zygote", "spare", "forked") and item["pid"] not in kernels + ] + def measured_read(app, proxy, path): before = proxy.totals() @@ -84,6 +131,7 @@ def run_scenario( cleanup.callback(daemon.shutdown) meter = api = None checkpoints, turns, wire, requests = [], [], {}, [] + pool_sessions = {} started = time.monotonic() def prepare(app): @@ -107,6 +155,8 @@ def run_scenario( ALBEDO_KERNEL_GRACE_SECONDS="30", ERL_FLAGS=harness.TEST_VM_FLAGS, ) + if name == "pool": + app.env["ALBEDO_TEST_TIME_SCALE"] = POOL_TIME_SCALE if executable: app.env["ALBEDO_DAEMON"] = executable if cli: @@ -167,22 +217,22 @@ def run_scenario( if name == "boot": app.session() checkpoint("idle", 1) - elif name in ("chat", "long", "coding"): + elif name in ("chat", "long", "coding", "kernel"): session = app.session() - if name == "coding": + if name in ("coding", "kernel"): (app.workspace / "calc.py").write_text( "def add(a, b):\n return a - b\n" ) (app.workspace / "test_calc.py").write_text( "import unittest\nfrom calc import add\nclass TestAdd(unittest.TestCase):\n def test_add(self):\n self.assertEqual(add(2, 3), 5)\n" ) - prompts = [ - "Execute exactly this Python cell once using your python tool, then reply done. Do not do anything else:\n```python\n" - + code - + "\n```" - for code in CODES + prompts = [cell(code) for code in CODES] + if name == "kernel": + prompts = [cell("print('ready')")] + [ + cell(KERNEL_WORK[i % len(KERNEL_WORK)]) + for i in range(KERNEL_WARMUP_CELLS + KERNEL_CELLS) ] - else: + elif name != "kernel": prompts = [ f"Turn {i + 1}. {SHORT}" for i in range(LONG_TURNS if name == "long" else 5) @@ -191,6 +241,14 @@ def run_scenario( for i, prompt in enumerate(prompts, 1): turn(session, prompt, i) checkpoint("turn", i) + if ( + name == "kernel" + and not record + and session not in checkpoints[-1]["kernels"] + ): + raise AssertionError( + f"session released its kernel after cell {i}" + ) if ( name == "coding" and "return a + b" not in (app.workspace / "calc.py").read_text() @@ -227,8 +285,9 @@ def run_scenario( elif name == "fan-out": for i in range(1, 21): turn(app.session(), SHORT, i) - if i in (1, 5, 20): - checkpoint("sessions", i) + # Every session: daemon PSS moves in allocator-carrier steps + # of a few MB, so a three-point slope is noisy. + checkpoint("sessions", i) elif name == "agents": parent = app.session() turn(parent, SHORT, 1) @@ -253,6 +312,43 @@ def run_scenario( app.idle(child, timeout=180) app.idle(parent, timeout=180) checkpoint("family", 4) + elif name == "pool": + assert meter is not None or record + first = app.session() + turn(first, cell("print('cold')"), 1) + checkpoint("cold", 1) + if meter: + await_pool( + meter, + lambda held, _: first not in held, + "first kernel released", + ) + turn(first, cell("print('wake')"), 2) + if meter: + await_pool( + meter, + lambda held, procs: any( + p["kind"] in ("spare", "forked") + and p["pid"] not in held.values() + for p in procs + ), + "spares warmed by a wake", + ) + checkpoint("warm", 2) + second = app.session() + turn(second, cell("print('adopted')"), 3) + checkpoint("adopted", 3) + if meter: + await_pool( + meter, + lambda held, procs: ( + not held + and not any(p["kind"] in ("spare", "forked") for p in procs) + ), + "kernels released and spares gone", + ) + checkpoint("rest", 3) + pool_sessions.update(first=first, second=second) checkpoint("final", len(turns)) provider_totals = recording.proxy.totals() provider_rows = [ @@ -280,6 +376,7 @@ def run_scenario( "turns": turns, "wire": wire, "requests": requests, + "pool_sessions": pool_sessions, } finally: cleanup.close()