"""Measure common tasks using real bonsai recordings and private snapshot daemons.""" from __future__ import annotations import argparse import os from pathlib import Path import platform import statistics import subprocess import sys import time from concurrent.futures import ThreadPoolExecutor from compare import compare, load from scenarios import ( KERNEL_WARMUP_CELLS, NAMES, WARMUP_TURNS, harness, pool_processes, run_scenario, ) from safety import SecretScanner import resources def summary(values): middle = statistics.median(values) return { "median": middle, "mad": statistics.median(abs(value - middle) for value in values), "samples": values, } def slope(points): center = statistics.mean(x for x, _ in points) average = statistics.mean(y for _, y in points) return sum((x - center) * (y - average) for x, y in points) / sum( (x - center) ** 2 for x, _ in points ) def robust_slope(points): """Theil-Sen: the median of every pairwise slope. A binary a socket or cache holds across one checkpoint moves BEAM memory 130 KB for that point alone, which a least-squares slope over 20 points cannot ignore.""" return statistics.median( (y2 - y1) / (x2 - x1) for i, (x1, y1) in enumerate(points) for x2, y2 in points[i + 1 :] if x2 != x1 ) def beam_kept(point): """BEAM memory at a checkpoint without empty process heap space, which a GC resizes in steps, and without binaries no process references (albedo_perf_probe:heap_slack/0 and owned_binary/0).""" beam = point["beam"] unowned = beam["binary"] - beam.get("owned_binary", beam["binary"]) return beam["total"] - beam.get("heap_slack", 0) - unowned def memory_metrics(run): checkpoints = run["checkpoints"] if not checkpoints: return {} final = checkpoints[-1] result = { "retained_tree_pss": final["tree_pss"], "retained_tree_uss": final["tree_uss"], } result.update({"beam_" + key: value for key, value in final["beam"].items()}) result["session_process_bytes"] = sum( item["bytes"] for item in final["sessions"].values() ) result["kernel_uss"] = sum( item["uss"] for item in final["processes"] if item["kind"] in ("kernel", "forked") ) return result # SQLite copies WAL frames into the database at checkpoints, which a timer # starts: how often a page is copied depends on when. Every other file's bytes # follow from what the daemon wrote. DATABASE = "home/albedo.sqlite" def appended(point): """Bytes write calls gave every file but the database, by a checkpoint.""" return sum( size for path, size in point["written_files"].items() if path != DATABASE ) def write_window(name, run): """The checkpoints a scenario's writes are counted between: long after warm-up, idle over its hour, any other from boot to the end.""" points = run["checkpoints"] if name == "long": first = next( p for p in points if (p["label"], p["count"]) == ("turn", WARMUP_TURNS) ) elif name == "idle": first = next(p for p in points if p["label"] == "idle") else: first = points[0] return first, points[-1] def written_files(first, last): """Bytes written per file between two checkpoints.""" changes = { path: size - first["written_files"].get(path, 0) for path, size in last["written_files"].items() } return {path: size for path, size in changes.items() if size} def write_report(name, runs): """What the tree wrote over the scenario's window, and to which files (each file's median over the runs, largest first).""" windows = [write_window(name, run) for run in runs] files = [written_files(first, last) for first, last in windows] paths = {path for row in files for path in row} medians = { path: statistics.median(row.get(path, 0) for row in files) for path in paths } return { "from": [windows[0][0]["label"], windows[0][0]["count"]], "written": summary( [ last["writes"]["written"] - first["writes"]["written"] for first, last in windows ] ), "boot": summary([run["checkpoints"][0]["writes"]["written"] for run in runs]), "files": dict(sorted(medians.items(), key=lambda item: -item[1])), } def aggregate(name, runs): memory = [memory_metrics(run) for run in runs] return { "runs": len(runs), "wall_ms": summary([run["wall_ms"] for run in runs]), "memory": {key: summary([row[key] for row in memory]) for key in memory[0]}, "wire": runs[0]["wire"], "requests": runs[0]["requests"], "recordings": runs[0]["recordings"], "observations": runs, } def warmup_retained(runs): """Memory after the warm-up turns minus before the first turn; not gated.""" def change(run, value): points = { (point["label"], point["count"]): point for point in run["checkpoints"] } return value(points[("turn", WARMUP_TURNS)]) - value(points[("created", 0)]) return { "turns": WARMUP_TURNS, "tree_uss": summary([change(run, lambda p: p["tree_uss"]) for run in runs]), "beam": summary([change(run, beam_kept) for run in runs]), } def held_kernel(point, session): """The process of the kernel a session holds at a checkpoint.""" pid = point["kernels"][session] return next(item for item in point["processes"] if item["pid"] == pid) def checkpoint_at(run, label, count): return next( p for p in run["checkpoints"] if (p["label"], p["count"]) == (label, count) ) def kernel_uss(run, label, count, field="uss"): point = checkpoint_at(run, label, count) session = next(iter(point["kernels"])) return held_kernel(point, session)[field] def python_anon(run, label, count): """Pss_Anon of every Python process at a checkpoint: the session's kernel and whatever pool (zygote, spares) it came with.""" point = checkpoint_at(run, label, count) return sum(p["pss_anon"] for p in point["processes"] if p["kind"] != "daemon") def pool_report(runs): """Pool cost when warm, and one session kernel started cold vs adopted; not gated.""" def at(run, label): return next(p for p in run["checkpoints"] if p["label"] == label) def kernel(run, label, which): item = held_kernel(at(run, label), run["pool_sessions"][which]) return item return { "warm_pss": summary( [ sum( p["pss"] for p in pool_processes( at(run, "warm")["kernels"], at(run, "warm")["processes"] ) ) for run in runs ] ), "first_kernel": { "kinds": [kernel(run, "cold", "first")["kind"] for run in runs], "uss": summary([kernel(run, "cold", "first")["uss"] for run in runs]), "pss": summary([kernel(run, "cold", "first")["pss"] for run in runs]), }, "adopted_kernel": { "kinds": [kernel(run, "adopted", "second")["kind"] for run in runs], "uss": summary([kernel(run, "adopted", "second")["uss"] for run in runs]), "pss": summary([kernel(run, "adopted", "second")["pss"] for run in runs]), }, } def derived(runs): metrics = {} def add(name, values): stats = summary(values) metrics[name] = { "value": stats["median"], "noise": stats["mad"], "samples": stats["samples"], } if len(values) == 1: metrics[name]["single_sample"] = True if runs["boot"][0]["checkpoints"]: add( "boot_tree_uss", [run["checkpoints"][0]["tree_uss"] for run in runs["boot"]] ) # USS, not PSS: PSS splits shared library pages with whatever other # process maps the same files (another BEAM on the host moved it 8 MB). for label, field in (("tree_uss", "tree_uss"), ("beam", "beam")): def value(point): return beam_kept(point) if field == "beam" else point[field] fit = robust_slope if field == "beam" else slope add( "idle_session_" + label, [ fit( [ (point["count"], value(point)) for point in run["checkpoints"] if point["label"] == "sessions" ] ) for run in runs["fan-out"] ], ) add( "retained_turn_" + label, [ fit( [ (point["count"], value(point)) for point in run["checkpoints"] if point["label"] == "turn" and point["count"] >= WARMUP_TURNS ] ) for run in runs["long"] ], ) # The daemon's own peak over a turn, kernels and jobs left out. Coding's # first turn starts the session's kernel: the daemon's side of that # is kernel_start_daemon_uss, the kernel's own the kernel gates. for metric, field in ( ("turn_peak_daemon_uss", "peak_daemon_uss"), ("turn_peak_beam_blocks", "peak_beam_blocks"), ): add( metric, [ max( turn[field] for name in ("chat", "coding", "long") for turn in runs[name][i]["turns"] if field in turn and (name, turn["turn"]) != ("coding", 1) ) for i in range(len(runs["long"])) ], ) add( "kernel_start_daemon_uss", [ next(t for t in run["turns"] if t["turn"] == 1)["peak_daemon_uss"] for run in runs["coding"] ], ) if runs["boot"][0]["checkpoints"]: add( "disk_written_boot", [run["checkpoints"][0]["writes"]["written"] for run in runs["boot"]], ) # A median, not a slope: a turn that lands on a WAL checkpoint, or # rewrites pages still dirty from the last one, is lumpy. add( "disk_written_per_turn", [ statistics.median( turn["written"] for turn in run["turns"] if turn["turn"] > WARMUP_TURNS ) for run in runs["long"] ], ) gated = [ ( next( p for p in run["checkpoints"] if (p["label"], p["count"]) == ("turn", WARMUP_TURNS) ), [p for p in run["checkpoints"] if p["label"] == "turn"][-1], ) for run in runs["long"] ] add( "disk_appended_per_turn", [ (appended(last) - appended(first)) / (last["count"] - first["count"]) for first, last in gated ], ) # Both ends of the window waited for the WAL's backfill, so this is # every page the gated turns' checkpoints copied into the database. add( "disk_backfilled_per_turn", [ ( last["written_files"].get(DATABASE, 0) - first["written_files"].get(DATABASE, 0) ) / (last["count"] - first["count"]) for first, last in gated ], ) add( "disk_written_idle_hour", [ last["writes"]["written"] - first["writes"]["written"] for first, last in (write_window("idle", run) for run in runs["idle"]) ], ) add( "request_bytes_per_turn", [ statistics.mean(turn["sent"] for turn in run["turns"]) for run in runs["long"] ], ) add( "request_bytes_slope", [ slope([(turn["turn"], turn["sent"]) for turn in run["turns"]]) for run in runs["long"] ], ) add("attach_bytes", [run["wire"]["attach_received"] for run in runs["long"]]) if runs["kernel"][0]["checkpoints"]: add( "kernel_rest_anon", [kernel_uss(run, "turn", 1, "pss_anon") for run in runs["kernel"]], ) add( "kernel_first_anon", [python_anon(run, "turn", 1) for run in runs["kernel"]], ) add( "kernel_forked_anon", [ held_kernel( next(p for p in run["checkpoints"] if p["label"] == "adopted"), run["pool_sessions"]["second"], )["pss_anon"] for run in runs["pool"] ], ) add( "kernel_cell_slope", [ slope( [ ( point["count"], kernel_uss(run, "turn", point["count"], "pss_anon"), ) for point in run["checkpoints"] if point["label"] == "turn" and point["count"] >= 1 + KERNEL_WARMUP_CELLS ] ) for run in runs["kernel"] ], ) add( "kernel_pool_pss", [ sum( item["pss"] for item in pool_processes(point["kernels"], point["processes"]) ) for run in runs["pool"] for point in run["checkpoints"] if point["label"] == "rest" ], ) return metrics def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--mode", choices=("replay", "record"), required=True, help="replay the recordings, or record them anew from bonsai at --url", ) parser.add_argument("--runs", type=int, default=5) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--baseline") parser.add_argument( "--jobs", type=int, help="repetitions of a scenario run at once, each on its own daemon " "(default: all of them)", ) parser.add_argument( "--recordings-dir", type=Path, default=Path(__file__).with_name("recordings") ) parser.add_argument("--url", help="bonsai's OpenAI-compatible base URL (record)") parser.add_argument( "--scenario", choices=NAMES, action="append", help="diagnostic subset; never used by CI", ) args = parser.parse_args() record = args.mode == "record" if args.runs < 1 or (record and args.runs != 1): parser.error("runs must be positive; recording requires --runs 1") if record != bool(args.url): parser.error("--url is required to record and unused in replay") jobs = args.jobs or args.runs if not record: # Five daemons replay at once, beside whatever else the runner has # on the same VM: a prompt's acknowledgement can take longer than the # harness's 20 s when the CPU is contended, and that latency is not # what the gate measures. A request gets the budget a turn has. harness.REQUEST_SECONDS = 180 scanner = SecretScanner() if not record and sys.platform != "linux": parser.error("replay memory gates require Linux /proc") harness.snapshot_daemon() names = args.scenario or NAMES raw = {} result = { "schema": 1, "commit": os.environ.get("PERF_COMMIT") or subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), "base_commit": None, "ref": os.environ.get("TANGLED_REF", os.environ.get("GITHUB_REF", "local")), "event": os.environ.get( "TANGLED_PIPELINE_KIND", os.environ.get("GITHUB_EVENT_NAME", "manual") ), "mode": args.mode, "provider": "bonsai" if record else "bonsai-recording", "runner": { "cpus": resources.cpu_count(), "memory_mib": min( [ int((directory / "memory.max").read_text()) for directory in resources.cgroups() if (directory / "memory.max").exists() and (directory / "memory.max").read_text().strip() != "max" ] or [os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")] ) / 1048576, "kernel": platform.release(), }, "scenarios": {}, "gated": {}, "verdict": "pass", } try: for name in names: print(f"{args.mode} {name} x{args.runs} ({jobs} at a time)", flush=True) started = time.monotonic() def repetition(_index, name=name): return run_scenario(name, args.url, recordings_dir=args.recordings_dir) with ThreadPoolExecutor(max_workers=jobs) as pool: raw[name] = list(pool.map(repetition, range(args.runs))) print(f" {name}: {time.monotonic() - started:.1f} s", flush=True) result["scenarios"][name] = aggregate(name, raw[name]) if name == "long" and raw[name][0]["checkpoints"]: result["scenarios"][name]["warmup_retained"] = warmup_retained( raw[name] ) if name == "pool" and not record and raw[name][0]["checkpoints"]: result["scenarios"][name]["pool"] = pool_report(raw[name]) if raw[name][0]["checkpoints"]: result["scenarios"][name]["writes"] = write_report(name, raw[name]) args.out.parent.mkdir(parents=True, exist_ok=True) scanner.write_json(args.out, result) # A recording is not a measurement: slow bonsai replies release kernels. if not args.scenario and not record: result["gated"] = derived(raw) compare(result, load(args.baseline) if args.baseline else None) scanner.write_json(args.out, result) return int(result["verdict"] == "fail") finally: harness.shutdown() if __name__ == "__main__": raise SystemExit(main())