Something went wrong. Try again.
small gleam coding and (not yet) persistent agent daemon with a detachable cli forked from okami.mom/albedo
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545"""Measure common tasks using real bonsai recordings and private snapshot daemons."""
from __future__ import annotations
import argparseimport osfrom pathlib import Pathimport platformimport statisticsimport subprocessimport sysimport timefrom concurrent.futures import ThreadPoolExecutor
from compare import compare, loadfrom scenarios import ( KERNEL_WARMUP_CELLS, NAMES, WARMUP_TURNS, harness, pool_processes, run_scenario,)from safety import SecretScannerimport resources
def summary(values): middle = statistics.median(values) return { "median": middle, "mad": statistics.median(abs(value - middle) for value in values), "samples": values, }
def slope(points): center = statistics.mean(x for x, _ in points) average = statistics.mean(y for _, y in points) return sum((x - center) * (y - average) for x, y in points) / sum( (x - center) ** 2 for x, _ in points )
def robust_slope(points): """Theil-Sen: the median of every pairwise slope. A binary a socket or cache holds across one checkpoint moves BEAM memory 130 KB for that point alone, which a least-squares slope over 20 points cannot ignore.""" return statistics.median( (y2 - y1) / (x2 - x1) for i, (x1, y1) in enumerate(points) for x2, y2 in points[i + 1 :] if x2 != x1 )
def beam_kept(point): """BEAM memory at a checkpoint without empty process heap space, which a GC resizes in steps, and without binaries no process references (albedo_perf_probe:heap_slack/0 and owned_binary/0).""" beam = point["beam"] unowned = beam["binary"] - beam.get("owned_binary", beam["binary"]) return beam["total"] - beam.get("heap_slack", 0) - unowned
def memory_metrics(run): checkpoints = run["checkpoints"] if not checkpoints: return {} final = checkpoints[-1] result = { "retained_tree_pss": final["tree_pss"], "retained_tree_uss": final["tree_uss"], } result.update({"beam_" + key: value for key, value in final["beam"].items()}) result["session_process_bytes"] = sum( item["bytes"] for item in final["sessions"].values() ) result["kernel_uss"] = sum( item["uss"] for item in final["processes"] if item["kind"] in ("kernel", "forked") ) return result
# SQLite copies WAL frames into the database at checkpoints, which a timer# starts: how often a page is copied depends on when. Every other file's bytes# follow from what the daemon wrote.DATABASE = "home/albedo.sqlite"
def appended(point): """Bytes write calls gave every file but the database, by a checkpoint.""" return sum( size for path, size in point["written_files"].items() if path != DATABASE )
def write_window(name, run): """The checkpoints a scenario's writes are counted between: long after warm-up, idle over its hour, any other from boot to the end.""" points = run["checkpoints"] if name == "long": first = next( p for p in points if (p["label"], p["count"]) == ("turn", WARMUP_TURNS) ) elif name == "idle": first = next(p for p in points if p["label"] == "idle") else: first = points[0] return first, points[-1]
def written_files(first, last): """Bytes written per file between two checkpoints.""" changes = { path: size - first["written_files"].get(path, 0) for path, size in last["written_files"].items() } return {path: size for path, size in changes.items() if size}
def write_report(name, runs): """What the tree wrote over the scenario's window, and to which files (each file's median over the runs, largest first).""" windows = [write_window(name, run) for run in runs] files = [written_files(first, last) for first, last in windows] paths = {path for row in files for path in row} medians = { path: statistics.median(row.get(path, 0) for row in files) for path in paths } return { "from": [windows[0][0]["label"], windows[0][0]["count"]], "written": summary( [ last["writes"]["written"] - first["writes"]["written"] for first, last in windows ] ), "boot": summary([run["checkpoints"][0]["writes"]["written"] for run in runs]), "files": dict(sorted(medians.items(), key=lambda item: -item[1])), }
def aggregate(name, runs): memory = [memory_metrics(run) for run in runs] return { "runs": len(runs), "wall_ms": summary([run["wall_ms"] for run in runs]), "memory": {key: summary([row[key] for row in memory]) for key in memory[0]}, "wire": runs[0]["wire"], "requests": runs[0]["requests"], "recordings": runs[0]["recordings"], "observations": runs, }
def warmup_retained(runs): """Memory after the warm-up turns minus before the first turn; not gated."""
def change(run, value): points = { (point["label"], point["count"]): point for point in run["checkpoints"] } return value(points[("turn", WARMUP_TURNS)]) - value(points[("created", 0)])
return { "turns": WARMUP_TURNS, "tree_uss": summary([change(run, lambda p: p["tree_uss"]) for run in runs]), "beam": summary([change(run, beam_kept) for run in runs]), }
def held_kernel(point, session): """The process of the kernel a session holds at a checkpoint.""" pid = point["kernels"][session] return next(item for item in point["processes"] if item["pid"] == pid)
def checkpoint_at(run, label, count): return next( p for p in run["checkpoints"] if (p["label"], p["count"]) == (label, count) )
def kernel_uss(run, label, count, field="uss"): point = checkpoint_at(run, label, count) session = next(iter(point["kernels"])) return held_kernel(point, session)[field]
def python_anon(run, label, count): """Pss_Anon of every Python process at a checkpoint: the session's kernel and whatever pool (zygote, spares) it came with.""" point = checkpoint_at(run, label, count) return sum(p["pss_anon"] for p in point["processes"] if p["kind"] != "daemon")
def pool_report(runs): """Pool cost when warm, and one session kernel started cold vs adopted; not gated."""
def at(run, label): return next(p for p in run["checkpoints"] if p["label"] == label)
def kernel(run, label, which): item = held_kernel(at(run, label), run["pool_sessions"][which]) return item
return { "warm_pss": summary( [ sum( p["pss"] for p in pool_processes( at(run, "warm")["kernels"], at(run, "warm")["processes"] ) ) for run in runs ] ), "first_kernel": { "kinds": [kernel(run, "cold", "first")["kind"] for run in runs], "uss": summary([kernel(run, "cold", "first")["uss"] for run in runs]), "pss": summary([kernel(run, "cold", "first")["pss"] for run in runs]), }, "adopted_kernel": { "kinds": [kernel(run, "adopted", "second")["kind"] for run in runs], "uss": summary([kernel(run, "adopted", "second")["uss"] for run in runs]), "pss": summary([kernel(run, "adopted", "second")["pss"] for run in runs]), }, }
def derived(runs): metrics = {}
def add(name, values): stats = summary(values) metrics[name] = { "value": stats["median"], "noise": stats["mad"], "samples": stats["samples"], } if len(values) == 1: metrics[name]["single_sample"] = True
if runs["boot"][0]["checkpoints"]: add( "boot_tree_uss", [run["checkpoints"][0]["tree_uss"] for run in runs["boot"]] ) # USS, not PSS: PSS splits shared library pages with whatever other # process maps the same files (another BEAM on the host moved it 8 MB). for label, field in (("tree_uss", "tree_uss"), ("beam", "beam")):
def value(point): return beam_kept(point) if field == "beam" else point[field]
fit = robust_slope if field == "beam" else slope
add( "idle_session_" + label, [ fit( [ (point["count"], value(point)) for point in run["checkpoints"] if point["label"] == "sessions" ] ) for run in runs["fan-out"] ], ) add( "retained_turn_" + label, [ fit( [ (point["count"], value(point)) for point in run["checkpoints"] if point["label"] == "turn" and point["count"] >= WARMUP_TURNS ] ) for run in runs["long"] ], ) # The daemon's own peak over a turn, kernels and jobs left out. Coding's # first turn starts the session's kernel: the daemon's side of that # is kernel_start_daemon_uss, the kernel's own the kernel gates. for metric, field in ( ("turn_peak_daemon_uss", "peak_daemon_uss"), ("turn_peak_beam_blocks", "peak_beam_blocks"), ): add( metric, [ max( turn[field] for name in ("chat", "coding", "long") for turn in runs[name][i]["turns"] if field in turn and (name, turn["turn"]) != ("coding", 1) ) for i in range(len(runs["long"])) ], ) add( "kernel_start_daemon_uss", [ next(t for t in run["turns"] if t["turn"] == 1)["peak_daemon_uss"] for run in runs["coding"] ], ) if runs["boot"][0]["checkpoints"]: add( "disk_written_boot", [run["checkpoints"][0]["writes"]["written"] for run in runs["boot"]], ) # A median, not a slope: a turn that lands on a WAL checkpoint, or # rewrites pages still dirty from the last one, is lumpy. add( "disk_written_per_turn", [ statistics.median( turn["written"] for turn in run["turns"] if turn["turn"] > WARMUP_TURNS ) for run in runs["long"] ], ) gated = [ ( next( p for p in run["checkpoints"] if (p["label"], p["count"]) == ("turn", WARMUP_TURNS) ), [p for p in run["checkpoints"] if p["label"] == "turn"][-1], ) for run in runs["long"] ] add( "disk_appended_per_turn", [ (appended(last) - appended(first)) / (last["count"] - first["count"]) for first, last in gated ], ) # Both ends of the window waited for the WAL's backfill, so this is # every page the gated turns' checkpoints copied into the database. add( "disk_backfilled_per_turn", [ ( last["written_files"].get(DATABASE, 0) - first["written_files"].get(DATABASE, 0) ) / (last["count"] - first["count"]) for first, last in gated ], ) add( "disk_written_idle_hour", [ last["writes"]["written"] - first["writes"]["written"] for first, last in (write_window("idle", run) for run in runs["idle"]) ], ) add( "request_bytes_per_turn", [ statistics.mean(turn["sent"] for turn in run["turns"]) for run in runs["long"] ], ) add( "request_bytes_slope", [ slope([(turn["turn"], turn["sent"]) for turn in run["turns"]]) for run in runs["long"] ], ) add("attach_bytes", [run["wire"]["attach_received"] for run in runs["long"]]) if runs["kernel"][0]["checkpoints"]: add( "kernel_rest_anon", [kernel_uss(run, "turn", 1, "pss_anon") for run in runs["kernel"]], ) add( "kernel_first_anon", [python_anon(run, "turn", 1) for run in runs["kernel"]], ) add( "kernel_forked_anon", [ held_kernel( next(p for p in run["checkpoints"] if p["label"] == "adopted"), run["pool_sessions"]["second"], )["pss_anon"] for run in runs["pool"] ], ) add( "kernel_cell_slope", [ slope( [ ( point["count"], kernel_uss(run, "turn", point["count"], "pss_anon"), ) for point in run["checkpoints"] if point["label"] == "turn" and point["count"] >= 1 + KERNEL_WARMUP_CELLS ] ) for run in runs["kernel"] ], ) add( "kernel_pool_pss", [ sum( item["pss"] for item in pool_processes(point["kernels"], point["processes"]) ) for run in runs["pool"] for point in run["checkpoints"] if point["label"] == "rest" ], ) return metrics
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--mode", choices=("replay", "record"), required=True, help="replay the recordings, or record them anew from bonsai at --url", ) parser.add_argument("--runs", type=int, default=5) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--baseline") parser.add_argument( "--jobs", type=int, help="repetitions of a scenario run at once, each on its own daemon " "(default: all of them)", ) parser.add_argument( "--recordings-dir", type=Path, default=Path(__file__).with_name("recordings") ) parser.add_argument("--url", help="bonsai's OpenAI-compatible base URL (record)") parser.add_argument( "--scenario", choices=NAMES, action="append", help="diagnostic subset; never used by CI", ) args = parser.parse_args() record = args.mode == "record" if args.runs < 1 or (record and args.runs != 1): parser.error("runs must be positive; recording requires --runs 1") if record != bool(args.url): parser.error("--url is required to record and unused in replay") jobs = args.jobs or args.runs if not record: # Five daemons replay at once, beside whatever else the runner has # on the same VM: a prompt's acknowledgement can take longer than the # harness's 20 s when the CPU is contended, and that latency is not # what the gate measures. A request gets the budget a turn has. harness.REQUEST_SECONDS = 180 scanner = SecretScanner() if not record and sys.platform != "linux": parser.error("replay memory gates require Linux /proc") harness.snapshot_daemon() names = args.scenario or NAMES raw = {} result = { "schema": 1, "commit": os.environ.get("PERF_COMMIT") or subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), "base_commit": None, "ref": os.environ.get("TANGLED_REF", os.environ.get("GITHUB_REF", "local")), "event": os.environ.get( "TANGLED_PIPELINE_KIND", os.environ.get("GITHUB_EVENT_NAME", "manual") ), "mode": args.mode, "provider": "bonsai" if record else "bonsai-recording", "runner": { "cpus": resources.cpu_count(), "memory_mib": min( [ int((directory / "memory.max").read_text()) for directory in resources.cgroups() if (directory / "memory.max").exists() and (directory / "memory.max").read_text().strip() != "max" ] or [os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")] ) / 1048576, "kernel": platform.release(), }, "scenarios": {}, "gated": {}, "verdict": "pass", } try: for name in names: print(f"{args.mode} {name} x{args.runs} ({jobs} at a time)", flush=True) started = time.monotonic()
def repetition(_index, name=name): return run_scenario(name, args.url, recordings_dir=args.recordings_dir)
with ThreadPoolExecutor(max_workers=jobs) as pool: raw[name] = list(pool.map(repetition, range(args.runs))) print(f" {name}: {time.monotonic() - started:.1f} s", flush=True) result["scenarios"][name] = aggregate(name, raw[name]) if name == "long" and raw[name][0]["checkpoints"]: result["scenarios"][name]["warmup_retained"] = warmup_retained( raw[name] ) if name == "pool" and not record and raw[name][0]["checkpoints"]: result["scenarios"][name]["pool"] = pool_report(raw[name]) if raw[name][0]["checkpoints"]: result["scenarios"][name]["writes"] = write_report(name, raw[name]) args.out.parent.mkdir(parents=True, exist_ok=True) scanner.write_json(args.out, result) # A recording is not a measurement: slow bonsai replies release kernels. if not args.scenario and not record: result["gated"] = derived(raw) compare(result, load(args.baseline) if args.baseline else None) scanner.write_json(args.out, result) return int(result["verdict"] == "fail") finally: harness.shutdown()
if __name__ == "__main__": raise SystemExit(main())