"""Render schema-v1 performance results without recomputing measurements."""
from __future__ import annotations
import argparse
import html
import json
from pathlib import Path
import re
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from safety import SecretScanner # noqa: E402
from records import COLLECTION, DID, LIVE_COLLECTION # noqa: E402
SCANNER = SecretScanner()
def write_file(path, content):
SCANNER.check(content, label=str(path))
if isinstance(content, bytes):
Path(path).write_bytes(content)
else:
Path(path).write_text(content)
def read_result(path):
content = Path(path).read_text()
SCANNER.check(content, label=str(path))
result = json.loads(content)
if result["schema"] != 1 or result["mode"] not in ("live", "replay"):
raise ValueError(f"unsupported performance result: {path}")
if not re.fullmatch(r"[0-9a-f]{40}", result["commit"]):
raise ValueError("commit must be a full lowercase SHA")
return result
def esc(value):
return html.escape(str(value), quote=True)
def number(value):
return "—" if value is None else f"{value:,.2f}".rstrip("0").rstrip(".")
def table(headers, rows):
head = "".join(f"
{esc(h)}
" for h in headers)
body = "".join(
"
"
+ "".join(
f'
{esc(v)}
'
for h, v in zip(headers, row, strict=True)
)
+ "
"
for row in rows
)
return f"
{head}
{body}
"
ECHARTS = "https://cdn.jsdelivr.net/npm/echarts@6.1.0/dist/echarts.min.js"
# sha384 of dist/echarts.min.js from the npm tarball, checked against the
# registry's sha512 and against jsDelivr's copy (byte-identical).
ECHARTS_SRI = "sha384-C2iskrW/uPW46KzOjrvJIQo4YkV8lkD+QS0CrDN18IIPIpT/g2USu8bTP3nvmIAD"
def summary(result):
"""The numbers pages compare across runs, small enough for every record."""
return {
"gated": result["gated"],
"scenarios": {
name: {
"memory": {m: v["median"] for m, v in scenario["memory"].items()},
"wire": scenario["wire"],
}
for name, scenario in result["scenarios"].items()
},
}
TILES = (
("boot_tree_uss", "Memory at boot"),
("idle_session_tree_uss", "Memory per idle session"),
("retained_turn_beam", "Memory kept per turn"),
("turn_peak_tree_uss", "Peak during a turn"),
("request_bytes_per_turn", "Sent to the model per turn"),
("attach_bytes", "Sent to open a long session"),
)
def size(value):
"""Bytes in decimal units with three significant figures."""
if value is None:
return "-"
for unit, scale in (("GB", 1e9), ("MB", 1e6), ("KB", 1e3)):
if abs(value) >= scale:
return f"{value / scale:.3g} {unit}"
return f"{value:.0f} B"
def median_observation(scenario):
"""The run whose final memory is the median, so its timeline is typical."""
runs = [o for o in scenario.get("observations", []) if o.get("checkpoints")]
if not runs:
return {"checkpoints": [], "turns": []}
finals = sorted(
range(len(runs)), key=lambda i: private(runs[i]["checkpoints"][-1]) or 0
)
return runs[finals[len(finals) // 2]]
def private(snapshot):
"""A snapshot's private memory (USS), or its PSS from results that predate USS."""
return snapshot.get("tree_uss", snapshot.get("tree_pss"))
def median(value):
"""A plain number, or the median of a {median, mad, samples} distribution."""
return value.get("median") if isinstance(value, dict) else value
def pool_report(pool):
"""The pool scenario's warm pool and the first and adopted kernel."""
if not pool:
return None
kernels = {
name: {
"kinds": sorted(set(pool[name].get("kinds", []))),
"uss": median(pool[name].get("uss")),
"pss": median(pool[name].get("pss")),
}
for name in ("first_kernel", "adopted_kernel")
if name in pool
}
return {"warmPss": median(pool.get("warm_pss")), **kernels}
def timeline(scenario):
"""What the scenario charts draw: settled memory and per-request bytes."""
observation = median_observation(scenario)
return {
"checkpoints": [
{
"label": c["label"],
"count": c["count"],
"memory": private(c),
}
for c in observation["checkpoints"]
],
# Keyed by turn number: checkpoints are sparse, and not every turn
# carries a peak.
"peaks": {
str(t["turn"]): t.get("peak_tree_uss", t.get("peak_tree_pss"))
for t in observation.get("turns", [])
if "peak_tree_uss" in t or "peak_tree_pss" in t
},
"requests": [
{"sent": r["sent"], "received": r["received"]}
for r in scenario.get("requests", [])
],
"wire": scenario.get("wire", {}),
"memory": {m: v["median"] for m, v in scenario.get("memory", {}).items()},
"wallMs": scenario.get("wall_ms", {}).get("median"),
"pool": pool_report(scenario.get("pool")),
"runs": scenario.get("runs"),
}
def page_data(result):
"""Everything the page script needs about this run, without raw samples."""
recordings = {
name: {
"recordedAt": scenario["recordings"]["recorded_at"],
"model": scenario["recordings"]["model"],
"stale": bool(scenario["recordings"]["stale_turns"]),
}
for name, scenario in result["scenarios"].items()
if scenario.get("recordings")
}
return {
"commit": result["commit"],
"baseCommit": result["base_commit"],
"ref": result["ref"],
"event": result["event"],
"verdict": result["verdict"],
"gated": {
name: {
**{
k: m[k] for k in ("value", "noise", "baseline", "budget", "verdict")
},
"singleSample": bool(m.get("single_sample")),
}
for name, m in result["gated"].items()
},
"scenarios": {n: timeline(s) for n, s in result["scenarios"].items()},
"recordings": recordings,
"warmup": warmup(result),
}
def warmup(result):
"""The long scenario's one-time warm-up, when the result carries it."""
measured = result["scenarios"].get("long", {}).get("warmup_retained")
if not measured:
return None
private = measured.get("tree_uss") or measured["tree_pss"]
return {"turns": measured["turns"], "memory": private["median"]}
def fallback(result):
"""The gate's numbers for readers without JavaScript."""
rows = "".join(
f"
{esc(label)}
{esc(size(result['gated'][key]['value']))}
"
f"
{esc(result['gated'][key]['verdict'])}
"
for key, label in TILES
if key in result["gated"]
)
return (
f""
)
def page(title, body, own=None):
scripts = ""
if own is not None:
payload = json.dumps(own, separators=(",", ":")).replace("<", "\\u003c")
scripts = (
f''
f''
''
)
return f"""
{esc(title)} · albedo perf
{body}{scripts}"""
SKELETON = """
Loading.
What moved
Scenarios
History on main
"""
def build_run_site(directory, result, own):
"""Write one run's immutable site: its page, its data and its assets."""
directory = Path(directory)
(directory / "data").mkdir(parents=True, exist_ok=True)
SCANNER.write_json(directory / "data/run.json", result)
own = {**own, "liveCollection": LIVE_COLLECTION, **page_data(result)}
body = SKELETON.replace("", fallback(result) + "")
write_file(
directory / "index.html",
page(f"{result['commit'][:7]} · {result['ref']}", body, own),
)
for asset in ("style.css", "app.js"):
write_file(directory / asset, Path(__file__).with_name(asset).read_bytes())
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Render one run's site for a local preview."
)
parser.add_argument("site", type=Path)
parser.add_argument("result", type=Path)
parser.add_argument("--did", default=DID, help="repo to read runs from")
parser.add_argument("--pds", help="PDS used when the DID does not resolve")
parser.add_argument("--created-at", help="when the run was published (ISO 8601)")
args = parser.parse_args()
result = read_result(args.result)
build_run_site(
args.site,
result,
{
"did": args.did,
"collection": COLLECTION,
"rkey": result["commit"],
"pds": args.pds,
"createdAt": args.created_at,
},
)