"""Every public call in the model's Python namespace is named in the text the model is given. Agents otherwise discover the API with dir() and help(), or miss parts of it entirely (files.write, cells.last_id, and bash's timeout were all real but undocumented). The session enables every installed extension but the compaction strategies, so the namespace and the request are the daemon's own: no list of plugins or of instruction sources is kept here. An MCP server is configured so the `mcp` binding loads, and every shipped binding must have loaded.""" import json from pathlib import Path import re import sys import unittest from harness import Albedo, Provider, cell_output, exclusive, python, text ROOT = Path(__file__).resolve().parents[2] PLUGINS = ROOT / "priv/python/albedo_plugins" FAKE_MCP = Path(__file__).with_name("fake_mcp_server.py") MARKER = "API_DOCS_PROBE" # The cell lists what the model must be told about: a label and the pattern # that names it, for each binding, each public member of an object binding, # and each member of a run job handle. INTERNAL is plumbing the kernel or the # remote relay calls, not part of the model's API. CHILD_ONLY is told only to # child agents, in the task instructions agents/extension.gleam gives them. # Members minted from this session's own catalogs, its slash commands and its # MCP servers, are data the model reads from commands.catalog() and mcp.tools(). PROBE = r""" import inspect, json, re, sys import albedo INTERNAL = {"run.Job.claim", "agents.Agents.spawn"} CHILD_ONLY = {"agents.progress"} minted = {f"commands.{row['method']}" for row in await commands.catalog()} minted |= { f"mcp.{key}" for key, held in vars(mcp).items() if isinstance(held, sys.modules["albedo_plugins.mcp"].Server) } needed = [] for name, value in albedo.bindings.items(): if isinstance(value, (dict, type)): needed.append([name, re.escape(name)]) continue if inspect.isfunction(value): needed.append([f"{name}(", re.escape(f"{name}(")]) continue kind = type(value) for attribute in dir(kind) + list(vars(value)): owner = f"{kind.__module__.split('.')[-1]}.{kind.__name__}.{attribute}" label = f"{name}.{attribute}" if attribute.startswith("_") or owner in INTERNAL or label in CHILD_ONLY | minted: continue needed.append([label, rf"\b{re.escape(name)}\.{re.escape(attribute)}\b"]) job = sys.modules["albedo_plugins.run"].Job members = [ name for name, _ in inspect.getmembers(job, callable) if not name.startswith("_") and f"run.Job.{name}" not in INTERNAL ] for name in members + ["id", "command", "exit_code", "duration", "timed_out"]: needed.append([f"job.{name}", rf"\bjob\.{name}\b"]) # The modules the kernel set up, as the daemon passed them (albedo_cells.open_link). loaded = json.loads(sys.argv[3] if sys.argv[1:2] == ["--run"] else sys.argv[1]) print(json.dumps({"needed": needed, "loaded": loaded})) """ def shipped_bindings(): """Every shipped plugin module with a `setup`: what some extension must declare for the kernel to load it.""" names = [] for path in sorted(PLUGINS.iterdir()): source = path / "__init__.py" if path.is_dir() else path if path.name.startswith("_") or source.suffix != ".py" or not source.exists(): continue if re.search(r"^(async )?def setup\(", source.read_text(), re.MULTILINE): names.append(path.stem if path.is_file() else path.name) return names # exclusive: the fixture configures a global MCP server and enables every # extension's binding. @exclusive class ApiDocsTest(unittest.TestCase): maxDiff = None def setUp(self): def reply(request): last = request["input"][-1] if last.get("type") == "function_call_output": return text("done") if MARKER in json.dumps(last.get("content", "")): return python(PROBE) return text("ok") def prepare(app): (app.home / "extensions.json").write_text( json.dumps( { "models": {"refreshHours": 0}, "mcp": { "servers": { "fake": { "type": "stdio", "command": sys.executable, "args": [str(FAKE_MCP)], } } }, } ) ) self.provider = Provider(reply) self.addCleanup(self.provider.close) self.app = Albedo(self.provider, protocol="responses", prepare=prepare) self.app.__enter__() self.addCleanup(self.app.__exit__, None, None, None) def catalog(self, session): with self.app.api( f"/sessions/{session}/catalog?kind=extensions&limit=200" ) as response: return json.load(response) def extensions(self, catalog): """The installed extensions the daemon runs.""" return [ row for row in catalog["discovery"]["candidates"] if row["kind"] == "extension" and not row["quarantined"] ] def test_every_public_call_is_named_for_the_model(self): session = self.app.session() # A session runs one compaction strategy; none declares a module. enabled = { row["preference_key"]: True for row in self.extensions(self.catalog(session)) if "compaction" not in row["extension"]["plugins"] } self.app.patch_session(session, {"selection": {"extensions": enabled}}) with self.app.api( f"/sessions/{session}/reload", {"target": "session"} ) as response: self.assertEqual(json.load(response)["session"]["state"], "applied") before = len(self.provider.requests) self.app.prompt(session, MARKER).close() self.app.idle(session, timeout=60) requests = [entry["request"] for entry in self.provider.requests[before:]] self.assertEqual(len(requests), 2, requests) given = json.dumps(requests[0]) probe = json.loads( cell_output( next( item["output"] for item in reversed(requests[1]["input"]) if item.get("type") == "function_call_output" ) ) ) self.assertEqual( sorted(set(shipped_bindings()) - set(probe["loaded"])), [], "shipped bindings that no extension loaded", ) missing = [ label for label, pattern in probe["needed"] if not re.search(pattern, given) ] self.assertEqual( sorted(set(missing)), [], "public API the model is never told about" ) if __name__ == "__main__": unittest.main()