Something went wrong. Try again.
MCP based ACP agent
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666"""memory — read-only SQL over the agent's own history, returned as DataFrames.
Use memory("sql", statement). Filtering, grouping, ordering and limiting belongin SQL; print only the evidence needed. Legacy list/search modes remain forcompatibility. In particular, search uses an FTS language, not literal substringmatching, and its index does not include tool-call arguments.
The database URI comes from execute's injected identity (register.db_uri), notfrom a guessed config path. Read-only connections protect crow.db againstaccidental writes; they do not sandbox the Python kernel or the filesystem."""
from __future__ import annotations
import asyncioimport jsonfrom contextlib import suppressfrom pathlib import Pathfrom typing import Any
from sqlalchemy import bindparam, textfrom sqlalchemy.exc import SQLAlchemyError
import crow_cli.memory as cm
from .register import db_uri as _rail_db_urifrom .register import subtoolfrom .results import _MEMORY_BYTES, MemoryResult, MemoryToolError
_MODES = ("list", "search", "sql")_ROLES = ("system", "user", "assistant", "tool")
_SESSIONS_LIMIT = 50_MESSAGES_LIMIT = 1000_SEARCH_LIMIT = 20# The byte cap is the real guard, but it only streams for the queries this# module composes — search goes through the memory package's FTS seam, which# materializes its hits. 10k rows at the live db's 6KB average is 60MB, which# is the biggest hole a limit= can dig._MAX_LIMIT = 10_000
_SNIPPET = 200 # last_text on the sessions frame_CALL_ARGS = 80 # chars of a tool call's arguments_EXCERPT = 200 # chars either side of a search match
# Eight columns each, which is not a coincidence: polars' repr elides the# middle of a frame wider than eight (first four, last four), and print(r.df)# is the LLM's whole view of the result._SESSION_COLS = ( "session_id", "last_activity", "msgs", "agents", "cwd", "model", "last_role", "last_text",)_MESSAGE_COLS = ( "id", "agent_idx", "fork_idx", "role", "created_at", "chars", "calls", "text",)_SEARCH_COLS = ( "id", "session_id", "agent_idx", "fork_idx", "role", "created_at", "rank", "excerpt",)
# The engine lives in a mutable CELL, not a module global, for the reason# web.py keeps its browser in one: importlib.reload re-executes this source in# the EXISTING module dict, so `_engine = None` at module level would drop the# handle on every reload() and leak its connection pool — file descriptors on# sqlite, a server session each on postgres._state: dict = globals().setdefault("_state", {})
def _dispose() -> None: """Drop the cached engine (test hygiene, and a db_uri that changed).""" engine = _state.get("engine") if engine is not None: with suppress(Exception): engine.dispose() _state["engine"] = None _state["uri"] = None
def _engine(): """The read-only engine for the injected crow.db, cached per uri.""" uri = _rail_db_uri() if uri is None: raise MemoryToolError( "no database — memory() reads the crow.db that execute's prologue" " injects on the identity rail; outside a kernel, point the rail" " at one with crow_cli.tools.register.begin_cell(db_uri=…)" ) if _state.get("uri") != uri: _dispose() if uri.startswith("sqlite:///"): # mode=ro cannot CREATE, so a missing file is "unable to open # database file" — true, and useless. Name the path instead. path = Path(uri.removeprefix("sqlite:///")) if not path.exists(): raise MemoryToolError( f"no database at {path} — nothing has been remembered yet" ) _state["engine"] = cm.get_ro_engine(uri) _state["uri"] = uri return _state["engine"]
def _stmt(sql: str, params: dict | None = None, expanding: tuple[str, ...] = ()): return text(sql).bindparams( *(bindparam(name, expanding=True) for name in expanding), **(params or {}), )
def _fetch(engine, stmt) -> tuple[list[tuple], list[str], bool]: """(rows, column names, truncated) for one statement, streamed.
Iterated, not fetchall()'d, and breaking out of the loop stops sqlite stepping the query — measured on the live db: a 1MB cap on ``select id, data from messages`` returns 316 rows in 1ms where the same query aggregated to completion takes 511ms. That is what makes the cap a guard rather than a decoration: 499MB of message JSON never has to fit in the kernel to be refused.
The first row is always kept, whatever it costs — one 12.8MB message (the live db's largest) must produce a frame with one row in it, not an empty one that looks like "no matches".
``stmt`` is a TextClause for the queries composed here and a plain string for mode="sql", which goes through exec_driver_sql: the model's statement verbatim, with no bindparam parser between it and the database. text() is careful about quoted literals and comments, but it does scan for ``:name``, and a statement that trips the scan fails with a complaint about SQLAlchemy's parameters ("A value is required for bind parameter 'role'") where the same statement through the driver fails with the database's own ("Incorrect number of bindings supplied") — this mode's contract is that what you wrote is what runs, so the database should be the one to say what is wrong with it. """ rows: list[tuple] = [] size = 0 truncated = False with engine.connect() as conn: conn = conn.execution_options(stream_results=True) result = conn.exec_driver_sql(stmt) if isinstance(stmt, str) else conn.execute(stmt) names = list(result.keys()) for row in result: cost = sum(len(v) if isinstance(v, str) else 8 for v in row) if rows and size + cost > _MEMORY_BYTES: truncated = True break rows.append(tuple(row)) size += cost return rows, names, truncated
def _scalar(engine, stmt) -> int: with engine.connect() as conn: return conn.execute(stmt).scalar() or 0
def _frame(columns: dict[str, list]) -> Any: import polars as pl
return pl.DataFrame(columns)
def _empty(cols: tuple[str, ...]) -> Any: return _frame({c: [] for c in cols})
def _cell(value: Any) -> Any: """A value polars can hold: scalars as themselves, everything else JSON.
Polars infers a Struct from the FIRST row of dicts, and a real messages table breaks that two ways (both measured on 1.44.1): ``content`` is a str in a user row and a list of blocks in an assistant one, which RAISES (``TypeError: unexpected value while building Series of type String``); and a row carrying a key the first row lacked — ``tool_calls`` — does not raise, the key is silently DROPPED from the inferred Struct. A JSON string column builds every time and loses nothing, is searchable as text, answers ``.str.json_path_match("$.content")``, and ``json.loads`` gets the object back out of ``.to_dicts()``. """ if value is None or isinstance(value, (str, int, float, bool, bytes)): return value return json.dumps(value, default=str, ensure_ascii=False)
def _data(raw: Any) -> dict: """The message dict, whichever way the dialect handed it over: raw SQL bypasses the ORM's JSON type, so sqlite returns text and postgres jsonb.""" if isinstance(raw, str): with suppress(ValueError): raw = json.loads(raw) return raw if isinstance(raw, dict) else {"content": str(raw or "")}
def _calls(data: dict) -> str: """``edit({"file_path": "/home/…"), terminal({"command": "pytest …")``.
Without this column a transcript frame is 40% blank rows: an assistant message that only calls tools has no content, and its tool calls are the interesting part of it — which file, which command. Arguments are cut at 80 chars because the frame is a view, and ``sql`` is the whole JSON. """ out = [] for call in data.get("tool_calls") or []: fn = (call or {}).get("function") or {} name = fn.get("name") or "?" args = fn.get("arguments") or "" out.append(f"{name}({args[:_CALL_ARGS]})" if args else name) return ", ".join(out)
def _snippet(raw: Any) -> str: text = " ".join(cm.message_text(_data(raw)).split()) return text[:_SNIPPET] + ("…" if len(text) > _SNIPPET else "")
def _excerpt(text: str, query: str) -> str: """The window around the match — what makes a hit legible.
polars' repr shows ~30 chars of a cell, so a frame of whole messages would print their openings and never the part that matched. Falls back token by token (bm25 ANDs tokens; they need not be adjacent, so the whole query is often not a substring) and then to the head of the message. """ low = text.lower() for needle in (query, *query.split()): needle = needle.strip().lower() if not needle: continue idx = low.find(needle) if idx < 0: continue start = max(0, idx - _EXCERPT) end = min(len(text), idx + len(needle) + _EXCERPT) return ( ("…" if start else "") + text[start:end] + ("…" if end < len(text) else "") ) head = text[: _EXCERPT * 2] return head + ("…" if len(text) > len(head) else "")
def _q_sessions(limit: int, include_forks: bool) -> MemoryResult: engine = _engine() where = "" if include_forks else "WHERE a.fork_idx = 1" and_where = "" if include_forks else "AND a.fork_idx = 1" agg = ( "SELECT a.session_id AS session_id, " # coalesce, not a bare max: a session with no messages would sort # NULL, and NULLs come FIRST on a postgres DESC and LAST on a sqlite # one — the same query, two different answers. "coalesce(max(m.created_at), max(a.created_at)) AS last_activity, " "count(DISTINCT m.id) AS msgs, count(DISTINCT a.agent_id) AS agents " "FROM agents a LEFT JOIN messages m ON m.agent_id = a.agent_id " f"{where} GROUP BY a.session_id ORDER BY last_activity DESC LIMIT :lim" ) rows, _, truncated = _fetch(engine, _stmt(agg, {"lim": limit})) total = _scalar( engine, _stmt( "SELECT count(DISTINCT session_id) FROM agents" + ("" if include_forks else " WHERE fork_idx = 1") ), ) cols = {c: [] for c in _SESSION_COLS} if not rows: return MemoryResult( df=_empty(_SESSION_COLS), subject="sessions", sql=agg, total=total )
sids = [r[0] for r in rows] # Two more queries for the page, not one per session: the shape this # replaces asked each session for its own last message (N+1). detail, _, _ = _fetch( engine, _stmt( "SELECT a.session_id, a.cwd, a.model_identifier FROM agents a " f"WHERE a.session_id IN :sids {and_where} ORDER BY a.agent_idx", {"sids": sids}, ("sids",), ), ) first: dict[str, tuple[str, str]] = {} for sid, cwd, model in detail: first.setdefault(sid, (cwd or "", model or "")) last, _, last_cut = _fetch( engine, _stmt( "SELECT session_id, role, data FROM (" " SELECT a.session_id AS session_id, m.role AS role, m.data AS data," " row_number() OVER (PARTITION BY a.session_id ORDER BY m.id DESC)" " AS rn FROM messages m JOIN agents a ON a.agent_id = m.agent_id" f" WHERE a.session_id IN :sids {and_where}) t WHERE rn = 1", {"sids": sids}, ("sids",), ), ) tail = {sid: (role, _snippet(data)) for sid, role, data in last} for sid, last_activity, msgs, agents in rows: cwd, model = first.get(sid, ("", "")) role, snippet = tail.get(sid, ("", "")) cols["session_id"].append(sid) cols["last_activity"].append(last_activity or "") cols["msgs"].append(msgs) cols["agents"].append(agents) cols["cwd"].append(cwd) cols["model"].append(model) cols["last_role"].append(role or "") cols["last_text"].append(snippet) return MemoryResult( df=_frame(cols), subject="sessions", sql=agg, total=total, truncated=truncated or last_cut, )
def _q_messages( session_id: str, roles: tuple[str, ...] | None, limit: int, include_forks: bool) -> MemoryResult: engine = _engine() # The id is a session_id OR a fork's wire agent_id — the same dual # identity delegation_tool_call_ids matches, because rlm hands back the # fork's wire id (a trunk's wire id IS its bare session id; a fork's is # its agent_id) and memory("list", session_id=…) is the documented way to # collect an async delegation. A typo'd id and a session that said nothing # look identical in an empty frame, and the first is a caller bug — so # neither matching is an error, the same ruling as a blank web query. is_session = bool( _scalar( engine, _stmt("SELECT count(*) FROM agents WHERE session_id = :sid", {"sid": session_id}), ) ) is_agent = not is_session and bool( _scalar( engine, _stmt("SELECT count(*) FROM agents WHERE agent_id = :sid", {"sid": session_id}), ) ) if not is_session and not is_agent: raise MemoryToolError( f"no session or fork {session_id!r} in the database —" " memory('list') lists the sessions there are" ) # A bare session_id scopes to the trunk (fork_idx=1) unless include_forks # folds the whole session in; a wire agent_id IS one fork, so it scopes to # exactly that agent and include_forks is moot. clauses = ["a.agent_id = :sid"] if is_agent else ["a.session_id = :sid"] if is_session and not include_forks: clauses.append("a.fork_idx = 1") params: dict[str, Any] = {"sid": session_id} expanding: tuple[str, ...] = () if roles: clauses.append("m.role IN :roles") params["roles"] = roles expanding = ("roles",) where = " AND ".join(clauses) join = "FROM messages m JOIN agents a ON a.agent_id = m.agent_id " total = _scalar( engine, _stmt(f"SELECT count(*) {join}WHERE {where}", params, expanding), ) sql = ( "SELECT m.id, a.agent_idx, a.fork_idx, m.role, m.created_at, m.data " f"{join}WHERE {where} ORDER BY m.id DESC LIMIT :lim" ) rows, _, truncated = _fetch( engine, _stmt(sql, {**params, "lim": limit}, expanding) ) # The tail, read forwards: DESC + LIMIT is how you ask for the LAST n, and # a transcript that arrives newest-first is one .reverse() away from being # wrong everywhere it is used. rows.reverse() cols = {c: [] for c in _MESSAGE_COLS} for mid, agent_idx, fork_idx, role, created_at, raw in rows: data = _data(raw) body = cm.message_text(data) cols["id"].append(mid) cols["agent_idx"].append(agent_idx) cols["fork_idx"].append(fork_idx) cols["role"].append(role) cols["created_at"].append(created_at or "") cols["chars"].append(len(body)) cols["calls"].append(_calls(data)) cols["text"].append(body) return MemoryResult( df=_frame(cols), subject=session_id, sql=sql, total=total, truncated=truncated, )
def _q_search( query: str, session_id: str | None, roles: tuple[str, ...] | None, limit: int, include_forks: bool,) -> MemoryResult: engine = _engine() hits = cm.search_messages( engine, query, limit=limit, session_id=session_id, roles=list(roles) if roles else None, include_forks=include_forks, ) cols = {c: [] for c in _SEARCH_COLS} for hit in hits: body = cm.message_text(hit["data"]) cols["id"].append(hit["id"]) cols["session_id"].append(hit["session_id"]) cols["agent_idx"].append(hit["agent_idx"]) cols["fork_idx"].append(hit["fork_idx"]) cols["role"].append(hit["role"]) cols["created_at"].append(hit["created_at"] or "") # bm25's own sign: lower is better, and the rows arrive that way. Not # negated into a "score" — a column that invites a descending sort and # then returns the worst hits first is worse than one with an # unfamiliar convention and an honest name. cols["rank"].append(hit["score"]) cols["excerpt"].append(_excerpt(body, query)) subject = query if session_id is None else f"{query} in {session_id}" return MemoryResult(df=_frame(cols), subject=subject)
def _q_sql(statement: str) -> MemoryResult: engine = _engine() rows, names, truncated = _fetch(engine, statement) cols: dict[str, list] = {name: [] for name in names} for row in rows: for name, value in zip(names, row): cols[name].append(_cell(value)) return MemoryResult( df=_frame(cols), subject=statement, sql=statement, truncated=truncated )
def _limit(value: int | None, default: int) -> int: if value is None: return default if value < 0: raise MemoryToolError(f"limit must be >= 0, got {value}") if value > _MAX_LIMIT: raise MemoryToolError( f"limit must be <= {_MAX_LIMIT}, got {value} — the frame is not" " the constraint, kernel RAM is; mode='sql' with your own LIMIT" " and fewer columns goes further" ) return int(value)
def _roles(value: Any) -> tuple[str, ...] | None: if value is None: return None if isinstance(value, str): value = (value,) # a string is a sequence of characters, not of roles roles = tuple(str(r) for r in value) for role in roles: if role not in _ROLES: raise MemoryToolError( f"unknown role {role!r} — expected one of: {', '.join(_ROLES)}" ) return roles
@subtool(tool="memory")async def memory( mode: str, target: str | None = None, session_id: str | None = None, roles: tuple[str, ...] | list[str] | str | None = None, limit: int | None = None, include_forks: bool | None = None,): """Query your history with SQL: ``r = await memory("sql", statement)``.
SQL is the primary interface. Put session/role filters, ordering, joins, and LIMIT in the statement, not in Python keyword arguments. The injected connection selects the right crow.db; do not open a guessed database path.
Args: mode: use "sql". target: one SQL statement. No bound parameters; escape a quote inside a SQL string by doubling it (O'Brien -> 'O''Brien'). session_id, roles, limit, include_forks: legacy list/search options; leave unset for SQL. Those modes remain for compatibility, not the recommended retrieval path.
Returns: MemoryResult: .df (polars DataFrame), .n_rows (returned row count), .sql (executed statement), .text (bounded rendering), .truncated (64MB result cap reached; narrow the query). .total is NOT a SQL match count — use COUNT(*) when you need one.
Only printed output reaches the model. print(r.df) truncates long cells; to read actual content, select a few rows, then print each content value with r.df.iter_rows(named=True). Never dump all history.
Retrieval recipe: Know the session? Read its USER messages first, ordered by m.id; user corrections outrank assistant summaries. Then inspect relevant assistant/tool messages. Discovering a session? Query messages joined to agents, with role/date/text filters and a small LIMIT. An empty result means that query matched nothing, not that work never happened.
For literal text, paths, hyphenated names, and tool-call arguments, scan CAST(m.data AS TEXT): LIKE '%self-improving%' or SQLite's instr(CAST(m.data AS TEXT), 'self-improving') > 0. LIKE treats % and _ as wildcards; instr matches a literal, case-sensitive substring. Narrow by session/role/date when possible; a global scan can be costly. Do not send these strings to legacy search: its FTS query language can reject punctuation, even quoted input. The FTS index also omits tool-call arguments; absence from FTS is not absence from memory.
Core schema (SQLite; discover more with PRAGMA table_info(table_name)): agents: agent_id PK, session_id, agent_idx, fork_idx, cwd, model_identifier, status, created_at. Join messages.agent_id to agents.agent_id. messages has NO session_id column. messages: id PK, agent_id, role, created_at, data (JSON), prompt_tokens, completion_tokens, total_tokens. JSON: json_extract(m.data, '$.content') AS content; also tool_calls, tool_call_id, reasoning_content. Content may itself be a JSON list of text/image blocks, not a plain string. Forks: a.fork_idx=1 is the trunk. Add this filter explicitly if wanted; SQL never hides forks. Compaction generations share a session_id; query the session, not only its newest agent_id. subtool_calls: session_id, agent_id, parent_tool_call_id, tool, mode, args, status, error, created_at — calls made inside execute. Also available: prompts, tasks, task_deliveries, session_tabs. PostgreSQL: use m.data->>'content' instead of json_extract; discover columns through information_schema.columns instead of PRAGMA.
Examples (SQLite): # Find a literal phrase, including matches in tool-call arguments. r = await memory("sql", "SELECT m.id, a.session_id, m.role FROM messages m " "JOIN agents a ON a.agent_id=m.agent_id " "WHERE m.role='user' AND m.data LIKE '%self-improving%' " "ORDER BY m.id DESC LIMIT 5") print(r.df)
# Read the user's brief/corrections; replace SESSION with the found id. r = await memory("sql", "SELECT m.id, json_extract(m.data,'$.content') AS content " "FROM messages m JOIN agents a ON a.agent_id=m.agent_id " "WHERE a.session_id='SESSION' AND m.role='user' " "ORDER BY m.id") for row in r.df.iter_rows(named=True): print(row["id"], row["content"])
# Inspect a matched message in full, not the table's clipped preview. r = await memory("sql", "SELECT data FROM messages WHERE id=123") for row in r.df.iter_rows(named=True): print(row["data"])
Raises: MemoryToolError: missing/invalid SQL, unsupported arguments, or a write rejected by the read-only connection. This protects crow.db, not the filesystem: the kernel itself is not a sandbox. """ if mode not in _MODES: raise MemoryToolError( f"unknown memory mode {mode!r} — expected one of: {', '.join(_MODES)}" ) forks = bool(include_forks) target = (target or "").strip()
if mode == "sql": for name, value in ( ("session_id", session_id), ("roles", roles), ("limit", limit), ("include_forks", include_forks), ): if value is not None: raise MemoryToolError( f"{name}= is for mode='list' and 'search', not 'sql' —" " write it into the statement" ) if not target: raise MemoryToolError("mode='sql' requires target= — the statement to run") return await _run(_q_sql, target)
if target and mode == "list": raise MemoryToolError( "mode='list' takes no target= — it lists sessions, or the messages" " of one when session_id= is given" ) if mode == "search" and not target: raise MemoryToolError( "mode='search' requires target= — the words to look for" ) if session_id is not None: session_id = session_id.strip() if not session_id: raise MemoryToolError("session_id= is blank") kept = _roles(roles) # A sessions list has no role column to filter, so accepting roles= here # would be an argument quietly ignored — the caller asked for something # and got something else with no signal. if kept is not None and mode == "list" and session_id is None: raise MemoryToolError( "roles= filters MESSAGES, and mode='list' without session_id=" " lists sessions — pass session_id= to list a session's messages" )
if mode == "search": return await _run( _q_search, target, session_id, kept, _limit(limit, _SEARCH_LIMIT), forks, ) if session_id is not None: return await _run( _q_messages, session_id, kept, _limit(limit, _MESSAGES_LIMIT), forks ) return await _run(_q_sessions, _limit(limit, _SESSIONS_LIMIT), forks)
async def _run(fn, *args) -> MemoryResult: """Blocking SQLAlchemy off the kernel's event loop, and the database's own errors in the tool's own vocabulary.
to_thread for the reason vision runs cv2 in one: a cell is already inside ipykernel's running loop, and a 500ms full scan on it stalls the kernel's heartbeats with it. """ try: return await asyncio.to_thread(fn, *args) except MemoryToolError: raise except SQLAlchemyError as exc: # SQLAlchemy renders "(sqlite3.OperationalError) <the actual problem>" # on the first line and then the statement, the parameters and a link # to its own docs — the model wrote the statement, it wants the # problem. raise MemoryToolError(str(exc).splitlines()[0]) from None