// Context compaction: when a thread's compiled context nears the model's window, older // messages are summarized into an ai.languid.conversation.summary record, and the compiled // context becomes: system prompt, the summary, then recent messages verbatim. Messages are // never deleted; the summary only changes what's compiled. // // Borrowed from: // - pi: token-based cut with recent turns kept verbatim; each summary updates the previous // one; chars/4 token estimates; transcript serialized as text (tool results truncated) so // the summarizer doesn't continue the conversation. // - Codex: framing the summary as a handoff from an earlier context, and keeping the user's // own words. // - a summary in the first person with "lookup hints" for evicted details, and a // memory-pressure warning so the agent saves what matters to its blocks before eviction. import type { Message as PiMessage, Model, SystemMessage } from '@earendil-works/pi-ai' import type { AgentConfig } from './records.ts' export const DEFAULT_RESERVE_TOKENS = 16_384 export const DEFAULT_KEEP_RECENT_TOKENS = 20_000 /** Above this share of the compaction threshold, the agent is told to save to its blocks. */ const PRESSURE_SHARE = 0.8 /** Cap on any one message's text inside a summarization request. */ const MAX_MESSAGE_CHARS = 20_000 /** Tool output kept per result when summarizing (pi uses the same). */ const MAX_TOOL_RESULT_CHARS = 2_000 /** An image counts as this many characters (~1200 tokens), as in pi. */ const IMAGE_CHARS = 4_800 const SUMMARY_MAX_TOKENS = 8_192 /** Room to plan for a summary in the context (the prompt asks for under 800 words). */ export const SUMMARY_TOKENS = 1_500 export interface CompactionSettings { contextWindow: number /** Compact when the context is above this many tokens. */ threshold: number /** Warn the agent (memory pressure) above this many tokens. */ pressure: number /** Recent tokens kept verbatim. */ keepRecent: number } /** * Limits for the agent's model. A client running turns on a smaller local model passes its window, * so compaction keeps the context inside it; a client can only shrink the window, never grow it. */ export function compactionSettings(config: AgentConfig, model: Model, clientWindow?: number): CompactionSettings { const own = model.contextWindow || 128_000 const window = clientWindow && clientWindow > 0 ? Math.min(own, clientWindow) : own // Small windows: don't reserve or keep more than a fraction of the window. const reserve = Math.min(config.compaction?.reserveTokens ?? DEFAULT_RESERVE_TOKENS, Math.floor(window * 0.25)) const keepRecent = Math.min(config.compaction?.keepRecentTokens ?? DEFAULT_KEEP_RECENT_TOKENS, Math.floor(window * 0.3)) const threshold = window - reserve return { contextWindow: window, threshold, pressure: Math.floor(threshold * PRESSURE_SHARE), keepRecent } } const contentChars = (content: unknown): number => typeof content === 'string' ? content.length : Array.isArray(content) ? content.reduce((n, c: any) => n + (c.type === 'image' ? IMAGE_CHARS : (c.text ?? c.thinking ?? '').length), 0) : 0 /** Estimated tokens of messages: characters / 4, as pi estimates. Tool definitions count too. */ export function estimateContext(messages: (SystemMessage | PiMessage)[]): number { let chars = 0 for (const m of messages as any[]) { if (m.role === 'system') { chars += contentChars(m.content) + (m.toolsAdded ? JSON.stringify(m.toolsAdded).length : 0) } else if (m.role === 'assistant') { for (const c of m.content) chars += c.type === 'toolCall' ? c.name.length + JSON.stringify(c.arguments).length : contentChars([c]) } else chars += contentChars(m.content) } return Math.ceil(chars / 4) } const text = (content: unknown) => typeof content === 'string' ? content : Array.isArray(content) ? content.filter((c: any) => c.type === 'text').map((c: any) => c.text).join('\n') : '' /** * Messages as a plain-text transcript, so the summarizer reads it rather than continuing it. * Thinking is left out; tool results are truncated. */ export function serializeConversation(messages: PiMessage[]): string { const out: string[] = [] for (const m of messages as any[]) { if (m.role === 'user') out.push(`[User]: ${text(m.content)}`) else if (m.role === 'assistant') { const said = text(m.content) if (said) out.push(`[Assistant]: ${said}`) const calls = m.content.filter((c: any) => c.type === 'toolCall') if (calls.length) out.push(`[Assistant tool calls]: ${calls.map((c: any) => `${c.name}(${JSON.stringify(c.arguments)})`).join('; ')}`) } else if (m.role === 'toolResult') { let r = text(m.content) if (r.length > MAX_TOOL_RESULT_CHARS) r = `${r.slice(0, MAX_TOOL_RESULT_CHARS)}…[${r.length - MAX_TOOL_RESULT_CHARS} chars truncated]` out.push(`[Tool result${m.isError ? ', error' : ''}]: ${r}`) } } return out.join('\n') } /** * Where to cut, given each live message record's estimated size (oldest first): the index of the * first message kept verbatim. Keeps about `keepRecent` tokens, starting at a user message when * one is in range so turns stay whole; a single turn larger than that is cut at an assistant * message. `force` (a manual compact) keeps only the latest turn. 0 means nothing to summarize. */ export function findCut(sizes: number[], roles: ('user' | 'assistant')[], keepRecent: number, force = false): number { if (force) return Math.max(0, roles.lastIndexOf('user')) let i = sizes.length let acc = 0 while (i > 0 && acc < keepRecent) acc += sizes[--i] if (i === 0) return 0 const nextUser = roles.indexOf('user', i) if (nextUser !== -1) return nextUser // The latest turn alone is over budget: cut inside it, at an assistant message. for (let j = i; j < roles.length; j++) if (roles[j] === 'assistant') return j return 0 } /** The summary as it appears at the start of the compiled thread. */ export function summaryMessage(summary: string, messageCount: number, createdAt: string): PiMessage { return { role: 'user', content: `\nThe ${messageCount} earliest messages of this conversation were compacted to fit your context window. ` + `They are still stored in your space; what follows is the summary written when they were evicted. ` + `Build on it rather than redoing work. Your memory blocks are separate and always current.\n\n${summary}\n`, timestamp: Date.parse(createdAt), } } export const SUMMARY_SYSTEM_PROMPT = `You are summarizing an agent's conversation so the agent can keep working after older messages leave its context window. Do NOT continue the conversation. Do NOT answer questions in it. ONLY output the summary.` const SECTIONS = `Write in the first person, as the agent, as a factual record. Use these sections: ## Goals What the user wants, in their own words where possible. Ongoing tasks and long-term aims. ## What happened The exchanges and work so far: what was asked, what I did, how it went. ## Important details Facts, names, decisions and their reasons, preferences, files and paths, URLs, IDs, commands and data needed to continue. Keep identifiers verbatim; don't paraphrase or truncate them. ## Errors and corrections What went wrong and how it was fixed. Quote the user's corrections and feedback verbatim. ## Current state Exactly what was in progress at the end of these messages, and the next step if the user asked for one. ## Lookup hints Topics and key terms for details too long to keep here, so they can be found in the stored messages later. My memory blocks persist separately and are always in my context: don't copy them. Be concise but complete; prefer specifics to generalities. Stay under 800 words.` /** The summarization request: the transcript chunk, the previous summary if any, then instructions. */ export function summaryPrompt(conversation: string, previous?: string, instructions?: string): string { let text = `\n${conversation}\n\n\n` if (previous) { text += `\n${previous}\n\n\n` text += 'The conversation above continues from the previous summary. Write one updated summary covering both: ' + 'keep everything from the previous summary that still matters, add what is new, update the current state.\n\n' } else { text += 'Summarize the conversation above.\n\n' } text += SECTIONS if (instructions) text += `\n\nAlso focus on: ${instructions}` return text } /** * Serialize messages for summarization and split them into chunks that each fit `budget` tokens, * so a span larger than the summarizer's window is summarized in several passes. */ export function serializeChunks(groups: PiMessage[][], budget: number): string[] { const chunks: string[] = [] let current = '' for (const group of groups) { let t = serializeConversation(group) if (t.length > MAX_MESSAGE_CHARS) { const half = MAX_MESSAGE_CHARS / 2 t = `${t.slice(0, half)}\n…[${t.length - MAX_MESSAGE_CHARS} chars omitted]…\n${t.slice(-half)}` } if (current && (current.length + t.length) / 4 > budget) { chunks.push(current) current = '' } current += (current ? '\n\n' : '') + t } if (current) chunks.push(current) return chunks } /** Tokens a summarization request may spend on transcript, for a given summarizer model. */ export function chunkBudget(model: Model): number { const window = model.contextWindow || 128_000 const output = Math.min(model.maxTokens || SUMMARY_MAX_TOKENS, SUMMARY_MAX_TOKENS) // Leave room for the instructions, the previous summary and the output. return Math.max(4_000, Math.floor((window - output - 4_000) * 0.8)) } export const summaryMaxTokens = (model: Model) => Math.min(model.maxTokens || SUMMARY_MAX_TOKENS, SUMMARY_MAX_TOKENS) export const PRESSURE_NOTE = '\nYour context is nearly full. Soon the oldest messages of this conversation will be replaced by a summary. ' + 'If anything in them should outlast that (facts about the user, decisions, ongoing work), save it to your memory blocks now. ' + 'If it is already saved, carry on normally.\n' // ---- idle compaction // // A long thread that goes quiet is summarized while the provider still has its prompt cached, so // the summary request reads the whole thread from the cache, and the next turn starts from a small // context instead of re-sending the long one at full price. /** * Default minutes of quiet before idle compaction, counted from when the thread's last model request * started (when the provider cached its prompt). Providers keep a prompt cached for about five * minutes after its last use (Anthropic's default, OpenAI, Together, DeepSeek); three leaves room * for the sweep's 30-second interval and a cache that expires a little early. */ export const DEFAULT_IDLE_MINUTES = 3 /** Default context size below which a quiet thread is left as it is. */ export const DEFAULT_IDLE_MIN_TOKENS = 100_000 export const IDLE_LIMITS = { minutes: { min: 1, max: 1440 }, minTokens: { min: 10_000, max: 2_000_000 } } export interface IdleSettings { enabled: boolean minutes: number minTokens: number } /** * Idle compaction for a thread whose model is `hosted` (runs on Languid's keys and credit): on by * default there, off for the owner's own keys, unless the agent's config says. minTokens stays * under the compaction threshold, so a thread that would be compacted anyway still counts. */ export function idleSettings(config: Pick, hosted: boolean, threshold?: number): IdleSettings { const idle = config.compaction?.idle const minutes = Math.min(IDLE_LIMITS.minutes.max, Math.max(IDLE_LIMITS.minutes.min, idle?.minutes ?? DEFAULT_IDLE_MINUTES)) let minTokens = Math.max(0, idle?.minTokens ?? DEFAULT_IDLE_MIN_TOKENS) if (threshold && threshold > 0) minTokens = Math.min(minTokens, Math.floor(threshold * 0.9)) return { enabled: idle?.enabled ?? hosted, minutes, minTokens } } /** What the idle check needs to know about one thread. */ export interface IdleThread { ephemeral?: boolean /** The owner turned idle compaction off for this thread. */ idleCompaction?: boolean /** Made for a wakeup or a Machine run. */ automated: boolean /** A turn is running in it. */ running: boolean /** Calls in it wait for the owner's answer. */ parked: boolean /** Its latest live message. */ last?: { rkey: string; role: 'user' | 'assistant'; stopReason?: string; createdAt: string } /** When the model request behind the latest reply started (ms), if this process saw it; else the reply's createdAt counts. */ requestAt?: number /** When its latest summary was written. */ summarizedAt?: string contextTokens: number } /** * Whether a thread is due an idle compaction: quiet for `minutes` since a finished reply, at least * `minTokens` long, and not summarized since. `tried` is the latest message an idle compaction was * already attempted after, so one quiet stretch gets at most one try. */ export function idleCompactionDue(t: IdleThread, s: IdleSettings, at = Date.now(), tried?: string): boolean { if (!s.enabled || t.ephemeral || t.idleCompaction === false || t.automated || t.running || t.parked) return false const last = t.last if (!last || last.role !== 'assistant' || last.stopReason !== 'stop') return false if (tried === last.rkey) return false if (at - (t.requestAt ?? Date.parse(last.createdAt)) < s.minutes * 60_000) return false if (t.summarizedAt && Date.parse(t.summarizedAt) >= Date.parse(last.createdAt)) return false return t.contextTokens >= s.minTokens } /** * The last message of a cache-friendly summary request: it follows the thread's own context, exactly * as the agent's turns send it, so the provider reads all of that from its prompt cache. `firstKept` * is the start of the first message that stays verbatim; the summary covers only what comes before. */ export function cachedSummaryPrompt(firstKept: { role: 'user' | 'assistant'; at: string; opening: string }, hasPrevious: boolean, instructions?: string): string { let text = '\n' text += `${SUMMARY_SYSTEM_PROMPT.replace('You are summarizing', 'Pause here: you are summarizing')} Do NOT call any tools.\n\n` text += `Summarize the conversation above, from its start up to (not including) the ${firstKept.role === 'user' ? 'user message' : 'assistant message'} from ${firstKept.at} that begins:\n` + `\n${firstKept.opening}\n\n` + 'That message and everything after it stay in the context word for word, so leave them out. ' + 'Under Current state, describe where things stood just before it.\n\n' if (hasPrevious) { text += 'The conversation starts with a of even older messages. Write one updated summary covering both: ' + 'keep everything from that summary that still matters, add what is new, update the current state.\n\n' } text += SECTIONS if (instructions) text += `\n\nAlso focus on: ${instructions}` return `${text}\n` } /** The opening of a message, for cachedSummaryPrompt to point at. */ export function messageOpening(messages: PiMessage[], chars = 300): string { for (const m of messages as any[]) { let t = text(m.content) if (!t && m.role === 'assistant') { const calls = m.content.filter((c: any) => c.type === 'toolCall') if (calls.length) t = `[tool calls: ${calls.map((c: any) => c.name).join(', ')}]` } if (t) return t.length > chars ? `${t.slice(0, chars)}…` : t } return '(a message with no text)' }