Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743const std = @import("std");
const provider = @import("provider.zig");const models = @import("models.zig");const tools = @import("tools.zig");const Abort = @import("abort.zig").Abort;const protocol = @import("protocol.zig");const Block = protocol.core.Block;const Notice = protocol.core.Notice;const Request = protocol.ui.Request;const UiSink = protocol.Sink(protocol.core.Message);
const Message = provider.Message;const ToolCall = provider.ToolCall;
/// Safety cap for one user prompt.pub const max_requests = 50;
/// How many prompts can wait while the agent is busy.pub const max_queued = 8;
pub const Agent = struct { // Three tasks run the agent, and none of them holds a lock. The owner holds // the attached UI and the queued prompts, and only it sends to the UI. The // runner runs one prompt at a time and sends through the owner. The // forwarder moves requests from the inbox to the owner, so the runner's // messages cannot take the room that requests need. The UI never waits on // the agent, so the waits cannot form a cycle. gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ, providers: Providers, system_prompt: []const u8, models: []const models.Entry, /// The level the user asked for. Only the owner uses it after the start. wanted_level: models.Level, /// The model and the level of the next request. The owner writes it and /// the runner reads it. selection: std.atomic.Value(u32), cwd: []const u8, /// Only the runner uses it. history: std.ArrayList(Message) = .empty, /// What every request so far used. Only the runner uses it. session: provider.Usage = .{}, /// Null once a request used a model without a price. session_cost: ?f64 = 0, /// Ends the running prompt. The owner clears it only before it gives the /// runner a prompt, when nothing waits on it. abort: Abort = .{}, /// Where the runner sends its messages for the UI. out: UiSink, inbox: std.Io.Queue(Request), inbox_buffer: [protocol.inbox_capacity]Request = undefined, /// Everything the owner handles. Never closed. events: std.Io.Queue(Event), events_buffer: [64]Event = undefined, /// Gives the runner its next prompt. prompts: std.Io.Queue([]const u8), prompt_buffer: [1][]const u8 = undefined, runner_task: ?std.Io.Future(void) = null, owner_task: ?std.Io.Future(void) = null, forwarder_task: ?std.Io.Future(void) = null,
const Event = union(enum) { request: Request, message: protocol.core.Message, /// The runner finished its prompt. done, stop, };
pub const Options = struct { /// The environment of child processes. environ: std.process.Environ, /// A model can be selected only when its provider is set here. providers: Providers, system_prompt: []const u8, /// The models a UI can select. Must outlive the agent. models: []const models.Entry = &models.entries, /// The index in `models` of the model to start with. Its provider /// must be set. model: usize, /// The reasoning level to start with. The agent uses the nearest /// level that the model supports. level: models.Level = models.default_level, cwd: []const u8, };
pub const Providers = std.EnumArray(models.Provider, ?provider.Provider);
const Selection = struct { /// An index in `models`. model: u16, level: models.Level,
fn pack(self: Selection) u32 { return @as(u32, self.model) << 8 | @backingInt(self.level); }
fn unpack(bits: u32) Selection { return .{ .model = @intCast(bits >> 8), .level = @fromBackingInt(@intCast(@as(std.meta.Tag(models.Level), @truncate(bits)))) }; } };
/// Allocates the agent with `gpa`. `deinit` frees it. The strings in /// `options` must outlive the agent. pub fn init(gpa: std.mem.Allocator, io: std.Io, options: Options) !*Agent { const self = try gpa.create(Agent); self.* = .{ .gpa = gpa, .io = io, .environ = options.environ, .providers = options.providers, .system_prompt = options.system_prompt, .models = options.models, .wanted_level = options.level, .selection = .init(Selection.pack(.{ .model = @intCast(options.model), .level = models.clamp(&options.models[options.model], options.level), })), .cwd = options.cwd, .out = .{ .ctx = self, .sendFn = post }, .inbox = .init(&self.inbox_buffer), .events = .init(&self.events_buffer), .prompts = .init(&self.prompt_buffer), }; return self; }
/// Starts reading the requests that arrive through `sink()` and running /// the prompts they submit. No UI is attached yet. pub fn start(self: *Agent) !void { errdefer self.stop(); self.runner_task = try self.io.concurrent(runnerMain, .{self}); self.owner_task = try self.io.concurrent(Owner.main, .{self}); self.forwarder_task = try self.io.concurrent(forwarderMain, .{self}); }
/// Where a UI sends its requests. Valid after `start`. Never blocks, so a /// UI can send from its event loop. Refuses a request when the inbox is /// full or the agent stopped. pub fn sink(self: *Agent) protocol.Sink(Request) { return .{ .ctx = self, .sendFn = sendRequest }; }
fn sendRequest(ctx: *anyopaque, request: Request) bool { const self: *Agent = @ptrCast(@alignCast(ctx)); return (self.inbox.putUncancelable(self.io, &.{request}, 0) catch 0) == 1; }
/// Ends the running prompt, stops the tasks and frees the prompts that /// wait. An attached UI must not block forever meanwhile. pub fn stop(self: *Agent) void { self.inbox.close(self.io); if (self.forwarder_task) |*task| { task.await(self.io); self.forwarder_task = null; } // After the last request, so the owner handles none after the stop. if (self.owner_task) |*task| { self.events.putOneUncancelable(self.io, .stop) catch unreachable; task.await(self.io); self.owner_task = null; } self.prompts.close(self.io); if (self.runner_task) |*task| { task.await(self.io); self.runner_task = null; } }
pub fn deinit(self: *Agent) void { self.stop(); for (self.history.items) |message| message.deinit(self.gpa); self.history.deinit(self.gpa); self.gpa.destroy(self); }
fn forwarderMain(self: *Agent) void { while (self.inbox.getOneUncancelable(self.io)) |request| { self.events.putOneUncancelable(self.io, .{ .request = request }) catch unreachable; } else |_| {} }
fn runnerMain(self: *Agent) void { while (self.prompts.getOneUncancelable(self.io)) |prompt| { self.runInner(prompt) catch |err| switch (err) { error.OutOfMemory => self.sendNotice(.err, "out of memory"), }; self.events.putOneUncancelable(self.io, .done) catch unreachable; } else |_| {} }
fn post(ctx: *anyopaque, message: protocol.core.Message) bool { const self: *Agent = @ptrCast(@alignCast(ctx)); self.events.putOneUncancelable(self.io, .{ .message = message }) catch unreachable; return true; }
fn send(self: *Agent, message: protocol.core.Message) void { if (!self.out.send(message)) { message.deinit(self.gpa); } }
fn sendNotice(self: *Agent, kind: Notice.Kind, text: []const u8) void { // A provider detail can hold server text that is not UTF-8, or end inside a code point. const copy = std.fmt.allocPrint(self.gpa, "{f}", .{std.unicode.fmtUtf8(text)}) catch return; self.send(.{ .notice = .{ .kind = kind, .text = copy } }); }
/// Runs one prompt and takes ownership of it. fn runInner(self: *Agent, prompt: []const u8) !void { self.history.append(self.gpa, .{ .user = prompt }) catch |err| { self.gpa.free(prompt); return err; };
var requests: usize = 0; while (true) { if (requests == max_requests) { self.sendNotice(.err, "request limit reached, the run ended"); return; } requests += 1;
// A change of the selection applies from the next request. const selection: Selection = .unpack(self.selection.load(.acquire)); const model = &self.models[selection.model]; var stream: Stream = .{ .agent = self }; defer stream.deinit(); var diagnostics: provider.Diagnostics = .{}; const reply = self.providers.get(model.provider).?.stream( .{ .system = self.system_prompt, .messages = self.history.items, .tools = &tools.definitions, .model = model, .level = selection.level, }, stream.sink(), &self.abort, &diagnostics, ); // A request that failed or was interrupted was billed too. if (stream.usage) |usage| { self.recordUsage(model, usage); } const finish = reply catch |err| { if (self.abort.isSet()) { self.sendNotice(.interrupted, "interrupted"); return; } if (err == error.OutOfMemory) { return error.OutOfMemory; } const detail = diagnostics.message(); self.sendNotice(.err, if (detail.len == 0) "provider error" else detail); return; };
// A partial answer is not part of the history: it may hold incomplete tool calls. if (self.abort.isSet()) { self.sendNotice(.interrupted, "interrupted"); return; } if (stream.out_of_memory) { return error.OutOfMemory; }
const calls = calls: { try self.history.ensureUnusedCapacity(self.gpa, 1); const text = try stream.text.toOwnedSlice(self.gpa); errdefer self.gpa.free(text); const thinking = try stream.thinking.toOwnedSlice(self.gpa); errdefer self.gpa.free(thinking); const calls = try stream.calls.toOwnedSlice(self.gpa); self.history.appendAssumeCapacity(.{ .assistant = .{ .text = text, .thinking = thinking, .tool_calls = calls } }); break :calls calls; };
if (finish == .filtered or finish == .other) { for (calls) |call| try self.recordPlain(call.id, "not run: the reply ended early"); if (finish == .filtered) { self.sendNotice(.err, "the provider's content filter stopped the reply"); } else if (diagnostics.message().len == 0) { self.sendNotice(.err, "the reply ended early"); } else { const prefix = "the reply ended early: "; var buffer: [prefix.len + diagnostics.buffer.len]u8 = undefined; self.sendNotice(.err, std.fmt.bufPrint(&buffer, prefix ++ "{s}", .{diagnostics.message()}) catch unreachable); } return; }
if (calls.len == 0) { if (finish == .length) { self.sendNotice(.err, "the reply reached the token limit"); } return; }
if (finish == .length) { for (calls) |call| try self.recordPlain(call.id, "the output was truncated; re-issue the call"); continue; }
for (calls, 0..) |call, index| { // The flag can be set between two calls, so it is checked before each one. if (self.abort.isSet()) { try self.skipRemaining(calls[index..]); self.sendNotice(.interrupted, "interrupted"); return; } var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); const result = tools.execute( .{ .io = self.io, .environ = self.environ, .abort = &self.abort }, arena_state.allocator(), call.name, call.args_json, ); // A tool reports an interrupt in its own result, and work it finished still happened. try self.recordResult(call.id, result); if (self.abort.isSet()) { try self.skipRemaining(calls[index + 1 ..]); self.sendNotice(.interrupted, "interrupted"); return; } } } }
/// Adds the usage of one request to the session and tells the UI. fn recordUsage(self: *Agent, model: *const models.Entry, usage: provider.Usage) void { const request_cost: ?f64 = if (model.cost) |prices| models.cost(prices, usage) else null; self.session.add(usage); self.session_cost = if (self.session_cost != null and request_cost != null) self.session_cost.? + request_cost.? else null; self.send(.{ .usage = .{ .request = usage, .request_cost = request_cost, .session = self.session, .session_cost = self.session_cost, .context_tokens = usage.total(), .context_window = model.context_window, } }); }
fn skipRemaining(self: *Agent, calls: []const ToolCall) !void { const skip = "not run: aborted by user"; for (calls) |call| try self.recordPlain(call.id, skip); }
/// Records a failed result that the agent wrote itself. fn recordPlain(self: *Agent, call_id: []const u8, text: []const u8) !void { var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); try self.recordResult(call_id, tools.plainResult(arena_state.allocator(), text, true)); }
fn recordResult(self: *Agent, call_id: []const u8, result: tools.Result) !void { try self.history.ensureUnusedCapacity(self.gpa, 1); const id = try self.gpa.dupe(u8, call_id); errdefer self.gpa.free(id); // Tool output can hold any bytes, and the request JSON needs UTF-8. const text = if (std.unicode.utf8ValidateSlice(result.text)) try self.gpa.dupe(u8, result.text) else try std.fmt.allocPrint(self.gpa, "{f}", .{std.unicode.fmtUtf8(result.text)}); self.history.appendAssumeCapacity(.{ .tool_result = .{ .call_id = id, .text = text } }); // A call without a result breaks every later request, so a failure here only hides the result from the UI. self.showResult(call_id, result) catch {}; }
fn showResult(self: *Agent, call_id: []const u8, result: tools.Result) !void { const id = try self.gpa.dupe(u8, call_id); errdefer self.gpa.free(id); const collapsed: Block = try .init(self.gpa, result.collapsed); errdefer collapsed.deinit(self.gpa); const expanded: Block = try .init(self.gpa, result.expanded); self.send(.{ .tool_result = .{ .call_id = id, .is_error = result.is_error, .collapsed = collapsed, .expanded = expanded, }, }); }};
/// The state of the owner task. No other task can reach it.const Owner = struct { agent: *Agent, ui: ?UiSink = null, /// The prompts that wait for the runner. queued: std.ArrayList([]const u8), /// The runner has a prompt. running: bool = false, stopping: bool = false,
fn main(agent: *Agent) void { var buffer: [max_queued][]const u8 = undefined; var self: Owner = .{ .agent = agent, .queued = .initBuffer(&buffer) }; while (!self.stopping or self.running) { switch (agent.events.getOneUncancelable(agent.io) catch unreachable) { .request => |request| self.handle(request), .message => |message| self.send(message), .done => self.finish(), .stop => self.stop(), } } }
fn handle(self: *Owner, request: Request) void { const id = request.id; switch (request.method) { .attach => |ui| self.attach(id, ui.sink, ui.version), .detach => |ui| self.detach(id, ui), .submit => |prompt| self.submit(id, prompt), .interrupt => { self.respond(id, .ok); if (self.running) { self.agent.abort.set(self.agent.io); } }, .select_model => |name| { defer self.agent.gpa.free(name); const index = models.indexOf(self.agent.models, name) orelse return self.respond(id, .unknown_model); if (!self.selectable(index)) { return self.respond(id, .unknown_model); } self.selectModel(id, index); }, .cycle_model => { const count = self.agent.models.len; var index: usize = self.selection().model; // The selected model can be selected, so the search ends there at the latest. for (0..count) |_| { index = (index + 1) % count; if (self.selectable(index)) { break; } } self.selectModel(id, index); }, .cycle_level => { const current = self.selection(); const level = models.nextLevel(&self.agent.models[current.model], current.level); // A model with one level keeps what the user asked for, for the next model. if (level != current.level) { self.agent.wanted_level = level; } self.select(id, .{ .model = current.model, .level = level }); }, } }
fn selection(self: *Owner) Agent.Selection { return .unpack(self.agent.selection.load(.acquire)); }
fn selectable(self: *Owner, index: usize) bool { return self.agent.providers.get(self.agent.models[index].provider) != null; }
/// Selects a model, at the level nearest to the one the user asked for. fn selectModel(self: *Owner, id: protocol.RequestId, index: usize) void { self.select(id, .{ .model = @intCast(index), .level = models.clamp(&self.agent.models[index], self.agent.wanted_level), }); }
/// Makes `selected` the selection of the next request, tells the UI and /// then answers the request that asked for it. fn select(self: *Owner, id: protocol.RequestId, selected: Agent.Selection) void { self.agent.selection.store(selected.pack(), .release); defer self.respond(id, .ok); const entry = &self.agent.models[selected.model]; const model = self.agent.gpa.dupe(u8, entry.name) catch return; self.send(.{ .selection = .{ .model = model, .level = if (models.reasons(entry)) selected.level else null } }); }
fn attach(self: *Owner, id: protocol.RequestId, ui: UiSink, version: u32) void { if (version != protocol.version) { _ = ui.send(.{ .response = .{ .id = id, .result = .incompatible } }); return; } if (self.ui != null) { _ = ui.send(.{ .response = .{ .id = id, .result = .already_attached } }); return; } const message: protocol.core.Message = .{ .response = .{ .id = id, .result = .{ .attached = self.session() }, }, }; if (!ui.send(message)) { message.deinit(self.agent.gpa); return; } self.ui = ui; }
/// A copy of the session for a UI. Empty when memory runs out, because /// the attach must still get its answer. fn session(self: *Owner) protocol.core.Session { const gpa = self.agent.gpa; const selected = self.selection(); const entry = &self.agent.models[selected.model]; const model = gpa.dupe(u8, entry.name) catch return .{ .model = "", .cwd = "" }; const cwd = gpa.dupe(u8, self.agent.cwd) catch { gpa.free(model); return .{ .model = "", .cwd = "" }; }; return .{ .model = model, .level = if (models.reasons(entry)) selected.level else null, .cwd = cwd }; }
fn detach(self: *Owner, id: protocol.RequestId, ui: UiSink) void { if (self.ui) |attached| { if (attached.eql(ui)) { self.ui = null; } } _ = ui.send(.{ .response = .{ .id = id, .result = .ok } }); }
/// Runs or queues a prompt and takes ownership of it. fn submit(self: *Owner, id: protocol.RequestId, prompt: []const u8) void { if (!self.running) { self.respond(id, .queued); self.send(.agent_start); return self.startPrompt(prompt); } self.queued.appendBounded(prompt) catch { self.agent.gpa.free(prompt); return self.respond(id, .refused); }; self.respond(id, .queued); }
fn startPrompt(self: *Owner, prompt: []const u8) void { self.send(.prompt_start); self.agent.abort.reset(); // The queue is empty while no prompt runs, and it closes only after the owner returns. self.agent.prompts.putOneUncancelable(self.agent.io, prompt) catch unreachable; self.running = true; }
fn finish(self: *Owner) void { self.running = false; if (self.agent.abort.isSet()) { self.returnQueued(); } if (self.queued.items.len > 0) { return self.startPrompt(self.queued.orderedRemove(0)); } self.send(.agent_end); }
fn stop(self: *Owner) void { self.stopping = true; for (self.queued.items) |prompt| self.agent.gpa.free(prompt); self.queued.clearRetainingCapacity(); self.agent.abort.set(self.agent.io); }
/// Sends the text of every queued prompt back and takes them off the queue. fn returnQueued(self: *Owner) void { const gpa = self.agent.gpa; const taken = self.queued.items; if (taken.len == 0) { return; } defer { for (taken) |prompt| gpa.free(prompt); self.queued.clearRetainingCapacity(); } var parts: [max_queued][]const u8 = undefined; var kept: usize = 0; for (taken) |prompt| { if (std.mem.trim(u8, prompt, &std.ascii.whitespace).len == 0) { continue; } parts[kept] = prompt; kept += 1; } const text = std.mem.join(gpa, "\n\n", parts[0..kept]) catch text: { self.sendNotice(.err, "out of memory; the queued prompts were lost"); break :text ""; }; self.send(.{ .prompts_returned = .{ .count = taken.len, .text = text } }); }
fn respond(self: *Owner, id: protocol.RequestId, result: protocol.core.Response.Result) void { self.send(.{ .response = .{ .id = id, .result = result } }); }
/// Sends a message to the attached UI, or drops it when none is attached. fn send(self: *Owner, message: protocol.core.Message) void { if (self.ui) |ui| { if (ui.send(message)) { return; } } message.deinit(self.agent.gpa); }
fn sendNotice(self: *Owner, kind: Notice.Kind, text: []const u8) void { const copy = self.agent.gpa.dupe(u8, text) catch return; self.send(.{ .notice = .{ .kind = kind, .text = copy } }); }};
/// Collects the streamed answer and forwards every delta toward the UI.const Stream = struct { agent: *Agent, text: std.ArrayList(u8) = .empty, thinking: std.ArrayList(u8) = .empty, calls: std.ArrayList(ToolCall) = .empty, usage: ?provider.Usage = null, /// Part of the reply was dropped. out_of_memory: bool = false,
fn deinit(self: *Stream) void { const gpa = self.agent.gpa; self.text.deinit(gpa); self.thinking.deinit(gpa); for (self.calls.items) |call| call.deinit(gpa); self.calls.deinit(gpa); }
fn sink(self: *Stream) provider.StreamSink { return .{ .ctx = self, .text_delta = onText, .thinking_delta = onThinking, .tool_call = onCall, .usage = onUsage }; }
fn onUsage(ctx: *anyopaque, usage: provider.Usage) void { const self: *Stream = @ptrCast(@alignCast(ctx)); self.usage = usage; }
fn onText(ctx: *anyopaque, delta: []const u8) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; self.text.appendSlice(gpa, delta) catch return self.lose(); const copy = gpa.dupe(u8, delta) catch return; self.agent.send(.{ .text_delta = copy }); }
fn onThinking(ctx: *anyopaque, delta: []const u8) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; self.thinking.appendSlice(gpa, delta) catch return self.lose(); const copy = gpa.dupe(u8, delta) catch return; self.agent.send(.{ .thinking_delta = copy }); }
fn onCall(ctx: *anyopaque, call: ToolCall) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; const owned = call.dupe(gpa) catch return self.lose(); self.calls.append(gpa, owned) catch { owned.deinit(gpa); return self.lose(); }; self.show(call) catch {}; }
fn lose(self: *Stream) void { self.out_of_memory = true; }
fn show(self: *Stream, call: ToolCall) !void { const gpa = self.agent.gpa; var arena_state = std.heap.ArenaAllocator.init(gpa); defer arena_state.deinit(); const shown = try tools.renderCall(arena_state.allocator(), call.name, call.args_json); const id = try gpa.dupe(u8, call.id); errdefer gpa.free(id); self.agent.send(.{ .tool_call = .{ .id = id, .call = try .init(gpa, shown) } }); }};
const testing = std.testing;const helpers = @import("testing.zig");const Scripted = helpers.Scripted;const EventLog = helpers.EventLog(protocol.core.Message);const toolCall = helpers.toolCall;
const test_model: models.Entry = .{ .name = "model", .id = "provider/model", .provider = .fake, .max_output = 0 };
fn testOptions(streamer: provider.Provider) Agent.Options { var providers: Agent.Providers = .initFill(null); providers.set(.fake, streamer); return .{ .environ = testing.environ, .providers = providers, .system_prompt = "system", .models = (&test_model)[0..1], .model = 0, .cwd = "/work" };}
/// A started agent with a scripted provider and an attached message log. Do/// not move it after `init`.const Harness = struct { log: EventLog, script: Scripted, agent: *Agent, next_id: protocol.RequestId,
fn init(self: *Harness, steps: []const Scripted.Step) !void { return self.initWith(steps, &test_model); }
fn initWith(self: *Harness, steps: []const Scripted.Step, model: *const models.Entry) !void { return self.initModels(steps, model[0..1], models.default_level); }
/// Starts with the first of `entries`. fn initModels(self: *Harness, steps: []const Scripted.Step, entries: []const models.Entry, level: models.Level) !void { self.log = .{}; self.script = .init(steps); self.next_id = 0; var options = testOptions(self.script.asProvider()); options.models = entries; options.level = level; self.agent = try Agent.init(testing.allocator, testing.io, options); try self.agent.start(); try self.send(.{ .attach = .{ .sink = self.log.sink() } }); // The attach response is the first message; its own test checks it. try self.log.waitFor(.response, 1); for (self.log.events.items) |event| event.deinit(testing.allocator); self.log.events.clearRetainingCapacity(); }
fn deinit(self: *Harness) void { // A failed test must not leave the agent task in a held step. self.script.release(); self.agent.deinit(); self.script.deinit(); self.log.deinit(); }
/// Submits a prompt and waits until the agent is idle again. fn run(self: *Harness, prompt: []const u8) !void { self.log.collect(); const ends = self.log.count(.agent_end); try self.submit(prompt); try self.log.waitFor(.agent_end, ends + 1); }
/// Submits a prompt and waits until the agent answers. fn submit(self: *Harness, prompt: []const u8) !void { self.log.collect(); const answered = self.log.count(.response); try self.send(.{ .submit = try testing.allocator.dupe(u8, prompt) }); try self.log.waitFor(.response, answered + 1); }
/// Sends a request and returns the answer to it. fn ask(self: *Harness, method: protocol.ui.Request.Method) !protocol.core.Response.Result { self.log.collect(); const answered = self.log.count(.response); const id = self.next_id; try self.send(method); try self.log.waitFor(.response, answered + 1); var index = self.log.events.items.len; while (index > 0) { index -= 1; switch (self.log.events.items[index]) { .response => |response| if (response.id == id) { return response.result; }, else => {}, } } return error.NoResponse; }
/// The last selection that the agent reported. fn selection(self: *Harness) ?protocol.core.Selection { var index = self.log.events.items.len; while (index > 0) { index -= 1; switch (self.log.events.items[index]) { .selection => |selected| return selected, else => {}, } } return null; }
/// Sends a request and takes ownership of its method. fn send(self: *Harness, method: protocol.ui.Request.Method) !void { const message: protocol.ui.Request = .{ .id = self.next_id, .method = method }; self.next_id += 1; if (!self.agent.sink().send(message)) { message.deinit(testing.allocator); return error.Closed; } }
/// Interrupts the agent and waits until the running prompt sees it. fn interrupt(self: *Harness) !void { try self.send(.interrupt); if (!self.agent.abort.wait(testing.io, .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } })) { return error.Timeout; } }
fn request(self: *Harness, index: usize) Scripted.Request { return self.script.requests.items[index]; }
/// The usage reports so far, in arrival order. fn usage(self: *Harness, arena: std.mem.Allocator) ![]const protocol.core.UsageReport { var reports: std.ArrayList(protocol.core.UsageReport) = .empty; for (self.log.events.items) |event| switch (event) { .usage => |report| try reports.append(arena, report), else => {}, }; return reports.items; }};
/// Waits until a writer sends one line through the FIFO at `path`.fn waitForLine(dir: std.Io.Dir, path: []const u8) !void { const Done = union(enum) { line: anyerror!void, late: std.Io.Cancelable!void }; var buffer: [2]Done = undefined; var select: std.Io.Select(Done) = .init(testing.io, &buffer); defer select.cancelDiscard(); try select.concurrent(.line, readLine, .{ dir, path }); try select.concurrent(.late, std.Io.sleep, .{ testing.io, .fromSeconds(10), .awake }); switch (try select.await()) { .line => |result| try result, .late => return error.Timeout, }}
fn readLine(dir: std.Io.Dir, path: []const u8) anyerror!void { const file = try dir.openFile(testing.io, path, .{}); defer file.close(testing.io); var buffer: [64]u8 = undefined; var reader = file.reader(testing.io, &buffer); _ = try reader.interface.takeDelimiterInclusive('\n');}
fn expectMessages(expected: []const Message, actual: []const Message) !void { try testing.expectEqual(expected.len, actual.len); for (expected, actual) |want, got| { try testing.expectEqual(std.meta.activeTag(want), std.meta.activeTag(got)); switch (want) { .user => |text| try testing.expectEqualStrings(text, got.user), .assistant => |assistant| { try testing.expectEqualStrings(assistant.text, got.assistant.text); try testing.expectEqualStrings(assistant.thinking, got.assistant.thinking); try testing.expectEqual(assistant.tool_calls.len, got.assistant.tool_calls.len); for (assistant.tool_calls, got.assistant.tool_calls) |want_call, got_call| { try testing.expectEqualStrings(want_call.id, got_call.id); try testing.expectEqualStrings(want_call.name, got_call.name); try testing.expectEqualStrings(want_call.args_json, got_call.args_json); } }, .tool_result => |result| { try testing.expectEqualStrings(result.call_id, got.tool_result.call_id); try testing.expectEqualStrings(result.text, got.tool_result.text); }, } }}
const Tag = EventLog.Tag;
test "a text reply streams between agent_start and agent_end" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{ "po", "ng" } }}); defer harness.deinit();
try harness.run("ping");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .queued }, harness.log.events.items[0].response); try testing.expectEqualStrings("pong", try harness.log.text(arena));}
test "attach answers with the model and the working directory" { var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{}); defer script.deinit(); const agent = try Agent.init(testing.allocator, testing.io, testOptions(script.asProvider())); defer agent.deinit(); try agent.start();
try testing.expect(agent.sink().send(.{ .id = 7, .method = .{ .attach = .{ .sink = log.sink() } } }));
try log.waitFor(.response, 1); const response = log.events.items[0].response; try testing.expectEqual(7, response.id); try testing.expectEqualStrings("model", response.result.attached.model); try testing.expectEqual(null, response.result.attached.level); try testing.expectEqualStrings("/work", response.result.attached.cwd);}
test "an attach with another protocol version is answered incompatible and does not attach" { var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{}); defer script.deinit(); const agent = try Agent.init(testing.allocator, testing.io, testOptions(script.asProvider())); defer agent.deinit(); try agent.start();
try testing.expect(agent.sink().send(.{ .id = 3, .method = .{ .attach = .{ .sink = log.sink(), .version = protocol.version + 1 } } })); try log.waitFor(.response, 1); try testing.expectEqual(protocol.core.Response{ .id = 3, .result = .incompatible }, log.events.items[0].response);
try testing.expect(agent.sink().send(.{ .id = 4, .method = .{ .attach = .{ .sink = log.sink() } } })); try log.waitFor(.response, 2); try testing.expectEqualStrings("model", log.events.items[1].response.result.attached.model);}
test "a second attach is answered on its own sink and leaves the first UI attached" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); var second: EventLog = .{}; defer second.deinit();
try harness.send(.{ .attach = .{ .sink = second.sink() } }); try second.waitFor(.response, 1); try harness.run("ping");
second.collect(); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .already_attached }, second.events.items[0].response); try testing.expectEqual(1, second.events.items.len); try testing.expectEqualStrings("pong", try harness.log.text(arena_state.allocator()));}
test "the detach response is the last message to the sink, and the running prompt continues for the next UI" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"}, .hold = true }}); defer harness.deinit();
try harness.submit("ping"); try harness.log.waitFor(.text_delta, 1); try harness.send(.{ .detach = harness.log.sink() }); try harness.log.waitFor(.response, 2); var next: EventLog = .{}; defer next.deinit(); try harness.send(.{ .attach = .{ .sink = next.sink() } }); try next.waitFor(.response, 1); harness.script.release(); try next.waitFor(.agent_end, 1);
harness.log.collect(); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .ok }, harness.log.events.items[4].response);}
test "a detach from a UI that is not attached is answered and leaves the attached UI" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); var other: EventLog = .{}; defer other.deinit();
try harness.send(.{ .detach = other.sink() }); try other.waitFor(.response, 1); try harness.run("ping");
try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .ok }, other.events.items[0].response); try testing.expectEqualStrings("pong", try harness.log.text(arena_state.allocator()));}
test "the first request holds the system prompt, the prompt and every tool" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit();
try harness.run("ping");
const request = harness.request(0); try testing.expectEqualStrings("system", request.system); try expectMessages(&.{.{ .user = "ping" }}, request.messages); try testing.expectEqual(tools.definitions.len, request.tools.len); for (tools.definitions, request.tools) |definition, name| try testing.expectEqualStrings(definition.name, name);}
test "a tool call runs and its result goes back to the model" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.path("out.txt"); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "write", args)}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqualStrings("hi", try dir.read("out.txt"));
const wrote = try std.fmt.allocPrint(arena, "wrote 2 bytes to {s}", .{path}); const result = harness.log.events.items[4].tool_result; try testing.expectEqualStrings("call-1", result.call_id); try testing.expect(!result.is_error);
try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "", .tool_calls = &.{toolCall("call-1", "write", args)} } }, .{ .tool_result = .{ .call_id = "call-1", .text = wrote } }, }, harness.request(1).messages);}
test "calls in one reply run in order and each gets its own result" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.path("out.txt"); const first = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "first" }, .{}); const second = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "second!" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"writing"}, .calls = &.{ toolCall("call-a", "write", first), toolCall("call-b", "write", second) }, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualStrings("second!", try dir.read("out.txt")); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "writing", .tool_calls = &.{ toolCall("call-a", "write", first), toolCall("call-b", "write", second) } } }, .{ .tool_result = .{ .call_id = "call-a", .text = try std.fmt.allocPrint(arena, "wrote 5 bytes to {s}", .{path}) } }, .{ .tool_result = .{ .call_id = "call-b", .text = try std.fmt.allocPrint(arena, "wrote 7 bytes to {s}", .{path}) } }, }, harness.request(1).messages);}
test "invalid UTF-8 in a tool result reaches the model and the UI as U+FFFD" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("latin1.txt", "caf\xe9\n"); const args = try std.json.Stringify.valueAlloc(arena, .{ .files = &.{.{ .path = path }} }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "read", args)}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
const shown = harness.log.events.items[4].tool_result.expanded.text; const sent = harness.request(1).messages[2].tool_result.text; try testing.expect(std.unicode.utf8ValidateSlice(shown)); try testing.expect(std.mem.indexOf(u8, shown, "1| caf\u{fffd}\n") != null); try testing.expect(std.unicode.utf8ValidateSlice(sent)); try testing.expect(std.mem.indexOf(u8, sent, "1| caf\u{fffd}\n") != null);}
test "a failed tool call is a result for the model, not the end of the run" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"sorry"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena)); const result = harness.log.events.items[4].tool_result; try testing.expect(result.is_error); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "", .tool_calls = &.{toolCall("call-1", "nope", "{}")} } }, .{ .tool_result = .{ .call_id = "call-1", .text = "unknown tool: nope" } }, }, harness.request(1).messages);}
/// Fails one allocation, the one with index `fail_index`.const OneFailure = struct { fail_index: usize, index: usize = 0,
fn allocator(self: *OneFailure) std.mem.Allocator { return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = std.mem.Allocator.noResize, .remap = std.mem.Allocator.noRemap, .free = free } }; }
fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { const self: *OneFailure = @ptrCast(@alignCast(ctx)); defer self.index += 1; if (self.index == self.fail_index) { return null; } return testing.allocator.rawAlloc(len, alignment, ret_addr); }
fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { _ = ctx; testing.allocator.rawFree(memory, alignment, ret_addr); }};
test "a failed allocation ends the run with the error or keeps the whole reply, and leaks nothing" { var fail_index: usize = 0; while (true) : (fail_index += 1) { var failing: OneFailure = .{ .fail_index = fail_index }; const gpa = failing.allocator(); var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{ .{ .text = &.{ "a", "b" }, .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer script.deinit(); const agent = Agent.init(gpa, testing.io, testOptions(script.asProvider())) catch continue; defer agent.deinit();
agent.out = log.sink(); const prompt = gpa.dupe(u8, "go") catch continue; agent.runInner(prompt) catch |err| { try testing.expectEqual(error.OutOfMemory, err); continue; }; const history = agent.history.items; try testing.expectEqual(4, history.len); try expectMessages(&.{.{ .assistant = .{ .text = "ab", .tool_calls = &.{toolCall("call-1", "nope", "{}")} } }}, history[1..2]); try expectMessages(&.{.{ .assistant = .{ .text = "done", .tool_calls = &.{} } }}, history[3..]); if (failing.index <= fail_index) { break; } }}
test "a reply cut off by the length limit answers its calls without running them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "write", args)}, .finish = .length }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectError(error.FileNotFound, dir.read("out.txt")); const result = harness.log.events.items[4].tool_result; try testing.expect(result.is_error); try testing.expect(std.mem.indexOf(u8, result.expanded.text, "truncated") != null); const messages = harness.request(1).messages; try testing.expectEqual(@as(usize, 3), messages.len); try testing.expectEqualStrings("call-1", messages[2].tool_result.call_id);}
test "a reply cut off by the length limit without calls ends with a notice" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .finish = .length }}); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[4].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the reply reached the token limit", notice.text);}
test "a reply stopped by the content filter ends with a notice and answers its calls without running them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .calls = &.{toolCall("call-1", "write", args)}, .finish = .filtered }, .{ .text = &.{"next"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .tool_result, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectError(error.FileNotFound, dir.read("out.txt")); const result = harness.log.events.items[5].tool_result; try testing.expect(result.is_error); try testing.expectEqualStrings("not run: the reply ended early", result.expanded.text); const notice = harness.log.events.items[6].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the provider's content filter stopped the reply", notice.text);
try harness.run("again"); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "par", .tool_calls = &.{toolCall("call-1", "write", args)} } }, .{ .tool_result = .{ .call_id = "call-1", .text = "not run: the reply ended early" } }, .{ .user = "again" }, }, harness.request(1).messages);}
test "a reply that ends for an unknown reason ends with a notice that names the reason" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .finish = .other, .detail = "finish reason: overloaded" }}); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[4].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the reply ended early: finish reason: overloaded", notice.text);}
test "a provider error ends the run with a notice that holds its detail as valid UTF-8" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .fail = error.Http, .detail = "503 overloaded \xff" }}); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[3].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("503 overloaded \u{FFFD}", notice.text);}
test "a run stops after max_requests requests" { var harness: Harness = undefined; const step: Scripted.Step = .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }; const steps: [max_requests + 1]Scripted.Step = @splat(step); try harness.init(&steps); defer harness.deinit();
try harness.run("go");
try testing.expectEqual(@as(usize, max_requests), harness.script.requests.items.len); const events = harness.log.events.items; const notice = events[events.len - 2].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expect(std.mem.indexOf(u8, notice.text, "request limit") != null);}
test "an interrupt while streaming keeps the partial answer out of the history" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .wait_for_abort = true }, .{ .text = &.{"pong"} }, }); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .ok }, harness.log.events.items[4].response); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[5].notice.kind);
try harness.run("second"); try expectMessages(&.{ .{ .user = "first" }, .{ .user = "second" } }, harness.request(1).messages);}
test "an interrupt that the provider does not report drops the reply and its calls" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .calls = &.{toolCall("call-1", "write", args)}, .hold = true, .finish = .tool_calls }, .{ .text = &.{"pong"} }, }); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.tool_call, 1); try harness.interrupt(); harness.script.release(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .response, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[6].notice.kind); try testing.expectError(error.FileNotFound, dir.read("out.txt"));
try harness.run("second"); try expectMessages(&.{ .{ .user = "first" }, .{ .user = "second" } }, harness.request(1).messages);}
test "an interrupt during a tool call keeps its own result and skips the calls after it" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); try testing.expect(!dir.call("bash", .{ .command = "mkfifo started", .timeout = 10, .cwd = dir.root }).is_error); const slow = try std.json.Stringify.valueAlloc(arena, .{ .command = "echo partial; echo > started; sleep 30", .timeout = 60, .cwd = dir.root }, .{}); const fast = try std.json.Stringify.valueAlloc(arena, .{ .command = "echo two", .timeout = 10, .cwd = dir.root }, .{}); const calls = [_]ToolCall{ toolCall("call-1", "bash", slow), toolCall("call-2", "bash", fast) };
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &calls, .finish = .tool_calls }, .{ .text = &.{"after"} }, }); defer harness.deinit();
const started = std.Io.Timestamp.now(testing.io, .awake); try harness.submit("go"); try waitForLine(dir.tmp.dir, "started"); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try testing.expect(started.untilNow(testing.io, .awake).nanoseconds < 10 * std.time.ns_per_s);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_call, .response, .tool_result, .tool_result, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[8].notice.kind); const first = harness.log.events.items[6].tool_result; try testing.expect(!first.is_error); try testing.expectStringStartsWith(first.expanded.text, "partial\n[aborted by user, elapsed "); const second = harness.log.events.items[7].tool_result; try testing.expect(second.is_error); try testing.expectEqualStrings("not run: aborted by user", second.expanded.text);
try harness.run("next"); const messages = harness.request(1).messages; try testing.expectEqual(5, messages.len); try testing.expectStringStartsWith(messages[2].tool_result.text, "partial\n[aborted by user, elapsed "); try expectMessages(&.{ .{ .tool_result = .{ .call_id = "call-2", .text = "not run: aborted by user" } }, .{ .user = "next" }, }, messages[3..]);}
test "thinking is shown and goes back to the model with its reply" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .thinking = &.{ "h", "m" }, .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"answer"} }, .{ .text = &.{"two"} }, }); defer harness.deinit();
try harness.run("first"); try testing.expectEqualSlices( Tag, &.{ .response, .agent_start, .prompt_start, .thinking_delta, .thinking_delta, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena), ); const sent = harness.request(1).messages; try testing.expectEqualStrings("hm", sent[1].assistant.thinking);
try harness.run("second"); try testing.expectEqualStrings("hm", harness.request(2).messages[1].assistant.thinking); try testing.expectEqualStrings("", harness.request(2).messages[3].assistant.thinking);}
test "each request names the model and a reasoning level that the model supports" { const leveled: models.Entry = .{ .name = "leveled", .id = "provider/leveled", .provider = .fake, .levels = .init(.{ .off = null, .low = "l", .medium = null, .high = "h", .max = null }), .max_output = 0, }; var harness: Harness = undefined; try harness.initWith(&.{.{ .text = &.{"a"} }}, &leveled); defer harness.deinit();
try harness.run("go");
try testing.expectEqualStrings("leveled", harness.request(0).model); try testing.expectEqual(models.Level.high, harness.request(0).level);}
const choices = [_]models.Entry{ .{ .name = "wide", .id = "provider/wide", .provider = .fake, .levels = .init(.{ .off = "none", .low = "low", .medium = null, .high = "high", .max = null }), .max_output = 0, }, // Its provider is not set. .{ .name = "away", .id = "provider/away", .provider = .workers_ai, .max_output = 0 }, .{ .name = "plain", .id = "provider/plain", .provider = .fake, .max_output = 0 }, .{ .name = "narrow", .id = "provider/narrow", .provider = .fake, .levels = .init(.{ .off = null, .low = "low", .medium = "medium", .high = null, .max = null }), .max_output = 0, },};
fn expectSelection(harness: *Harness, model: []const u8, level: ?models.Level) !void { const selected = harness.selection().?; try testing.expectEqualStrings(model, selected.model); try testing.expectEqual(level, selected.level);}
test "attach answers with the reasoning level of a model that reasons" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .medium); defer harness.deinit(); var log: EventLog = .{}; defer log.deinit();
try harness.send(.{ .detach = harness.log.sink() }); try harness.log.waitFor(.response, 1); try harness.send(.{ .attach = .{ .sink = log.sink() } }); try log.waitFor(.response, 1);
const session = log.events.items[0].response.result.attached; try testing.expectEqualStrings("wide", session.model); try testing.expectEqual(models.Level.high, session.level.?);}
test "cycling the level steps through the levels of the model and wraps around" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .high); defer harness.deinit();
try testing.expectEqual(.ok, try harness.ask(.cycle_level)); try expectSelection(&harness, "wide", .off); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "wide", .low); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "wide", .high);}
test "cycling the model skips a model without a provider and wraps around" { var harness: Harness = undefined; try harness.initModels(&.{.{ .text = &.{"a"} }}, &choices, .high); defer harness.deinit();
try testing.expectEqual(.ok, try harness.ask(.cycle_model)); try expectSelection(&harness, "plain", null); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "narrow", .medium);
try harness.run("go"); try testing.expectEqualStrings("narrow", harness.request(0).model); try testing.expectEqual(models.Level.medium, harness.request(0).level);
_ = try harness.ask(.cycle_model); try expectSelection(&harness, "wide", .high);}
test "a model without reasoning stays off, and the level the user chose comes back with the next model" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .low); defer harness.deinit();
_ = try harness.ask(.cycle_model); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "plain", null); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "narrow", .low);}
test "a model is selected by name, and an unknown name or a missing provider is refused" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .high); defer harness.deinit();
try testing.expectEqual(.ok, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "narrow") })); try expectSelection(&harness, "narrow", .medium);
try testing.expectEqual(.unknown_model, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "nope") })); try testing.expectEqual(.unknown_model, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "away") })); try testing.expectEqual(@as(usize, 1), harness.log.count(.selection)); try expectSelection(&harness, "narrow", .medium);}
test "a change during a request applies from the next request of the same prompt" { var harness: Harness = undefined; try harness.initModels(&.{ .{ .text = &.{"a"}, .calls = &.{toolCall("call-1", "nope", "{}")}, .hold = true, .finish = .tool_calls }, .{ .text = &.{"done"} }, }, &choices, .high); defer harness.deinit();
try harness.submit("go"); try harness.log.waitFor(.text_delta, 1); _ = try harness.ask(.cycle_level); _ = try harness.ask(.cycle_model); _ = try harness.ask(.cycle_model); harness.script.release(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualStrings("wide", harness.request(0).model); try testing.expectEqual(models.Level.high, harness.request(0).level); try testing.expectEqualStrings("narrow", harness.request(1).model); try testing.expectEqual(models.Level.low, harness.request(1).level);}
const priced_model: models.Entry = .{ .name = "priced", .id = "provider/priced", .provider = .fake, .context_window = 2000, .max_output = 0, .cost = .{ .input = 2, .output = 10, .cache_read = 1 },};
test "each request reports its usage and the session totals" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.initWith(&.{ .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .usage = .{ .input = 1000, .output = 100, .cache_read = 0 }, .finish = .tool_calls }, .{ .text = &.{"done"}, .usage = .{ .input = 200, .output = 50, .cache_read = 1000 } }, .{ .text = &.{"again"}, .usage = .{ .input = 10, .output = 5, .cache_read = 1250 } }, }, &priced_model); defer harness.deinit();
try harness.run("go"); try harness.run("more");
const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 3), reports.len);
try testing.expectEqual(provider.Usage{ .input = 1000, .output = 100 }, reports[0].request); try testing.expectEqual(reports[0].request, reports[0].session); try testing.expectApproxEqAbs(@as(f64, 0.003), reports[0].request_cost.?, 1e-12); try testing.expectApproxEqAbs(@as(f64, 0.003), reports[0].session_cost.?, 1e-12); try testing.expectEqual(@as(u64, 1100), reports[0].context_tokens); try testing.expectEqual(@as(u32, 2000), reports[0].context_window);
try testing.expectEqual(provider.Usage{ .input = 200, .output = 50, .cache_read = 1000 }, reports[1].request); try testing.expectEqual(provider.Usage{ .input = 1200, .output = 150, .cache_read = 1000 }, reports[1].session); try testing.expectApproxEqAbs(@as(f64, 0.0019), reports[1].request_cost.?, 1e-12); try testing.expectApproxEqAbs(@as(f64, 0.0049), reports[1].session_cost.?, 1e-12); try testing.expectEqual(@as(u64, 1250), reports[1].context_tokens);
try testing.expectEqual(provider.Usage{ .input = 1210, .output = 155, .cache_read = 2250 }, reports[2].session); try testing.expectEqual(@as(u64, 1265), reports[2].context_tokens);}
test "usage arrives after the reply it belongs to and before the tool results" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"a"}, .calls = &.{toolCall("call-1", "nope", "{}")}, .usage = .{ .input = 1 }, .finish = .tool_calls }, .{ .text = &.{"b"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices( Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .usage, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena), );}
test "a model without a price reports usage without a cost" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"hi"}, .usage = .{ .input = 7, .output = 2 } }}); defer harness.deinit();
try harness.run("go");
const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 1), reports.len); try testing.expectEqual(provider.Usage{ .input = 7, .output = 2 }, reports[0].session); try testing.expectEqual(null, reports[0].request_cost); try testing.expectEqual(null, reports[0].session_cost); try testing.expectEqual(@as(u32, 0), reports[0].context_window);}
test "an interrupted request and a failed request still report their usage" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.initWith(&.{ .{ .text = &.{"par"}, .usage = .{ .input = 30, .output = 3 }, .wait_for_abort = true }, .{ .usage = .{ .input = 40, .output = 4 }, .fail = error.Provider }, }, &priced_model); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try harness.run("second");
const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 2), reports.len); try testing.expectEqual(provider.Usage{ .input = 30, .output = 3 }, reports[0].request); try testing.expectEqual(provider.Usage{ .input = 70, .output = 7 }, reports[1].session);}
test "a prompt queued during a run is answered and runs next in the same busy period" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"one"}, .hold = true }, .{ .text = &.{"two"} }, }); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.submit("second"); harness.script.release(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .prompt_start, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .queued }, harness.log.events.items[4].response); try expectMessages(&.{ .{ .user = "first" }, .{ .assistant = .{ .text = "one", .tool_calls = &.{} } }, .{ .user = "second" }, }, harness.request(1).messages);}
test "a full queue refuses a prompt" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"a"}, .hold = true }}); defer harness.deinit(); try harness.submit("running"); try harness.log.waitFor(.text_delta, 1);
for (0..max_queued) |_| try harness.submit("queued"); try harness.submit("refused"); const events = harness.log.events.items; var queued: usize = 0; for (events) |message| queued += @intFromBool(message == .response and message.response.result == .queued); try testing.expectEqual(max_queued + 1, queued); try testing.expect(events[events.len - 1].response.result == .refused);}
test "an interrupt returns the queued prompts in order and runs none of them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .wait_for_abort = true }}); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.submit("second"); try harness.submit(" "); try harness.submit("third"); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .response, .response, .response, .notice, .prompts_returned, .agent_end }, try harness.log.tags(arena)); const returned = harness.log.events.items[9].prompts_returned; try testing.expectEqual(3, returned.count); try testing.expectEqualStrings("second\n\nthird", returned.text); try testing.expectEqual(1, harness.script.requests.items.len);}
test "stopping the agent drops the queued prompts" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"a"}, .wait_for_abort = true }}); defer harness.deinit();
try harness.submit("first"); try harness.submit("second"); try harness.log.waitFor(.text_delta, 1); harness.agent.stop(); try testing.expectEqual(@as(usize, 1), harness.script.requests.items.len);}
test "stopping the agent during a run ends the run" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .wait_for_abort = true }}); defer harness.deinit();
try harness.submit("go"); try harness.log.waitFor(.text_delta, 1); harness.agent.stop(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[4].notice.kind);}