Something went wrong. Try again.
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721const std = @import("std");
const provider = @import("provider.zig");const models = @import("models.zig");const tools = @import("tools.zig");const Abort = @import("abort.zig").Abort;const protocol = @import("protocol.zig");const Block = protocol.core.Block;const Notice = protocol.core.Notice;const Request = protocol.ui.Request;const UiSink = protocol.Sink(protocol.core.Message);
const Message = provider.Message;const ToolCall = provider.ToolCall;
/// Safety cap for one user prompt.pub const max_requests = 50;
/// How many prompts can wait while the agent is busy.pub const max_queued = 8;
pub const Agent = struct { // Three tasks run the agent, and none of them holds a lock. The owner holds // the attached UI and the queued prompts, and only it sends to the UI. The // runner runs one prompt at a time and sends through the owner. The // forwarder moves requests from the inbox to the owner, so the runner's // messages cannot take the room that requests need. The UI never waits on // the agent, so the waits cannot form a cycle. gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ, providers: Providers, system_prompt: []const u8, models: []const models.Entry, /// The level the user asked for. Only the owner uses it after the start. wanted_level: models.Level, /// The model and the level of the next request. The owner writes it and /// the runner reads it. selection: std.atomic.Value(u32), cwd: []const u8, /// Only the runner uses it. history: std.ArrayList(Message) = .empty, /// What every request so far used. Only the runner uses it. session: provider.Usage = .{}, /// Null once a request used a model without a price. session_cost: ?f64 = 0, /// Ends the running prompt. The owner clears it only before it gives the /// runner a prompt, when nothing waits on it. abort: Abort = .{}, /// Where the runner sends its messages for the UI. out: UiSink, inbox: std.Io.Queue(Request), inbox_buffer: [protocol.inbox_capacity]Request = undefined, /// Everything the owner handles. Never closed. events: std.Io.Queue(Event), events_buffer: [64]Event = undefined, /// Gives the runner its next prompt. prompts: std.Io.Queue([]const u8), prompt_buffer: [1][]const u8 = undefined, runner_task: ?std.Io.Future(void) = null, owner_task: ?std.Io.Future(void) = null, forwarder_task: ?std.Io.Future(void) = null,
const Event = union(enum) { request: Request, message: protocol.core.Message, /// The runner finished its prompt. done, stop, };
pub const Options = struct { /// The environment of child processes. environ: std.process.Environ, /// A model can be selected only when its provider is set here. providers: Providers, system_prompt: []const u8, /// The models a UI can select. Must outlive the agent. models: []const models.Entry = &models.entries, /// The index in `models` of the model to start with. Its provider /// must be set. model: usize, /// The reasoning level to start with. The agent uses the nearest /// level that the model supports. level: models.Level = models.default_level, cwd: []const u8, };
pub const Providers = std.EnumArray(models.Provider, ?provider.Provider);
const Selection = struct { /// An index in `models`. model: u16, level: models.Level,
fn pack(self: Selection) u32 { return @as(u32, self.model) << 8 | @backingInt(self.level); }
fn unpack(bits: u32) Selection { return .{ .model = @intCast(bits >> 8), .level = @fromBackingInt(@intCast(@as(std.meta.Tag(models.Level), @truncate(bits)))) }; } };
/// Allocates the agent with `gpa`. `deinit` frees it. The strings in /// `options` must outlive the agent. pub fn init(gpa: std.mem.Allocator, io: std.Io, options: Options) !*Agent { const self = try gpa.create(Agent); self.* = .{ .gpa = gpa, .io = io, .environ = options.environ, .providers = options.providers, .system_prompt = options.system_prompt, .models = options.models, .wanted_level = options.level, .selection = .init(Selection.pack(.{ .model = @intCast(options.model), .level = models.clamp(&options.models[options.model], options.level), })), .cwd = options.cwd, .out = .{ .ctx = self, .sendFn = post }, .inbox = .init(&self.inbox_buffer), .events = .init(&self.events_buffer), .prompts = .init(&self.prompt_buffer), }; return self; }
/// Starts reading the requests that arrive through `sink()` and running /// the prompts they submit. No UI is attached yet. pub fn start(self: *Agent) !void { errdefer self.stop(); self.runner_task = try self.io.concurrent(runnerMain, .{self}); self.owner_task = try self.io.concurrent(Owner.main, .{self}); self.forwarder_task = try self.io.concurrent(forwarderMain, .{self}); }
/// Where a UI sends its requests. Valid after `start`. Never blocks, so a /// UI can send from its event loop. Refuses a request when the inbox is /// full or the agent stopped. pub fn sink(self: *Agent) protocol.Sink(Request) { return .{ .ctx = self, .sendFn = sendRequest }; }
fn sendRequest(ctx: *anyopaque, request: Request) bool { const self: *Agent = @ptrCast(@alignCast(ctx)); return (self.inbox.putUncancelable(self.io, &.{request}, 0) catch 0) == 1; }
/// Ends the running prompt, stops the tasks and frees the prompts that /// wait. An attached UI must not block forever meanwhile. pub fn stop(self: *Agent) void { self.inbox.close(self.io); if (self.forwarder_task) |*task| { task.await(self.io); self.forwarder_task = null; } // After the last request, so the owner handles none after the stop. if (self.owner_task) |*task| { self.events.putOneUncancelable(self.io, .stop) catch unreachable; task.await(self.io); self.owner_task = null; } self.prompts.close(self.io); if (self.runner_task) |*task| { task.await(self.io); self.runner_task = null; } }
pub fn deinit(self: *Agent) void { self.stop(); for (self.history.items) |message| message.deinit(self.gpa); self.history.deinit(self.gpa); self.gpa.destroy(self); }
fn forwarderMain(self: *Agent) void { while (self.inbox.getOneUncancelable(self.io)) |request| { self.events.putOneUncancelable(self.io, .{ .request = request }) catch unreachable; } else |_| {} }
fn runnerMain(self: *Agent) void { while (self.prompts.getOneUncancelable(self.io)) |prompt| { self.runInner(prompt) catch |err| switch (err) { error.OutOfMemory => self.sendNotice(.err, "out of memory"), }; self.events.putOneUncancelable(self.io, .done) catch unreachable; } else |_| {} }
fn post(ctx: *anyopaque, message: protocol.core.Message) bool { const self: *Agent = @ptrCast(@alignCast(ctx)); self.events.putOneUncancelable(self.io, .{ .message = message }) catch unreachable; return true; }
fn send(self: *Agent, message: protocol.core.Message) void { if (!self.out.send(message)) { message.deinit(self.gpa); } }
fn sendNotice(self: *Agent, kind: Notice.Kind, text: []const u8) void { // A provider detail can hold server text that is not UTF-8, or end inside a code point. const copy = std.fmt.allocPrint(self.gpa, "{f}", .{std.unicode.fmtUtf8(text)}) catch return; self.send(.{ .notice = .{ .kind = kind, .text = copy } }); }
/// Runs one prompt and takes ownership of it. fn runInner(self: *Agent, prompt: []const u8) !void { self.history.append(self.gpa, .{ .user = prompt }) catch |err| { self.gpa.free(prompt); return err; };
var requests: usize = 0; while (true) { if (requests == max_requests) { self.sendNotice(.err, "request limit reached, the run ended"); return; } requests += 1;
// A change of the selection applies from the next request. const selection: Selection = .unpack(self.selection.load(.acquire)); const model = &self.models[selection.model]; var stream: Stream = .{ .agent = self }; defer stream.deinit(); var diagnostics: provider.Diagnostics = .{}; const reply = self.providers.get(model.provider).?.stream( .{ .system = self.system_prompt, .messages = self.history.items, .tools = &tools.definitions, .model = model, .level = selection.level, }, stream.sink(), &self.abort, &diagnostics, ); // A request that failed or was interrupted was billed too. if (stream.usage) |usage| { self.recordUsage(model, usage); } const finish = reply catch |err| { if (self.abort.isSet()) { self.sendNotice(.interrupted, "interrupted"); return; } if (err == error.OutOfMemory) { return error.OutOfMemory; } const detail = diagnostics.message(); self.sendNotice(.err, if (detail.len == 0) "provider error" else detail); return; };
// A partial answer is not part of the history: it may hold incomplete tool calls. if (self.abort.isSet()) { self.sendNotice(.interrupted, "interrupted"); return; } if (stream.out_of_memory) { return error.OutOfMemory; }
const calls = calls: { try self.history.ensureUnusedCapacity(self.gpa, 1); const text = try stream.text.toOwnedSlice(self.gpa); errdefer self.gpa.free(text); const thinking = try stream.thinking.toOwnedSlice(self.gpa); errdefer self.gpa.free(thinking); const calls = try stream.calls.toOwnedSlice(self.gpa); self.history.appendAssumeCapacity(.{ .assistant = .{ .text = text, .thinking = thinking, .tool_calls = calls } }); break :calls calls; };
if (finish == .filtered or finish == .other) { for (calls) |call| try self.recordPlain(call.id, "not run: the reply ended early"); if (finish == .filtered) { self.sendNotice(.err, "the provider's content filter stopped the reply"); } else if (diagnostics.message().len == 0) { self.sendNotice(.err, "the reply ended early"); } else { const prefix = "the reply ended early: "; var buffer: [prefix.len + diagnostics.buffer.len]u8 = undefined; self.sendNotice(.err, std.fmt.bufPrint(&buffer, prefix ++ "{s}", .{diagnostics.message()}) catch unreachable); } return; }
if (calls.len == 0) { if (finish == .length) { self.sendNotice(.err, "the reply reached the token limit"); } return; }
if (finish == .length) { for (calls) |call| try self.recordPlain(call.id, "the output was truncated; re-issue the call"); continue; }
for (calls, 0..) |call, index| { // The flag can be set between two calls, so it is checked before each one. if (self.abort.isSet()) { try self.skipRemaining(calls[index..]); self.sendNotice(.interrupted, "interrupted"); return; } var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); const result = tools.execute( .{ .io = self.io, .environ = self.environ, .abort = &self.abort }, arena_state.allocator(), call.name, call.args_json, ); // A tool reports an interrupt in its own result, and work it finished still happened. try self.recordResult(call.id, result); if (self.abort.isSet()) { try self.skipRemaining(calls[index + 1 ..]); self.sendNotice(.interrupted, "interrupted"); return; } } } }
/// Adds the usage of one request to the session and tells the UI. fn recordUsage(self: *Agent, model: *const models.Entry, usage: provider.Usage) void { const request_cost: ?f64 = if (model.cost) |prices| models.cost(prices, usage) else null; self.session.add(usage); self.session_cost = if (self.session_cost != null and request_cost != null) self.session_cost.? + request_cost.? else null; self.send(.{ .usage = .{ .request = usage, .request_cost = request_cost, .session = self.session, .session_cost = self.session_cost, .context_tokens = usage.total(), .context_window = model.context_window, } }); }
fn skipRemaining(self: *Agent, calls: []const ToolCall) !void { const skip = "not run: aborted by user"; for (calls) |call| try self.recordPlain(call.id, skip); }
/// Records a failed result that the agent wrote itself. fn recordPlain(self: *Agent, call_id: []const u8, text: []const u8) !void { var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); try self.recordResult(call_id, tools.plainResult(arena_state.allocator(), text, true)); }
fn recordResult(self: *Agent, call_id: []const u8, result: tools.Result) !void { try self.history.ensureUnusedCapacity(self.gpa, 1); const id = try self.gpa.dupe(u8, call_id); errdefer self.gpa.free(id); // Tool output can hold any bytes, and the request JSON needs UTF-8. const text = if (std.unicode.utf8ValidateSlice(result.text)) try self.gpa.dupe(u8, result.text) else try std.fmt.allocPrint(self.gpa, "{f}", .{std.unicode.fmtUtf8(result.text)}); self.history.appendAssumeCapacity(.{ .tool_result = .{ .call_id = id, .text = text } }); // A call without a result breaks every later request, so a failure here only hides the result from the UI. self.showResult(call_id, result) catch {}; }
fn showResult(self: *Agent, call_id: []const u8, result: tools.Result) !void { const id = try self.gpa.dupe(u8, call_id); errdefer self.gpa.free(id); const collapsed: Block = try .init(self.gpa, result.collapsed); errdefer collapsed.deinit(self.gpa); const expanded: Block = try .init(self.gpa, result.expanded); self.send(.{ .tool_result = .{ .call_id = id, .is_error = result.is_error, .collapsed = collapsed, .expanded = expanded, }, }); }};
/// The state of the owner task. No other task can reach it.const Owner = struct { agent: *Agent, ui: ?UiSink = null, /// The prompts that wait for the runner. queued: std.ArrayList([]const u8), /// The runner has a prompt. running: bool = false, stopping: bool = false,
fn main(agent: *Agent) void { var buffer: [max_queued][]const u8 = undefined; var self: Owner = .{ .agent = agent, .queued = .initBuffer(&buffer) }; while (!self.stopping or self.running) { switch (agent.events.getOneUncancelable(agent.io) catch unreachable) { .request => |request| self.handle(request), .message => |message| self.send(message), .done => self.finish(), .stop => self.stop(), } } }
fn handle(self: *Owner, request: Request) void { const id = request.id; switch (request.method) { .attach => |ui| self.attach(id, ui), .detach => |ui| self.detach(id, ui), .submit => |prompt| self.submit(id, prompt), .interrupt => { self.respond(id, .ok); if (self.running) { self.agent.abort.set(self.agent.io); } }, .select_model => |name| { defer self.agent.gpa.free(name); const index = models.indexOf(self.agent.models, name) orelse return self.respond(id, .unknown_model); if (!self.selectable(index)) { return self.respond(id, .unknown_model); } self.selectModel(id, index); }, .cycle_model => { const count = self.agent.models.len; var index: usize = self.selection().model; // The selected model can be selected, so the search ends there at the latest. for (0..count) |_| { index = (index + 1) % count; if (self.selectable(index)) { break; } } self.selectModel(id, index); }, .cycle_level => { const current = self.selection(); const level = models.nextLevel(&self.agent.models[current.model], current.level); // A model with one level keeps what the user asked for, for the next model. if (level != current.level) { self.agent.wanted_level = level; } self.select(id, .{ .model = current.model, .level = level }); }, } }
fn selection(self: *Owner) Agent.Selection { return .unpack(self.agent.selection.load(.acquire)); }
fn selectable(self: *Owner, index: usize) bool { return self.agent.providers.get(self.agent.models[index].provider) != null; }
/// Selects a model, at the level nearest to the one the user asked for. fn selectModel(self: *Owner, id: protocol.RequestId, index: usize) void { self.select(id, .{ .model = @intCast(index), .level = models.clamp(&self.agent.models[index], self.agent.wanted_level), }); }
/// Makes `selected` the selection of the next request, tells the UI and /// then answers the request that asked for it. fn select(self: *Owner, id: protocol.RequestId, selected: Agent.Selection) void { self.agent.selection.store(selected.pack(), .release); defer self.respond(id, .ok); const entry = &self.agent.models[selected.model]; const model = self.agent.gpa.dupe(u8, entry.name) catch return; self.send(.{ .selection = .{ .model = model, .level = if (models.reasons(entry)) selected.level else null } }); }
fn attach(self: *Owner, id: protocol.RequestId, ui: UiSink) void { if (self.ui != null) { _ = ui.send(.{ .response = .{ .id = id, .result = .already_attached } }); return; } const message: protocol.core.Message = .{ .response = .{ .id = id, .result = .{ .attached = self.session() }, }, }; if (!ui.send(message)) { message.deinit(self.agent.gpa); return; } self.ui = ui; }
/// A copy of the session for a UI. Empty when memory runs out, because /// the attach must still get its answer. fn session(self: *Owner) protocol.core.Session { const gpa = self.agent.gpa; const selected = self.selection(); const entry = &self.agent.models[selected.model]; const model = gpa.dupe(u8, entry.name) catch return .{ .model = "", .cwd = "" }; const cwd = gpa.dupe(u8, self.agent.cwd) catch { gpa.free(model); return .{ .model = "", .cwd = "" }; }; return .{ .model = model, .level = if (models.reasons(entry)) selected.level else null, .cwd = cwd }; }
fn detach(self: *Owner, id: protocol.RequestId, ui: UiSink) void { if (self.ui) |attached| { if (attached.eql(ui)) { self.ui = null; } } _ = ui.send(.{ .response = .{ .id = id, .result = .ok } }); }
/// Runs or queues a prompt and takes ownership of it. fn submit(self: *Owner, id: protocol.RequestId, prompt: []const u8) void { if (!self.running) { self.respond(id, .queued); self.send(.agent_start); return self.startPrompt(prompt); } self.queued.appendBounded(prompt) catch { self.agent.gpa.free(prompt); return self.respond(id, .refused); }; self.respond(id, .queued); }
fn startPrompt(self: *Owner, prompt: []const u8) void { self.send(.prompt_start); self.agent.abort.reset(); // The queue is empty while no prompt runs, and it closes only after the owner returns. self.agent.prompts.putOneUncancelable(self.agent.io, prompt) catch unreachable; self.running = true; }
fn finish(self: *Owner) void { self.running = false; if (self.agent.abort.isSet()) { self.returnQueued(); } if (self.queued.items.len > 0) { return self.startPrompt(self.queued.orderedRemove(0)); } self.send(.agent_end); }
fn stop(self: *Owner) void { self.stopping = true; for (self.queued.items) |prompt| self.agent.gpa.free(prompt); self.queued.clearRetainingCapacity(); self.agent.abort.set(self.agent.io); }
/// Sends the text of every queued prompt back and takes them off the queue. fn returnQueued(self: *Owner) void { const gpa = self.agent.gpa; const taken = self.queued.items; if (taken.len == 0) { return; } defer { for (taken) |prompt| gpa.free(prompt); self.queued.clearRetainingCapacity(); } var parts: [max_queued][]const u8 = undefined; var kept: usize = 0; for (taken) |prompt| { if (std.mem.trim(u8, prompt, &std.ascii.whitespace).len == 0) { continue; } parts[kept] = prompt; kept += 1; } const text = std.mem.join(gpa, "\n\n", parts[0..kept]) catch text: { self.sendNotice(.err, "out of memory; the queued prompts were lost"); break :text ""; }; self.send(.{ .prompts_returned = .{ .count = taken.len, .text = text } }); }
fn respond(self: *Owner, id: protocol.RequestId, result: protocol.core.Response.Result) void { self.send(.{ .response = .{ .id = id, .result = result } }); }
/// Sends a message to the attached UI, or drops it when none is attached. fn send(self: *Owner, message: protocol.core.Message) void { if (self.ui) |ui| { if (ui.send(message)) { return; } } message.deinit(self.agent.gpa); }
fn sendNotice(self: *Owner, kind: Notice.Kind, text: []const u8) void { const copy = self.agent.gpa.dupe(u8, text) catch return; self.send(.{ .notice = .{ .kind = kind, .text = copy } }); }};
/// Collects the streamed answer and forwards every delta toward the UI.const Stream = struct { agent: *Agent, text: std.ArrayList(u8) = .empty, thinking: std.ArrayList(u8) = .empty, calls: std.ArrayList(ToolCall) = .empty, usage: ?provider.Usage = null, /// Part of the reply was dropped. out_of_memory: bool = false,
fn deinit(self: *Stream) void { const gpa = self.agent.gpa; self.text.deinit(gpa); self.thinking.deinit(gpa); for (self.calls.items) |call| call.deinit(gpa); self.calls.deinit(gpa); }
fn sink(self: *Stream) provider.StreamSink { return .{ .ctx = self, .text_delta = onText, .thinking_delta = onThinking, .tool_call = onCall, .usage = onUsage }; }
fn onUsage(ctx: *anyopaque, usage: provider.Usage) void { const self: *Stream = @ptrCast(@alignCast(ctx)); self.usage = usage; }
fn onText(ctx: *anyopaque, delta: []const u8) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; self.text.appendSlice(gpa, delta) catch return self.lose(); const copy = gpa.dupe(u8, delta) catch return; self.agent.send(.{ .text_delta = copy }); }
fn onThinking(ctx: *anyopaque, delta: []const u8) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; self.thinking.appendSlice(gpa, delta) catch return self.lose(); const copy = gpa.dupe(u8, delta) catch return; self.agent.send(.{ .thinking_delta = copy }); }
fn onCall(ctx: *anyopaque, call: ToolCall) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; const owned = call.dupe(gpa) catch return self.lose(); self.calls.append(gpa, owned) catch { owned.deinit(gpa); return self.lose(); }; self.show(call) catch {}; }
fn lose(self: *Stream) void { self.out_of_memory = true; }
fn show(self: *Stream, call: ToolCall) !void { const gpa = self.agent.gpa; var arena_state = std.heap.ArenaAllocator.init(gpa); defer arena_state.deinit(); const shown = try tools.renderCall(arena_state.allocator(), call.name, call.args_json); const id = try gpa.dupe(u8, call.id); errdefer gpa.free(id); self.agent.send(.{ .tool_call = .{ .id = id, .call = try .init(gpa, shown) } }); }};
const testing = std.testing;const helpers = @import("testing.zig");const Scripted = helpers.Scripted;const EventLog = helpers.EventLog(protocol.core.Message);const toolCall = helpers.toolCall;
const test_model: models.Entry = .{ .name = "model", .id = "provider/model", .provider = .fake, .max_output = 0 };
fn testOptions(streamer: provider.Provider) Agent.Options { var providers: Agent.Providers = .initFill(null); providers.set(.fake, streamer); return .{ .environ = testing.environ, .providers = providers, .system_prompt = "system", .models = (&test_model)[0..1], .model = 0, .cwd = "/work" };}
/// A started agent with a scripted provider and an attached message log. Do/// not move it after `init`.const Harness = struct { log: EventLog, script: Scripted, agent: *Agent, next_id: protocol.RequestId,
fn init(self: *Harness, steps: []const Scripted.Step) !void { return self.initWith(steps, &test_model); }
fn initWith(self: *Harness, steps: []const Scripted.Step, model: *const models.Entry) !void { return self.initModels(steps, model[0..1], models.default_level); }
/// Starts with the first of `entries`. fn initModels(self: *Harness, steps: []const Scripted.Step, entries: []const models.Entry, level: models.Level) !void { self.log = .{}; self.script = .init(steps); self.next_id = 0; var options = testOptions(self.script.asProvider()); options.models = entries; options.level = level; self.agent = try Agent.init(testing.allocator, testing.io, options); try self.agent.start(); try self.send(.{ .attach = self.log.sink() }); // The attach response is the first message; its own test checks it. try self.log.waitFor(.response, 1); for (self.log.events.items) |event| event.deinit(testing.allocator); self.log.events.clearRetainingCapacity(); }
fn deinit(self: *Harness) void { // A failed test must not leave the agent task in a held step. self.script.release(); self.agent.deinit(); self.script.deinit(); self.log.deinit(); }
/// Submits a prompt and waits until the agent is idle again. fn run(self: *Harness, prompt: []const u8) !void { self.log.collect(); const ends = self.log.count(.agent_end); try self.submit(prompt); try self.log.waitFor(.agent_end, ends + 1); }
/// Submits a prompt and waits until the agent answers. fn submit(self: *Harness, prompt: []const u8) !void { self.log.collect(); const answered = self.log.count(.response); try self.send(.{ .submit = try testing.allocator.dupe(u8, prompt) }); try self.log.waitFor(.response, answered + 1); }
/// Sends a request and returns the answer to it. fn ask(self: *Harness, method: protocol.ui.Request.Method) !protocol.core.Response.Result { self.log.collect(); const answered = self.log.count(.response); const id = self.next_id; try self.send(method); try self.log.waitFor(.response, answered + 1); var index = self.log.events.items.len; while (index > 0) { index -= 1; switch (self.log.events.items[index]) { .response => |response| if (response.id == id) { return response.result; }, else => {}, } } return error.NoResponse; }
/// The last selection that the agent reported. fn selection(self: *Harness) ?protocol.core.Selection { var index = self.log.events.items.len; while (index > 0) { index -= 1; switch (self.log.events.items[index]) { .selection => |selected| return selected, else => {}, } } return null; }
/// Sends a request and takes ownership of its method. fn send(self: *Harness, method: protocol.ui.Request.Method) !void { const message: protocol.ui.Request = .{ .id = self.next_id, .method = method }; self.next_id += 1; if (!self.agent.sink().send(message)) { message.deinit(testing.allocator); return error.Closed; } }
/// Interrupts the agent and waits until the running prompt sees it. fn interrupt(self: *Harness) !void { try self.send(.interrupt); if (!self.agent.abort.wait(testing.io, .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } })) { return error.Timeout; } }
fn request(self: *Harness, index: usize) Scripted.Request { return self.script.requests.items[index]; }
/// The usage reports so far, in arrival order. fn usage(self: *Harness, arena: std.mem.Allocator) ![]const protocol.core.UsageReport { var reports: std.ArrayList(protocol.core.UsageReport) = .empty; for (self.log.events.items) |event| switch (event) { .usage => |report| try reports.append(arena, report), else => {}, }; return reports.items; }};
/// Waits until a writer sends one line through the FIFO at `path`.fn waitForLine(dir: std.Io.Dir, path: []const u8) !void { const Done = union(enum) { line: anyerror!void, late: std.Io.Cancelable!void }; var buffer: [2]Done = undefined; var select: std.Io.Select(Done) = .init(testing.io, &buffer); defer select.cancelDiscard(); try select.concurrent(.line, readLine, .{ dir, path }); try select.concurrent(.late, std.Io.sleep, .{ testing.io, .fromSeconds(10), .awake }); switch (try select.await()) { .line => |result| try result, .late => return error.Timeout, }}
fn readLine(dir: std.Io.Dir, path: []const u8) anyerror!void { const file = try dir.openFile(testing.io, path, .{}); defer file.close(testing.io); var buffer: [64]u8 = undefined; var reader = file.reader(testing.io, &buffer); _ = try reader.interface.takeDelimiterInclusive('\n');}
fn expectMessages(expected: []const Message, actual: []const Message) !void { try testing.expectEqual(expected.len, actual.len); for (expected, actual) |want, got| { try testing.expectEqual(std.meta.activeTag(want), std.meta.activeTag(got)); switch (want) { .user => |text| try testing.expectEqualStrings(text, got.user), .assistant => |assistant| { try testing.expectEqualStrings(assistant.text, got.assistant.text); try testing.expectEqualStrings(assistant.thinking, got.assistant.thinking); try testing.expectEqual(assistant.tool_calls.len, got.assistant.tool_calls.len); for (assistant.tool_calls, got.assistant.tool_calls) |want_call, got_call| { try testing.expectEqualStrings(want_call.id, got_call.id); try testing.expectEqualStrings(want_call.name, got_call.name); try testing.expectEqualStrings(want_call.args_json, got_call.args_json); } }, .tool_result => |result| { try testing.expectEqualStrings(result.call_id, got.tool_result.call_id); try testing.expectEqualStrings(result.text, got.tool_result.text); }, } }}
const Tag = EventLog.Tag;
test "a text reply streams between agent_start and agent_end" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{ "po", "ng" } }}); defer harness.deinit();
try harness.run("ping");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .queued }, harness.log.events.items[0].response); try testing.expectEqualStrings("pong", try harness.log.text(arena));}
test "attach answers with the model and the working directory" { var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{}); defer script.deinit(); const agent = try Agent.init(testing.allocator, testing.io, testOptions(script.asProvider())); defer agent.deinit(); try agent.start();
try testing.expect(agent.sink().send(.{ .id = 7, .method = .{ .attach = log.sink() } }));
try log.waitFor(.response, 1); const response = log.events.items[0].response; try testing.expectEqual(7, response.id); try testing.expectEqualStrings("model", response.result.attached.model); try testing.expectEqual(null, response.result.attached.level); try testing.expectEqualStrings("/work", response.result.attached.cwd);}
test "a second attach is answered on its own sink and leaves the first UI attached" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); var second: EventLog = .{}; defer second.deinit();
try harness.send(.{ .attach = second.sink() }); try second.waitFor(.response, 1); try harness.run("ping");
second.collect(); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .already_attached }, second.events.items[0].response); try testing.expectEqual(1, second.events.items.len); try testing.expectEqualStrings("pong", try harness.log.text(arena_state.allocator()));}
test "the detach response is the last message to the sink, and the running prompt continues for the next UI" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"}, .hold = true }}); defer harness.deinit();
try harness.submit("ping"); try harness.log.waitFor(.text_delta, 1); try harness.send(.{ .detach = harness.log.sink() }); try harness.log.waitFor(.response, 2); var next: EventLog = .{}; defer next.deinit(); try harness.send(.{ .attach = next.sink() }); try next.waitFor(.response, 1); harness.script.release(); try next.waitFor(.agent_end, 1);
harness.log.collect(); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .ok }, harness.log.events.items[4].response);}
test "a detach from a UI that is not attached is answered and leaves the attached UI" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); var other: EventLog = .{}; defer other.deinit();
try harness.send(.{ .detach = other.sink() }); try other.waitFor(.response, 1); try harness.run("ping");
try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .ok }, other.events.items[0].response); try testing.expectEqualStrings("pong", try harness.log.text(arena_state.allocator()));}
test "the first request holds the system prompt, the prompt and every tool" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit();
try harness.run("ping");
const request = harness.request(0); try testing.expectEqualStrings("system", request.system); try expectMessages(&.{.{ .user = "ping" }}, request.messages); try testing.expectEqual(tools.definitions.len, request.tools.len); for (tools.definitions, request.tools) |definition, name| try testing.expectEqualStrings(definition.name, name);}
test "a tool call runs and its result goes back to the model" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.path("out.txt"); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "write", args)}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqualStrings("hi", try dir.read("out.txt"));
const wrote = try std.fmt.allocPrint(arena, "wrote 2 bytes to {s}", .{path}); const result = harness.log.events.items[4].tool_result; try testing.expectEqualStrings("call-1", result.call_id); try testing.expect(!result.is_error);
try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "", .tool_calls = &.{toolCall("call-1", "write", args)} } }, .{ .tool_result = .{ .call_id = "call-1", .text = wrote } }, }, harness.request(1).messages);}
test "calls in one reply run in order and each gets its own result" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.path("out.txt"); const first = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "first" }, .{}); const second = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "second!" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"writing"}, .calls = &.{ toolCall("call-a", "write", first), toolCall("call-b", "write", second) }, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualStrings("second!", try dir.read("out.txt")); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "writing", .tool_calls = &.{ toolCall("call-a", "write", first), toolCall("call-b", "write", second) } } }, .{ .tool_result = .{ .call_id = "call-a", .text = try std.fmt.allocPrint(arena, "wrote 5 bytes to {s}", .{path}) } }, .{ .tool_result = .{ .call_id = "call-b", .text = try std.fmt.allocPrint(arena, "wrote 7 bytes to {s}", .{path}) } }, }, harness.request(1).messages);}
test "invalid UTF-8 in a tool result reaches the model and the UI as U+FFFD" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("latin1.txt", "caf\xe9\n"); const args = try std.json.Stringify.valueAlloc(arena, .{ .files = &.{.{ .path = path }} }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "read", args)}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
const shown = harness.log.events.items[4].tool_result.expanded.text; const sent = harness.request(1).messages[2].tool_result.text; try testing.expect(std.unicode.utf8ValidateSlice(shown)); try testing.expect(std.mem.indexOf(u8, shown, "1| caf\u{fffd}\n") != null); try testing.expect(std.unicode.utf8ValidateSlice(sent)); try testing.expect(std.mem.indexOf(u8, sent, "1| caf\u{fffd}\n") != null);}
test "a failed tool call is a result for the model, not the end of the run" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"sorry"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena)); const result = harness.log.events.items[4].tool_result; try testing.expect(result.is_error); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "", .tool_calls = &.{toolCall("call-1", "nope", "{}")} } }, .{ .tool_result = .{ .call_id = "call-1", .text = "unknown tool: nope" } }, }, harness.request(1).messages);}
/// Fails one allocation, the one with index `fail_index`.const OneFailure = struct { fail_index: usize, index: usize = 0,
fn allocator(self: *OneFailure) std.mem.Allocator { return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = std.mem.Allocator.noResize, .remap = std.mem.Allocator.noRemap, .free = free } }; }
fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { const self: *OneFailure = @ptrCast(@alignCast(ctx)); defer self.index += 1; if (self.index == self.fail_index) { return null; } return testing.allocator.rawAlloc(len, alignment, ret_addr); }
fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { _ = ctx; testing.allocator.rawFree(memory, alignment, ret_addr); }};
test "a failed allocation ends the run with the error or keeps the whole reply, and leaks nothing" { var fail_index: usize = 0; while (true) : (fail_index += 1) { var failing: OneFailure = .{ .fail_index = fail_index }; const gpa = failing.allocator(); var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{ .{ .text = &.{ "a", "b" }, .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer script.deinit(); const agent = Agent.init(gpa, testing.io, testOptions(script.asProvider())) catch continue; defer agent.deinit();
agent.out = log.sink(); const prompt = gpa.dupe(u8, "go") catch continue; agent.runInner(prompt) catch |err| { try testing.expectEqual(error.OutOfMemory, err); continue; }; const history = agent.history.items; try testing.expectEqual(4, history.len); try expectMessages(&.{.{ .assistant = .{ .text = "ab", .tool_calls = &.{toolCall("call-1", "nope", "{}")} } }}, history[1..2]); try expectMessages(&.{.{ .assistant = .{ .text = "done", .tool_calls = &.{} } }}, history[3..]); if (failing.index <= fail_index) { break; } }}
test "a reply cut off by the length limit answers its calls without running them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "write", args)}, .finish = .length }, .{ .text = &.{"done"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectError(error.FileNotFound, dir.read("out.txt")); const result = harness.log.events.items[4].tool_result; try testing.expect(result.is_error); try testing.expect(std.mem.indexOf(u8, result.expanded.text, "truncated") != null); const messages = harness.request(1).messages; try testing.expectEqual(@as(usize, 3), messages.len); try testing.expectEqualStrings("call-1", messages[2].tool_result.call_id);}
test "a reply cut off by the length limit without calls ends with a notice" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .finish = .length }}); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[4].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the reply reached the token limit", notice.text);}
test "a reply stopped by the content filter ends with a notice and answers its calls without running them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .calls = &.{toolCall("call-1", "write", args)}, .finish = .filtered }, .{ .text = &.{"next"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .tool_result, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectError(error.FileNotFound, dir.read("out.txt")); const result = harness.log.events.items[5].tool_result; try testing.expect(result.is_error); try testing.expectEqualStrings("not run: the reply ended early", result.expanded.text); const notice = harness.log.events.items[6].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the provider's content filter stopped the reply", notice.text);
try harness.run("again"); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "par", .tool_calls = &.{toolCall("call-1", "write", args)} } }, .{ .tool_result = .{ .call_id = "call-1", .text = "not run: the reply ended early" } }, .{ .user = "again" }, }, harness.request(1).messages);}
test "a reply that ends for an unknown reason ends with a notice that names the reason" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .finish = .other, .detail = "finish reason: overloaded" }}); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[4].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the reply ended early: finish reason: overloaded", notice.text);}
test "a provider error ends the run with a notice that holds its detail as valid UTF-8" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .fail = error.Http, .detail = "503 overloaded \xff" }}); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[3].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("503 overloaded \u{FFFD}", notice.text);}
test "a run stops after max_requests requests" { var harness: Harness = undefined; const step: Scripted.Step = .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }; const steps: [max_requests + 1]Scripted.Step = @splat(step); try harness.init(&steps); defer harness.deinit();
try harness.run("go");
try testing.expectEqual(@as(usize, max_requests), harness.script.requests.items.len); const events = harness.log.events.items; const notice = events[events.len - 2].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expect(std.mem.indexOf(u8, notice.text, "request limit") != null);}
test "an interrupt while streaming keeps the partial answer out of the history" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .wait_for_abort = true }, .{ .text = &.{"pong"} }, }); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .ok }, harness.log.events.items[4].response); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[5].notice.kind);
try harness.run("second"); try expectMessages(&.{ .{ .user = "first" }, .{ .user = "second" } }, harness.request(1).messages);}
test "an interrupt that the provider does not report drops the reply and its calls" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{});
var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .calls = &.{toolCall("call-1", "write", args)}, .hold = true, .finish = .tool_calls }, .{ .text = &.{"pong"} }, }); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.tool_call, 1); try harness.interrupt(); harness.script.release(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .response, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[6].notice.kind); try testing.expectError(error.FileNotFound, dir.read("out.txt"));
try harness.run("second"); try expectMessages(&.{ .{ .user = "first" }, .{ .user = "second" } }, harness.request(1).messages);}
test "an interrupt during a tool call keeps its own result and skips the calls after it" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); try testing.expect(!dir.call("bash", .{ .command = "mkfifo started", .timeout = 10, .cwd = dir.root }).is_error); const slow = try std.json.Stringify.valueAlloc(arena, .{ .command = "echo partial; echo > started; sleep 30", .timeout = 60, .cwd = dir.root }, .{}); const fast = try std.json.Stringify.valueAlloc(arena, .{ .command = "echo two", .timeout = 10, .cwd = dir.root }, .{}); const calls = [_]ToolCall{ toolCall("call-1", "bash", slow), toolCall("call-2", "bash", fast) };
var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &calls, .finish = .tool_calls }, .{ .text = &.{"after"} }, }); defer harness.deinit();
const started = std.Io.Timestamp.now(testing.io, .awake); try harness.submit("go"); try waitForLine(dir.tmp.dir, "started"); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try testing.expect(started.untilNow(testing.io, .awake).nanoseconds < 10 * std.time.ns_per_s);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_call, .response, .tool_result, .tool_result, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[8].notice.kind); const first = harness.log.events.items[6].tool_result; try testing.expect(!first.is_error); try testing.expectStringStartsWith(first.expanded.text, "partial\n[aborted by user, elapsed "); const second = harness.log.events.items[7].tool_result; try testing.expect(second.is_error); try testing.expectEqualStrings("not run: aborted by user", second.expanded.text);
try harness.run("next"); const messages = harness.request(1).messages; try testing.expectEqual(5, messages.len); try testing.expectStringStartsWith(messages[2].tool_result.text, "partial\n[aborted by user, elapsed "); try expectMessages(&.{ .{ .tool_result = .{ .call_id = "call-2", .text = "not run: aborted by user" } }, .{ .user = "next" }, }, messages[3..]);}
test "thinking is shown and goes back to the model with its reply" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .thinking = &.{ "h", "m" }, .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"answer"} }, .{ .text = &.{"two"} }, }); defer harness.deinit();
try harness.run("first"); try testing.expectEqualSlices( Tag, &.{ .response, .agent_start, .prompt_start, .thinking_delta, .thinking_delta, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena), ); const sent = harness.request(1).messages; try testing.expectEqualStrings("hm", sent[1].assistant.thinking);
try harness.run("second"); try testing.expectEqualStrings("hm", harness.request(2).messages[1].assistant.thinking); try testing.expectEqualStrings("", harness.request(2).messages[3].assistant.thinking);}
test "each request names the model and a reasoning level that the model supports" { const leveled: models.Entry = .{ .name = "leveled", .id = "provider/leveled", .provider = .fake, .levels = .init(.{ .off = null, .low = "l", .medium = null, .high = "h", .max = null }), .max_output = 0, }; var harness: Harness = undefined; try harness.initWith(&.{.{ .text = &.{"a"} }}, &leveled); defer harness.deinit();
try harness.run("go");
try testing.expectEqualStrings("leveled", harness.request(0).model); try testing.expectEqual(models.Level.high, harness.request(0).level);}
const choices = [_]models.Entry{ .{ .name = "wide", .id = "provider/wide", .provider = .fake, .levels = .init(.{ .off = "none", .low = "low", .medium = null, .high = "high", .max = null }), .max_output = 0, }, // Its provider is not set. .{ .name = "away", .id = "provider/away", .provider = .workers_ai, .max_output = 0 }, .{ .name = "plain", .id = "provider/plain", .provider = .fake, .max_output = 0 }, .{ .name = "narrow", .id = "provider/narrow", .provider = .fake, .levels = .init(.{ .off = null, .low = "low", .medium = "medium", .high = null, .max = null }), .max_output = 0, },};
fn expectSelection(harness: *Harness, model: []const u8, level: ?models.Level) !void { const selected = harness.selection().?; try testing.expectEqualStrings(model, selected.model); try testing.expectEqual(level, selected.level);}
test "attach answers with the reasoning level of a model that reasons" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .medium); defer harness.deinit(); var log: EventLog = .{}; defer log.deinit();
try harness.send(.{ .detach = harness.log.sink() }); try harness.log.waitFor(.response, 1); try harness.send(.{ .attach = log.sink() }); try log.waitFor(.response, 1);
const session = log.events.items[0].response.result.attached; try testing.expectEqualStrings("wide", session.model); try testing.expectEqual(models.Level.high, session.level.?);}
test "cycling the level steps through the levels of the model and wraps around" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .high); defer harness.deinit();
try testing.expectEqual(.ok, try harness.ask(.cycle_level)); try expectSelection(&harness, "wide", .off); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "wide", .low); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "wide", .high);}
test "cycling the model skips a model without a provider and wraps around" { var harness: Harness = undefined; try harness.initModels(&.{.{ .text = &.{"a"} }}, &choices, .high); defer harness.deinit();
try testing.expectEqual(.ok, try harness.ask(.cycle_model)); try expectSelection(&harness, "plain", null); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "narrow", .medium);
try harness.run("go"); try testing.expectEqualStrings("narrow", harness.request(0).model); try testing.expectEqual(models.Level.medium, harness.request(0).level);
_ = try harness.ask(.cycle_model); try expectSelection(&harness, "wide", .high);}
test "a model without reasoning stays off, and the level the user chose comes back with the next model" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .low); defer harness.deinit();
_ = try harness.ask(.cycle_model); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "plain", null); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "narrow", .low);}
test "a model is selected by name, and an unknown name or a missing provider is refused" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .high); defer harness.deinit();
try testing.expectEqual(.ok, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "narrow") })); try expectSelection(&harness, "narrow", .medium);
try testing.expectEqual(.unknown_model, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "nope") })); try testing.expectEqual(.unknown_model, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "away") })); try testing.expectEqual(@as(usize, 1), harness.log.count(.selection)); try expectSelection(&harness, "narrow", .medium);}
test "a change during a request applies from the next request of the same prompt" { var harness: Harness = undefined; try harness.initModels(&.{ .{ .text = &.{"a"}, .calls = &.{toolCall("call-1", "nope", "{}")}, .hold = true, .finish = .tool_calls }, .{ .text = &.{"done"} }, }, &choices, .high); defer harness.deinit();
try harness.submit("go"); try harness.log.waitFor(.text_delta, 1); _ = try harness.ask(.cycle_level); _ = try harness.ask(.cycle_model); _ = try harness.ask(.cycle_model); harness.script.release(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualStrings("wide", harness.request(0).model); try testing.expectEqual(models.Level.high, harness.request(0).level); try testing.expectEqualStrings("narrow", harness.request(1).model); try testing.expectEqual(models.Level.low, harness.request(1).level);}
const priced_model: models.Entry = .{ .name = "priced", .id = "provider/priced", .provider = .fake, .context_window = 2000, .max_output = 0, .cost = .{ .input = 2, .output = 10, .cache_read = 1 },};
test "each request reports its usage and the session totals" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.initWith(&.{ .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .usage = .{ .input = 1000, .output = 100, .cache_read = 0 }, .finish = .tool_calls }, .{ .text = &.{"done"}, .usage = .{ .input = 200, .output = 50, .cache_read = 1000 } }, .{ .text = &.{"again"}, .usage = .{ .input = 10, .output = 5, .cache_read = 1250 } }, }, &priced_model); defer harness.deinit();
try harness.run("go"); try harness.run("more");
const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 3), reports.len);
try testing.expectEqual(provider.Usage{ .input = 1000, .output = 100 }, reports[0].request); try testing.expectEqual(reports[0].request, reports[0].session); try testing.expectApproxEqAbs(@as(f64, 0.003), reports[0].request_cost.?, 1e-12); try testing.expectApproxEqAbs(@as(f64, 0.003), reports[0].session_cost.?, 1e-12); try testing.expectEqual(@as(u64, 1100), reports[0].context_tokens); try testing.expectEqual(@as(u32, 2000), reports[0].context_window);
try testing.expectEqual(provider.Usage{ .input = 200, .output = 50, .cache_read = 1000 }, reports[1].request); try testing.expectEqual(provider.Usage{ .input = 1200, .output = 150, .cache_read = 1000 }, reports[1].session); try testing.expectApproxEqAbs(@as(f64, 0.0019), reports[1].request_cost.?, 1e-12); try testing.expectApproxEqAbs(@as(f64, 0.0049), reports[1].session_cost.?, 1e-12); try testing.expectEqual(@as(u64, 1250), reports[1].context_tokens);
try testing.expectEqual(provider.Usage{ .input = 1210, .output = 155, .cache_read = 2250 }, reports[2].session); try testing.expectEqual(@as(u64, 1265), reports[2].context_tokens);}
test "usage arrives after the reply it belongs to and before the tool results" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"a"}, .calls = &.{toolCall("call-1", "nope", "{}")}, .usage = .{ .input = 1 }, .finish = .tool_calls }, .{ .text = &.{"b"} }, }); defer harness.deinit();
try harness.run("go");
try testing.expectEqualSlices( Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .usage, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena), );}
test "a model without a price reports usage without a cost" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"hi"}, .usage = .{ .input = 7, .output = 2 } }}); defer harness.deinit();
try harness.run("go");
const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 1), reports.len); try testing.expectEqual(provider.Usage{ .input = 7, .output = 2 }, reports[0].session); try testing.expectEqual(null, reports[0].request_cost); try testing.expectEqual(null, reports[0].session_cost); try testing.expectEqual(@as(u32, 0), reports[0].context_window);}
test "an interrupted request and a failed request still report their usage" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.initWith(&.{ .{ .text = &.{"par"}, .usage = .{ .input = 30, .output = 3 }, .wait_for_abort = true }, .{ .usage = .{ .input = 40, .output = 4 }, .fail = error.Provider }, }, &priced_model); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try harness.run("second");
const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 2), reports.len); try testing.expectEqual(provider.Usage{ .input = 30, .output = 3 }, reports[0].request); try testing.expectEqual(provider.Usage{ .input = 70, .output = 7 }, reports[1].session);}
test "a prompt queued during a run is answered and runs next in the same busy period" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"one"}, .hold = true }, .{ .text = &.{"two"} }, }); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.submit("second"); harness.script.release(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .prompt_start, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .queued }, harness.log.events.items[4].response); try expectMessages(&.{ .{ .user = "first" }, .{ .assistant = .{ .text = "one", .tool_calls = &.{} } }, .{ .user = "second" }, }, harness.request(1).messages);}
test "a full queue refuses a prompt" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"a"}, .hold = true }}); defer harness.deinit(); try harness.submit("running"); try harness.log.waitFor(.text_delta, 1);
for (0..max_queued) |_| try harness.submit("queued"); try harness.submit("refused"); const events = harness.log.events.items; var queued: usize = 0; for (events) |message| queued += @intFromBool(message == .response and message.response.result == .queued); try testing.expectEqual(max_queued + 1, queued); try testing.expect(events[events.len - 1].response.result == .refused);}
test "an interrupt returns the queued prompts in order and runs none of them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .wait_for_abort = true }}); defer harness.deinit();
try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.submit("second"); try harness.submit(" "); try harness.submit("third"); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1);
try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .response, .response, .response, .notice, .prompts_returned, .agent_end }, try harness.log.tags(arena)); const returned = harness.log.events.items[9].prompts_returned; try testing.expectEqual(3, returned.count); try testing.expectEqualStrings("second\n\nthird", returned.text); try testing.expectEqual(1, harness.script.requests.items.len);}
test "stopping the agent drops the queued prompts" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"a"}, .wait_for_abort = true }}); defer harness.deinit();
try harness.submit("first"); try harness.submit("second"); try harness.log.waitFor(.text_delta, 1); harness.agent.stop(); try testing.expectEqual(@as(usize, 1), harness.script.requests.items.len);}
test "stopping the agent during a run ends the run" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .wait_for_abort = true }}); defer harness.deinit();
try harness.submit("go"); try harness.log.waitFor(.text_delta, 1); harness.agent.stop(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[4].notice.kind);}