const std = @import("std"); const provider = @import("provider.zig"); const models = @import("models.zig"); const tools = @import("tools.zig"); const Abort = @import("abort.zig").Abort; const protocol = @import("protocol.zig"); const Block = protocol.core.Block; const Notice = protocol.core.Notice; const Request = protocol.ui.Request; const UiSink = protocol.Sink(protocol.core.Message); const Message = provider.Message; const ToolCall = provider.ToolCall; /// Safety cap for one user prompt. pub const max_requests = 50; /// How many prompts can wait while the agent is busy. pub const max_queued = 8; pub const Agent = struct { // Three tasks run the agent, and none of them holds a lock. The owner holds // the attached UI and the queued prompts, and only it sends to the UI. The // runner runs one prompt at a time and sends through the owner. The // forwarder moves requests from the inbox to the owner, so the runner's // messages cannot take the room that requests need. The UI never waits on // the agent, so the waits cannot form a cycle. gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ, providers: Providers, system_prompt: []const u8, models: []const models.Entry, /// The level the user asked for. Only the owner uses it after the start. wanted_level: models.Level, /// The model and the level of the next request. The owner writes it and /// the runner reads it. selection: std.atomic.Value(u32), cwd: []const u8, /// Only the runner uses it. history: std.ArrayList(Message) = .empty, /// What every request so far used. Only the runner uses it. session: provider.Usage = .{}, /// Null once a request used a model without a price. session_cost: ?f64 = 0, /// Ends the running prompt. The owner clears it only before it gives the /// runner a prompt, when nothing waits on it. abort: Abort = .{}, /// Where the runner sends its messages for the UI. out: UiSink, inbox: std.Io.Queue(Request), inbox_buffer: [protocol.inbox_capacity]Request = undefined, /// Everything the owner handles. Never closed. events: std.Io.Queue(Event), events_buffer: [64]Event = undefined, /// Gives the runner its next prompt. prompts: std.Io.Queue([]const u8), prompt_buffer: [1][]const u8 = undefined, runner_task: ?std.Io.Future(void) = null, owner_task: ?std.Io.Future(void) = null, forwarder_task: ?std.Io.Future(void) = null, const Event = union(enum) { request: Request, message: protocol.core.Message, /// The runner finished its prompt. done, stop, }; pub const Options = struct { /// The environment of child processes. environ: std.process.Environ, /// A model can be selected only when its provider is set here. providers: Providers, system_prompt: []const u8, /// The models a UI can select. Must outlive the agent. models: []const models.Entry = &models.entries, /// The index in `models` of the model to start with. Its provider /// must be set. model: usize, /// The reasoning level to start with. The agent uses the nearest /// level that the model supports. level: models.Level = models.default_level, cwd: []const u8, }; pub const Providers = std.EnumArray(models.Provider, ?provider.Provider); const Selection = struct { /// An index in `models`. model: u16, level: models.Level, fn pack(self: Selection) u32 { return @as(u32, self.model) << 8 | @backingInt(self.level); } fn unpack(bits: u32) Selection { return .{ .model = @intCast(bits >> 8), .level = @fromBackingInt(@intCast(@as(std.meta.Tag(models.Level), @truncate(bits)))) }; } }; /// Allocates the agent with `gpa`. `deinit` frees it. The strings in /// `options` must outlive the agent. pub fn init(gpa: std.mem.Allocator, io: std.Io, options: Options) !*Agent { const self = try gpa.create(Agent); self.* = .{ .gpa = gpa, .io = io, .environ = options.environ, .providers = options.providers, .system_prompt = options.system_prompt, .models = options.models, .wanted_level = options.level, .selection = .init(Selection.pack(.{ .model = @intCast(options.model), .level = models.clamp(&options.models[options.model], options.level), })), .cwd = options.cwd, .out = .{ .ctx = self, .sendFn = post }, .inbox = .init(&self.inbox_buffer), .events = .init(&self.events_buffer), .prompts = .init(&self.prompt_buffer), }; return self; } /// Starts reading the requests that arrive through `sink()` and running /// the prompts they submit. No UI is attached yet. pub fn start(self: *Agent) !void { errdefer self.stop(); self.runner_task = try self.io.concurrent(runnerMain, .{self}); self.owner_task = try self.io.concurrent(Owner.main, .{self}); self.forwarder_task = try self.io.concurrent(forwarderMain, .{self}); } /// Where a UI sends its requests. Valid after `start`. Never blocks, so a /// UI can send from its event loop. Refuses a request when the inbox is /// full or the agent stopped. pub fn sink(self: *Agent) protocol.Sink(Request) { return .{ .ctx = self, .sendFn = sendRequest }; } fn sendRequest(ctx: *anyopaque, request: Request) bool { const self: *Agent = @ptrCast(@alignCast(ctx)); return (self.inbox.putUncancelable(self.io, &.{request}, 0) catch 0) == 1; } /// Ends the running prompt, stops the tasks and frees the prompts that /// wait. An attached UI must not block forever meanwhile. pub fn stop(self: *Agent) void { self.inbox.close(self.io); if (self.forwarder_task) |*task| { task.await(self.io); self.forwarder_task = null; } // After the last request, so the owner handles none after the stop. if (self.owner_task) |*task| { self.events.putOneUncancelable(self.io, .stop) catch unreachable; task.await(self.io); self.owner_task = null; } self.prompts.close(self.io); if (self.runner_task) |*task| { task.await(self.io); self.runner_task = null; } } pub fn deinit(self: *Agent) void { self.stop(); for (self.history.items) |message| message.deinit(self.gpa); self.history.deinit(self.gpa); self.gpa.destroy(self); } fn forwarderMain(self: *Agent) void { while (self.inbox.getOneUncancelable(self.io)) |request| { self.events.putOneUncancelable(self.io, .{ .request = request }) catch unreachable; } else |_| {} } fn runnerMain(self: *Agent) void { while (self.prompts.getOneUncancelable(self.io)) |prompt| { self.runInner(prompt) catch |err| switch (err) { error.OutOfMemory => self.sendNotice(.err, "out of memory"), }; self.events.putOneUncancelable(self.io, .done) catch unreachable; } else |_| {} } fn post(ctx: *anyopaque, message: protocol.core.Message) bool { const self: *Agent = @ptrCast(@alignCast(ctx)); self.events.putOneUncancelable(self.io, .{ .message = message }) catch unreachable; return true; } fn send(self: *Agent, message: protocol.core.Message) void { if (!self.out.send(message)) { message.deinit(self.gpa); } } fn sendNotice(self: *Agent, kind: Notice.Kind, text: []const u8) void { // A provider detail can hold server text that is not UTF-8, or end inside a code point. const copy = std.fmt.allocPrint(self.gpa, "{f}", .{std.unicode.fmtUtf8(text)}) catch return; self.send(.{ .notice = .{ .kind = kind, .text = copy } }); } /// Runs one prompt and takes ownership of it. fn runInner(self: *Agent, prompt: []const u8) !void { self.history.append(self.gpa, .{ .user = prompt }) catch |err| { self.gpa.free(prompt); return err; }; var requests: usize = 0; while (true) { if (requests == max_requests) { self.sendNotice(.err, "request limit reached, the run ended"); return; } requests += 1; // A change of the selection applies from the next request. const selection: Selection = .unpack(self.selection.load(.acquire)); const model = &self.models[selection.model]; var stream: Stream = .{ .agent = self }; defer stream.deinit(); var diagnostics: provider.Diagnostics = .{}; const reply = self.providers.get(model.provider).?.stream( .{ .system = self.system_prompt, .messages = self.history.items, .tools = &tools.definitions, .model = model, .level = selection.level, }, stream.sink(), &self.abort, &diagnostics, ); // A request that failed or was interrupted was billed too. if (stream.usage) |usage| { self.recordUsage(model, usage); } const finish = reply catch |err| { if (self.abort.isSet()) { self.sendNotice(.interrupted, "interrupted"); return; } if (err == error.OutOfMemory) { return error.OutOfMemory; } const detail = diagnostics.message(); self.sendNotice(.err, if (detail.len == 0) "provider error" else detail); return; }; // A partial answer is not part of the history: it may hold incomplete tool calls. if (self.abort.isSet()) { self.sendNotice(.interrupted, "interrupted"); return; } if (stream.out_of_memory) { return error.OutOfMemory; } const calls = calls: { try self.history.ensureUnusedCapacity(self.gpa, 1); const text = try stream.text.toOwnedSlice(self.gpa); errdefer self.gpa.free(text); const thinking = try stream.thinking.toOwnedSlice(self.gpa); errdefer self.gpa.free(thinking); const calls = try stream.calls.toOwnedSlice(self.gpa); self.history.appendAssumeCapacity(.{ .assistant = .{ .text = text, .thinking = thinking, .tool_calls = calls } }); break :calls calls; }; if (finish == .filtered or finish == .other) { for (calls) |call| try self.recordPlain(call.id, "not run: the reply ended early"); if (finish == .filtered) { self.sendNotice(.err, "the provider's content filter stopped the reply"); } else if (diagnostics.message().len == 0) { self.sendNotice(.err, "the reply ended early"); } else { const prefix = "the reply ended early: "; var buffer: [prefix.len + diagnostics.buffer.len]u8 = undefined; self.sendNotice(.err, std.fmt.bufPrint(&buffer, prefix ++ "{s}", .{diagnostics.message()}) catch unreachable); } return; } if (calls.len == 0) { if (finish == .length) { self.sendNotice(.err, "the reply reached the token limit"); } return; } if (finish == .length) { for (calls) |call| try self.recordPlain(call.id, "the output was truncated; re-issue the call"); continue; } for (calls, 0..) |call, index| { // The flag can be set between two calls, so it is checked before each one. if (self.abort.isSet()) { try self.skipRemaining(calls[index..]); self.sendNotice(.interrupted, "interrupted"); return; } var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); const result = tools.execute( .{ .io = self.io, .environ = self.environ, .abort = &self.abort }, arena_state.allocator(), call.name, call.args_json, ); // A tool reports an interrupt in its own result, and work it finished still happened. try self.recordResult(call.id, result); if (self.abort.isSet()) { try self.skipRemaining(calls[index + 1 ..]); self.sendNotice(.interrupted, "interrupted"); return; } } } } /// Adds the usage of one request to the session and tells the UI. fn recordUsage(self: *Agent, model: *const models.Entry, usage: provider.Usage) void { const request_cost: ?f64 = if (model.cost) |prices| models.cost(prices, usage) else null; self.session.add(usage); self.session_cost = if (self.session_cost != null and request_cost != null) self.session_cost.? + request_cost.? else null; self.send(.{ .usage = .{ .request = usage, .request_cost = request_cost, .session = self.session, .session_cost = self.session_cost, .context_tokens = usage.total(), .context_window = model.context_window, } }); } fn skipRemaining(self: *Agent, calls: []const ToolCall) !void { const skip = "not run: aborted by user"; for (calls) |call| try self.recordPlain(call.id, skip); } /// Records a failed result that the agent wrote itself. fn recordPlain(self: *Agent, call_id: []const u8, text: []const u8) !void { var arena_state = std.heap.ArenaAllocator.init(self.gpa); defer arena_state.deinit(); try self.recordResult(call_id, tools.plainResult(arena_state.allocator(), text, true)); } fn recordResult(self: *Agent, call_id: []const u8, result: tools.Result) !void { try self.history.ensureUnusedCapacity(self.gpa, 1); const id = try self.gpa.dupe(u8, call_id); errdefer self.gpa.free(id); // Tool output can hold any bytes, and the request JSON needs UTF-8. const text = if (std.unicode.utf8ValidateSlice(result.text)) try self.gpa.dupe(u8, result.text) else try std.fmt.allocPrint(self.gpa, "{f}", .{std.unicode.fmtUtf8(result.text)}); self.history.appendAssumeCapacity(.{ .tool_result = .{ .call_id = id, .text = text } }); // A call without a result breaks every later request, so a failure here only hides the result from the UI. self.showResult(call_id, result) catch {}; } fn showResult(self: *Agent, call_id: []const u8, result: tools.Result) !void { const id = try self.gpa.dupe(u8, call_id); errdefer self.gpa.free(id); const collapsed: Block = try .init(self.gpa, result.collapsed); errdefer collapsed.deinit(self.gpa); const expanded: Block = try .init(self.gpa, result.expanded); self.send(.{ .tool_result = .{ .call_id = id, .is_error = result.is_error, .collapsed = collapsed, .expanded = expanded, }, }); } }; /// The state of the owner task. No other task can reach it. const Owner = struct { agent: *Agent, ui: ?UiSink = null, /// The prompts that wait for the runner. queued: std.ArrayList([]const u8), /// The runner has a prompt. running: bool = false, stopping: bool = false, fn main(agent: *Agent) void { var buffer: [max_queued][]const u8 = undefined; var self: Owner = .{ .agent = agent, .queued = .initBuffer(&buffer) }; while (!self.stopping or self.running) { switch (agent.events.getOneUncancelable(agent.io) catch unreachable) { .request => |request| self.handle(request), .message => |message| self.send(message), .done => self.finish(), .stop => self.stop(), } } } fn handle(self: *Owner, request: Request) void { const id = request.id; switch (request.method) { .attach => |ui| self.attach(id, ui), .detach => |ui| self.detach(id, ui), .submit => |prompt| self.submit(id, prompt), .interrupt => { self.respond(id, .ok); if (self.running) { self.agent.abort.set(self.agent.io); } }, .select_model => |name| { defer self.agent.gpa.free(name); const index = models.indexOf(self.agent.models, name) orelse return self.respond(id, .unknown_model); if (!self.selectable(index)) { return self.respond(id, .unknown_model); } self.selectModel(id, index); }, .cycle_model => { const count = self.agent.models.len; var index: usize = self.selection().model; // The selected model can be selected, so the search ends there at the latest. for (0..count) |_| { index = (index + 1) % count; if (self.selectable(index)) { break; } } self.selectModel(id, index); }, .cycle_level => { const current = self.selection(); const level = models.nextLevel(&self.agent.models[current.model], current.level); // A model with one level keeps what the user asked for, for the next model. if (level != current.level) { self.agent.wanted_level = level; } self.select(id, .{ .model = current.model, .level = level }); }, } } fn selection(self: *Owner) Agent.Selection { return .unpack(self.agent.selection.load(.acquire)); } fn selectable(self: *Owner, index: usize) bool { return self.agent.providers.get(self.agent.models[index].provider) != null; } /// Selects a model, at the level nearest to the one the user asked for. fn selectModel(self: *Owner, id: protocol.RequestId, index: usize) void { self.select(id, .{ .model = @intCast(index), .level = models.clamp(&self.agent.models[index], self.agent.wanted_level), }); } /// Makes `selected` the selection of the next request, tells the UI and /// then answers the request that asked for it. fn select(self: *Owner, id: protocol.RequestId, selected: Agent.Selection) void { self.agent.selection.store(selected.pack(), .release); defer self.respond(id, .ok); const entry = &self.agent.models[selected.model]; const model = self.agent.gpa.dupe(u8, entry.name) catch return; self.send(.{ .selection = .{ .model = model, .level = if (models.reasons(entry)) selected.level else null } }); } fn attach(self: *Owner, id: protocol.RequestId, ui: UiSink) void { if (self.ui != null) { _ = ui.send(.{ .response = .{ .id = id, .result = .already_attached } }); return; } const message: protocol.core.Message = .{ .response = .{ .id = id, .result = .{ .attached = self.session() }, }, }; if (!ui.send(message)) { message.deinit(self.agent.gpa); return; } self.ui = ui; } /// A copy of the session for a UI. Empty when memory runs out, because /// the attach must still get its answer. fn session(self: *Owner) protocol.core.Session { const gpa = self.agent.gpa; const selected = self.selection(); const entry = &self.agent.models[selected.model]; const model = gpa.dupe(u8, entry.name) catch return .{ .model = "", .cwd = "" }; const cwd = gpa.dupe(u8, self.agent.cwd) catch { gpa.free(model); return .{ .model = "", .cwd = "" }; }; return .{ .model = model, .level = if (models.reasons(entry)) selected.level else null, .cwd = cwd }; } fn detach(self: *Owner, id: protocol.RequestId, ui: UiSink) void { if (self.ui) |attached| { if (attached.eql(ui)) { self.ui = null; } } _ = ui.send(.{ .response = .{ .id = id, .result = .ok } }); } /// Runs or queues a prompt and takes ownership of it. fn submit(self: *Owner, id: protocol.RequestId, prompt: []const u8) void { if (!self.running) { self.respond(id, .queued); self.send(.agent_start); return self.startPrompt(prompt); } self.queued.appendBounded(prompt) catch { self.agent.gpa.free(prompt); return self.respond(id, .refused); }; self.respond(id, .queued); } fn startPrompt(self: *Owner, prompt: []const u8) void { self.send(.prompt_start); self.agent.abort.reset(); // The queue is empty while no prompt runs, and it closes only after the owner returns. self.agent.prompts.putOneUncancelable(self.agent.io, prompt) catch unreachable; self.running = true; } fn finish(self: *Owner) void { self.running = false; if (self.agent.abort.isSet()) { self.returnQueued(); } if (self.queued.items.len > 0) { return self.startPrompt(self.queued.orderedRemove(0)); } self.send(.agent_end); } fn stop(self: *Owner) void { self.stopping = true; for (self.queued.items) |prompt| self.agent.gpa.free(prompt); self.queued.clearRetainingCapacity(); self.agent.abort.set(self.agent.io); } /// Sends the text of every queued prompt back and takes them off the queue. fn returnQueued(self: *Owner) void { const gpa = self.agent.gpa; const taken = self.queued.items; if (taken.len == 0) { return; } defer { for (taken) |prompt| gpa.free(prompt); self.queued.clearRetainingCapacity(); } var parts: [max_queued][]const u8 = undefined; var kept: usize = 0; for (taken) |prompt| { if (std.mem.trim(u8, prompt, &std.ascii.whitespace).len == 0) { continue; } parts[kept] = prompt; kept += 1; } const text = std.mem.join(gpa, "\n\n", parts[0..kept]) catch text: { self.sendNotice(.err, "out of memory; the queued prompts were lost"); break :text ""; }; self.send(.{ .prompts_returned = .{ .count = taken.len, .text = text } }); } fn respond(self: *Owner, id: protocol.RequestId, result: protocol.core.Response.Result) void { self.send(.{ .response = .{ .id = id, .result = result } }); } /// Sends a message to the attached UI, or drops it when none is attached. fn send(self: *Owner, message: protocol.core.Message) void { if (self.ui) |ui| { if (ui.send(message)) { return; } } message.deinit(self.agent.gpa); } fn sendNotice(self: *Owner, kind: Notice.Kind, text: []const u8) void { const copy = self.agent.gpa.dupe(u8, text) catch return; self.send(.{ .notice = .{ .kind = kind, .text = copy } }); } }; /// Collects the streamed answer and forwards every delta toward the UI. const Stream = struct { agent: *Agent, text: std.ArrayList(u8) = .empty, thinking: std.ArrayList(u8) = .empty, calls: std.ArrayList(ToolCall) = .empty, usage: ?provider.Usage = null, /// Part of the reply was dropped. out_of_memory: bool = false, fn deinit(self: *Stream) void { const gpa = self.agent.gpa; self.text.deinit(gpa); self.thinking.deinit(gpa); for (self.calls.items) |call| call.deinit(gpa); self.calls.deinit(gpa); } fn sink(self: *Stream) provider.StreamSink { return .{ .ctx = self, .text_delta = onText, .thinking_delta = onThinking, .tool_call = onCall, .usage = onUsage }; } fn onUsage(ctx: *anyopaque, usage: provider.Usage) void { const self: *Stream = @ptrCast(@alignCast(ctx)); self.usage = usage; } fn onText(ctx: *anyopaque, delta: []const u8) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; self.text.appendSlice(gpa, delta) catch return self.lose(); const copy = gpa.dupe(u8, delta) catch return; self.agent.send(.{ .text_delta = copy }); } fn onThinking(ctx: *anyopaque, delta: []const u8) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; self.thinking.appendSlice(gpa, delta) catch return self.lose(); const copy = gpa.dupe(u8, delta) catch return; self.agent.send(.{ .thinking_delta = copy }); } fn onCall(ctx: *anyopaque, call: ToolCall) void { const self: *Stream = @ptrCast(@alignCast(ctx)); const gpa = self.agent.gpa; const owned = call.dupe(gpa) catch return self.lose(); self.calls.append(gpa, owned) catch { owned.deinit(gpa); return self.lose(); }; self.show(call) catch {}; } fn lose(self: *Stream) void { self.out_of_memory = true; } fn show(self: *Stream, call: ToolCall) !void { const gpa = self.agent.gpa; var arena_state = std.heap.ArenaAllocator.init(gpa); defer arena_state.deinit(); const shown = try tools.renderCall(arena_state.allocator(), call.name, call.args_json); const id = try gpa.dupe(u8, call.id); errdefer gpa.free(id); self.agent.send(.{ .tool_call = .{ .id = id, .call = try .init(gpa, shown) } }); } }; const testing = std.testing; const helpers = @import("testing.zig"); const Scripted = helpers.Scripted; const EventLog = helpers.EventLog(protocol.core.Message); const toolCall = helpers.toolCall; const test_model: models.Entry = .{ .name = "model", .id = "provider/model", .provider = .fake, .max_output = 0 }; fn testOptions(streamer: provider.Provider) Agent.Options { var providers: Agent.Providers = .initFill(null); providers.set(.fake, streamer); return .{ .environ = testing.environ, .providers = providers, .system_prompt = "system", .models = (&test_model)[0..1], .model = 0, .cwd = "/work" }; } /// A started agent with a scripted provider and an attached message log. Do /// not move it after `init`. const Harness = struct { log: EventLog, script: Scripted, agent: *Agent, next_id: protocol.RequestId, fn init(self: *Harness, steps: []const Scripted.Step) !void { return self.initWith(steps, &test_model); } fn initWith(self: *Harness, steps: []const Scripted.Step, model: *const models.Entry) !void { return self.initModels(steps, model[0..1], models.default_level); } /// Starts with the first of `entries`. fn initModels(self: *Harness, steps: []const Scripted.Step, entries: []const models.Entry, level: models.Level) !void { self.log = .{}; self.script = .init(steps); self.next_id = 0; var options = testOptions(self.script.asProvider()); options.models = entries; options.level = level; self.agent = try Agent.init(testing.allocator, testing.io, options); try self.agent.start(); try self.send(.{ .attach = self.log.sink() }); // The attach response is the first message; its own test checks it. try self.log.waitFor(.response, 1); for (self.log.events.items) |event| event.deinit(testing.allocator); self.log.events.clearRetainingCapacity(); } fn deinit(self: *Harness) void { // A failed test must not leave the agent task in a held step. self.script.release(); self.agent.deinit(); self.script.deinit(); self.log.deinit(); } /// Submits a prompt and waits until the agent is idle again. fn run(self: *Harness, prompt: []const u8) !void { self.log.collect(); const ends = self.log.count(.agent_end); try self.submit(prompt); try self.log.waitFor(.agent_end, ends + 1); } /// Submits a prompt and waits until the agent answers. fn submit(self: *Harness, prompt: []const u8) !void { self.log.collect(); const answered = self.log.count(.response); try self.send(.{ .submit = try testing.allocator.dupe(u8, prompt) }); try self.log.waitFor(.response, answered + 1); } /// Sends a request and returns the answer to it. fn ask(self: *Harness, method: protocol.ui.Request.Method) !protocol.core.Response.Result { self.log.collect(); const answered = self.log.count(.response); const id = self.next_id; try self.send(method); try self.log.waitFor(.response, answered + 1); var index = self.log.events.items.len; while (index > 0) { index -= 1; switch (self.log.events.items[index]) { .response => |response| if (response.id == id) { return response.result; }, else => {}, } } return error.NoResponse; } /// The last selection that the agent reported. fn selection(self: *Harness) ?protocol.core.Selection { var index = self.log.events.items.len; while (index > 0) { index -= 1; switch (self.log.events.items[index]) { .selection => |selected| return selected, else => {}, } } return null; } /// Sends a request and takes ownership of its method. fn send(self: *Harness, method: protocol.ui.Request.Method) !void { const message: protocol.ui.Request = .{ .id = self.next_id, .method = method }; self.next_id += 1; if (!self.agent.sink().send(message)) { message.deinit(testing.allocator); return error.Closed; } } /// Interrupts the agent and waits until the running prompt sees it. fn interrupt(self: *Harness) !void { try self.send(.interrupt); if (!self.agent.abort.wait(testing.io, .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } })) { return error.Timeout; } } fn request(self: *Harness, index: usize) Scripted.Request { return self.script.requests.items[index]; } /// The usage reports so far, in arrival order. fn usage(self: *Harness, arena: std.mem.Allocator) ![]const protocol.core.UsageReport { var reports: std.ArrayList(protocol.core.UsageReport) = .empty; for (self.log.events.items) |event| switch (event) { .usage => |report| try reports.append(arena, report), else => {}, }; return reports.items; } }; /// Waits until a writer sends one line through the FIFO at `path`. fn waitForLine(dir: std.Io.Dir, path: []const u8) !void { const Done = union(enum) { line: anyerror!void, late: std.Io.Cancelable!void }; var buffer: [2]Done = undefined; var select: std.Io.Select(Done) = .init(testing.io, &buffer); defer select.cancelDiscard(); try select.concurrent(.line, readLine, .{ dir, path }); try select.concurrent(.late, std.Io.sleep, .{ testing.io, .fromSeconds(10), .awake }); switch (try select.await()) { .line => |result| try result, .late => return error.Timeout, } } fn readLine(dir: std.Io.Dir, path: []const u8) anyerror!void { const file = try dir.openFile(testing.io, path, .{}); defer file.close(testing.io); var buffer: [64]u8 = undefined; var reader = file.reader(testing.io, &buffer); _ = try reader.interface.takeDelimiterInclusive('\n'); } fn expectMessages(expected: []const Message, actual: []const Message) !void { try testing.expectEqual(expected.len, actual.len); for (expected, actual) |want, got| { try testing.expectEqual(std.meta.activeTag(want), std.meta.activeTag(got)); switch (want) { .user => |text| try testing.expectEqualStrings(text, got.user), .assistant => |assistant| { try testing.expectEqualStrings(assistant.text, got.assistant.text); try testing.expectEqualStrings(assistant.thinking, got.assistant.thinking); try testing.expectEqual(assistant.tool_calls.len, got.assistant.tool_calls.len); for (assistant.tool_calls, got.assistant.tool_calls) |want_call, got_call| { try testing.expectEqualStrings(want_call.id, got_call.id); try testing.expectEqualStrings(want_call.name, got_call.name); try testing.expectEqualStrings(want_call.args_json, got_call.args_json); } }, .tool_result => |result| { try testing.expectEqualStrings(result.call_id, got.tool_result.call_id); try testing.expectEqualStrings(result.text, got.tool_result.text); }, } } } const Tag = EventLog.Tag; test "a text reply streams between agent_start and agent_end" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{ "po", "ng" } }}); defer harness.deinit(); try harness.run("ping"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .queued }, harness.log.events.items[0].response); try testing.expectEqualStrings("pong", try harness.log.text(arena)); } test "attach answers with the model and the working directory" { var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{}); defer script.deinit(); const agent = try Agent.init(testing.allocator, testing.io, testOptions(script.asProvider())); defer agent.deinit(); try agent.start(); try testing.expect(agent.sink().send(.{ .id = 7, .method = .{ .attach = log.sink() } })); try log.waitFor(.response, 1); const response = log.events.items[0].response; try testing.expectEqual(7, response.id); try testing.expectEqualStrings("model", response.result.attached.model); try testing.expectEqual(null, response.result.attached.level); try testing.expectEqualStrings("/work", response.result.attached.cwd); } test "a second attach is answered on its own sink and leaves the first UI attached" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); var second: EventLog = .{}; defer second.deinit(); try harness.send(.{ .attach = second.sink() }); try second.waitFor(.response, 1); try harness.run("ping"); second.collect(); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .already_attached }, second.events.items[0].response); try testing.expectEqual(1, second.events.items.len); try testing.expectEqualStrings("pong", try harness.log.text(arena_state.allocator())); } test "the detach response is the last message to the sink, and the running prompt continues for the next UI" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"}, .hold = true }}); defer harness.deinit(); try harness.submit("ping"); try harness.log.waitFor(.text_delta, 1); try harness.send(.{ .detach = harness.log.sink() }); try harness.log.waitFor(.response, 2); var next: EventLog = .{}; defer next.deinit(); try harness.send(.{ .attach = next.sink() }); try next.waitFor(.response, 1); harness.script.release(); try next.waitFor(.agent_end, 1); harness.log.collect(); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .ok }, harness.log.events.items[4].response); } test "a detach from a UI that is not attached is answered and leaves the attached UI" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); var other: EventLog = .{}; defer other.deinit(); try harness.send(.{ .detach = other.sink() }); try other.waitFor(.response, 1); try harness.run("ping"); try testing.expectEqual(protocol.core.Response{ .id = 1, .result = .ok }, other.events.items[0].response); try testing.expectEqualStrings("pong", try harness.log.text(arena_state.allocator())); } test "the first request holds the system prompt, the prompt and every tool" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"pong"} }}); defer harness.deinit(); try harness.run("ping"); const request = harness.request(0); try testing.expectEqualStrings("system", request.system); try expectMessages(&.{.{ .user = "ping" }}, request.messages); try testing.expectEqual(tools.definitions.len, request.tools.len); for (tools.definitions, request.tools) |definition, name| try testing.expectEqualStrings(definition.name, name); } test "a tool call runs and its result goes back to the model" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.path("out.txt"); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "hi" }, .{}); var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "write", args)}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqualStrings("hi", try dir.read("out.txt")); const wrote = try std.fmt.allocPrint(arena, "wrote 2 bytes to {s}", .{path}); const result = harness.log.events.items[4].tool_result; try testing.expectEqualStrings("call-1", result.call_id); try testing.expect(!result.is_error); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "", .tool_calls = &.{toolCall("call-1", "write", args)} } }, .{ .tool_result = .{ .call_id = "call-1", .text = wrote } }, }, harness.request(1).messages); } test "calls in one reply run in order and each gets its own result" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.path("out.txt"); const first = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "first" }, .{}); const second = try std.json.Stringify.valueAlloc(arena, .{ .path = path, .content = "second!" }, .{}); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"writing"}, .calls = &.{ toolCall("call-a", "write", first), toolCall("call-b", "write", second) }, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit(); try harness.run("go"); try testing.expectEqualStrings("second!", try dir.read("out.txt")); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "writing", .tool_calls = &.{ toolCall("call-a", "write", first), toolCall("call-b", "write", second) } } }, .{ .tool_result = .{ .call_id = "call-a", .text = try std.fmt.allocPrint(arena, "wrote 5 bytes to {s}", .{path}) } }, .{ .tool_result = .{ .call_id = "call-b", .text = try std.fmt.allocPrint(arena, "wrote 7 bytes to {s}", .{path}) } }, }, harness.request(1).messages); } test "invalid UTF-8 in a tool result reaches the model and the UI as U+FFFD" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("latin1.txt", "caf\xe9\n"); const args = try std.json.Stringify.valueAlloc(arena, .{ .files = &.{.{ .path = path }} }, .{}); var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "read", args)}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer harness.deinit(); try harness.run("go"); const shown = harness.log.events.items[4].tool_result.expanded.text; const sent = harness.request(1).messages[2].tool_result.text; try testing.expect(std.unicode.utf8ValidateSlice(shown)); try testing.expect(std.mem.indexOf(u8, shown, "1| caf\u{fffd}\n") != null); try testing.expect(std.unicode.utf8ValidateSlice(sent)); try testing.expect(std.mem.indexOf(u8, sent, "1| caf\u{fffd}\n") != null); } test "a failed tool call is a result for the model, not the end of the run" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"sorry"} }, }); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena)); const result = harness.log.events.items[4].tool_result; try testing.expect(result.is_error); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "", .tool_calls = &.{toolCall("call-1", "nope", "{}")} } }, .{ .tool_result = .{ .call_id = "call-1", .text = "unknown tool: nope" } }, }, harness.request(1).messages); } /// Fails one allocation, the one with index `fail_index`. const OneFailure = struct { fail_index: usize, index: usize = 0, fn allocator(self: *OneFailure) std.mem.Allocator { return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = std.mem.Allocator.noResize, .remap = std.mem.Allocator.noRemap, .free = free } }; } fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { const self: *OneFailure = @ptrCast(@alignCast(ctx)); defer self.index += 1; if (self.index == self.fail_index) { return null; } return testing.allocator.rawAlloc(len, alignment, ret_addr); } fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { _ = ctx; testing.allocator.rawFree(memory, alignment, ret_addr); } }; test "a failed allocation ends the run with the error or keeps the whole reply, and leaks nothing" { var fail_index: usize = 0; while (true) : (fail_index += 1) { var failing: OneFailure = .{ .fail_index = fail_index }; const gpa = failing.allocator(); var log: EventLog = .{}; defer log.deinit(); var script: Scripted = .init(&.{ .{ .text = &.{ "a", "b" }, .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"done"} }, }); defer script.deinit(); const agent = Agent.init(gpa, testing.io, testOptions(script.asProvider())) catch continue; defer agent.deinit(); agent.out = log.sink(); const prompt = gpa.dupe(u8, "go") catch continue; agent.runInner(prompt) catch |err| { try testing.expectEqual(error.OutOfMemory, err); continue; }; const history = agent.history.items; try testing.expectEqual(4, history.len); try expectMessages(&.{.{ .assistant = .{ .text = "ab", .tool_calls = &.{toolCall("call-1", "nope", "{}")} } }}, history[1..2]); try expectMessages(&.{.{ .assistant = .{ .text = "done", .tool_calls = &.{} } }}, history[3..]); if (failing.index <= fail_index) { break; } } } test "a reply cut off by the length limit answers its calls without running them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{}); var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &.{toolCall("call-1", "write", args)}, .finish = .length }, .{ .text = &.{"done"} }, }); defer harness.deinit(); try harness.run("go"); try testing.expectError(error.FileNotFound, dir.read("out.txt")); const result = harness.log.events.items[4].tool_result; try testing.expect(result.is_error); try testing.expect(std.mem.indexOf(u8, result.expanded.text, "truncated") != null); const messages = harness.request(1).messages; try testing.expectEqual(@as(usize, 3), messages.len); try testing.expectEqualStrings("call-1", messages[2].tool_result.call_id); } test "a reply cut off by the length limit without calls ends with a notice" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .finish = .length }}); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[4].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the reply reached the token limit", notice.text); } test "a reply stopped by the content filter ends with a notice and answers its calls without running them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{}); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .calls = &.{toolCall("call-1", "write", args)}, .finish = .filtered }, .{ .text = &.{"next"} }, }); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .tool_result, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectError(error.FileNotFound, dir.read("out.txt")); const result = harness.log.events.items[5].tool_result; try testing.expect(result.is_error); try testing.expectEqualStrings("not run: the reply ended early", result.expanded.text); const notice = harness.log.events.items[6].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the provider's content filter stopped the reply", notice.text); try harness.run("again"); try expectMessages(&.{ .{ .user = "go" }, .{ .assistant = .{ .text = "par", .tool_calls = &.{toolCall("call-1", "write", args)} } }, .{ .tool_result = .{ .call_id = "call-1", .text = "not run: the reply ended early" } }, .{ .user = "again" }, }, harness.request(1).messages); } test "a reply that ends for an unknown reason ends with a notice that names the reason" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .finish = .other, .detail = "finish reason: overloaded" }}); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[4].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("the reply ended early: finish reason: overloaded", notice.text); } test "a provider error ends the run with a notice that holds its detail as valid UTF-8" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .fail = error.Http, .detail = "503 overloaded \xff" }}); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .notice, .agent_end }, try harness.log.tags(arena)); const notice = harness.log.events.items[3].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expectEqualStrings("503 overloaded \u{FFFD}", notice.text); } test "a run stops after max_requests requests" { var harness: Harness = undefined; const step: Scripted.Step = .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }; const steps: [max_requests + 1]Scripted.Step = @splat(step); try harness.init(&steps); defer harness.deinit(); try harness.run("go"); try testing.expectEqual(@as(usize, max_requests), harness.script.requests.items.len); const events = harness.log.events.items; const notice = events[events.len - 2].notice; try testing.expectEqual(Notice.Kind.err, notice.kind); try testing.expect(std.mem.indexOf(u8, notice.text, "request limit") != null); } test "an interrupt while streaming keeps the partial answer out of the history" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .wait_for_abort = true }, .{ .text = &.{"pong"} }, }); defer harness.deinit(); try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .ok }, harness.log.events.items[4].response); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[5].notice.kind); try harness.run("second"); try expectMessages(&.{ .{ .user = "first" }, .{ .user = "second" } }, harness.request(1).messages); } test "an interrupt that the provider does not report drops the reply and its calls" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); const args = try std.json.Stringify.valueAlloc(arena, .{ .path = try dir.path("out.txt"), .content = "hi" }, .{}); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"par"}, .calls = &.{toolCall("call-1", "write", args)}, .hold = true, .finish = .tool_calls }, .{ .text = &.{"pong"} }, }); defer harness.deinit(); try harness.submit("first"); try harness.log.waitFor(.tool_call, 1); try harness.interrupt(); harness.script.release(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .response, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[6].notice.kind); try testing.expectError(error.FileNotFound, dir.read("out.txt")); try harness.run("second"); try expectMessages(&.{ .{ .user = "first" }, .{ .user = "second" } }, harness.request(1).messages); } test "an interrupt during a tool call keeps its own result and skips the calls after it" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var dir = try tools.TestDir.init(); defer dir.deinit(); try testing.expect(!dir.call("bash", .{ .command = "mkfifo started", .timeout = 10, .cwd = dir.root }).is_error); const slow = try std.json.Stringify.valueAlloc(arena, .{ .command = "echo partial; echo > started; sleep 30", .timeout = 60, .cwd = dir.root }, .{}); const fast = try std.json.Stringify.valueAlloc(arena, .{ .command = "echo two", .timeout = 10, .cwd = dir.root }, .{}); const calls = [_]ToolCall{ toolCall("call-1", "bash", slow), toolCall("call-2", "bash", fast) }; var harness: Harness = undefined; try harness.init(&.{ .{ .calls = &calls, .finish = .tool_calls }, .{ .text = &.{"after"} }, }); defer harness.deinit(); const started = std.Io.Timestamp.now(testing.io, .awake); try harness.submit("go"); try waitForLine(dir.tmp.dir, "started"); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try testing.expect(started.untilNow(testing.io, .awake).nanoseconds < 10 * std.time.ns_per_s); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .tool_call, .tool_call, .response, .tool_result, .tool_result, .notice, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[8].notice.kind); const first = harness.log.events.items[6].tool_result; try testing.expect(!first.is_error); try testing.expectStringStartsWith(first.expanded.text, "partial\n[aborted by user, elapsed "); const second = harness.log.events.items[7].tool_result; try testing.expect(second.is_error); try testing.expectEqualStrings("not run: aborted by user", second.expanded.text); try harness.run("next"); const messages = harness.request(1).messages; try testing.expectEqual(5, messages.len); try testing.expectStringStartsWith(messages[2].tool_result.text, "partial\n[aborted by user, elapsed "); try expectMessages(&.{ .{ .tool_result = .{ .call_id = "call-2", .text = "not run: aborted by user" } }, .{ .user = "next" }, }, messages[3..]); } test "thinking is shown and goes back to the model with its reply" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .thinking = &.{ "h", "m" }, .calls = &.{toolCall("call-1", "nope", "{}")}, .finish = .tool_calls }, .{ .text = &.{"answer"} }, .{ .text = &.{"two"} }, }); defer harness.deinit(); try harness.run("first"); try testing.expectEqualSlices( Tag, &.{ .response, .agent_start, .prompt_start, .thinking_delta, .thinking_delta, .tool_call, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena), ); const sent = harness.request(1).messages; try testing.expectEqualStrings("hm", sent[1].assistant.thinking); try harness.run("second"); try testing.expectEqualStrings("hm", harness.request(2).messages[1].assistant.thinking); try testing.expectEqualStrings("", harness.request(2).messages[3].assistant.thinking); } test "each request names the model and a reasoning level that the model supports" { const leveled: models.Entry = .{ .name = "leveled", .id = "provider/leveled", .provider = .fake, .levels = .init(.{ .off = null, .low = "l", .medium = null, .high = "h", .max = null }), .max_output = 0, }; var harness: Harness = undefined; try harness.initWith(&.{.{ .text = &.{"a"} }}, &leveled); defer harness.deinit(); try harness.run("go"); try testing.expectEqualStrings("leveled", harness.request(0).model); try testing.expectEqual(models.Level.high, harness.request(0).level); } const choices = [_]models.Entry{ .{ .name = "wide", .id = "provider/wide", .provider = .fake, .levels = .init(.{ .off = "none", .low = "low", .medium = null, .high = "high", .max = null }), .max_output = 0, }, // Its provider is not set. .{ .name = "away", .id = "provider/away", .provider = .workers_ai, .max_output = 0 }, .{ .name = "plain", .id = "provider/plain", .provider = .fake, .max_output = 0 }, .{ .name = "narrow", .id = "provider/narrow", .provider = .fake, .levels = .init(.{ .off = null, .low = "low", .medium = "medium", .high = null, .max = null }), .max_output = 0, }, }; fn expectSelection(harness: *Harness, model: []const u8, level: ?models.Level) !void { const selected = harness.selection().?; try testing.expectEqualStrings(model, selected.model); try testing.expectEqual(level, selected.level); } test "attach answers with the reasoning level of a model that reasons" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .medium); defer harness.deinit(); var log: EventLog = .{}; defer log.deinit(); try harness.send(.{ .detach = harness.log.sink() }); try harness.log.waitFor(.response, 1); try harness.send(.{ .attach = log.sink() }); try log.waitFor(.response, 1); const session = log.events.items[0].response.result.attached; try testing.expectEqualStrings("wide", session.model); try testing.expectEqual(models.Level.high, session.level.?); } test "cycling the level steps through the levels of the model and wraps around" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .high); defer harness.deinit(); try testing.expectEqual(.ok, try harness.ask(.cycle_level)); try expectSelection(&harness, "wide", .off); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "wide", .low); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "wide", .high); } test "cycling the model skips a model without a provider and wraps around" { var harness: Harness = undefined; try harness.initModels(&.{.{ .text = &.{"a"} }}, &choices, .high); defer harness.deinit(); try testing.expectEqual(.ok, try harness.ask(.cycle_model)); try expectSelection(&harness, "plain", null); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "narrow", .medium); try harness.run("go"); try testing.expectEqualStrings("narrow", harness.request(0).model); try testing.expectEqual(models.Level.medium, harness.request(0).level); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "wide", .high); } test "a model without reasoning stays off, and the level the user chose comes back with the next model" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .low); defer harness.deinit(); _ = try harness.ask(.cycle_model); _ = try harness.ask(.cycle_level); try expectSelection(&harness, "plain", null); _ = try harness.ask(.cycle_model); try expectSelection(&harness, "narrow", .low); } test "a model is selected by name, and an unknown name or a missing provider is refused" { var harness: Harness = undefined; try harness.initModels(&.{}, &choices, .high); defer harness.deinit(); try testing.expectEqual(.ok, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "narrow") })); try expectSelection(&harness, "narrow", .medium); try testing.expectEqual(.unknown_model, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "nope") })); try testing.expectEqual(.unknown_model, try harness.ask(.{ .select_model = try testing.allocator.dupe(u8, "away") })); try testing.expectEqual(@as(usize, 1), harness.log.count(.selection)); try expectSelection(&harness, "narrow", .medium); } test "a change during a request applies from the next request of the same prompt" { var harness: Harness = undefined; try harness.initModels(&.{ .{ .text = &.{"a"}, .calls = &.{toolCall("call-1", "nope", "{}")}, .hold = true, .finish = .tool_calls }, .{ .text = &.{"done"} }, }, &choices, .high); defer harness.deinit(); try harness.submit("go"); try harness.log.waitFor(.text_delta, 1); _ = try harness.ask(.cycle_level); _ = try harness.ask(.cycle_model); _ = try harness.ask(.cycle_model); harness.script.release(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqualStrings("wide", harness.request(0).model); try testing.expectEqual(models.Level.high, harness.request(0).level); try testing.expectEqualStrings("narrow", harness.request(1).model); try testing.expectEqual(models.Level.low, harness.request(1).level); } const priced_model: models.Entry = .{ .name = "priced", .id = "provider/priced", .provider = .fake, .context_window = 2000, .max_output = 0, .cost = .{ .input = 2, .output = 10, .cache_read = 1 }, }; test "each request reports its usage and the session totals" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.initWith(&.{ .{ .calls = &.{toolCall("call-1", "nope", "{}")}, .usage = .{ .input = 1000, .output = 100, .cache_read = 0 }, .finish = .tool_calls }, .{ .text = &.{"done"}, .usage = .{ .input = 200, .output = 50, .cache_read = 1000 } }, .{ .text = &.{"again"}, .usage = .{ .input = 10, .output = 5, .cache_read = 1250 } }, }, &priced_model); defer harness.deinit(); try harness.run("go"); try harness.run("more"); const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 3), reports.len); try testing.expectEqual(provider.Usage{ .input = 1000, .output = 100 }, reports[0].request); try testing.expectEqual(reports[0].request, reports[0].session); try testing.expectApproxEqAbs(@as(f64, 0.003), reports[0].request_cost.?, 1e-12); try testing.expectApproxEqAbs(@as(f64, 0.003), reports[0].session_cost.?, 1e-12); try testing.expectEqual(@as(u64, 1100), reports[0].context_tokens); try testing.expectEqual(@as(u32, 2000), reports[0].context_window); try testing.expectEqual(provider.Usage{ .input = 200, .output = 50, .cache_read = 1000 }, reports[1].request); try testing.expectEqual(provider.Usage{ .input = 1200, .output = 150, .cache_read = 1000 }, reports[1].session); try testing.expectApproxEqAbs(@as(f64, 0.0019), reports[1].request_cost.?, 1e-12); try testing.expectApproxEqAbs(@as(f64, 0.0049), reports[1].session_cost.?, 1e-12); try testing.expectEqual(@as(u64, 1250), reports[1].context_tokens); try testing.expectEqual(provider.Usage{ .input = 1210, .output = 155, .cache_read = 2250 }, reports[2].session); try testing.expectEqual(@as(u64, 1265), reports[2].context_tokens); } test "usage arrives after the reply it belongs to and before the tool results" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"a"}, .calls = &.{toolCall("call-1", "nope", "{}")}, .usage = .{ .input = 1 }, .finish = .tool_calls }, .{ .text = &.{"b"} }, }); defer harness.deinit(); try harness.run("go"); try testing.expectEqualSlices( Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .tool_call, .usage, .tool_result, .text_delta, .agent_end }, try harness.log.tags(arena), ); } test "a model without a price reports usage without a cost" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"hi"}, .usage = .{ .input = 7, .output = 2 } }}); defer harness.deinit(); try harness.run("go"); const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 1), reports.len); try testing.expectEqual(provider.Usage{ .input = 7, .output = 2 }, reports[0].session); try testing.expectEqual(null, reports[0].request_cost); try testing.expectEqual(null, reports[0].session_cost); try testing.expectEqual(@as(u32, 0), reports[0].context_window); } test "an interrupted request and a failed request still report their usage" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.initWith(&.{ .{ .text = &.{"par"}, .usage = .{ .input = 30, .output = 3 }, .wait_for_abort = true }, .{ .usage = .{ .input = 40, .output = 4 }, .fail = error.Provider }, }, &priced_model); defer harness.deinit(); try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try harness.run("second"); const reports = try harness.usage(arena); try testing.expectEqual(@as(usize, 2), reports.len); try testing.expectEqual(provider.Usage{ .input = 30, .output = 3 }, reports[0].request); try testing.expectEqual(provider.Usage{ .input = 70, .output = 7 }, reports[1].session); } test "a prompt queued during a run is answered and runs next in the same busy period" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{ .{ .text = &.{"one"}, .hold = true }, .{ .text = &.{"two"} }, }); defer harness.deinit(); try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.submit("second"); harness.script.release(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .prompt_start, .text_delta, .agent_end }, try harness.log.tags(arena)); try testing.expectEqual(protocol.core.Response{ .id = 2, .result = .queued }, harness.log.events.items[4].response); try expectMessages(&.{ .{ .user = "first" }, .{ .assistant = .{ .text = "one", .tool_calls = &.{} } }, .{ .user = "second" }, }, harness.request(1).messages); } test "a full queue refuses a prompt" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"a"}, .hold = true }}); defer harness.deinit(); try harness.submit("running"); try harness.log.waitFor(.text_delta, 1); for (0..max_queued) |_| try harness.submit("queued"); try harness.submit("refused"); const events = harness.log.events.items; var queued: usize = 0; for (events) |message| queued += @intFromBool(message == .response and message.response.result == .queued); try testing.expectEqual(max_queued + 1, queued); try testing.expect(events[events.len - 1].response.result == .refused); } test "an interrupt returns the queued prompts in order and runs none of them" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .wait_for_abort = true }}); defer harness.deinit(); try harness.submit("first"); try harness.log.waitFor(.text_delta, 1); try harness.submit("second"); try harness.submit(" "); try harness.submit("third"); try harness.interrupt(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqualSlices(Tag, &.{ .response, .agent_start, .prompt_start, .text_delta, .response, .response, .response, .response, .notice, .prompts_returned, .agent_end }, try harness.log.tags(arena)); const returned = harness.log.events.items[9].prompts_returned; try testing.expectEqual(3, returned.count); try testing.expectEqualStrings("second\n\nthird", returned.text); try testing.expectEqual(1, harness.script.requests.items.len); } test "stopping the agent drops the queued prompts" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"a"}, .wait_for_abort = true }}); defer harness.deinit(); try harness.submit("first"); try harness.submit("second"); try harness.log.waitFor(.text_delta, 1); harness.agent.stop(); try testing.expectEqual(@as(usize, 1), harness.script.requests.items.len); } test "stopping the agent during a run ends the run" { var harness: Harness = undefined; try harness.init(&.{.{ .text = &.{"par"}, .wait_for_abort = true }}); defer harness.deinit(); try harness.submit("go"); try harness.log.waitFor(.text_delta, 1); harness.agent.stop(); try harness.log.waitFor(.agent_end, 1); try testing.expectEqual(Notice.Kind.interrupted, harness.log.events.items[4].notice.kind); }