//! Test helpers: a scripted provider that keeps every request, an event log //! that waits for events, and a watchdog for queues. const std = @import("std"); const protocol = @import("protocol.zig"); const provider = @import("provider.zig"); const Abort = @import("abort.zig").Abort; const expired = @import("abort.zig").expired; const io = std.testing.io; /// How long a test waits for an event before it fails. const event_timeout: std.Io.Timeout = .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } }; /// A provider that plays one step per request and keeps a copy of each request. pub const Scripted = struct { arena_state: std.heap.ArenaAllocator, steps: []const Step, requests: std.ArrayList(Request) = .empty, released: std.Io.Event = .unset, pub const Step = struct { thinking: []const []const u8 = &.{}, text: []const []const u8 = &.{}, calls: []const provider.ToolCall = &.{}, usage: ?provider.Usage = null, /// After the deltas, wait for `release`. hold: bool = false, /// After the deltas, wait for the abort signal and fail with `error.Aborted`. wait_for_abort: bool = false, fail: ?provider.StreamError = null, /// The detail the provider reports. detail: []const u8 = "", finish: provider.Finish = .stop, }; pub const Request = struct { system: []const u8, messages: []const provider.Message, tools: []const []const u8, model: []const u8, level: @import("models.zig").Level, }; pub fn init(steps: []const Step) Scripted { return .{ .arena_state = .init(std.testing.allocator), .steps = steps }; } pub fn deinit(self: *Scripted) void { self.arena_state.deinit(); } pub fn asProvider(self: *Scripted) provider.Provider { return .{ .ctx = self, .streamFn = stream }; } /// Lets every step that holds continue. pub fn release(self: *Scripted) void { self.released.set(io); } fn stream( ctx: *anyopaque, request: provider.Request, out: provider.StreamSink, abort: *Abort, diagnostics: *provider.Diagnostics, ) provider.StreamError!provider.Finish { const self: *Scripted = @ptrCast(@alignCast(ctx)); const index = self.requests.items.len; try self.record(request); if (index >= self.steps.len) { return diagnostics.fail("the script has no step for this request", .{}); } const step = self.steps[index]; for (step.thinking) |delta| out.thinkingDelta(delta); for (step.text) |delta| out.textDelta(delta); for (step.calls) |call| out.toolCall(call); if (step.usage) |usage| out.reportUsage(usage); if (step.hold) { self.released.waitUncancelable(io); } if (step.wait_for_abort) { _ = abort.wait(io, .none); return error.Aborted; } diagnostics.print("{s}", .{step.detail}); if (step.fail) |failure| { return failure; } return step.finish; } fn record(self: *Scripted, request: provider.Request) error{OutOfMemory}!void { const arena = self.arena_state.allocator(); const messages = try arena.alloc(provider.Message, request.messages.len); for (request.messages, messages) |message, *copy| copy.* = try copyMessage(arena, message); const names = try arena.alloc([]const u8, request.tools.len); for (request.tools, names) |definition, *name| name.* = try arena.dupe(u8, definition.name); try self.requests.append(arena, .{ .system = try arena.dupe(u8, request.system), .messages = messages, .tools = names, .model = try arena.dupe(u8, request.model.name), .level = request.level, }); } fn copyMessage(arena: std.mem.Allocator, message: provider.Message) error{OutOfMemory}!provider.Message { return switch (message) { .user => |text| .{ .user = try arena.dupe(u8, text) }, .assistant => |assistant| blk: { const calls = try arena.alloc(provider.ToolCall, assistant.tool_calls.len); for (assistant.tool_calls, calls) |call, *copy| copy.* = try call.dupe(arena); break :blk .{ .assistant = .{ .text = try arena.dupe(u8, assistant.text), .thinking = try arena.dupe(u8, assistant.thinking), .tool_calls = calls, } }; }, .tool_result => |result| .{ .tool_result = .{ .call_id = try arena.dupe(u8, result.call_id), .text = try arena.dupe(u8, result.text), } }, }; } }; /// Receives events and keeps them in arrival order. The test thread reads /// `events`, which `collect` fills. Do not move it after the first event. pub fn EventLog(comptime T: type) type { return struct { const Self = @This(); mutex: std.Io.Mutex = .init, incoming: std.ArrayList(T) = .empty, events: std.ArrayList(T) = .empty, arrived: std.Io.Event = .unset, pub const Tag = std.meta.Tag(T); pub fn deinit(self: *Self) void { self.collect(); for (self.events.items) |event| event.deinit(std.testing.allocator); self.events.deinit(std.testing.allocator); self.incoming.deinit(std.testing.allocator); } pub fn sink(self: *Self) protocol.Sink(T) { return .{ .ctx = self, .sendFn = send }; } /// Waits until `count` events with `tag` have arrived in total. pub fn waitFor(self: *Self, tag: Tag, wanted: usize) error{Timeout}!void { const deadline = event_timeout.toDeadline(io); while (true) { // Reset before collecting so an event that arrives after the collect is not missed. self.arrived.reset(); self.collect(); if (self.count(tag) >= wanted) { return; } self.arrived.waitTimeout(io, deadline) catch |err| switch (err) { // A spurious wakeup also reports a timeout. error.Timeout => if (expired(io, deadline)) { return error.Timeout; }, error.Canceled => unreachable, }; } } pub fn count(self: *const Self, tag: Tag) usize { var total: usize = 0; for (self.events.items) |event| total += @intFromBool(event == tag); return total; } /// The tags of every event so far, in arrival order. pub fn tags(self: *const Self, arena: std.mem.Allocator) error{OutOfMemory}![]const Tag { const out = try arena.alloc(Tag, self.events.items.len); for (self.events.items, out) |event, *tag| tag.* = event; return out; } /// Every text delta so far, joined. pub fn text(self: *const Self, arena: std.mem.Allocator) error{OutOfMemory}![]const u8 { var out: std.ArrayList(u8) = .empty; for (self.events.items) |event| switch (event) { .text_delta => |delta| try out.appendSlice(arena, delta), else => {}, }; return out.items; } /// Takes the events that arrived since the last collect. pub fn collect(self: *Self) void { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); self.events.appendSlice(std.testing.allocator, self.incoming.items) catch { for (self.incoming.items) |event| event.deinit(std.testing.allocator); }; self.incoming.clearRetainingCapacity(); } fn send(ctx: *anyopaque, event: T) bool { const self: *Self = @ptrCast(@alignCast(ctx)); { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); self.incoming.append(std.testing.allocator, event) catch return false; } self.arrived.set(io); return true; } }; } /// Closes a queue after a while, so a test that waits for an event fails and /// does not hang. pub fn Watchdog(comptime Queue: type) type { return struct { future: std.Io.Future(void), pub fn start(queue: *Queue) !@This() { return .{ .future = try io.concurrent(closeLater, .{queue}) }; } pub fn stop(self: *@This()) void { self.future.cancel(io); } fn closeLater(queue: *Queue) void { io.sleep(.fromSeconds(10), .awake) catch return; queue.close(error.Timeout); } }; } /// A tool call for a scripted step. pub fn toolCall(id: []const u8, name: []const u8, args_json: []const u8) provider.ToolCall { return .{ .id = id, .name = name, .args_json = args_json }; }