Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246//! Test helpers: a scripted provider that keeps every request, an event log//! that waits for events, and a watchdog for queues.
const std = @import("std");
const protocol = @import("protocol.zig");const provider = @import("provider.zig");const Abort = @import("abort.zig").Abort;const expired = @import("abort.zig").expired;
const io = std.testing.io;
/// How long a test waits for an event before it fails.const event_timeout: std.Io.Timeout = .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } };
/// A provider that plays one step per request and keeps a copy of each request.pub const Scripted = struct { arena_state: std.heap.ArenaAllocator, steps: []const Step, requests: std.ArrayList(Request) = .empty, released: std.Io.Event = .unset,
pub const Step = struct { thinking: []const []const u8 = &.{}, text: []const []const u8 = &.{}, calls: []const provider.ToolCall = &.{}, usage: ?provider.Usage = null, /// After the deltas, wait for `release`. hold: bool = false, /// After the deltas, wait for the abort signal and fail with `error.Aborted`. wait_for_abort: bool = false, fail: ?provider.StreamError = null, /// The detail the provider reports. detail: []const u8 = "", finish: provider.Finish = .stop, };
pub const Request = struct { system: []const u8, messages: []const provider.Message, tools: []const []const u8, model: []const u8, level: @import("models.zig").Level, };
pub fn init(steps: []const Step) Scripted { return .{ .arena_state = .init(std.testing.allocator), .steps = steps }; }
pub fn deinit(self: *Scripted) void { self.arena_state.deinit(); }
pub fn asProvider(self: *Scripted) provider.Provider { return .{ .ctx = self, .streamFn = stream }; }
/// Lets every step that holds continue. pub fn release(self: *Scripted) void { self.released.set(io); }
fn stream( ctx: *anyopaque, request: provider.Request, out: provider.StreamSink, abort: *Abort, diagnostics: *provider.Diagnostics, ) provider.StreamError!provider.Finish { const self: *Scripted = @ptrCast(@alignCast(ctx)); const index = self.requests.items.len; try self.record(request); if (index >= self.steps.len) { return diagnostics.fail("the script has no step for this request", .{}); }
const step = self.steps[index]; for (step.thinking) |delta| out.thinkingDelta(delta); for (step.text) |delta| out.textDelta(delta); for (step.calls) |call| out.toolCall(call); if (step.usage) |usage| out.reportUsage(usage); if (step.hold) { self.released.waitUncancelable(io); } if (step.wait_for_abort) { _ = abort.wait(io, .none); return error.Aborted; } diagnostics.print("{s}", .{step.detail}); if (step.fail) |failure| { return failure; } return step.finish; }
fn record(self: *Scripted, request: provider.Request) error{OutOfMemory}!void { const arena = self.arena_state.allocator(); const messages = try arena.alloc(provider.Message, request.messages.len); for (request.messages, messages) |message, *copy| copy.* = try copyMessage(arena, message); const names = try arena.alloc([]const u8, request.tools.len); for (request.tools, names) |definition, *name| name.* = try arena.dupe(u8, definition.name); try self.requests.append(arena, .{ .system = try arena.dupe(u8, request.system), .messages = messages, .tools = names, .model = try arena.dupe(u8, request.model.name), .level = request.level, }); }
fn copyMessage(arena: std.mem.Allocator, message: provider.Message) error{OutOfMemory}!provider.Message { return switch (message) { .user => |text| .{ .user = try arena.dupe(u8, text) }, .assistant => |assistant| blk: { const calls = try arena.alloc(provider.ToolCall, assistant.tool_calls.len); for (assistant.tool_calls, calls) |call, *copy| copy.* = try call.dupe(arena); break :blk .{ .assistant = .{ .text = try arena.dupe(u8, assistant.text), .thinking = try arena.dupe(u8, assistant.thinking), .tool_calls = calls, } }; }, .tool_result => |result| .{ .tool_result = .{ .call_id = try arena.dupe(u8, result.call_id), .text = try arena.dupe(u8, result.text), } }, }; }};
/// Receives events and keeps them in arrival order. The test thread reads/// `events`, which `collect` fills. Do not move it after the first event.pub fn EventLog(comptime T: type) type { return struct { const Self = @This();
mutex: std.Io.Mutex = .init, incoming: std.ArrayList(T) = .empty, events: std.ArrayList(T) = .empty, arrived: std.Io.Event = .unset,
pub const Tag = std.meta.Tag(T);
pub fn deinit(self: *Self) void { self.collect(); for (self.events.items) |event| event.deinit(std.testing.allocator); self.events.deinit(std.testing.allocator); self.incoming.deinit(std.testing.allocator); }
pub fn sink(self: *Self) protocol.Sink(T) { return .{ .ctx = self, .sendFn = send }; }
/// Waits until `count` events with `tag` have arrived in total. pub fn waitFor(self: *Self, tag: Tag, wanted: usize) error{Timeout}!void { const deadline = event_timeout.toDeadline(io); while (true) { // Reset before collecting so an event that arrives after the collect is not missed. self.arrived.reset(); self.collect(); if (self.count(tag) >= wanted) { return; } self.arrived.waitTimeout(io, deadline) catch |err| switch (err) { // A spurious wakeup also reports a timeout. error.Timeout => if (expired(io, deadline)) { return error.Timeout; }, error.Canceled => unreachable, }; } }
pub fn count(self: *const Self, tag: Tag) usize { var total: usize = 0; for (self.events.items) |event| total += @intFromBool(event == tag); return total; }
/// The tags of every event so far, in arrival order. pub fn tags(self: *const Self, arena: std.mem.Allocator) error{OutOfMemory}![]const Tag { const out = try arena.alloc(Tag, self.events.items.len); for (self.events.items, out) |event, *tag| tag.* = event; return out; }
/// Every text delta so far, joined. pub fn text(self: *const Self, arena: std.mem.Allocator) error{OutOfMemory}![]const u8 { var out: std.ArrayList(u8) = .empty; for (self.events.items) |event| switch (event) { .text_delta => |delta| try out.appendSlice(arena, delta), else => {}, }; return out.items; }
/// Takes the events that arrived since the last collect. pub fn collect(self: *Self) void { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); self.events.appendSlice(std.testing.allocator, self.incoming.items) catch { for (self.incoming.items) |event| event.deinit(std.testing.allocator); }; self.incoming.clearRetainingCapacity(); }
fn send(ctx: *anyopaque, event: T) bool { const self: *Self = @ptrCast(@alignCast(ctx)); { self.mutex.lockUncancelable(io); defer self.mutex.unlock(io); self.incoming.append(std.testing.allocator, event) catch return false; } self.arrived.set(io); return true; } };}
/// Closes a queue after a while, so a test that waits for an event fails and/// does not hang.pub fn Watchdog(comptime Queue: type) type { return struct { future: std.Io.Future(void),
pub fn start(queue: *Queue) !@This() { return .{ .future = try io.concurrent(closeLater, .{queue}) }; }
pub fn stop(self: *@This()) void { self.future.cancel(io); }
fn closeLater(queue: *Queue) void { io.sleep(.fromSeconds(10), .awake) catch return; queue.close(error.Timeout); } };}
/// A tool call for a scripted step.pub fn toolCall(id: []const u8, name: []const u8, args_json: []const u8) provider.ToolCall { return .{ .id = id, .name = name, .args_json = args_json };}