const std = @import("std"); const models = @import("models.zig"); const tools = @import("tools.zig"); const Abort = @import("abort.zig").Abort; pub const ToolCall = struct { id: []const u8, name: []const u8, args_json: []const u8, pub fn dupe(self: ToolCall, gpa: std.mem.Allocator) error{OutOfMemory}!ToolCall { const id = try gpa.dupe(u8, self.id); errdefer gpa.free(id); const name = try gpa.dupe(u8, self.name); errdefer gpa.free(name); return .{ .id = id, .name = name, .args_json = try gpa.dupe(u8, self.args_json) }; } pub fn deinit(self: ToolCall, gpa: std.mem.Allocator) void { gpa.free(self.id); gpa.free(self.name); gpa.free(self.args_json); } }; pub const Assistant = struct { text: []const u8, /// What the model reasoned before this reply. Some models need it back /// while they continue the same turn. thinking: []const u8 = "", tool_calls: []const ToolCall, }; pub const ToolResult = struct { call_id: []const u8, text: []const u8, }; pub const Message = union(enum) { user: []const u8, assistant: Assistant, tool_result: ToolResult, pub fn deinit(self: Message, gpa: std.mem.Allocator) void { switch (self) { .user => |text| gpa.free(text), .assistant => |assistant| { gpa.free(assistant.text); gpa.free(assistant.thinking); for (assistant.tool_calls) |call| call.deinit(gpa); gpa.free(assistant.tool_calls); }, .tool_result => |result| { gpa.free(result.call_id); gpa.free(result.text); }, } } }; pub const Request = struct { system: []const u8, messages: []const Message, tools: []const tools.Definition, model: *const models.Entry, /// A level that the model supports, or `off` for a model without reasoning. level: models.Level = .off, }; /// The tokens one reply was billed for, or the sum over several replies. pub const Usage = struct { /// Prompt tokens that were not read from a cache. input: u64 = 0, /// Includes reasoning tokens. output: u64 = 0, /// Prompt tokens read from a cache. cache_read: u64 = 0, /// Prompt tokens written to a cache. cache_write: u64 = 0, pub fn total(self: Usage) u64 { return self.input + self.output + self.cache_read + self.cache_write; } pub fn add(self: *Usage, other: Usage) void { self.input += other.input; self.output += other.output; self.cache_read += other.cache_read; self.cache_write += other.cache_write; } }; pub const Finish = enum { stop, /// The reply reached the output token limit. length, tool_calls, /// A content filter of the provider stopped the reply. filtered, /// Another reason, which the diagnostics may name. other, }; /// Receives a reply while it streams. The slices are valid only during the /// call, so a receiver that keeps one must copy it. The callbacks can run on /// another thread than the one that called `Provider.stream`, but never two at /// once, and none runs after `Provider.stream` returns. pub const StreamSink = struct { ctx: *anyopaque, text_delta: *const fn (ctx: *anyopaque, delta: []const u8) void, thinking_delta: *const fn (ctx: *anyopaque, delta: []const u8) void, /// Called once the arguments of a call are complete. tool_call: *const fn (ctx: *anyopaque, call: ToolCall) void, /// Called at most once, and only when the provider reported usage. A /// reply that failed or was aborted can still report it. usage: *const fn (ctx: *anyopaque, usage: Usage) void, pub fn textDelta(self: StreamSink, delta: []const u8) void { self.text_delta(self.ctx, delta); } pub fn thinkingDelta(self: StreamSink, delta: []const u8) void { self.thinking_delta(self.ctx, delta); } pub fn toolCall(self: StreamSink, call: ToolCall) void { self.tool_call(self.ctx, call); } pub fn reportUsage(self: StreamSink, usage: Usage) void { self.usage(self.ctx, usage); } }; pub const StreamError = error{ Aborted, Http, Provider, OutOfMemory }; /// Explains why a stream failed or ended early. Text beyond the buffer is cut off. pub const Diagnostics = struct { buffer: [1024]u8 = undefined, len: usize = 0, pub fn message(self: *const Diagnostics) []const u8 { return self.buffer[0..self.len]; } /// Replaces the message. pub fn print(self: *Diagnostics, comptime fmt: []const u8, args: anytype) void { var writer: std.Io.Writer = .fixed(&self.buffer); writer.print(fmt, args) catch {}; self.len = writer.end; } /// Replaces the message and returns `error.Provider`. pub fn fail(self: *Diagnostics, comptime fmt: []const u8, args: anytype) StreamError { self.print(fmt, args); return error.Provider; } /// Maps a transport error to a stream error. An error other than /// cancellation or out of memory becomes `error.Http`, and its name /// replaces the message. pub fn httpError(self: *Diagnostics, err: anyerror) StreamError { return switch (err) { error.Canceled => error.Aborted, error.OutOfMemory => error.OutOfMemory, else => { self.print("{s}", .{@errorName(err)}); return error.Http; }, }; } }; pub const Provider = struct { ctx: *anyopaque, streamFn: *const fn (ctx: *anyopaque, request: Request, out: StreamSink, abort: *Abort, diagnostics: *Diagnostics) StreamError!Finish, /// Streams one reply to `out` and returns how it ended. Once `abort` is /// set, it should return `error.Aborted` soon; `out` may already have /// received part of the reply. After `error.Http`, `error.Provider` or an /// `other` finish, `diagnostics` may explain why. pub fn stream(self: Provider, request: Request, out: StreamSink, abort: *Abort, diagnostics: *Diagnostics) StreamError!Finish { return self.streamFn(self.ctx, request, out, abort, diagnostics); } }; test "a diagnostic replaces the last one and is cut off at the buffer size" { var diagnostics: Diagnostics = .{}; diagnostics.print("first {d}", .{1}); diagnostics.print("{s}", .{"second"}); try std.testing.expectEqualStrings("second", diagnostics.message()); const long: [2000]u8 = @splat('x'); diagnostics.print("{s}", .{&long}); try std.testing.expectEqualStrings(long[0..diagnostics.buffer.len], diagnostics.message()); }