From 7cc566bfc419eda4128ef2dc2cc59949df47b0ea Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Wed, 7 Oct 2026 01:48:33 -0400 Subject: [PATCH] openai_api: plain http streams over a caller-owned socket, decoder fast paths for delta events, 500 tok/s mock benchmark --- src/albedo/openai_api.gleam | 13 +- src/albedo/openai_api/albedo_openai_json.erl | 84 +++++ .../openai_api/albedo_openai_transport.erl | 300 ++++++++++++++++-- src/albedo/openai_api/chat.gleam | 101 ++++-- src/albedo/openai_api/fields.gleam | 52 +++ src/albedo/openai_api/reasoning.gleam | 17 + src/albedo/openai_api/responses.gleam | 66 +++- src/albedo/openai_api/sse.gleam | 14 +- src/albedo/openai_api/transport.gleam | 5 +- test/manual/albedo_openai_rate_bench.erl | 89 ++++++ test/manual/mock_openai_server.py | 210 ++++++++++++ test/manual/stream_rate_benchmark.gleam | 111 +++++++ .../albedo_openai_transport_test_server.erl | 15 +- test/openai_api/transport_test.gleam | 33 +- 14 files changed, 1026 insertions(+), 84 deletions(-) create mode 100644 src/albedo/openai_api/fields.gleam create mode 100644 test/manual/albedo_openai_rate_bench.erl create mode 100644 test/manual/mock_openai_server.py create mode 100644 test/manual/stream_rate_benchmark.gleam diff --git a/src/albedo/openai_api.gleam b/src/albedo/openai_api.gleam index 1ca50c5d..e91155a7 100644 --- a/src/albedo/openai_api.gleam +++ b/src/albedo/openai_api.gleam @@ -18,8 +18,9 @@ //// Supports text, image input, and function tools. Audio and custom tools are //// unsupported. Function arguments must be validated before execution. //// Requires Erlang/OTP 27+ for native JSON. The bounded incremental parser owns -//// SSE framing; Gun owns streaming HTTP, flow control, and TLS verification. -//// Requests stay as iodata. +//// SSE framing. Plain HTTP/1.1 is read straight from a socket the caller +//// owns; Gun owns HTTPS (HTTP/2, flow control, TLS verification). Requests +//// stay as iodata. import albedo/clock @@ -164,7 +165,7 @@ pub fn exchange( |> result.map_error(transport_error), ) use <- transport.with_connection(connection) - use first <- result.try(receive(connection)) + use #(first, connection) <- result.try(receive(connection)) case first { transport.Headers(status, headers, final) if status >= 200 && status < 300 -> { case event_stream(headers) || !exchange.require_event_stream, final { @@ -247,7 +248,7 @@ fn pump( thinking: Thinking, on_event: fn(Event) -> Control, ) -> Result(Turn, Error) { - use message <- result.try(receive(connection)) + use #(message, connection) <- result.try(receive(connection)) case message { transport.Data(bytes, final) -> { let now = clock.monotonic_ms() @@ -357,7 +358,7 @@ fn elapsed(thinking: Thinking, now: Int) -> Int { fn receive( connection: transport.Connection, -) -> Result(transport.Message, Error) { +) -> Result(#(transport.Message, transport.Connection), Error) { transport.receive(connection) |> result.map_error(transport_error) } @@ -395,7 +396,7 @@ fn http_error( Error(types.HttpError(status, body)) } False -> { - use message <- result.try(receive(connection)) + use #(message, connection) <- result.try(receive(connection)) case message { transport.Data(bytes, final) -> { let remaining = 65_536 - size diff --git a/src/albedo/openai_api/albedo_openai_json.erl b/src/albedo/openai_api/albedo_openai_json.erl index b3955e1f..9419d6c9 100644 --- a/src/albedo/openai_api/albedo_openai_json.erl +++ b/src/albedo/openai_api/albedo_openai_json.erl @@ -1,5 +1,8 @@ -module(albedo_openai_json). -export([encode/1, null/0, flatten/1, data_url/2, base64_string/1, semantically_empty/1, object_fields/1]). +-export([event/1, string_field/2, optional_string_field/2, int_field/2, + optional_int_field/2, int_field_or/3, list_field/2, object_field/2, + missing/2, empty_except/2]). encode(Value) -> json:encode(Value). @@ -74,3 +77,84 @@ base64_clean(<>) base64_clean(Rest); base64_clean(<<>>) -> true; base64_clean(_) -> false. + +%% Field reads over objects json:decode produced, whose strings are already +%% valid UTF-8. Each answers {error, nil} for any shape its decode equivalent +%% would not take as is; the caller then runs that decoder for the verdict. + +%% An event object and its string `type`. +event(Data) -> + try json:decode(Data) of + #{<<"type">> := Kind} = Event when is_binary(Kind) -> {ok, {Kind, Event}}; + _ -> {error, nil} + catch + error:_ -> {error, nil} + end. + +string_field(Object, Key) -> + case Object of + #{Key := Value} when is_binary(Value) -> {ok, Value}; + _ -> {error, nil} + end. + +int_field(Object, Key) -> + case Object of + #{Key := Value} when is_integer(Value) -> {ok, Value}; + _ -> {error, nil} + end. + +list_field(Object, Key) -> + case Object of + #{Key := Value} when is_list(Value) -> {ok, Value}; + _ -> {error, nil} + end. + +object_field(Object, Key) -> + case Object of + #{Key := Value} when is_map(Value) -> {ok, Value}; + _ -> {error, nil} + end. + +%% An absent or null field is none. +optional_string_field(Object, Key) when is_map(Object) -> + case Object of + #{Key := null} -> {ok, none}; + #{Key := Value} when is_binary(Value) -> {ok, {some, Value}}; + #{Key := _} -> {error, nil}; + _ -> {ok, none} + end; +optional_string_field(_, _) -> {error, nil}. + +optional_int_field(Object, Key) when is_map(Object) -> + case Object of + #{Key := null} -> {ok, none}; + #{Key := Value} when is_integer(Value) -> {ok, {some, Value}}; + #{Key := _} -> {error, nil}; + _ -> {ok, none} + end; +optional_int_field(_, _) -> {error, nil}. + +%% An absent field is Default; a present one must be an integer. +int_field_or(Object, Key, Default) when is_map(Object) -> + case Object of + #{Key := Value} when is_integer(Value) -> {ok, Value}; + #{Key := _} -> {error, nil}; + _ -> {ok, Default} + end; +int_field_or(_, _, _) -> {error, nil}. + +missing(Object, Key) -> + case Object of + #{Key := null} -> true; + #{Key := _} -> false; + _ -> true + end. + +%% Whether every field outside Keys is null or an empty list. +empty_except(Object, Keys) when is_map(Object) -> + maps:fold(fun + (_, null, Empty) -> Empty; + (_, [], Empty) -> Empty; + (Key, _, Empty) -> Empty andalso lists:member(Key, Keys) + end, true, Object); +empty_except(_, _) -> false. diff --git a/src/albedo/openai_api/albedo_openai_transport.erl b/src/albedo/openai_api/albedo_openai_transport.erl index 6ae86f07..a0ef56ec 100644 --- a/src/albedo/openai_api/albedo_openai_transport.erl +++ b/src/albedo/openai_api/albedo_openai_transport.erl @@ -3,10 +3,21 @@ -export([open/4, receive_message/1, close/1, with_connection/2, materialize/1]). -define(FLOW, 1). +%% Bounds on what a plain HTTP response may make the direct client buffer +%% before its body: the head, a chunk-size line, and the trailers. +-define(MAX_HEAD, 65536). +-define(MAX_CHUNK_LINE, 1024). +%% Socket messages the direct client lets queue before the socket pauses; it +%% re-arms on tcp_passive instead of once per packet. +-define(ACTIVE, 16). +-define(NO_LOCAL_SOCKET, <<"connection failed: this machine could not open " + "a local socket (EADDRNOTAVAIL or EINVAL); its ephemeral ports may be " + "exhausted by another program's connections">>). open(URL, Headers, Body, Timeout) when is_binary(URL), is_list(Headers), is_integer(Timeout), Timeout > 0 -> case parse_url(URL) of + {ok, #{transport := tcp} = Request} -> open_direct(Request, Headers, Body, Timeout); {ok, Request} -> open_request(Request, Headers, Body, Timeout); error -> {error, invalid_url} end; @@ -43,6 +54,236 @@ open_request(#{host := Host, port := Port, target := Target, transport_error(<<"could not start HTTP transport">>, Reason) end. +%% Plain HTTP/1.1 is spoken over a socket the caller owns, so each streamed +%% chunk wakes one process rather than a connection process and then its +%% caller; the socket closes when its owner exits. TLS endpoints stay on Gun, +%% which negotiates HTTP/2 and verifies certificates. The connection carries +%% the response state: {head, Buffer} until the headers, then +%% {body, Framing, Buffer}, where Buffer holds bytes not yet framed. +open_direct(#{host := Host, port := Port, target := Target}, Headers, Body, Timeout) -> + Options = [binary, {active, false}, {packet, raw}, {nodelay, true}, + {send_timeout, Timeout}, {send_timeout_close, true}], + case gen_tcp:connect(Host, Port, Options, Timeout) of + {ok, Socket} -> + try send_direct(Socket, Host, Port, Target, Headers, Body) of + ok -> + ok = inet:setopts(Socket, [{active, ?ACTIVE}]), + {ok, {direct, self(), Socket, Timeout, {head, <<>>}}}; + {error, Reason} -> + gen_tcp:close(Socket), + {error, {transport_error, Reason}} + catch + error:{attested_body, Why} -> + gen_tcp:close(Socket), + {error, {transport_error, Why}}; + _Class:_Reason -> + gen_tcp:close(Socket), + {error, {transport_error, <<"request failed">>}} + end; + {error, timeout} -> + {error, timed_out}; + {error, Reason} when Reason =:= einval; Reason =:= eaddrnotavail -> + {error, {transport_error, ?NO_LOCAL_SOCKET}}; + {error, Reason} -> + transport_error(<<"connection failed">>, Reason) + end. + +send_direct(Socket, Host, Port, Target, Headers, Body) -> + {Segments, Length, Hasher} = + case segments(Body) of + plain -> {[{data, Body}], iolist_size(Body), none}; + {Parts, Size, Marker} -> {Parts, Size, hasher(Marker)} + end, + Head = [<<"POST ">>, Target, <<" HTTP/1.1\r\nhost: ">>, host_header(Host, Port), + <<"\r\nconnection: close\r\ncontent-length: ">>, integer_to_binary(Length), + <<"\r\n">>, [[Name, <<": ">>, Value, <<"\r\n">>] || {Name, Value} <- Headers], + <<"\r\n">>], + Send = fun(_IsFin, Data) -> + case gen_tcp:send(Socket, Data) of + ok -> ok; + {error, Reason} -> transport_error(<<"request failed">>, Reason) + end + end, + then(Send(nofin, Head), fun() -> write(Send, fun() -> ok end, Segments, Hasher) end). + +host_header(Host, 80) -> bracketed(Host); +host_header(Host, Port) -> [bracketed(Host), $:, integer_to_binary(Port)]. + +bracketed(Host) -> + case lists:member($:, Host) of + true -> [$[, Host, $]]; + false -> Host + end. + +direct_message({direct, Owner, Socket, Timeout, {head, Buffer}} = Connection) -> + case response_head(Buffer) of + {ok, Status, _Headers, Rest} when Status >= 100, Status < 200, Status =/= 101 -> + direct_message(setelement(5, Connection, {head, Rest})); + {ok, Status, Headers, Rest} -> + Framing = framing(Status, Headers), + Next = {direct, Owner, Socket, Timeout, {body, Framing, Rest}}, + {ok, {{headers, Status, Headers, Framing =:= done}, Next}}; + more when byte_size(Buffer) > ?MAX_HEAD -> + {error, {transport_error, <<"response failed: response head too large">>}}; + more -> + await(Connection, Buffer, fun(More) -> {head, More} end, closed_early); + {error, Reason} -> + transport_error(<<"response failed">>, Reason) + end; +direct_message({direct, Owner, Socket, Timeout, {body, Framing, Buffer}} = Connection) -> + case unframe(Framing, Buffer) of + {Data, Next, Rest} when Data =/= []; Next =:= done -> + State = {body, Next, Rest}, + {ok, {{data, payload(Data), Next =:= done}, {direct, Owner, Socket, Timeout, State}}}; + {[], Next, Rest} -> + OnClose = case Next of + close -> close; + _ -> closed_early + end, + await(Connection, Rest, fun(More) -> {body, Next, More} end, OnClose); + {error, Reason} -> + {error, {transport_error, <<"response failed: ", Reason/binary>>}} + end. + +%% Waits for more bytes after Buffer, then reads on from State(Bytes). A close +%% ends a close-delimited body and fails any other response. +await({direct, Owner, Socket, Timeout, _} = Connection, Buffer, State, OnClose) -> + receive + {tcp, Socket, Bytes} -> + More = case Buffer of + <<>> -> Bytes; + _ -> <> + end, + direct_message({direct, Owner, Socket, Timeout, State(More)}); + {tcp_passive, Socket} -> + case inet:setopts(Socket, [{active, ?ACTIVE}]) of + ok -> await(Connection, Buffer, State, OnClose); + {error, _} -> closed(Owner, Socket, Timeout, OnClose) + end; + {tcp_closed, Socket} -> closed(Owner, Socket, Timeout, OnClose); + {tcp_error, Socket, Reason} -> transport_error(<<"response failed">>, Reason) + after Timeout -> + {error, timed_out} + end. + +closed(Owner, Socket, Timeout, close) -> + {ok, {{data, <<>>, true}, {direct, Owner, Socket, Timeout, {body, done, <<>>}}}}; +closed(_, _, _, closed_early) -> + {error, {transport_error, <<"response failed: the connection closed before the response ended">>}}. + +flush_socket(Socket) -> + receive + {tcp, Socket, _} -> flush_socket(Socket); + {tcp_passive, Socket} -> flush_socket(Socket); + {tcp_closed, Socket} -> flush_socket(Socket); + {tcp_error, Socket, _} -> flush_socket(Socket) + after 0 -> ok + end. + +response_head(Buffer) -> + case erlang:decode_packet(http_bin, Buffer, []) of + {ok, {http_response, _Version, Status, _Reason}, Rest} -> + response_headers(Rest, Status, []); + {ok, _, _} -> {error, malformed_status_line}; + {more, _} -> more; + {error, Reason} -> {error, Reason} + end. + +response_headers(Buffer, Status, Headers) -> + case erlang:decode_packet(httph_bin, Buffer, []) of + {ok, {http_header, _, Name, _, Value}, Rest} -> + response_headers(Rest, Status, [{header_name(Name), Value} | Headers]); + {ok, http_eoh, Rest} -> {ok, Status, lists:reverse(Headers), Rest}; + {ok, _, _} -> {error, malformed_header}; + {more, _} -> more; + {error, Reason} -> {error, Reason} + end. + +header_name(Name) when is_atom(Name) -> string:lowercase(atom_to_binary(Name)); +header_name(Name) -> string:lowercase(Name). + +framing(Status, _) when Status =:= 204; Status =:= 304 -> done; +framing(_, Headers) -> + Chunked = [Value || {<<"transfer-encoding">>, Value} <- Headers, + string:find(string:lowercase(Value), <<"chunked">>) =/= nomatch], + Length = [Value || {<<"content-length">>, Value} <- Headers], + case {Chunked, Length} of + {[_ | _], _} -> {chunked, size}; + {[], [Value | _]} -> + case string:to_integer(string:trim(Value)) of + {0, <<>>} -> done; + {Size, <<>>} when Size > 0 -> {length, Size}; + _ -> close + end; + {[], []} -> close + end. + +payload([]) -> <<>>; +payload([Data]) -> Data; +payload(Data) -> iolist_to_binary(Data). + +%% The body bytes in Buffer, the framing state after them, and the bytes +%% left over for the next read. +unframe(done, _) -> {[], done, <<>>}; +unframe(close, Buffer) -> {[Buffer || Buffer =/= <<>>], close, <<>>}; +unframe({length, Size}, Buffer) when byte_size(Buffer) >= Size -> + {[binary:part(Buffer, 0, Size)], done, <<>>}; +unframe({length, Size}, Buffer) -> + {[Buffer || Buffer =/= <<>>], {length, Size - byte_size(Buffer)}, <<>>}; +unframe({chunked, State}, Buffer) -> + case chunks(State, Buffer, []) of + {Data, done, Rest} -> {Data, done, Rest}; + {Data, Next, Rest} -> {Data, {chunked, Next}, Rest}; + {error, _} = Error -> Error + end. + +%% Chunked transfer coding: size reads a chunk-size line, {data, N} has N +%% bytes of chunk data left, crlf ends a chunk, trailers skips trailer lines. +chunks({data, Size}, Buffer, Data) -> + case Buffer of + <> -> chunks(crlf, Rest, [Chunk | Data]); + <<>> -> {lists:reverse(Data), {data, Size}, <<>>}; + _ -> {lists:reverse([Buffer | Data]), {data, Size - byte_size(Buffer)}, <<>>} + end; +chunks(crlf, <<"\r\n", Rest/binary>>, Data) -> chunks(size, Rest, Data); +chunks(crlf, Buffer, Data) when Buffer =:= <<>>; Buffer =:= <<"\r">> -> + {lists:reverse(Data), crlf, Buffer}; +chunks(crlf, _, _) -> {error, <<"malformed chunk">>}; +chunks(Line, Buffer, Data) when Line =:= size; Line =:= trailers -> + %% A one-byte pattern is a memchr; a two-byte one is compiled per call. + case binary:match(Buffer, <<"\n">>) of + nomatch when byte_size(Buffer) > ?MAX_CHUNK_LINE, Line =:= size -> + {error, <<"chunk size line too long">>}; + nomatch when byte_size(Buffer) > ?MAX_HEAD -> + {error, <<"trailers too large">>}; + nomatch -> {lists:reverse(Data), Line, Buffer}; + {0, 1} -> {error, <<"malformed chunk line">>}; + {At, 1} -> + case Buffer of + <> -> + case {Line, chunk_size(Text, 0, 0)} of + {trailers, _} when At =:= 1 -> {lists:reverse(Data), done, Rest}; + {trailers, _} -> chunks(trailers, Rest, Data); + {size, {ok, 0}} -> chunks(trailers, Rest, Data); + {size, {ok, Size}} -> chunks({data, Size}, Rest, Data); + {size, error} -> {error, <<"malformed chunk size">>} + end; + _ -> {error, <<"malformed chunk line">>} + end + end. + +%% The hex size before any chunk extension; Digits counts the digits read. +chunk_size(<>, Size, Digits) when C >= $0, C =< $9 -> + chunk_size(Rest, Size * 16 + C - $0, Digits + 1); +chunk_size(<>, Size, Digits) when C >= $a, C =< $f -> + chunk_size(Rest, Size * 16 + C - $a + 10, Digits + 1); +chunk_size(<>, Size, Digits) when C >= $A, C =< $F -> + chunk_size(Rest, Size * 16 + C - $A + 10, Digits + 1); +chunk_size(<>, Size, Digits) when Digits > 0, (C =:= $; orelse C =:= $\s orelse C =:= $\t) -> + {ok, Size}; +chunk_size(<<>>, Size, Digits) when Digits > 0 -> {ok, Size}; +chunk_size(_, _, _) -> error. + %% Gun defers owner death while a close-delimited response is still streaming. %% A killed caller cannot run with_connection's cleanup, so close it explicitly. close_when_owner_exits(Pid) -> @@ -67,9 +308,7 @@ await_connection(Pid, Monitor, Target, Headers, Body, Timeout) -> %% gun exits badarg when connect/4 answers einval or eaddrnotavail. {error, {down, badarg}} -> cleanup(Pid, Monitor), - {error, {transport_error, <<"connection failed: this machine could not open " - "a local socket (EADDRNOTAVAIL or EINVAL); its ephemeral ports may be " - "exhausted by another program's connections">>}}; + {error, {transport_error, ?NO_LOCAL_SOCKET}}; {error, Reason} -> cleanup(Pid, Monitor), transport_error(<<"connection failed">>, Reason) @@ -102,7 +341,9 @@ send(Pid, Target, Headers, Body, Timeout) -> Sized = [{<<"content-length">>, integer_to_binary(Length)} | Headers], Stream = gun:post(Pid, Target, Sized, #{flow => ?FLOW}), Deadline = erlang:monotonic_time(millisecond) + Timeout, - case write(Pid, Stream, Segments, Deadline, hasher(Marker)) of + Send = fun(IsFin, Data) -> gun:data(Pid, Stream, IsFin, Data) end, + Drain = fun() -> drained(Pid, Deadline) end, + case write(Send, Drain, Segments, hasher(Marker)) of ok -> {ok, Stream}; {error, Reason} -> gun:cancel(Pid, Stream), @@ -163,24 +404,25 @@ read_payload(Size, Read) -> _ -> {error, <<"a stored image payload is missing or damaged">>} end. -write(_, _, [], _, _) -> ok; -write(Pid, Stream, [{data, Data} | Rest], Deadline, Hasher) -> - gun:data(Pid, Stream, fin(Rest), Data), - write(Pid, Stream, Rest, Deadline, feed(Hasher, Data)); -write(Pid, Stream, [{image, Size, Read} | Rest], Deadline, Hasher) -> +%% Send(IsFin, Data) writes one segment; Drain() waits, after an image, until +%% the writer has taken it, so at most one image payload is held at a time. +write(_, _, [], _) -> ok; +write(Send, Drain, [{data, Data} | Rest], Hasher) -> + then(Send(fin(Rest), Data), fun() -> write(Send, Drain, Rest, feed(Hasher, Data)) end); +write(Send, Drain, [{image, Size, Read} | Rest], Hasher) -> case read_payload(Size, Read) of {ok, Payload} -> - gun:data(Pid, Stream, fin(Rest), Payload), - case drained(Pid, Deadline) of - ok -> write(Pid, Stream, Rest, Deadline, feed(Hasher, Payload)); - Error -> Error - end; + then(Send(fin(Rest), Payload), fun() -> + then(Drain(), fun() -> write(Send, Drain, Rest, feed(Hasher, Payload)) end) + end); {error, _} = Error -> Error end; -write(Pid, Stream, [{attest, _, _} = Segment | Rest], Deadline, Hasher) -> +write(Send, Drain, [{attest, _, _} = Segment | Rest], Hasher) -> Value = marker_value(Segment, Rest, Hasher), - gun:data(Pid, Stream, fin(Rest), Value), - write(Pid, Stream, Rest, Deadline, done). + then(Send(fin(Rest), Value), fun() -> write(Send, Drain, Rest, done) end). + +then(ok, Next) -> Next(); +then({error, _} = Error, _) -> Error. %% The exact binary sent over the wire, with payloads read and markers spliced. materialize(Body) -> @@ -223,26 +465,29 @@ drained(Pid, Deadline) -> end end. -receive_message({connection, Owner, Pid, Stream, Monitor, Timeout}) +%% The next message, and the connection to read the one after it from. +receive_message({connection, Owner, Pid, Stream, Monitor, Timeout} = Connection) when Owner =:= self() -> case gun:await(Pid, Stream, Timeout, Monitor) of {inform, _Status, _Headers} -> - receive_message({connection, Owner, Pid, Stream, Monitor, Timeout}); + receive_message(Connection); {response, IsFin, Status, Headers} -> - {ok, {headers, Status, Headers, IsFin =:= fin}}; + {ok, {{headers, Status, Headers, IsFin =:= fin}, Connection}}; {data, IsFin, Bytes} -> maybe_replenish(IsFin, Pid, Stream), - {ok, {data, Bytes, IsFin =:= fin}}; + {ok, {{data, Bytes, IsFin =:= fin}, Connection}}; {trailers, _Headers} -> - {ok, {data, <<>>, true}}; + {ok, {{data, <<>>, true}, Connection}}; {error, timeout} -> {error, timed_out}; {error, Reason} -> transport_error(<<"response failed">>, Reason); _Other -> - receive_message({connection, Owner, Pid, Stream, Monitor, Timeout}) + receive_message(Connection) end; -receive_message({connection, _, _, _, _, _}) -> +receive_message({direct, Owner, _, _, _} = Connection) when Owner =:= self() -> + direct_message(Connection); +receive_message(_) -> {error, {transport_error, <<"connection used by a process that does not own it">>}}. maybe_replenish(nofin, Pid, Stream) -> @@ -250,6 +495,13 @@ maybe_replenish(nofin, Pid, Stream) -> maybe_replenish(fin, _, _) -> ok. +close({direct, Owner, Socket, _, _}) -> + gen_tcp:close(Socket), + case Owner =:= self() of + true -> flush_socket(Socket); + false -> ok + end, + nil; close({connection, Owner, Pid, Stream, Monitor, _Timeout}) -> case Owner =:= self() of true -> diff --git a/src/albedo/openai_api/chat.gleam b/src/albedo/openai_api/chat.gleam index a7da1a31..38a11112 100644 --- a/src/albedo/openai_api/chat.gleam +++ b/src/albedo/openai_api/chat.gleam @@ -1,3 +1,4 @@ +import albedo/openai_api/fields import albedo/openai_api/reasoning import albedo/openai_api/replay import albedo/openai_api/types @@ -73,30 +74,82 @@ pub fn feed( types.InvalidEvent("invalid chat JSON: " <> string.inspect(error)) }), ) - let error_decoder = - decode.optionally_at( - ["error", "message"], - None, - decode.map(decode.string, Some), - ) - case decode.run(value, error_decoder) { - Ok(Some(message)) -> Error(types.ProviderError(message)) - _ -> { - use chunk <- result.try( - decode.run(value, chunk_decoder()) - |> result.map_error(fn(error) { - types.InvalidEvent( - "invalid chat completion chunk: " <> string.inspect(error), - ) - }), - ) - apply_chunk(state, chunk) - } - } + use chunk <- result.try(case text_chunk(value) { + Ok(chunk) -> Ok(chunk) + Error(Nil) -> decode_chunk(value) + }) + apply_chunk(state, chunk) } } } +fn decode_chunk(value: dynamic.Dynamic) -> Result(Chunk, types.Error) { + let error_decoder = + decode.optionally_at( + ["error", "message"], + None, + decode.map(decode.string, Some), + ) + case decode.run(value, error_decoder) { + Ok(Some(message)) -> Error(types.ProviderError(message)) + _ -> + decode.run(value, chunk_decoder()) + |> result.map_error(fn(error) { + types.InvalidEvent( + "invalid chat completion chunk: " <> string.inspect(error), + ) + }) + } +} + +/// The chunk most of a stream is, read without the decoders: one choice at +/// index 0, no finish, usage or error, and a delta of only text, refusal +/// and reasoning text. Error(Nil) for anything else, which they then read. +fn text_chunk(value: dynamic.Dynamic) -> Result(Chunk, Nil) { + use <- require( + fields.missing(value, "error") && fields.missing(value, "usage"), + ) + use id <- result.try(fields.optional_string(value, "id")) + use choices <- result.try(fields.list(value, "choices")) + use choice <- result.try(case choices { + [choice] -> Ok(choice) + _ -> Error(Nil) + }) + use index <- result.try(fields.int(choice, "index")) + use <- require(index == 0 && fields.missing(choice, "finish_reason")) + use delta <- result.try(fields.object(choice, "delta")) + use <- require(fields.empty_except(delta, text_fields)) + use role <- result.try(fields.optional_string(delta, "role")) + use <- require(role == None || role == Some("assistant")) + use content <- result.try(fields.optional_string(delta, "content")) + use refusal <- result.try(fields.optional_string(delta, "refusal")) + use reasoning <- result.map(reasoning.read_text(delta)) + let delta = Delta(role, non_empty(content), non_empty(refusal), reasoning, []) + Chunk(id, [Choice(0, delta, None)], None) +} + +const text_fields = [ + "role", + "content", + "refusal", + "reasoning", + "reasoning_content", +] + +fn require(condition: Bool, next: fn() -> Result(a, Nil)) -> Result(a, Nil) { + case condition { + True -> next() + False -> Error(Nil) + } +} + +fn non_empty(value: Option(String)) -> Option(String) { + case value { + Some("") -> None + other -> other + } +} + fn chunk_decoder() -> decode.Decoder(Chunk) { use id <- decode.optional_field("id", None, decode.optional(decode.string)) use choices <- decode.field("choices", decode.list(choice_decoder())) @@ -165,13 +218,7 @@ fn object_fields( ) -> Result(Dict(String, dynamic.Dynamic), Dict(String, dynamic.Dynamic)) fn non_empty_string() -> decode.Decoder(Option(String)) { - decode.optional(decode.string) - |> decode.map(fn(value) { - case value { - Some("") -> None - other -> other - } - }) + decode.optional(decode.string) |> decode.map(non_empty) } fn tool_fragment_decoder() -> decode.Decoder(ToolFragment) { diff --git a/src/albedo/openai_api/fields.gleam b/src/albedo/openai_api/fields.gleam new file mode 100644 index 00000000..044f7f43 --- /dev/null +++ b/src/albedo/openai_api/fields.gleam @@ -0,0 +1,52 @@ +//// Field reads over objects `json.parse` produced, without the per-field +//// bookkeeping of `gleam/dynamic/decode`. Each returns `Error(Nil)` for any +//// shape its decoder equivalent would not take as is, so a caller reads the +//// common shape here and falls back to that decoder, which owns the verdict. + +import gleam/dynamic.{type Dynamic} +import gleam/option.{type Option} + +/// A parsed event and its `type` string, as +/// `decode.field("type", decode.string)` reads it. +@external(erlang, "albedo_openai_json", "event") +pub fn event(data: String) -> Result(#(String, Dynamic), Nil) + +/// `decode.field(key, decode.string)`. +@external(erlang, "albedo_openai_json", "string_field") +pub fn string(object: Dynamic, key: String) -> Result(String, Nil) + +/// `decode.field(key, decode.int)`. +@external(erlang, "albedo_openai_json", "int_field") +pub fn int(object: Dynamic, key: String) -> Result(Int, Nil) + +/// `decode.optional_field(key, default, decode.int)`. +@external(erlang, "albedo_openai_json", "int_field_or") +pub fn int_or(object: Dynamic, key: String, default: Int) -> Result(Int, Nil) + +/// `decode.optional_field(key, None, decode.optional(decode.string))`. +@external(erlang, "albedo_openai_json", "optional_string_field") +pub fn optional_string( + object: Dynamic, + key: String, +) -> Result(Option(String), Nil) + +/// `decode.optional_field(key, None, decode.optional(decode.int))`. +@external(erlang, "albedo_openai_json", "optional_int_field") +pub fn optional_int(object: Dynamic, key: String) -> Result(Option(Int), Nil) + +/// The elements of a list field. +@external(erlang, "albedo_openai_json", "list_field") +pub fn list(object: Dynamic, key: String) -> Result(List(Dynamic), Nil) + +/// An object field. +@external(erlang, "albedo_openai_json", "object_field") +pub fn object(object: Dynamic, key: String) -> Result(Dynamic, Nil) + +/// Whether a field is absent or null. +@external(erlang, "albedo_openai_json", "missing") +pub fn missing(object: Dynamic, key: String) -> Bool + +/// Whether `object` is an object whose fields outside `keys` are all null +/// or empty lists. +@external(erlang, "albedo_openai_json", "empty_except") +pub fn empty_except(object: Dynamic, keys: List(String)) -> Bool diff --git a/src/albedo/openai_api/reasoning.gleam b/src/albedo/openai_api/reasoning.gleam index c6e658ab..0c4a46b6 100644 --- a/src/albedo/openai_api/reasoning.gleam +++ b/src/albedo/openai_api/reasoning.gleam @@ -1,5 +1,6 @@ //// Compatible-provider reasoning fields. Preserve their names and opaque metadata. +import albedo/openai_api/fields import albedo/openai_api/types import gleam/dict.{type Dict} import gleam/dynamic.{type Dynamic} @@ -50,6 +51,22 @@ pub fn decoder() -> decode.Decoder(Delta) { decode.success(Delta(text, details)) } +/// What `decoder` reads from an object without `reasoning_details`, read +/// directly; Error(Nil) when a text field is neither a string nor null. +pub fn read_text(object: Dynamic) -> Result(Delta, Nil) { + use reasoning <- result.try(fields.optional_string(object, "reasoning")) + use content <- result.map(fields.optional_string(object, "reasoning_content")) + let text = + [#("reasoning", reasoning), #("reasoning_content", content)] + |> list.fold(dict.new(), fn(text, field) { + case field { + #(key, Some(value)) -> dict.insert(text, key, value) + #(_, None) -> text + } + }) + Delta(text, []) +} + fn detail_decoder() -> decode.Decoder(DetailDelta) { use fields <- decode.then(decode.dict(decode.string, decode.dynamic)) use index <- decode.field("index", decode.int) diff --git a/src/albedo/openai_api/responses.gleam b/src/albedo/openai_api/responses.gleam index 33401e21..5232e257 100644 --- a/src/albedo/openai_api/responses.gleam +++ b/src/albedo/openai_api/responses.gleam @@ -1,3 +1,4 @@ +import albedo/openai_api/fields import albedo/openai_api/replay import albedo/openai_api/types import gleam/dict.{type Dict} @@ -50,17 +51,18 @@ pub fn feed( let State(terminal:, ..) = state case terminal { True -> Error(types.InvalidEvent("response event after terminal event")) - False -> { - use envelope <- result.try( - json.parse(data, envelope_decoder()) - |> result.map_error(fn(error) { - types.InvalidEvent( - "invalid Responses event: " <> string.inspect(error), - ) - }), - ) - dispatch(state, envelope) - } + False -> + case fields.event(data) { + Ok(#(kind, value)) -> dispatch(state, Envelope(kind, value)) + Error(Nil) -> + json.parse(data, envelope_decoder()) + |> result.map_error(fn(error) { + types.InvalidEvent( + "invalid Responses event: " <> string.inspect(error), + ) + }) + |> result.try(dispatch(state, _)) + } } } @@ -137,7 +139,13 @@ fn text_delta( decode.success(#(output_index, item_id, content_index, delta)) } let context = "response.output_text.delta" - use #(output_index, item_id, content_index, delta) <- result.try(run( + let fast = { + use #(output_index, item_id, delta) <- result.try(delta_fields(value)) + use content_index <- result.map(fields.int_or(value, "content_index", 0)) + #(output_index, item_id, content_index, delta) + } + use #(output_index, item_id, content_index, delta) <- result.try(read( + fast, value, decoder, context, @@ -165,12 +173,28 @@ fn arguments_delta( decode.success(#(output_index, item_id, delta)) } let context = "response.function_call_arguments.delta" - use #(output_index, item_id, delta) <- result.try(run(value, decoder, context)) + use #(output_index, item_id, delta) <- result.try(read( + delta_fields(value), + value, + decoder, + context, + )) use index <- result.try(resolve_index(state, output_index, item_id, context)) let name = dict.get(state.call_names, index) |> result.unwrap("") Ok(#(state, [types.ArgumentsDelta(index, name, delta)], None)) } +/// A delta event's `output_index`, `item_id` and `delta`, read without a +/// decoder when they have the types the delta decoders require. +fn delta_fields( + value: dynamic.Dynamic, +) -> Result(#(Option(Int), Option(String), String), Nil) { + use output_index <- result.try(fields.optional_int(value, "output_index")) + use item_id <- result.try(fields.optional_string(value, "item_id")) + use delta <- result.map(fields.string(value, "delta")) + #(output_index, item_id, delta) +} + /// A function call's item opens with its name, before any of its arguments. fn output_item_added( state: State, @@ -295,7 +319,8 @@ fn reasoning_delta( use delta <- decode.field("delta", decode.string) decode.success(types.ThinkingDelta(delta)) } - use event <- result.try(run(value, decoder, context)) + let fast = fields.string(value, "delta") |> result.map(types.ThinkingDelta) + use event <- result.try(read(fast, value, decoder, context)) Ok(#(state, [event], None)) } @@ -433,6 +458,19 @@ fn run( }) } +/// `fast` when it read the event, else what `decoder` makes of it. +fn read( + fast: Result(a, Nil), + value: dynamic.Dynamic, + decoder: decode.Decoder(a), + context: String, +) -> Result(a, types.Error) { + case fast { + Ok(read) -> Ok(read) + Error(Nil) -> run(value, decoder, context) + } +} + fn record_id( state: State, incoming: Option(String), diff --git a/src/albedo/openai_api/sse.gleam b/src/albedo/openai_api/sse.gleam index 98ccbe88..6fa4f0cd 100644 --- a/src/albedo/openai_api/sse.gleam +++ b/src/albedo/openai_api/sse.gleam @@ -162,13 +162,12 @@ fn complete_line( parser: Parser, events: List(Event), ) -> Result(#(Parser, List(Event)), Error) { - use line <- result.try( - parser.fragments - |> list.reverse - |> bit_array.concat - |> utf8 - |> result.replace_error(InvalidUtf8), - ) + // A line that arrived in one chunk, as most do, is used without a copy. + let bytes = case parser.fragments { + [fragment] -> fragment + fragments -> fragments |> list.reverse |> bit_array.concat + } + use line <- result.try(utf8(bytes) |> result.replace_error(InvalidUtf8)) // Comments are free; every other line counts toward the event budget. let bytes = parser.event_bytes + parser.line_bytes let parser = Parser(..parser, fragments: [], line_bytes: 0, pending_cr: False) @@ -195,6 +194,7 @@ fn complete_line( fn dispatch(parser: Parser, events: List(Event)) -> #(Parser, List(Event)) { let events = case parser.data { [] -> events + [data] -> [Event(parser.name, data), ..events] data -> [ Event(parser.name, data |> list.reverse |> string.join("\n")), ..events diff --git a/src/albedo/openai_api/transport.gleam b/src/albedo/openai_api/transport.gleam index a3ec2928..3070dd17 100644 --- a/src/albedo/openai_api/transport.gleam +++ b/src/albedo/openai_api/transport.gleam @@ -23,9 +23,10 @@ pub fn open( timeout_ms: Int, ) -> Result(Connection, Error) -/// Wait for the next response headers or body chunk. +/// Wait for the next response headers or body chunk; the connection it +/// returns reads the one after. @external(erlang, "albedo_openai_transport", "receive_message") -pub fn receive(connection: Connection) -> Result(Message, Error) +pub fn receive(connection: Connection) -> Result(#(Message, Connection), Error) /// Close the connection and discard only its pending transport messages. @external(erlang, "albedo_openai_transport", "close") diff --git a/test/manual/albedo_openai_rate_bench.erl b/test/manual/albedo_openai_rate_bench.erl new file mode 100644 index 00000000..93b03ad2 --- /dev/null +++ b/test/manual/albedo_openai_rate_bench.erl @@ -0,0 +1,89 @@ +-module(albedo_openai_rate_bench). +-export([env/2, token/0, measure/3]). + +%% Helpers for manual/stream_rate_benchmark: one stream against +%% test/manual/mock_openai_server.py, measured from the calling process. + +env(Name, Default) -> + case os:getenv(binary_to_list(Name)) of + false -> Default; + Value -> unicode:characters_to_binary(Value) + end. + +%% Called from the stream callback for every text delta. +token() -> + put(arrivals, [os:system_time(nanosecond) | get(arrivals)]), + nil. + +%% Runs Run in a fresh process; returns {Result, Metrics} where Metrics is a +%% list of {Name, Number} pairs. VM CPU is microstate accounting over every +%% scheduler and thread, so the connection process and the poller count too. +measure(Port, Tokens, Run) -> + {ok, _} = application:ensure_all_started(gun), + Owner = self(), + erlang:garbage_collect(), + {Reductions0, _} = statistics(reductions), + {Runtime0, _} = statistics(runtime), + msacc:start(), + msacc:reset(), + {Pid, Ref} = spawn_monitor(fun() -> + put(arrivals, []), + Start = erlang:monotonic_time(microsecond), + Result = Run(), + End = erlang:monotonic_time(microsecond), + {reductions, Reductions} = process_info(self(), reductions), + {memory, Memory} = process_info(self(), memory), + Owner ! {self(), done, Result, End - Start, Reductions, Memory, get(arrivals)} + end), + receive + {Pid, done, Result, Elapsed, Worker, Memory, Arrivals} -> + Stats = msacc:stats(), + Runtime = msacc:stats(system_runtime, Stats), + msacc:stop(), + {Runtime1, _} = statistics(runtime), + case os:getenv("ALBEDO_BENCH_MSACC") of false -> ok; _ -> msacc:print(Stats) end, + {Reductions1, _} = statistics(reductions), + demonitor(Ref, [flush]), + Received = length(Arrivals), + Latency = latency(Port, lists:reverse(Arrivals)), + Per = fun(X) -> X / max(1, Tokens) end, + {Result, [{Name, float(Value)} || {Name, Value} <- [ + {<<"tokens">>, Received}, + {<<"wall_ms">>, Elapsed / 1000}, + {<<"tok_per_s">>, Received / (Elapsed / 1000000)}, + {<<"vm_cpu_us_per_tok">>, Per(Runtime)}, + {<<"vm_cpu_pct">>, 100 * Runtime / Elapsed}, + {<<"os_cpu_us_per_tok">>, Per(1000 * (Runtime1 - Runtime0))}, + {<<"worker_reds_per_tok">>, Per(Worker)}, + {<<"vm_reds_per_tok">>, Per(Reductions1 - Reductions0)}, + {<<"worker_memory_kb">>, Memory / 1024} + | Latency + ]]}; + {'DOWN', Ref, process, Pid, Reason} -> error({benchmark_worker_crashed, Reason}) + end. + +%% Send-to-callback latency per token, from the mock's send times. +latency(Port, Arrivals) -> + Sent = timings(Port), + case length(Sent) =:= length(Arrivals) of + false -> [{<<"latency_mismatch">>, length(Sent)}]; + true -> + Sorted = lists:sort([(A - S) / 1000 || {S, A} <- lists:zip(Sent, Arrivals)]), + [{<<"lat_p50_us">>, pct(Sorted, 0.50)}, {<<"lat_p99_us">>, pct(Sorted, 0.99)}, + {<<"lat_max_us">>, lists:last(Sorted)}] + end. + +pct(Sorted, P) -> lists:nth(max(1, round(P * length(Sorted))), Sorted). + +timings(Port) -> + {ok, Socket} = gen_tcp:connect({127, 0, 0, 1}, Port, [binary, {active, false}]), + ok = gen_tcp:send(Socket, <<"GET /timings HTTP/1.1\r\nhost: mock\r\n\r\n">>), + Response = read_all(Socket, []), + [_, Body] = binary:split(Response, <<"\r\n\r\n">>), + json:decode(Body). + +read_all(Socket, Acc) -> + case gen_tcp:recv(Socket, 0, 5000) of + {ok, Data} -> read_all(Socket, [Data | Acc]); + {error, closed} -> iolist_to_binary(lists:reverse(Acc)) + end. diff --git a/test/manual/mock_openai_server.py b/test/manual/mock_openai_server.py new file mode 100644 index 00000000..2a370136 --- /dev/null +++ b/test/manual/mock_openai_server.py @@ -0,0 +1,210 @@ +"""A local OpenAI-compatible streaming server for transport benchmarks. + +Run explicitly: python3 test/manual/mock_openai_server.py [port] +The base URL picks the stream: http://127.0.0.1:PORT///v1 +streams text deltas at tokens per second (0 for as fast as +the socket takes them), one SSE event per write, over HTTP/1.1 chunked +encoding. GET /timings returns the send time (ns, CLOCK_REALTIME) of every +delta of the last stream, so a client can compute per-token latency. +""" + +import asyncio +import json +import sys +import time + +WORDS = [ + "the", + " quick", + " brown", + " fox", + " jumps", + " over", + " lazy", + " dog", + ".", + "\n", +] +timings: list[int] = [] + + +def sse(name: str | None, payload: object) -> bytes: + data = json.dumps(payload, separators=(",", ":")) + head = f"event: {name}\n" if name else "" + return f"{head}data: {data}\n\n".encode() + + +def chunked(body: bytes) -> bytes: + return b"%x\r\n%s\r\n" % (len(body), body) + + +def message(text: str, status: str) -> dict: + content = [{"type": "output_text", "text": text, "annotations": [], "logprobs": []}] + return { + "id": "msg_0001", + "type": "message", + "status": status, + "role": "assistant", + "content": content, + } + + +def responses(tokens: list[str]): + created = { + "id": "resp_0001", + "object": "response", + "status": "in_progress", + "model": "mock", + "output": [], + } + yield ( + None, + sse( + "response.created", + {"type": "response.created", "sequence_number": 0, "response": created}, + ), + ) + added = { + "type": "response.output_item.added", + "sequence_number": 1, + "output_index": 0, + "item": message("", "in_progress"), + } + yield None, sse("response.output_item.added", added) + for i, token in enumerate(tokens): + delta = { + "type": "response.output_text.delta", + "sequence_number": i + 2, + "item_id": "msg_0001", + "output_index": 0, + "content_index": 0, + "delta": token, + "logprobs": [], + "obfuscation": "x7Qp2LmN", + } + yield i, sse("response.output_text.delta", delta) + full = message("".join(tokens), "completed") + done = { + "type": "response.output_item.done", + "sequence_number": len(tokens) + 2, + "output_index": 0, + "item": full, + } + yield None, sse("response.output_item.done", done) + usage = { + "input_tokens": 1200, + "input_tokens_details": {"cached_tokens": 1000}, + "output_tokens": len(tokens), + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 1200 + len(tokens), + } + final = {**created, "status": "completed", "output": [full], "usage": usage} + yield ( + None, + sse( + "response.completed", + { + "type": "response.completed", + "sequence_number": len(tokens) + 3, + "response": final, + }, + ), + ) + + +def chat(tokens: list[str]): + def chunk(delta: dict, finish: str | None) -> dict: + choice = {"index": 0, "delta": delta, "logprobs": None, "finish_reason": finish} + return { + "id": "chatcmpl-0001", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mock", + "service_tier": "default", + "system_fingerprint": "fp_0001", + "choices": [choice], + "usage": None, + "obfuscation": "x7Qp2LmN", + } + + yield ( + None, + sse(None, chunk({"role": "assistant", "content": "", "refusal": None}, None)), + ) + for i, token in enumerate(tokens): + yield i, sse(None, chunk({"content": token}, None)) + yield None, sse(None, chunk({}, "stop")) + usage = { + "prompt_tokens": 1200, + "completion_tokens": len(tokens), + "prompt_tokens_details": {"cached_tokens": 1000}, + } + yield ( + None, + sse( + None, + { + "id": "chatcmpl-0001", + "object": "chat.completion.chunk", + "choices": [], + "usage": usage, + }, + ), + ) + yield None, b"data: [DONE]\n\n" + + +async def stream(writer: asyncio.StreamWriter, path: str) -> None: + _, rate, count, *_ = path.split("/") + tokens = [WORDS[i % len(WORDS)] for i in range(int(count))] + events = chat(tokens) if path.endswith("/chat/completions") else responses(tokens) + head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\nconnection: close\r\n\r\n" + writer.write(head.encode()) + timings.clear() + interval = 1 / float(rate) if float(rate) > 0 else 0.0 + started = time.monotonic() + for index, event in events: + if index is not None and interval: + delay = started + index * interval - time.monotonic() + if delay > 0: + await asyncio.sleep(delay) + if index is not None: + timings.append(time.time_ns()) + writer.write(chunked(event)) + await writer.drain() + writer.write(b"0\r\n\r\n") + await writer.drain() + + +async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + request = (await reader.readuntil(b"\r\n")).decode() + method, path, _ = request.split(" ", 2) + length = 0 + while (line := await reader.readuntil(b"\r\n")) != b"\r\n": + name, _, value = line.decode().partition(":") + if name.strip().lower() == "content-length": + length = int(value) + await reader.readexactly(length) + if method == "GET" and path == "/timings": + body = json.dumps(timings).encode() + head = f"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {len(body)}\r\nconnection: close\r\n\r\n" + writer.write(head.encode() + body) + else: + await stream(writer, path) + await writer.drain() + except (ConnectionError, asyncio.IncompleteReadError): + pass + finally: + writer.close() + + +async def main(port: int) -> None: + server = await asyncio.start_server(handle, "127.0.0.1", port) + print(f"listening on http://127.0.0.1:{port}", flush=True) + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + asyncio.run(main(int(sys.argv[1]) if len(sys.argv) > 1 else 8765)) diff --git a/test/manual/stream_rate_benchmark.gleam b/test/manual/stream_rate_benchmark.gleam new file mode 100644 index 00000000..5b4a0e32 --- /dev/null +++ b/test/manual/stream_rate_benchmark.gleam @@ -0,0 +1,111 @@ +//// Run explicitly with: gleam run -m manual/stream_rate_benchmark +//// Streams from test/manual/mock_openai_server.py (start it first; port from +//// ALBEDO_BENCH_PORT, default 8765) through the real transport, at 500 +//// tokens/s and unthrottled, and prints VM CPU and reductions per token and +//// send-to-callback latency. Run with the daemon's ERL_FLAGS (the +//// vm_defaults in priv/bin/albedo-daemon: two schedulers, no busy waiting), +//// so scheduler spinning does not count as work. +//// ALBEDO_BENCH_PROTOCOL=responses|chat_completions runs one protocol, +//// ALBEDO_BENCH_UNTHROTTLED=0 skips the unthrottled run, and +//// ALBEDO_BENCH_MSACC=1 prints each run's microstate accounting. + +import albedo/openai_api as openai +import albedo/openai_api/types +import gleam/float +import gleam/int +import gleam/io +import gleam/json +import gleam/list +import gleam/result +import gleam/string + +@external(erlang, "albedo_openai_rate_bench", "env") +fn env(name: String, default: String) -> String + +@external(erlang, "albedo_openai_rate_bench", "token") +fn token() -> Nil + +@external(erlang, "albedo_openai_rate_bench", "measure") +fn measure( + port: Int, + tokens: Int, + run: fn() -> Result(types.Turn, types.Error), +) -> #(Result(types.Turn, types.Error), List(#(String, Float))) + +const runs = 3 + +pub fn main() -> Nil { + let assert Ok(port) = int.parse(env("ALBEDO_BENCH_PORT", "8765")) + let only = env("ALBEDO_BENCH_PROTOCOL", "") + let unthrottled = env("ALBEDO_BENCH_UNTHROTTLED", "1") == "1" + [types.Responses, types.ChatCompletions] + |> list.filter(fn(protocol) { + only == "" || only == types.protocol_name(protocol) + }) + |> list.each(fn(protocol) { + bench(port, protocol, 500, 2000) + case unthrottled { + True -> bench(port, protocol, 0, 20_000) + False -> Nil + } + }) +} + +fn bench(port: Int, protocol: types.Protocol, rate: Int, tokens: Int) -> Nil { + let base = + "http://127.0.0.1:" + <> int.to_string(port) + <> "/" + <> int.to_string(rate) + <> "/" + <> int.to_string(tokens) + <> "/v1" + let client = openai.client(protocol, base, "") + let request = openai.request("mock", [types.User("go")]) + // one warm-up run loads modules and opens nothing persistent + let _ = measure(port, tokens, fn() { stream(client, request) }) + let results = + list.repeat(Nil, runs) + |> list.map(fn(_) { + measure(port, tokens, fn() { stream(client, request) }) + }) + let label = + types.protocol_name(protocol) + <> " @ " + <> case rate { + 0 -> "unthrottled" + _ -> int.to_string(rate) <> " tok/s" + } + case list.find(results, fn(run) { result.is_error(run.0) }) { + Ok(#(Error(error), _)) -> io.println(label <> ": " <> string.inspect(error)) + _ -> io.println(label <> " " <> median(list.map(results, fn(r) { r.1 }))) + } +} + +fn stream( + client: types.Client, + request: types.Request, +) -> Result(types.Turn, types.Error) { + openai.stream(client, request, fn(event) { + case event { + types.TextDelta(..) -> token() + _ -> Nil + } + types.Continue + }) +} + +/// Each metric's median across runs, as one JSON object. +fn median(runs: List(List(#(String, Float)))) -> String { + let assert [first, ..] = runs + list.map(first, fn(metric) { + let values = + list.filter_map(runs, fn(run) { list.key_find(run, metric.0) }) + |> list.sort(float.compare) + let assert Ok(middle) = + list.drop(values, list.length(values) / 2) |> list.first + #(metric.0, json.float(float.to_precision(middle, 2))) + }) + |> json.object + |> json.to_string +} diff --git a/test/openai_api/albedo_openai_transport_test_server.erl b/test/openai_api/albedo_openai_transport_test_server.erl index 84358c1a..9dc42790 100644 --- a/test/openai_api/albedo_openai_transport_test_server.erl +++ b/test/openai_api/albedo_openai_transport_test_server.erl @@ -37,7 +37,7 @@ owner_after_headers(Fixture) -> Parent = self(), {Owner, Monitor} = spawn_monitor(fun() -> {ok, Connection} = albedo_openai_transport:open(url(Fixture), [], <<>>, 2000), - {ok, {headers, 200, _, false}} = + {ok, {{headers, 200, _, false}, _}} = albedo_openai_transport:receive_message(Connection), Parent ! {transport_owner_ready, self()}, receive stop -> albedo_openai_transport:close(Connection) end @@ -125,6 +125,19 @@ respond(Socket, stream, Owner) -> receive after 40 -> ok end, ok = gen_tcp:send(Socket, <<"3\r\ntwo\r\n0\r\n\r\n">>), wait_for_close(Socket, Owner); +respond(Socket, chunked_trickle, Owner) -> + ok = inet:setopts(Socket, [{nodelay, true}]), + Response = <<"HTTP/1.1 100 Continue\r\n\r\n", + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n", + "Transfer-Encoding: chunked\r\n\r\n", + "5;name=value\r\nhello\r\n1\r\n \r\nA\r\nworld, all\r\n", + "0\r\nx-trailer: yes\r\n\r\n">>, + [begin ok = gen_tcp:send(Socket, <>), receive after 1 -> ok end end + || <> <= Response], + wait_for_close(Socket, Owner); +respond(Socket, content_length, Owner) -> + ok = gen_tcp:send(Socket, <<"HTTP/1.1 200 OK\r\ncontent-length: 16\r\n\r\nhello world, all">>), + wait_for_close(Socket, Owner); respond(Socket, close_delimited, Owner) -> ok = gen_tcp:send(Socket, [ <<"HTTP/1.0 200 OK\r\n">>, diff --git a/test/openai_api/transport_test.gleam b/test/openai_api/transport_test.gleam index 70ccbf6b..e67baacc 100644 --- a/test/openai_api/transport_test.gleam +++ b/test/openai_api/transport_test.gleam @@ -1,7 +1,10 @@ -//// Hard process death cannot execute callback cleanup; real socket ownership -//// and receive deadlines require native transport probes beyond daemon output. +//// Hard process death cannot execute callback cleanup; real socket ownership, +//// receive deadlines, and responses split at every byte require native +//// transport probes beyond daemon output. import albedo/openai_api/transport +import gleam/bit_array +import gleam/list import gleam/string_tree import gleeunit/should @@ -11,6 +14,8 @@ type Owner type Mode { CloseDelimited + ChunkedTrickle + ContentLength } @external(erlang, "albedo_openai_transport_test_server", "start") @@ -41,10 +46,32 @@ pub fn close_delimited_transport_releases_owned_sockets_test() -> Nil { let idle = start(CloseDelimited) let assert Ok(connection) = transport.open(url(idle), [], string_tree.new(), 2000) - let assert Ok(transport.Headers(200, _, False)) = + let assert Ok(#(transport.Headers(200, _, False), connection)) = transport.receive(connection) transport.receive(connection) |> should.equal(Error(transport.TimedOut)) transport.close(connection) await_closed(idle) |> should.be_true stop(idle) } + +pub fn plain_http_bodies_end_where_their_framing_says_test() -> Nil { + use mode <- list.each([ChunkedTrickle, ContentLength]) + let fixture = start(mode) + let assert Ok(connection) = + transport.open(url(fixture), [], string_tree.new(), 2000) + let assert Ok(#(transport.Headers(200, _, False), connection)) = + transport.receive(connection) + body(connection, <<>>) |> should.equal(<<"hello world, all">>) + transport.close(connection) + stop(fixture) +} + +fn body(connection: transport.Connection, read: BitArray) -> BitArray { + let assert Ok(#(transport.Data(bytes, final), connection)) = + transport.receive(connection) + let read = bit_array.append(read, bytes) + case final { + True -> read + False -> body(connection, read) + } +} -- knotmirror 0.0.1