From b2e29fd276eda1ce2e5549e543d93bbfc594de1f Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Wed, 30 Sep 2026 19:18:59 +0300 Subject: [PATCH] requests: fit history images over the provider's edge by appending image fit rows to the transcript Signed-off-by: dawn <90008@klbr.net> --- native/render/Cargo.lock | 91 +++++++++++ native/render/Cargo.toml | 3 +- native/render/src/fit.rs | 73 +++++++++ native/render/src/grid.rs | 45 ++++-- native/render/src/main.rs | 31 +++- native/render/tests/cli.rs | 25 ++- robot-docs/extensions.md | 2 +- robot-docs/models.md | 2 + src/albedo/daemon/albedo_conversation.erl | 21 ++- src/albedo/daemon/albedo_images.erl | 1 + src/albedo/daemon/conversation.gleam | 81 +++++++++- src/albedo/daemon/image_fit.gleam | 143 ++++++++++++++++++ src/albedo/daemon/images.gleam | 6 +- src/albedo/daemon/session.gleam | 35 +++++ src/albedo/daemon/session_history.gleam | 21 +++ src/albedo/daemon/session_run.gleam | 16 ++ src/albedo/daemon/transcript.gleam | 7 + src/albedo/harness/albedo_render.erl | 96 ++++++++++++ .../harness/extensions/python/cells.gleam | 2 +- .../python/migrations/cell_images.gleam | 2 +- .../snapcompact/albedo_snapcompact.erl | 69 +-------- src/albedo/harness/loop.gleam | 28 ++++ test/daemon/session_run_test.gleam | 1 + 23 files changed, 710 insertions(+), 91 deletions(-) create mode 100644 native/render/src/fit.rs create mode 100644 src/albedo/daemon/image_fit.gleam create mode 100644 src/albedo/harness/albedo_render.erl diff --git a/native/render/Cargo.lock b/native/render/Cargo.lock index 35cfa4b..527473b 100644 --- a/native/render/Cargo.lock +++ b/native/render/Cargo.lock @@ -25,6 +25,7 @@ dependencies = [ "arborium-highlight", "arborium-theme", "fontdue", + "image", "png", "unicode-width", ] @@ -683,12 +684,30 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "cc" version = "1.4.7" @@ -793,6 +812,32 @@ dependencies = [ "foldhash", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "libc" version = "0.2.189" @@ -831,6 +876,25 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "png" version = "0.18.1" @@ -844,6 +908,18 @@ dependencies = [ "miniz_oxide 0.8.9", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "regex" version = "1.13.1" @@ -932,3 +1008,18 @@ name = "zlib-rs" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/native/render/Cargo.toml b/native/render/Cargo.toml index eb24e84..e311566 100644 --- a/native/render/Cargo.toml +++ b/native/render/Cargo.toml @@ -2,7 +2,7 @@ name = "albedo-render" version = "0.1.0" edition = "2024" -description = "Renders a range of a source file as syntax-highlighted PNG pages" +description = "Renders a range of a source file as syntax-highlighted PNG pages, and fits images to a size" license = "WTFPL" publish = false @@ -10,6 +10,7 @@ publish = false arborium-highlight = "2.18.2" arborium-theme = "2.18.2" fontdue = "0.9.4" +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } png = "0.18.1" unicode-width = "0.2.2" diff --git a/native/render/src/fit.rs b/native/render/src/fit.rs new file mode 100644 index 0000000..e750682 --- /dev/null +++ b/native/render/src/fit.rs @@ -0,0 +1,73 @@ +//! An image scaled down so neither edge passes a provider's limit, for +//! screenshots and photos albedo did not render itself. The aspect ratio is +//! kept; a JPEG stays a JPEG, anything else becomes a PNG. + +use image::codecs::jpeg::JpegEncoder; +use image::imageops::FilterType; +use image::{ImageFormat, ImageReader}; +use std::io::Cursor; + +pub struct Fitted { + pub bytes: Vec, + pub width: u32, + pub height: u32, + pub extension: &'static str, +} + +pub fn fit(bytes: &[u8], edge: u32) -> Result { + let reader = ImageReader::new(Cursor::new(bytes)) + .with_guessed_format() + .map_err(|e| e.to_string())?; + let format = reader.format().ok_or("not a PNG, JPEG, or WebP image")?; + let image = reader + .decode() + .map_err(|e| format!("cannot decode the image: {e}"))?; + let (width, height) = scaled(image.width(), image.height(), edge); + let resized = image.resize_exact(width, height, FilterType::Lanczos3); + let mut out = Cursor::new(Vec::new()); + let extension = match format { + ImageFormat::Jpeg => { + JpegEncoder::new_with_quality(&mut out, 90) + .encode_image(&resized.to_rgb8()) + .map_err(|e| e.to_string())?; + "jpg" + } + _ => { + resized + .write_to(&mut out, ImageFormat::Png) + .map_err(|e| e.to_string())?; + "png" + } + }; + Ok(Fitted { + bytes: out.into_inner(), + width, + height, + extension, + }) +} + +/// The size `width` × `height` shrinks to so its longer edge is `edge`; an +/// image already inside keeps its size. +fn scaled(width: u32, height: u32, edge: u32) -> (u32, u32) { + let longer = width.max(height); + if longer <= edge { + return (width, height); + } + let shrink = + |side: u32| ((side as u64 * edge as u64 + longer as u64 / 2) / longer as u64).max(1) as u32; + (shrink(width), shrink(height)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_longer_edge_lands_on_the_limit() { + assert_eq!(scaled(2304, 1212, 2000), (2000, 1052)); + assert_eq!(scaled(1000, 8000, 2000), (250, 2000)); + assert_eq!(scaled(1200, 800, 2000), (1200, 800)); + assert_eq!(scaled(9000, 1, 2000), (2000, 1)); + } +} diff --git a/native/render/src/grid.rs b/native/render/src/grid.rs index 7a6863a..eac8d7a 100644 --- a/native/render/src/grid.rs +++ b/native/render/src/grid.rs @@ -44,13 +44,16 @@ fn parse_bdf(bytes: &[u8]) -> Result { let mut fields = line.split_whitespace(); match fields.next() { Some("ENCODING") => { - encoding = fields.next().and_then(|n| n.parse::().ok()).and_then(|n| { - if (0..=0x10FFFF).contains(&n) { - Some(n as u32) - } else { - None - } - }); + encoding = fields + .next() + .and_then(|n| n.parse::().ok()) + .and_then(|n| { + if (0..=0x10FFFF).contains(&n) { + Some(n as u32) + } else { + None + } + }); } Some("BBX") => { let mut number = || fields.next().and_then(|n| n.parse::().ok()); @@ -67,10 +70,20 @@ fn parse_bdf(bytes: &[u8]) -> Result { let _ = (w, x, y); } Some("ENDCHAR") => { - if let (Some(code), Some((w, h, x, y)), Some(rows)) = (encoding, box_, bitmap.take()) + if let (Some(code), Some((w, h, x, y)), Some(rows)) = + (encoding, box_, bitmap.take()) { if rows.len() == h { - glyphs.insert(code, Glyph { width: w, height: h, off_x: x, off_y: y, rows }); + glyphs.insert( + code, + Glyph { + width: w, + height: h, + off_x: x, + off_y: y, + rows, + }, + ); } } encoding = None; @@ -203,9 +216,15 @@ fn encode(width: usize, height: usize, pixels: Vec) -> Result encoder.set_depth(png::BitDepth::One); encoder.set_compression(png::Compression::High); let mut writer = encoder.write_header().expect("header fits in memory"); - writer.write_image_data(&bits).expect("pixels match the header"); + writer + .write_image_data(&bits) + .expect("pixels match the header"); drop(writer); - Ok(Frame { width: width as u32, height: height as u32, png: out }) + Ok(Frame { + width: width as u32, + height: height as u32, + png: out, + }) } #[cfg(test)] @@ -234,7 +253,9 @@ mod tests { #[test] fn every_ascii_glyph_leaves_ink() { - let text: String = (0x21u32..=0x7e).map(|c| char::from_u32(c).unwrap()).collect(); + let text: String = (0x21u32..=0x7e) + .map(|c| char::from_u32(c).unwrap()) + .collect(); assert!(ink(&text) > 1000, "the printable ASCII range is covered"); } } diff --git a/native/render/src/main.rs b/native/render/src/main.rs index bf06769..0929768 100644 --- a/native/render/src/main.rs +++ b/native/render/src/main.rs @@ -16,7 +16,12 @@ //! albedo-render --snapcompact[-dir] PATH --out DIR --advance N --pitch N //! --width N: dense pixel-font frames, one `image PATH WxH` stdout line //! per frame. Unknown glyphs render as '?'. +//! +//! albedo-render --fit EDGE IMAGE --out DIR: IMAGE (PNG, JPEG, or WebP) +//! scaled so neither edge passes EDGE, written to DIR/fit.png (or +//! fit.jpg for a JPEG), with one `image PATH WxH` stdout line. +mod fit; mod grid; mod highlight; mod layout; @@ -44,12 +49,15 @@ struct Request { advance: usize, pitch: usize, frame_width: usize, + fit: Option, } fn main() -> ExitCode { let parsed = parse(std::env::args().skip(1)); match parsed.and_then(|request| { - if request.snapcompact && request.snapcompact_dir { + if let Some(edge) = request.fit { + run_fit(&request, edge) + } else if request.snapcompact && request.snapcompact_dir { run_snapcompact_dir(&request) } else if request.snapcompact { run_snapcompact(&request) @@ -86,6 +94,7 @@ fn parse(mut args: impl Iterator) -> Result { advance: 11, pitch: 16, frame_width: 1568, + fit: None, }; let mut out = None; while let Some(arg) = args.next() { @@ -116,6 +125,7 @@ fn parse(mut args: impl Iterator) -> Result { "--advance" => request.advance = number(6, 24)?, "--pitch" => request.pitch = number(10, 64)?, "--width" => request.frame_width = number(256, 4096)?, + "--fit" => request.fit = Some(number(16, 16384)? as u32), "--out" => out = Some(PathBuf::from(value()?)), flag if flag.starts_with("--") => return Err(format!("unknown option {flag}")), _ if file.is_none() => file = Some(PathBuf::from(arg)), @@ -124,7 +134,7 @@ fn parse(mut args: impl Iterator) -> Result { } request.file = file.ok_or("a file to render is required")?; request.out = out.ok_or("--out DIR is required")?; - if request.snapcompact { + if request.fit.is_some() || request.snapcompact { if !request.snapcompact_dir && !request.file.is_file() { return Err(format!("{}: not a file", request.file.display())); } @@ -159,8 +169,7 @@ fn run_snapcompact_dir(request: &Request) -> Result { } let mut report = String::new(); for (index, path) in names.iter().enumerate() { - let text = std::fs::read_to_string(path) - .map_err(|e| format!("{}: {e}", path.display()))?; + let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; let frame = grid::render(&text, request.advance, request.pitch, request.frame_width)?; let out = request.out.join(format!("snap-{}.png", index + 1)); std::fs::write(&out, &frame.png).map_err(|e| format!("{}: {e}", out.display()))?; @@ -174,6 +183,20 @@ fn run_snapcompact_dir(request: &Request) -> Result { Ok(report) } +fn run_fit(request: &Request, edge: u32) -> Result { + let bytes = + std::fs::read(&request.file).map_err(|e| format!("{}: {e}", request.file.display()))?; + let fitted = fit::fit(&bytes, edge)?; + let path = request.out.join(format!("fit.{}", fitted.extension)); + std::fs::write(&path, &fitted.bytes).map_err(|e| format!("{}: {e}", path.display()))?; + Ok(format!( + "image {} {}x{}\n", + path.display(), + fitted.width, + fitted.height + )) +} + fn run_snapcompact(request: &Request) -> Result { let text = std::fs::read_to_string(&request.file) .map_err(|e| format!("{}: {e}", request.file.display()))?; diff --git a/native/render/tests/cli.rs b/native/render/tests/cli.rs index 46d0524..6917a10 100644 --- a/native/render/tests/cli.rs +++ b/native/render/tests/cli.rs @@ -1,4 +1,5 @@ -//! The binary end to end: a file in, PNG pages and a text report out. +//! The binary end to end: a file in, PNG pages and a text report out, or an +//! image in and the same image fitted to an edge out. use std::path::{Path, PathBuf}; use std::process::Command; @@ -173,3 +174,25 @@ fn without_line_numbers_the_gutter_keeps_only_wrap_marks() { assert_eq!(size(&bare), size(&numbered)); assert_ne!(bare, numbered); } + +#[test] +fn a_page_fits_inside_an_edge_and_keeps_its_aspect() { + let scratch = Scratch::new("fit"); + let source: String = (1..=60).map(|n| format!("line {n}\n")).collect(); + let file = scratch.file("sample.txt", source.as_bytes()); + let pages = scratch.0.join("pages"); + std::fs::create_dir_all(&pages).unwrap(); + assert!(render(&file, &pages, &["--start", "1", "--end", "60"]).0); + let page = pages.join("view-1.png"); + let (width, height) = png_size(&page); + let (ok, report, _) = render(&page, &scratch.0, &["--fit", "400"]); + assert!(ok); + let fitted = scratch.0.join("fit.png"); + let (w, h) = png_size(&fitted); + assert_eq!(report, format!("image {} {w}x{h}\n", fitted.display())); + assert_eq!(w.max(h), 400); + let expected = width as f64 / height as f64; + assert!((w as f64 / h as f64 - expected).abs() < 0.02); + let (ok, _, error) = render(&file, &scratch.0, &["--fit", "400"]); + assert!(!ok && error.contains("PNG, JPEG, or WebP")); +} diff --git a/robot-docs/extensions.md b/robot-docs/extensions.md index 5b8165a..5b265ff 100644 --- a/robot-docs/extensions.md +++ b/robot-docs/extensions.md @@ -117,7 +117,7 @@ optional. `await view_code(path, start_line=1, end_line=None, *, columns=79)` re the whole file is highlighted, so a range starting inside a comment or string is colored correctly. lines wrap at `columns`, with an arrow in the gutter on each continuation row; tabs expand to 4, wide characters take two cells. up to 4 pages of at most 80 display rows, split evenly at line boundaries, so wrap-heavy code fits fewer source lines per page. the returned text names each page's lines and the call that continues. `view_diff` leaves out files git does not track and asks git for a repository first, so outside one it says so. -rendering is `albedo-render`, a rust binary in `native/render` (arborium's tree-sitter grammars, bundled JetBrains Mono), run as a supervised `run` job. `native/render/install.sh` builds it into `priv/bin`; `PATH` also works. a language without a compiled-in grammar renders unhighlighted. +rendering is `albedo-render`, a rust binary in `native/render` (arborium's tree-sitter grammars, bundled JetBrains Mono), run as a supervised `run` job. `native/render/install.sh` builds it into `priv/bin`; `PATH` also works. a language without a compiled-in grammar renders unhighlighted. the same binary renders snapcompact frames and fits oversized history images to a provider's edge (`--fit`, see [models](models.md)); every daemon call goes through `albedo_render.erl`. ## work diff --git a/robot-docs/models.md b/robot-docs/models.md index e26e06f..426a33e 100644 --- a/robot-docs/models.md +++ b/robot-docs/models.md @@ -64,3 +64,5 @@ Antigravity is not on models.dev, and its model ids and required `model_enum` la A model transport contributes `ModelProviderPlugin(ModelProvider(catalog_provider, resolve))`. `catalog_provider` is the models.dev namespace used by `/login` and `/model`; `resolve` returns the profile's `extension.Upstream`: its endpoint, the replay protocol it produces, a `stream` function over albedo's request types, `explain`, which turns a provider failure into an actionable message, and `images`, the largest image edge every request it sends accepts (`extension.any_images`, albedo's own 16384px bound, when the provider states none). `openai` builds upstreams from the shared `openai_api` client with `openai.upstream`; a provider with another wire format supplies its own `stream`. Its extension should require the catalog or transport extension whose metadata and wire behavior it reuses. For example, `openai` declares the `openai` namespace, while `codex` also declares `openai` and inherits the shared transport through its dependency. New provider extensions get catalog-backed discovery by declaring their namespace. A provider whose backend reports its own models, as Codex does, can also answer from a `ModelsPlugin` of its own. `claude` declares 2000px. Anthropic takes an 8000px image on its own, but once a request carries more than 20 images every image in it, earlier turns and tool results included, must fit in 2000px, and a working session passes 20 quickly. A tool image over the limit never reaches the model: the python tool reports it under `image_errors` (`2304x1212 image is over this model's 2000px edge limit`) so the model can show a smaller one, and a submitted image over it is refused before the transcript keeps it. + +History can still hold images over the limit: ones an earlier, looser provider accepted before a model switch, or ones from before limits were declared. At the start of each model step, before the request is prepared, the loop has `albedo-render --fit EDGE` scale each such image down (aspect kept, Lanczos; a JPEG stays JPEG, anything else becomes PNG) and appends an image fit row to the transcript (`transcript.ImageFit`): the copy, stored in `images` like any payload, the original's payload hash, and a user note from origin `image scaled` (`An earlier image was scaled from WxH to WxH to fit this model's EDGEpx edge limit; the transcript keeps the original.`). Loading the history applies each fit to every row before it, so the transcript alone says what any request carried: provider request rows, `/context`, forks, and restarts all see the copy, and a fork from before the fit row starts from the original. Readers that only show rows see the fit as its note, which the chat shows once. The original row is never rewritten. A fit is permanent: switching back to a looser provider keeps the copy (fitting per edge is a TODO in `daemon/image_fit.gleam`). Snapcompact frames never need fitting, because they are rendered to fit. Without `albedo-render` the turn fails before sending, naming the binary, rather than getting the provider's 400. diff --git a/src/albedo/daemon/albedo_conversation.erl b/src/albedo/daemon/albedo_conversation.erl index 1b3b42b..2461875 100644 --- a/src/albedo/daemon/albedo_conversation.erl +++ b/src/albedo/daemon/albedo_conversation.erl @@ -1,5 +1,5 @@ -module(albedo_conversation). --export([pack/1,unpack/2,unpack_trace/1,pack_list/1,unpack_list/2,row_atoms/0,elide_tool_images/1,elision_marker/1]). +-export([pack/1,pack_fit/3,unpack/2,unpack_fit/2,unpack_trace/1,pack_list/1,unpack_list/2,row_atoms/0,elide_tool_images/1,elision_marker/1]). pack(Input) -> term_to_binary({1,albedo_images:pack(Input)}). %% binary_to_term/2 with `safe` rejects atoms that do not exist yet; a stored @@ -16,6 +16,25 @@ unpack(Bytes,Read) -> {1,{tool_output,Id,Text,Images}=Input} when is_binary(Id),is_binary(Text),is_list(Images) -> attached(Input,Read); {1,{replay,{replay_item,responses,#{<<"type">> := Type}}}=Input} when is_binary(Type) -> {ok,Input}; {1,{replay,{replay_item,chat_completions,#{<<"role">> := <<"assistant">>}}}=Input} -> {ok,Input}; + %% An image fit reads as its note everywhere but the history fold. + {1,{image_fit,{user,Text},Source,_}} when is_binary(Text),is_binary(Source) -> {ok,{user,Text}}; + _ -> {error,nil} + catch _:_ -> {error,nil} end. + +%% An image fit row (see transcript.ImageFit): its note, the source payload +%% hash, and the fitted image, already stored. +pack_fit(Note,Source,Image) -> + term_to_binary({1,{image_fit,{user,Note},Source,albedo_images:pack_image(Image)}}). + +%% {ok, ImageFit} for an image fit row, with its image's reader attached; +%% {error, nil} for any other row. +unpack_fit(Bytes,Read) -> + try binary_to_term(Bytes,[safe]) of + {1,{image_fit,{user,Note},Source,Image}} when is_binary(Note),is_binary(Source) -> + case albedo_images:load(Image,Read) of + {ok,Loaded} -> {ok,{image_fit,Note,Source,Loaded}}; + error -> {error,nil} + end; _ -> {error,nil} catch _:_ -> {error,nil} end. diff --git a/src/albedo/daemon/albedo_images.erl b/src/albedo/daemon/albedo_images.erl index 9ac7d8b..9063b91 100644 --- a/src/albedo/daemon/albedo_images.erl +++ b/src/albedo/daemon/albedo_images.erl @@ -126,6 +126,7 @@ hashes(Payload) -> {1, {user_image, _, {image, _, {stored_data, Hash, _}, _, _, _}}} -> [Hash]; {1, {tool_output, _, _, Images}} when is_list(Images) -> [Hash || {image, _, {stored_data, Hash, _}, _, _, _} <- Images]; + {1, {image_fit, _, _, {image, _, {stored_data, Hash, _}, _, _, _}}} -> [Hash]; _ -> [] catch _:_ -> [] end. diff --git a/src/albedo/daemon/conversation.gleam b/src/albedo/daemon/conversation.gleam index c543b1a..e43a377 100644 --- a/src/albedo/daemon/conversation.gleam +++ b/src/albedo/daemon/conversation.gleam @@ -1,4 +1,5 @@ import albedo/daemon/events +import albedo/daemon/image_fit import albedo/daemon/images import albedo/daemon/mail import albedo/daemon/migrations/conversation_columns @@ -442,6 +443,10 @@ pub fn load_sources( load_source_pages(store, id, -1, upper, []) } +/// A loaded row, and the image fit it is when it is one. +type LoadedRow = + #(transcript.SourcedEntry, Result(transcript.ImageFit, Nil)) + /// Rows per store round trip. A transcript is read a page at a time so the /// raw rows and their decoded entries are only ever live for one page, not /// the whole transcript at once. @@ -452,7 +457,7 @@ fn load_source_pages( id: String, after: Int, upper: Int, - pages: List(List(transcript.SourcedEntry)), + pages: List(List(LoadedRow)), ) -> Result(List(transcript.SourcedEntry), String) { let read = images.reader(store) let page = @@ -468,7 +473,12 @@ fn load_source_pages( ], source_row(), )) - use entries <- result.try(list.try_map(rows, sourced_entry(id, _, read))) + use entries <- result.try( + list.try_map(rows, fn(row) { + sourced_entry(id, row, read) + |> result.map(fn(entry) { #(entry, unpack_fit(row.1, read)) }) + }), + ) Ok(#(entries, list.last(rows) |> result.map(fn(row) { row.0 }))) }) // New rows may land between pages; the captured upper bound keeps one view. @@ -477,13 +487,37 @@ fn load_source_pages( Ok(#(entries, Ok(last))) -> case list.length(entries) == load_page_rows { True -> load_source_pages(store, id, last, upper, [entries, ..pages]) - False -> Ok(list.flatten(list.reverse([entries, ..pages]))) + False -> Ok(fitted(list.flatten(list.reverse([entries, ..pages])))) } Ok(#(entries, Error(_))) -> - Ok(list.flatten(list.reverse([entries, ..pages]))) + Ok(fitted(list.flatten(list.reverse([entries, ..pages])))) } } +/// The entries with each image fit applied to every row before it, so a +/// history read up to any row is what the model was sent at that point. +fn fitted(rows: List(LoadedRow)) -> List(transcript.SourcedEntry) { + list.fold(rows, [], fn(earlier, row) { + let #(sourced, fit) = row + let earlier = case fit { + Error(_) -> earlier + Ok(fit) -> + list.map(earlier, fn(earlier: transcript.SourcedEntry) { + let entry = earlier.entry + transcript.SourcedEntry( + ..earlier, + entry: transcript.Entry( + ..entry, + input: image_fit.apply(entry.input, fit), + ), + ) + }) + } + [sourced, ..earlier] + }) + |> list.reverse +} + /// A transcript row as the reads select it: seq, payload, timestamp, /// provider, thought_ms. fn source_row() -> decode.Decoder( @@ -844,6 +878,36 @@ pub fn entries( entries } +/// Appends image fit rows (see transcript.ImageFit), storing each fitted +/// payload, and returns their shared daemon time. +pub fn commit_fits( + store: store.Store, + id: String, + fits: List(transcript.ImageFit), + provider: String, +) -> Result(Int, String) { + let timestamp = usage.now() + store.query(store, fn(db) { + store.transaction(db, fn() { + list.try_each(fits, fn(fit) { + use stored <- result.try(images.store_images(db, [fit.image])) + let assert [image] = stored + store.run( + db, + "INSERT INTO transcript(session,payload,timestamp,provider) VALUES(?,?,?,?)", + [ + sqlight.text(id), + sqlight.blob(pack_fit(fit.note, fit.source, image)), + sqlight.int(timestamp), + sqlight.text(provider), + ], + ) + }) + |> result.replace(timestamp) + }) + }) +} + /// Atomically appends transcript inputs and returns their shared daemon time. /// Empty commits still update the session stage and return the operation time. pub fn commit( @@ -1082,6 +1146,15 @@ pub fn record_usage( @external(erlang, "albedo_conversation", "pack") fn pack(input: types.Input) -> BitArray +@external(erlang, "albedo_conversation", "pack_fit") +fn pack_fit(note: String, source: String, image: types.Image) -> BitArray + +@external(erlang, "albedo_conversation", "unpack_fit") +fn unpack_fit( + bytes: BitArray, + read: fn(String) -> Result(String, Nil), +) -> Result(transcript.ImageFit, Nil) + @external(erlang, "albedo_conversation", "unpack") fn unpack( bytes: BitArray, diff --git a/src/albedo/daemon/image_fit.gleam b/src/albedo/daemon/image_fit.gleam new file mode 100644 index 0000000..3071550 --- /dev/null +++ b/src/albedo/daemon/image_fit.gleam @@ -0,0 +1,143 @@ +//// History images fitted to a provider's edge by appending to the transcript. +//// +//// New images over the limit are refused when attached, so what needs fitting +//// is history an earlier, looser provider accepted. Each such image is scaled +//// once, in albedo-render, and the copy is appended as an image fit row (see +//// transcript.ImageFit): loading the history swaps it in for every earlier +//// occurrence, so the transcript alone says what each request carried. +//// +//// TODO: a fit is permanent, so a switch back to a looser provider keeps the +//// copy. Fitting per edge would record the edge on the row and apply a fit +//// only while the provider needs it, which also needs the limit each request +//// ran under in the transcript. + +import albedo/daemon/image +import albedo/daemon/note +import albedo/daemon/transcript +import albedo/openai_api/types +import gleam/bit_array +import gleam/crypto +import gleam/int +import gleam/list +import gleam/option +import gleam/result +import gleam/string + +@external(erlang, "albedo_render", "fit") +fn render_fit( + data: String, + edge: Int, +) -> Result(#(String, String, Int, Int, Int), String) + +/// A fit for every distinct image in `inputs` over `limits`, oldest first. +pub fn needed( + limits: types.ImageLimits, + inputs: List(types.Input), +) -> Result(List(transcript.ImageFit), String) { + inputs + |> list.flat_map(images) + |> list.filter(fn(image) { + option.is_some(types.image_refusal(limits, image)) + }) + |> list.try_fold([], fn(fits, original) { + let source = source(original) + case list.any(fits, fn(fit: transcript.ImageFit) { fit.source == source }) { + True -> Ok(fits) + False -> { + use data <- result.try( + payload(original) + |> result.replace_error(failed( + limits.max_edge, + "its payload is missing", + )), + ) + use image <- result.map( + render_fit(data, limits.max_edge) + |> result.try(fn(fitted) { + let #(mime, data, width, height, bytes) = fitted + image.validate(mime, data, width, height, bytes) + }) + |> result.map_error(failed(limits.max_edge, _)), + ) + [ + transcript.ImageFit( + note(original, image, limits.max_edge), + source, + image, + ), + ..fits + ] + } + } + }) + |> result.map(list.reverse) +} + +/// `input` with every image `fit` stands in for swapped for its copy. +pub fn apply(input: types.Input, fit: transcript.ImageFit) -> types.Input { + let swap = fn(image) { + case source(image) == fit.source { + True -> fit.image + False -> image + } + } + case input { + types.UserImage(text, image) -> types.UserImage(text, swap(image)) + types.ToolOutput(id, text, images) -> + types.ToolOutput(id, text, list.map(images, swap)) + _ -> input + } +} + +fn images(input: types.Input) -> List(types.Image) { + case input { + types.UserImage(_, image) -> [image] + types.ToolOutput(_, _, images) -> images + _ -> [] + } +} + +/// The hash albedo_images.erl stores `image`'s payload under. +fn source(image: types.Image) -> String { + case types.image_data(image) { + types.StoredData(hash, _, _) -> hash + types.InlineData(data) -> hash(data) + } +} + +fn payload(image: types.Image) -> Result(String, Nil) { + case types.image_data(image) { + types.InlineData(data) -> Ok(data) + types.StoredData(_, _, read) -> read() + } +} + +fn hash(data: String) -> String { + crypto.hash(crypto.Sha256, bit_array.from_string(data)) + |> bit_array.base16_encode + |> string.lowercase +} + +fn failed(edge: Int, reason: String) -> String { + "an image in this session is over this model's " + <> int.to_string(edge) + <> "px edge limit and could not be fitted: " + <> reason +} + +fn note(original: types.Image, fitted: types.Image, edge: Int) -> String { + let size = fn(image) { + let #(_, width, height, _) = types.image_meta(image) + int.to_string(width) <> "x" <> int.to_string(height) + } + note.wrap( + "image scaled", + "An earlier image was scaled from " + <> size(original) + <> " to " + <> size(fitted) + <> " to fit this model's " + <> int.to_string(edge) + <> "px edge limit; the transcript keeps the original.", + ) +} diff --git a/src/albedo/daemon/images.gleam b/src/albedo/daemon/images.gleam index 57488a5..d8aeb3b 100644 --- a/src/albedo/daemon/images.gleam +++ b/src/albedo/daemon/images.gleam @@ -60,9 +60,9 @@ pub fn externalize( Ok(stored) } -/// Stores a cell's inline images in the caller's transaction. The returned -/// references acquire their live reader when the cell is loaded later. -pub fn store_cell_images( +/// Stores inline images, such as a cell's, in the caller's transaction. The +/// returned references acquire their live reader when their row is loaded. +pub fn store_images( db: sqlight.Connection, images: List(types.Image), ) -> Result(List(types.Image), String) { diff --git a/src/albedo/daemon/session.gleam b/src/albedo/daemon/session.gleam index 06d2fa6..c716419 100644 --- a/src/albedo/daemon/session.gleam +++ b/src/albedo/daemon/session.gleam @@ -8,6 +8,7 @@ import albedo/daemon/events as view import albedo/daemon/family import albedo/daemon/images import albedo/daemon/mail +import albedo/daemon/note import albedo/daemon/requests import albedo/daemon/session_extensions import albedo/daemon/session_history @@ -16,6 +17,7 @@ import albedo/daemon/session_provider import albedo/daemon/session_run import albedo/daemon/session_state import albedo/daemon/session_submission +import albedo/daemon/transcript import albedo/daemon/turn.{type Submission, Submission} import albedo/daemon/usage import albedo/harness/cache_fade @@ -133,6 +135,7 @@ pub type Message { Option(Int), Subject(Result(#(Int, Option(Int)), String)), ) + CommitFits(String, List(transcript.ImageFit), Subject(Result(Nil, String))) RecordContext(String, context_snapshot.Snapshot, Bool, Subject(Nil)) RecordUsage(String, usage.Metadata, Subject(Result(Nil, String))) ReportPin(String, Option(Int), Subject(Nil)) @@ -688,6 +691,37 @@ fn handle(state: State, message: Message) { } _ -> answer(state, reply, Error("stale run")) } + CommitFits(id, fits, reply) -> + case turn.owner(state.activity, id) { + Some(_) -> + case + conversation.commit_fits( + runtime.ledger(state.host), + state.info.id, + fits, + state.info.provider, + ) + { + Ok(timestamp) -> { + let state = + list.fold(fits, state, fn(state, fit) { + let #(origin, body) = + note.parse(fit.note) |> option.unwrap(#("note", fit.note)) + session_state.emit( + state, + view.user(body, origin, None, Some(timestamp)), + ) + }) + answer( + session_history.remember_fits(state, fits, timestamp), + reply, + Ok(Nil), + ) + } + Error(error) -> answer(state, reply, Error(error)) + } + None -> answer(state, reply, Error("stale run")) + } DrainSteering(id, reply) -> case turn.live(state.activity, id), state.steering { False, _ -> answer(state, reply, Error("cancelled")) @@ -1857,6 +1891,7 @@ fn start_worker( session_run.Messages( Publish, Commit, + CommitFits, RecordContext, RecordUsage, DrainSteering, diff --git a/src/albedo/daemon/session_history.gleam b/src/albedo/daemon/session_history.gleam index b1ffcf7..a1e27a5 100644 --- a/src/albedo/daemon/session_history.gleam +++ b/src/albedo/daemon/session_history.gleam @@ -2,6 +2,7 @@ import albedo/daemon/conversation import albedo/daemon/events as view +import albedo/daemon/image_fit import albedo/daemon/projection import albedo/daemon/session_state import albedo/daemon/transcript @@ -93,6 +94,26 @@ pub fn remember( remember_response(state, inputs, timestamp, None) } +/// Remembers image fits as `conversation.commit_fits` wrote them: the copies +/// stand in for every earlier image, and each note follows. +pub fn remember_fits( + state: session_state.State(message), + fits: List(transcript.ImageFit), + timestamp: Int, +) -> session_state.State(message) { + let history = + option.map(state.history, fn(history) { + list.map(history, fn(entry) { + transcript.Entry( + ..entry, + input: list.fold(fits, entry.input, image_fit.apply), + ) + }) + }) + session_state.State(..state, history:) + |> remember(list.map(fits, fn(fit) { types.User(fit.note) }), timestamp) +} + /// Remembers committed inputs as `conversation.commit_response` wrote them. pub fn remember_response( state: session_state.State(message), diff --git a/src/albedo/daemon/session_run.gleam b/src/albedo/daemon/session_run.gleam index f493179..ade6dda 100644 --- a/src/albedo/daemon/session_run.gleam +++ b/src/albedo/daemon/session_run.gleam @@ -4,6 +4,7 @@ import albedo/daemon/context_snapshot import albedo/daemon/conversation import albedo/daemon/requests import albedo/daemon/session_state +import albedo/daemon/transcript import albedo/daemon/turn import albedo/daemon/usage import albedo/harness/extension @@ -23,6 +24,8 @@ pub type Messages(message) { Option(Int), Subject(Result(#(Int, Option(Int)), String)), ) -> message, + fits: fn(String, List(transcript.ImageFit), Subject(Result(Nil, String))) -> + message, context: fn(String, context_snapshot.Snapshot, Bool, Subject(Nil)) -> message, usage: fn(String, usage.Metadata, Subject(Result(Nil, String))) -> message, @@ -135,6 +138,17 @@ pub fn commit_fn( } } +pub fn fits_fn( + owner: Subject(message), + run_id: String, + messages: Messages(message), + waiting timeout: Int, +) -> fn(List(transcript.ImageFit)) -> Result(Nil, String) { + fn(fits) { + confirm(owner, "image fit", timeout, messages.fits(run_id, fits, _)) + } +} + pub fn usage_fn( owner: Subject(message), run_id: String, @@ -194,6 +208,7 @@ pub fn start( client, publish_fn(owner, run_id, messages, stop, 30_000), commit_fn(owner, run_id, messages, 10_000), + fits_fn(owner, run_id, messages, 10_000), fn(request, observation, compacted) { let snapshot = context_snapshot.from_request( @@ -276,6 +291,7 @@ pub fn start_background( // never block on it either. fn(_event) { True }, fn(_inputs, _stage, _thought) { Error("a background call never commits") }, + fn(_fits) { Error("a background call never commits") }, fn(_request, _observation, _compacted) { Nil }, fn(_metadata) { Ok(Nil) }, fn() { Ok([]) }, diff --git a/src/albedo/daemon/transcript.gleam b/src/albedo/daemon/transcript.gleam index 58c8829..68ea78f 100644 --- a/src/albedo/daemon/transcript.gleam +++ b/src/albedo/daemon/transcript.gleam @@ -23,3 +23,10 @@ pub type SourceRef { pub type SourcedEntry { SourcedEntry(source: SourceRef, entry: Entry) } + +/// A transcript row that swaps every earlier image whose payload hashes to +/// `source` for `image`, a copy scaled to a provider's edge. Readers that +/// only show rows see `note`, a user note that tells the model about it. +pub type ImageFit { + ImageFit(note: String, source: String, image: types.Image) +} diff --git a/src/albedo/harness/albedo_render.erl b/src/albedo/harness/albedo_render.erl new file mode 100644 index 0000000..a61a746 --- /dev/null +++ b/src/albedo/harness/albedo_render.erl @@ -0,0 +1,96 @@ +%% Runs the optional albedo-render binary (native/render). It is a subprocess, +%% so a crash is an error, never a daemon loss. snapcompact renders frames +%% through it, and requests fit oversized images with it. +-module(albedo_render). +-export([run/1, scratch/1, parse_image_line/1, fit/2]). + +-define(RENDER_TIMEOUT_MS, 30000). + +%% The image in canonical base64, scaled so neither edge passes Edge: +%% {ok, {Mime, Base64, Width, Height, DecodedBytes}} or {error, Reason}. +fit(Base64, Edge) -> + Scratch = scratch("albedo-fit-"), + try + ok = file:make_dir(Scratch), + In = filename:join(Scratch, "in"), + ok = file:write_file(In, base64:decode(Base64)), + case run(["--fit", integer_to_list(Edge), In, "--out", Scratch]) of + {ok, Output} -> + [Line | _] = binary:split(Output, <<"\n">>), + case parse_image_line(Line) of + {ok, Path, W, H} -> + {ok, Data} = file:read_file(Path), + {ok, {mime(Path), base64:encode(Data), W, H, byte_size(Data)}}; + _ -> {error, <<"unparseable renderer output">>} + end; + {error, Reason} -> {error, Reason} + end + catch _:_ -> + {error, <<"the image could not be fitted">>} + after + _ = file:del_dir_r(Scratch) + end. + +mime(Path) -> + case filename:extension(Path) of + <<".jpg">> -> <<"image/jpeg">>; + _ -> <<"image/png">> + end. + +%% "image PATH WxH", the line the renderer prints per image it wrote. +parse_image_line(<<"image ", Rest/binary>>) -> + Toks = binary:split(Rest, <<" ">>, [global]), + case lists:reverse(Toks) of + [Dims | RevPath] when length(RevPath) > 0 -> + Path = iolist_to_binary(lists:join(<<" ">>, lists:reverse(RevPath))), + case binary:split(Dims, <<"x">>) of + [W, H] -> + case catch {binary_to_integer(W), binary_to_integer(H)} of + {WI, HI} when is_integer(WI), is_integer(HI), WI > 0, HI > 0 -> + {ok, Path, WI, HI}; + _ -> error + end; + _ -> error + end; + _ -> error + end; +parse_image_line(<<>>) -> skip; +parse_image_line(_) -> error. + +%% A unique scratch dir keeps concurrent renders from colliding. +scratch(Prefix) -> + Tmp = case os:getenv("TMPDIR") of false -> "/tmp"; T -> T end, + filename:join(Tmp, Prefix ++ integer_to_list(erlang:unique_integer([positive]))). + +%% The binary's stdout, stderr merged in so failures explain themselves. +run(Args) -> + case renderer() of + Bin when is_list(Bin), Bin =/= "" -> + Port = open_port({spawn_executable, Bin}, + [{args, Args}, exit_status, binary, hide, stderr_to_stdout]), + drain(Port, []); + _ -> + {error, <<"albedo-render is not built; run native/render/install.sh">>} + end. + +renderer() -> + Bin = case code:priv_dir(albedo) of + {error, _} -> ""; + Priv -> filename:join(Priv, "bin/albedo-render") + end, + case filelib:is_file(Bin) of + true -> Bin; + false -> os:find_executable("albedo-render") + end. + +drain(Port, Acc) -> + receive + {Port, {data, Data}} -> drain(Port, [Data | Acc]); + {Port, {exit_status, 0}} -> {ok, iolist_to_binary(lists:reverse(Acc))}; + {Port, {exit_status, _}} -> + {error, iolist_to_binary(["renderer failed: " | lists:reverse(Acc)])}; + {Port, eof} -> drain(Port, Acc) + after ?RENDER_TIMEOUT_MS -> + catch erlang:port_close(Port), + {error, <<"the renderer timed out">>} + end. diff --git a/src/albedo/harness/extensions/python/cells.gleam b/src/albedo/harness/extensions/python/cells.gleam index cdc010a..864554f 100644 --- a/src/albedo/harness/extensions/python/cells.gleam +++ b/src/albedo/harness/extensions/python/cells.gleam @@ -118,7 +118,7 @@ pub fn finish( store.transaction(db, fn() { use saved <- result.try(case outcome { Ok(finished) -> - images.store_cell_images(db, finished.images) + images.store_images(db, finished.images) |> result.map(fn(stored) { Ok(python.Outcome(..finished, images: stored)) }) diff --git a/src/albedo/harness/extensions/python/migrations/cell_images.gleam b/src/albedo/harness/extensions/python/migrations/cell_images.gleam index 5e068a1..c16c0fc 100644 --- a/src/albedo/harness/extensions/python/migrations/cell_images.gleam +++ b/src/albedo/harness/extensions/python/migrations/cell_images.gleam @@ -89,7 +89,7 @@ fn migrate_image_pages( ) case decoded { Ok(outcome) -> { - use stored <- result.try(images.store_cell_images( + use stored <- result.try(images.store_images( db, outcome.images, )) diff --git a/src/albedo/harness/extensions/snapcompact/albedo_snapcompact.erl b/src/albedo/harness/extensions/snapcompact/albedo_snapcompact.erl index 60a2b82..da5c512 100644 --- a/src/albedo/harness/extensions/snapcompact/albedo_snapcompact.erl +++ b/src/albedo/harness/extensions/snapcompact/albedo_snapcompact.erl @@ -1,11 +1,9 @@ -%% Renders snapcompact frames through albedo-render's grid renderer and -%% normalizes archive text into the continuous cell stream the grid layout -%% needs. The renderer is a subprocess, so a crash is an error, never a -%% daemon loss. +%% Renders snapcompact frames through albedo-render's grid renderer (see +%% albedo_render.erl) and normalizes archive text into the continuous cell +%% stream the grid layout needs. -module(albedo_snapcompact). -export([render_frames/4, paginate/2, now_ms/0, sha256/1, normalize/1, format_args/1]). --define(RENDER_TIMEOUT_MS, 30000). %% U+2588 FULL BLOCK in UTF-8: the grid layout's line marker. -define(BLOCK, <<226, 150, 136>>). @@ -14,7 +12,7 @@ %% The CLI writes OUT/snap-N.png and prints one "image PATH WxH" line per %% frame on stdout; stderr is merged in so failures explain themselves. render_frames(Chunks, Advance, Pitch, Width) -> - Scratch = scratch(), + Scratch = albedo_render:scratch("albedo-snapcompact-"), try ok = file:make_dir(Scratch), Out = filename:join(Scratch, "out"), @@ -27,7 +25,7 @@ render_frames(Chunks, Advance, Pitch, Width) -> "--advance", integer_to_list(Advance), "--pitch", integer_to_list(Pitch), "--width", integer_to_list(Width)], - case run(renderer(), Args) of + case albedo_render:run(Args) of {ok, Output} -> collect_frames(Output, []); {error, Reason} -> @@ -47,12 +45,12 @@ collect_frames(<<>>, Frames) -> collect_frames(Output, Frames) -> case binary:split(Output, <<"\n">>) of [Line, Rest] -> - case parse_image_line(Line) of + case albedo_render:parse_image_line(Line) of {ok, Path, W, H} -> read_frame(Path, W, H, Rest, Frames); error -> {error, <<"unparseable renderer output">>} end; [Line] -> - case parse_image_line(Line) of + case albedo_render:parse_image_line(Line) of {ok, Path, W, H} -> read_frame(Path, W, H, <<>>, Frames); skip -> collect_frames(<<>>, Frames); error -> {error, <<"unparseable renderer output">>} @@ -66,65 +64,12 @@ read_frame(Path, W, H, Rest, Frames) -> _ -> {error, <<"the renderer produced no frame file">>} end. -parse_image_line(<<"image ", Rest/binary>>) -> - Toks = binary:split(Rest, <<" ">>, [global]), - case lists:reverse(Toks) of - [Dims | RevPath] when length(RevPath) > 0 -> - Path = iolist_to_binary(lists:join(<<" ">>, lists:reverse(RevPath))), - case binary:split(Dims, <<"x">>) of - [W, H] -> - case catch {binary_to_integer(W), binary_to_integer(H)} of - {WI, HI} when is_integer(WI), is_integer(HI), WI > 0, HI > 0 -> - {ok, Path, WI, HI}; - _ -> error - end; - _ -> error - end; - _ -> error - end; -parse_image_line(<<>>) -> skip; -parse_image_line(_) -> error. - sha256(Text) -> binary:encode_hex(crypto:hash(sha256, Text), lowercase). now_ms() -> erlang:system_time(millisecond). -%% A unique scratch dir keeps concurrent renders from colliding. -scratch() -> - Tmp = case os:getenv("TMPDIR") of false -> "/tmp"; T -> T end, - filename:join(Tmp, "albedo-snapcompact-" ++ integer_to_list(erlang:unique_integer([positive]))). - -renderer() -> - Bin = case code:priv_dir(albedo) of - {error, _} -> ""; - Priv -> filename:join(Priv, "bin/albedo-render") - end, - case filelib:is_file(Bin) of - true -> Bin; - false -> os:find_executable("albedo-render") - end. - -run(Bin, Args) when is_list(Bin), Bin =/= "" -> - Port = open_port({spawn_executable, Bin}, - [{args, Args}, exit_status, binary, hide, stderr_to_stdout]), - drain(Port, []); -run(_, _) -> - {error, <<"albedo-render is not built; run native/render/install.sh">>}. - -drain(Port, Acc) -> - receive - {Port, {data, Data}} -> drain(Port, [Data | Acc]); - {Port, {exit_status, 0}} -> {ok, iolist_to_binary(lists:reverse(Acc))}; - {Port, {exit_status, _}} -> - {error, iolist_to_binary(["renderer failed: " | lists:reverse(Acc)])}; - {Port, eof} -> drain(Port, Acc) - after ?RENDER_TIMEOUT_MS -> - catch erlang:port_close(Port), - {error, <<"the renderer timed out">>} - end. - %% One pass: escape sequences stripped, tabs expanded to four spaces, and %% newline runs folded into single full-block cells, so the archive is one %% continuous character stream that wraps positionally. diff --git a/src/albedo/harness/loop.gleam b/src/albedo/harness/loop.gleam index 3a1690f..9a0e706 100644 --- a/src/albedo/harness/loop.gleam +++ b/src/albedo/harness/loop.gleam @@ -2,7 +2,9 @@ import albedo/daemon/conversation import albedo/daemon/events as view +import albedo/daemon/image_fit import albedo/daemon/requests +import albedo/daemon/transcript import albedo/daemon/usage import albedo/harness/cache_fade import albedo/harness/cache_ttl @@ -33,6 +35,8 @@ pub type Loop { /// links to. commit: fn(List(types.Input), conversation.Stage, Option(Int)) -> Result(#(Int, Option(Int)), String), + /// Appends image fit rows ahead of the request that needs them. + commit_fits: fn(List(transcript.ImageFit)) -> Result(Nil, String), record_context: fn(types.Request, Option(compaction.Observation), Bool) -> Nil, record_usage: fn(usage.Metadata) -> Result(Nil, String), @@ -83,6 +87,7 @@ pub fn run( step: Int, ) -> Result(Nil, String) { use _ <- result.try(checkpoint(state)) + use inputs <- result.try(fit_history(state, inputs)) let original = list.reverse(inputs) use prepared <- result.try(prepare(state, original, False)) let #(state, history) = settle_pin(state, original, prepared.inputs) @@ -156,6 +161,29 @@ pub fn run( } } +/// `inputs`, newest first, with the images the provider would refuse swapped +/// for fitted copies. The fits are committed first, so a history loaded from +/// the transcript carries the same copies. +fn fit_history( + state: Loop, + inputs: List(types.Input), +) -> Result(List(types.Input), String) { + use fits <- result.try(image_fit.needed( + state.upstream.images, + list.reverse(inputs), + )) + case fits { + [] -> Ok(inputs) + _ -> { + use _ <- result.try(state.commit_fits(fits)) + let notes = list.map(fits, fn(fit) { types.User(fit.note) }) + list.map(inputs, fn(input) { list.fold(fits, input, image_fit.apply) }) + |> list.append(list.reverse(notes), _) + |> Ok + } + } +} + /// Force the session's selected strategy without sending an ordinary assistant turn. /// The durable transcript remains unchanged; the next request uses the saved projection. pub fn compact(state: Loop, inputs: List(types.Input)) -> Result(Nil, String) { diff --git a/test/daemon/session_run_test.gleam b/test/daemon/session_run_test.gleam index 8462770..7732dc2 100644 --- a/test/daemon/session_run_test.gleam +++ b/test/daemon/session_run_test.gleam @@ -89,6 +89,7 @@ fn messages() -> session_run.Messages(Owner) { session_run.Messages( publish: fn(_, _, reply) { Publish(reply) }, commit: fn(_, _, _, _, reply) { Commit(reply) }, + fits: fn(_, _, _) { Unused }, context: fn(_, _, _, _) { Unused }, usage: fn(_, _, _) { Unused }, drain: fn(_, _) { Unused }, -- knotmirror 0.0.1