From 5b66fe77f8ea729a47c8fef4e6b1580372e05ebf Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Sat, 31 Oct 2026 18:56:35 -0400 Subject: [PATCH] read a sealed build digest instead of hashing the app tree on every boot --- cli/cmd/albedo/build_identity.go | 14 ++++++++++- cli/cmd/albedo/build_identity_test.go | 20 +++++++++++++++ nix/packages/daemon.nix | 6 +++++ robot-docs/kernel.md | 4 +-- robot-docs/runtime.md | 15 ++++++++++++ src/albedo/daemon/albedo_daemon.erl | 35 ++++++++++++++++++++++++--- test/e2e/daemon_test.py | 11 +++++++++ test/snapshot-daemon.sh | 13 +++++----- 8 files changed, 105 insertions(+), 13 deletions(-) diff --git a/cli/cmd/albedo/build_identity.go b/cli/cmd/albedo/build_identity.go index 8b4b74c8..98f2cd12 100644 --- a/cli/cmd/albedo/build_identity.go +++ b/cli/cmd/albedo/build_identity.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "sort" + "strings" "albedo/cli/internal/daemon" ) @@ -70,13 +71,24 @@ func hasEbin(dir string) bool { return err == nil && info.IsDir() } -// digestTree hashes a build the way the daemon hashes its own running tree +// digestTree reads a sealed build identity or hashes a mutable tree the way +// the daemon hashes its own running tree // (albedo_daemon:build_digest/0): sha256 over the sorted relative paths and // bytes of every regular file under ebin/ and priv/, symlinks followed. An // unreadable path fails the whole digest, the call the daemon's walker makes // too, and the shared fixture in test/fixtures/build-digest pins both // implementations to one constant. func digestTree(appDir string) string { + if file, err := os.Open(filepath.Join(appDir, ".build-digest")); err == nil { + data, err := io.ReadAll(io.LimitReader(file, 66)) + file.Close() + if err == nil && len(data) == 65 && data[64] == '\n' { + digest := string(data[:64]) + if decoded, err := hex.DecodeString(digest); err == nil && len(decoded) == sha256.Size && digest == strings.ToLower(digest) { + return digest + } + } + } var paths []string for _, sub := range []string{"ebin", "priv"} { root := filepath.Join(appDir, sub) diff --git a/cli/cmd/albedo/build_identity_test.go b/cli/cmd/albedo/build_identity_test.go index 159d137f..bd7ce499 100644 --- a/cli/cmd/albedo/build_identity_test.go +++ b/cli/cmd/albedo/build_identity_test.go @@ -3,6 +3,7 @@ package main import ( "os" "path/filepath" + "strings" "testing" ) @@ -42,6 +43,25 @@ func TestDigestTreeFailsClosedOnUnreadablePaths(t *testing.T) { } } +func TestDigestTreeUsesOnlyValidSealedIdentities(t *testing.T) { + root := t.TempDir() + beam := filepath.Join(root, "ebin", "module.beam") + write(t, beam, "original beam") + want := digestTree(root) + sidecar := filepath.Join(root, ".build-digest") + write(t, sidecar, want+"\n") + write(t, beam, "a mutable copy must discard its seal") + if got := digestTree(root); got != want { + t.Fatalf("sealed identity = %q, want %q", got, want) + } + for _, malformed := range []string{want, want + "\nextra", "z" + want[1:] + "\n", strings.ToUpper(want) + "\n"} { + write(t, sidecar, malformed) + if got := digestTree(root); got == want || got == "" { + t.Fatalf("malformed sidecar %q did not fall back to the changed tree: %q", malformed, got) + } + } +} + func TestAppDirRecognizesTheKnownLayouts(t *testing.T) { root := t.TempDir() diff --git a/nix/packages/daemon.nix b/nix/packages/daemon.nix index f1c7623b..a400fa21 100644 --- a/nix/packages/daemon.nix +++ b/nix/packages/daemon.nix @@ -103,6 +103,12 @@ in --prefix PATH : ${lib.makeBinPath [erlang python311 bash coreutils ripgrep render chrome-headless-shell]} runHook postInstall ''; + # Sealed last: fixup rewrites shebangs and strips files under priv. + postFixup = '' + ALBEDO_BUILD_APP="$out/lib/albedo/albedo" ${erlang}/bin/erl +S 2:2 \ + -pa "$out/lib/albedo"/*/ebin -noshell \ + -eval 'application:ensure_all_started(crypto), ok = albedo_daemon:seal_build(unicode:characters_to_binary(os:getenv("ALBEDO_BUILD_APP"))), halt().' + ''; meta = { description = "coding agent daemon with its runtime and native helpers"; license = lib.licenses.mpl20; diff --git a/robot-docs/kernel.md b/robot-docs/kernel.md index 8687a3b3..1c1c89e3 100644 --- a/robot-docs/kernel.md +++ b/robot-docs/kernel.md @@ -495,8 +495,8 @@ At startup the python extension scans this home's run directories, skipping ever linked, staged or orphaned kernel. The native discovery entry point lists the run directory first and avoids starting an interpreter when it is absent, empty, or contains only those recorded paths. Unknown entries still go through Python's -full identity validation before any ownership record is created. A matching live process identity with no record -is recorded in `kernel_orphans` for the ordinary idle-limit reaper. Old kernels +full identity validation before any ownership record is created. A matching live +process identity with no record is recorded in `kernel_orphans` for the ordinary idle-limit reaper. Old kernels without an identity file, stale identities and other homes are left alone. Replacing a published kernel after an attach failure first moves its record into diff --git a/robot-docs/runtime.md b/robot-docs/runtime.md index d71498f6..3b8197bd 100644 --- a/robot-docs/runtime.md +++ b/robot-docs/runtime.md @@ -101,6 +101,21 @@ to match. Polling and HTTP probe budgets remain unchanged. A stalled startup reports the log path and leaves the detached process running; check that log before retrying. A child exit reports its startup log immediately. +## sealed build identity + +An immutable application may carry `.build-digest` beside `ebin` and `priv`, +outside the hashed roots. It contains the existing tree digest as 64 lowercase +hex digits followed by one newline. `albedo_daemon:seal_build/1` writes it after +the builder has finished the tree. The daemon and CLI read this identity instead +of scanning every file. Missing or malformed sidecars fall back to the original +sorted-path-and-bytes hash. Reads are bounded to 66 bytes. + +Nix packages seal after bytecode compilation and helper installation. Daemon +snapshots copy their priv trees, compile Python bytecode, then seal; they no +longer refer back to mutable checkout priv files. An editable copy must omit or +remove its seal before changing files. Source checkout builds remain unsealed. +The daemon still memoizes the identity it booted with. + ## code loading Erlang loads modules on their first call. Before readiness, `albedo_code:prepare/1` diff --git a/src/albedo/daemon/albedo_daemon.erl b/src/albedo/daemon/albedo_daemon.erl index f3535511..d712e5da 100644 --- a/src/albedo/daemon/albedo_daemon.erl +++ b/src/albedo/daemon/albedo_daemon.erl @@ -1,6 +1,6 @@ -module(albedo_daemon). -include_lib("kernel/include/file.hrl"). --export([physical_memory_mb/0,kernel_pressure/0,kernel_pressure/1,set_kernel_pressure/1,set_kernel_budget/1,set_kernel_average/1,defaults/0,refuse_home/1,env/1,free_port/0,bind_failure/1,forget_webhook_port/1,ready/4,read_config/1,directory/1,shutdown/0,rss/1,collect_idle/0,watch_parent/1,hold/1,http_request/1,build_digest/0,tree_digest/1]). +-export([physical_memory_mb/0,kernel_pressure/0,kernel_pressure/1,set_kernel_pressure/1,set_kernel_budget/1,set_kernel_average/1,defaults/0,refuse_home/1,env/1,free_port/0,bind_failure/1,forget_webhook_port/1,ready/4,read_config/1,directory/1,shutdown/0,rss/1,collect_idle/0,watch_parent/1,hold/1,http_request/1,build_digest/0,tree_digest/1,seal_build/1]). env(Name) -> case os:getenv(binary_to_list(Name)) of false -> <<>>; Value -> unicode:characters_to_binary(Value) end. %% Home and authentication belong to the daemon, including direct starts. defaults() -> @@ -53,7 +53,8 @@ read_config(Home) -> directory(Path) -> filename:pathtype(Path) =:= absolute andalso filelib:is_dir(Path). %% The content digest of this daemon's own build: sha256 over the sorted %% relative paths and bytes of every regular file under the albedo app's -%% ebin/ and priv/ trees, symlinks followed. Memoized for the VM's life, so a +%% ebin/ and priv/ trees, symlinks followed, or the builder's sealed sidecar. +%% Memoized for the VM's life, so a %% tree edited under a running daemon keeps the digest it booted with. Empty %% when the tree cannot be read. The CLI hashes its candidate build with the %% same recipe (cli/cmd/albedo/build_identity.go); the shared fixture in @@ -73,7 +74,35 @@ build_digest() -> compute_build_digest() -> case code:lib_dir(albedo) of {error, _} -> <<>>; - AppDir -> tree_digest(AppDir) + AppDir -> + case read_build_seal(AppDir) of + {ok, <>} -> + case valid_digest(Digest) of + true -> Digest; + false -> tree_digest(AppDir) + end; + _ -> tree_digest(AppDir) + end + end. + +read_build_seal(AppDir) -> + case file:open(filename:join(AppDir, <<".build-digest">>), [read, raw, binary]) of + {ok, File} -> + Result = file:read(File, 66), + _ = file:close(File), + Result; + Error -> Error + end. + +valid_digest(<<>>) -> true; +valid_digest(<>) when C >= $0, C =< $9; C >= $a, C =< $f -> valid_digest(Rest); +valid_digest(_) -> false. + +%% Builders seal the completed app; remove the sidecar before editing its files. +seal_build(AppDir) -> + case tree_digest(AppDir) of + <<>> -> {error, unreadable_build}; + Digest -> file:write_file(filename:join(AppDir, <<".build-digest">>), <>) end. tree_digest(AppDir) -> diff --git a/test/e2e/daemon_test.py b/test/e2e/daemon_test.py index 62b2ed42..b63261f7 100644 --- a/test/e2e/daemon_test.py +++ b/test/e2e/daemon_test.py @@ -20,6 +20,8 @@ import unittest import urllib.error import urllib.parse +import harness + from inspect_support import Inspection from harness import ( Albedo, @@ -175,6 +177,15 @@ def delayed_request(app, method, path, body, headers): class DaemonTest(unittest.TestCase): + def test_server_reports_the_sealed_build_identity(self): + assert harness._executable is not None + stamp = Path(harness._executable).parent / "albedo" / ".build-digest" + expected = stamp.read_text().strip() + self.assertRegex(expected, r"^[0-9a-f]{64}$") + with Albedo() as app: + with app.api("/server") as response: + self.assertEqual(json.load(response)["digest"], expected) + @exclusive def test_empty_kernel_discovery_does_not_start_an_interpreter(self): def prepare(app): diff --git a/test/snapshot-daemon.sh b/test/snapshot-daemon.sh index f2847e0d..7b822824 100755 --- a/test/snapshot-daemon.sh +++ b/test/snapshot-daemon.sh @@ -20,15 +20,14 @@ for ebin in "$root"/build/dev/erlang/*/ebin; do name="$(basename "$package")" mkdir -p "$out/$name" cp -R "$ebin" "$out/$name/ebin" - # A priv that links to a source tree keeps pointing at it, as under - # `gleam run`. One the build itself holds (a dependency's NIF) is copied: - # a rebuild by another toolchain deletes it under a running daemon. - if [ -L "$package/priv" ]; then - ln -s "$(cd "$package/priv" && pwd -P)" "$out/$name/priv" - elif [ -d "$package/priv" ]; then - cp -R "$package/priv" "$out/$name/priv" + # A sealed snapshot owns its priv tree too; later source edits cannot change it. + if [ -d "$package/priv" ]; then + cp -RL "$package/priv" "$out/$name/priv" fi done +# Python caches are build artifacts, not changes to a sealed snapshot. +python3 -m compileall -q -f --invalidation-mode checked-hash "$out/albedo/priv/python" +ALBEDO_BUILD_APP="$out/albedo" "$root/priv/bin/albedo-daemon" erl -pa "$out"/*/ebin -noshell -eval 'application:ensure_all_started(crypto), ok = albedo_daemon:seal_build(unicode:characters_to_binary(os:getenv("ALBEDO_BUILD_APP"))), halt().' cat >"$out/albedo-daemon" <