From 2bbcb61448287cd2be58a5ff2c840e50c2b14c82 Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Wed, 30 Sep 2026 01:53:28 -0400 Subject: [PATCH] fix: let extensions contribute their sqlite migrations --- agents.md | 2 +- robot-docs/extensions.md | 26 ++++- robot-docs/migrations.md | 61 +++++++----- src/albedo/daemon/migrations.gleam | 15 +-- src/albedo/daemon/migrations/backup.gleam | 11 --- src/albedo/daemon/server.gleam | 24 +++-- src/albedo/harness/extension.gleam | 43 +++++++- .../extensions/paperclips/extension.gleam | 4 + .../extensions/paperclips/ledger.gleam | 9 +- .../paperclips/migrations/title.gleam} | 0 .../harness/extensions/python/extension.gleam | 5 + .../python}/migrations/cell_images.gleam | 20 +++- .../harness/extensions/work/extension.gleam | 4 + .../harness/extensions/work/ledger.gleam | 6 +- .../extensions/work/migrations/cwd.gleam} | 0 src/albedo/harness/runtime.gleam | 9 ++ test/e2e/migrations_test.py | 89 ++++++++++++++++- test/harness/migration_test.gleam | 97 +++++++++++++++++++ test/harness/tool_image_test.gleam | 2 +- test/harness/work_test.gleam | 53 ---------- 20 files changed, 351 insertions(+), 129 deletions(-) rename src/albedo/{daemon/migrations/paperclips_title.gleam => harness/extensions/paperclips/migrations/title.gleam} (100%) rename src/albedo/{daemon => harness/extensions/python}/migrations/cell_images.gleam (90%) rename src/albedo/{daemon/migrations/work_cwd.gleam => harness/extensions/work/migrations/cwd.gleam} (100%) create mode 100644 test/harness/migration_test.gleam diff --git a/agents.md b/agents.md index 973abee2..fb0af4f8 100644 --- a/agents.md +++ b/agents.md @@ -45,7 +45,7 @@ extension.Extension( ) ``` -An extension bundles plugins: context (system instructions), tool (model tools + Python modules + host routes), command (one definition drives the CLI menu, user invocation, and the kernel's typed `commands` bindings), managed (stateful contributions that observe session events), models/model-provider (catalog, wire protocol, login), service (HTTP under `//`), compaction, and fold. Selection is per session through `/extensions`; at most one compaction strategy may be enabled, and changes require an idle session. +An extension bundles plugins: context (system instructions), tool (model tools + Python modules + host routes), command (one definition drives the CLI menu, user invocation, and the kernel's typed `commands` bindings), managed (stateful contributions that observe session events), models/model-provider (catalog, wire protocol, login), service (HTTP under `//`), migration (extension-owned SQLite upgrades applied by the host), compaction, and fold. Selection is per session through `/extensions`; at most one compaction strategy may be enabled, and changes require an idle session. ### `cli/` — the terminal (Go) diff --git a/robot-docs/extensions.md b/robot-docs/extensions.md index 1c3ad9f1..1d597d68 100644 --- a/robot-docs/extensions.md +++ b/robot-docs/extensions.md @@ -1,6 +1,6 @@ # extensions -an extension is a named bundle of plugins. it can contribute any number of context, tool, command, managed, models, provider, login, service, compaction, or fold plugins. `python`, `run`, `work`, `mail`, `agents`, `schedule`, `paperclips`, `files`, `memory`, `instructions`, `commands`, `skills`, `models`, `openai`, `codex`, `antigravity`, `alibaba`, `claude`, `rolling`, `snapcompact-memory`, `lcm-memory`, and `remote` are enabled by default; `mcp`, `view`, [`proxy`](proxy.md), `webhooks`, [`warm`](cache-warming.md), and the `snapcompact` and `lcm` compaction strategies are installed and disabled until enabled. plugin contributions compose inside them. +an extension is a named bundle of plugins. it can contribute any number of context, tool, command, managed, models, provider, login, service, migration, compaction, or fold plugins. `python`, `run`, `work`, `mail`, `agents`, `schedule`, `paperclips`, `files`, `memory`, `instructions`, `commands`, `skills`, `models`, `openai`, `codex`, `antigravity`, `alibaba`, `claude`, `rolling`, `snapcompact-memory`, `lcm-memory`, and `remote` are enabled by default; `mcp`, `view`, [`proxy`](proxy.md), `webhooks`, [`warm`](cache-warming.md), and the `snapcompact` and `lcm` compaction strategies are installed and disabled until enabled. plugin contributions compose inside them. ## select extensions @@ -46,6 +46,30 @@ a managed contribution also hears its session through `observe(session, event)`, `CompactionPlugin` supplies a history strategy; `FoldPlugin` supplies stored folds any strategy reads through `context.prior`; see [compaction](compaction.md). one enabled compaction strategy owns the request-history view in a built-in session. a strategy receives chronological history and must preserve tool call/result associations; it must not replace the durable transcript. +## sqlite migrations + +extensions own their tables and their SQLite migration implementations. contribute +`MigrationPlugin(SchemaMigration(apply))` for column/index upgrades and +`MigrationPlugin(DataMigration(name, run))` for data rewrites; keep the callbacks +under the owning extension, not `daemon/migrations/`. + +`extension.install` calls each installed owner's `initialise(ledger)` to create +its tables, then applies that owner's schema callbacks on the serialized store +connection before proceeding to the next owner. `runtime.migrate(host, backup)` +collects and applies data callbacks in registry/plugin order after the daemon's +core data upgrades and before sessions start. `run(ledger, backup)` returns a +rewritten-row count. installed-but-disabled extensions still upgrade their +storage; session selection/reload does not apply migrations. failures stop the +phase, and data callbacks own their idempotency markers, paging, and transactions. +embedding hosts must prepare core storage and invoke `runtime.migrate` before +opening sessions. see [migrations](migrations.md) for startup order and backups. + +```gleam +plugins: [ + extension.MigrationPlugin(extension.SchemaMigration(title.apply)), +], +``` + ## extension settings optional `$ALBEDO_HOME/extensions.json` (default `~/.albedo/extensions.json`) holds named extension settings separately from provider credentials and session selection. settings are read when preparing a worker; changing a file does not mutate an existing session composition. missing sections use extension defaults. invalid settings fail preparation without replacing the live worker. diff --git a/robot-docs/migrations.md b/robot-docs/migrations.md index 93701410..4e27c36b 100644 --- a/robot-docs/migrations.md +++ b/robot-docs/migrations.md @@ -1,32 +1,46 @@ # existing storage migrations -this is an organization of the migrations already shipped, not a new migration -framework or schema version. the implementations live under -`src/albedo/daemon/migrations/`; `src/albedo/daemon/migrations.gleam` is the -explicit ordered entrypoint for startup data upgrades. +this organizes migrations already shipped; it does not introduce a schema +version. core implementations live under `src/albedo/daemon/migrations/`; +extension implementations live under their own `migrations/` directories and +contribute `MigrationPlugin` values. the host applies them, without importing +extension-specific migration modules. ## startup order -`server.prepare_storage` still runs before sessions start: +`runtime.start` installs the extension registry before `server.prepare_storage`. +for each installed extension, `extension.install` first calls its table +initialiser, then applies its contributed `SchemaMigration` callbacks on the +store-owned SQLite connection, in plugin order. work contributes `cwd.apply`; +paperclips contributes `title.apply`. their ledgers create tables but do not run +upgrades themselves. all installed owners upgrade, even when disabled for +sessions; session selection and live reload never rerun migrations. + +`server.prepare_storage` then runs before sessions start: 1. `conversation.initialise` creates the core tables, then calls `conversation_columns.apply`: the existing session columns first, transcript columns second. it then runs its existing domain-local session recovery (missing activity/title from transcript), then creates `sessions_activity`. 2. quota, mail, and family initialise their tables, unchanged. -3. `migrations.run(ledger, backup)` runs `image_store.run`, then - `cell_images.run`, returning `#(transcript_rows_moved, cell_results_moved)` - for the existing server log messages. `image_store.run` first externalizes - legacy transcript images, then converts image-table TEXT data to decoded BLOB. +3. `migrations.run(ledger, backup)` runs the core `image_store.run`, which first + externalizes legacy transcript images, then converts image-table TEXT data + to decoded BLOB. `runtime.migrate(host, backup)` then collects installed + extensions' `DataMigration` callbacks in registry/plugin order and applies + them with the same backup path. python contributes `cell_images.run`, which + requires the core image and marker tables. callbacks report named row counts; + the server logs nonzero extension results as `migration : rows`. + a failed callback stops startup and prevents later migrations from running. 4. the existing best-effort filesystem credential migration runs; then provider assignment from legacy configuration runs, unchanged. -schema creation is still owned by each subsystem. work and paperclips upgrade -only when their extension initialisers run, not unconditionally at core startup: -`work_cwd.apply` checks `PRAGMA table_info(work)`, adds the existing reserved -`__albedo_legacy__` cwd default when absent, then creates `work_cwd_id`; -`paperclips_title.apply` adds the existing non-null empty-default title column. -these calls remain after each owner's table creation. +schema creation is still owned by each subsystem. `work/migrations/cwd.apply` +checks `PRAGMA table_info(work)`, adds the existing reserved `__albedo_legacy__` +cwd default when absent, then creates `work_cwd_id`; +`paperclips/migrations/title.apply` adds the existing non-null empty-default title +column. these upgrades run immediately after their owner's table creation, so +later extension initialisers can use the upgraded schema. embedding hosts call +`runtime.migrate` after preparing core storage and before opening sessions. ## compatibility and interruption @@ -43,8 +57,8 @@ these calls remain after each owner's table creation. transaction helper. there is no new all-migrations transaction or transaction policy change. markers are written at the original completion points, and interrupted runs recheck remaining rows. -- `backup.gleam` preserves the two existing backup call patterns: transcript - and BLOB migration check for an existing file before directory creation and +- core `backup.gleam` and python's cell migration preserve the two existing + backup call patterns: transcript and BLOB migration check for an existing file before directory creation and `VACUUM INTO`, retaining the `image store backup failed: ` error prefix; cell migration ensures the directory first and copies on the first page containing inline images, outside its transaction. the server still supplies one shared @@ -64,14 +78,15 @@ these calls remain after each owner's table creation. | `images.migrate` / `migrate_legacy` / transcript pages | `migrations/image_store.run` / `migrate_legacy` / transcript pages | | `images.migrate_blobs` / BLOB pages | `migrations/image_store.migrate_blobs` / BLOB pages | | `images.backup_before_migration` | `migrations/backup.image_store` | -| `python/cells.migrate_images` / cell pages | `migrations/cell_images.run` / cell pages | -| `python/cells.backup_before_migration` | `migrations/backup.cell_images` | -| `work/ledger.initialise`: cwd alteration and index | `migrations/work_cwd.apply` | -| `paperclips/ledger.initialise`: title addition | `migrations/paperclips_title.apply` | +| `python/cells.migrate_images` / cell pages | `python/migrations/cell_images.run` / cell pages | +| `python/cells.backup_before_migration` | `python/migrations/cell_images.backup_before_migration` | +| `work/ledger.initialise`: cwd alteration and index | `work/migrations/cwd.apply`, contributed by work | +| `paperclips/ledger.initialise`: title addition | `paperclips/migrations/title.apply`, contributed by paperclips | `images.migrate` and `cells.migrate_images` are removed rather than wrapped: -wrappers would introduce import cycles between image/cell domain modules and -migration modules. direct callers use the migration modules instead. +the migration implementations are separate from the live domain readers and +writers. startup discovers extension upgrades through plugins, not direct calls +to their implementation modules. ## intentionally not extracted diff --git a/src/albedo/daemon/migrations.gleam b/src/albedo/daemon/migrations.gleam index 279e6444..50fcdd15 100644 --- a/src/albedo/daemon/migrations.gleam +++ b/src/albedo/daemon/migrations.gleam @@ -1,15 +1,10 @@ -//// Ordered data upgrades after core schema initialisation, before sessions start. -//// Schema column upgrades remain in their owning initialisers. +//// Ordered core data upgrades after schema initialisation, before sessions start. -import albedo/daemon/migrations/cell_images import albedo/daemon/migrations/image_store import albedo/daemon/store -import gleam/result -/// Transcript references, image TEXT to BLOB, then cell references. Both image -/// steps share the startup backup path. Returns rewritten transcript/cell counts. -pub fn run(ledger: store.Store, backup: String) -> Result(#(Int, Int), String) { - use transcript_moved <- result.try(image_store.run(ledger, backup)) - use cells_moved <- result.try(cell_images.run(ledger, backup)) - Ok(#(transcript_moved, cells_moved)) +/// Transcript references and image TEXT to BLOB. Extension upgrades follow via +/// the runtime; this module knows only core storage. +pub fn run(ledger: store.Store, backup: String) -> Result(Int, String) { + image_store.run(ledger, backup) } diff --git a/src/albedo/daemon/migrations/backup.gleam b/src/albedo/daemon/migrations/backup.gleam index af925ad4..3b107e86 100644 --- a/src/albedo/daemon/migrations/backup.gleam +++ b/src/albedo/daemon/migrations/backup.gleam @@ -17,17 +17,6 @@ pub fn image_store(ledger: store.Store, backup: String) -> Result(Nil, String) { } } -pub fn cell_images( - db: sqlight.Connection, - path: String, -) -> Result(Nil, String) { - ensure_dir(path) - case backup_exists(path) { - True -> Ok(Nil) - False -> store.run(db, "VACUUM INTO ?", [sqlight.text(path)]) - } -} - @external(erlang, "albedo_images", "ensure_dir") fn ensure_dir(path: String) -> Nil diff --git a/src/albedo/daemon/server.gleam b/src/albedo/daemon/server.gleam index 078e94c1..d5a86392 100644 --- a/src/albedo/daemon/server.gleam +++ b/src/albedo/daemon/server.gleam @@ -561,19 +561,17 @@ fn prepare_storage( <> "/backups/albedo-before-image-store-" <> int.to_string(usage.now()) <> ".sqlite" - use #(moved, cells_moved) <- result.try(migrations.run( - runtime.ledger(host), - backup, - )) - case cells_moved { - 0 -> Nil - rows -> - io.println( - "image store: moved images out of " - <> int.to_string(rows) - <> " cell results", - ) - } + use moved <- result.try(migrations.run(runtime.ledger(host), backup)) + use upgraded <- result.try(runtime.migrate(host, backup)) + list.each(upgraded, fn(migration) { + case migration.1 { + 0 -> Nil + rows -> + io.println( + "migration " <> migration.0 <> ": " <> int.to_string(rows) <> " rows", + ) + } + }) case moved { 0 -> Nil rows -> diff --git a/src/albedo/harness/extension.gleam b/src/albedo/harness/extension.gleam index 65a8a139..a07db3b2 100644 --- a/src/albedo/harness/extension.gleam +++ b/src/albedo/harness/extension.gleam @@ -233,6 +233,19 @@ pub type Plugin { /// HTTP routes the daemon serves under `//` while the extension /// is enabled globally. ServicePlugin(service: Service) + /// SQLite upgrades owned by this extension, applied by the host at startup. + MigrationPlugin(migration: Migration) +} + +pub type Migration { + /// Applied immediately after the owning extension creates its tables. + SchemaMigration(apply: fn(sqlight.Connection) -> Result(Nil, String)) + /// Applied after core data upgrades, with the same pre-upgrade backup path. + /// Returns the number of rows rewritten for startup reporting. + DataMigration( + name: String, + run: fn(store.Store, String) -> Result(Int, String), + ) } /// Handles a request below the extension's mount point. The daemon token does @@ -330,7 +343,34 @@ pub fn install( ) }), ) - list.try_each(installed, fn(extension) { extension.initialise(ledger) }) + list.try_each(installed, fn(extension) { + use _ <- result.try(extension.initialise(ledger)) + list.try_each(extension.plugins, fn(plugin) { + case plugin { + MigrationPlugin(SchemaMigration(apply)) -> store.query(ledger, apply) + _ -> Ok(Nil) + } + }) + }) +} + +/// Installed extensions' data upgrades, in registry/plugin order, not session +/// selection order. Each migration owns its markers and transaction boundaries. +pub fn migrate( + installed: List(Extension), + ledger: store.Store, + backup: String, +) -> Result(List(#(String, Int)), String) { + plugin_values(installed, fn(_, plugin) { + case plugin { + MigrationPlugin(DataMigration(name, run)) -> Ok(#(name, run)) + _ -> Error(Nil) + } + }) + |> list.try_map(fn(migration) { + migration.1(ledger, backup) + |> result.map(fn(count) { #(migration.0, count) }) + }) } fn validate_registry( @@ -988,6 +1028,7 @@ pub fn summaries( ModelProviderPlugin(_) -> "model_provider" LoginPlugin(_) -> "login" ServicePlugin(_) -> "service" + MigrationPlugin(_) -> "migration" } }), ) diff --git a/src/albedo/harness/extensions/paperclips/extension.gleam b/src/albedo/harness/extensions/paperclips/extension.gleam index 787ae5f8..b02b39ca 100644 --- a/src/albedo/harness/extensions/paperclips/extension.gleam +++ b/src/albedo/harness/extensions/paperclips/extension.gleam @@ -1,6 +1,7 @@ import albedo/harness/extension as harness_extension import albedo/harness/extensions/paperclips/command as paperclips_command import albedo/harness/extensions/paperclips/ledger as paperclips +import albedo/harness/extensions/paperclips/migrations/title import albedo/harness/extensions/paperclips/rpc pub fn extension() -> harness_extension.Extension { @@ -9,6 +10,9 @@ pub fn extension() -> harness_extension.Extension { "A vent channel: the model records friction, the user reviews it in /paperclips.", ["python"], [ + harness_extension.MigrationPlugin(harness_extension.SchemaMigration( + title.apply, + )), harness_extension.ToolPlugin(vent_instructions, [], ["paperclips"], []), // The prepared workspace scopes both the Python route and /paperclips. harness_extension.ManagedPlugin(fn(store, _, workspace) { diff --git a/src/albedo/harness/extensions/paperclips/ledger.gleam b/src/albedo/harness/extensions/paperclips/ledger.gleam index 5937726c..0902df40 100644 --- a/src/albedo/harness/extensions/paperclips/ledger.gleam +++ b/src/albedo/harness/extensions/paperclips/ledger.gleam @@ -2,7 +2,6 @@ //// the user to review. The model writes and lists; the user triages through //// /paperclips. Rows live in the shared ledger store under one table. -import albedo/daemon/migrations/paperclips_title import albedo/daemon/store as storage import gleam/dynamic/decode import gleam/json @@ -49,13 +48,9 @@ pub type Error { pub type Store = storage.Store -/// Runs once at install against the shared ledger store. `add_columns` -/// carries tables installed before a column existed. +/// Creates the table at install; the extension contributes its column upgrades. pub fn initialise(ledger: Store) -> Result(Nil, String) { - storage.query(ledger, fn(db) { - use _ <- result.try(storage.exec(db, schema)) - paperclips_title.apply(db) - }) + storage.query(ledger, storage.exec(_, schema)) } const schema = " diff --git a/src/albedo/daemon/migrations/paperclips_title.gleam b/src/albedo/harness/extensions/paperclips/migrations/title.gleam similarity index 100% rename from src/albedo/daemon/migrations/paperclips_title.gleam rename to src/albedo/harness/extensions/paperclips/migrations/title.gleam diff --git a/src/albedo/harness/extensions/python/extension.gleam b/src/albedo/harness/extensions/python/extension.gleam index d15c906e..31895970 100644 --- a/src/albedo/harness/extensions/python/extension.gleam +++ b/src/albedo/harness/extensions/python/extension.gleam @@ -3,6 +3,7 @@ import albedo/harness/extension import albedo/harness/extensions/python/cells as journal import albedo/harness/extensions/python/kernel as python +import albedo/harness/extensions/python/migrations/cell_images import albedo/harness/extensions/python/rpc as cells import albedo/openai_api/types import gleam/dynamic/decode @@ -85,6 +86,10 @@ pub fn extension() -> extension.Extension { "A persistent Python namespace with saved cell recovery.", [], [ + extension.MigrationPlugin(extension.DataMigration( + "cell_images", + cell_images.run, + )), extension.ToolPlugin( "Python has a persistent namespace, top-level await, cells.read/info/trace and cells.run for saved-source repair (all async), and output.read/output.list for bounded retained output (synchronous; awaiting them also works). cells.last_id is the id of the latest cell, and await cells.list(limit=20) lists this session's cells newest first with their status and first line, even after their output has rolled out. output.read(id, offset=0, limit=4000) returns up to limit characters. output.list() names every retained channel, which is also how to find earlier cells: cells, background jobs, and 'native' for bytes written to fd 1/2 while no cell was running. show_image(source) returns a PNG, JPEG, or WebP (bytes or a file path) to you with this cell's result, so you see it after the cell ends; at most 4 images and 5 MiB per cell.", [extension.Tool(definition(), invoke, recover)], diff --git a/src/albedo/daemon/migrations/cell_images.gleam b/src/albedo/harness/extensions/python/migrations/cell_images.gleam similarity index 90% rename from src/albedo/daemon/migrations/cell_images.gleam rename to src/albedo/harness/extensions/python/migrations/cell_images.gleam index 3b45bb6c..5e068a19 100644 --- a/src/albedo/daemon/migrations/cell_images.gleam +++ b/src/albedo/harness/extensions/python/migrations/cell_images.gleam @@ -1,7 +1,6 @@ //// Existing migration of inline cell screenshots into shared image payloads. import albedo/daemon/images -import albedo/daemon/migrations/backup as snapshot import albedo/daemon/store import albedo/harness/extensions/python/kernel as python import gleam/dynamic/decode @@ -74,7 +73,7 @@ fn migrate_image_pages( case moved == 0 && list.any(rows, fn(row) { has_inline_image(row.1) }) { - True -> snapshot.cell_images(db, backup) + True -> backup_before_migration(db, backup) False -> Ok(Nil) }, ) @@ -130,6 +129,23 @@ fn migrate_image_pages( } } +fn backup_before_migration( + db: sqlight.Connection, + path: String, +) -> Result(Nil, String) { + ensure_dir(path) + case backup_exists(path) { + True -> Ok(Nil) + False -> store.run(db, "VACUUM INTO ?", [sqlight.text(path)]) + } +} + +@external(erlang, "albedo_images", "ensure_dir") +fn ensure_dir(path: String) -> Nil + +@external(erlang, "albedo_images", "backup_exists") +fn backup_exists(path: String) -> Bool + @external(erlang, "albedo_native", "pack_cell") fn pack(outcome: Result(python.Outcome, python.Error)) -> BitArray diff --git a/src/albedo/harness/extensions/work/extension.gleam b/src/albedo/harness/extensions/work/extension.gleam index 77140f2f..cdb7f701 100644 --- a/src/albedo/harness/extensions/work/extension.gleam +++ b/src/albedo/harness/extensions/work/extension.gleam @@ -1,6 +1,7 @@ import albedo/harness/extension as harness_extension import albedo/harness/extensions/work/command as work_command import albedo/harness/extensions/work/ledger as work +import albedo/harness/extensions/work/migrations/cwd import albedo/harness/extensions/work/rpc pub fn extension() -> harness_extension.Extension { @@ -9,6 +10,9 @@ pub fn extension() -> harness_extension.Extension { "A durable revision-checked work ledger shared by humans and agents.", ["python"], [ + harness_extension.MigrationPlugin(harness_extension.SchemaMigration( + cwd.apply, + )), harness_extension.ToolPlugin( "Humans and agents share this revision-checked work ledger; every call is async. await work.list(after=0, limit=50), work.get(id), work.create(title, notes=\"\", parent=None), work.update(id, revision=, title=, notes=, status=), and work.delete(id, revision=) return item records: item.id and item[\"id\"] both work, and so do revision, status, title, notes, and parent. Status is open, active, blocked, done, or cancelled. Pass the revision you last read; a stale one raises WorkError, so get the item again. Keep execution status separate from work status. People manage the same ledger with /work, and their changes reach you as notes.", [], diff --git a/src/albedo/harness/extensions/work/ledger.gleam b/src/albedo/harness/extensions/work/ledger.gleam index fc246984..b7267cb2 100644 --- a/src/albedo/harness/extensions/work/ledger.gleam +++ b/src/albedo/harness/extensions/work/ledger.gleam @@ -1,4 +1,3 @@ -import albedo/daemon/migrations/work_cwd import albedo/daemon/store as storage import gleam/dynamic/decode import gleam/erlang/process @@ -311,8 +310,5 @@ pub fn to_json(item: Item) -> json.Json { } pub fn initialise(store: Store) -> Result(Nil, String) { - storage.query(store, fn(db) { - use _ <- result.try(storage.exec(db, schema)) - work_cwd.apply(db) - }) + storage.query(store, storage.exec(_, schema)) } diff --git a/src/albedo/daemon/migrations/work_cwd.gleam b/src/albedo/harness/extensions/work/migrations/cwd.gleam similarity index 100% rename from src/albedo/daemon/migrations/work_cwd.gleam rename to src/albedo/harness/extensions/work/migrations/cwd.gleam diff --git a/src/albedo/harness/runtime.gleam b/src/albedo/harness/runtime.gleam index dc460d6c..c3c23051 100644 --- a/src/albedo/harness/runtime.gleam +++ b/src/albedo/harness/runtime.gleam @@ -200,6 +200,15 @@ pub fn ledger(runtime: Runtime) -> work.Store { runtime.work } +/// Apply installed extensions' data upgrades after core storage is ready, before +/// opening sessions. Embedding hosts supply their own pre-upgrade backup path. +pub fn migrate( + runtime: Runtime, + backup: String, +) -> Result(List(#(String, Int)), String) { + extension.migrate(runtime.extensions, runtime.work, backup) +} + fn checked_id(id: String) -> Result(String, python.Error) { case string.trim(id) == "" || string.byte_size(id) > 256 { True -> diff --git a/test/e2e/migrations_test.py b/test/e2e/migrations_test.py index 177d6645..877dc009 100644 --- a/test/e2e/migrations_test.py +++ b/test/e2e/migrations_test.py @@ -1,4 +1,4 @@ -"""Existing image migrations survive real startup, paging, backups, and restarts. +"""Owner-contributed schema and image migrations survive real startup and restarts. Legacy ETF fixtures cannot be written by the current daemon. They are seeded while the shared daemon is stopped, then exercised through its normal readers. @@ -41,6 +41,93 @@ def legacy_cell(cell_id): @exclusive class MigrationsTest(unittest.TestCase): + def test_installed_extensions_upgrade_legacy_tables_before_their_routes_run(self): + provider = Provider( + lambda request: python( + "items = await work.list()\n" + "assert not items\n" + "item = await work.create('new scoped item')\n" + "assert (await work.get(item.id)).title == 'new scoped item'\n" + "legacy = [v for v in await vents() if v['message'] == 'legacy vent']\n" + "assert len(legacy) == 1 and legacy[0]['title'] == ''\n" + "await vent('bug', 'new vent', title='new title')\n" + "assert any(v['title'] == 'new title' for v in await vents())\n" + "print('upgraded extension routes usable')" + ) + if request["messages"][-1].get("role") == "user" + else text("done") + ) + self.addCleanup(provider.close) + with Albedo(provider) as app: + session = app.session() + database = app.home / "albedo.sqlite" + previous_cwds, previous_titles = [], [] + legacy_work = None + + def prepare(_app): + nonlocal legacy_work + with sqlite3.connect(database) as db: + previous_cwds.extend(db.execute("SELECT cwd,id FROM work")) + previous_titles.extend( + db.execute("SELECT title,id FROM paperclips") + ) + db.execute("DROP INDEX work_cwd_id") + db.execute("ALTER TABLE work DROP COLUMN cwd") + db.execute("ALTER TABLE paperclips DROP COLUMN title") + legacy_work = db.execute( + "INSERT INTO work(title) VALUES('legacy isolated item')" + ).lastrowid + db.execute( + "INSERT INTO paperclips(topic,message,cwd) VALUES('bug','legacy vent',?)", + (str(app.workspace),), + ) + settings_path = app.home / "extensions.json" + settings = json.loads(settings_path.read_text()) + settings.setdefault("enabled", {}).update(work=False, paperclips=False) + settings_path.write_text(json.dumps(settings)) + + def restore(_app): + with sqlite3.connect(database) as db: + db.executemany("UPDATE work SET cwd=? WHERE id=?", previous_cwds) + db.executemany( + "UPDATE paperclips SET title=? WHERE id=?", previous_titles + ) + + try: + app.restart(prepare=prepare) + # Installed storage still upgrades when its routes are disabled. + with sqlite3.connect(database) as db: + self.assertEqual( + db.execute( + "SELECT cwd FROM work WHERE id=?", (legacy_work,) + ).fetchone(), + ("__albedo_legacy__",), + ) + self.assertIsNotNone( + db.execute( + "SELECT name FROM sqlite_master WHERE name='work_cwd_id' AND type='index'" + ).fetchone() + ) + for name in ("work", "paperclips"): + app.api( + f"/sessions/{session}/extensions", + {"name": name, "enabled": True}, + ).close() + app.restart() + app.prompt(session, "use the upgraded extension ledgers").close() + app.idle(session) + results = [ + json.loads(event["result"]) + for event in app.events(session) + if event.get("type") == "tool" and event.get("name") == "python" + ] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["status"], "ok", results[0]) + self.assertIn("upgraded extension routes usable", results[0]["output"]) + finally: + # Restore other fixtures' fields erased by the legacy schema seed. + app.restart(prepare=restore) + def test_startup_migrates_legacy_images_once_with_a_consistent_backup(self): probe = "" diff --git a/test/harness/migration_test.gleam b/test/harness/migration_test.gleam new file mode 100644 index 00000000..b5490127 --- /dev/null +++ b/test/harness/migration_test.gleam @@ -0,0 +1,97 @@ +//// The built-in daemon cannot install arbitrary test extensions. Probe the +//// cross-extension contract here: schema-before-use, deferred data, order, +//// backup propagation, disabled-but-installed owners, and failure short-circuit. + +import albedo/daemon/store +import albedo/harness/extension +import gleam/dynamic/decode +import gleam/result +import gleeunit/should +import sqlight + +pub fn contributed_migrations_keep_their_phases_and_stop_on_failure_test() { + let assert Ok(ledger) = store.start(":memory:", "") + let installed = [ + extension.Extension( + "owner", + "", + [], + [ + extension.MigrationPlugin( + extension.SchemaMigration(fn(db) { + store.add_columns(db, "owner", [#("revision", "INTEGER DEFAULT 1")]) + }), + ), + extension.MigrationPlugin( + extension.DataMigration("owner-data", fn(ledger, backup) { + use _ <- result.try(store.read( + ledger, + "SELECT 1 FROM core_ready", + [], + decode.dynamic, + )) + store.write( + ledger, + "INSERT INTO applied(name,backup) VALUES('owner',?)", + [sqlight.text(backup)], + ) + |> result.replace(1) + }), + ), + ], + fn(ledger) { + store.query(ledger, store.exec( + _, + "CREATE TABLE owner(id INTEGER); CREATE TABLE applied(name TEXT,backup TEXT)", + )) + }, + ), + extension.Extension( + "sibling", + "", + [], + [ + extension.MigrationPlugin( + extension.DataMigration("sibling-data", fn(ledger, backup) { + store.write( + ledger, + "INSERT INTO applied(name,backup) VALUES('sibling',?)", + [sqlight.text(backup)], + ) + |> result.replace(1) + }), + ), + ], + fn(ledger) { + // This initialiser depends on the previous owner's schema upgrade. + store.write(ledger, "INSERT INTO owner(revision) VALUES(2)", []) + }, + ), + ] + // Neither extension is enabled for sessions, but both own durable tables. + extension.install(installed, [], ledger) |> should.equal(Ok(Nil)) + let assert Ok([2]) = + store.read( + ledger, + "SELECT revision FROM owner", + [], + decode.field(0, decode.int, decode.success), + ) + let assert Ok([]) = + store.read(ledger, "SELECT * FROM applied", [], decode.dynamic) + let assert Error(_) = extension.migrate(installed, ledger, "/backup") + let assert Ok([]) = + store.read(ledger, "SELECT * FROM applied", [], decode.dynamic) + let assert Ok(_) = + store.query(ledger, store.exec(_, "CREATE TABLE core_ready(id INTEGER)")) + extension.migrate(installed, ledger, "/backup") + |> should.equal(Ok([#("owner-data", 1), #("sibling-data", 1)])) + let assert Ok(rows) = + store.read(ledger, "SELECT name,backup FROM applied ORDER BY rowid", [], { + use name <- decode.field(0, decode.string) + use backup <- decode.field(1, decode.string) + decode.success(#(name, backup)) + }) + rows |> should.equal([#("owner", "/backup"), #("sibling", "/backup")]) + store.close(ledger) +} diff --git a/test/harness/tool_image_test.gleam b/test/harness/tool_image_test.gleam index 54328540..56f45fa9 100644 --- a/test/harness/tool_image_test.gleam +++ b/test/harness/tool_image_test.gleam @@ -2,10 +2,10 @@ //// changes; synthetic corrupted and legacy records cannot arise in E2E. import albedo/daemon/images -import albedo/daemon/migrations/cell_images import albedo/daemon/store import albedo/harness/extensions/python/cells import albedo/harness/extensions/python/kernel as python +import albedo/harness/extensions/python/migrations/cell_images import albedo/openai_api/types import gleam/dynamic import gleam/dynamic/decode diff --git a/test/harness/work_test.gleam b/test/harness/work_test.gleam index 7dd6e57e..26fc7930 100644 --- a/test/harness/work_test.gleam +++ b/test/harness/work_test.gleam @@ -1,11 +1,8 @@ //// Concurrent revision updates must have exactly one winner. import albedo/harness/extensions/work/ledger as work -import gleam/dynamic/decode import gleam/erlang/process -import gleam/int import gleam/option.{None, Some} -import sqlight pub fn only_one_concurrent_edit_wins_test() { let assert Ok(store) = work.start(":memory:") @@ -51,53 +48,3 @@ pub fn ledger_items_are_scoped_to_cwd_test() { let assert Error(work.Invalid(_)) = work.list(store, "", 0, 10) work.close(store) } - -@external(erlang, "erlang", "unique_integer") -fn unique_integer() -> Int - -@external(erlang, "albedo_session", "discard") -fn discard(path: String) -> Nil - -pub fn old_schema_migrates_before_cwd_index_test() { - let path = - "/tmp/albedo-work-legacy-" <> int.to_string(unique_integer()) <> ".db" - let assert Ok(db) = sqlight.open(path) - let assert Ok(_) = - sqlight.exec( - "CREATE TABLE work (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, notes TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'open', parent INTEGER, session TEXT, run TEXT, revision INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL DEFAULT '')", - db, - ) - let assert Ok(_) = - sqlight.exec("INSERT INTO work(title) VALUES('legacy')", db) - let assert Ok(_) = sqlight.close(db) - let assert Ok(store) = work.start(path) - let assert Ok(_) = work.initialise(store) - work.close(store) - let assert Ok(check) = sqlight.open(path) - let assert Ok([title]) = - sqlight.query( - "SELECT title FROM work", - check, - [], - decode.field(0, decode.string, decode.success), - ) - let assert True = title == "legacy" - let assert Ok([cwd]) = - sqlight.query( - "SELECT cwd FROM work", - check, - [], - decode.field(0, decode.string, decode.success), - ) - let assert True = cwd == "__albedo_legacy__" - let assert Ok([index]) = - sqlight.query( - "SELECT name FROM sqlite_master WHERE type='index' AND name='work_cwd_id'", - check, - [], - decode.field(0, decode.string, decode.success), - ) - let assert True = index == "work_cwd_id" - let assert Ok(_) = sqlight.close(check) - discard(path) -} -- knotmirror 0.0.1