Something went wrong. Try again.
Mirror of https://github.com/kualta/site — web infrastructure
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260import { expect, test } from "bun:test";import { Database } from "bun:sqlite";import { createHash } from "node:crypto";import { readdirSync, readFileSync } from "node:fs";import { mkdir, mkdtemp, rm, stat, utimes, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";import { archiveResponse } from "../../src/lib/publisher/archive.ts";import { memoryArchive, memoryMedia } from "../../src/lib/publisher/memory-archive.ts";import { archiveClient } from "../publisher/archive.mjs";import { originalsName, startOffload } from "./offload.mjs";
const sha = (text) => createHash("sha256").update(text).digest("hex");
/** The site's archive routes, answered in memory, as the grade app's fetch reaches them; `hold` can stall a request. */function site({ hold = () => null } = {}) { const sqlite = new Database(":memory:"); sqlite.exec(readFileSync("migrations/0002_publisher.sql", "utf8")); const db = { prepare: (sql) => { let values = []; const statement = { bind: (...v) => { values = v; return statement; }, first: async () => sqlite.query(sql).get(...values), all: async () => ({ results: sqlite.query(sql).all(...values) }), }; return statement; }, }; const archive = memoryArchive(); const media = memoryMedia(); const fetcher = async (url, init = {}) => { const rest = new URL(url).pathname.replace(/^\/api\/publisher\/helper\/archive\//, ""); const held = hold(rest, init); if (held) return held; const headers = new Headers(init.headers); if (init.body !== undefined) headers.set("Content-Length", String(Buffer.byteLength(init.body))); const request = new Request(url, { method: init.method, headers, body: init.body }); return archiveResponse(rest, request, { db, media: media.bucket, archive: archive.bucket }); }; return archiveClient({ origin: "https://post.kualta.dev", token: "token", fetcher });}
/** A card (DCIM/DJI_001, each clip with its .LRF) and a Movies/Original, with these clips on them. */async function place(card, mac = []) { const root = await mkdtemp(join(tmpdir(), "offload-")); const dcim = join(root, "DCIM/DJI_001"); const original = join(root, "Original"); await mkdir(dcim, { recursive: true }); await mkdir(original); const make = async (dir, [name, body, day], lrf) => { const path = join(dir, name); await writeFile(path, body); const when = new Date("2026-11-18T12:00:00Z"); await utimes(path, when, when); if (lrf) await writeFile(path.replace(/\.MP4$/, ".LRF"), "low-res"); return { name, path, size: Buffer.byteLength(body), mtime: when.getTime(), day }; }; const camera = []; for (const clip of card) camera.push(await make(dcim, clip, true)); const onMac = []; for (const clip of mac) onMac.push(await make(original, clip, false)); return { root, dcim, original, camera, mac: onMac, done: () => rm(root, { recursive: true, force: true }) };}
const files = (dir) => readdirSync(dir).sort();const plenty = async () => 1e12;
test("the card goes to the Mac and the Mac to the archive, each deleted once the next has it, by day", async () => { const at = await place( [ ["DJI_20261118203054_0316_D.MP4", "a clip long enough to go in parts", "2026-11-18"], ["DJI_20261118203937_0317_D.MP4", "short", "2026-11-18"], ["DJI_20261119090000_0318_D.MP4", "the next day", "2026-11-19"], ], [["DJI_20261117090000_0300_D.MP4", "already on the Mac", "2026-11-17"]], ); try { const request = site(); const landed = []; const removed = []; const run = startOffload({ camera: at.camera, mac: at.mac, dir: at.original, request, room: plenty, keepFree: 0, part: 8, landed: (clip, sha256) => landed.push([clip.name, sha256]), removed: (clip, where) => removed.push([where, clip.name]), }); await run.done; expect(run.state).toMatchObject({ running: false, finished: "done", error: null }); expect(run.state.copy).toMatchObject({ total: 3, done: 3, ended: true, error: null }); expect(run.state.send).toMatchObject({ total: 4, done: 4, current: [] }); expect(run.state.freed).toEqual({ camera: 50, mac: 68 }); expect(Object.fromEntries(landed)).toEqual({ "DJI_20261118203054_0316_D.MP4": sha("a clip long enough to go in parts"), "DJI_20261118203937_0317_D.MP4": sha("short"), "DJI_20261119090000_0318_D.MP4": sha("the next day"), "DJI_20261117090000_0300_D.MP4": sha("already on the Mac"), }); expect(removed.filter(([where]) => where === "camera").map(([, name]) => name)).toEqual( at.camera.map((c) => c.name), ); expect(removed.filter(([where]) => where === "mac")).toHaveLength(4); // Nothing left on the card, .LRF included, nor on the Mac, half-copied files included. expect(files(at.dcim)).toEqual([]); expect(files(at.original)).toEqual([]);
const day = await request(`projects/${originalsName("2026-11-18")}`); expect(day.files.map((f) => [f.path, f.size, f.mtime])).toEqual([ ["assets/DJI_20261118203054_0316_D.MP4", 33, Date.parse("2026-11-18T12:00:00Z")], ["assets/DJI_20261118203937_0317_D.MP4", 5, Date.parse("2026-11-18T12:00:00Z")], ]); expect((await request(`projects/${originalsName("2026-11-17")}`)).files.map((f) => f.path)).toEqual([ "assets/DJI_20261117090000_0300_D.MP4", ]); const blob = await request(`blobs/${sha("a clip long enough to go in parts")}`, { raw: true }); expect(await blob.text()).toBe("a clip long enough to go in parts"); } finally { await at.done(); }});
test("a clip comes off the card only when the Mac has room for it, made by the clips going up", async () => { const at = await place([ ["DJI_20261118100000_0001_D.MP4", "twenty bytes, first.", "2026-11-18"], ["DJI_20261118100100_0002_D.MP4", "twenty bytes, second", "2026-11-18"], ["DJI_20261118100200_0003_D.MP4", "twenty bytes, third.", "2026-11-18"], ]); try { // Room for one clip at a time: 30 bytes, less what's in Original. const used = async () => { let total = 0; for (const name of files(at.original)) total += (await stat(join(at.original, name))).size; return total; }; const most = []; let waited = false; const run = startOffload({ camera: at.camera, mac: [], dir: at.original, request: site(), room: async () => { waited ||= run?.state.waiting === "room"; return 30 - (await used()); }, keepFree: 0, landed: async () => most.push(files(at.original).length), }); await run.done; expect(run.state).toMatchObject({ finished: "done", error: null }); expect(run.state.send.done).toBe(3); expect(most).toEqual([1, 1, 1]); expect(waited).toBe(true); expect(files(at.dcim)).toEqual([]); expect(files(at.original)).toEqual([]); } finally { await at.done(); }});
test("with no room on the Mac and nothing there to go up, the copy says so and the card keeps its clips", async () => { const at = await place([["DJI_20261118100000_0001_D.MP4", "a clip", "2026-11-18"]]); try { const run = startOffload({ camera: at.camera, mac: [], dir: at.original, request: site(), room: async () => 3e9 }); await run.done; expect(run.state.finished).toBe("done"); expect(run.state.copy.error).toBe("No room on this Mac for the next clip: 3.0 GB free, 5.0 GB kept free"); expect(files(at.dcim)).toEqual(["DJI_20261118100000_0001_D.LRF", "DJI_20261118100000_0001_D.MP4"]); } finally { await at.done(); }});
test("what the archive holds already is only deleted; what it's lost goes again", async () => { const at = await place( [["DJI_20261118100000_0001_D.MP4", "in the archive", "2026-11-18"]], [["DJI_20261117100000_0001_D.MP4", "recorded, but the archive lost it", "2026-11-17"]], ); try { const request = site(); await request(`blobs/${sha("in the archive")}`, { method: "PUT", body: "in the archive" }); const record = { "DJI_20261118100000_0001_D.MP4": { sha256: sha("in the archive"), size: 14 }, "DJI_20261117100000_0001_D.MP4": { sha256: sha("recorded, but the archive lost it"), size: 33 }, }; const landed = []; const run = startOffload({ camera: at.camera, mac: at.mac, record, dir: at.original, request, room: plenty, landed: (clip) => landed.push(clip.name), }); await run.done; expect(run.state.copy.total).toBe(0); expect(landed).toEqual(["DJI_20261117100000_0001_D.MP4"]); expect(files(at.dcim)).toEqual([]); expect(files(at.original)).toEqual([]); } finally { await at.done(); }});
test("stopping cuts the part in flight, keeps what's in, and keeps the clip on the Mac", async () => { const at = await place([], [["DJI_20261117100000_0001_D.MP4", "a clip long enough to go in parts", "2026-11-17"]]); try { let stalled = null; const request = site({ hold: (rest, init) => { if (!/^uploads\/\w+\/2$/.test(rest)) return null; return new Promise((_, reject) => { stalled?.(); init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true }); }); }, }); const run = startOffload({ camera: at.camera, mac: at.mac, dir: at.original, request, room: plenty, part: 8 }); await new Promise((resolve) => { stalled = resolve; }); expect(run.state.send.current).toMatchObject([{ name: "DJI_20261117100000_0001_D.MP4", sent: 8 }]); run.stop(); await run.done; expect(run.state).toMatchObject({ finished: "stopped", error: null, running: false }); expect(run.state.send.done).toBe(0); expect(files(at.original)).toEqual(["DJI_20261117100000_0001_D.MP4"]); } finally { await at.done(); }});
test("a request that fails stops it all, saying why, and nothing is deleted that isn't in", async () => { const at = await place( [["DJI_20261118100000_0001_D.MP4", "off the card", "2026-11-18"]], [["DJI_20261117100000_0001_D.MP4", "on the Mac", "2026-11-17"]], ); try { const request = site({ hold: (rest) => (rest === "missing" ? Response.json({ error: "Archive unavailable" }, { status: 400 }) : null), }); const run = startOffload({ camera: at.camera, mac: at.mac, dir: at.original, request, room: plenty, together: 1 }); await run.done; expect(run.state).toMatchObject({ finished: null, error: "Archive unavailable", running: false }); expect(run.state.send.done).toBe(0); expect(files(at.original).filter((f) => f.endsWith(".MP4")).length).toBeGreaterThanOrEqual(1); } finally { await at.done(); }});