#!/usr/bin/env node // The grade app's local server. Clips come from sources, each a shoot day: the camera's card ("camera:2026-11-08", // read only, nothing is copied off it; ⌘⌫ only moves clips to the card's Trash) and ~/Movies/Original on this Mac // ("mac:2026-11-08"). Exports go to ~/Movies/Exported, which holds nothing else. The app's own files live apart: each // clip's grade in one grades.json and what each export was made with in one exports.json (Application Support), the // metadata, thumbnails and previews it can remake in Caches. Clips are known by file name, so a clip graded off the // card and the same clip copied into Original share one grade and one export. Listens on 127.0.0.1 only. import { execFile } from "node:child_process"; import { createReadStream } from "node:fs"; import { mkdir, readFile, readdir, rename, rm, stat, statfs, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import { homedir } from "node:os"; import { basename, dirname, extname, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; import { composeLut, gradeKey, normalizeGrade, parseCube, PROFILES, resolveGrade, writeCube } from "./color.mjs"; import { startOffload } from "./offload.mjs"; import { archivedFootage, comparable, ledgerCloud, mergeLedger, pictureName, unionLedgers, youtubeUploads, } from "./ledger.mjs"; import { cameraDays, exportClip, exportFrame, makeCalibration, makeProxy, makeThumb, probe, VIDEO } from "./media.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const MOVIES = process.env.GRADE_ROOT || join(homedir(), "Movies"); const ORIGINAL = join(MOVIES, "Original"); const EXPORTED = join(MOVIES, "Exported"); const DATA = process.env.GRADE_DATA || join(homedir(), "Library/Application Support/Grade"); const CACHE = process.env.GRADE_CACHE || join(homedir(), "Library/Caches/Grade"); const LUTS = process.env.GRADE_LUTS || join(DATA, "luts"); const GRADES = join(DATA, "grades.json"); const EXPORTS = join(DATA, "exports.json"); const CAMERA = process.env.GRADE_CAMERA || "/Volumes/Pocket4P/DCIM"; const CAMERA_NAME = process.env.GRADE_CAMERA_NAME || "Pocket 4P"; const PORT = Number(process.env.GRADE_PORT || 4777); const EXPORT_LUT_SIZE = 65; // The crossposter helper's folder in this checkout: its pairing reaches the site's archive, and its footage.json says // what went up to YouTube. const PUBLISHER = process.env.GRADE_PUBLISHER || join(here, "../../.publisher"); /** OpenFilm's projects: the films, whose clips the ledger says. */ const PROJECTS = process.env.GRADE_PROJECTS || join(MOVIES, "OpenFilm/Projects"); const LEDGER = join(DATA, "ledger.json"); /** How often the ledger looks again; the archive's projects are asked every 10 minutes of that. */ const LEDGER_EVERY = Number(process.env.GRADE_LEDGER_EVERY || 30_000); const ARCHIVE_EVERY = 10 * 60_000; /** The pages that may frame the app: the composer's Grade tab, and the site run locally. Kept in step with app.mjs. */ const FRAMED_BY = ["https://post.kualta.dev", "http://localhost:4321", "http://127.0.0.1:4321"]; /** What each clip offloaded to the archive was, by file name: its SHA-256, size and shoot day. */ const OFFLOADED = join(DATA, "offloaded.json"); /** The look an offloaded log clip without a grade gets, as settings, until it's graded by hand. */ const DEFAULT_LOOK = process.env.GRADE_DEFAULT_LOOK || "opendrt-colorful"; /** Clips come off the card only while this much stays free on the Mac, for exports and the system. */ const KEEP_FREE = Number(process.env.GRADE_KEEP_FREE || 5e9); // ---- base looks: a .cube per profile that takes that profile's code values ------------------------------------ const P4P = [ ["01", "Neutral", "Start here"], ["02", "FlatBase", "Start here"], ["03", "Natural", "Everyday"], ["12", "Punch", "Everyday"], ["04", "PrintFilm", "Film"], ["06", "Bleach", "Film"], ["11", "RetroFade", "Film"], ["10", "Mono", "Film"], ["05", "TealAmber", "Situational"], ["07", "NightNeon", "Situational"], ["08", "Golden", "Situational"], ["09", "Nordic", "Situational"], ]; const DJI = "dji/DJI OSMO Pocket 4P"; /** One generated file per log profile, from make_luts.py: `name(profile)` is its path under the LUT folder. */ const perLogProfile = (name) => Object.fromEntries(["dlog2", "dlog", "hlg"].map((p) => [p, name(p)])); /** A make_luts.py pack: /-.cube for each [slug, name], with ids `-`. */ const pack = (group, dir, looks) => looks.map(([slug, name]) => ({ id: `${dir}-${slug}`, name, group, files: perLogProfile((p) => `${dir}/${p}-${slug}.cube`) })); const REGISTRY = [ { id: "none", name: "As shot", group: "Camera", files: { dlog2: null, dlog: null, hlg: null, rec709: null } }, { id: "dji", name: "Rec.709", group: "DJI", files: { dlog2: `${DJI} D-Log2 to Rec.709 V1.0 size65.cube`, dlog: `${DJI} D-Log to Rec.709 V2.0 size33.cube` } }, { id: "dji-vivid", name: "Rec.709 vivid", group: "DJI", files: { dlog2: `${DJI} D-Log2 to Rec.709 vivid V1.0 size65.cube`, dlog: `${DJI} D-Log to Rec.709 vivid V2.0 size33.cube` } }, { id: "aces", name: "ACES 2.0", group: "ACES", files: perLogProfile((p) => `${p}-to-rec709-aces2.cube`) }, { id: "aces1", name: "ACES 1.0", group: "ACES", files: perLogProfile((p) => `${p}-to-rec709-aces1.cube`) }, ...pack("ARRI", "arri", [["reveal", "ALEXA 35 REVEAL"], ["classic", "ALEXA Classic"]]), ...pack("AgX", "agx", [["base", "Base"], ["punchy", "Punchy"], ["high", "High contrast"], ["low", "Low contrast"]]), ...pack("OpenDRT", "opendrt", [["default", "Default"], ["colorful", "Colorful"], ["umbra", "Umbra"]]), ...pack("Fujifilm", "fujifilm", [ ["provia", "Provia"], ["velvia", "Velvia"], ["astia", "Astia"], ["classic-chrome", "Classic Chrome"], ["reala-ace", "Reala Ace"], ["pro-neg-std", "Pro Neg. Std"], ["classic-neg", "Classic Neg."], ["eterna", "Eterna"], ["eterna-bb", "Eterna Bleach Bypass"], ["acros", "Acros"], ]), ...pack("Sony", "sony", [["s709", "s709"], ["cool", "Cool"], ["teal-orange", "Teal & orange"], ["vintage", "Vintage"], ["warm", "Warm"]]), ...P4P.map(([n, name, group]) => ({ id: `p4p-${n}`, name, group: `P4P · ${group}`, files: { dlog2: `p4p/P4P_${n}_${name}_size65.cube` } })), ]; async function exists(p) { return stat(p).then(() => true, () => false); } /** Base looks available per profile: the registry's files that are present, plus /custom//*.cube. */ async function bases() { const out = Object.fromEntries(Object.keys(PROFILES).map((p) => [p, []])); for (const b of REGISTRY) for (const [profile, file] of Object.entries(b.files)) if (file === null || (await exists(join(LUTS, file)))) out[profile].push({ id: b.id, name: b.name, group: b.group, file }); for (const profile of Object.keys(PROFILES)) { const dir = join(LUTS, "custom", profile); const files = await readdir(dir).catch(() => []); for (const f of files.filter((f) => f.toLowerCase().endsWith(".cube")).sort()) out[profile].push({ id: `custom:${f}`, name: basename(f, extname(f)), group: "Custom", file: join("custom", profile, f) }); } return out; } const cubes = new Map(); async function loadCube(file) { const path = join(LUTS, file); const { mtimeMs } = await stat(path); const hit = cubes.get(path); if (hit && hit.mtimeMs === mtimeMs) return hit.lut; const lut = parseCube(await readFile(path, "utf8")); cubes.set(path, { mtimeMs, lut }); return lut; } // ---- sources ------------------------------------------------------------------------------------------------------ function httpError(status, message) { return Object.assign(new Error(message), { status }); } async function readJson(path, fallback) { try { return JSON.parse(await readFile(path, "utf8")); } catch (e) { if (e.code === "ENOENT") return fallback; throw e; } } async function writeJson(path, data) { await mkdir(dirname(path), { recursive: true }); const tmp = `${path}.part`; await writeFile(tmp, `${JSON.stringify(data, null, 2)}\n`); await rename(tmp, path); } /** Changes to one of the app's { version, clips } files, made one after another so no save drops another's. */ const updating = new Map(); function updateClips(path, change) { const run = (updating.get(path) ?? Promise.resolve()).then(async () => { const data = await readJson(path, { version: 1, clips: {} }); data.version = 1; data.clips ??= {}; change(data.clips); await writeJson(path, data); }); updating.set(path, run.catch(() => {})); return run; } const readClips = async (path) => (await readJson(path, { clips: {} })).clips ?? {}; const stemOf = (name) => name.slice(0, name.length - extname(name).length); const isVideo = (f) => VIDEO.test(f) && !f.startsWith("."); const CAMERA_ID = /^camera:(\d{4}-\d{2}-\d{2})$/; const MAC_ID = /^mac:(\d{4}-\d{2}-\d{2})$/; const localDay = (d) => `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; /** The clips in Movies/Original by shoot day: the recording date in a DJI name, or else the file's own date. */ async function originalDays() { const days = new Map(); for (const name of (await readdir(ORIGINAL).catch(() => [])).filter(isVideo).sort()) { const path = join(ORIGINAL, name); const m = /^DJI_(\d{4})(\d{2})(\d{2})\d{6}_/.exec(name); const day = m ? `${m[1]}-${m[2]}-${m[3]}` : localDay((await stat(path)).mtime); if (!days.has(day)) days.set(day, []); days.get(day).push({ name, path }); } return days; } // The card's listing, read at most every few seconds so polling stays cheap and a plugged-in camera shows up quickly. let cardCache = null; async function card() { if (cardCache && Date.now() - cardCache.at < 3000) return cardCache.days; const days = await cameraDays(CAMERA); cardCache = { at: Date.now(), days }; return days; } /** * A source by id: a shoot day on the card or in Movies/Original, and where its clips are (`paths`, name → file). * A camera day that isn't on the card (unplugged) is a 404. */ async function source(id) { const cam = CAMERA_ID.exec(id); if (cam) { const list = (await card())?.get(cam[1]); if (!list) throw httpError(404, `${CAMERA_NAME} isn't connected, or has no clips from ${cam[1]}`); return { id, kind: "camera", label: cam[1], paths: new Map(list.map((c) => [c.name, c.path])) }; } const mac = MAC_ID.exec(id); if (!mac) throw httpError(400, "bad source"); const list = (await originalDays()).get(mac[1]); if (!list) throw httpError(404, `no clips from ${mac[1]} in ${ORIGINAL}`); return { id, kind: "mac", label: mac[1], paths: new Map(list.map((c) => [c.name, c.path])) }; } const proxyFile = (name) => join(CACHE, "proxies", `${stemOf(name)}.mp4`); const thumbFile = (name) => join(CACHE, "thumbs", `${stemOf(name)}.jpg`); const exportedFile = (name) => join(EXPORTED, `${stemOf(name)}.mp4`); /** Clip names (without extension) that have an export in Movies/Exported, made here or put there by hand. */ async function exportedStems() { const files = await readdir(EXPORTED).catch(() => []); return new Set(files.filter((f) => f.endsWith(".mp4") && !f.startsWith(".")).map(stemOf)); } /** Everything the picker lists: the camera's days, then the days in Movies/Original, each with how many are graded. */ async function library() { const exported = await exportedStems(); const listed = (prefix, days) => [...days] .sort((a, b) => b[0].localeCompare(a[0])) .map(([day, list]) => ({ id: `${prefix}:${day}`, label: day, clips: list.length, graded: list.filter((c) => exported.has(stemOf(c.name))).length })); const days = await card(); const camera = { name: CAMERA_NAME, path: CAMERA, mounted: Boolean(days), days: listed("camera", days ?? new Map()) }; return { original: ORIGINAL, exported: EXPORTED, luts: LUTS, camera, folders: listed("mac", await originalDays()) }; } /** * Camera metadata per clip, probed once (eight at a time) and kept in the cache's meta.json by path, size and * modification time, plus the file's size (`bytes`). */ async function metas(src, names) { const file = join(CACHE, "meta.json"); const cache = await readClips(file); const fresh = {}; const out = {}; const todo = [...names]; async function worker() { for (let name = todo.shift(); name !== undefined; name = todo.shift()) { const path = src.paths.get(name); const s = await stat(path); const hit = cache[path]; if (hit && hit.size === s.size && hit.mtimeMs === s.mtimeMs && "videoBitrate" in hit.meta) out[name] = hit.meta; else { out[name] = await probe(path); fresh[path] = { size: s.size, mtimeMs: s.mtimeMs, meta: out[name] }; } out[name] = { ...out[name], bytes: s.size }; } } await Promise.all(Array.from({ length: 8 }, worker)); if (Object.keys(fresh).length) await updateClips(file, (clips) => Object.assign(clips, fresh)); return out; } // ---- background jobs: thumbnails (automatic), previews (on request), exports --------------------------------------- const jobs = { thumb: { queue: [], busy: 0, failed: {} }, proxy: { queue: [], current: null, failed: {} }, export: { queue: [], current: null, done: [], failed: [], stopped: null }, }; const key = (id, name) => `${id}/${name}`; const jobOf = (src, name, meta) => ({ k: key(src.id, name), id: src.id, name, path: src.paths.get(name), meta }); function queueThumb(src, name, meta) { const job = jobOf(src, name, meta); if (jobs.thumb.failed[job.k] || jobs.thumb.queue.some((j) => j.k === job.k)) return; jobs.thumb.queue.push(job); pumpThumbs(); } function pumpThumbs() { while (jobs.thumb.busy < 4 && jobs.thumb.queue.length) { const job = jobs.thumb.queue.shift(); jobs.thumb.busy++; makeThumb(job.path, thumbFile(job.name), { still: job.meta.still, at: Math.min(1, job.meta.duration / 2) }) .catch((e) => { jobs.thumb.failed[job.k] = String(e.message || e); }) .finally(() => { jobs.thumb.busy--; pumpThumbs(); }); } } /** Ask for previews: they're made one at a time, the latest request first, so the clip you just picked comes next. */ function queueProxies(src, names, meta) { for (const name of [...names].reverse()) { const job = jobOf(src, name, meta[name]); if (jobs.proxy.current?.k === job.k) continue; delete jobs.proxy.failed[job.k]; jobs.proxy.queue = jobs.proxy.queue.filter((j) => j.k !== job.k); jobs.proxy.queue.unshift(job); } pumpProxies(); } let proxyBusy = false; async function pumpProxies() { if (proxyBusy) return; proxyBusy = true; while (jobs.proxy.queue.length) { const job = jobs.proxy.queue.shift(); jobs.proxy.current = job; const dst = proxyFile(job.name); try { if (!(await exists(dst))) await makeProxy(job.path, dst, job.meta); } catch (e) { jobs.proxy.failed[job.k] = String(e.message || e); } jobs.proxy.current = null; } proxyBusy = false; } let exportBusy = false; let exportAbort = null; async function pumpExports() { if (exportBusy) return; exportBusy = true; while (jobs.export.queue.length) { const job = jobs.export.queue.shift(); jobs.export.current = { ...job, progress: 0 }; exportAbort = new AbortController(); try { await exportOne( job.id, job.name, job.quality, (p) => { jobs.export.current.progress = p; }, exportAbort.signal, ); jobs.export.done.unshift({ ...job, at: new Date().toISOString() }); jobs.export.done.length = Math.min(jobs.export.done.length, 100); } catch (e) { if (!exportAbort.signal.aborted) jobs.export.failed.unshift({ ...job, error: String(e.message || e) }); } exportAbort = null; jobs.export.current = null; } exportBusy = false; scheduleLedger(2000); } /** Stop exporting: drop the queue and stop the clip in progress (its half-written file is removed). */ function stopExports() { const count = jobs.export.queue.length + (jobs.export.current ? 1 : 0); jobs.export.queue = []; exportAbort?.abort(); jobs.export.stopped = { at: new Date().toISOString(), count }; return count; } // ---- moving clips to the Trash (⌘⌫) ------------------------------------------------------------------------------- /** Where trashed clips go instead of the system Trash: a folder, for the tests. */ const TRASH_DIR = process.env.GRADE_TRASH; async function moveToTrash(files) { if (TRASH_DIR) { await mkdir(TRASH_DIR, { recursive: true }); for (const f of files) await rename(f, join(TRASH_DIR, basename(f))); return; } // macOS 15's trash(8): -s fails instead of skipping a file it can't move. A card's Trash is its own .Trashes. await new Promise((resolve, reject) => { execFile("/usr/bin/trash", ["-s", ...files], (err, _out, stderr) => { if (err) reject(new Error(stderr.trim() || err.message)); else resolve(); }); }); const left = []; for (const f of files) if (await exists(f)) left.push(basename(f)); if (left.length) throw new Error(`couldn't move ${left.join(", ")} to the Trash`); } /** The trash request in progress: requests run one after another. */ let trashing = Promise.resolve(); /** * Move clips to the Trash with the files the camera wrote beside them under the same name (DJI's low-res .LRF), and * drop the app's preview and thumbnail for them. Their grades and graded copies stay: the edit uses those. * One request at a time, and a file already gone is skipped: a second ⌘⌫ on a clip can arrive with a card listing * (cached for a few seconds) that still has it. */ function trashClips(src, names) { const run = trashing.then(() => trashNow(src, names)); trashing = run.catch(() => {}); return run; } async function trashNow(src, names) { for (const name of names) { if (!src.paths.has(name)) throw httpError(400, `unknown clip ${name}`); const exporting = (j) => j?.id === src.id && j?.name === name; if (exporting(jobs.export.current) || jobs.export.queue.some(exporting)) throw httpError(409, `${name} is exporting: stop exporting first`); } const trashed = []; for (const name of names) { const path = src.paths.get(name); const siblings = (await readdir(dirname(path))).filter((f) => f !== name && !f.startsWith(".") && stemOf(f) === stemOf(name)); const files = siblings.map((f) => join(dirname(path), f)); if (await exists(path)) files.unshift(path); if (files.length) await moveToTrash(files); trashed.push(...files.map((f) => basename(f))); const k = key(src.id, name); jobs.thumb.queue = jobs.thumb.queue.filter((j) => j.k !== k); jobs.proxy.queue = jobs.proxy.queue.filter((j) => j.k !== k); await rm(proxyFile(name), { force: true }); await rm(thumbFile(name), { force: true }); } cardCache = null; scheduleLedger(2000); return trashed; } /** * Bake one clip's grade from its original, wherever that is, and write only the graded copy and its record. The LUT * is written beside its final name and moved into place only once the export has worked, so a failed or stopped * re-export leaves the previous export, its LUT and its record as they were. */ /** A clip's saved grade as it will be applied, and the one LUT that bakes it (what the preview shows). */ async function clipLut(src, name) { const meta = (await metas(src, [name]))[name]; const available = (await bases())[meta.profile]; const grade = resolveGrade((await readClips(GRADES))[name], available.map((b) => b.id)); const base = available.find((b) => b.id === grade.base); const lut = composeLut({ profile: meta.profile, grade, shotK: meta.cameraKelvin, base: base.file ? await loadCube(base.file) : null, size: EXPORT_LUT_SIZE }); return { meta, grade, base, cube: writeCube(EXPORT_LUT_SIZE, lut, `${stemOf(name)} · ${base.name} · grade app`) }; } /** Export quality, a whole percent from 10 to 100 (full, near lossless). */ const exportQuality = (q) => Math.min(100, Math.max(10, Math.round(Number(q) || 100))); /** Run `use` with a clip's LUT written to a file of its own in the cache, gone again afterwards. */ async function withLutFile(name, cube, use) { const file = join(CACHE, "luts", `${stemOf(name)}.${process.hrtime.bigint()}.cube`); await mkdir(dirname(file), { recursive: true }); await writeFile(file, cube); try { return await use(file); } finally { await rm(file, { force: true }); } } /** * Bake one clip's grade from its original, wherever that is, into Movies/Exported/.mp4, and record what it was * made with in exports.json. The file is written under a temporary name and moved into place only once the export * has worked, so a failed or stopped re-export leaves the previous export and its record as they were. */ async function exportOne(id, name, quality, onProgress, signal) { const src = await source(id); if (!src.paths.has(name)) throw new Error(`${name} isn't in ${src.label} anymore`); const { meta, grade, base, cube } = await clipLut(src, name); const path = src.paths.get(name); await withLutFile(name, cube, (lut) => exportClip({ src: path, dst: exportedFile(name), cube: lut, matrix: meta.matrix, deflicker: grade.deflicker, duration: meta.duration, quality, videoBitrate: meta.videoBitrate, onProgress, signal, }), ); await updateClips(EXPORTS, (clips) => { clips[name] = { source: src.kind === "camera" ? `${CAMERA_NAME}: ${relative(dirname(CAMERA), path)}` : relative(MOVIES, path), file: relative(MOVIES, exportedFile(name)), profile: meta.profile, cameraKelvin: meta.cameraKelvin, base: { id: base.id, name: base.name, file: base.file }, grade, key: gradeKey(grade), quality, exportedAt: new Date().toISOString(), tool: "scripts/grade", }; }); } /** * One frame of a clip, graded with its saved grade, as Movies/Exported/_f.png at full size. `frame` * counts from 0; the seek lands a quarter frame early so ffmpeg keeps exactly that frame. */ async function frameOne(src, name, frame) { const { meta, grade, cube } = await clipLut(src, name); const dst = join(EXPORTED, `${stemOf(name)}_f${String(frame).padStart(4, "0")}.png`); const time = Math.max(0, (frame - 0.25) / (meta.fps || 24)); await withLutFile(name, cube, (lut) => exportFrame({ src: src.paths.get(name), dst, cube: lut, matrix: meta.matrix, deflicker: grade.deflicker, time }), ); return relative(MOVIES, dst); } // The decoder calibration clip and what ffmpeg reads from it, made once per library (version bumps remake it). const CALIBRATION_VERSION = 2; const calibrationVideo = join(CACHE, "calibration.mp4"); let calibration = null; function calibrationData() { calibration ??= (async () => { const file = join(CACHE, "calibration.json"); const cached = await readJson(file, null); if (cached?.version === CALIBRATION_VERSION && (await exists(calibrationVideo))) return cached; const expected = await makeCalibration(calibrationVideo); const data = { version: CALIBRATION_VERSION, expected }; await writeJson(file, data); return data; })().catch((e) => { calibration = null; throw e; }); return calibration; } // ---- the footage ledger (ledger.mjs) ---------------------------------------------------------------------------------- const ledger = { /** The ledger as last merged; null until the first look. */ current: null, /** Whether the archive's copy has been read back, so sending this Mac's can't drop clips only it knows. */ restored: false, /** What the archive holds, as comparable(): the ledger is sent when it differs. */ sent: null, savedAt: null, error: null, archiveError: null, cloud: false, archiveAt: 0, manifests: new Map(), /** Clips whose metadata couldn't be read: not tried again every look. */ unreadable: new Set(), running: false, timer: null, }; /** The clips with a thumbnail in the cache, by name without extension, lowercase. */ /** The OpenFilm projects on this Mac (folders holding a film.html), each with the video files in its assets. */ async function localProjects() { const found = []; for (const entry of await readdir(PROJECTS, { withFileTypes: true }).catch(() => [])) { const dir = join(PROJECTS, entry.name); if (!entry.isDirectory() || !(await exists(join(dir, "film.html")))) continue; const files = []; for (const assets of ["assets", "zh/assets"]) files.push(...(await readdir(join(dir, assets)).catch(() => [])).filter(isVideo)); found.push({ name: entry.name, files }); } return found; } async function thumbStems() { return new Set((await thumbFiles()).keys()); } /** The cache's thumbnails: clip, lowercase, → file name. */ async function thumbFiles() { const files = await readdir(join(CACHE, "thumbs")).catch(() => []); return new Map(files.filter((f) => f.endsWith(".jpg")).map((f) => [stemOf(f).toLowerCase(), f])); } /** * Sends the archive a few clips' pictures it hasn't got (for the composer's Footage tab), marking each clip's record * `thumb` once it's there; says whether more are waiting. */ async function sendThumbs(cloud, at = 24) { const files = await thumbFiles(); const waiting = Object.entries(ledger.current.clips).filter( ([key, clip]) => !clip.thumb && pictureName(key) && files.has(key.toLowerCase()), ); for (const [key, clip] of waiting.slice(0, at)) { try { await cloud.putThumb(key, await readFile(join(CACHE, "thumbs", files.get(key.toLowerCase())))); clip.thumb = true; } catch (e) { ledger.error = `Couldn't send a picture to Cloudflare: ${e.message}`; return false; } } return waiting.length > at; } /** A file's size and modification time, or null once it's gone. */ const sizeOf = async (path) => { const s = await stat(path).catch(() => null); return s ? { size: s.size, mtime: s.mtimeMs } : null; }; /** * What the ledger is made from, as mergeLedger takes it. Clips whose profile isn't known yet are probed a few at a time * (none while an export reads the card); `more` says some are left. */ async function observe(cloud) { const at = new Date().toISOString(); const days = await card(); const listed = async (list) => (await Promise.all(list.map(async (c) => ({ name: c.name, path: c.path, ...(await sizeOf(c.path)) })))).filter((c) => c.size !== undefined); const cameraClips = await listed([...(days?.values() ?? [])].flat()); const macClips = await listed([...(await originalDays()).values()].flat()); const known = await readClips(join(CACHE, "meta.json")); const metasByClip = new Map(); const unknown = []; for (const c of [...cameraClips, ...macClips]) { const meta = known[c.path]?.meta; if (meta) metasByClip.set(stemOf(c.name).toLowerCase(), { profile: meta.profile, duration: meta.duration }); else if (!ledger.unreadable.has(c.path)) unknown.push(c); } const batch = jobs.export.current ? [] : unknown.slice(0, 8); for (const c of batch) { try { const meta = (await metas({ id: "ledger", paths: new Map([[c.name, c.path]]) }, [c.name]))[c.name]; metasByClip.set(stemOf(c.name).toLowerCase(), { profile: meta.profile, duration: meta.duration }); } catch { ledger.unreadable.add(c.path); } } const exportedFiles = (await readdir(EXPORTED).catch(() => [])).filter((f) => f.endsWith(".mp4") && !f.startsWith(".")); const exported = await listed(exportedFiles.map((name) => ({ name, path: join(EXPORTED, name) }))); // The ledger's small pictures, made like the clip list's: from the original (its still, or a frame), or from the // graded copy once the original is gone. A dozen each look, none while an export reads the card. const pictured = await thumbStems(); const unpictured = []; for (const c of [...cameraClips, ...macClips, ...exported]) { const lower = stemOf(c.name).toLowerCase(); if (pictured.has(lower)) continue; const meta = known[c.path]?.meta ?? (c.path.startsWith(EXPORTED) ? { still: null, duration: 1 } : null); if (!meta) continue; // One picture a clip: from the first copy that can make it. pictured.add(lower); unpictured.push({ ...c, meta }); } const pictures = jobs.export.current ? [] : unpictured.slice(0, 12); for (const c of pictures) queueThumb({ id: "ledger", paths: new Map([[c.name, c.path]]) }, c.name, c.meta); let archive = null; if (cloud && Date.now() - ledger.archiveAt > ARCHIVE_EVERY) { try { archive = await archivedFootage(cloud.request, ledger.manifests); ledger.archiveAt = Date.now(); ledger.archiveError = null; } catch (e) { // Asked again in two minutes rather than ten. ledger.archiveAt = Date.now() - ARCHIVE_EVERY + 2 * 60_000; ledger.archiveError = `Couldn't read the Cloudflare archive: ${e.message}`; } } const seen = { at, camera: { name: CAMERA_NAME, mounted: Boolean(days), clips: cameraClips }, mac: macClips, metas: metasByClip, exports: await readClips(EXPORTS), exported, grades: Object.keys(await readClips(GRADES)), youtube: await youtubeUploads(PUBLISHER), archive, projects: await localProjects(), }; const moreToProbe = unknown.length > batch.length && batch.length > 0; const moreToPicture = unpictured.length > pictures.length && pictures.length > 0; return { seen, more: moreToProbe || moreToPicture }; } /** * One look: read back the ledger kept here, and the archive's copy once it can be reached, then merge in what's seen * now, keep it here and send it to the archive if anything in it changed. */ async function ledgerLook() { const cloud = await ledgerCloud(PUBLISHER); ledger.cloud = Boolean(cloud); ledger.error = null; if (!ledger.current) ledger.current = await readJson(LEDGER, null); if (cloud && !ledger.restored) { try { const remote = await cloud.read(); ledger.current = unionLedgers(ledger.current, remote); ledger.sent = comparable(remote); // When the archive's copy was made: unchanged since, there's nothing to send, and it's still the time it was saved. ledger.savedAt = remote?.updatedAt ?? null; ledger.restored = true; } catch (e) { ledger.error = `Couldn't read the ledger in Cloudflare: ${e.message}`; } } const { seen, more } = await observe(cloud); const before = comparable(ledger.current); ledger.current = mergeLedger(ledger.current, seen); const morePictures = cloud && ledger.restored ? await sendThumbs(cloud) : false; const now = comparable(ledger.current); if (now !== before || !(await exists(LEDGER))) await writeJson(LEDGER, ledger.current); if (cloud && ledger.restored && now !== ledger.sent) { try { await cloud.write(ledger.current); ledger.sent = now; ledger.savedAt = ledger.current.updatedAt; } catch (e) { ledger.error = `Couldn't save the ledger to Cloudflare: ${e.message}`; } } return more || morePictures; } /** Looks again in `ms`; sooner after an export or a trash, and every few seconds while clips wait to be probed. */ function scheduleLedger(ms = LEDGER_EVERY) { clearTimeout(ledger.timer); ledger.timer = setTimeout(async () => { if (ledger.running) return scheduleLedger(1000); ledger.running = true; let more = false; try { more = await ledgerLook(); } catch (e) { ledger.error = String(e.message || e); console.error(e); } finally { ledger.running = false; } scheduleLedger(more ? Math.min(5000, LEDGER_EVERY) : LEDGER_EVERY); }, ms); } // ---- offloading: the card to this Mac, this Mac to the archive (offload.mjs) ----------------------------------------- /** The run going, or the last one: offload.mjs's `{ state, stop, done }`. */ let offloading = null; /** Set while a run is being started: the clips' profiles are read first. */ let starting = false; /** While a run goes, the composer counts as watching, so graded clips go up to YouTube meanwhile (the helper's job). */ let footageWatch = null; const sizeOfAll = (clips) => clips.reduce((sum, clip) => sum + clip.size, 0); async function roomOnMac() { const { bavail, bsize } = await statfs(MOVIES); return bavail * bsize; } /** The clips in a place, the card ("camera") or Movies/Original ("mac"), each with its size, time and shoot day. */ async function placeClips(scope) { const days = scope === "camera" ? ((await card()) ?? new Map()) : await originalDays(); const clips = []; for (const [day, list] of days) for (const clip of list) { const s = await stat(clip.path).catch(() => null); if (s) clips.push({ name: clip.name, path: clip.path, size: s.size, mtime: Math.round(s.mtimeMs), day }); } return clips.sort((a, b) => a.name.localeCompare(b.name)); } /** A clip an export is making, or is queued to make: it stays where it is. */ const exporting = (clip) => jobs.export.current?.name === clip.name || jobs.export.queue.some((job) => job.name === clip.name); function offloadState() { if (!offloading) return null; const { samples, ...state } = offloading.state; const [first, last] = [samples[0], samples.at(-1)]; const speed = first && last && last[0] > first[0] ? ((last[1] - first[1]) / (last[0] - first[0])) * 1000 : null; const inFlight = state.send.current.reduce((sum, clip) => sum + clip.sent, 0); return { ...state, sentBytes: state.send.doneBytes + inFlight, speed }; } /** What an offload would take: the card's clips, the Mac's not in the archive yet, the room on the Mac; and the run. */ async function offloadSummary() { const record = await readClips(OFFLOADED); const mounted = Boolean(await card()); const camera = await placeClips("camera"); const mac = (await placeClips("mac")).filter((clip) => record[clip.name]?.size !== clip.size); return { camera: { mounted, clips: camera.length, bytes: sizeOfAll(camera) }, mac: { clips: mac.length, bytes: sizeOfAll(mac) }, room: { free: await roomOnMac(), keepFree: KEEP_FREE }, run: offloadState(), }; } /** * Gives ungraded log clips the default look, as settings: they can be changed before any export. A clip nobody has * opened yet is probed for its profile, eight at a time. */ async function applyDefaultLook(clips) { const [grades, looks, known] = await Promise.all([readClips(GRADES), bases(), readClips(join(CACHE, "meta.json"))]); const todo = clips.filter((clip) => !grades[clip.name]); const added = {}; async function worker() { for (let clip = todo.shift(); clip; clip = todo.shift()) { const profile = known[clip.path]?.meta?.profile ?? (await probe(clip.path).then((meta) => meta.profile, () => null)); if (looks[profile]?.some((look) => look.id === DEFAULT_LOOK)) added[clip.name] = normalizeGrade({ base: DEFAULT_LOOK }); } } await Promise.all(Array.from({ length: 8 }, worker)); // A grade made meanwhile stays. await updateClips(GRADES, (all) => { for (const [name, grade] of Object.entries(added)) all[name] ??= grade; }); } /** Starts the card (if it's plugged in) and Movies/Original going: it goes on in the background until done or stopped. */ async function startOffloading() { if (starting || offloading?.state.running) throw httpError(409, "Already offloading"); starting = true; try { await beginOffload(); } finally { starting = false; } } async function beginOffload() { const cloud = await ledgerCloud(PUBLISHER); if (!cloud) throw httpError(409, "Pair the crossposter helper first: the offload goes through it"); const [camera, mac, record] = await Promise.all([placeClips("camera"), placeClips("mac"), readClips(OFFLOADED)]); await mkdir(ORIGINAL, { recursive: true }); // Before any clip moves: the look goes by name, the probing by where each clip is now. await applyDefaultLook([...camera, ...mac]).catch((e) => console.error("grade: default look:", e.message)); offloading = startOffload({ camera, mac, record, dir: ORIGINAL, request: cloud.request, room: roomOnMac, keepFree: KEEP_FREE, removable: (clip) => !exporting(clip), landed: (clip, sha256) => { void updateClips(OFFLOADED, (all) => { all[clip.name] = { sha256, size: clip.size, day: clip.day, at: new Date().toISOString() }; }); // The ledger reads the archive again, so the clip shows as in Cloudflare. ledger.archiveAt = 0; scheduleLedger(2000); }, removed: () => { cardCache = null; }, }); const watch = () => cloud.watchFootage().catch(() => {}); void watch(); clearInterval(footageWatch); footageWatch = setInterval(watch, 60_000); void offloading.done.finally(() => { clearInterval(footageWatch); footageWatch = null; cardCache = null; scheduleLedger(2000); }); } // ---- HTTP ----------------------------------------------------------------------------------------------------------- function send(res, status, body, type = "application/json") { const data = type === "application/json" ? JSON.stringify(body) : body; res.writeHead(status, { "Content-Type": type, "Cache-Control": "no-store" }); res.end(data); } async function body(req) { let size = 0; const chunks = []; for await (const c of req) { size += c.length; if (size > 2 * 1024 * 1024) throw httpError(413, "request too large"); chunks.push(c); } return JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}"); } const TYPES = { ".html": "text/html; charset=utf-8", ".mjs": "text/javascript", ".js": "text/javascript", ".css": "text/css", ".mp4": "video/mp4", ".mov": "video/quicktime", ".jpg": "image/jpeg", ".svg": "image/svg+xml" }; /** A file, with single-range support so the browser can seek video. */ async function serveFile(req, res, path, cache = "no-cache") { const s = await stat(path).catch(() => null); if (!s?.isFile()) throw httpError(404, "not found"); const type = TYPES[extname(path).toLowerCase()] || "application/octet-stream"; const range = /^bytes=(\d*)-(\d*)$/.exec(req.headers.range || ""); if (range) { let start = range[1] === "" ? s.size - Number(range[2]) : Number(range[1]); let end = range[1] !== "" && range[2] !== "" ? Number(range[2]) : s.size - 1; start = Math.max(0, start); end = Math.min(s.size - 1, end); if (start > end) { res.writeHead(416, { "Content-Range": `bytes */${s.size}` }); return res.end(); } res.writeHead(206, { "Content-Type": type, "Content-Length": end - start + 1, "Content-Range": `bytes ${start}-${end}/${s.size}`, "Accept-Ranges": "bytes", "Cache-Control": cache }); return createReadStream(path, { start, end }).pipe(res); } res.writeHead(200, { "Content-Type": type, "Content-Length": s.size, "Accept-Ranges": "bytes", "Cache-Control": cache }); if (req.method === "HEAD") return res.end(); createReadStream(path).pipe(res); } /** A clip's preview: ready, being made, waiting, failed, or not asked for yet. */ function proxyState(k, ready) { if (ready) return "ready"; if (jobs.proxy.failed[k]) return "failed"; if (jobs.proxy.current?.k === k) return "working"; if (jobs.proxy.queue.some((j) => j.k === k)) return "queued"; return "none"; } async function sourcePayload(id) { const src = await source(id); const names = [...src.paths.keys()]; const meta = await metas(src, names); const [allGrades, records, graded] = await Promise.all([readClips(GRADES), readClips(EXPORTS), exportedStems()]); const grades = {}; const clips = []; for (const name of names) { const k = key(id, name); if (allGrades[name]) grades[name] = allGrades[name]; const thumb = await exists(thumbFile(name)); if (!thumb) queueThumb(src, name, meta[name]); let exported = null; if (graded.has(stemOf(name))) { // An export without the app's record (put there by hand) counts as graded; its grade just isn't known. const r = records[name]; exported = r ? { key: r.key, at: r.exportedAt } : { key: null, at: (await stat(exportedFile(name))).mtime.toISOString() }; } clips.push({ name, stem: stemOf(name), ...meta[name], thumb, proxy: proxyState(k, await exists(proxyFile(name))), proxyError: jobs.proxy.failed[k] ?? null, exported, }); } return { id, kind: src.kind, label: src.label, clips, grades }; } async function route(req, res) { const url = new URL(req.url, "http://localhost"); const p = decodeURIComponent(url.pathname); const mutating = req.method !== "GET" && req.method !== "HEAD"; // A page on another origin cannot send this header without a preflight this server never grants. if (mutating && req.headers["x-grade"] !== "1") throw httpError(403, "missing X-Grade header"); if (p === "/" || p === "/index.html") { // Only the composer's Grade tab (post.kualta.dev/grade, or the site run locally) may show the app in a frame. res.setHeader("Content-Security-Policy", `frame-ancestors 'self' ${FRAMED_BY.join(" ")}`); return serveFile(req, res, join(here, "ui/index.html")); } if (p === "/color.mjs") return serveFile(req, res, join(here, "color.mjs")); if (/^\/ui\/\w[\w.-]*$/.test(p)) return serveFile(req, res, join(here, p)); if (p === "/api/library" && req.method === "GET") return send(res, 200, await library()); if (p === "/api/ledger" && req.method === "GET") return send(res, 200, { ledger: ledger.current, cloud: ledger.cloud, savedAt: ledger.savedAt, error: ledger.error ?? ledger.archiveError, }); // Where a clip is, by its name, for the Footage tab's Grade: the card's day holding it, else Movies/Original's. if (p === "/api/find" && req.method === "GET") { const wanted = stemOf(url.searchParams.get("clip") ?? "").toLowerCase(); const places = [ ["camera", (await card()) ?? new Map()], ["mac", await originalDays()], ]; for (const [kind, days] of places) for (const [day, list] of days) { const hit = list.find((c) => stemOf(c.name).toLowerCase() === wanted); if (hit) return send(res, 200, { source: `${kind}:${day}`, name: hit.name }); } throw httpError(404, "That clip isn't on the camera or this Mac"); } if (p === "/api/bases" && req.method === "GET") return send(res, 200, await bases()); if (p === "/api/calibration" && req.method === "GET") { const { expected } = await calibrationData(); return send(res, 200, { video: `/media/calibration.mp4?v=${CALIBRATION_VERSION}`, expected }); } if (p === "/media/calibration.mp4") { await calibrationData(); return serveFile(req, res, calibrationVideo); } let m = /^\/api\/sources\/([^/]+)$/.exec(p); if (m && req.method === "GET") return send(res, 200, await sourcePayload(m[1])); m = /^\/api\/sources\/([^/]+)\/grades$/.exec(p); if (m && req.method === "PUT") { const src = await source(m[1]); const incoming = Object.entries((await body(req)).clips ?? {}); for (const [name] of incoming) if (!src.paths.has(name)) throw httpError(400, `unknown clip ${name}`); await updateClips(GRADES, (clips) => Object.assign(clips, Object.fromEntries(incoming))); return send(res, 200, { ok: true }); } m = /^\/api\/sources\/([^/]+)\/prepare$/.exec(p); if (m && req.method === "POST") { const src = await source(m[1]); const names = ((await body(req)).names ?? []).filter((n) => src.paths.has(n)).slice(0, 50); queueProxies(src, names, await metas(src, names)); return send(res, 200, { queued: names.length }); } m = /^\/api\/sources\/([^/]+)\/trash$/.exec(p); if (m && req.method === "POST") { const src = await source(m[1]); const names = (await body(req)).clips ?? []; if (!Array.isArray(names) || !names.length) throw httpError(400, "no clips to move to the Trash"); return send(res, 200, { trashed: await trashClips(src, names) }); } m = /^\/api\/lut\/(\w+)\/([^/]+)$/.exec(p); if (m && req.method === "GET") { const base = (await bases())[m[1]]?.find((b) => b.id === m[2]); if (!base?.file) throw httpError(404, "no such base"); const lut = await loadCube(base.file); res.writeHead(200, { "Content-Type": "application/octet-stream", "X-Lut-Size": String(lut.size), "Cache-Control": "no-cache" }); return res.end(Buffer.from(lut.data.buffer, lut.data.byteOffset, lut.data.byteLength)); } // A clip's preview proxy, its thumbnail, or the original itself (the preview, streamed off the card). m = /^\/media\/([^/]+)\/(proxy|thumb|original)\/([^/]+)$/.exec(p); if (m) { const [, id, kind, file] = m; const src = await source(id); const name = [...src.paths.keys()].find((n) => stemOf(n) === stemOf(file)); if (!name) throw httpError(404, "no such clip"); const path = { proxy: () => proxyFile(name), thumb: () => thumbFile(name), original: () => src.paths.get(name) }[kind](); return serveFile(req, res, path); } if (p === "/api/export" && req.method === "POST") { const { source: id, names, quality: q } = await body(req); const src = await source(String(id)); const quality = exportQuality(q); for (const name of names ?? []) { if (!src.paths.has(name)) throw httpError(400, `unknown clip ${name}`); if (jobs.export.current?.id === src.id && jobs.export.current?.name === name) continue; const queued = jobs.export.queue.find((j) => j.id === src.id && j.name === name); if (queued) queued.quality = quality; else jobs.export.queue.push({ id: src.id, name, quality }); } jobs.export.failed = jobs.export.failed.filter((f) => !(f.id === src.id && (names ?? []).includes(f.name))); pumpExports(); return send(res, 200, { queued: jobs.export.queue.length }); } if (p === "/api/export/stop" && req.method === "POST") return send(res, 200, { stopped: stopExports() }); if (p === "/api/export-frame" && req.method === "POST") { const { source: id, name, frame } = await body(req); const src = await source(String(id)); if (!src.paths.has(name)) throw httpError(400, `unknown clip ${name}`); if (!Number.isInteger(frame) || frame < 0) throw httpError(400, "frame must be a whole number from 0"); return send(res, 200, { file: await frameOne(src, name, frame) }); } if (p === "/api/jobs" && req.method === "GET") return send(res, 200, { thumbs: jobs.thumb.queue.length + jobs.thumb.busy, proxy: { queued: jobs.proxy.queue.length, current: jobs.proxy.current && { id: jobs.proxy.current.id, name: jobs.proxy.current.name } }, export: jobs.export, offload: offloadState(), }); // Offloading: the card to this Mac, this Mac to the archive. if (p === "/api/offload" && req.method === "GET") return send(res, 200, await offloadSummary()); if (p === "/api/offload" && req.method === "POST") { await startOffloading(); return send(res, 200, await offloadSummary()); } if (p === "/api/offload/stop" && req.method === "POST") { offloading?.stop(); return send(res, 200, { stopping: Boolean(offloading?.state.running) }); } throw httpError(404, "not found"); } const server = createServer(async (req, res) => { // Only this machine's names: a page that rebinds its own domain to 127.0.0.1 still sends its own Host. const host = String(req.headers.host || "").replace(/:\d+$/, ""); if (!["127.0.0.1", "localhost", "[::1]"].includes(host)) return send(res, 421, { error: "wrong host" }); try { await route(req, res); } catch (e) { if (!res.headersSent) send(res, e.status || 500, { error: String(e.message || e) }); if (!e.status) console.error(e); } }); for (const dir of [ORIGINAL, EXPORTED, DATA, CACHE]) await mkdir(dir, { recursive: true }); server.listen(PORT, "127.0.0.1", () => { const address = `http://127.0.0.1:${PORT}`; console.log(`grade: ${address} (movies ${MOVIES}, camera ${CAMERA}, data ${DATA}, LUTs ${LUTS})`); if (process.argv.includes("--open")) execFile("open", [address], () => {}); scheduleLedger(Math.min(2000, LEDGER_EVERY)); });