#!/usr/bin/env node /** * Back up, inspect and restore a Fablewright instance. * * An instance is one SQLite database, an `auth-secret` file and an uploads * directory. A backup is one directory holding all three and a manifest. * * The database is copied with `VACUUM INTO`: one consistent file, no write lock. * Copying the database, its journal and its shared-memory file by hand gives a * backup that opens and is wrong. * * Restore keeps the database it replaces, so a failed upgrade is one restore * from the version that worked. * * This file is plain JavaScript with no imports from the app. It runs while the * app is stopped, on a container with no shell tooling and no running server. */ import { createHash, randomBytes, scryptSync } from 'node:crypto'; import { appendFileSync, chmodSync, closeSync, constants as fsConstants, copyFileSync, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, readSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { fileURLToPath } from 'node:url'; import process from 'node:process'; import { readDbState, resolveDatabasePath, unexpectedMigrations } from './db-state.mjs'; import { clampRetentionDays, cutoffFor, DEFAULT_RETENTION_DAYS, expiryQueries, HISTORY_TABLES, KEEP_ITEM_VERSIONS, KEEP_PROJECT_VERSIONS, KEEP_SCENE_VERSIONS } from './history-retention.mjs'; import { deleteAccountReferences } from './account-references.mjs'; /** The app root, so the tool finds `package.json` and `drizzle/` wherever it runs. */ const APP_ROOT = fileURLToPath(new URL('..', import.meta.url)); const MIGRATIONS_FOLDER = join(APP_ROOT, 'drizzle'); const FORMAT = 1; const DRILL_FORMAT = 1; const DATABASE_FILE = 'fablewright.db'; const MANIFEST_FILE = 'manifest.json'; const SECRET_FILE = 'auth-secret'; const UPLOADS_DIR = 'uploads'; const IMPORTS_DIR = 'imports'; /** * How long a file must have sat on disk before a sweep may delete it. The app * publishes a file and then commits the row that names it, so a file inside this * window may belong to a row that has not landed yet. The same window covers an * import, which publishes every picture in the archive before its transaction * ends, and the staging copies an upload writes before it renames them. */ export const SWEEP_GRACE_MS = 60 * 60 * 1000; /** * Beside the database, not in it: a restore replaces the database, and the record * of a drill has to outlive the thing it proves. */ export const DRILLS_FILE = 'restore-drills.ndjson'; /** Re-exported from the shared retention policy for existing callers and tests. */ export { DEFAULT_RETENTION_DAYS, KEEP_ITEM_VERSIONS, KEEP_PROJECT_VERSIONS, KEEP_SCENE_VERSIONS }; const DIRECTORY_PATTERN = /^fablewright-\d{4}-\d{2}-\d{2}T[\d-]+Z-v/; /** * A backup holds these four names and nothing else. The manifest says which of * them the directory should hold, so anything else means this is not a directory * the tool wrote. */ const BACKUP_ENTRIES = new Set([MANIFEST_FILE, DATABASE_FILE, SECRET_FILE, UPLOADS_DIR]); const SHA256_HEX = /^[a-f0-9]{64}$/; const COPY_CHUNK_BYTES = 1024 * 1024; /** `O_NOFOLLOW` is absent on Windows, where a link needs a privilege to make one. */ const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; const CREATE_FLAGS = fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC; /** * One backup inventory: every inspected category, its display noun, the SQLite * table and optional filter it counts, and whether zero counts stay visible. * Keeping every table here updates inspection, manifest checks, formatting, and * restore-drill comparisons together. */ const BACKUP_CONTENTS = [ { key: 'projects', noun: 'project', table: 'project', resting: true }, { key: 'scenes', noun: 'scene', table: 'scene', resting: true }, { key: 'notes', noun: 'note', table: 'note', resting: true }, { key: 'noteTypes', noun: 'note type', table: 'note_type' }, { key: 'storylines', noun: 'storyline', table: 'storyline' }, { key: 'plotCards', noun: 'plot card', table: 'plot_card' }, { key: 'accounts', noun: 'account', table: 'user' }, { key: 'passkeys', noun: 'passkey', table: 'passkey' }, { key: 'sceneVersions', noun: 'scene version', table: 'scene_snapshot' }, { key: 'itemVersions', noun: 'item version', table: 'item_version' }, { key: 'checkpoints', noun: 'checkpoint', table: 'project_checkpoint' }, { key: 'images', noun: 'image', table: 'note_image', where: 'image_id is not null' }, { key: 'feedback', noun: 'feedback item', table: 'scene_feedback' }, { key: 'shares', noun: 'secret link', table: 'project_share' }, { key: 'invitations', noun: 'invitation', table: 'project_invitation' }, { key: 'writingDays', noun: 'writing day', table: 'writing_day' }, { key: 'writingGoals', noun: 'writing goal', table: 'writing_goal' }, { key: 'importReports', noun: 'import report', table: 'import_report' } ]; /** * `VACUUM INTO` takes no write lock and refuses to overwrite, so the target must * not exist. It checkpoints and compacts, so the copy is one file with no journal * beside it. */ function vacuumInto(databasePath, target) { const db = new DatabaseSync(databasePath); try { db.prepare('VACUUM INTO ?').run(target); } finally { db.close(); } } function inspect(copyPath) { const db = new DatabaseSync(copyPath, { readOnly: true }); try { const integrity = db.prepare('PRAGMA integrity_check').get()?.integrity_check ?? 'unknown'; const foreignKeys = db.prepare('PRAGMA foreign_key_check').all().length; return { integrity, foreignKeyProblems: foreignKeys, counts: Object.fromEntries( BACKUP_CONTENTS.map((entry) => [ entry.key, entry.where ? countWhere(db, entry.table, entry.where) : countRows(db, entry.table) ]) ) }; } finally { db.close(); } } /** Live rows in a table, or null when the table is missing. */ function countRows(db, table) { try { const columns = db .prepare(`pragma table_info(${table})`) .all() .map((column) => column.name); const live = columns.includes('deleted_at') ? ' where deleted_at is null' : ''; return db.prepare(`select count(*) as n from ${table}${live}`).get().n; } catch { return null; } } /** Rows that match a condition, or null when the table is missing. */ function countWhere(db, table, condition) { try { return db.prepare(`select count(*) as n from ${table} where ${condition}`).get().n; } catch { return null; } } function sha256Of(path) { return createHash('sha256').update(readFileSync(path)).digest('hex'); } /** `2026-10-26T20-15-00Z`: sorts in a listing and survives every filesystem. */ function stampOf(date) { return date .toISOString() .replace(/\.\d{3}Z$/, 'Z') .replace(/:/g, '-'); } function buildIdentity() { let version = 'unknown'; try { version = JSON.parse(readFileSync(join(APP_ROOT, 'package.json'), 'utf8')).version ?? version; } catch { // A build without its manifest still backs up. It cannot name its version. } return { version, commit: process.env.APP_COMMIT ?? 'unknown' }; } function describeBuild(build) { return `${build.version} (${build.commit})`; } /** Where the instance lives: flags first, then the environment. */ function locate(flags) { const databasePath = flags.database ?? resolveDatabasePath(process.env.DATABASE_URL) ?? (process.env.DATA_DIR ? join(process.env.DATA_DIR, DATABASE_FILE) : null); if (!databasePath) { throw new Error('No database. Pass --database, or set DATABASE_URL or DATA_DIR.'); } const dataDir = flags['data-dir'] ?? dirname(databasePath); const outDir = flags.out ?? join(dataDir, 'backups'); return { databasePath, dataDir, outDir }; } /** * Looser than `locate`: a drill on a clean machine runs before that machine has * an instance, so `--data-dir` alone is enough. With no source at all, the * working directory holds the record. */ function locateDrill(flags) { const databasePath = flags.database ?? resolveDatabasePath(process.env.DATABASE_URL) ?? (process.env.DATA_DIR ? join(process.env.DATA_DIR, DATABASE_FILE) : null); const dataDir = flags['data-dir'] ?? (databasePath ? dirname(databasePath) : process.cwd()); return { databasePath, dataDir, outDir: flags.out ?? join(dataDir, 'backups') }; } /** * The copy is built in a hidden staging directory and renamed into place, so a * backup that fails halfway never wears the name of one that worked. * * A `databaseOnly` backup is for a deployment that copies the rest of the data * directory itself. It is consistent, but not restorable on its own. */ export function takeBackup(options = {}) { const databasePath = options.databasePath; const dataDir = options.dataDir ?? dirname(databasePath); const outDir = options.outDir ?? join(dataDir, 'backups'); const keep = options.keep ?? 8; const databaseOnly = options.databaseOnly === true; const now = options.now ?? new Date(); if (!databasePath) { throw new Error('No database. Pass --database, or set DATABASE_URL or DATA_DIR.'); } if (!existsSync(databasePath)) { throw new Error(`No database at ${databasePath}.`); } mkdirSync(outDir, { recursive: true }); const build = buildIdentity(); const staging = mkdtempSync(join(outDir, '.staging-')); const copyPath = join(staging, DATABASE_FILE); try { vacuumInto(databasePath, copyPath); const report = inspect(copyPath); if (report.integrity !== 'ok' || report.foreignKeyProblems > 0) { throw new Error( `The copy failed its own integrity test: ${report.integrity}, ${report.foreignKeyProblems} foreign key problems.` ); } const secretPath = join(dataDir, SECRET_FILE); const secret = databaseOnly ? false : existsSync(secretPath); if (secret) { copyFileSync(secretPath, join(staging, SECRET_FILE)); chmodSync(join(staging, SECRET_FILE), 0o600); } const uploads = databaseOnly ? null : copyUploads(join(dataDir, UPLOADS_DIR), join(staging, UPLOADS_DIR)); const state = readDbState({ databasePath: copyPath, migrationsFolder: MIGRATIONS_FOLDER }); const manifest = { format: FORMAT, createdAt: now.toISOString(), madeBy: describeBuild(build), source: databasePath, database: { file: DATABASE_FILE, bytes: statSync(copyPath).size, sha256: sha256Of(copyPath) }, secret, uploads, schema: { applied: state.applied.length, newest: state.applied.at(-1) ?? null, lastVersion: state.version, lastOpenedAt: state.openedAt }, integrity: report.integrity, contents: report.counts }; writeFileSync(join(staging, MANIFEST_FILE), `${JSON.stringify(manifest, null, '\t')}\n`); const target = uniqueDirectory(outDir, `fablewright-${stampOf(now)}-v${build.version}`); renameSync(staging, target); return { dir: target, manifest, pruned: prune(outDir, keep, target) }; } catch (error) { rmSync(staging, { recursive: true, force: true }); throw error; } } /** Null when there is no uploads directory, or it is empty. */ function copyUploads(source, target) { if (!existsSync(source) || !statSync(source).isDirectory()) return null; // The instance's own uploads directory may be a mount, so the walk starts at its // contents. Anything below it has to be a regular file or a directory, because a // link copied into a backup is a link every later reader follows. const walk = walkTree(source); if (walk.problems.length > 0) { throw new Error(`The uploads directory cannot be backed up:\n ${walk.problems.join('\n ')}`); } if (walk.files.length === 0) return null; mkdirSync(target, { recursive: true }); for (const relative of walk.directories) { mkdirSync(join(target, relative), { recursive: true }); } let bytes = 0; for (const file of walk.files) { copyRegularFile(file.path, join(target, file.relative), 0o644); bytes += file.bytes; } return { files: walk.files.length, bytes }; } /** Every file below a directory, so a nested folder counts as its contents. */ function countFiles(path) { return readdirSync(path, { withFileTypes: true }).reduce( (total, entry) => total + (entry.isDirectory() ? countFiles(join(path, entry.name)) : 1), 0 ); } function directorySize(path) { return readdirSync(path, { withFileTypes: true }).reduce( (total, entry) => total + (entry.isDirectory() ? directorySize(join(path, entry.name)) : statSync(join(path, entry.name)).size), 0 ); } /** * Opens without following a link at the end of the path. `lstat` then copy leaves * a window in which a link can be swapped in, and a crafted backup reaching * outside its own directory is exactly what that window is worth to an attacker. */ function openRefusingLinks(path, flags, mode) { try { return openSync(path, flags | NO_FOLLOW, mode); } catch (error) { if (error?.code === 'ELOOP') { throw new Error(`${path} is a link, and this tool does not follow links.`, { cause: error }); } throw error; } } /** * Copies one regular file, from an open handle so nothing can change underneath * the check. `O_NONBLOCK` keeps a FIFO from blocking the open before it can be * refused, which is what a crafted backup would rather have than a copy. */ function copyRegularFile(source, target, mode) { const input = openRefusingLinks(source, fsConstants.O_RDONLY | fsConstants.O_NONBLOCK); try { if (!fstatSync(input).isFile()) { throw new Error(`${source} is not a regular file, so it is not copied.`); } const output = openRefusingLinks(target, CREATE_FLAGS, mode); try { const buffer = Buffer.allocUnsafe(COPY_CHUNK_BYTES); for (;;) { const read = readSync(input, buffer, 0, COPY_CHUNK_BYTES, null); if (read === 0) break; writeSync(output, buffer, 0, read); } } finally { closeSync(output); } } finally { closeSync(input); } } function lstatIfAny(path) { try { return lstatSync(path); } catch { return null; } } /** * A symbol link where a directory belongs makes every name below it a path into * somebody else's tree. A restore writes there and a sweep deletes what it finds * there, so neither starts until the path is known to be a real directory. A path * that is absent is fine: a restore creates it, and a sweep has nothing to read. */ function assertRealDirectory(path) { const stats = lstatIfAny(path); if (stats !== null && !stats.isDirectory()) { throw new Error( `${path} is not a directory. Nothing here follows a link, so look at it before you run this again.` ); } } /** * Every entry below a directory, read with `lstat` so a link is seen as a link and * never as its target. Only regular files and directories pass: a backup that * carries a FIFO makes a reader wait forever, and one that carries a link makes * every later reader reach outside the directory it claims to hold. * * Both halves of the tool walk with this, so what `verify` refuses is exactly * what `restore` will not copy. */ function walkTree(root) { const files = []; const directories = []; const problems = []; const visit = (absolute, relative) => { const stats = lstatIfAny(absolute); if (stats === null) { problems.push(`${relative} could not be read.`); return; } if (stats.isSymbolicLink()) { problems.push(`${relative} is a link. A backup holds files, not links.`); return; } if (stats.isDirectory()) { directories.push(relative); for (const name of readdirSync(absolute).sort()) { visit(join(absolute, name), `${relative}/${name}`); } return; } if (!stats.isFile()) { problems.push(`${relative} is not a file or a directory.`); return; } files.push({ path: absolute, relative, bytes: stats.size }); }; const topLevel = readdirSync(root).sort(); for (const name of topLevel) visit(join(root, name), name); return { files, directories, topLevel, problems }; } /** A value as it appears in JSON, for a message about a field the tool refuses. */ function shown(value) { if (value === undefined) return 'nothing'; return JSON.stringify(value) ?? String(value); } /** A name no earlier backup holds, so two backups in one second both survive. */ function uniqueDirectory(outDir, name) { let candidate = join(outDir, name); let suffix = 2; while (existsSync(candidate)) { candidate = join(outDir, `${name}-${suffix}`); suffix += 1; } return candidate; } /** Removes the oldest backups above `keep`, never the new one. */ function prune(outDir, keep, current) { if (keep <= 0) return []; const existing = readdirSync(outDir) .filter((name) => DIRECTORY_PATTERN.test(name)) .sort(); const excess = existing.length - keep; if (excess <= 0) return []; const removed = existing.filter((name) => name !== basename(current)).slice(0, excess); for (const name of removed) rmSync(join(outDir, name), { recursive: true, force: true }); return removed; } /** * The manifest is data from a directory an operator hands the tool, so no value in * it becomes a path and nothing else on disk is read until its shape is known. A * field that can be absent is a check that can be skipped, so every field the * writer writes is required here, and a digest the manifest does not name is a * failure rather than a comparison that does not happen. * * Every problem is collected, so one run names all of them. */ function readManifest(dir) { const problems = []; const manifestPath = join(dir, MANIFEST_FILE); const stats = lstatIfAny(manifestPath); if (stats === null) { return { manifest: null, problems: ['The backup has no manifest.json, so nothing can be compared against it.'] }; } if (!stats.isFile()) { return { manifest: null, problems: [`${MANIFEST_FILE} is not a regular file.`] }; } let manifest; try { manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); } catch { return { manifest: null, problems: ['manifest.json is not valid JSON.'] }; } if (typeof manifest !== 'object' || manifest === null || Array.isArray(manifest)) { return { manifest: null, problems: ['manifest.json does not hold a JSON object.'] }; } const count = (value) => Number.isInteger(value) && value >= 0; // A table the writer's build did not have is recorded as null, not as zero, so // a row count is either a number or null and never absent. const counted = (value) => count(value) || value === null; const database = manifest.database; const uploads = manifest.uploads; const schema = manifest.schema; const contents = manifest.contents; if (manifest.format !== FORMAT) { problems.push(`The manifest is format ${shown(manifest.format)}, not format ${FORMAT}.`); } if (typeof manifest.createdAt !== 'string' || Number.isNaN(Date.parse(manifest.createdAt))) { problems.push('The manifest does not record when the backup was made.'); } if (typeof database !== 'object' || database === null) { problems.push('The manifest does not describe a database file.'); } else { if (database.file !== DATABASE_FILE) { problems.push( `The manifest names ${shown(database.file)} as the database, not ${DATABASE_FILE}.` ); } if (!count(database.bytes)) { problems.push('The manifest does not record the size of the database.'); } if (typeof database.sha256 !== 'string' || !SHA256_HEX.test(database.sha256)) { problems.push( 'The manifest does not record a database digest, so the file cannot be checked.' ); } } if (typeof manifest.secret !== 'boolean') { problems.push('The manifest does not say whether the backup holds the signing secret.'); } if ( uploads !== null && (typeof uploads !== 'object' || !count(uploads?.files) || !count(uploads?.bytes)) ) { problems.push('The manifest does not describe the uploads the backup holds.'); } if (manifest.integrity !== 'ok') { problems.push('The manifest does not record a clean integrity check.'); } if (typeof schema !== 'object' || schema === null || !count(schema.applied)) { problems.push('The manifest does not record which migrations the database had.'); } if (typeof contents !== 'object' || contents === null || Array.isArray(contents)) { problems.push('The manifest does not record what the database holds.'); } else { const uncounted = BACKUP_CONTENTS.filter(({ key }) => !counted(contents[key])); if (uncounted.length > 0) { problems.push( `The manifest does not record how many ${uncounted.map(({ noun }) => noun).join(', ')} the database holds.` ); } } return { manifest: problems.length === 0 ? manifest : null, problems }; } /** * Problems are collected rather than thrown, so one run tells an operator * everything wrong with the file they are about to restore. */ export function verifyBackup(dir) { const problems = []; const report = { ok: false, problems, manifest: null, database: null, schema: null, contents: null, integrity: null, foreignKeyProblems: null, secret: false, uploads: null }; // The backup directory is named by the operator, so a link to it is the // operator's own choice. Everything below it is walked without following one. if (!dir || !existsSync(dir) || !statSync(dir).isDirectory()) { problems.push(`No backup directory at ${dir}.`); return report; } const read = readManifest(dir); problems.push(...read.problems); if (read.manifest === null) return report; report.manifest = read.manifest; const walk = walkTree(dir); const byName = new Map(walk.files.map((file) => [file.relative, file])); problems.push(...walk.problems); for (const name of walk.topLevel) { if (!BACKUP_ENTRIES.has(name)) { problems.push(`${name} is not part of a backup. Nothing removed it, so this is not one.`); } } const database = byName.get(DATABASE_FILE); if (!database) { problems.push(`The backup holds no database file named ${DATABASE_FILE}.`); } else { const digest = sha256Of(database.path); report.database = { path: database.path, bytes: database.bytes, sha256: digest }; if (digest !== read.manifest.database.sha256) { problems.push( 'The database digest does not match the manifest. The file changed after it was written.' ); } if (database.bytes !== read.manifest.database.bytes) { problems.push( `The database holds ${sizeText(database.bytes)}, but the manifest says ${sizeText(read.manifest.database.bytes)}.` ); } try { const inspected = inspect(database.path); report.contents = inspected.counts; report.integrity = inspected.integrity; report.foreignKeyProblems = inspected.foreignKeyProblems; if (inspected.integrity !== 'ok') { problems.push(`SQLite reports the file as ${inspected.integrity}.`); } if (inspected.foreignKeyProblems > 0) { problems.push(`${inspected.foreignKeyProblems} rows point at a row that is not there.`); } } catch (error) { problems.push(`The database could not be opened: ${messageOf(error)}`); } const state = readDbState({ databasePath: database.path, migrationsFolder: MIGRATIONS_FOLDER }); report.schema = state; // A read that reported an error answers "no migrations" too, so only the // error is worth naming in that case. if (state.error !== null) { problems.push(`The migrations could not be read: ${state.error}`); } else if (state.applied.length === 0) { problems.push('The database has no migrations applied, so it is not one this build wrote.'); } } if (read.manifest.secret) { if (byName.has(SECRET_FILE)) { report.secret = true; } else { problems.push( 'The manifest says the backup holds the signing secret, but there is no auth-secret file.' ); } } else if (byName.has(SECRET_FILE)) { problems.push('The backup holds an auth-secret file the manifest does not name.'); } /* * Uploads are the half of a backup SQLite cannot check: a copy that lost the * directory still opens, and every picture is gone. The walk counted them, and * the manifest has to agree in both directions: a directory it does not name is * a directory `restore` was never told about. */ const uploadsPresent = walk.directories.includes(UPLOADS_DIR); const uploadFiles = walk.files.filter((file) => file.relative.startsWith(`${UPLOADS_DIR}/`)); if (read.manifest.uploads === null) { if (uploadsPresent) { problems.push( 'The manifest says the backup holds no uploads, but there is an uploads directory.' ); } } else if (!uploadsPresent) { problems.push( `The manifest says it holds ${plural(read.manifest.uploads.files, 'file')}, but there is no uploads directory.` ); } else { const bytes = uploadFiles.reduce((total, file) => total + file.bytes, 0); report.uploads = { files: uploadFiles.length, bytes }; if ( report.uploads.files !== read.manifest.uploads.files || bytes !== read.manifest.uploads.bytes ) { problems.push( `The uploads directory holds ${plural(report.uploads.files, 'file')} (${report.uploads.bytes} bytes), but the manifest says ${plural(read.manifest.uploads.files, 'file')} (${read.manifest.uploads.bytes} bytes).` ); } } report.ok = problems.length === 0; return report; } /** * The database it replaces is renamed, never deleted, and its journal and shared * memory file move with it: a journal beside a newly copied database belongs to * a different file, and SQLite applies it. */ export function restoreBackup(options = {}) { const databasePath = options.databasePath; const dataDir = options.dataDir ?? dirname(databasePath); const dir = options.dir; const report = verifyBackup(dir); if (!report.ok) { throw new Error(`The backup failed its own check:\n ${report.problems.join('\n ')}`); } // The walk is taken again here, and it is the one the copy follows, so what is // written is the tree that was just read. A link is refused rather than skipped: // a backup that gained one between the check and the copy is not the backup the // operator verified. const walk = walkTree(dir); if (walk.problems.length > 0) { throw new Error( `The backup changed while it was being checked:\n ${walk.problems.join('\n ')}` ); } const uploadDirectories = walk.directories.filter( (relative) => relative === UPLOADS_DIR || relative.startsWith(`${UPLOADS_DIR}/`) ); const uploadFiles = walk.files.filter((file) => file.relative.startsWith(`${UPLOADS_DIR}/`)); // The instance is checked before anything in it is replaced. A restore that has // already moved the database aside cannot be undone by refusing the next step, // and a link where a directory belongs sends every file written below it // somewhere else. for (const relative of uploadDirectories) assertRealDirectory(join(dataDir, relative)); mkdirSync(dataDir, { recursive: true }); const previousStamp = stampOf(options.now ?? new Date()); const previous = `${databasePath}.pre-restore-${previousStamp}`; let kept = null; if (existsSync(databasePath)) { renameSync(databasePath, previous); kept = previous; for (const suffix of ['-wal', '-shm']) { if (existsSync(databasePath + suffix)) renameSync(databasePath + suffix, previous + suffix); } } copyRegularFile(join(dir, DATABASE_FILE), databasePath, 0o644); if (report.secret) { copyRegularFile(join(dir, SECRET_FILE), join(dataDir, SECRET_FILE), 0o600); } let restoredUploads = null; if (uploadDirectories.length > 0) { for (const relative of uploadDirectories) { mkdirSync(join(dataDir, relative), { recursive: true }); } for (const file of uploadFiles) { copyRegularFile(file.path, join(dataDir, file.relative), 0o644); } restoredUploads = { files: uploadFiles.length, bytes: uploadFiles.reduce((total, file) => total + file.bytes, 0) }; } return { databasePath, previous: kept, secret: report.secret, uploads: restoredUploads }; } function messageOf(error) { return error instanceof Error ? error.message : String(error); } /** `3 scenes`, `1 scene`. */ function plural(count, noun) { return `${count} ${noun}${count === 1 ? '' : 's'}`; } /** Digits grouped by three, the same in every locale and container. */ function thousands(value) { return String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ','); } function describeCounts(counts) { return BACKUP_CONTENTS.filter( ({ key, resting }) => typeof counts?.[key] === 'number' && (counts[key] > 0 || resting) ) .map(({ key, noun }) => plural(counts[key], noun)) .join(', '); } /** Scene and word counts, so a check can tell a full restore from an empty one. */ function readProse(db) { try { const row = db .prepare( `select count(*) as scenes, coalesce(sum(word_count), 0) as words, sum(case when word_count > 0 and trim(body_text) = '' then 1 else 0 end) as empty from scene where deleted_at is null` ) .get(); return { scenes: Number(row.scenes), words: Number(row.words), empty: Number(row.empty ?? 0) }; } catch { return null; } } function readAccounts(db) { try { return { users: Number(db.prepare('select count(*) as n from user').get().n), passwords: Number( db .prepare( "select count(distinct user_id) as n from account where provider_id = 'credential' and password is not null and length(password) > 0" ) .get().n ), sessions: Number(db.prepare('select count(*) as n from session').get().n) }; } catch { return null; } } /** Every upload file the copy names: note images, and the files writers imported. */ function readUploadReferences(db) { const images = []; const imports = []; try { const rows = db .prepare('select image_id, bytes from note_image where image_id is not null') .all(); for (const row of rows) images.push({ imageId: String(row.image_id), bytes: Number(row.bytes ?? 0) }); } catch { // A database from before note images existed names no picture. } try { const rows = db .prepare('select source_id from import_report where source_id is not null') .all(); for (const row of rows) imports.push(String(row.source_id)); } catch { // A database from before import reports existed names no file. } return { images, imports }; } function appendDrillRecord(dataDir, record) { const path = join(dataDir, DRILLS_FILE); mkdirSync(dataDir, { recursive: true }); appendFileSync(path, `${JSON.stringify(record)}\n`); return path; } /** * Restores a backup into a directory nobody serves, then reads it back, so a * backup that has never been restored stops being a hope. * * Every check reads the copy, never the instance: a drill that read the live * database would pass on the morning the backup stopped being restorable. */ export function drillRestore(options = {}) { const started = Date.now(); const dataDir = options.dataDir; if (!dataDir) throw new Error('No data directory. Pass --data-dir, or set DATABASE_URL or DATA_DIR.'); const outDir = options.outDir ?? join(dataDir, 'backups'); const scratchRoot = options.scratchDir ?? tmpdir(); const keep = options.keep ?? 8; const now = options.now ?? new Date(); const checks = []; const problems = []; const check = (id, label, ok, detail) => { checks.push({ id, label, ok, detail }); if (!ok) problems.push(`${label}: ${detail}`); }; const tookBackup = options.backupDir === undefined; let backup = null; let restored = null; try { const dir = options.backupDir ?? takeBackup({ databasePath: options.databasePath, dataDir, outDir, keep, now }).dir; const report = verifyBackup(dir); const manifest = report.manifest; backup = { dir, createdAt: manifest?.createdAt ?? null, madeBy: manifest?.madeBy ?? null, sha256: manifest?.database?.sha256 ?? null, applied: manifest?.schema?.applied ?? null, newest: manifest?.schema?.newest ?? null, lastVersion: manifest?.schema?.lastVersion ?? null }; if (!report.ok) { check('backup', 'The backup', false, report.problems.join(' ')); } else { check( 'backup', 'The backup', true, `sound, made ${manifest.createdAt} by ${manifest.madeBy}` ); // An empty directory is what a clean server has before a restore. Nothing // here is served. const scratch = mkdtempSync(join(scratchRoot, 'fablewright-drill-')); try { const written = restoreBackup({ dir, databasePath: join(scratch, DATABASE_FILE), dataDir: scratch }); const restoredPath = written.databasePath; const copy = inspect(restoredPath); const bytes = statSync(restoredPath).size; restored = { bytes, uploads: written.uploads }; const intact = copy.integrity === 'ok' && copy.foreignKeyProblems === 0; check( 'database', 'Database', intact, intact ? `integrity ok, no foreign key problems, ${megabytes(bytes)}` : `${copy.integrity} from SQLite, ${plural(copy.foreignKeyProblems, 'foreign key problem')}` ); const state = readDbState({ databasePath: restoredPath, migrationsFolder: MIGRATIONS_FOLDER }); const unexpected = unexpectedMigrations(state); const sameLevel = state.applied.length === backup.applied && (state.applied.at(-1) ?? null) === backup.newest; check( 'schema', 'Schema', state.error === null && sameLevel && unexpected.length === 0, state.error !== null ? `the migrations could not be read: ${state.error}` : unexpected.length > 0 ? `${plural(unexpected.length, 'migration')} this build does not ship: ${unexpected.join(', ')}` : sameLevel ? `${plural(state.applied.length, 'migration')}, newest ${state.applied.at(-1) ?? 'none'}` : `${backup.applied ?? '?'} migrations in the backup, ${state.applied.length} after the restore` ); // The manifest is required to count every table, so a count the copy does // not match is a row the restore lost. const recorded = manifest.contents; const wrong = BACKUP_CONTENTS.filter(({ key }) => copy.counts[key] !== recorded[key]); check( 'contents', 'What it holds', wrong.length === 0, wrong.length === 0 ? describeCounts(copy.counts) : wrong .map( ({ key, noun }) => `${plural(recorded[key], noun)} in the backup, ${copy.counts[key] ?? 'nothing'} after the restore` ) .join('; ') ); const db = new DatabaseSync(restoredPath, { readOnly: true }); let prose = null; let accounts = null; let references = null; try { prose = readProse(db); accounts = readAccounts(db); references = readUploadReferences(db); } finally { db.close(); } check( 'words', 'Words', prose !== null && prose.empty === 0, prose === null ? 'the copy has no scene table' : prose.empty === 0 ? `${thousands(prose.words)} words in ${plural(prose.scenes, 'scene')}` : `${plural(prose.empty, 'scene')} count words and hold no text` ); const secretFile = join(scratch, SECRET_FILE); const held = manifest.secret === true && existsSync(secretFile); const secret = held ? readFileSync(secretFile) : null; const sameSecret = secret !== null && secret.equals(readFileSync(join(dir, SECRET_FILE))); // A deployment may keep the secret in the environment instead of the // data directory. The backup then holds none, and a host that still sets // BETTER_AUTH_SECRET restores, which is worth more than a failure. const fromEnvironment = !held && (process.env.BETTER_AUTH_SECRET ?? '').length > 0; check( 'secret', 'Signing secret', sameSecret || fromEnvironment, sameSecret ? `restored, ${plural(secret.length, 'byte')}, the same as the backup` : fromEnvironment ? 'not in the backup. This host sets BETTER_AUTH_SECRET, so a restore needs that variable' : held ? 'the restored secret differs from the one in the backup, so every session ends' : 'the backup holds no signing secret, so every session ends' ); check( 'accounts', 'Accounts', accounts !== null && (accounts.users === 0 || accounts.passwords > 0), accounts === null ? 'the copy has no account table' : accounts.users === 0 ? 'no accounts yet' : accounts.passwords > 0 ? [ `${plural(accounts.passwords, 'account')} with a password`, accounts.sessions > 0 ? `${plural(accounts.sessions, 'session')} kept` : null ] .filter(Boolean) .join(', ') : `${plural(accounts.users, 'account')}, and none of them holds a password` ); // A picture comes back with the database, so a backup that never held // one is the failure worth naming. A file the database names but the // backup does not was already gone when the backup was taken. const uploadsDir = join(scratch, UPLOADS_DIR); const present = existsSync(uploadsDir); const files = present ? countFiles(uploadsDir) : 0; const fileBytes = present ? directorySize(uploadsDir) : 0; const named = (references?.images.length ?? 0) + (references?.imports.length ?? 0); let missing = 0; for (const image of references?.images ?? []) { if (!existsSync(join(uploadsDir, `${image.imageId}.webp`))) missing += 1; else if (image.bytes > 0 && !existsSync(join(uploadsDir, `${image.imageId}.orig`))) { missing += 1; } } for (const source of references?.imports ?? []) { if (!existsSync(join(uploadsDir, IMPORTS_DIR, source))) missing += 1; } const claimed = manifest.uploads ?? null; const uploadsMatch = claimed ? files === claimed.files && fileBytes === claimed.bytes : named === 0; const uploadsDetail = !uploadsMatch ? claimed ? `${plural(claimed.files, 'file')} in the backup, ${plural(files, 'file')} after the restore` : `the backup holds no uploads, but the database names ${plural(named, 'file')}` : [ claimed ? `${plural(files, 'file')} (${sizeText(fileBytes)}) restored` : 'none to restore', named > 0 && missing === 0 ? 'every file the database names is here' : null, missing > 0 ? `${plural(missing, 'file')} the database names were already missing from the backup` : null ] .filter(Boolean) .join('; '); check('uploads', 'Images and files', uploadsMatch, uploadsDetail); } finally { rmSync(scratch, { recursive: true, force: true }); } } } catch (error) { // A backup that never existed is a command that did not run, not a restore // that failed. It throws rather than record a drill that never happened. if (backup === null) throw error; // A throw after the backup is a failed drill, so it gets a check of its own // and the passed count cannot add up to the total. check('error', 'The drill', false, messageOf(error)); } const record = { format: DRILL_FORMAT, at: now.toISOString(), ok: problems.length === 0, madeBy: describeBuild(buildIdentity()), // A backup given by `--file` came from an instance this machine cannot name. source: tookBackup ? (options.databasePath ?? null) : null, backup, restored, checks, problems, durationMs: Date.now() - started }; let recordPath = null; let recordError = null; try { recordPath = appendDrillRecord(dataDir, record); } catch (error) { recordError = messageOf(error); } return { ...record, recordPath, recordError }; } /** * better-auth's parameters, not preferences: N 16384, r 16, p 1, a 64-byte key, * and a hex salt before a colon. `account-recovery.spec.ts` proves the format * against the library, so an upstream change fails the suite rather than an * operator's rescue. A recovery needs no build, browser, session or mailer. */ const SCRYPT = { N: 16384, r: 16, p: 1, dkLen: 64 }; /** The admin screen's alphabet, so a copied password is not misread. */ const PASSWORD_ALPHABET = 'abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'; export function hashPassword(password) { const salt = randomBytes(16).toString('hex'); const key = scryptSync(password.normalize('NFKC'), salt, SCRYPT.dkLen, { N: SCRYPT.N, r: SCRYPT.r, p: SCRYPT.p, maxmem: 128 * SCRYPT.N * SCRYPT.r * 2 }); return `${salt}:${key.toString('hex')}`; } export function generatePassword(length = 16) { const limit = 256 - (256 % PASSWORD_ALPHABET.length); let password = ''; while (password.length < length) { for (const byte of randomBytes(length * 2)) { if (byte >= limit) continue; password += PASSWORD_ALPHABET[byte % PASSWORD_ALPHABET.length]; if (password.length === length) break; } } return password; } /** * Sets the password and ends every session for the account. A new password * alone leaves anyone already signed in still signed in, which defeats the * recovery. */ export function resetPassword({ databasePath, email, password }) { if (!existsSync(databasePath)) { throw new Error(`There is no database at ${databasePath}.`); } const db = new DatabaseSync(databasePath); try { const user = db.prepare('select id, email from user where email = ?').get(email.toLowerCase()); if (!user) { throw new Error(`No account uses ${email}.`); } const now = Date.now(); const hash = hashPassword(password); db.exec('begin'); try { const updated = db .prepare( "update account set password = ?, updated_at = ? where user_id = ? and provider_id = 'credential'" ) .run(hash, now, user.id); // An account made by another provider has no password row to update. if (updated.changes === 0) { db.prepare( `insert into account (id, account_id, provider_id, user_id, password, created_at, updated_at) values (?, ?, 'credential', ?, ?, ?, ?)` ).run(randomBytes(16).toString('hex'), user.id, user.id, hash, now, now); } const sessions = db.prepare('delete from session where user_id = ?').run(user.id); db.prepare('update user set updated_at = ? where id = ?').run(now, user.id); db.exec('commit'); return { email: user.email, sessions: sessions.changes }; } catch (error) { db.exec('rollback'); throw error; } } finally { db.close(); } } /** * Images sit at the top level, named by image id and an extension; the files * writers imported sit in `uploads/imports/`. Without `apply` it only counts. */ export function pruneUploads(options = {}) { const databasePath = options.databasePath; const dataDir = options.dataDir ?? dirname(databasePath); const apply = options.apply === true; const graceMs = options.graceMs ?? SWEEP_GRACE_MS; const uploadsDir = join(dataDir, UPLOADS_DIR); if (!existsSync(databasePath)) { throw new Error(`There is no database at ${databasePath}.`); } // One instant answers both questions. Every file is judged against the // reference set read at this moment, so a row that commits later can only name // a file the grace window still protects. const now = Date.now(); const db = new DatabaseSync(databasePath, { readOnly: true }); const referencedIds = new Set(); const referencedSources = new Set(); try { if (hasTable(db, 'note_image')) { const rows = db.prepare('select image_id from note_image where image_id is not null').all(); for (const row of rows) { if (row.image_id) referencedIds.add(row.image_id); } } // A database from before the import report existed has no such table. if (hasTable(db, 'import_report')) { const rows = db .prepare('select source_id from import_report where source_id is not null') .all(); for (const row of rows) { if (row.source_id) referencedSources.add(row.source_id); } } } finally { db.close(); } const result = { totalFiles: 0, totalBytes: 0, referencedFiles: 0, unreferencedFiles: 0, unreferencedBytes: 0, recentFiles: 0, recentBytes: 0, removedFiles: 0, removedBytes: 0, failedFiles: [] }; if (!existsSync(uploadsDir)) { return result; } assertRealDirectory(uploadsDir); // One decision for a file with no row behind it, wherever the file lives. const sweep = (filePath, stat, isReferenced) => { result.totalFiles += 1; result.totalBytes += stat.size; if (isReferenced) { result.referencedFiles += 1; return; } result.unreferencedFiles += 1; result.unreferencedBytes += stat.size; if (insideGraceWindow(stat, now, graceMs)) { result.recentFiles += 1; result.recentBytes += stat.size; return; } if (!apply) return; try { rmSync(filePath, { force: true }); } catch (error) { // A file the tool cannot delete stays, and is reported rather than // ending the sweep: the rest of the reclaim is still worth having. result.failedFiles.push(`${filePath}: ${messageOf(error)}`); return; } result.removedFiles += 1; result.removedBytes += stat.size; }; for (const name of readdirSync(uploadsDir)) { const filePath = join(uploadsDir, name); let stat; try { stat = statSync(filePath); } catch { continue; } if (!stat.isFile()) continue; const dot = name.indexOf('.'); const prefix = dot > 0 ? name.slice(0, dot) : name; sweep(filePath, stat, referencedIds.has(prefix)); } // An import source is named `.` and the row holds the whole // name, so the file cannot be matched without its extension. const importsDir = join(uploadsDir, IMPORTS_DIR); if (existsSync(importsDir) && statSync(importsDir).isDirectory()) { for (const name of readdirSync(importsDir)) { const filePath = join(importsDir, name); let stat; try { stat = statSync(filePath); } catch { continue; } if (!stat.isFile()) continue; sweep(filePath, stat, referencedSources.has(name)); } } return result; } /** * `apply` drops the `.orig` copies: each file goes first, then the row that claims * it, so a row saying `bytes > 0` never outlives its file. The display copy has * to be there first, because without it the original is the only copy left. */ export function dropOriginals(options = {}) { const databasePath = options.databasePath; const dataDir = options.dataDir ?? dirname(databasePath); const apply = options.apply === true; const graceMs = options.graceMs ?? SWEEP_GRACE_MS; const uploadsDir = join(dataDir, UPLOADS_DIR); if (!existsSync(databasePath)) { throw new Error(`There is no database at ${databasePath}.`); } // A sweep deletes what it finds, so it refuses a tree it did not write. assertRealDirectory(uploadsDir); const now = Date.now(); const result = { totalOriginals: 0, totalBytes: 0, droppableOriginals: 0, droppableBytes: 0, droppedOriginals: 0, droppedBytes: 0, keptOriginals: 0, recentOriginals: 0, missingOriginals: 0, failedFiles: [] }; // A dry run opens the database read-only, so it cannot write by accident. const db = new DatabaseSync(databasePath, { readOnly: !apply }); try { if (!hasTable(db, 'note_image')) return result; const clearBytes = apply ? db.prepare('update note_image set bytes = 0 where image_id = ? and bytes > 0') : null; // Every image id whose original is still on disk, so the row that names it is // not blamed for a claim the disk has kept. const onDisk = new Set(); if (existsSync(uploadsDir) && lstatSync(uploadsDir).isDirectory()) { for (const name of readdirSync(uploadsDir)) { if (!name.endsWith('.orig')) continue; const imageId = name.slice(0, -'.orig'.length); const filePath = join(uploadsDir, name); let stat; try { stat = statSync(filePath); } catch { // A file listed but already gone is not a problem to report. continue; } if (!stat.isFile()) continue; result.totalOriginals += 1; result.totalBytes += stat.size; onDisk.add(imageId); if (insideGraceWindow(stat, now, graceMs)) { result.recentOriginals += 1; continue; } // The display copy is renamed into place before the original, so its // presence is what makes the original a duplicate rather than the only copy. let display = null; try { display = statSync(join(uploadsDir, `${imageId}.webp`)); } catch { display = null; } if (!display?.isFile() || display.size === 0) { result.keptOriginals += 1; continue; } result.droppableOriginals += 1; result.droppableBytes += stat.size; if (!apply) continue; try { rmSync(filePath, { force: true }); } catch (error) { // The row keeps its claim, so the original stays reachable. result.failedFiles.push(`${filePath}: ${messageOf(error)}`); continue; } clearBytes.run(imageId); result.droppedOriginals += 1; result.droppedBytes += stat.size; } } // A row that claims a full-size copy the disk has already lost. Clearing the // claim is safe only when the display copy survived; otherwise the row keeps // the evidence and the backup drill keeps reporting it. const claimed = db .prepare('select image_id from note_image where bytes > 0 and image_id is not null') .all(); for (const row of claimed) { if (onDisk.has(row.image_id)) continue; let display = null; try { display = statSync(join(uploadsDir, `${row.image_id}.webp`)); } catch { display = null; } if (!display?.isFile() || display.size === 0) continue; result.missingOriginals += 1; if (apply) clearBytes.run(row.image_id); } return result; } finally { db.close(); } } /** * `saved` is the condition that protects a version the writer asked for. A * version the app took on its own has none, and expires by age alone. * * `source = 'manual'` is that mark, never a label. The app writes labels itself * (a guest edit, a conflict, a restore), so reading a label as a bookmark let a * guest link grow one scene's history for ever. */ function historyDays(db) { if (!hasTable(db, 'instance_setting')) return DEFAULT_RETENTION_DAYS; const columns = db.prepare('pragma table_info(instance_setting)').all(); if (!columns.some((column) => column.name === 'history_retention_days')) { return DEFAULT_RETENTION_DAYS; } const row = db .prepare('select history_retention_days as days from instance_setting where id = 1') .get(); return clampRetentionDays(row?.days); } /** * Runs the shared retention policy from `history-retention.mjs` directly against * SQLite while the server is stopped. Without `apply`, it reports eligible * counts without deleting rows. */ export function pruneHistory(options = {}) { const databasePath = options.databasePath; const apply = options.apply === true; const includeKept = options.includeKept === true; const vacuum = options.vacuum === true; if (!existsSync(databasePath)) { throw new Error(`There is no database at ${databasePath}.`); } const bytesBefore = statSync(databasePath).size; const db = new DatabaseSync(databasePath); try { const days = options.days === undefined ? historyDays(db) : clampDays(options.days); const cutoff = cutoffFor(options.now ?? new Date(), days); const result = { days, cutoff: cutoff === null ? null : cutoff.toISOString(), sceneVersions: 0, itemVersions: 0, projectVersions: 0, held: { sceneVersions: 0, itemVersions: 0, projectVersions: 0 }, removed: apply, bytesBefore, bytesAfter: bytesBefore, reclaimedBytes: null }; if (cutoff === null) return result; const cutoffMs = cutoff.getTime(); db.exec('begin'); try { for (const spec of HISTORY_TABLES) { if (!hasTable(db, spec.table)) continue; result.held[spec.key] = Number( db.prepare(`select count(*) as n from ${spec.table}`).get()?.n ?? 0 ); const { countSql, deleteSql, params } = expiryQueries(spec, cutoffMs, includeKept); const doomed = Number(db.prepare(countSql).get(...params)?.n ?? 0); result[spec.key] = doomed; if (apply && doomed > 0) db.prepare(deleteSql).run(...params); } db.exec('commit'); } catch (error) { db.exec('rollback'); throw error; } if (apply && vacuum) { db.exec('vacuum'); // A vacuum in WAL mode leaves its pages in the journal until a checkpoint // writes them, and the file shrinks only then. db.exec('pragma wal_checkpoint(truncate)'); result.bytesAfter = statSync(databasePath).size; result.reclaimedBytes = Math.max(0, result.bytesBefore - result.bytesAfter); } return result; } finally { db.close(); } } function clampDays(value) { const days = value === true ? Number.NaN : Number(value); if (!Number.isInteger(days) || days < 0) { throw new Error('--days takes a whole number, 0 or more. 0 keeps everything.'); } return days; } function megabytes(bytes) { return `${(bytes / 1_000_000).toFixed(1)} MB`; } /** The same size in the unit that does not read as zero for a small file. */ function sizeText(bytes) { if (bytes < 1000) return `${bytes} B`; return bytes < 1_000_000 ? `${Math.round(bytes / 1000)} KB` : megabytes(bytes); } const USAGE = `fablewright: back up, inspect or restore an instance, or manage a password and an account. node scripts/ops.mjs backup [--database PATH] [--data-dir DIR] [--out DIR] [--keep N] [--database-only] node scripts/ops.mjs verify --file DIR node scripts/ops.mjs restore --file DIR [--database PATH] [--data-dir DIR] node scripts/ops.mjs drill [--file DIR] [--database PATH] [--data-dir DIR] [--out DIR] [--scratch DIR] [--keep N] node scripts/ops.mjs prune-uploads [--database PATH] [--data-dir DIR] [--originals] [--grace SECONDS] [--apply] node scripts/ops.mjs prune-history [--database PATH] [--days N] [--all] [--apply] [--vacuum] node scripts/ops.mjs reset-password --email ADDRESS [--password TEXT | --generate] node scripts/ops.mjs delete-account --email ADDRESS [--delete-shared-projects] The database comes from --database, then DATABASE_URL, then DATA_DIR. Backups go to /backups. The newest 8 stay unless --keep says otherwise. --keep 0 removes nothing. --database-only copies the database and nothing else, for a deployment that copies the rest of the data directory itself. Stop the app before restore. A running app keeps writing to the database it opened, and a restore moves that file aside. drill restores a backup into a temporary directory and checks the result: the words, the history, the images, the signing secret and the accounts. Without --file it takes a backup first. It does not restore over the live database, so it can run with the app up. --scratch DIR puts the copy on another disk. The outcome is appended to /restore-drills.ndjson. prune-uploads removes orphaned files in uploads/, including the files writers imported that no import report names. Pass --originals to drop stored originals and keep only display WebP copies. Pass --apply to delete; without it, it reports what would be removed. A file newer than the grace window (--grace, one hour by default) is left for a later run, because the app publishes a file before it commits the row that names it. An original is never dropped while its display copy is missing, because then it is the only copy. prune-history removes recovery history older than the window: automatic scene versions, note and card versions, and the project versions taken before a structural change. Scene and project versions saved by hand stay unless --all removes those too. Without --days it uses the instance setting, which is 365 days until an admin changes it; a window of 0 keeps everything. Every scene, note, card and project keeps its newest few versions. Pass --apply to delete, and --vacuum to give the space back to the disk. reset-password works with the app stopped and ends every session for the account. It prints a generated password once; a password you pass with --password is not printed. delete-account removes the account and the projects only it can open. It refuses a project other people can open unless --delete-shared-projects says to remove those too, which takes their words with it.`; export function main(argv = process.argv.slice(2)) { const [command, ...rest] = argv; const flags = parseFlags(rest); if (!command || command === 'help' || command === '--help' || flags.help === true) { console.log(USAGE); return command ? 0 : 1; } try { if (command === 'backup') return runBackup(flags); if (command === 'verify') return runVerify(flags); if (command === 'restore') return runRestore(flags); if (command === 'drill') return runDrill(flags); if (command === 'prune-uploads') return runPruneUploads(flags); if (command === 'prune-history') return runPruneHistory(flags); if (command === 'reset-password') return runResetPassword(flags); if (command === 'delete-account') return runDeleteAccount(flags); } catch (error) { console.error(`fablewright: ${messageOf(error)}`); return 1; } console.error(USAGE); return 1; } /** `--name value` and `--name=value`, into one plain object. */ function parseFlags(args) { const flags = {}; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (!arg.startsWith('--')) continue; const [name, inline] = arg.slice(2).split('=', 2); if (inline !== undefined) { flags[name] = inline; } else if (args[index + 1] !== undefined && !args[index + 1].startsWith('--')) { flags[name] = args[index + 1]; index += 1; } else { flags[name] = true; } } return flags; } function runBackup(flags) { const { databasePath, dataDir, outDir } = locate(flags); const keep = flags.keep === undefined ? 8 : Number(flags.keep); const databaseOnly = flags['database-only'] === true; if (flags.keep === true || !Number.isInteger(keep) || keep < 0) { throw new Error('--keep takes a whole number, 0 or more. 0 removes nothing.'); } const result = takeBackup({ databasePath, dataDir, outDir, keep, databaseOnly }); const { manifest } = result; console.log(`fablewright: backup written to ${result.dir}`); console.log( ` database ${megabytes(manifest.database.bytes)}, ${manifest.schema.applied} migrations, newest ${manifest.schema.newest ?? 'none'}` ); if (databaseOnly) { console.log(' holds the database only'); } else { console.log( ` secret ${manifest.secret ? 'included' : 'not included; keep an environment-supplied secret separately'}` ); console.log( ` uploads ${manifest.uploads ? `${manifest.uploads.files} files` : 'none yet'}` ); } console.log(` checked integrity ${manifest.integrity}`); for (const name of result.pruned) console.log(` removed ${name}`); console.log( ' next copy this directory off the machine. A backup on the same disk can be lost with the instance.' ); return 0; } function runVerify(flags) { if (!flags.file || flags.file === true) throw new Error('verify needs --file DIR.'); const report = verifyBackup(flags.file); if (!report.ok) { console.error(`fablewright: ${flags.file} cannot be trusted.`); for (const problem of report.problems) console.error(` ${problem}`); return 1; } const { manifest, database, schema, contents } = report; console.log(`fablewright: ${flags.file} passed verification.`); console.log(` made ${manifest.createdAt} by ${manifest.madeBy}`); console.log(` database ${megabytes(database.bytes)}, digest matches the manifest`); console.log( ` integrity ${report.integrity}, ${report.foreignKeyProblems} foreign key problems` ); console.log( ` schema ${schema.applied.length} migrations, newest ${schema.applied.at(-1) ?? 'none'}, last opened by ${schema.version ?? 'an unrecorded build'}` ); console.log( ` contents ${contents.projects ?? '?'} projects, ${contents.scenes ?? '?'} scenes, ${contents.notes ?? '?'} notes` ); console.log(` secret ${report.secret ? 'included' : 'not included'}`); console.log( ` uploads ${ report.uploads ? `${plural(report.uploads.files, 'file')} (${sizeText(report.uploads.bytes)})` : 'none in this backup' }` ); return 0; } function runRestore(flags) { if (!flags.file || flags.file === true) throw new Error('restore needs --file DIR.'); const { databasePath, dataDir } = locate(flags); const result = restoreBackup({ dir: flags.file, databasePath, dataDir }); console.log(`fablewright: restored ${databasePath} from ${flags.file}`); console.log( ` previous ${result.previous ? `kept as ${result.previous}` : 'there was no database to replace'}` ); console.log(` secret ${result.secret ? 'restored' : 'left alone, the backup held none'}`); if (result.uploads) { console.log(` uploads restored ${result.uploads.files} files`); } else { console.log(' uploads left alone, the backup held none'); } console.log(' next start the app with the version this backup was taken from.'); return 0; } function runPruneHistory(flags) { if (flags.days === true) throw new Error('prune-history needs a number after --days.'); const { databasePath } = locate(flags); const result = pruneHistory({ databasePath, days: flags.days === undefined ? undefined : flags.days, includeKept: flags.all === true, apply: flags.apply === true, vacuum: flags.vacuum === true }); if (result.cutoff === null) { console.log('fablewright: the window is 0 days, so no history expires.'); console.log(` held ${heldText(result.held)}`); console.log(' next pass --days N to choose a window, or set it on the Instance page.'); return 0; } const due = result.sceneVersions + result.itemVersions + result.projectVersions; const end = result.cutoff.slice(0, 10); console.log( result.removed ? `fablewright: removed ${due} ${due === 1 ? 'version' : 'versions'} older than ${end}.` : `fablewright: ${due} ${due === 1 ? 'version is' : 'versions are'} older than ${end}.` ); console.log(` window ${result.days} days`); console.log(` scenes ${versionCount(result.sceneVersions, result.held.sceneVersions)}`); console.log(` notes ${versionCount(result.itemVersions, result.held.itemVersions)}`); console.log(` projects ${versionCount(result.projectVersions, result.held.projectVersions)}`); if (flags.all !== true) { console.log(' kept versions saved by hand'); } if (!result.removed) { console.log( due > 0 ? ' dry run: pass --apply to remove them. Add --vacuum to shrink the file.' : ' nothing to do.' ); return 0; } console.log(` file ${megabytes(result.bytesBefore)} before`); console.log( result.reclaimedBytes === null ? ' file unchanged. Pass --vacuum to give the space back to the disk.' : ` file ${megabytes(result.bytesAfter)} after, ${megabytes(result.reclaimedBytes)} reclaimed` ); return 0; } /** `12 of 40 versions`, so a dry run says how much of the history is due. */ function versionCount(due, held) { return `${due} of ${held} ${held === 1 ? 'version' : 'versions'}`; } function heldText(held) { const total = held.sceneVersions + held.itemVersions + held.projectVersions; return `${total} ${total === 1 ? 'version' : 'versions'}`; } function runDrill(flags) { for (const name of ['file', 'out', 'scratch']) { if (flags[name] === true) throw new Error(`drill needs a directory after --${name}.`); } const { databasePath, dataDir, outDir } = locateDrill(flags); const keep = flags.keep === undefined ? 8 : Number(flags.keep); if (flags.keep === true || !Number.isInteger(keep) || keep < 0) { throw new Error('--keep takes a whole number, 0 or more. 0 removes nothing.'); } const result = drillRestore({ databasePath, dataDir, outDir, keep, backupDir: flags.file ?? undefined, scratchDir: flags.scratch ?? undefined }); const passed = result.checks.filter((check) => check.ok).length; console.log( `fablewright: ${result.ok ? 'the backup restores' : 'the drill failed'}. ${passed} of ${result.checks.length} checks passed.` ); for (const check of result.checks) { console.log(` ${check.ok ? 'ok ' : 'FAIL'} ${check.label.padEnd(18)}${check.detail}`); } console.log( result.recordPath ? ` recorded ${result.recordPath}` : ` recorded nothing. ${result.recordError ?? 'There is no data directory.'}` ); console.log( result.ok ? ' next the drill did not restore over the live database. Copy the backup off this machine.' : ' next the restore is not proven. Read the failed checks before trusting the backup.' ); return result.ok ? 0 : 1; } function runPruneUploads(flags) { const { databasePath, dataDir } = locate(flags); const apply = flags.apply === true; const graceMs = graceFrom(flags) ?? SWEEP_GRACE_MS; if (flags.originals) { const result = dropOriginals({ databasePath, dataDir, apply, graceMs }); if (apply) { console.log( `fablewright: dropped ${plural(result.droppedOriginals, 'full-size original')} (${megabytes(result.droppedBytes)} reclaimed).` ); } else { console.log( `fablewright: found ${plural(result.totalOriginals, 'full-size original')} (${megabytes(result.totalBytes)}).` ); console.log( ` droppable: ${plural(result.droppableOriginals, 'file')} (${megabytes(result.droppableBytes)})` ); console.log(' dry-run: pass --apply to drop originals from the database and disk.'); } if (result.keptOriginals > 0) { console.log( ` kept ${plural(result.keptOriginals, 'original')} whose display copy is missing.` ); console.log(' Those are the only copy of the image, so they are not dropped.'); } if (result.recentOriginals > 0) { console.log( ` kept ${plural(result.recentOriginals, 'original')} newer than ${graceWindow(graceMs)}.` ); } if (result.missingOriginals > 0) { console.log( apply ? ` cleared the byte count on ${plural(result.missingOriginals, 'row')} whose original was already gone.` : ` ${plural(result.missingOriginals, 'row')} name an original the disk no longer holds.` ); } if (apply && result.keptOriginals === 0) { console.log(' display copies remain intact.'); } return reportFailedFiles(result.failedFiles); } const result = pruneUploads({ databasePath, dataDir, apply, graceMs }); if (apply) { console.log( `fablewright: pruned ${plural(result.removedFiles, 'unreferenced file')} (${megabytes(result.removedBytes)} reclaimed).` ); console.log(` kept ${plural(result.referencedFiles, 'referenced file')}.`); } else { console.log( `fablewright: scanned ${result.totalFiles} uploads (${megabytes(result.totalBytes)}).` ); console.log(` referenced: ${plural(result.referencedFiles, 'file')}`); console.log( ` unreferenced: ${plural(result.unreferencedFiles, 'file')} (${megabytes(result.unreferencedBytes)})` ); if (result.unreferencedFiles > 0) { console.log(' dry-run: pass --apply to delete unreferenced files from disk.'); } } if (result.recentFiles > 0) { console.log( ` ${plural(result.recentFiles, 'file')} newer than ${graceWindow(graceMs)}, so a later run takes them.` ); } return reportFailedFiles(result.failedFiles); } /** * A file inside the grace window may be an upload whose row has not landed yet, * so the window is what keeps a live instance's words and pictures out of a sweep. * Zero turns the window off, which is a sweep an operator runs with the app stopped. */ function insideGraceWindow(stat, now, graceMs) { return graceMs > 0 && now - stat.mtimeMs < graceMs; } function graceFrom(flags) { if (flags.grace === undefined) return undefined; if (flags.grace === true) throw new Error('--grace needs a number of seconds.'); const seconds = Number(flags.grace); if (!Number.isInteger(seconds) || seconds < 0) { throw new Error('--grace takes a whole number of seconds, 0 or more.'); } return seconds * 1000; } function graceWindow(ms) { if (ms === 0) return 'the grace window, which is off'; const minutes = Math.round(ms / 60_000); if (minutes < 60) return `the ${plural(minutes, 'minute')} grace window`; return `the ${plural(Math.round(minutes / 60), 'hour')} grace window`; } /** A sweep that could not delete everything says so, and exits for a script to see. */ function reportFailedFiles(failedFiles) { if (failedFiles.length === 0) return 0; console.error(`fablewright: ${plural(failedFiles.length, 'file')} could not be deleted.`); for (const line of failedFiles) console.error(` ${line}`); console.error(' Each one is still on disk, and its row was left alone.'); return 1; } /** True when a table exists, so a tool works against an older database too. */ function hasTable(db, name) { const row = db .prepare("select count(*) as n from sqlite_master where type = 'table' and name = ?") .get(name); return Number(row?.n ?? 0) > 0; } /** * The operator's tool, not a writer's: the web refuses to delete the only * account, because an instance with no accounts can be claimed by anyone. At a * shell that is a deliberate decision. * * The rows that reference a user are text ids with no foreign key, so nothing * cascades. The projects and the search index are removed here by hand. */ export function deleteAccount({ databasePath, email, deleteSharedProjects = false }) { if (!existsSync(databasePath)) { throw new Error(`There is no database at ${databasePath}.`); } const db = new DatabaseSync(databasePath); db.exec('pragma foreign_keys = on'); try { const user = db.prepare('select id, email from user where email = ?').get(email.toLowerCase()); if (!user) { throw new Error(`No account uses ${email}.`); } const owned = db .prepare( `select p.id, p.title, (select count(*) from project_member m where m.project_id = p.id and m.user_id <> p.owner_id) as others from project p where p.owner_id = ?` ) .all(user.id); const shared = owned.filter((project) => Number(project.others) > 0); if (shared.length > 0 && !deleteSharedProjects) { const names = shared.map((project) => project.title).join(', '); throw new Error( `This account owns projects with other people in them: ${names}. Pass --delete-shared-projects to remove them too, which takes everyone's words.` ); } const indexed = hasTable(db, 'search_index'); db.exec('begin'); try { for (const project of owned) { if (indexed) db.prepare('delete from search_index where project_id = ?').run(project.id); db.prepare('delete from project where id = ?').run(project.id); } deleteAccountReferences(user.id, { hasTable: (table) => hasTable(db, table), run: (statement, userId) => db.prepare(statement).run(userId) }); db.prepare('delete from user where id = ?').run(user.id); db.exec('commit'); return { email: user.email, projects: owned.length, shared: shared.length }; } catch (error) { db.exec('rollback'); throw error; } } finally { db.close(); } } function runResetPassword(flags) { if (!flags.email || flags.email === true) { throw new Error('reset-password needs --email ADDRESS.'); } const given = flags.password === undefined || flags.password === true ? null : String(flags.password); if (given && flags.generate) { throw new Error('Pass --password or --generate, not both.'); } const password = given ?? generatePassword(); if (password.length < 10) { throw new Error('Use at least 10 characters, which is the rule the sign-in form applies.'); } const { databasePath } = locate(flags); const result = resetPassword({ databasePath, email: String(flags.email), password }); console.log(`fablewright: the password for ${result.email} is set.`); console.log(given ? ' password the one you passed in --password' : ` password ${password}`); console.log(` sessions ${result.sessions} ended, so nobody stays signed in`); console.log(' next sign in with it, then change it from your account page.'); return 0; } function runDeleteAccount(flags) { if (!flags.email || flags.email === true) { throw new Error('delete-account needs --email ADDRESS.'); } const { databasePath } = locate(flags); const result = deleteAccount({ databasePath, email: String(flags.email), deleteSharedProjects: flags['delete-shared-projects'] === true }); console.log(`fablewright: ${result.email} is deleted.`); console.log( ` projects ${result.projects} ${result.projects === 1 ? 'project' : 'projects'} removed, with every word in them` ); if (result.shared > 0) { console.log(` shared ${result.shared} of those had other people in them`); } console.log( ' next earlier backups and project archives can still contain the deleted work.' ); return 0; } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { process.exitCode = main(); }