Something went wrong. Try again.
A free, open-source, self-hosted place to write a novel.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775import { zipSync, strToU8 } from 'fflate';import { and, asc, eq, inArray, isNull, or } from 'drizzle-orm';import { db, tables } from './db/index.ts';import { toMarkdown, bodyToText } from './markdown.ts';import { countWords } from '#lib/words.ts';import { readImage } from './uploads.ts';import { EXPORT_FORMAT, EXPORT_VERSION } from '#lib/export-format.ts';import type { MatterPosition } from '#lib/matter.ts';import type { SnapshotSource } from '#lib/snapshot-sources.ts';import { bodyChapters, matterPages } from '#lib/matter.ts';
// Import reads the lossless JSON. Markdown is only a reading copy.export { EXPORT_FORMAT, EXPORT_VERSION };
export type ExportArchive = { filename: string; bytes: Uint8Array; sceneCount: number; wordCount: number;};
// A file can leave out timestamps and deletion flags and still import.type JsonTimestamps = { createdAt?: string; updatedAt?: string; deletedAt?: string | null;};
export type JsonScene = JsonTimestamps & { id: string; title: string; body: unknown; bodyText: string; wordCount: number; status: string; synopsis?: string; /** Optional, because an older export carries no viewpoint. */ viewpoint?: string; ordinal: number; revision?: number; lastOpenedAt?: string | null; storylineId?: string | null; storyline: string | null;};
export type JsonChapter = JsonTimestamps & { id: string; title: string; kind: string; matter?: MatterPosition | null; synopsis?: string; ordinal: number; scenes: JsonScene[];};
export type JsonPart = JsonTimestamps & { id: string; title: string; ordinal: number; chapters: JsonChapter[];};
export type JsonBook = JsonTimestamps & { id: string; title: string; ordinal: number; parts: JsonPart[]; chapters: JsonChapter[];};
/** * Histories use this tree's ids, including trash, so import can remap them. * Checkpoint payloads keep their own ids so recovery can import them afresh. * Version 1 files can omit timestamps, id-based references, and recovery arrays. */export type ManuscriptJson = { format: typeof EXPORT_FORMAT; version: number; exportedAt: string; project: { id: string; title: string; subtitle: string | null; description: string | null; /** The pen name a Word manuscript and an EPUB print as the creator. */ author?: string | null; accent?: string; createdAt?: string; updatedAt?: string; }; feedback?: { id: string; sceneId: string | null; chapterId?: string | null; anchorStart?: number | null; anchorPrefix?: string; anchorSuffix?: string; author: string; message: string; quote: string; suggestion: string; revision: number; resolved: boolean; createdAt: string; }[]; books: JsonBook[]; notes: (JsonTimestamps & { id: string; title: string; summary: string; noteTypeId?: string | null; type: string | null; body: unknown; bodyText: string; ordinal: number; revision?: number; })[]; noteImages?: { noteId: string; imageId: string | null; mime: string; bytes: number; width: number; height: number; alt: string; originalName: string; }[]; noteTypes: { id: string; name: string; pluralName: string; accent: string; template?: unknown; ordinal: number; }[]; storylines: (JsonTimestamps & { id: string; title: string; accent: string; ordinal: number; })[]; plotCards: { id: string; storylineId: string; sceneId: string | null; chapterId: string | null; title: string; body: string; ordinal: number; createdAt?: string; updatedAt?: string; deletedAt?: string | null; }[]; sceneNotes: { sceneId: string; noteId: string }[]; plotCardNotes: { cardId: string; noteId: string }[]; sceneSnapshots?: { id: string; sceneId: string; body: unknown; bodyText: string; wordCount: number; label: string | null; source: SnapshotSource; createdAt: string; }[]; itemVersions?: { id: string; kind: 'note' | 'plot_card'; itemId: string; title: string; summary: string; body: unknown; bodyText: string; source: 'edit' | 'pre_restore' | 'conflict'; createdAt: string; }[]; checkpoints?: { id: string; label: string; source: 'manual' | 'structure'; manuscript: ManuscriptJson; createdAt: string; }[];};
function timestamps(row: { createdAt: Date; updatedAt: Date; deletedAt?: Date | null }) { return { createdAt: row.createdAt.toISOString(), updatedAt: row.updatedAt.toISOString(), ...('deletedAt' in row ? { deletedAt: row.deletedAt?.toISOString() ?? null } : {}) };}
export function buildManuscriptJson( projectId: string, options: { includeCheckpoints?: boolean } = {}): ManuscriptJson { // The caller may already hold a transaction. SQLite cannot nest BEGIN. const read = () => readManuscriptJson(projectId, options.includeCheckpoints !== false); return db.$client.isTransaction ? read() : db.transaction(read);}
function readManuscriptJson(projectId: string, includeCheckpoints: boolean): ManuscriptJson { const project = db.select().from(tables.project).where(eq(tables.project.id, projectId)).get();
if (!project) throw new Error('Project not found.');
const books = db .select() .from(tables.book) .where(eq(tables.book.projectId, projectId)) .orderBy(asc(tables.book.ordinal), asc(tables.book.createdAt)) .all();
const bookIds = books.map((row) => row.id); const parts = bookIds.length ? db .select() .from(tables.part) .where(inArray(tables.part.bookId, bookIds)) .orderBy(asc(tables.part.ordinal), asc(tables.part.createdAt)) .all() : []; const chapters = bookIds.length ? db .select() .from(tables.chapter) .where(inArray(tables.chapter.bookId, bookIds)) .orderBy(asc(tables.chapter.ordinal), asc(tables.chapter.createdAt)) .all() : []; const chapterIds = chapters.map((row) => row.id); const scenes = chapterIds.length ? db .select() .from(tables.scene) .where(inArray(tables.scene.chapterId, chapterIds)) .orderBy(asc(tables.scene.ordinal), asc(tables.scene.createdAt)) .all() : [];
const storylines = db .select() .from(tables.storyline) .where(eq(tables.storyline.projectId, projectId)) .orderBy(asc(tables.storyline.ordinal), asc(tables.storyline.createdAt)) .all(); const storylineTitles = new Map(storylines.map((row) => [row.id, row.title]));
const notes = db .select() .from(tables.note) .where(eq(tables.note.projectId, projectId)) .orderBy(asc(tables.note.ordinal), asc(tables.note.createdAt)) .all(); const referencedTypeIds = notes.flatMap((row) => (row.noteTypeId ? [row.noteTypeId] : [])); const noteTypes = db .select() .from(tables.noteType) .where( or( eq(tables.noteType.projectId, projectId), referencedTypeIds.length ? and(isNull(tables.noteType.projectId), inArray(tables.noteType.id, referencedTypeIds)) : undefined ) ) .orderBy(asc(tables.noteType.ordinal), asc(tables.noteType.id)) .all(); const noteTypeNames = new Map(noteTypes.map((row) => [row.id, row.name]));
const noteImageRows = db .select({ noteId: tables.noteImage.noteId, imageId: tables.noteImage.imageId, mime: tables.noteImage.mime, bytes: tables.noteImage.bytes, width: tables.noteImage.width, height: tables.noteImage.height, alt: tables.noteImage.alt, originalName: tables.noteImage.originalName }) .from(tables.noteImage) .innerJoin(tables.note, eq(tables.noteImage.noteId, tables.note.id)) .where(eq(tables.note.projectId, projectId)) .all();
const storylineIds = storylines.map((row) => row.id); const plotCards = storylineIds.length ? db .select() .from(tables.plotCard) .where(inArray(tables.plotCard.storylineId, storylineIds)) .orderBy(asc(tables.plotCard.ordinal), asc(tables.plotCard.createdAt)) .all() : [];
const localSceneIds = new Set(scenes.map((row) => row.id)); const localChapterIds = new Set(chapterIds); const localNoteIds = new Set(notes.map((row) => row.id)); const localCardIds = new Set(plotCards.map((row) => row.id)); const sceneNotes = scenes.length ? db .select() .from(tables.sceneNote) .where(inArray(tables.sceneNote.sceneId, [...localSceneIds])) .all() .filter((row) => localNoteIds.has(row.noteId)) : []; const cardNoteRows = plotCards.length ? db .select() .from(tables.plotCardNote) .where(inArray(tables.plotCardNote.cardId, [...localCardIds])) .all() .filter((row) => localNoteIds.has(row.noteId)) : []; const sceneSnapshots = scenes.length ? db .select() .from(tables.sceneSnapshot) .where(inArray(tables.sceneSnapshot.sceneId, [...localSceneIds])) .orderBy(asc(tables.sceneSnapshot.createdAt), asc(tables.sceneSnapshot.id)) .all() : []; const itemVersions = db .select() .from(tables.itemVersion) .where(eq(tables.itemVersion.projectId, projectId)) .orderBy(asc(tables.itemVersion.createdAt), asc(tables.itemVersion.id)) .all();
for (const version of itemVersions) { const localIds = version.kind === 'note' ? localNoteIds : localCardIds; if (!localIds.has(version.itemId)) { throw new Error('An item version refers to an item outside this project.'); } }
const checkpoints = includeCheckpoints ? db .select() .from(tables.projectCheckpoint) .where(eq(tables.projectCheckpoint.projectId, projectId)) .orderBy(asc(tables.projectCheckpoint.createdAt), asc(tables.projectCheckpoint.id)) .all() : [];
const scenesByChapter = new Map<string, JsonScene[]>();
for (const scene of scenes) { const list = scenesByChapter.get(scene.chapterId) ?? []; const storylineId = scene.storylineId && storylineTitles.has(scene.storylineId) ? scene.storylineId : null;
list.push({ id: scene.id, title: scene.title, body: scene.body, bodyText: scene.bodyText, wordCount: scene.wordCount, status: scene.status, synopsis: scene.synopsis, viewpoint: scene.viewpoint, ordinal: scene.ordinal, revision: scene.revision, lastOpenedAt: scene.lastOpenedAt?.toISOString() ?? null, storylineId, storyline: storylineId ? storylineTitles.get(storylineId)! : null, ...timestamps(scene) }); scenesByChapter.set(scene.chapterId, list); }
const chaptersByBook = new Map<string, JsonChapter[]>(); const chaptersByPart = new Map<string, JsonChapter[]>(); const partBooks = new Map(parts.map((row) => [row.id, row.bookId]));
for (const row of chapters) { if (row.partId && partBooks.get(row.partId) !== row.bookId) { throw new Error('A chapter refers to a part outside its book.'); }
const chapter: JsonChapter = { id: row.id, title: row.title, kind: row.kind, matter: row.matter, synopsis: row.synopsis, ordinal: row.ordinal, scenes: scenesByChapter.get(row.id) ?? [], ...timestamps(row) }; const map = row.partId ? chaptersByPart : chaptersByBook; const parentId = row.partId ?? row.bookId; const list = map.get(parentId) ?? []; list.push(chapter); map.set(parentId, list); }
const partsByBook = new Map<string, JsonPart[]>();
for (const row of parts) { const list = partsByBook.get(row.bookId) ?? []; list.push({ id: row.id, title: row.title, ordinal: row.ordinal, chapters: chaptersByPart.get(row.id) ?? [], ...timestamps(row) }); partsByBook.set(row.bookId, list); }
return { feedback: db .select() .from(tables.sceneFeedback) .where(eq(tables.sceneFeedback.projectId, projectId)) .all() .map( ({ id, sceneId, chapterId, anchorStart, anchorPrefix, anchorSuffix, author, message, quote, suggestion, revision, resolved, createdAt }) => ({ id, sceneId, chapterId, anchorStart, anchorPrefix, anchorSuffix, author, message, quote, suggestion, revision, resolved, createdAt: createdAt.toISOString() }) ), format: EXPORT_FORMAT, version: EXPORT_VERSION, exportedAt: new Date().toISOString(), project: { id: project.id, title: project.title, subtitle: project.subtitle, description: project.description, author: project.author, accent: project.accent, createdAt: project.createdAt.toISOString(), updatedAt: project.updatedAt.toISOString() }, books: books.map((row) => ({ id: row.id, title: row.title, ordinal: row.ordinal, parts: partsByBook.get(row.id) ?? [], chapters: chaptersByBook.get(row.id) ?? [], ...timestamps(row) })), notes: notes.map((row) => ({ id: row.id, title: row.title, summary: row.summary, noteTypeId: row.noteTypeId && noteTypeNames.has(row.noteTypeId) ? row.noteTypeId : null, type: row.noteTypeId ? (noteTypeNames.get(row.noteTypeId) ?? null) : null, body: row.body, bodyText: row.bodyText, ordinal: row.ordinal, revision: row.revision, ...timestamps(row) })), noteImages: noteImageRows.map((row) => ({ noteId: row.noteId, imageId: row.imageId, mime: row.mime, bytes: row.bytes, width: row.width, height: row.height, alt: row.alt, originalName: row.originalName })), noteTypes: noteTypes.map((row) => ({ id: row.id, name: row.name, pluralName: row.pluralName, accent: row.accent, template: row.template, ordinal: row.ordinal })), storylines: storylines.map((row) => ({ id: row.id, title: row.title, accent: row.accent, ordinal: row.ordinal, ...timestamps(row) })), plotCards: plotCards.map((row) => ({ id: row.id, storylineId: row.storylineId, sceneId: row.sceneId && localSceneIds.has(row.sceneId) ? row.sceneId : null, chapterId: row.chapterId && localChapterIds.has(row.chapterId) ? row.chapterId : null, title: row.title, body: row.body, ordinal: row.ordinal, ...timestamps(row) })), sceneNotes: sceneNotes.map((row) => ({ sceneId: row.sceneId, noteId: row.noteId })), plotCardNotes: cardNoteRows.map((row) => ({ cardId: row.cardId, noteId: row.noteId })), sceneSnapshots: sceneSnapshots.map((row) => ({ id: row.id, sceneId: row.sceneId, body: row.body, bodyText: row.bodyText, wordCount: row.wordCount, label: row.label, source: row.source, createdAt: row.createdAt.toISOString() })), itemVersions: itemVersions.map((row) => ({ id: row.id, kind: row.kind, itemId: row.itemId, title: row.title, summary: row.summary, body: row.body, bodyText: row.bodyText, source: row.source, createdAt: row.createdAt.toISOString() })), checkpoints: checkpoints.map((row) => ({ id: row.id, label: row.label, source: row.source, manuscript: row.manuscript as ManuscriptJson, createdAt: row.createdAt.toISOString() })) };}
/** Slashes, leading dots, and control characters can turn a title into a path. */export function safeName(title: string, fallback: string): string { const cleaned = title // eslint-disable-next-line no-control-regex -- Titles can contain pasted control characters. .replace(/[<>:"/\\|?*\u0000-\u001f]/g, ' ') .replace(/\s+/g, ' ') .trim() .replace(/^[.\s]+/, '');
// A UTF-16 cut can split an emoji and break the download header. const shortened = [...cleaned].slice(0, 60).join('').trim();
return shortened.length > 0 ? shortened : fallback;}
/** HTTP needs an ASCII fallback as well as the writer's exact filename. */export function contentDisposition(filename: string): string { const name = filename.trim().length > 0 ? filename : 'manuscript.zip';
return `attachment; filename="${asciiFilename(name)}"; filename*=UTF-8''${encodeRFC5987(name)}`;}
function asciiFilename(filename: string): string { const dot = filename.lastIndexOf('.'); const stem = dot > 0 ? filename.slice(0, dot) : filename; const extension = dot > 0 ? filename.slice(dot).replace(/[^A-Za-z0-9.]/g, '') : '';
const folded = stem .normalize('NFD') .replace(/[\u0300-\u036f]/g, '') .replace(/[Ææ]/g, (char) => (char === 'Æ' ? 'AE' : 'ae')) .replace(/[Øø]/g, (char) => (char === 'Ø' ? 'O' : 'o')) .replace(/ß/g, 'ss') .replace(/[Đđ]/g, (char) => (char === 'Đ' ? 'D' : 'd')) .replace(/[Łł]/g, (char) => (char === 'Ł' ? 'L' : 'l')) .replace(/[Þþ]/g, (char) => (char === 'Þ' ? 'Th' : 'th')) .replace(/[^\x20-\x7e]/g, ' ') .replace(/\s+/g, ' ') .replace(/-+/g, '-') .replace(/\s*-\s*/g, '-') .trim() .replace(/^[.\s-]+/, '') .replace(/[.\s-]+$/, '') .replace(/\\/g, '\\\\') .replace(/"/g, '\\"');
return /[A-Za-z0-9]/.test(folded) ? `${folded}${extension}` : 'manuscript.zip';}
function encodeRFC5987(value: string): string { return encodeURIComponent(value.toWellFormed()).replace( /['()*]/g, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}` );}
function prefix(index: number): string { return String(index + 1).padStart(2, '0');}
export type ManuscriptFiles = { files: Map<string, string>; count: number; words: number;};
/** Every image whose note is in the manuscript, live or in the trash. */function exportedNoteImages(manuscript: ManuscriptJson) { const noteIds = new Set(manuscript.notes.map((note) => note.id));
return (manuscript.noteImages ?? []).filter((img) => img.imageId && noteIds.has(img.noteId));}
export function buildMarkdownFiles(manuscript: ManuscriptJson): ManuscriptFiles { const files = new Map<string, string>(); let count = 0; let words = 0;
const chapterFolder = (chapter: JsonChapter, index: number) => `chapters/${prefix(index)}-${safeName(chapter.title, 'chapter')}`;
const writeChapter = (chapter: JsonChapter, folder: string) => { if (chapter.deletedAt) return;
for (const [index, scene] of chapter.scenes.filter((row) => !row.deletedAt).entries()) { const sceneFile = `${folder}/scenes/${prefix(index)}-${safeName(scene.title, 'scene')}.md`; files.set(sceneFile, `# ${scene.title}\n\n${toMarkdown(scene.body)}\n`); count += 1; words += scene.wordCount || countWords(bodyToText(scene.body)); } };
for (const [bookIndex, book] of manuscript.books.filter((row) => !row.deletedAt).entries()) { const root = `books/${prefix(bookIndex)}-${safeName(book.title, 'book')}`; for (const position of ['front', 'back'] as const) { for (const [index, page] of matterPages(book.chapters, position) .filter((row) => !row.deletedAt) .entries()) { writeChapter( page, `${root}/${position}-matter/${prefix(index)}-${safeName(page.title, 'page')}` ); } } const liveParts = book.parts.filter((row) => !row.deletedAt); let chapterIndex = liveParts.reduce( (sum, part) => sum + part.chapters.filter((row) => !row.deletedAt).length, 0 );
for (const [partIndex, part] of liveParts.entries()) { const partFolder = `${root}/parts/${prefix(partIndex)}-${safeName(part.title, 'part')}`; files.set(`${partFolder}/_part.md`, `# ${part.title}\n`);
for (const [index, chapter] of part.chapters.filter((row) => !row.deletedAt).entries()) { writeChapter(chapter, `${partFolder}/${chapterFolder(chapter, index)}`); } }
for (const chapter of bodyChapters(book.chapters).filter((row) => !row.deletedAt)) { writeChapter(chapter, `${root}/${chapterFolder(chapter, chapterIndex)}`); chapterIndex += 1; } }
const noteImageByNoteId = new Map((manuscript.noteImages ?? []).map((img) => [img.noteId, img]));
for (const [index, note] of manuscript.notes.filter((row) => !row.deletedAt).entries()) { const file = `notes/${prefix(index)}-${safeName(note.title, 'note')}.md`; const img = noteImageByNoteId.get(note.id); // An original that was never kept is not in the archive, but the display copy is. const imageFile = img?.imageId ? `../images/${img.imageId}.${img.bytes > 0 ? 'orig' : 'webp'}` : null; const imageLink = imageFile ? `\n\n` : ''; const type = note.type ? `*${note.type}*\n\n` : ''; files.set(file, `# ${note.title}\n\n${imageLink}${type}${toMarkdown(note.body)}\n`); }
const exportedImages = exportedNoteImages(manuscript); const trashedNoteIds = new Set( manuscript.notes.filter((note) => note.deletedAt).map((note) => note.id) ); const trashedImages = exportedImages.filter((img) => trashedNoteIds.has(img.noteId)); const withOriginal = exportedImages.filter((img) => img.bytes > 0 && img.imageId);
const imageLines: string[] = []; if (exportedImages.length > 0) { imageLines.push( `${exportedImages.length} ${exportedImages.length === 1 ? 'note image was' : 'note images were'} exported (${withOriginal.length} with a full-size original in images/).` ); } if (trashedImages.length > 0) { imageLines.push( trashedImages.length === 1 ? '1 image belongs to a note in the trash and travels so a restore keeps it.' : `${trashedImages.length} images belong to notes in the trash and travel so a restore keeps them.` ); } imageLines.push( 'Note images travel in this project archive and not in manuscript.json alone. DOCX and EPUB exports render the manuscript alone.' );
const readme = [ `# ${manuscript.project.title}`, manuscript.project.subtitle ? `\n${manuscript.project.subtitle}` : '', '', 'Exported from Fablewright.', '', '`manuscript.json` holds the whole project, including deleted content and recovery history.', 'Import that file to create a new project. Deleted content stays deleted.', '', 'The Markdown files contain only live content for reading, git, and other tools.', 'Import reads the JSON because Markdown cannot keep every document detail.', '', ...imageLines, '' ].join('\n'); files.set('README.md', readme);
return { files, count, words };}
export function buildExport(projectId: string): ExportArchive { const manuscript = buildManuscriptJson(projectId); const { files, count, words } = buildMarkdownFiles(manuscript); const entries: Record<string, Uint8Array> = { 'manuscript.json': strToU8(JSON.stringify(manuscript, null, 2)) };
if (manuscript.noteImages) { for (const noteImg of exportedNoteImages(manuscript)) { if (!noteImg.imageId) continue;
// The display copy is always on disk, so it always travels. The original // travels only when the row says the instance kept one. const display = readImage(noteImg.imageId, 'display'); if (display) { entries[`images/${noteImg.imageId}.webp`] = display.bytes; }
if (noteImg.bytes > 0) { const original = readImage(noteImg.imageId, 'original'); if (original) { entries[`images/${noteImg.imageId}.orig`] = original.bytes; } } } }
for (const [path, contents] of files) entries[path] = strToU8(contents);
const bytes = zipSync(entries, { level: 6 }); const stamp = new Date().toISOString().slice(0, 10);
return { filename: `${safeName(manuscript.project.title, 'manuscript')}-${stamp}.zip`, bytes, sceneCount: count, wordCount: words };}