#!/usr/bin/env node import { collectAnalytics } from "./analytics.mjs"; import { mkdir, readFile, writeFile, chmod, rm } from "node:fs/promises"; import { createInterface } from "node:readline/promises"; import { stdin, stdout } from "node:process"; import { resolve, join, dirname, basename } from "node:path"; import { fileURLToPath } from "node:url"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import ffmpeg from "ffmpeg-static"; import ffprobe from "ffprobe-static"; import { deletePublication } from "./delete.mjs"; import { createDraft } from "./draft.mjs"; import { archiveClient, archiveProject, archiveRecentMedia, cleanupProject, listArchived, projectFolders, restoreProject, } from "./archive.mjs"; import { publish } from "./runner.mjs"; import { connectAccount } from "./connections.mjs"; import { DAILY, FOOTAGE, dueClips, readState, uploadFootage, watchFootage } from "./footage.mjs"; import { createOAuth, ensureClientKey } from "./oauth.mjs"; import { reconcilePostRecords, syncPostRecord } from "./post-record.mjs"; import { install, uninstall } from "./service.mjs"; import { sweepChromeClones } from "./clones.mjs"; import { archiveStories, crosspostStories, storiesRoot } from "./stories.mjs"; const exec = promisify(execFile); const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const stateDir = join(root, ".publisher"); const configFile = join(stateDir, "config.json"); process.umask(0o077); process.env.FFMPEG_PATH ||= ffmpeg; process.env.FFPROBE_PATH ||= ffprobe.path; await mkdir(stateDir, { recursive: true, mode: 0o700 }); let config = { origin: "https://post.kualta.dev", platforms: [] }; try { config = JSON.parse(await readFile(configFile, "utf8")); } catch (e) { if (e.code !== "ENOENT") throw e; } async function save() { await writeFile(configFile, JSON.stringify(config), { mode: 0o600 }); await chmod(configFile, 0o600); } async function api(path, data) { const response = await fetch(`${config.origin}/api/publisher/helper/${path}`, { method: data === undefined ? "GET" : "POST", headers: { Authorization: `Bearer ${config.token}`, ...(data === undefined ? {} : { "Content-Type": "application/json" }), }, body: data === undefined ? undefined : JSON.stringify(data), signal: AbortSignal.timeout(120_000), }); if (!response.ok) { // The site says why, such as media that has expired; History shows it. const data = await response.json().catch(() => ({})); throw new Error(data.error || `Publisher request failed (${response.status})`); } return response; } async function python(script, args, signal) { let output; try { output = await exec(join(stateDir, "venv/bin/python"), [join(root, "scripts/publisher/vendor", script), ...args], { timeout: 20 * 60_000, maxBuffer: 2 * 1024 * 1024, signal, }); } catch (e) { output = e; } const line = String(output.stdout || "") .trim() .split("\n") .at(-1); try { return JSON.parse(line); } catch { throw new Error("Browser publisher stopped without a confirmed result"); } } const [command] = process.argv.slice(2); if (command === "connect") { const prompt = createInterface({ input: stdin, output: stdout }); try { const origin = (await prompt.question("Publisher URL [https://post.kualta.dev]: ")).trim() || "https://post.kualta.dev"; const url = new URL(origin); if ( url.origin !== "https://post.kualta.dev" && !(url.protocol === "http:" && ["localhost", "127.0.0.1"].includes(url.hostname)) ) throw new Error("Use post.kualta.dev or localhost"); const token = (await prompt.question("Pairing key: ")).trim(); if (!/^[a-f0-9]{64}$/.test(token)) throw new Error("Invalid pairing key"); config.origin = url.origin; config.token = token; await api("heartbeat", {}); await save(); console.log("Connected. Run bun run publisher install, then connect accounts in the composer."); } finally { prompt.close(); } } else if (command === "start") { if (!config.token) throw new Error("Connect the helper first"); await exec("exiftool", ["-ver"]); await exec(process.env.FFMPEG_PATH, ["-version"]); const oauth = await createOAuth(stateDir); await rm(join(stateDir, "connection-key.json"), { force: true }); if (config.atproto?.password || !(await oauth.hasSession())) { delete config.atproto; config.platforms = config.platforms.filter((p) => !["grain", "bluesky"].includes(p)); await save(); } console.log("Publisher ready. Keep this terminal open. Ctrl+C stops the helper."); setInterval(() => api("heartbeat", {}).catch(() => {}), 30_000).unref(); // Every Chrome launch copies the 1.4 GB app; copies left by interrupted launches would fill the disk. const sweep = () => sweepChromeClones() .then((removed) => removed.length && console.log(`Removed ${removed.length} leftover Chrome copies`)) .catch((error) => console.error(`Chrome copy cleanup failed: ${error.message}`)); void sweep(); setInterval(sweep, 10 * 60_000).unref(); // Graded clips go up to the footage archive's channel beside the publishing, and only while the composer's Archive // tab is open, so they never take the network unseen. The helper asks every 10 seconds; closing the tab stops the // upload mid-part, to resume there the next time it opens. void watchFootage({ stateDir, loadConfig: async () => JSON.parse(await readFile(configFile, "utf8")), ask: async (status) => (await api("footage", status ?? undefined)).json(), client: (current) => archiveClient({ origin: current.origin, token: current.token }), log: (line) => console.log(line), }); // Instagram's story archive is copied to disk on start, then hourly when nothing else is waiting, so new stories // reach Pinkleap while they're still up. The folder goes to the Cloudflare archive on start and after any change. let storiesDue = 0; let storiesChanged = true; while (true) { try { config = JSON.parse(await readFile(configFile, "utf8")); const { target, post, connection, deletion, analytics } = await ( await api("claim", { platforms: config.platforms, deletePublications: true, analytics: true, carousels: true }) ).json(); if (analytics) { const result = await collectAnalytics(analytics.target, { stateDir }); await api("analytics-result", { target_id: analytics.target_id, claim: analytics.claim, ...result }); continue; } if (deletion) { const result = await deletePublication(deletion, { oauth, stateDir }); // Retry delivery of the receipt, never the deletion itself. for (let attempt = 0; attempt < 5; attempt++) { try { await api("deletion-result", { target_id: deletion.target_id, claim: deletion.claim, ...result }); if (result.state === "deleted") await syncPostRecord(deletion.post.id, { api, oauth }); break; } catch { if (attempt < 4) await new Promise((resolve) => setTimeout(resolve, 5000)); else console.error("Deletion receipt was not saved. Check the platform."); } } console.log(`${deletion.target.platform}: ${result.state}`); continue; } if (connection) { const result = await connectAccount(connection, { stateDir, config, oauth, api, save }); // Retry the receipt only; never reopen or repeat an account action. for (let attempt = 0; attempt < 5; attempt++) { try { await api("accounts/result", { id: connection.id, claim: connection.claim, ...result }); break; } catch (error) { if (attempt === 4) console.error("Connection receipt could not be saved. Refresh the composer."); else await new Promise((resolve) => setTimeout(resolve, 5000)); } } console.log(`${connection.platform}: ${result.message}`); continue; } if (target) { console.log(`Publishing ${target.platform}…`); // Stop in the composer releases the target; this ends the attempt once the laptop notices. const stop = new AbortController(); const watch = setInterval(() => { api(`targets/${target.id}?claim=${target.claim}`) .then((response) => response.json()) .then(({ active }) => active === false && stop.abort()) .catch(() => {}); }, 5000); const result = await publish(target, post, { api, config, oauth, stateDir, python, signal: stop.signal, }).finally(() => clearInterval(watch)); if (stop.signal.aborted && result.state !== "succeeded") { console.log(`${target.platform}: stopped`); continue; } // Only delivery acknowledgements retry. Never repeat a publishing action. let delivered = false; for (let i = 0; i < 5 && !delivered; i++) { try { const response = await (await api("result", { id: target.id, claim: target.claim, ...result })).json(); delivered = response.ok; } catch { await new Promise((resolve) => setTimeout(resolve, 5000)); } } console.log( `${target.platform}: ${result.state}${delivered ? "" : " (receipt not saved; check the platform)"}`, ); if (delivered && result.state === "succeeded") await syncPostRecord(post.id, { api, oauth }); } else if (config.platforms.includes("instagram") && Date.now() >= storiesDue) { storiesDue = Date.now() + 60 * 60_000; const { saved, removed, total } = await archiveStories({ stateDir }); if (saved) console.log(`Saved ${saved} Instagram stories (${total} in ${storiesRoot})`); if (removed) console.log(`${removed} saved Instagram stories were deleted there; their files stay`); storiesChanged ||= saved > 0 || removed > 0; if (config.platforms.includes("bluesky")) { const { posted, left, unposted } = await crosspostStories({ oauth }); if (posted) console.log(`Sent ${posted} Instagram stories to Pinkleap${left ? ` (${left} to go)` : ""}`); if (unposted) console.log(`Took ${unposted} stories deleted on Instagram off Pinkleap`); storiesChanged ||= posted > 0 || unposted > 0; } if (storiesChanged) { const { sent } = await archiveProject(storiesRoot, { name: "instagram-stories", origin: config.origin, token: config.token, keepCache: true, }); storiesChanged = false; if (sent) console.log(`Archived Instagram stories to Cloudflare (${(sent / 1e6).toFixed(1)} MB sent)`); } } else await new Promise((resolve) => setTimeout(resolve, 5000)); } catch (e) { // Timed, so a run of failures in the log can be matched to what happened then. console.error(`${new Date().toISOString()} ${e.message}`); await new Promise((resolve) => setTimeout(resolve, 10_000)); } } } else if (command === "install") { if (!config.token) throw new Error("Connect the helper first"); const log = await install({ root, stateDir }); console.log(`Helper runs in the background and starts at login. Log: ${log}`); } else if (command === "uninstall") { await uninstall(); console.log("Background helper removed."); } else if (command === "draft") { try { if (!config.token) throw new Error("Connect the helper first"); const spec = process.argv[3]; if (!spec) throw new Error("Usage: bun run publisher draft "); const { url, reencoded } = await createDraft(resolve(spec), { origin: config.origin, token: config.token }); if (reencoded) console.error("Re-encoded the video to fit the upload limit."); console.log(url); } catch (error) { console.error(error.message); process.exitCode = 1; } } else if (["archive", "restore", "archives", "archive-media", "cleanup"].includes(command)) { try { if (!config.token) throw new Error("Connect the helper first"); const auth = { origin: config.origin, token: config.token }; const args = process.argv.slice(3); const flag = (name) => { const at = args.indexOf(name); return at === -1 ? undefined : args.splice(at, 2)[1]; }; const gb = (bytes) => `${(bytes / 1e9).toFixed(2)} GB`; if (command === "archive") { const keepCache = args.includes("--keep-cache"); const folders = args.filter((arg) => arg !== "--keep-cache"); if (!folders.length) throw new Error("Usage: bun run publisher archive … [--keep-cache]"); for (const folder of folders) { const done = await archiveProject(resolve(folder), { ...auth, keepCache, log: (line) => console.error(line) }); console.log( `${done.name}: archived ${done.files} files, ${gb(done.bytes)} (${gb(done.sent)} new) as ${done.key}` + (done.cleared ? `; cleared ${gb(done.cleared)} of Studio's cache` : ""), ); } } else if (command === "restore") { const at = flag("--at"); const [name, dest] = args; if (!name) throw new Error("Usage: bun run publisher restore [folder] [--at ]"); const done = await restoreProject(name, { ...auth, at, dest, log: (line) => console.error(line) }); console.log(`${done.name}: restored ${done.files} files, ${gb(done.bytes)}, to ${done.dest}`); } else if (command === "archives") { for (const p of await listArchived(auth)) console.log(`${p.name} last archived ${p.archivedAt}${p.versions > 1 ? `, ${p.versions} times` : ""}`); } else if (command === "cleanup") { const posted = args.includes("--posted"); const dryRun = args.includes("--dry-run"); let folders = args.filter((arg) => !arg.startsWith("--")); if (args.includes("--all")) { if (posted) throw new Error("--posted vouches for one film: name its folder instead of --all"); folders = await projectFolders(); } if (!folders.length) throw new Error("Usage: bun run publisher cleanup … | --all [--posted] [--dry-run]"); const size = (files) => gb(files.reduce((sum, file) => sum + file.size, 0)); const names = (files) => files.map((file) => file.name).join(", "); for (const folder of folders) { try { const done = await cleanupProject(resolve(folder), { ...auth, posted, dryRun, log: (l) => console.error(l) }); if (done.skipped) { console.log(`${done.name}: kept, ${done.skipped}`); continue; } const states = new Set(done.posts.flatMap((p) => p.platforms.map((t) => `${t.platform} ${t.state}`))); const out = states.size ? `posted: ${[...states].join(", ")}` : "posted, as you confirmed"; if (done.dryRun) { const listed = done.clips.length ? `: ${names(done.clips)}` : ""; console.log( `${done.name}: ${out}. Would archive it again, then delete ${done.clips.length} exported clips ` + `(${size(done.clips)}) whose bytes the archive holds${listed}, and the project (${gb(done.bytes)})`, ); continue; } console.log( `${done.name}: ${out}. Archived (${gb(done.archived.sent)} new); deleted ${done.deleted.length} exported ` + `clips (${size(done.deleted)}) and the project (${gb(done.bytes)})` + (done.kept.length ? `; kept ${names(done.kept)}, exported again since it was archived` : "") + `. Bring it back with: bun run publisher restore ${done.name}`, ); } catch (error) { console.error(`${basename(folder)}: ${error.message}`); process.exitCode = 1; } } } else { const done = await archiveRecentMedia(auth); console.log( `Checked ${done.posts} posts: copied ${done.copied} uploads to the archive; ${done.gone} had expired`, ); } } catch (error) { console.error(error.message); process.exitCode = 1; } } else if (command === "footage") { try { if (!config.token) throw new Error("Connect the helper first"); const args = process.argv.slice(3); const at = args.indexOf("--daily"); if (at !== -1) { const daily = Number(args[at + 1]); if (!Number.isInteger(daily) || daily < 1) throw new Error("--daily takes how many clips a day, 1 or more"); config.footage = { ...config.footage, daily }; await save(); } if (!config.platforms.includes(FOOTAGE)) throw new Error("Connect the YouTube footage archive in the composer's Accounts first"); const request = archiveClient({ origin: config.origin, token: config.token }); if (args.includes("--now")) { const done = await uploadFootage({ stateDir, config, request, log: (line) => console.error(line) }); if (done.busy) console.log("The helper is putting clips up right now"); else console.log(`${done.uploaded} clips up, ${done.left} to go`); } const state = await readState(stateDir); const due = await dueClips(state, { request }); const exported = due.filter((clip) => clip.from === "exported").length; const today = state.days[new Date().toISOString().slice(0, 10)] ?? 0; const videos = Object.values(state.clips).flat().length; console.log( `youtube.com/${config.footage?.channel}: ${videos} videos of ${Object.keys(state.clips).length} clips up, ` + `${today} of ${config.footage?.daily ?? DAILY} today. ` + `Waiting: ${exported} in Exported, ${due.length - exported} only in the archive.`, ); if (state.pausedUntil > Date.now()) console.log(`Paused until ${new Date(state.pausedUntil).toLocaleString()} after: ${state.lastError?.message}`); } catch (error) { console.error(error.message); process.exitCode = 1; } } else if (command === "post-records") { let oauth; try { if (!config.token) throw new Error("Connect the helper first"); const args = process.argv.slice(3); const write = args.includes("--write"); const job = args.includes("--job") ? args[args.indexOf("--job") + 1] : undefined; if (args.includes("--job") && !/^[a-f0-9-]{36}$/.test(job ?? "")) throw new Error("Usage: bun run publisher post-records [--write] [--job ]"); if (write) oauth = await createOAuth(stateDir).catch((error) => { // The running helper holds the sign-in; two processes refreshing it would end it. if (error.code === "EADDRINUSE") throw new Error( "The helper is running and holds the Bluesky sign-in. Stop it (bun run publisher uninstall, or Ctrl+C " + "where it runs), run this again, then start it (bun run publisher install).", ); throw error; }); const done = await reconcilePostRecords({ api, job, authorization: await oauth?.agent() }); const would = write ? "" : "would be "; console.log( `${done.create} ${would}created, ${done.update} ${would}updated, ${done.delete} ${would}deleted, ` + `${done.none} already right` + (done.failed ? `, ${done.failed} FAILED` : "") + (done.orphans ? `, ${done.orphans} records without a post` : "") + (write ? "" : ". Nothing was written; add --write to do it."), ); if (done.failed) process.exitCode = 1; } catch (error) { console.error(error.message); process.exitCode = 1; } finally { await oauth?.close(); } } else if (command === "oauth-key") { const file = join(stateDir, "oauth/client-key.json"); const { created, jwk } = await ensureClientKey(file, `publisher-${new Date().toISOString().slice(0, 10)}`); console.log( created ? `Made a signing key at ${file}. Put its public half below in jwks.keys of src/lib/publisher/oauth-client.json, deploy the site, restart the helper, then connect Grain / Bluesky again.` : `${file} already holds this key; jwks.keys in src/lib/publisher/oauth-client.json must list it:`, ); console.log(JSON.stringify(jwk)); } else console.log( "Usage: bun run publisher connect | start | install | uninstall | draft | oauth-key |\n" + " archive … [--keep-cache] | restore [folder] [--at