Something went wrong. Try again.
A simple DAV sync application / cli
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197import { readFileSync, rmSync, writeFileSync } from "node:fs";import { mkdir } from "node:fs/promises";import { join } from "node:path";import type { FetchLike } from "@ordner.fyi/dav-client";import { detect, type Probe } from "./detect.ts";import type { Outcome, SyncEvent } from "../shared.ts";import { execute } from "./execute.ts";import { ignorer } from "./ignore.ts";import { LocalSide } from "./local.ts";import { isBelow } from "./paths.ts";import { plan } from "./plan.ts";import { KeychainUnavailable } from "../keychain.ts";import { explain, RemoteSide } from "./remote.ts";import { isTransient } from "./transient.ts";import type { StateStore } from "./state.ts";import type { Action, FolderConfig, Plan, States } from "./types.ts";
/** How long locally deleted files stay in .davsync/trash. */const TRASH_DAYS = 30;
export type { SyncEvent };
export interface SyncOptions { /** Detect and plan, log the plan, change nothing. */ dryRun?: boolean; /** Where per-folder lock files go. */ lockDir: string; onEvent?: (event: SyncEvent) => void; /** The folder's password (from the keychain); may throw KeychainUnavailable. Without it, none is sent. */ password?: (folder: FolderConfig) => string; /** For tests: talk to an in-process server. */ fetch?: FetchLike;}
export interface SyncResult { run: number; /** skipped: something that passes by itself (the network, a locked keychain) stopped it; try again later. */ status: "ok" | "partial" | "failed" | "planned" | "skipped"; plan: Plan | null; outcomes: Outcome[]; error: string | null;}
/** One sync of one folder: scan both sides, detect, plan, execute, log. */export async function syncFolder(folder: FolderConfig, store: StateStore, options: SyncOptions): Promise<SyncResult> { const release = await lock(options.lockDir, folder.id); store.interruptRuns(folder.id); const run = store.startRun(folder.id, options.dryRun ? "planned" : "running"); const emit = (event: SyncEvent) => options.onEvent?.(event); const phase = (name: "scanning" | "detecting" | "planning" | "executing") => emit({ type: "phase", folder: folder.id, phase: name }); let result: SyncResult; try { const local = new LocalSide(folder.localPath); const password = folder.username && options.password ? options.password(folder) : ""; const remote = new RemoteSide({ remoteUrl: folder.remoteUrl, username: folder.username, password, ...(options.fetch ? { fetch: options.fetch } : {}), }); const ignored = ignorer(folder.ignore); const base = store.loadBase(folder.id);
phase("scanning"); if (!options.dryRun) await local.tidy(TRASH_DAYS * 86_400_000); const [localNow, remoteNow] = await Promise.all([local.scan(ignored), remote.scan(base, ignored, folder.trustCollectionEtags)]); guardAgainstEmpty(base.size, localNow.size, "local folder", folder.localPath); guardAgainstEmpty(base.size, remoteNow.size, "remote folder", folder.remoteUrl); const states: States = { base, local: localNow, remote: remoteNow, readOnly: remote.readOnly };
phase("detecting"); const probe: Probe = { headLocal: (snapshot) => local.head(snapshot), hashLocal: (snapshot) => local.hash(snapshot), headRemote: (snapshot) => remote.head(snapshot), }; const changes = await detect(states, probe);
phase("planning"); const actions = plan(states, changes.local, changes.remote); const thePlan: Plan = { actions, local: changes.local, remote: changes.remote }; emit({ type: "planned", folder: folder.id, run, actions: actions.length });
if (options.dryRun) { store.transaction(() => actions.forEach((action, index) => store.logAction(run, index, action, "planned"))); const summary = countBy(actions.map((action) => action.type)); store.finishRun(run, "planned", summary, null); emit({ type: "finished", folder: folder.id, run, status: "planned", summary, error: null }); return { run, status: "planned", plan: thePlan, outcomes: [], error: null }; }
phase("executing"); const runTag = `${Date.now()}-${run}`; const outcomes = await execute({ folder: folder.id, local, remote, store, states, runTag, onAction: (index, action, outcome) => { store.logAction(run, index, action, outcome.status, outcome.detail); emit({ type: "action", folder: folder.id, run, index, action, outcome }); } }, actions);
if (folder.trustCollectionEtags) rememberCollectionEtags(folder.id, store, states, actions, outcomes);
const failed = outcomes.some((outcome) => outcome.status !== "done" && outcome.status !== "held"); const status = failed ? "partial" : "ok"; const summary = countBy(actions.map((action, index) => (outcomes[index]!.status === "done" ? action.type : outcomes[index]!.status))); store.finishRun(run, status, summary, null); store.pruneRuns(folder.id); emit({ type: "finished", folder: folder.id, run, status, summary, error: null }); result = { run, status, plan: thePlan, outcomes, error: null }; } catch (error) { const transient = isTransient(error); const status = transient ? "skipped" : "failed"; const message = transient ? `${transientReason(error, folder)}; trying again at the next sync` : error instanceof Error ? error.message : String(error); store.finishRun(run, status, null, message); emit({ type: "finished", folder: folder.id, run, status, summary: {}, error: message }); result = { run, status, plan: null, outcomes: [], error: message }; } finally { release(); } return result;}
/** What stopped a sync that will be tried again, in words. */function transientReason(error: unknown, folder: FolderConfig): string { if (error instanceof KeychainUnavailable) return `The keychain is locked or not available (${error.message})`; return explain(error, folder.serverUrl, "").message.replace(/\.$/, "");}
/** * A side that comes back empty although files were synced before is far more likely * an unmounted disk, a wrong URL or an emptied share than a person deleting * everything, and syncing it would delete everything on the other side. Refuse. */function guardAgainstEmpty(known: number, found: number, what: string, where: string): void { if (known > 0 && found === 0) { throw new Error( `The ${what} (${where}) is empty, but ${known} entries were synced before. Refusing to delete them on the other side; ` + `if this is intended, remove the folder from davsync and add it again.`, ); }}
/** * With trusted collection etags, a directory's remote etag is remembered once * everything below it was handled, so the next scan can skip it while it stays the same. */function rememberCollectionEtags(folder: string, store: StateStore, states: States, actions: Action[], outcomes: Outcome[]): void { const troubled = actions .filter((_, index) => outcomes[index]!.status !== "done") .map((action) => ("path" in action ? action.path : action.type === "move" ? action.to : action.base)); store.transaction(() => { for (const [path, entry] of states.base) { const scanned = states.remote.get(path); if (entry.kind !== "dir" || !scanned || scanned.etag === entry.remote.etag) continue; if (troubled.some((other) => other === path || isBelow(other, path))) continue; const updated = { ...entry, remote: { ...entry.remote, etag: scanned.etag } }; store.putBase(folder, updated); states.base.set(path, updated); } });}
function countBy(keys: string[]): Record<string, number> { const counts: Record<string, number> = {}; for (const key of keys) counts[key] = (counts[key] ?? 0) + 1; return counts;}
/** * One sync per folder at a time, across processes (the service and the setup * window may both want to). A lock whose process is gone is taken over. */async function lock(dir: string, folder: string): Promise<() => void> { await mkdir(dir, { recursive: true }); const file = join(dir, `${folder}.lock`); for (let attempt = 0; attempt < 2; attempt++) { try { writeFileSync(file, String(process.pid), { flag: "wx" }); return () => rmSync(file, { force: true }); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; const holder = Number(readFileSync(file, "utf8")); if (holder && holder !== process.pid && isAlive(holder)) break; if (holder === process.pid) break; rmSync(file, { force: true }); } } throw new Error("This folder is already being synced");}
function isAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; }}