Something went wrong. Try again.
various tools I have been using throughout the years
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709use anyhow::{Context, Result};use clap::{Args, Parser};use console::style;use similar::{ChangeTag, TextDiff};use std::collections::{BTreeSet, HashSet};use std::fs;use std::io::Write;use std::path::{Path, PathBuf};use std::process::{Command, ExitCode};
/// dam (diff-and-merge) — sync line-oriented text files between a local/// directory and a remote host over rsync.////// dam merges files as the de-duplicated union of their local and remote/// lines: local lines keep their original order and remote-only lines are/// appended. It is intended for list-like data (allow-lists, hosts files,/// playlists, word lists), not for arbitrary documents — blank lines and/// duplicate lines are not preserved by the merge.////// --local and --remote may be directories or single files. When --local is/// an existing directory dam syncs the whole tree (recursing into/// subdirectories); otherwise it treats both paths as a single file. To mirror/// a directory into a new local location, create that directory first.////// Because the merge is a union, dam never removes a line: a line deleted on/// one side reappears from the other. Removing data is a manual edit on both/// sides, or a one-shot rsync.////// dam has a plan/apply workflow, with three mutually exclusive modes:////// --plan (default) Show the changes each direction would make. Writes/// nothing./// --pull Apply the merge to the local directory (remote → local)./// --push Apply the merge to the remote host (local → remote).////// Progress and the summary are written to stderr and the plan diff to stdout,/// so `dam --plan > changes.diff` captures just the differences.////// Examples:/// # See what would change on either side/// dam --host myserver --local ~/lists --remote /srv/lists////// # Bring the local directory up to date/// dam --host myserver --local ~/lists --remote /srv/lists --pull////// # Bring the remote up to date/// dam --host myserver --local ~/lists --remote /srv/lists --push////// # Fail (exit 1) if the two sides have drifted — handy in CI/// dam --host myserver --local ~/lists --remote /srv/lists --plan --exit-code#[derive(Parser, Debug)]#[command( author, version, about = "Diff-and-merge line-oriented text files between a local directory and a remote host over rsync", arg_required_else_help = true, verbatim_doc_comment)]pub struct Cli { /// SSH host alias (as configured in ~/.ssh/config) to sync with. #[arg(long, required = true)] host: String,
/// Local directory (or single file) containing the line-oriented text. #[arg(long, required = true)] local: PathBuf,
/// Remote directory (or single file) to sync from/to. #[arg(long, required = true)] remote: PathBuf,
#[command(flatten)] mode: ModeFlags,
#[command(flatten)] elevate: ElevateFlags,
/// Exit with status 1 if --plan finds pending changes. Processing errors /// always exit with status 2. #[arg(long)] exit_code: bool,
/// I/O timeout in seconds handed to rsync (0 disables the timeout). #[arg(long, default_value_t = 30)] timeout: u64,}
/// The mutually exclusive plan/pull/push flags. `--plan` (preview) is the/// default when none is given.#[derive(Args, Debug)]#[group(required = false, multiple = false)]struct ModeFlags { /// Show the changes each direction would make, without writing anything /// (the default). #[arg(long)] plan: bool,
/// Apply the merge to the local directory (remote → local). #[arg(long)] pull: bool,
/// Apply the merge to the remote host (local → remote). #[arg(long)] push: bool,}
impl ModeFlags { fn mode(&self) -> Mode { if self.push { Mode::Push } else if self.pull { Mode::Pull } else { Mode::Plan } }}
/// The optional privilege-elevation flags. The remote rsync runs unprivileged/// unless one is given; the local rsync is never elevated.#[derive(Args, Debug)]#[group(required = false, multiple = false)]struct ElevateFlags { /// Elevate the remote rsync with `sudo` (via --rsync-path). #[arg(long)] sudo: bool,
/// Elevate the remote rsync with `doas` (via --rsync-path). #[arg(long)] doas: bool,}
impl ElevateFlags { /// The command to prefix rsync with, if any. fn command(&self) -> Option<&'static str> { if self.sudo { Some("sudo") } else if self.doas { Some("doas") } else { None } }}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]enum Mode { Plan, Pull, Push,}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]enum FileStatus { Created, Updated, Unchanged,}
#[derive(Debug, Default)]struct Tally { created: u32, updated: u32, unchanged: u32, errors: u32,}
impl Tally { fn record(&mut self, status: FileStatus) { match status { FileStatus::Created => self.created += 1, FileStatus::Updated => self.updated += 1, FileStatus::Unchanged => self.unchanged += 1, } }
/// Number of files that would change (or did change). fn changed(&self) -> u32 { self.created + self.updated }
fn print(&self, header: &str) { eprintln!("\n{header}:"); eprintln!(" Created: {}", self.created); eprintln!(" Updated: {}", self.updated); eprintln!(" Unchanged: {}", self.unchanged); if self.errors > 0 { eprintln!(" Errors: {}", self.errors); } }}
struct FileSyncWorker { host_alias: String, local_path: PathBuf, remote_path: PathBuf, mode: Mode, timeout: u64, elevate: Option<&'static str>,}
/// One file to reconcile: where it lives locally, where its downloaded remote/// copy is, and where a push stages its merged result. `label` is used in/// progress and diff output.struct Pair { label: String, local: PathBuf, remote: PathBuf, staging: PathBuf,}
impl FileSyncWorker { /// Single-file mode when --local is not an existing directory; otherwise /// the whole directory tree is synced. fn single_file(&self) -> bool { !self.local_path.is_dir() }
fn run(&self) -> Result<Tally> { let single_file = self.single_file();
// Always pull the remote into a temp dir so we have both sides to // compare; nothing is uploaded unless --push has changes to apply. let remote_tmp = tempfile::tempdir().context("Failed to create temporary directory")?; let remote_dir = remote_tmp.path(); self.download(remote_dir, single_file)?;
let action = match self.mode { Mode::Plan => "Planning", Mode::Pull => "Pulling", Mode::Push => "Pushing", }; let unit = if single_file { "file" } else { "files" }; eprintln!("{action} {unit} for {}", self.host_alias);
// Staging dir holds the files we will upload when pushing. let staging_tmp = tempfile::tempdir().context("Failed to create staging directory")?; let staging = staging_tmp.path();
let mut tally = Tally::default(); for pair in self.build_pairs(remote_dir, staging, single_file)? { match self.process(&pair) { Ok(status) => tally.record(status), Err(e) => { eprintln!("Error processing {}: {e:#}", pair.label); tally.errors += 1; } } }
let header = match self.mode { Mode::Plan => "Plan summary", Mode::Pull => "Pull complete", Mode::Push => "Push complete", }; tally.print(header);
if self.mode == Mode::Push && tally.changed() > 0 { self.upload(staging, single_file)?; }
Ok(tally) }
/// Enumerate the file pair(s) to process: one per file in single-file /// mode, or one per file present in either directory tree otherwise. fn build_pairs( &self, remote_dir: &Path, staging: &Path, single_file: bool, ) -> Result<Vec<Pair>> { if single_file { let name = self.remote_file_name()?; return Ok(vec![Pair { label: self.local_path.display().to_string(), local: self.local_path.clone(), remote: remote_dir.join(name), staging: staging.join(name), }]); }
let mut names = BTreeSet::new(); collect_files(&self.local_path, &self.local_path, &mut names)?; collect_files(remote_dir, remote_dir, &mut names)?; Ok(names .into_iter() .map(|rel| Pair { label: rel.display().to_string(), local: self.local_path.join(&rel), remote: remote_dir.join(&rel), staging: staging.join(&rel), }) .collect()) }
fn process(&self, pair: &Pair) -> Result<FileStatus> { let local = read_optional(&pair.local)?; let remote = read_optional(&pair.remote)?; let merged = merge(local.as_deref(), remote.as_deref());
match self.mode { Mode::Plan => Ok(plan( &pair.label, local.as_deref(), remote.as_deref(), &merged, )), Mode::Pull => apply(&pair.label, local.as_deref(), &merged, &pair.local, "local"), Mode::Push => apply( &pair.label, remote.as_deref(), &merged, &pair.staging, "remote", ), } }
/// The file name component of --remote, required in single-file mode. fn remote_file_name(&self) -> Result<&std::ffi::OsStr> { self.remote_path .file_name() .context("--remote must name a file in single-file mode") }
fn download(&self, dest: &Path, single_file: bool) -> Result<()> { // A trailing slash copies a directory's *contents*; without one rsync // copies the single file into the temp dir under its own name. let source = if single_file { format!("{}:{}", self.host_alias, self.remote_path.display()) } else { format!("{}:{}/", self.host_alias, self.remote_path.display()) }; self.rsync(&source, &dest.to_string_lossy()) }
fn upload(&self, staging: &Path, single_file: bool) -> Result<()> { let (source, dest) = if single_file { let name = self.remote_file_name()?; ( staging.join(name).to_string_lossy().into_owned(), format!("{}:{}", self.host_alias, self.remote_path.display()), ) } else { ( format!("{}/", staging.display()), format!("{}:{}/", self.host_alias, self.remote_path.display()), ) }; eprintln!("Uploading to {dest}"); self.rsync(&source, &dest) }
/// Run rsync with non-interactive SSH so it fails fast in automation /// instead of blocking on a password or host-key prompt. When elevation /// is requested it is applied to the *remote* rsync via --rsync-path; the /// local rsync always runs unprivileged. fn rsync(&self, source: &str, dest: &str) -> Result<()> { let mut cmd = Command::new("rsync"); cmd.arg("-az").arg("-e").arg("ssh -o BatchMode=yes"); if let Some(elevate) = self.elevate { cmd.arg(format!("--rsync-path={elevate} rsync")); } if self.timeout > 0 { cmd.arg(format!("--timeout={}", self.timeout)); } cmd.arg(source).arg(dest);
let status = cmd .status() .context("Failed to execute rsync (is it installed?)")?; if !status.success() { anyhow::bail!("rsync exited with status: {status}"); } Ok(()) }}
/// Which end of the sync a planned change applies to.#[derive(Clone, Copy)]enum Side { Local, Remote,}
impl Side { /// The tag printed in front of this side's diff, colour-coded so local and /// remote are easy to tell apart at a glance. fn tag(self) -> console::StyledObject<&'static str> { match self { Side::Local => style(" local ").black().on_cyan().bold(), Side::Remote => style(" remote ").black().on_magenta().bold(), } }}
/// Preview the changes both directions would make to a single file, printing/// a per-side diff for each side that has pending changes. Writes nothing.fn plan(label: &str, local: Option<&str>, remote: Option<&str>, merged: &str) -> FileStatus { let local_current = local.unwrap_or(""); let remote_current = remote.unwrap_or(""); let local_changes = local_current != merged; let remote_changes = remote_current != merged;
if !local_changes && !remote_changes { return FileStatus::Unchanged; }
println!("{}", style(label).bold().underlined()); if local_changes { print_side_diff(Side::Local, local_current, merged); } if remote_changes { print_side_diff(Side::Remote, remote_current, merged); } println!(); FileStatus::Updated}
/// Apply the merged content to one side, writing `dest` atomically. `current`/// is that side's existing content (`None` if the file is absent there).fn apply( label: &str, current: Option<&str>, merged: &str, dest: &Path, side: &str,) -> Result<FileStatus> { let exists = current.is_some(); if current.unwrap_or("") == merged { return Ok(FileStatus::Unchanged); }
// Never create an empty new file. if !merged.is_empty() || exists { write_atomic(dest, merged)?; }
if exists { eprintln!("Updating {label} on {side}"); Ok(FileStatus::Updated) } else { eprintln!("Creating {label} on {side}"); Ok(FileStatus::Created) }}
/// Recursively collect the paths of regular files under `dir`, expressed/// relative to `root`. Symlinks are skipped to avoid traversal loops.fn collect_files(root: &Path, dir: &Path, out: &mut BTreeSet<PathBuf>) -> Result<()> { for entry in fs::read_dir(dir).with_context(|| format!("Failed to read {}", dir.display()))? { let entry = entry?; let path = entry.path(); let file_type = entry.file_type()?; if file_type.is_dir() { collect_files(root, &path, out)?; } else if file_type.is_file() { let rel = path .strip_prefix(root) .with_context(|| format!("{} is not under {}", path.display(), root.display()))?; out.insert(rel.to_path_buf()); } } Ok(())}
/// Read a file, returning `None` if it does not exist.fn read_optional(path: &Path) -> Result<Option<String>> { match fs::read_to_string(path) { Ok(s) => Ok(Some(s)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(e) => Err(e).with_context(|| format!("Failed to read {}", path.display())), }}
/// Write `contents` to `path` atomically: stage in a temp file in the same/// directory and rename into place, so a crash mid-write can't truncate or/// corrupt the target.fn write_atomic(path: &Path, contents: &str) -> Result<()> { let dir = path .parent() .with_context(|| format!("{} has no parent directory", path.display()))?; fs::create_dir_all(dir).with_context(|| format!("Failed to create {}", dir.display()))?;
let mut tmp = tempfile::NamedTempFile::new_in(dir) .with_context(|| format!("Failed to create temp file in {}", dir.display()))?; tmp.write_all(contents.as_bytes()) .with_context(|| format!("Failed to write {}", path.display()))?; tmp.persist(path) .map_err(|e| e.error) .with_context(|| format!("Failed to persist {}", path.display()))?; Ok(())}
/// Merge two sides into the file content dam would write.////// When the file exists on both sides the result is the de-duplicated union/// of their non-empty lines: the local lines in their original order, then/// the lines only the remote has. Order is never changed, so pushing a/// reordered file propagates that order instead of fighting it, and merging/// an already-merged file is a no-op. When the file exists on only one side/// that side's content is propagated verbatim.fn merge(local: Option<&str>, remote: Option<&str>) -> String { match (local, remote) { (Some(l), Some(r)) => { let merged = union_lines(l, r); if merged.is_empty() { String::new() } else { format!("{}\n", merged.join("\n")) } } (Some(only), None) | (None, Some(only)) => only.to_string(), (None, None) => String::new(), }}
/// De-duplicated union of the non-blank lines of `a` and `b`, in first-seen/// order: all of `a`'s lines, then the lines only `b` has. A trailing/// carriage return is stripped so CRLF and LF inputs compare equal.fn union_lines(a: &str, b: &str) -> Vec<String> { let mut seen = HashSet::new(); a.lines() .chain(b.lines()) .map(|line| line.trim_end_matches('\r')) .filter(|line| !line.trim().is_empty()) .filter(|line| seen.insert(*line)) .map(str::to_string) .collect()}
/// Print one side's planned change to stdout: a tagged header naming the side/// with its added/removed counts, followed by only the changed lines.////// `current` is that side's existing content and `merged` the proposed result,/// so `+` lines are gained and `-` lines lost. Unchanged lines are omitted to/// keep the plan focused on the delta.fn print_side_diff(side: Side, current: &str, merged: &str) { // Diff the lines themselves rather than the raw text: `lines()` drops the // terminators, so a missing trailing newline never shows up as a change. let current: Vec<&str> = current.lines().collect(); let merged: Vec<&str> = merged.lines().collect(); let diff = TextDiff::from_slices(¤t, &merged);
// `true` marks an inserted line, `false` a deleted one; equal lines are // dropped here so the body below is purely the delta. let mut changes: Vec<(bool, &str)> = Vec::new(); let (mut added, mut removed) = (0u32, 0u32); for change in diff.iter_all_changes() { match change.tag() { ChangeTag::Insert => added += 1, ChangeTag::Delete => removed += 1, ChangeTag::Equal => continue, } changes.push((change.tag() == ChangeTag::Insert, change.value())); }
println!( " {} {} {}", side.tag(), style(format!("+{added}")).green(), style(format!("-{removed}")).red(), ); for (inserted, line) in changes { if inserted { println!(" {}", style(format!("+{line}")).green()); } else { println!(" {}", style(format!("-{line}")).red()); } }}
/// Parse the command line and run the configured sync, returning the process/// exit code.////// # Errors////// Returns an error when the sync cannot run at all: rsync fails or is/// missing, or a temporary directory cannot be created. Per-file processing/// failures do not abort the run; they are reported on stderr and mapped to/// exit code 2.pub fn run() -> Result<ExitCode> { let cli = Cli::parse();
let mode = cli.mode.mode();
let worker = FileSyncWorker { host_alias: cli.host, local_path: cli.local, remote_path: cli.remote, mode, timeout: cli.timeout, elevate: cli.elevate.command(), }; let tally = worker.run()?;
let code = if tally.errors > 0 { ExitCode::from(2) } else if cli.exit_code && mode == Mode::Plan && tally.changed() > 0 { ExitCode::from(1) } else { ExitCode::SUCCESS }; Ok(code)}
#[cfg(test)]mod tests { use super::*;
#[test] fn union_dedups_and_drops_blank_lines() { let got = union_lines("banana\napple\n\napple", "cherry\napple"); assert_eq!(got, vec!["banana", "apple", "cherry"]); }
#[test] fn union_keeps_local_order_and_appends_remote_extras() { let got = union_lines("file10\nfile2\nfile1", "file0\nfile2"); assert_eq!(got, vec!["file10", "file2", "file1", "file0"]); }
#[test] fn union_normalises_crlf() { let got = union_lines("a\r\nb\r\n", "b\r\nc\r"); assert_eq!(got, vec!["a", "b", "c"]); }
#[test] fn merge_unions_when_both_present() { assert_eq!(merge(Some("b\na"), Some("c\na")), "b\na\nc\n"); }
#[test] fn merge_keeps_local_order_when_sides_hold_the_same_lines() { // A reordered playlist: pushing must propagate the local order, so a // follow-up plan sees the local side as unchanged (no push/pull loop). let merged = merge(Some("t3\nt1\nt2\n"), Some("t1\nt2\nt3\n")); assert_eq!(merged, "t3\nt1\nt2\n"); assert_eq!( plan("f", Some("t3\nt1\nt2\n"), Some(&merged), &merged), FileStatus::Unchanged ); }
#[test] fn merge_is_idempotent() { // Once both sides hold the merged content, further runs change nothing. let merged = merge(Some("z\nm\na\n"), Some("m\nq\n")); assert_eq!(merge(Some(&merged), Some(&merged)), merged); }
#[test] fn merge_propagates_single_side_verbatim() { // Order is preserved and nothing is sorted when only one side has it. assert_eq!(merge(Some("z\na\n"), None), "z\na\n"); assert_eq!(merge(None, Some("z\na\n")), "z\na\n"); }
#[test] fn merge_is_empty_when_neither_present() { assert_eq!(merge(None, None), ""); }
#[test] fn merge_of_blank_only_content_is_empty() { assert_eq!(merge(Some("\n\n"), Some(" ")), ""); }
#[test] fn plan_reports_unchanged_when_sides_match() { let merged = merge(Some("a\nb\n"), Some("a\nb\n")); assert_eq!( plan("f", Some("a\nb\n"), Some("a\nb\n"), &merged), FileStatus::Unchanged ); }
#[test] fn plan_reports_change_when_sides_differ() { let merged = merge(Some("a"), Some("b")); assert_eq!( plan("f", Some("a"), Some("b"), &merged), FileStatus::Updated ); }
#[test] fn apply_creates_updates_and_skips() { let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("f");
// Absent target with content -> Created and written. let created = apply("f", None, "a\nb\n", &dest, "local").unwrap(); assert_eq!(created, FileStatus::Created); assert_eq!(fs::read_to_string(&dest).unwrap(), "a\nb\n");
// Existing target, new content -> Updated. let updated = apply("f", Some("a\nb\n"), "a\nb\nc\n", &dest, "local").unwrap(); assert_eq!(updated, FileStatus::Updated); assert_eq!(fs::read_to_string(&dest).unwrap(), "a\nb\nc\n");
// No difference -> Unchanged, no write needed. let same = apply("f", Some("x"), "x", &dest, "local").unwrap(); assert_eq!(same, FileStatus::Unchanged); }}