Something went wrong. Try again.
Convert CSV banking statements to hledger journals
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575//! The end-to-end acceptance suite (`M1-AC-*`).//!//! These are the cross-cutting cases from REQUIREMENTS.md that only mean//! anything against the finished pipeline. Every other test in this repository//! is about one module; each of these runs a whole command against a project//! on disk and checks what a user would see.//!//! Acceptance IDs that are already exercised where they belong are not//! repeated here. `M1-AC-002` lives with discovery, `M1-AC-008` through//! `M1-AC-010` with the interactive session, and so on; this file holds the//! ones whose statement spans the pipeline.
const std = @import("std");
// -- tests -------------------------------------------------------------------
const cli = @import("cli.zig");const diag = @import("diag.zig");const journal_parse = @import("journal/parse.zig");const Manifest = @import("config/Manifest.zig");
const expect = std.testing.expect;const expectEqual = std.testing.expectEqual;const expectEqualStrings = std.testing.expectEqualStrings;const test_io = std.testing.io;const Writer = std.Io.Writer;const Session = @import("ui/Session.zig");
const manifest_text = @embedFile("testdata/manifest_text.yaml");
const common_rules = @embedFile("testdata/rules_text.yaml");
/// Two rows, two rules, in an order that would be visible if it were lost.const current_csv = "Date,Amount,Posting\n2026-09-11,-1.50,market\n2026-09-12,-9.90,rail\n";
/// The approved journal a matching history pair holds.const approved = @embedFile("testdata/approved.journal");
const history_csv = "Date,Amount,Posting\n2026-03-01,-2.00,market\n";
pub const Project = struct { tmp: std.testing.TmpDir,
const Outcome = struct { has_error: bool, report: []const u8, };
fn init(project_manifest: []const u8, rules_text: []const u8, csv_text: []const u8) !Project { var project: Project = .{ .tmp = std.testing.tmpDir(.{ .iterate = true }) }; try project.write("project.yaml", project_manifest); try project.write("rules.yaml", rules_text); try project.tmp.dir.createDirPath(test_io, "data"); try project.write("data/2026.csv", csv_text); return project; }
/// Adds a historical pair: a CSV and the approved journal beside it. fn withHistory(self: *Project, journal_text: []const u8) !void { try self.write("data/2025.csv", history_csv); try self.write("data/2025.csv.journal", journal_text); }
fn deinit(self: *Project) void { self.tmp.cleanup(); }
fn write(self: *Project, path: []const u8, data: []const u8) !void { try self.tmp.dir.writeFile(test_io, .{ .sub_path = path, .data = data }); }
fn exists(self: *Project, path: []const u8) bool { _ = self.tmp.dir.statFile(test_io, path, .{}) catch return false; return true; }
fn read(self: *Project, arena: std.mem.Allocator, path: []const u8) ![]const u8 { return self.tmp.dir.readFileAlloc(test_io, path, arena, .unlimited); }
fn run(self: *Project, arena: *std.heap.ArenaAllocator, command: cli.Invocation.Command) Outcome { var allocating = std.Io.Writer.Allocating.init(arena.allocator()); defer allocating.deinit();
const failure = cli.execute(arena.allocator(), test_io, &allocating.writer, self.tmp.dir, .{ .command = command, .manifest_path = "project.yaml", }) catch { return .{ .has_error = true, .report = "out of memory\n" }; }; const report = allocating.toOwnedSlice() catch { return .{ .has_error = true, .report = "out of memory\n" }; };
return .{ .has_error = failure, .report = report }; }};
test "M1-AC-001: a successful conversion" { var project = try Project.init(manifest_text, common_rules, current_csv); defer project.deinit(); try project.withHistory(approved); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
// One unpaired CSV whose rows each match exactly one rule, a history pair // that is structurally equal, valid saved rules, and no target yet. try expect(!project.exists("data/2026.csv.journal")); const outcome = project.run(&arena, .convert); try expect(!outcome.has_error);
const written = try project.read(alloc, "data/2026.csv.journal"); try expectEqualStrings( \\2026-09-11 market \\ assets:bank -1.50 EUR \\ expenses:food \\ \\2026-09-12 rail \\ assets:bank -9.90 EUR \\ expenses:travel \\ , written);
// Parseable, one two-posting transaction per row, in source order. var diags = diag.List.init(alloc); const parsed = try journal_parse.parse(alloc, "data/2026.csv.journal", written, &diags); try expectEqual(@as(usize, 2), parsed.transactions.len); try expectEqualStrings("market", parsed.transactions[0].narration); try expectEqualStrings("rail", parsed.transactions[1].narration); for (parsed.transactions) |transaction| { try expectEqualStrings("assets:bank", transaction.postings[0].account); try expect(transaction.postings[0].amount != null); // M1-JRN-005: the destination amount is hledger's to infer. try expect(transaction.postings[1].amount == null); }
// M1-SAFE-003: only the target appeared. try expectEqualStrings(current_csv, try project.read(alloc, "data/2026.csv")); try expectEqualStrings(approved, try project.read(alloc, "data/2025.csv.journal"));}
test "M1-AC-005: a formatting-only historical difference still passes" { // Deeper indentation, a wider gap before the amount, CRLF endings, a // trailing blank line and a wider decimal scale. The parsed structure and // the exact decimal value are unchanged. // // Surrounding whitespace on the narration is deliberately not here. It is // not formatting under M1-HIST-009, which compares narrations exactly -- // see DESIGN_M1.md section 11.3. const reformatted = "2026-03-01 market\r\n" ++ "\t assets:bank \t -2.000 EUR\r\n" ++ " expenses:food\r\n" ++ "\r\n";
var project = try Project.init(manifest_text, common_rules, current_csv); defer project.deinit(); try project.withHistory(reformatted); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .convert); try expect(!outcome.has_error); try expect(project.exists("data/2026.csv.journal"));}
test "M1-AC-006: every structural historical difference is caught and named" { const cases = [_]struct { journal: []const u8, what: []const u8 }{ .{ .what = "account", .journal = "2026-03-01 market\n assets:other -2.00 EUR\n expenses:food\n", }, .{ .what = "amount value", .journal = "2026-03-01 market\n assets:bank -2.01 EUR\n expenses:food\n", }, .{ .what = "currency", .journal = "2026-03-01 market\n assets:bank -2.00 USD\n expenses:food\n", }, .{ .what = "date", .journal = "2026-03-02 market\n assets:bank -2.00 EUR\n expenses:food\n", }, .{ .what = "narration", .journal = "2026-03-01 grocer\n assets:bank -2.00 EUR\n expenses:food\n", }, .{ .what = "comment", .journal = "2026-03-01 market\n assets:bank -2.00 EUR ; note\n expenses:food\n", }, .{ .what = "posting order", .journal = "2026-03-01 market\n expenses:food -2.00 EUR\n assets:bank\n", }, .{ .what = "transaction order", .journal = approved ++ "\n2026-03-01 market\n assets:bank -2.00 EUR\n expenses:food\n", }, };
for (cases) |case| { var project = try Project.init(manifest_text, common_rules, current_csv); defer project.deinit(); try project.withHistory(case.journal); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .convert); try expect(outcome.has_error); // Identified, not merely refused: the report says where. try expect(std.mem.find(u8, outcome.report, "2025.csv.journal") != null); // And finalization did not happen. try expect(!project.exists("data/2026.csv.journal")); }}
test "M1-AC-020: a CSV with no data rows converts to a zero-byte journal" { var project = try Project.init(manifest_text, common_rules, current_csv); defer project.deinit(); try project.withHistory(approved); try project.write("data/2026.csv", "Date,Amount,Posting\n"); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .convert); try expect(!outcome.has_error); try expectEqualStrings("", try project.read(arena.allocator(), "data/2026.csv.journal"));}
// -- determinism and safe failure --------------------------------------------
test "M1-SAFE-001: the same project twice gives the same everything" { // Two independent runs over the same inputs must agree byte for byte: // report, exit code and journal. That covers the parts of a run that // could vary with allocation addresses, map iteration or accumulated // arena state. // // It does not, on its own, prove discovery is ordered. The files are // created in opposite orders here in the hope of shaking that out, but // tmpfs hands both directories back in the same order regardless, so // removing discovery's sort does not fail this test -- it fails // "M1-SAFE-001: history comes back in a deterministic order" in // discover.zig, which is where that particular guarantee is pinned. const history = [_][]const u8{ "2019", "2023", "2021", "2020", "2022" };
var first = try Project.init(manifest_text, common_rules, current_csv); defer first.deinit(); var second = try Project.init(manifest_text, common_rules, current_csv); defer second.deinit(); for ([_]*Project{ &first, &second }, 0..) |project, which| { for (0..history.len) |step| { const year = history[if (which == 0) step else history.len - 1 - step]; const csv = try std.testing.allocator.print( "data/{s}.csv", .{year}, ); defer std.testing.allocator.free(csv); const journal = try std.testing.allocator.print( "data/{s}.csv.journal", .{year}, ); defer std.testing.allocator.free(journal); try project.write(csv, history_csv); try project.write(journal, approved); } }
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
// Checking: matching and diagnostics, in the report. const checked_once = first.run(&arena, .check); const checked_twice = second.run(&arena, .check); try expectEqual(checked_once.has_error, checked_twice.has_error); try expectEqualStrings(checked_once.report, checked_twice.report);
// Generation: the journal itself. try expect(!first.run(&arena, .convert).has_error); try expect(!second.run(&arena, .convert).has_error); try expectEqualStrings( try first.read(alloc, "data/2026.csv.journal"), try second.read(alloc, "data/2026.csv.journal"), );}
test "M1-SAFE-001: a failing project reports the same diagnostics twice" { const ambiguous = @embedFile("testdata/ambiguous.yaml");
var first = try Project.init(manifest_text, common_rules, current_csv); defer first.deinit(); try first.write("rules.yaml", ambiguous); var second = try Project.init(manifest_text, common_rules, current_csv); defer second.deinit(); try second.write("rules.yaml", ambiguous);
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
// Two rules match every row and one matches none, so there is a spread of // diagnostics to get in a stable order. const once = first.run(&arena, .check); const twice = second.run(&arena, .check); try expect(once.has_error); try expectEqualStrings(once.report, twice.report);}
test "M1-SAFE-004: an amount too wide for the representation is a diagnostic" { var project = try Project.init(manifest_text, common_rules, current_csv); defer project.deinit(); // M1-CSV-019 states no bound; A1 accepts that a cell beyond an i64 // coefficient is diagnosed rather than represented. try project.write( "data/2026.csv", "Date,Amount,Posting\n2026-09-11," ++ &@as([100]u8, @splat('9')) ++ ",market\n", ); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .convert); try expect(outcome.has_error); try expect(!project.exists("data/2026.csv.journal"));}
test "M1-SAFE-004: running out of memory creates no target and leaves the rules alone" { // Every allocation count from none to well past a whole run, so the // failure lands at every stage in turn: reading, parsing, classifying, // generating and writing. var limit: usize = 0; while (limit < 600) : (limit += 7) { var project = try Project.init(manifest_text, common_rules, current_csv); defer project.deinit();
var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = limit, }); var arena = std.heap.ArenaAllocator.init(failing.allocator()); defer arena.deinit();
const outcome = project.run(&arena, .convert);
// Either the run finished, or it failed. Never a half-written target: // the journal is linked into place complete or not at all. if (!outcome.has_error) { const written = try project.read(std.testing.allocator, "data/2026.csv.journal"); defer std.testing.allocator.free(written); try expectEqualStrings( \\2026-09-11 market \\ assets:bank -1.50 EUR \\ expenses:food \\ \\2026-09-12 rail \\ assets:bank -9.90 EUR \\ expenses:travel \\ , written); }
// M1-SAFE-004: and the rules file is whatever it was. const rules_now = try project.read(std.testing.allocator, "rules.yaml"); defer std.testing.allocator.free(rules_now); try expectEqualStrings(common_rules, rules_now); }}
const common_project_manifest = @embedFile("testdata/project_manifest.yaml");const project_rules = @embedFile("testdata/project_rules.yaml");const one_match = "Date,Amount,Posting\n2026-09-11,-1.50,market\n";
test "M1-CLI-003: check validates without creating anything" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .check); try expect(!outcome.has_error); try expect(!project.exists("data/2026.csv.journal"));}
test "M1-CLI-005: convert creates exactly one target" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .convert); try expect(!outcome.has_error); try expect(project.exists("data/2026.csv.journal"));
const written = try project.tmp.dir.readFileAlloc( test_io, "data/2026.csv.journal", arena.allocator(), .unlimited, ); try expectEqualStrings( \\2026-09-11 market \\ assets:bank -1.50 EUR \\ expenses:food \\ , written);}
test "M1-JRN-014: converting twice refuses the second time" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
try expect(!project.run(&arena, .convert).has_error); // The target now exists, so discovery no longer finds an unpaired CSV. try expect(project.run(&arena, .convert).has_error);}
test "M1-AC-003: an unmatched row is named, and convert creates nothing" { var project = try Project.init(common_project_manifest, project_rules, "Date,Amount,Posting\n2026-09-11,-1.50,unknown\n"); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const checked = project.run(&arena, .check); try expect(checked.has_error); try expect(std.mem.find(u8, checked.report, "2026.csv") != null); try expect(std.mem.find(u8, checked.report, "1") != null);
try expect(project.run(&arena, .convert).has_error); try expect(!project.exists("data/2026.csv.journal"));}
test "M1-AC-004: an ambiguous row names every matching rule" { const overlapping = @embedFile("testdata/an_ambiguous_row_names_every_matching_rule.yaml"); var project = try Project.init(common_project_manifest, overlapping, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .check); try expect(outcome.has_error); try expect(std.mem.find(u8, outcome.report, "food") != null); try expect(std.mem.find(u8, outcome.report, "negative") != null); try expect(!project.exists("data/2026.csv.journal"));}
test "M1-CLI-004: check reports the status of every current row" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .check); try expect(std.mem.find(u8, outcome.report, "food") != null);}
test "M1-HIST-001: history is validated and summarized" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); try project.write("data/2025.csv", one_match); try project.write("data/2025.csv.journal", @embedFile("testdata/history_is_validated_and_summarized.journal")); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
try expect(!project.run(&arena, .check).has_error);}
test "M1-HIST-010: a historical difference fails the command" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); try project.write("data/2025.csv", one_match); try project.write("data/2025.csv.journal", @embedFile("testdata/a_historical_difference_fails_the_command.journal")); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
const outcome = project.run(&arena, .check); try expect(outcome.has_error); try expect(std.mem.find(u8, outcome.report, "2025.csv.journal") != null); try expect(!project.exists("data/2026.csv.journal"));}
test "M1-CLI-006: a malformed manifest fails with a non-zero status" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); try project.write("project.yaml", "schema:\n version: 2\n"); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
try expect(project.run(&arena, .check).has_error);}
test "M1-CLI-006: a missing manifest fails with a non-zero status" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
var dev_null = Writer.Discarding.init(&.{});
const failure = try cli.execute( arena.allocator(), test_io, &dev_null.writer, project.tmp.dir, try cli.Invocation.parse(&.{ "csv2journal", "check", "absent.yaml" }), ); try expect(failure);}
test "M1-PROJ-018: interactive refuses to start without a loadable manifest" { var project = try Project.init(common_project_manifest, project_rules, one_match); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
var allocating = Writer.Allocating.init(arena.allocator()); defer allocating.deinit();
// That this test returns at all is the assertion: had the server started, // it would block until the process was signalled. const failure = try cli.execute(arena.allocator(), test_io, &allocating.writer, project.tmp.dir, try cli.Invocation.parse( &.{ "csv2journal", "interactive", "absent.yaml" }, )); try expect(failure);
const report = try allocating.toOwnedSlice();
try expect(std.mem.find(u8, report, "absent.yaml") != null); try expect(std.mem.find(u8, report, "does not exist") != null);}
test "M1-CSV-023: a CSV with no data rows converts to a zero-byte journal" { var project = try Project.init(common_project_manifest, project_rules, "Date,Amount,Posting\n"); defer project.deinit(); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();
_ = project.run(&arena, .convert); const written = try project.tmp.dir.readFileAlloc( test_io, "data/2026.csv.journal", arena.allocator(), .unlimited, ); try expectEqualStrings("", written);}
test "M1-UI-021: interactive finalization writes exactly what convert writes" { var converted = try Project.init(common_project_manifest, project_rules, one_match); defer converted.deinit(); var interactive = try Project.init(common_project_manifest, project_rules, one_match); defer interactive.deinit();
var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator();
try expect(!converted.run(&arena, .convert).has_error);
// The same project, finalized through a session instead. The rules on // disk are already the candidate, so nothing needs saving first. const diags = try diag.List.create(alloc); const manifest = try Manifest.loadFromFileVerified(arena.allocator(), test_io, interactive.tmp.dir, "project.yaml", diags); const session = try Session.open(alloc, test_io, interactive.tmp.dir, &manifest, diags); try expect(session.view.finalizable); try expect(try session.finalize(alloc));
try expectEqualStrings( try converted.tmp.dir.readFileAlloc(test_io, "data/2026.csv.journal", alloc, .unlimited), try interactive.tmp.dir.readFileAlloc(test_io, "data/2026.csv.journal", alloc, .unlimited), );}