//! Orchestration (M1-RULE-006..010, M1-CSV-022, 024, M1-HIST-001..003). //! //! Runs a prepared candidate over rows: classification, generation, and //! historical validation. //! //! Two absences are load-bearing. Nothing here takes an `Io`, so this module //! cannot touch the filesystem at all -- which is M1-HIST-012 (never modify an //! approved journal or historical CSV) made structural rather than promised. //! And there is no parameter, flag or field anywhere that would let a //! historical difference be accepted, suppressed or recorded as an exception, //! which is M1-HIST-011. //! //! Rule order never decides anything (M1-RULE-006, M1-RULE-007): every rule is //! evaluated for every row and the number of matches classifies it, so there //! is no first match to win and nowhere to put a priority. const std = @import("std"); const diag = @import("diag.zig"); const eval = @import("expr/eval.zig"); const fields = @import("csv/fields.zig"); const journal_parse = @import("journal/parse.zig"); const LapTimer = @import("util/time.zig").LapTimer; const row = @import("row.zig"); const rules = @import("config/rules.zig"); const Snapshot = @import("Snapshot.zig"); const template = @import("template.zig"); const Transaction = @import("journal/Transaction.zig"); const Journal = @import("Journal.zig"); const log = std.log.scoped(.engine); pub const Candidate = @import("engine/Candidate.zig"); pub const Profile = @import("engine/Profile.zig"); pub const Error = Candidate.Error; pub const prepare = Candidate.prepare; pub const RowStatus = enum { multiple, one, unavailable, zero }; pub const RowResult = struct { /// Every decoded cell of the record, which the interface displays and /// searches (M1-UI-005, M1-UI-006). cells: []const []const u8, /// Every rule that matched, in declaration order (M1-RULE-010). matches: []const []const u8, physical_row: usize, status: RowStatus, }; /// Every data row of one file, and the columns the profile's aliases resolved /// to in it. The layout depends on the file's header, so it is only known /// once the file has been read. pub const Classification = struct { layout: fields.Layout, rows: []const RowResult, }; /// Classifies every data row. Rows are never filtered: M1-CSV-022 requires a /// row with a date, amount, currency, action or template error to be evaluated /// like any other. pub fn classify( arena: std.mem.Allocator, candidate: Candidate, csv_path: []const u8, csv_text: []const u8, diags: *diag.List, ) Error!Classification { var rows = fields.Rows.init(arena, csv_path, csv_text, candidate.profile.rowOptions(), diags) catch return error.EngineFailed; var results: std.ArrayList(RowResult) = .empty; var timer: LapTimer = .start(); while (true) { const data = (rows.next() catch return error.EngineFailed) orelse break; try results.append( arena, try classifyRow(arena, .{ .candidate = candidate, .layout = rows.layout, .data = data, .csv_path = csv_path, }, diags), ); timer.lap(); } log.debug("classified rows: {f}", .{timer.elapsed()}); return .{ .layout = rows.layout, .rows = results.items }; } /// M1-UI-007: the rows of a file whose rules could not be prepared. M1-RULE-013 /// makes classification unavailable for the complete candidate, so no rule is /// evaluated and every row is unavailable. Why is already a diagnostic, once; /// repeating it on every row would bury it. pub fn classifyUnevaluable( arena: std.mem.Allocator, profile: Profile, csv_path: []const u8, csv_text: []const u8, diags: *diag.List, ) Error!Classification { var rows = fields.Rows.init(arena, csv_path, csv_text, profile.rowOptions(), diags) catch return error.EngineFailed; var results: std.ArrayList(RowResult) = .empty; while (true) { const data = (rows.next() catch return error.EngineFailed) orelse break; try results.append(arena, .{ .physical_row = data.physical_row, .status = .unavailable, .matches = &.{}, .cells = data.cells, }); } return .{ .layout = rows.layout, .rows = results.items }; } /// One data row in the file it came from, with the candidate that interprets /// it. These travel together through classification and generation. const RowSite = struct { candidate: Candidate, csv_path: []const u8, data: fields.DataRow, layout: fields.Layout, }; /// The rule that matched a row, and what it says to generate. const Matched = struct { action: rules.Action, name: []const u8, }; fn classifyRow( arena: std.mem.Allocator, site: RowSite, diags: *diag.List, ) Error!RowResult { const candidate = site.candidate; const layout = site.layout; const data = site.data; const csv_path = site.csv_path; const values = try arena.alloc(eval.Field, layout.columns.len); for (layout.columns, 0..) |column, index| { values[index] = .{ .alias = column.field.alias, .value = data.cells[column.index] }; } var matches: std.ArrayList([]const u8) = .empty; var unavailable = false; // M1-RULE-008: every rule, every row. One rule being unusable for this row // does not stop the others being evaluated (M1-RULE-020). for (candidate.rules) |rule| { const outcome = rule.program.evaluate(.{ .fields = values }); const value = outcome.value orelse { unavailable = true; if (outcome.failure) |failure| { try diags.addRow( csv_path, data.physical_row, "rule '{s}': field '{s}' holds '{s}', which is not a decimal", .{ rule.name, failure.field, failure.cell }, ); } continue; }; if (value) try matches.append(arena, rule.name); } const status: RowStatus = if (unavailable) .unavailable else switch (matches.items.len) { 0 => .zero, 1 => .one, else => .multiple, }; switch (status) { .zero => try diags.addRow(csv_path, data.physical_row, "matches no rule", .{}), // M1-RULE-010: name every one of them. .multiple => try diags.addRow( csv_path, data.physical_row, "matches {d} rules: {s}", .{ matches.items.len, try joinNames(arena, matches.items) }, ), .one, .unavailable => {}, } return .{ .physical_row = data.physical_row, .status = status, .matches = matches.items, .cells = data.cells, }; } /// Builds the journal for one CSV. Every row is attempted even after one /// fails, so a single bad row does not hide the rest (M1-DIAG-004). pub fn generate( arena: std.mem.Allocator, candidate: Candidate, csv_path: []const u8, csv_text: []const u8, diags: *diag.List, ) Error!Journal { var rows = fields.Rows.init(arena, csv_path, csv_text, candidate.profile.rowOptions(), diags) catch return error.EngineFailed; const profile = candidate.profile; const mapping = row.ColumnMapping.load(profile.transaction, rows.layout, profile.amount_format) catch return error.EngineFailed; var transactions: std.ArrayList(Transaction) = .empty; var blocked = false; while (true) { const data = (rows.next() catch return error.EngineFailed) orelse break; const site: RowSite = .{ .candidate = candidate, .layout = rows.layout, .data = data, .csv_path = csv_path, }; const result = try classifyRow(arena, site, diags); if (result.status != .one) { blocked = true; continue; } // M1-CSV-024: a conversion error blocks finalization but has already // been reported against its row. const converted = mapping.convert(data, csv_path, diags) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return error.EngineFailed, }; if (!converted.isComplete()) { blocked = true; continue; } const rule = candidate.findRule(result.matches[0]).?; const action = rule.action orelse { try diags.addRow( csv_path, data.physical_row, "rule '{s}' matched but has no usable action", .{rule.name}, ); blocked = true; continue; }; const transaction = buildTransaction( arena, site, converted, .{ .name = rule.name, .action = action }, diags, ) catch { blocked = true; continue; }; try transactions.append(arena, transaction); } if (blocked) return error.EngineFailed; return .{ .transactions = transactions.items }; } fn buildTransaction( arena: std.mem.Allocator, site: RowSite, converted: row.Converted, matched: Matched, diags: *diag.List, ) !Transaction { const candidate = site.candidate; const layout = site.layout; const data = site.data; const csv_path = site.csv_path; const rule_name = matched.name; const action = matched.action; const values = try arena.alloc(template.Field, layout.columns.len); for (layout.columns, 0..) |column, index| { values[index] = .{ .alias = column.field.alias, .value = data.cells[column.index] }; } const rendering: template.Values = .{ .fields = values, .source_filename = csv_path, .physical_row = data.physical_row, .rule_name = rule_name, }; const request: template.Request = .{ .template = candidate.profile.narration, .values = rendering, .kind = .narration, .key_path = "profile.transaction.narrationTemplate", }; const narration = try request.render(arena, diags); return .{ .date = converted.date.?, .narration = narration, .postings = .{ .{ // M1-JRN-004: the profile default unless the rule overrides it. .account = action.source_account orelse candidate.profile.source_account, .amount = .{ .quantity = converted.amount.?, .currency = converted.currency.? }, .comment = try renderComment( arena, candidate.profile.source_comment, rendering, "profile.transaction.postingCommentTemplates.source", diags, ), }, .{ .account = action.target_account, // M1-JRN-005: left for hledger to infer. .amount = null, .comment = try renderComment( arena, candidate.profile.destination_comment, rendering, "profile.transaction.postingCommentTemplates.destination", diags, ), }, }, }; } /// M1-TPL-010: an absent template, or one that renders empty, emits no comment. fn renderComment( arena: std.mem.Allocator, maybe: ?template.Template, values: template.Values, key_path: []const u8, diags: *diag.List, ) !?[]const u8 { const configured = maybe orelse return null; const request: template.Request = .{ .template = configured, .values = values, .kind = .comment, .key_path = key_path, }; const rendered = try request.render(arena, diags); return if (rendered.len == 0) null else rendered; } /// M1-HIST-001: every discovered pair, and none is valid. M1-HIST-002: with /// the complete candidate, not just an edited rule. pub fn validateHistory( arena: std.mem.Allocator, candidate: Candidate, history: []const Snapshot.HistoryFiles, diags: *diag.List, ) Error!void { var failed = false; for (history) |pair| { // M1-DIAG-004: one pair failing must not stop the others being checked. const regenerated = generate(arena, candidate, pair.csv_path, pair.csv, diags) catch { failed = true; continue; }; const approved = journal_parse.parse(arena, pair.journal_path, pair.journal, diags) catch { failed = true; continue; }; approved.compare(regenerated, .{ .diags = diags, .location = .{ .csv = pair.csv_path, .journal = pair.journal_path }, }) catch { failed = true; }; } if (failed) return error.EngineFailed; } fn joinNames(arena: std.mem.Allocator, names: []const []const u8) Error![]const u8 { var out: std.ArrayList(u8) = .empty; for (names, 0..) |name, index| { if (index > 0) try out.appendSlice(arena, ", "); try out.appendSlice(arena, name); } return out.items; } // -- tests ------------------------------------------------------------------- test { // Tests added to candidate.zig would otherwise never be collected // (PROCESS.md ยง4). _ = Candidate; } const expect = std.testing.expect; const expectEqual = std.testing.expectEqual; const expectEqualStrings = std.testing.expectEqualStrings; const expectError = std.testing.expectError; const Decimal = @import("Decimal.zig"); const test_manifest = @embedFile("testdata/test_manifest.yaml"); const two_rules = @embedFile("testdata/two_rules.yaml"); const test_csv = "Date,Amount,Posting\n2026-09-11,-1.50,market\n2026-09-12,-800.00,landlord\n"; fn prepareWith( arena: *std.heap.ArenaAllocator, rules_text: []const u8, ) Error!struct { candidate: Candidate, diags: *diag.List } { const alloc = arena.allocator(); const diags = diag.List.create(alloc) catch return error.OutOfMemory; const candidate = try prepare(alloc, .{ .path = "project.yaml", .text = test_manifest }, .{ .path = "rules.yaml", .text = rules_text }, diags); return .{ .candidate = candidate, .diags = diags }; } test "M1-RULE-009: each row is classified as zero, one or multiple" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); const results = (try classify( arena.allocator(), setup.candidate, "2026.csv", test_csv, setup.diags, )).rows; try expectEqual(@as(usize, 2), results.len); try expectEqual(RowStatus.one, results[0].status); try expectEqualStrings("food", results[0].matches[0]); try expectEqual(RowStatus.one, results[1].status); try expectEqualStrings("rent", results[1].matches[0]); } test "M1-RULE-009: a row matching nothing is classified zero" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); const results = (try classify( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\n2026-09-11,-1.00,unknown\n", setup.diags, )).rows; try expectEqual(RowStatus.zero, results[0].status); try expectEqual(@as(usize, 0), results[0].matches.len); } test "M1-RULE-010: a multiply matched row names every matching rule" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const overlapping = @embedFile("testdata/overlapping.yaml"); const setup = try prepareWith(&arena, overlapping); const results = (try classify( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\n2026-09-11,-1.50,market\n", setup.diags, )).rows; try expectEqual(RowStatus.multiple, results[0].status); try expectEqual(@as(usize, 2), results[0].matches.len); try expectEqualStrings("first", results[0].matches[0]); try expectEqualStrings("second", results[0].matches[1]); } test "M1-RULE-010: zero and multiple diagnostics name the file and row" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); _ = try classify( arena.allocator(), setup.candidate, "bank/2026.csv", "Date,Amount,Posting\n2026-09-11,-1.00,unknown\n", setup.diags, ); try expect(!setup.diags.isEmpty()); const first_diag = setup.diags.items.items[0]; try expectEqualStrings("bank/2026.csv", first_diag.file.?); try expectEqual(@as(usize, 1), first_diag.physical_row.?); } test "M1-AC-010: reordering rules changes neither matching nor output" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const reversed = @embedFile("testdata/reversed.yaml"); const alloc = arena.allocator(); const original = try prepareWith(&arena, two_rules); const swapped = try prepareWith(&arena, reversed); const first = try generate(alloc, original.candidate, "2026.csv", test_csv, original.diags); const second = try generate(alloc, swapped.candidate, "2026.csv", test_csv, swapped.diags); const first_str = try arena.allocator().print("{f}", .{first}); const second_str = try arena.allocator().print("{f}", .{second}); try expectEqualStrings(first_str, second_str); } test "M1-RULE-008: a rule with an unusable action still takes part in matching" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const broken = @embedFile("testdata/broken.yaml"); const setup = try prepareWith(&arena, broken); const results = (try classify( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\n2026-09-11,-1.50,market\n", setup.diags, )).rows; // The condition was evaluated; only generation is blocked. try expectEqual(RowStatus.one, results[0].status); try expectEqualStrings("food", results[0].matches[0]); } test "M1-CSV-022: a row with a conversion error still enters evaluation" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); // The date is unusable, but the condition is about posting. const results = (try classify( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\nnot-a-date,-1.50,market\n", setup.diags, )).rows; try expectEqual(RowStatus.one, results[0].status); } test "M1-RULE-020: an unparseable numeric field leaves the row unavailable" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const numeric = @embedFile("testdata/numeric.yaml"); const setup = try prepareWith(&arena, numeric); const results = (try classify( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\n2026-09-11,N/A,market\n", setup.diags, )).rows; try expectEqual(RowStatus.unavailable, results[0].status); } test "M1-RULE-013: an invalid expression makes the whole candidate unusable" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const bad = @embedFile("testdata/bad.yaml"); try expectError(error.EngineFailed, prepareWith(&arena, bad)); } test "M1-RULE-008: an unknown alias in a condition fails the candidate" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const unknown = @embedFile("testdata/unknown.yaml"); try expectError(error.EngineFailed, prepareWith(&arena, unknown)); } test "M1-PROJ-018: malformed configuration fails preparation" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.EngineFailed, prepareWith(&arena, "schema:\n version: 2\n")); } test "M1-JRN-001: one transaction per row, in source order" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); const result = try generate( arena.allocator(), setup.candidate, "2026.csv", test_csv, setup.diags, ); try expectEqual(@as(usize, 2), result.transactions.len); try expectEqualStrings("market", result.transactions[0].narration); try expectEqualStrings("landlord", result.transactions[1].narration); try expectEqualStrings("assets:bank", result.transactions[0].postings[0].account); try expectEqualStrings("expenses:food", result.transactions[0].postings[1].account); try expectEqualStrings("by food", result.transactions[0].postings[1].comment.?); } test "M1-CSV-017: the profile's separator decodes every file's amounts" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const manifest_text = try std.mem.replaceOwned(u8, alloc, test_manifest, "decimalSeparator: dot", "decimalSeparator: comma"); const diags = try diag.List.create(alloc); const prepared = try prepare(alloc, .{ .path = "project.yaml", .text = manifest_text }, .{ .path = "rules.yaml", .text = two_rules }, diags); const generated = try generate(alloc, prepared, "2026.csv", "Date,Amount,Posting\n2026-09-11,\"-1,50\",market\n", diags); const quantity = generated.transactions[0].postings[0].amount.?.quantity; try expect(quantity.eql(try Decimal.parseLiteral("-1.5"))); } test "M1-RULE-016: a grouped cell reaches conversion and predicates alike" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const manifest_text = try std.mem.replaceOwned(u8, alloc, test_manifest, "decimalSeparator: dot", "decimalSeparator: comma\n groupSeparator: dot"); const rules_text = \\schema: \\ version: 1 \\rules: \\ big: \\ match: 'amount <= -1000' \\ set: \\ targetAccount: "expenses:big" \\ ; const diags = try diag.List.create(alloc); const prepared = try prepare(alloc, .{ .path = "project.yaml", .text = manifest_text }, .{ .path = "rules.yaml", .text = rules_text }, diags); const generated = try generate(alloc, prepared, "2026.csv", "Date,Amount,Posting\n2026-09-11,\"-1.234,50\",market\n", diags); const postings = generated.transactions[0].postings; try expect(postings[0].amount.?.quantity.eql(try Decimal.parseLiteral("-1234.5"))); try expectEqualStrings("expenses:big", postings[1].account); } test "M1-RULE-009: generation is refused when a row is not uniquely matched" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); try expectError(error.EngineFailed, generate( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\n2026-09-11,-1.00,unknown\n", setup.diags, )); } test "M1-HIST-001: no historical pairs is valid" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); try validateHistory(arena.allocator(), setup.candidate, &.{}, setup.diags); } test "M1-HIST-002: history is validated with the whole ruleset" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); const approved = @embedFile("testdata/history_is_validated_with_the_whole_ruleset.journal"); try validateHistory(arena.allocator(), setup.candidate, &.{.{ .csv_path = "2025.csv", .journal_path = "2025.csv.journal", .csv = test_csv, .journal = approved, }}, setup.diags); } test "M1-HIST-003: a historical difference is reported and fails" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); const approved = @embedFile("testdata/a_historical_difference_is_reported_and_fails.journal"); try expectError(error.EngineFailed, validateHistory( arena.allocator(), setup.candidate, &.{.{ .csv_path = "2025.csv", .journal_path = "2025.csv.journal", .csv = test_csv, .journal = approved, }}, setup.diags, )); try expect(!setup.diags.isEmpty()); } test "M1-HIST-003: a historical row matching nothing fails like a current one" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); try expectError(error.EngineFailed, validateHistory( arena.allocator(), setup.candidate, &.{.{ .csv_path = "2025.csv", .journal_path = "2025.csv.journal", .csv = "Date,Amount,Posting\n2026-09-11,-1.00,unknown\n", .journal = "", }}, setup.diags, )); } test "M1-CSV-023: a CSV with no data rows yields an empty journal" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const setup = try prepareWith(&arena, two_rules); const result = try generate( arena.allocator(), setup.candidate, "2026.csv", "Date,Amount,Posting\n", setup.diags, ); try expectEqual(@as(usize, 0), result.transactions.len); } test "M1-TPL-002: source and destination comment templates are independent" { // Four manifests differing only in which of the two templates is present. const base = @embedFile("testdata/source_and_destination_comment_templates_are_independent.yaml"); const cases = [_]struct { destination: ?[]const u8, source: ?[]const u8, templates: []const u8, }{ .{ .templates = "", .source = null, .destination = null }, .{ .templates = " postingCommentTemplates:\n source: \"from {posting}\"\n", .source = "from market", .destination = null, }, .{ .templates = " postingCommentTemplates:\n destination: \"by {ruleName}\"\n", .source = null, .destination = "by food", }, .{ .templates = " postingCommentTemplates:\n source: \"from {posting}\"\n" ++ " destination: \"by {ruleName}\"\n", .source = "from market", .destination = "by food", }, }; for (cases) |case| { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const manifest_text = try alloc.print("{s}{s}", .{ base, case.templates }); const diags = try diag.List.create(alloc); const prepared = try prepare(alloc, .{ .path = "project.yaml", .text = manifest_text }, .{ .path = "rules.yaml", .text = two_rules }, diags); const generated = try generate(alloc, prepared, "2026.csv", test_csv, diags); const postings = generated.transactions[0].postings; // Omitting either template emits no comment on that posting, and // neither omission affects the other. if (case.source) |text| { try expectEqualStrings(text, postings[0].comment.?); } else { try expectEqual(@as(?[]const u8, null), postings[0].comment); } if (case.destination) |text| { try expectEqualStrings(text, postings[1].comment.?); } else { try expectEqual(@as(?[]const u8, null), postings[1].comment); } } }