Something went wrong. Try again.
Convert CSV banking statements to hledger journals
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548//! Preamble, header and alias resolution (M1-CSV-008, 009, 011..014, 023).//!//! Sits on the record stream from reader.zig and decides which records are//! data, how wide they must be, and which column each alias names.//!//! The width rules differ by mode and the difference is deliberate. In header//! mode a data row must match the header exactly, because a differing width//! means the file is not what the profile describes (M1-CSV-013). In//! headerless mode a row need only reach the highest aliased index, and//! anything past it is ignored (M1-CSV-014) -- there is no header to disagree//! with, so trailing columns are simply not this profile's business.
const std = @import("std");const diag = @import("../diag.zig");const Lexer = @import("../expr/lexer.zig").Lexer;const reader = @import("reader.zig");const template = @import("../template.zig");
pub const Error = error{ InvalidCsv, InvalidConfig, OutOfMemory };
/// M1-CSV-011: exactly one of these selects a column.pub const Selector = union(enum) { header: []const u8, index: usize,};
pub const FieldSpec = struct { alias: []const u8, selector: Selector,};
pub const Options = struct { fields: []const FieldSpec, /// M1-CSV-009. header: bool, reader: reader.Options, /// M1-CSV-008. skip_records: usize,};
/// Where one alias landed in a particular file, with how the profile asked/// for it, so the interface can say both.pub const Column = struct { field: FieldSpec, index: usize,};
pub const Layout = struct { /// Resolved columns in profile declaration order. columns: []const Column, /// The exact width a data row must have in header mode (M1-CSV-013); /// null when headerless. header_width: ?usize, /// One past the highest aliased index (M1-CSV-014). minimum_width: usize,
pub fn indexOf(self: Layout, alias: []const u8) ?usize { for (self.columns) |column| { if (std.mem.eql(u8, column.field.alias, alias)) return column.index; } return null; }};
pub const DataRow = struct { /// Every cell of the record, including ones no alias selects: the /// interface searches them all (M1-UI-006). cells: []const []const u8, physical_row: usize,
pub fn value(self: DataRow, index: usize) []const u8 { return self.cells[index]; }};
pub const Rows = struct { arena: std.mem.Allocator, diags: *diag.List, file: []const u8, header_mode: bool, layout: Layout, records: reader.Reader,
pub fn init( arena: std.mem.Allocator, file: []const u8, source: []const u8, options: Options, diags: *diag.List, ) Error!Rows { try validateAliases(arena, file, options.fields, diags);
var records = try reader.Reader.init(arena, file, source, options.reader, diags);
// M1-CSV-008: the preamble is discarded, but the reader has already // counted it, so physical rows stay aligned with the file. var skipped: usize = 0; while (skipped < options.skip_records) : (skipped += 1) { if ((try records.next()) == null) { try diags.add(.{ .message = "file ends before the configured preamble does", .file = file, }); return error.InvalidCsv; } }
const layout = if (options.header) try resolveWithHeader(arena, file, &records, options.fields, diags) else try resolveHeaderless(arena, file, options.fields, diags);
return .{ .arena = arena, .file = file, .diags = diags, .header_mode = options.header, .records = records, .layout = layout, }; }
pub fn next(self: *Rows) Error!?DataRow { const record = (try self.records.next()) orelse return null;
if (self.layout.header_width) |width| { // M1-CSV-013: exactly the header's width, neither more nor less. if (record.fields.len != width) { try self.diags.addRow( self.file, record.physical_row, "row has {d} fields but the header has {d}", .{ record.fields.len, width }, ); return error.InvalidCsv; } } else if (record.fields.len < self.layout.minimum_width) { // M1-CSV-014: must reach the highest aliased index. try self.diags.addRow( self.file, record.physical_row, "row has {d} fields but an alias selects index {d}", .{ record.fields.len, self.layout.minimum_width - 1 }, ); return error.InvalidCsv; }
return .{ .physical_row = record.physical_row, .cells = record.fields }; }};
/// M1-CSV-011 and M1-RULE-014.fn validateAliases( arena: std.mem.Allocator, file: []const u8, fields: []const FieldSpec, diags: *diag.List,) Error!void { if (fields.len == 0) { try reportConfig(diags, file, "profile.csv.fields", "at least one field is required", .{}); return error.InvalidConfig; }
for (fields, 0..) |field, position| { const path = try arena.print("profile.csv.fields.{s}", .{field.alias});
if (!Lexer.isIdentifier(field.alias)) { try reportConfig(diags, file, path, "'{s}' is not a valid alias", .{field.alias}); return error.InvalidConfig; } // M1-RULE-014: the expression language reserves these, so a field // named after one could never be referenced. if (Lexer.isKeyword(field.alias)) { try reportConfig(diags, file, path, "'{s}' is a reserved keyword", .{field.alias}); return error.InvalidConfig; } // M1-TPL-006: likewise a template built-in, which would otherwise be // shadowed and unreachable. if (template.isReserved(field.alias)) { try reportConfig(diags, file, path, "'{s}' is reserved for a template placeholder", .{field.alias}); return error.InvalidConfig; } for (fields[0..position]) |earlier| { if (std.mem.eql(u8, earlier.alias, field.alias)) { try reportConfig(diags, file, path, "alias '{s}' is declared twice", .{field.alias}); return error.InvalidConfig; } } }}
fn resolveWithHeader( arena: std.mem.Allocator, file: []const u8, records: *reader.Reader, fields: []const FieldSpec, diags: *diag.List,) Error!Layout { // M1-CSV-009: the header must be there. const header = (try records.next()) orelse { try diags.add(.{ .message = "header mode is configured but the file has no header record", .file = file }); return error.InvalidCsv; };
// M1-CSV-013: uniqueness covers every cell, including ones no alias // selects, because a duplicate makes the file ambiguous whether or not // this profile happens to look at it. for (header.fields, 0..) |cell, position| { for (header.fields[0..position]) |earlier| { if (std.mem.eql(u8, earlier, cell)) { try diags.addRow(file, header.physical_row, "header cell '{s}' appears more than once", .{cell}); return error.InvalidCsv; } } }
const columns = try arena.alloc(Column, fields.len); var highest: usize = 0; for (fields, 0..) |field, position| { const index = switch (field.selector) { .header => |name| findHeader(header.fields, name) orelse { try diags.addRow(file, header.physical_row, "no header cell matches '{s}'", .{name}); return error.InvalidCsv; }, // M1-CSV-012: an index selector is allowed here too, but must land // inside the header. .index => |index| blk: { if (index >= header.fields.len) { try diags.addRow( file, header.physical_row, "alias '{s}' selects index {d} but the header has {d} columns", .{ field.alias, index, header.fields.len }, ); return error.InvalidCsv; } break :blk index; }, }; columns[position] = .{ .field = field, .index = index }; highest = @max(highest, index); }
return .{ .columns = columns, .header_width = header.fields.len, .minimum_width = highest + 1, };}
fn resolveHeaderless( arena: std.mem.Allocator, file: []const u8, fields: []const FieldSpec, diags: *diag.List,) Error!Layout { const columns = try arena.alloc(Column, fields.len); var highest: usize = 0; for (fields, 0..) |field, position| { const index = switch (field.selector) { // M1-CSV-012: there is no header to match against. .header => |name| { const path = try arena.print("profile.csv.fields.{s}", .{field.alias}); try reportConfig( diags, file, path, "header selector '{s}' needs header mode", .{name}, ); return error.InvalidConfig; }, .index => |index| index, }; columns[position] = .{ .field = field, .index = index }; highest = @max(highest, index); }
return .{ .columns = columns, .header_width = null, .minimum_width = highest + 1 };}
/// M1-CSV-012: the exact decoded cell, so case and spacing matter.fn findHeader(cells: []const []const u8, name: []const u8) ?usize { for (cells, 0..) |cell, index| { if (std.mem.eql(u8, cell, name)) return index; } return null;}
fn reportConfig( diags: *diag.List, file: []const u8, path: []const u8, comptime fmt: []const u8, args: anytype,) Error!void { try diags.addConfig(.{ .file = file, .key_path = path }, fmt, args);}
// -- tests -------------------------------------------------------------------
const expect = std.testing.expect;const expectEqual = std.testing.expectEqual;const expectEqualStrings = std.testing.expectEqualStrings;const expectError = std.testing.expectError;
fn optionsFor(skip: usize, header: bool, fields: []const FieldSpec) Options { return .{ .reader = .{ .delimiter = .comma, .line_ending = .lf }, .skip_records = skip, .header = header, .fields = fields, };}
fn openRows( arena: *std.heap.ArenaAllocator, source: []const u8, options: Options,) Error!Rows { const alloc = arena.allocator(); const diags = diag.List.create(alloc) catch return error.OutOfMemory; return Rows.init(alloc, "bank.csv", source, options, diags);}
fn collect( arena: *std.heap.ArenaAllocator, source: []const u8, options: Options,) Error![]const DataRow { const alloc = arena.allocator(); var rows = try openRows(arena, source, options); var list: std.ArrayList(DataRow) = .empty; while (try rows.next()) |row| try list.append(alloc, row); return list.items;}
const by_index: []const FieldSpec = &.{ .{ .alias = "date", .selector = .{ .index = 0 } }, .{ .alias = "amount", .selector = .{ .index = 1 } },};
const by_header: []const FieldSpec = &.{ .{ .alias = "date", .selector = .{ .header = "Datum" } }, .{ .alias = "amount", .selector = .{ .header = "Betrag" } },};
test "M1-CSV-008: skipped records are discarded but still counted" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "junk\nmore junk\na,1\nb,2\n", optionsFor(2, false, by_index)); try expectEqual(@as(usize, 2), rows.len); // Physical rows keep counting through the preamble (M1-TPL-004). try expectEqual(@as(usize, 2), rows[0].physical_row); try expectEqual(@as(usize, 3), rows[1].physical_row);}
test "M1-CSV-009: in header mode the record after the preamble is the header" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "junk\nDatum,Betrag\na,1\n", optionsFor(1, true, by_header)); try expectEqual(@as(usize, 1), rows.len); try expectEqual(@as(usize, 2), rows[0].physical_row); try expectEqualStrings("a", rows[0].cells[0]);}
test "M1-CSV-009: in headerless mode every record after the preamble is data" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "a,1\nb,2\n", optionsFor(0, false, by_index)); try expectEqual(@as(usize, 2), rows.len); try expectEqual(@as(usize, 0), rows[0].physical_row);}
test "M1-CSV-009: header mode requires the header record to exist" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, openRows(&arena, "junk\n", optionsFor(1, true, by_header))); try expectError(error.InvalidCsv, openRows(&arena, "", optionsFor(0, true, by_header)));}
test "M1-CSV-023: zero data rows is valid" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const with_header = try collect(&arena, "Datum,Betrag\n", optionsFor(0, true, by_header)); try expectEqual(@as(usize, 0), with_header.len);
const headerless = try collect(&arena, "", optionsFor(0, false, by_index)); try expectEqual(@as(usize, 0), headerless.len);}
test "M1-CSV-011: the fields mapping must not be empty" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidConfig, openRows(&arena, "a,1\n", optionsFor(0, false, &.{})));}
test "M1-CSV-011: an alias must use the identifier grammar" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); for ([_][]const u8{ "", "1st", "a-b", "a b", "wä" }) |bad| { const fields: []const FieldSpec = &.{.{ .alias = bad, .selector = .{ .index = 0 } }}; try expectError(error.InvalidConfig, openRows(&arena, "a\n", optionsFor(0, false, fields))); }}
test "M1-RULE-014: a keyword may not be a field alias" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); for ([_][]const u8{ "and", "or", "not" }) |keyword| { const fields: []const FieldSpec = &.{.{ .alias = keyword, .selector = .{ .index = 0 } }}; try expectError(error.InvalidConfig, openRows(&arena, "a\n", optionsFor(0, false, fields))); } // One merely containing a keyword is fine. const accepted: []const FieldSpec = &.{.{ .alias = "android", .selector = .{ .index = 0 } }}; _ = try openRows(&arena, "a\n", optionsFor(0, false, accepted));}
test "M1-AC-015: an alias named after a template built-in is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); for ([_][]const u8{ "sourceFilename", "physicalRow", "ruleName" }) |reserved| { const spec: []const FieldSpec = &.{.{ .alias = reserved, .selector = .{ .index = 0 } }}; try expectError(error.InvalidConfig, openRows(&arena, "a\n", optionsFor(0, false, spec))); }}
test "M1-CSV-011: duplicate aliases are rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const fields: []const FieldSpec = &.{ .{ .alias = "a", .selector = .{ .index = 0 } }, .{ .alias = "a", .selector = .{ .index = 1 } }, }; try expectError(error.InvalidConfig, openRows(&arena, "x,y\n", optionsFor(0, false, fields)));}
test "M1-CSV-012: an index selector works in either mode" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const headerless = try collect(&arena, "a,1\n", optionsFor(0, false, by_index)); try expectEqualStrings("1", headerless[0].value(1));
var rows = try openRows(&arena, "Datum,Betrag\na,1\n", optionsFor(0, true, by_index)); const row = (try rows.next()).?; try expectEqualStrings("1", row.value(rows.layout.indexOf("amount").?));}
test "M1-CSV-012: a header selector is rejected in headerless mode" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidConfig, openRows(&arena, "a,1\n", optionsFor(0, false, by_header)));}
test "M1-CSV-012: a header selector matches the exact decoded cell" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Case and surrounding space are part of the cell. try expectError(error.InvalidCsv, openRows(&arena, "datum,Betrag\na,1\n", optionsFor(0, true, by_header))); try expectError(error.InvalidCsv, openRows(&arena, " Datum,Betrag\na,1\n", optionsFor(0, true, by_header)));}
test "M1-CSV-012: a selected index must be within the header width" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const fields: []const FieldSpec = &.{.{ .alias = "far", .selector = .{ .index = 5 } }}; try expectError(error.InvalidCsv, openRows(&arena, "A,B\na,b\n", optionsFor(0, true, fields)));}
test "M1-CSV-013: every header cell must be unique, selected or not" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // The duplicate is in columns no alias mentions, and still fails. const fields: []const FieldSpec = &.{.{ .alias = "date", .selector = .{ .header = "Datum" } }}; try expectError( error.InvalidCsv, openRows(&arena, "Datum,Extra,Extra\na,b,c\n", optionsFor(0, true, fields)), );}
test "M1-CSV-013: a header selector that matches nothing is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, openRows(&arena, "Datum,Other\na,1\n", optionsFor(0, true, by_header)));}
test "M1-CSV-013: a data row must have exactly the header's width" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, collect(&arena, "Datum,Betrag\na\n", optionsFor(0, true, by_header))); try expectError(error.InvalidCsv, collect(&arena, "Datum,Betrag\na,1,extra\n", optionsFor(0, true, by_header)));}
test "M1-CSV-014: headerless rows must reach the highest aliased index" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, collect(&arena, "a\n", optionsFor(0, false, by_index)));}
test "M1-CSV-014: headerless extra trailing fields are ignored" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "a,1,ignored,also\n", optionsFor(0, false, by_index)); try expectEqual(@as(usize, 1), rows.len); try expectEqualStrings("1", rows[0].value(1));}
test "M1-AC-019: the header and width scenarios end to end" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Duplicate header fails. try expectError(error.InvalidCsv, openRows(&arena, "A,A\n1,2\n", optionsFor(0, true, &.{ .{ .alias = "a", .selector = .{ .header = "A" } }, }))); // Width mismatch fails. try expectError(error.InvalidCsv, collect(&arena, "A,B\n1\n", optionsFor(0, true, &.{ .{ .alias = "a", .selector = .{ .header = "A" } }, }))); // Headerless: extras ignored, missing index fails. const rows = try collect(&arena, "1,2,3\n", optionsFor(0, false, &.{ .{ .alias = "a", .selector = .{ .index = 1 } }, })); try expectEqualStrings("2", rows[0].value(1)); try expectError(error.InvalidCsv, collect(&arena, "1\n", optionsFor(0, false, &.{ .{ .alias = "a", .selector = .{ .index = 1 } }, })));}
test "M1-DIAG-001: a width failure names the file and physical row" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const diags = try diag.List.create(alloc); var rows = try Rows.init(alloc, "bank/2026.csv", "A,B\n1,2\n3\n", .{ .reader = .{ .delimiter = .comma, .line_ending = .lf }, .skip_records = 0, .header = true, .fields = &.{.{ .alias = "a", .selector = .{ .header = "A" } }}, }, diags); _ = rows.next() catch {}; _ = rows.next() catch {};
try expect(!diags.isEmpty()); const last = diags.items.items[diags.items.items.len - 1]; try expectEqualStrings("bank/2026.csv", last.file.?); try expectEqual(@as(usize, 2), last.physical_row.?);}