Something went wrong. Try again.
Convert CSV banking statements to hledger journals
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438//! CSV record stream (M1-CSV-001..007, M1-CSV-010).//!//! zarko does the lexing. What it does not do is any of the strictness//! milestone one requires, so this module wraps it://!//! * encoding and line-ending discipline are checked over the raw bytes//! before parsing begins (M1-CSV-001, M1-CSV-003);//! * the physical row ordinal is counted here, because zarko keeps none//! (M1-TPL-004, M1-DIAG-001);//! * a physically blank record is rejected (M1-CSV-010).//!//! That last one cannot be decided from the parsed fields. A blank line and a//! record holding one quoted empty field both come back as a single empty//! field, so the record's byte span is inspected instead.//!//! Two behaviours are zarko's already and are covered by pinning tests rather//! than code: a trailing line ending yields no extra record, and a final//! record may end at EOF (M1-CSV-004). M1-CSV-005 makes that pinned behaviour//! part of the specification, so a dependency change should fail those tests.
const std = @import("std");const diag = @import("../diag.zig");const zarko = @import("zarko");
pub const Error = error{ InvalidCsv, OutOfMemory };
/// M1-CSV-002.pub const Delimiter = enum { comma, semicolon, tab,
pub fn byte(self: Delimiter) u8 { return switch (self) { .comma => ',', .semicolon => ';', .tab => '\t', }; }};
/// M1-CSV-003. zarko also offers a lone CR; milestone one does not.pub const LineEnding = enum { crlf, lf,
pub fn bytes(self: LineEnding) []const u8 { return switch (self) { .lf => "\n", .crlf => "\r\n", }; }
fn dialectValue(self: LineEnding) zarko.LineEnding { return switch (self) { .lf => .lf, .crlf => .crlf, }; }};
pub const Options = struct { delimiter: Delimiter, line_ending: LineEnding,};
pub const Record = struct { /// Exact decoded cells: no trimming, no normalization (M1-CSV-007). fields: []const []const u8, /// Zero-based ordinal over every record in the file, including preamble /// and header records a later stage skips (M1-CSV-008, M1-TPL-004). physical_row: usize,};
pub const Reader = struct { arena: std.mem.Allocator, diags: *diag.List, file: []const u8, next_row: usize, options: Options, parser: zarko.Parser,
pub fn init( arena: std.mem.Allocator, file: []const u8, raw_source: []const u8, options: Options, diags: *diag.List, ) Error!Reader { // Removed before anything else sees the bytes, so it cannot reach a // cell, a diagnostic, or the line-ending scan. const source = withoutMark(raw_source);
// M1-CSV-001 and M1-CSV-003 are decided over the raw bytes, before // zarko sees them. zarko tolerates line endings that disagree with its // dialect, so leaving this to the parser would let them through. if (encodingProblem(source)) |problem| { try report(diags, file, "{s}", .{problem}); return error.InvalidCsv; } if (lineEndingProblem(source, options.line_ending)) |offset| { try report( diags, file, "line ending at byte {d} is not the configured {s}", .{ offset, @tagName(options.line_ending) }, ); return error.InvalidCsv; }
return .{ .arena = arena, .file = file, .diags = diags, .options = options, .parser = zarko.Parser.init(arena, source, .{ .separator = options.delimiter.byte(), // M1-CSV-005: the quote byte is always a double quote. .quote = '"', .line_ending = options.line_ending.dialectValue(), }), .next_row = 0, }; }
pub fn next(self: *Reader) Error!?Record { const span_start = self.parser.pos; const row = self.next_row;
const record = self.parser.next() catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, // zarko carries no position, so the row counted here is the // diagnostic's only locator (risk R4). error.UnterminatedQuote => { try self.reportRow(row, "quoted field is not closed", .{}); return error.InvalidCsv; }, else => { try self.reportRow(row, "malformed CSV record", .{}); return error.InvalidCsv; }, } orelse return null;
self.next_row += 1;
if (isPhysicallyBlank(self.parser.input[span_start..self.parser.pos], self.options.line_ending)) { try self.reportRow(row, "record is physically blank", .{}); return error.InvalidCsv; }
return .{ .physical_row = row, .fields = record.fields }; }
fn reportRow( self: *Reader, row: usize, comptime fmt: []const u8, args: anytype, ) Error!void { try self.diags.addRow(self.file, row, fmt, args); }};
/// M1-CSV-010. A blank line and a record of one quoted empty field decode/// identically, so the decision is made on the bytes the record occupied.fn isPhysicallyBlank(span: []const u8, ending: LineEnding) bool { const terminator = ending.bytes(); const body = if (std.mem.endsWith(u8, span, terminator)) span[0 .. span.len - terminator.len] else span; return body.len == 0;}
const utf8_bom = "\xEF\xBB\xBF";
/// M1-CSV-001. A leading mark is framing, not data: exporters add one to files/// the user cannot regenerate, and leaving it in would put it inside the first/// header cell, where an exact header match would then fail against a file/// that visibly says the right thing (M1-CSV-012).fn withoutMark(source: []const u8) []const u8 { if (std.mem.startsWith(u8, source, utf8_bom)) return source[utf8_bom.len..]; return source;}
/// M1-CSV-001. Decoding performs no charset conversion, so anything that is/// not UTF-8 is refused. A UTF-16 mark starts with a byte that is never valid/// UTF-8, so it is caught here.fn encodingProblem(source: []const u8) ?[]const u8 { if (!std.unicode.utf8ValidateSlice(source)) return "file is not valid UTF-8"; return null;}
/// M1-CSV-003. Every line break must be the configured sequence, including/// inside a quoted field, so this needs no quote tracking: a stray byte is/// illegal wherever it appears.fn lineEndingProblem(source: []const u8, ending: LineEnding) ?usize { switch (ending) { // Any CR at all is wrong: a lone one, or the other ending. .lf => return std.mem.findScalar(u8, source, '\r'),
.crlf => { var i: usize = 0; while (i < source.len) : (i += 1) { switch (source[i]) { '\r' => { if (i + 1 >= source.len or source[i + 1] != '\n') return i; i += 1; }, // Reached only when not consumed as part of a CRLF above. '\n' => return i, else => {}, } } return null; }, }}
fn report( diags: *diag.List, file: []const u8, comptime fmt: []const u8, args: anytype,) Error!void { try diags.addFmt(fmt, args, .{ .file = file });}
// -- tests -------------------------------------------------------------------
const expect = std.testing.expect;const expectEqual = std.testing.expectEqual;const expectEqualStrings = std.testing.expectEqualStrings;const expectError = std.testing.expectError;
fn readAll( arena: *std.heap.ArenaAllocator, source: []const u8, options: Options,) Error![]const Record { const alloc = arena.allocator(); const diags = diag.List.create(alloc) catch return error.OutOfMemory; var reader = try Reader.init(alloc, "bank.csv", source, options, diags); var list: std.ArrayList(Record) = .empty; while (try reader.next()) |record| try list.append(alloc, record); return list.items;}
const comma_lf: Options = .{ .delimiter = .comma, .line_ending = .lf };const comma_crlf: Options = .{ .delimiter = .comma, .line_ending = .crlf };
test "M1-CSV-001: a leading byte-order mark is ignored, not kept" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "\xEF\xBB\xBFa,b\n", comma_lf); try expectEqual(@as(usize, 1), records.len); // It must not survive into the first cell, or an exact header match would // fail against a file that plainly says "a" (M1-CSV-012). try expectEqualStrings("a", records[0].fields[0]);}
test "M1-CSV-001: a mark elsewhere in the file is ordinary data" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Only a leading mark is framing; U+FEFF inside a cell is content. const records = try readAll(&arena, "a,\xEF\xBB\xBFb\n", comma_lf); try expectEqualStrings("\xEF\xBB\xBFb", records[0].fields[1]);}
test "M1-CSV-001: invalid UTF-8 is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,\xC3\x28\n", comma_lf));}
test "M1-CSV-001: non-ASCII UTF-8 survives unchanged" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "Wä,café\n", comma_lf); try expectEqualStrings("Wä", records[0].fields[0]); try expectEqualStrings("café", records[0].fields[1]);}
test "M1-CSV-002: each supported delimiter separates fields" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const by_comma = try readAll(&arena, "a,b\n", comma_lf); try expectEqual(@as(usize, 2), by_comma[0].fields.len);
const by_semicolon = try readAll(&arena, "a;b\n", .{ .delimiter = .semicolon, .line_ending = .lf }); try expectEqual(@as(usize, 2), by_semicolon[0].fields.len);
const by_tab = try readAll(&arena, "a\tb\n", .{ .delimiter = .tab, .line_ending = .lf }); try expectEqual(@as(usize, 2), by_tab[0].fields.len);
// A delimiter that is not the configured one is ordinary data. const literal = try readAll(&arena, "a;b\n", comma_lf); try expectEqual(@as(usize, 1), literal[0].fields.len); try expectEqualStrings("a;b", literal[0].fields[0]);}
test "M1-CSV-003: in lf mode any carriage return is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,b\r\n", comma_lf)); try expectError(error.InvalidCsv, readAll(&arena, "a,b\rc\n", comma_lf)); try expectError(error.InvalidCsv, readAll(&arena, "a,\"x\ry\"\n", comma_lf));}
test "M1-CSV-003: in crlf mode a lone LF or CR is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,b\n", comma_crlf)); try expectError(error.InvalidCsv, readAll(&arena, "a,b\rc\r\n", comma_crlf)); try expectError(error.InvalidCsv, readAll(&arena, "a,b\r\nc,d\n", comma_crlf));}
test "M1-AC-018: a lone LF inside a quoted field fails under crlf" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,\"x\ny\"\r\n", comma_crlf)); // The same file with the configured ending throughout is fine. const rows = try readAll(&arena, "a,\"x\r\ny\"\r\n", comma_crlf); try expectEqualStrings("x\r\ny", rows[0].fields[1]);}
test "M1-CSV-004: a final record may end at EOF without a line ending" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a,b\nc,d", comma_lf); try expectEqual(@as(usize, 2), records.len); try expectEqualStrings("d", records[1].fields[1]);}
test "M1-CSV-004: a trailing line ending yields no extra record" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a,b\nc,d\n", comma_lf); try expectEqual(@as(usize, 2), records.len);}
test "M1-CSV-005: a quote opens a field only as its first byte" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a\"b,c\n", comma_lf); try expectEqualStrings("a\"b", records[0].fields[0]); try expectEqualStrings("c", records[0].fields[1]);}
test "M1-CSV-006: a quoted field carries delimiters and line breaks" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const with_delimiter = try readAll(&arena, "\"a,b\",c\n", comma_lf); try expectEqual(@as(usize, 2), with_delimiter[0].fields.len); try expectEqualStrings("a,b", with_delimiter[0].fields[0]);
const with_break = try readAll(&arena, "\"a\nb\",c\n", comma_lf); try expectEqualStrings("a\nb", with_break[0].fields[0]);}
test "M1-CSV-006: a doubled quote is a literal quote" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "\"a\"\"b\"\n", comma_lf); try expectEqualStrings("a\"b", records[0].fields[0]);}
test "M1-CSV-006: unclosed quoting is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "\"unterminated\n", comma_lf));}
test "M1-CSV-007: cells keep their exact bytes" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, " padded ,UPPER,\ttab\n", comma_lf); try expectEqualStrings(" padded ", records[0].fields[0]); try expectEqualStrings("UPPER", records[0].fields[1]); try expectEqualStrings("\ttab", records[0].fields[2]);}
test "M1-CSV-010: a physically blank record is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,b\n\nc,d\n", comma_lf)); try expectError(error.InvalidCsv, readAll(&arena, "\na,b\n", comma_lf));}
test "M1-CSV-010: a record of empty fields is not physically blank" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const separators = try readAll(&arena, ",,\n", comma_lf); try expectEqual(@as(usize, 3), separators[0].fields.len); try expectEqualStrings("", separators[0].fields[0]);
// A single quoted empty field is a record, though it decodes to the same // one empty field a blank line would produce. const quoted = try readAll(&arena, "\"\"\n", comma_lf); try expectEqual(@as(usize, 1), quoted[0].fields.len); try expectEqualStrings("", quoted[0].fields[0]);}
test "M1-DIAG-001: physical rows are zero-based and count every record" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a\nb\nc\n", comma_lf); try expectEqual(@as(usize, 0), records[0].physical_row); try expectEqual(@as(usize, 1), records[1].physical_row); try expectEqual(@as(usize, 2), records[2].physical_row);}
test "M1-TPL-004: a quoted line break does not advance the physical row" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "\"a\nb\"\nc\n", comma_lf); try expectEqual(@as(usize, 2), records.len); try expectEqual(@as(usize, 0), records[0].physical_row); try expectEqual(@as(usize, 1), records[1].physical_row);}
test "M1-CSV-024: a rejection names the source file" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const diags = try diag.List.create(alloc); _ = Reader.init(alloc, "bank/2026.csv", "a,b\r\n", comma_lf, diags) catch {}; try expect(!diags.isEmpty()); try expectEqualStrings("bank/2026.csv", diags.items.items[0].file.?);}
test "an empty file yields no records" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "", comma_lf); try expectEqual(@as(usize, 0), records.len);}