Something went wrong. Try again.
Convert CSV banking statements to hledger journals
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444//! Exact decimals (M1-CSV-018, M1-CSV-019, M1-RULE-016, M1-HIST-008).//!//! M1-CSV-019 forbids binary floating point outright. An amount is an i64//! coefficient and a u8 scale: "7.50" is coef 750, scale 2. Nothing is stored//! as text, because every clause of M1-CSV-019 falls out of the pair -- a//! leading '+' and redundant integer zeroes never survive parsing, the scale//! is the source scale, and -0.00 has coefficient zero, so its sign is gone//! before rendering is reached.//!//! Range. i64 holds 18 full digits. A cell whose digits exceed that is//! reported as an overflow rather than represented approximately (DESIGN_M1.md//! assumption A1); M1-CSV-019 states no bound, so this is a deliberate one.
const std = @import("std");
pub const Error = error{ /// Does not match the grammar: empty, whitespace, grouping the format /// does not allow, exponent, the wrong separator, a missing integer /// part (".5") or a missing fractional part ("1."). InvalidFormat, /// More digits than an i64 coefficient holds, or more than 255 fractional /// digits. Overflow,};
/// How a *cell* spells an amount (M1-CSV-017). Rule literals are always/// `dot` (M1-RULE-016).pub const Format = struct { decimal: Separator, /// Null when the profile configures no grouping. group: ?Separator = null,
pub const comma: Format = .{ .decimal = .comma }; pub const dot: Format = .{ .decimal = .dot };
/// Reads the integer part starting at `start` into `coef` and returns /// where it ends (M1-CSV-018). Grouping is optional, but a cell that /// groups has one to three digits first and exactly three in every /// group after that. fn scanInteger(self: Format, text: []const u8, start: usize, coef: *i64) Error!usize { const group_byte: ?u8 = if (self.group) |group| group.byte() else null; var grouped = false; // Digits since the start or since the last group separator. var run: usize = 0; var i = start; while (i < text.len) : (i += 1) { if (isDigit(text[i])) { coef.* = try pushDigit(coef.*, text[i]); run += 1; } else if (text[i] == group_byte) { if (run == 0 or run > 3 or (grouped and run != 3)) return error.InvalidFormat; grouped = true; run = 0; } else break; } // ".5" has no integer digits; "3.907." and "1.2345" end on a bad group. if (run == 0 or (grouped and run != 3)) return error.InvalidFormat; return i; }};
/// A byte a profile may configure within an amount cell.pub const Separator = enum { comma, dot,
fn byte(self: Separator) u8 { return switch (self) { .dot => '.', .comma => ',', }; }};
/// Scale is a u8, so this is the most fractional digits representable.const max_scale = 255;
/// Above this the cross-scaling in `order` could overflow i128. An i64/// coefficient is at most 9.3e18, and 9.3e18 * 1e19 still fits.const max_safe_shift = 19;
const Decimal = @This();
coef: i64,scale: u8,
pub const zero: Decimal = .{ .coef = 0, .scale = 0 };
/// Parses an amount cell (M1-CSV-018):////// [+-]? INTEGER ( SEP DIGIT+ )?/// INTEGER := DIGIT+ | DIGIT{1,3} ( GROUP DIGIT{3} )+////// The second form of INTEGER only when the format has a group separator./// Everything outside that shape is rejected, including surrounding/// whitespace, exponents, and non-ASCII digits.pub fn parse(text: []const u8, cell_format: Format) Error!Decimal { var i: usize = 0;
var negative = false; if (i < text.len and (text[i] == '+' or text[i] == '-')) { negative = text[i] == '-'; i += 1; }
var coef: i64 = 0; i = try cell_format.scanInteger(text, i, &coef);
var scale: u8 = 0; if (i < text.len) { if (text[i] != cell_format.decimal.byte()) return error.InvalidFormat; i += 1;
const fraction_start = i; while (i < text.len and isDigit(text[i])) : (i += 1) { coef = try pushDigit(coef, text[i]); } // "1." has no fractional digits. if (i == fraction_start) return error.InvalidFormat;
const digits = i - fraction_start; if (digits > max_scale) return error.Overflow; scale = @intCast(digits); }
// Anything left over is trailing junk: whitespace, a second // separator, an exponent. if (i != text.len) return error.InvalidFormat;
// Negating zero yields zero, which is how -0.00 loses its sign // (M1-CSV-019). return .{ .coef = if (negative) -coef else coef, .scale = scale };}
/// A decimal literal in a rule expression. Always dot, whatever the/// profile configures for cells (M1-RULE-016).pub fn parseLiteral(text: []const u8) Error!Decimal { return parse(text, .dot);}
/// Always emits '.', never ',' (M1-CSV-019). The source scale is preserved,/// so 100.0 stays 100.0.pub fn format(self: Decimal, writer: *std.Io.Writer) std.Io.Writer.Error!void { if (self.coef < 0) { try writer.writeAll("-"); }
var digit_buf: [40]u8 = undefined; // zlinter-disable-next-line no_swallow_error - an i64 coefficient is at most 19 digits, the buffer holds 40 const digits = std.mem.print(&digit_buf, "{d}", .{@abs(self.coef)}) catch unreachable;
if (self.scale == 0) { try writer.print("{s}", .{digits}); } else if (digits.len > self.scale) { const split = digits.len - self.scale; try writer.print("{s}.{s}", .{ digits[0..split], digits[split..] }); } else { // Value is below one: "0." then enough zeroes to reach the digits. try writer.writeAll("0."); const padding = self.scale - digits.len; for (0..padding) |_| { try writer.writeAll("0"); } try writer.writeAll(digits); }}
/// Compares by mathematical value (M1-HIST-008, M1-RULE-016). Sign/// spelling, leading integer zeroes and trailing fractional zeroes are all/// insignificant, so 100.0 and 100.00 are equal.pub fn order(self: Decimal, other: Decimal) std.math.Order { const lhs = self.canonical(); const rhs = other.canonical();
if (lhs.scale == rhs.scale) return std.math.order(lhs.coef, rhs.coef);
// Zero compares by the other side's sign whatever its scale. if (lhs.coef == 0) return std.math.order(0, rhs.coef); if (rhs.coef == 0) return std.math.order(lhs.coef, 0);
const shift = if (lhs.scale < rhs.scale) rhs.scale - lhs.scale else lhs.scale - rhs.scale; if (shift > max_safe_shift) { // Raising by more than 19 makes that side exceed any i64 // coefficient the other can hold, so its sign decides. const dominant = if (lhs.scale < rhs.scale) lhs.coef else rhs.coef; const dominant_is_lhs = lhs.scale < rhs.scale; const dominant_greater = dominant > 0; if (dominant_is_lhs) { return if (dominant_greater) .gt else .lt; } return if (dominant_greater) .lt else .gt; }
var lhs_value: i128 = lhs.coef; var rhs_value: i128 = rhs.coef; if (lhs.scale < rhs.scale) { lhs_value *= pow10(shift); } else { rhs_value *= pow10(shift); } return std.math.order(lhs_value, rhs_value);}
pub fn eql(self: Decimal, other: Decimal) bool { return self.order(other) == .eq;}
/// Strips trailing fractional zeroes, giving each value one form. Two/// equal values always canonicalise identically.fn canonical(self: Decimal) Decimal { if (self.coef == 0) return .{ .coef = 0, .scale = 0 }; var coef = self.coef; var scale = self.scale; while (scale > 0 and @rem(coef, 10) == 0) { coef = @divExact(coef, 10); scale -= 1; } return .{ .coef = coef, .scale = scale };}
fn isDigit(c: u8) bool { return c >= '0' and c <= '9';}
/// Appends one decimal digit, reporting overflow rather than wrapping (A1).fn pushDigit(coef: i64, digit: u8) Error!i64 { const scaled = @mulWithOverflow(coef, 10); if (scaled[1] != 0) return error.Overflow; const summed = @addWithOverflow(scaled[0], @as(i64, digit - '0')); if (summed[1] != 0) return error.Overflow; return summed[0];}
fn pow10(exponent: u8) i128 { var result: i128 = 1; var remaining = exponent; while (remaining > 0) : (remaining -= 1) result *= 10; return result;}
// -- tests -------------------------------------------------------------------
const expect = std.testing.expect;const expectEqual = std.testing.expectEqual;const expectError = std.testing.expectError;const expectFmt = std.testing.expectFmt;
fn parsed(text: []const u8, cell_format: Format) !Decimal { return Decimal.parse(text, cell_format);}
test "M1-CSV-018: accepts sign, integer digits and an optional fraction" { try expectEqual(Decimal{ .coef = 0, .scale = 0 }, try parsed("0", .dot)); try expectEqual(Decimal{ .coef = 1, .scale = 0 }, try parsed("1", .dot)); try expectEqual(Decimal{ .coef = 1, .scale = 0 }, try parsed("+1", .dot)); try expectEqual(Decimal{ .coef = -1, .scale = 0 }, try parsed("-1", .dot)); try expectEqual(Decimal{ .coef = 15, .scale = 1 }, try parsed("1.5", .dot)); try expectEqual(Decimal{ .coef = -15, .scale = 1 }, try parsed("-1.5", .dot)); try expectEqual(Decimal{ .coef = 150, .scale = 2 }, try parsed("1.50", .dot)); try expectEqual(Decimal{ .coef = 7, .scale = 0 }, try parsed("007", .dot));}
test "M1-CSV-018: rejects a missing integer part and a missing fraction" { try expectError(error.InvalidFormat, parsed(".5", .dot)); try expectError(error.InvalidFormat, parsed("1.", .dot)); try expectError(error.InvalidFormat, parsed(".", .dot));}
test "M1-CSV-018: rejects grouping, exponents and trailing junk" { try expectError(error.InvalidFormat, parsed("1,000", .dot)); try expectError(error.InvalidFormat, parsed("1e5", .dot)); try expectError(error.InvalidFormat, parsed("1E5", .dot)); try expectError(error.InvalidFormat, parsed("1.2.3", .dot)); try expectError(error.InvalidFormat, parsed("1a", .dot));}
test "M1-CSV-018: rejects surrounding whitespace" { try expectError(error.InvalidFormat, parsed(" 1", .dot)); try expectError(error.InvalidFormat, parsed("1 ", .dot)); try expectError(error.InvalidFormat, parsed("\t1", .dot)); try expectError(error.InvalidFormat, parsed("1.5\n", .dot));}
test "M1-CSV-018: rejects empty, bare signs and non-ASCII digits" { try expectError(error.InvalidFormat, parsed("", .dot)); try expectError(error.InvalidFormat, parsed("-", .dot)); try expectError(error.InvalidFormat, parsed("+", .dot)); try expectError(error.InvalidFormat, parsed("+-1", .dot)); try expectError(error.InvalidFormat, parsed("abc", .dot)); // Arabic-Indic one: a digit to Unicode, not to this grammar. try expectError(error.InvalidFormat, parsed("\u{0661}", .dot));}
test "M1-CSV-017: the configured separator is the only one accepted" { try expectEqual(Decimal{ .coef = 15, .scale = 1 }, try parsed("1,5", .comma)); try expectError(error.InvalidFormat, parsed("1.5", .comma)); try expectError(error.InvalidFormat, parsed("1,5", .dot));}
const german: Format = .{ .decimal = .comma, .group = .dot };
test "M1-CSV-018: a group separator splits the integer part into threes" { try expectEqual(Decimal{ .coef = 390707, .scale = 2 }, try parsed("3.907,07", german)); try expectEqual(Decimal{ .coef = 1234567, .scale = 0 }, try parsed("1.234.567", german)); try expectEqual(Decimal{ .coef = -12345, .scale = 1 }, try parsed("-1.234,5", german)); try expectEqual(Decimal{ .coef = 90700, .scale = 2 }, try parsed("0.907,00", german)); const dotted: Format = .{ .decimal = .dot, .group = .comma }; try expectEqual(Decimal{ .coef = 123456, .scale = 2 }, try parsed("1,234.56", dotted));}
test "M1-CSV-018: grouping is permitted, not required" { try expectEqual(Decimal{ .coef = 390707, .scale = 2 }, try parsed("3907,07", german)); try expectEqual(Decimal{ .coef = 90707, .scale = 2 }, try parsed("907,07", german));}
test "M1-CSV-018: every group after the first has exactly three digits" { for ([_][]const u8{ "39.07,07", // a short group "1.23.456,00", // a short group before a full one "1.234567,00", // a long group "1234.567,00", // a first group of four ".907,07", // no first group "3.,07", // an empty group "3..907", "3.907.", // a trailing separator "3.907.,07", "1,234.56", // the separators swapped }) |bad| { try expectError(error.InvalidFormat, parsed(bad, german)); }}
test "M1-CSV-019: rendering drops a leading plus and redundant integer zeroes" { try expectFmt("7.50", "{f}", .{try parsed("+007.50", .dot)}); try expectFmt("1", "{f}", .{try parsed("+1", .dot)}); try expectFmt("0", "{f}", .{try parsed("000", .dot)});}
test "M1-CSV-019: rendering preserves the source fractional scale" { try expectFmt("100.0", "{f}", .{try parsed("100.0", .dot)}); try expectFmt("100.00", "{f}", .{try parsed("100.00", .dot)}); try expectFmt("100", "{f}", .{try parsed("100", .dot)});}
test "M1-CSV-019: negative zero loses its sign" { try expectFmt("0.00", "{f}", .{try parsed("-0.00", .dot)}); try expectFmt("0", "{f}", .{try parsed("-0", .dot)});}
test "M1-CSV-019: a comma-separated cell still renders with a dot" { try expectFmt("1.5", "{f}", .{try parsed("1,5", .comma)}); try expectFmt("3907.07", "{f}", .{try parsed("3.907,07", german)});}
test "M1-CSV-019: values below one render with a leading zero" { try expectFmt("0.001", "{f}", .{try parsed("0.001", .dot)}); try expectFmt("-0.001", "{f}", .{try parsed("-0.001", .dot)}); try expectFmt("0.10", "{f}", .{try parsed("0.10", .dot)});}
test "M1-CSV-019: negative values keep their sign" { try expectFmt("-1.5", "{f}", .{try parsed("-1.5", .dot)}); try expectFmt("-100", "{f}", .{try parsed("-100", .dot)});}
test "M1-RULE-016: literals use a dot whatever the cell separator is" { try expectEqual(Decimal{ .coef = 10000, .scale = 2 }, try Decimal.parseLiteral("100.00")); try expectError(error.InvalidFormat, Decimal.parseLiteral("100,00"));}
test "M1-AC-011: literal 100.00 matches cell 100,0 under a comma separator" { const literal = try Decimal.parseLiteral("100.00"); const cell = try parsed("100,0", .comma); try expect(literal.eql(cell));}
test "M1-HIST-008: scale, sign spelling and leading zeroes are insignificant" { try expect((try parsed("100.0", .dot)).eql(try parsed("100.00", .dot))); try expect((try parsed("+100", .dot)).eql(try parsed("100", .dot))); try expect((try parsed("007", .dot)).eql(try parsed("7", .dot))); try expect((try parsed("-0.00", .dot)).eql(try parsed("0", .dot)));}
test "M1-HIST-008: different values are not equal" { try expect(!(try parsed("100.0", .dot)).eql(try parsed("100.01", .dot))); try expect(!(try parsed("1", .dot)).eql(try parsed("-1", .dot)));}
test "M1-RULE-016: ordering follows mathematical value" { try expectEqual(std.math.Order.lt, (try parsed("1.5", .dot)).order(try parsed("2", .dot))); try expectEqual(std.math.Order.gt, (try parsed("2", .dot)).order(try parsed("1.5", .dot))); try expectEqual(std.math.Order.lt, (try parsed("-1", .dot)).order(Decimal.zero)); try expectEqual(std.math.Order.gt, (try parsed("0.1", .dot)).order(try parsed("0.09", .dot)));}
test "M1-RULE-016: comparison holds when scales differ beyond i128 cross-scaling" { // 25 fractional digits: the shift exceeds max_safe_shift, so sign decides. const tiny = try parsed("0.0000000000000000000000001", .dot); const one = try parsed("1", .dot); const minus_one = try parsed("-1", .dot);
try expectEqual(std.math.Order.gt, one.order(tiny)); try expectEqual(std.math.Order.lt, tiny.order(one)); try expectEqual(std.math.Order.lt, minus_one.order(tiny)); try expectEqual(std.math.Order.gt, tiny.order(minus_one));}
test "M1-RULE-016: zero compares correctly against any scale" { const tiny = try parsed("0.0000000000000000000000001", .dot); try expectEqual(std.math.Order.lt, Decimal.zero.order(tiny)); try expectEqual(std.math.Order.gt, tiny.order(Decimal.zero)); try expectEqual(std.math.Order.eq, Decimal.zero.order(try parsed("0.000", .dot)));}
test "A1: digits beyond an i64 coefficient overflow rather than round" { try expectError(error.Overflow, parsed("99999999999999999999", .dot)); try expectError(error.Overflow, parsed("-99999999999999999999", .dot)); // Well inside the range still parses. try expectEqual(@as(i64, 999999999999999999), (try parsed("999999999999999999", .dot)).coef);}
test "A1: more than 255 fractional digits overflows" { var text: [300]u8 = undefined; text[0] = '0'; text[1] = '.'; @memset(text[2..], '0'); try expectError(error.Overflow, parsed(&text, .dot));}
test "M1-CSV-019: the widest magnitude is maxInt(i64), not minInt(i64)" { // The magnitude is accumulated before the sign is applied, so the most // negative i64 is one step out of reach. That asymmetry is load-bearing: // the journal parser reads amounts through this same function, so a // Decimal it cannot produce is also one it could never read back. Widening // this without widening emission would create a journal we can write and // not parse. try std.testing.expectError(error.Overflow, Decimal.parseLiteral("-9223372036854775808"));
const widest = try Decimal.parseLiteral("-9223372036854775807"); try std.testing.expectEqual(@as(i64, -std.math.maxInt(i64)), widest.coef);
try expectFmt("-9223372036854775807", "{f}", .{widest});}