Something went wrong. Try again.
Convert CSV banking statements to hledger journals
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504//! Expression tokens (M1-RULE-014, M1-RULE-015, M1-RULE-021).//!//! Tokenising is where three of the grammar's sharper rules live. A keyword is//! only a keyword when it is a whole token, so `android` is one identifier.//! Operators take their longest valid form, so `<=` never lexes as `<` then//! `=`. And only ASCII space, tab, CR and LF separate tokens: every other//! byte that starts no token is rejected, which is how M1-RULE-014 disposes of//! non-ASCII whitespace and of comment syntax the language does not have.
const std = @import("std");
pub const Error = error{ InvalidExpression, OutOfMemory };
/// The regex flags M1-RULE-003 admits. Mapping them onto the engine is/// M1-RULE-017 and belongs to the regex module.pub const Flags = struct { case_insensitive: bool = false, dot_all: bool = false, multiline: bool = false,};
pub const RegexLiteral = struct { flags: Flags, /// The sequence between the delimiters, with `\/` already reduced to `/` /// (M1-RULE-021). Every other backslash is untouched. pattern: []const u8,};
pub const Token = union(enum) { /// Raw text; the value is parsed by the decimal module (M1-RULE-016). decimal: []const u8, eq, ge, gt, identifier: []const u8, keyword_and, keyword_not, keyword_or, le, lparen, lt, matches, ne, regex: RegexLiteral, rparen, /// Decoded value, escapes already applied (M1-RULE-015). string: []const u8,};
pub const Lexer = struct { arena: std.mem.Allocator, pos: usize, source: []const u8, /// Byte offset where the most recent token began, for diagnostics. token_start: usize,
pub fn init(arena: std.mem.Allocator, source: []const u8) Lexer { return .{ .arena = arena, .source = source, .pos = 0, .token_start = 0 }; }
pub fn next(self: *Lexer) Error!?Token { self.skipSeparators(); if (self.pos >= self.source.len) return null; self.token_start = self.pos;
return switch (self.source[self.pos]) { '(' => self.single(.lparen), ')' => self.single(.rparen), '"' => try self.lexString(), '/' => try self.lexRegex(), '=' => try self.twoByte('=', .eq), '!' => try self.twoByte('=', .ne), '~' => try self.twoByte('=', .matches), '<' => self.oneOrTwo('=', .le, .lt), '>' => self.oneOrTwo('=', .ge, .gt), '+', '-', '0'...'9' => try self.lexDecimal(), 'A'...'Z', 'a'...'z', '_' => self.lexIdentifier(), // Anything else starts no token: a stray byte, a '#', or the // leading byte of non-ASCII whitespace (M1-RULE-014). else => error.InvalidExpression, }; }
/// M1-RULE-014: only these four separate tokens. fn skipSeparators(self: *Lexer) void { while (self.pos < self.source.len) : (self.pos += 1) { switch (self.source[self.pos]) { ' ', '\t', '\r', '\n' => {}, else => return, } } }
fn single(self: *Lexer, token: Token) Token { self.pos += 1; return token; }
/// An operator that exists only in its two-byte form. fn twoByte(self: *Lexer, second: u8, token: Token) Error!Token { if (self.pos + 1 >= self.source.len or self.source[self.pos + 1] != second) { return error.InvalidExpression; } self.pos += 2; return token; }
/// M1-RULE-021: prefer the longer form. fn oneOrTwo(self: *Lexer, second: u8, long: Token, short: Token) Token { if (self.pos + 1 < self.source.len and self.source[self.pos + 1] == second) { self.pos += 2; return long; } self.pos += 1; return short; }
fn lexIdentifier(self: *Lexer) Token { const start = self.pos; while (self.pos < self.source.len and isIdentifierChar(self.source[self.pos])) { self.pos += 1; } const text = self.source[start..self.pos]; if (std.mem.eql(u8, text, "and")) return .keyword_and; if (std.mem.eql(u8, text, "or")) return .keyword_or; if (std.mem.eql(u8, text, "not")) return .keyword_not; return .{ .identifier = text }; }
/// M1-RULE-003: [+-]?[0-9]+("." [0-9]+)? fn lexDecimal(self: *Lexer) Error!Token { const start = self.pos; if (self.source[self.pos] == '+' or self.source[self.pos] == '-') self.pos += 1;
const integer_start = self.pos; while (self.pos < self.source.len and isDigit(self.source[self.pos])) self.pos += 1; if (self.pos == integer_start) return error.InvalidExpression;
if (self.pos < self.source.len and self.source[self.pos] == '.') { self.pos += 1; const fraction_start = self.pos; while (self.pos < self.source.len and isDigit(self.source[self.pos])) self.pos += 1; // "1." has no fractional digits. if (self.pos == fraction_start) return error.InvalidExpression; }
return .{ .decimal = self.source[start..self.pos] }; }
fn isDigit(c: u8) bool { return c >= '0' and c <= '9'; }
/// M1-RULE-015: double quotes, and only these five escapes. fn lexString(self: *Lexer) Error!Token { self.pos += 1; var decoded: std.ArrayList(u8) = .empty;
while (true) { if (self.pos >= self.source.len) return error.InvalidExpression; const c = self.source[self.pos]; switch (c) { '"' => { self.pos += 1; return .{ .string = decoded.items }; }, // A literal line break never appears inside a string. '\n', '\r' => return error.InvalidExpression, '\\' => { if (self.pos + 1 >= self.source.len) return error.InvalidExpression; const escaped: u8 = switch (self.source[self.pos + 1]) { '"' => '"', '\\' => '\\', 'n' => '\n', 'r' => '\r', 't' => '\t', else => return error.InvalidExpression, }; try decoded.append(self.arena, escaped); self.pos += 2; }, else => { try decoded.append(self.arena, c); self.pos += 1; }, } } }
/// M1-RULE-017 and M1-RULE-021: `/pattern/flags`, where `\/` is a literal /// slash and every other backslash reaches the engine untouched. fn lexRegex(self: *Lexer) Error!Token { self.pos += 1; var pattern: std.ArrayList(u8) = .empty;
while (true) { if (self.pos >= self.source.len) return error.InvalidExpression; const c = self.source[self.pos]; if (c == '\n' or c == '\r') return error.InvalidExpression;
if (c == '\\') { if (self.pos + 1 >= self.source.len) return error.InvalidExpression; const escaped = self.source[self.pos + 1]; if (escaped == '\n' or escaped == '\r') return error.InvalidExpression; if (escaped == '/') { try pattern.append(self.arena, '/'); } else { try pattern.append(self.arena, '\\'); try pattern.append(self.arena, escaped); } self.pos += 2; continue; }
if (c == '/') { self.pos += 1; break; }
try pattern.append(self.arena, c); self.pos += 1; }
return .{ .regex = .{ .pattern = pattern.items, .flags = try self.lexFlags() } }; }
/// Every identifier character after the closing delimiter is a flag, so a /// run like `sand` is rejected rather than read as `s` then the keyword. fn lexFlags(self: *Lexer) Error!Flags { const start = self.pos; while (self.pos < self.source.len and isIdentifierChar(self.source[self.pos])) { self.pos += 1; }
var flags: Flags = .{}; for (self.source[start..self.pos]) |flag| switch (flag) { 'i' => { if (flags.case_insensitive) return error.InvalidExpression; flags.case_insensitive = true; }, 'm' => { if (flags.multiline) return error.InvalidExpression; flags.multiline = true; }, 's' => { if (flags.dot_all) return error.InvalidExpression; flags.dot_all = true; }, else => return error.InvalidExpression, }; return flags; }
/// M1-RULE-003's identifier grammar. Field aliases and template /// placeholders use the same shape, so this is its one definition. pub fn isIdentifier(text: []const u8) bool { if (text.len == 0) return false; switch (text[0]) { 'A'...'Z', 'a'...'z', '_' => {}, else => return false, } for (text[1..]) |c| { if (!isIdentifierChar(c)) return false; } return true; }
pub fn isIdentifierChar(c: u8) bool { return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '_'; }
/// M1-RULE-014: these three are reserved and may not be field aliases. pub fn isKeyword(name: []const u8) bool { return std.mem.eql(u8, name, "and") or std.mem.eql(u8, name, "or") or std.mem.eql(u8, name, "not"); }};
// -- tests -------------------------------------------------------------------
const expect = std.testing.expect;const expectEqual = std.testing.expectEqual;const expectEqualStrings = std.testing.expectEqualStrings;const expectError = std.testing.expectError;
fn tokensOf(arena: *std.heap.ArenaAllocator, source: []const u8) Error![]const Token { const alloc = arena.allocator(); var list: std.ArrayList(Token) = .empty; var lexer = Lexer.init(alloc, source); while (try lexer.next()) |token| try list.append(alloc, token); return list.items;}
fn firstToken(arena: *std.heap.ArenaAllocator, source: []const u8) Error!Token { const tokens = try tokensOf(arena, source); return tokens[0];}
test "M1-RULE-021: a keyword is only recognised as a complete token" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // The requirement names this case: android is one identifier. try expectEqualStrings("android", (try firstToken(&arena, "android")).identifier); try expectEqualStrings("andy", (try firstToken(&arena, "andy")).identifier); try expectEqualStrings("nothing", (try firstToken(&arena, "nothing")).identifier); try expectEqual(Token.keyword_and, try firstToken(&arena, "and")); try expectEqual(Token.keyword_or, try firstToken(&arena, "or")); try expectEqual(Token.keyword_not, try firstToken(&arena, "not"));}
test "M1-RULE-014: keywords are case-sensitive and lowercase only" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectEqualStrings("AND", (try firstToken(&arena, "AND")).identifier); try expectEqualStrings("And", (try firstToken(&arena, "And")).identifier); try expectEqualStrings("NOT", (try firstToken(&arena, "NOT")).identifier);}
test "M1-RULE-014: keywords are reserved and cannot be field aliases" { try expect(Lexer.isKeyword("and")); try expect(Lexer.isKeyword("or")); try expect(Lexer.isKeyword("not")); try expect(!Lexer.isKeyword("android")); try expect(!Lexer.isKeyword("AND"));}
test "M1-RULE-021: operators take their longest valid form" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectEqual(Token.le, try firstToken(&arena, "<=")); try expectEqual(Token.ge, try firstToken(&arena, ">=")); try expectEqual(Token.lt, try firstToken(&arena, "<")); try expectEqual(Token.gt, try firstToken(&arena, ">")); try expectEqual(Token.eq, try firstToken(&arena, "==")); try expectEqual(Token.ne, try firstToken(&arena, "!=")); try expectEqual(Token.matches, try firstToken(&arena, "~="));}
test "M1-RULE-021: an incomplete operator is a syntax error" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "=")); try expectError(error.InvalidExpression, tokensOf(&arena, "!")); try expectError(error.InvalidExpression, tokensOf(&arena, "~"));}
test "M1-RULE-021: tokens need no whitespace between them" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const tokens = try tokensOf(&arena, "a==1"); try expectEqual(@as(usize, 3), tokens.len); try expectEqualStrings("a", tokens[0].identifier); try expectEqual(Token.eq, tokens[1]); try expectEqualStrings("1", tokens[2].decimal);}
test "M1-RULE-021: whitespace must not split a token" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // "an droid" is two identifiers, not one. const tokens = try tokensOf(&arena, "an droid"); try expectEqual(@as(usize, 2), tokens.len); // A split operator is not an operator. try expectError(error.InvalidExpression, tokensOf(&arena, "< ="));}
test "M1-RULE-014: ASCII space, tab, CR and LF are ignored between tokens" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const tokens = try tokensOf(&arena, " a\t==\r\n1 "); try expectEqual(@as(usize, 3), tokens.len);}
test "M1-RULE-014: other Unicode whitespace is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // U+00A0 no-break space and U+2003 em space. try expectError(error.InvalidExpression, tokensOf(&arena, "a\u{00A0}== 1")); try expectError(error.InvalidExpression, tokensOf(&arena, "a\u{2003}== 1"));}
test "M1-RULE-014: the language has no comments" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "a == 1 # comment")); // "//" is an empty regex literal, not a comment opener. It tokenises // here and parses as a regex; the pinned engine is what refuses it, by // declining to compile an empty pattern (see expr/regex.zig). const tokens = try tokensOf(&arena, "// comment"); try expect(tokens[0] == .regex); try expectEqualStrings("comment", tokens[1].identifier);}
test "M1-RULE-015: strings use double quotes and the five allowed escapes" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectEqualStrings("plain", (try firstToken(&arena, "\"plain\"")).string); try expectEqualStrings("a\"b", (try firstToken(&arena, "\"a\\\"b\"")).string); try expectEqualStrings("a\\b", (try firstToken(&arena, "\"a\\\\b\"")).string); try expectEqualStrings("a\nb", (try firstToken(&arena, "\"a\\nb\"")).string); try expectEqualStrings("a\rb", (try firstToken(&arena, "\"a\\rb\"")).string); try expectEqualStrings("a\tb", (try firstToken(&arena, "\"a\\tb\"")).string);}
test "M1-RULE-015: an unknown escape is a syntax error" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "\"a\\qb\"")); try expectError(error.InvalidExpression, tokensOf(&arena, "\"a\\u0041\""));}
test "M1-RULE-015: a literal line break in a string is a syntax error" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "\"a\nb\"")); try expectError(error.InvalidExpression, tokensOf(&arena, "\"a\rb\"")); try expectError(error.InvalidExpression, tokensOf(&arena, "\"unterminated"));}
test "M1-RULE-015: single quotes do not delimit a string" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "'x'"));}
test "M1-RULE-015: strings keep exact UTF-8 without normalization" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Precomposed e-acute and the decomposed pair stay distinct. try expectEqualStrings("\u{00E9}", (try firstToken(&arena, "\"\u{00E9}\"")).string); try expectEqualStrings("e\u{0301}", (try firstToken(&arena, "\"e\u{0301}\"")).string);}
test "M1-RULE-003: decimals carry an optional sign and fraction" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectEqualStrings("1", (try firstToken(&arena, "1")).decimal); try expectEqualStrings("-1", (try firstToken(&arena, "-1")).decimal); try expectEqualStrings("+1.5", (try firstToken(&arena, "+1.5")).decimal); try expectEqualStrings("100.00", (try firstToken(&arena, "100.00")).decimal);}
test "M1-RULE-003: a malformed decimal is a syntax error" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "1.")); try expectError(error.InvalidExpression, tokensOf(&arena, ".5")); try expectError(error.InvalidExpression, tokensOf(&arena, "-"));}
test "M1-RULE-017: a regex literal carries its pattern and flags" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const bare = (try firstToken(&arena, "/food/")).regex; try expectEqualStrings("food", bare.pattern); try expect(!bare.flags.case_insensitive);
const flagged = (try firstToken(&arena, "/food/ims")).regex; try expect(flagged.flags.case_insensitive); try expect(flagged.flags.multiline); try expect(flagged.flags.dot_all);}
test "M1-RULE-021: an escaped slash becomes a literal slash in the pattern" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const token = (try firstToken(&arena, "/a\\/b/")).regex; try expectEqualStrings("a/b", token.pattern);}
test "M1-RULE-017: other backslashes pass through to the engine" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const token = (try firstToken(&arena, "/\\d+\\s/")).regex; try expectEqualStrings("\\d+\\s", token.pattern);}
test "M1-RULE-017: a repeated or unknown flag is a syntax error" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "/a/ii")); try expectError(error.InvalidExpression, tokensOf(&arena, "/a/x")); try expectError(error.InvalidExpression, tokensOf(&arena, "/a/sand"));}
test "M1-RULE-017: a literal line break in a regex is a syntax error" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidExpression, tokensOf(&arena, "/a\nb/")); try expectError(error.InvalidExpression, tokensOf(&arena, "/unterminated"));}
test "parentheses are their own tokens" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const tokens = try tokensOf(&arena, "(a)"); try expectEqual(Token.lparen, tokens[0]); try expectEqual(Token.rparen, tokens[2]);}