//! CSV record stream (M1-CSV-001..007, M1-CSV-010). //! //! zarko does the lexing. What it does not do is any of the strictness //! milestone one requires, so this module wraps it: //! //! * encoding and line-ending discipline are checked over the raw bytes //! before parsing begins (M1-CSV-001, M1-CSV-003); //! * the physical row ordinal is counted here, because zarko keeps none //! (M1-TPL-004, M1-DIAG-001); //! * a physically blank record is rejected (M1-CSV-010). //! //! That last one cannot be decided from the parsed fields. A blank line and a //! record holding one quoted empty field both come back as a single empty //! field, so the record's byte span is inspected instead. //! //! Two behaviours are zarko's already and are covered by pinning tests rather //! than code: a trailing line ending yields no extra record, and a final //! record may end at EOF (M1-CSV-004). M1-CSV-005 makes that pinned behaviour //! part of the specification, so a dependency change should fail those tests. const std = @import("std"); const diag = @import("../diag.zig"); const zarko = @import("zarko"); pub const Error = error{ InvalidCsv, OutOfMemory }; /// M1-CSV-002. pub const Delimiter = enum { comma, semicolon, tab, pub fn byte(self: Delimiter) u8 { return switch (self) { .comma => ',', .semicolon => ';', .tab => '\t', }; } }; /// M1-CSV-003. zarko also offers a lone CR; milestone one does not. pub const LineEnding = enum { crlf, lf, pub fn bytes(self: LineEnding) []const u8 { return switch (self) { .lf => "\n", .crlf => "\r\n", }; } fn dialectValue(self: LineEnding) zarko.LineEnding { return switch (self) { .lf => .lf, .crlf => .crlf, }; } }; pub const Options = struct { delimiter: Delimiter, line_ending: LineEnding, }; pub const Record = struct { /// Exact decoded cells: no trimming, no normalization (M1-CSV-007). fields: []const []const u8, /// Zero-based ordinal over every record in the file, including preamble /// and header records a later stage skips (M1-CSV-008, M1-TPL-004). physical_row: usize, }; pub const Reader = struct { arena: std.mem.Allocator, diags: *diag.List, file: []const u8, next_row: usize, options: Options, parser: zarko.Parser, pub fn init( arena: std.mem.Allocator, file: []const u8, raw_source: []const u8, options: Options, diags: *diag.List, ) Error!Reader { // Removed before anything else sees the bytes, so it cannot reach a // cell, a diagnostic, or the line-ending scan. const source = withoutMark(raw_source); // M1-CSV-001 and M1-CSV-003 are decided over the raw bytes, before // zarko sees them. zarko tolerates line endings that disagree with its // dialect, so leaving this to the parser would let them through. if (encodingProblem(source)) |problem| { try report(diags, file, "{s}", .{problem}); return error.InvalidCsv; } if (lineEndingProblem(source, options.line_ending)) |offset| { try report( diags, file, "line ending at byte {d} is not the configured {s}", .{ offset, @tagName(options.line_ending) }, ); return error.InvalidCsv; } return .{ .arena = arena, .file = file, .diags = diags, .options = options, .parser = zarko.Parser.init(arena, source, .{ .separator = options.delimiter.byte(), // M1-CSV-005: the quote byte is always a double quote. .quote = '"', .line_ending = options.line_ending.dialectValue(), }), .next_row = 0, }; } pub fn next(self: *Reader) Error!?Record { const span_start = self.parser.pos; const row = self.next_row; const record = self.parser.next() catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, // zarko carries no position, so the row counted here is the // diagnostic's only locator (risk R4). error.UnterminatedQuote => { try self.reportRow(row, "quoted field is not closed", .{}); return error.InvalidCsv; }, else => { try self.reportRow(row, "malformed CSV record", .{}); return error.InvalidCsv; }, } orelse return null; self.next_row += 1; if (isPhysicallyBlank(self.parser.input[span_start..self.parser.pos], self.options.line_ending)) { try self.reportRow(row, "record is physically blank", .{}); return error.InvalidCsv; } return .{ .physical_row = row, .fields = record.fields }; } fn reportRow( self: *Reader, row: usize, comptime fmt: []const u8, args: anytype, ) Error!void { try self.diags.addRow(self.file, row, fmt, args); } }; /// M1-CSV-010. A blank line and a record of one quoted empty field decode /// identically, so the decision is made on the bytes the record occupied. fn isPhysicallyBlank(span: []const u8, ending: LineEnding) bool { const terminator = ending.bytes(); const body = if (std.mem.endsWith(u8, span, terminator)) span[0 .. span.len - terminator.len] else span; return body.len == 0; } const utf8_bom = "\xEF\xBB\xBF"; /// M1-CSV-001. A leading mark is framing, not data: exporters add one to files /// the user cannot regenerate, and leaving it in would put it inside the first /// header cell, where an exact header match would then fail against a file /// that visibly says the right thing (M1-CSV-012). fn withoutMark(source: []const u8) []const u8 { if (std.mem.startsWith(u8, source, utf8_bom)) return source[utf8_bom.len..]; return source; } /// M1-CSV-001. Decoding performs no charset conversion, so anything that is /// not UTF-8 is refused. A UTF-16 mark starts with a byte that is never valid /// UTF-8, so it is caught here. fn encodingProblem(source: []const u8) ?[]const u8 { if (!std.unicode.utf8ValidateSlice(source)) return "file is not valid UTF-8"; return null; } /// M1-CSV-003. Every line break must be the configured sequence, including /// inside a quoted field, so this needs no quote tracking: a stray byte is /// illegal wherever it appears. fn lineEndingProblem(source: []const u8, ending: LineEnding) ?usize { switch (ending) { // Any CR at all is wrong: a lone one, or the other ending. .lf => return std.mem.findScalar(u8, source, '\r'), .crlf => { var i: usize = 0; while (i < source.len) : (i += 1) { switch (source[i]) { '\r' => { if (i + 1 >= source.len or source[i + 1] != '\n') return i; i += 1; }, // Reached only when not consumed as part of a CRLF above. '\n' => return i, else => {}, } } return null; }, } } fn report( diags: *diag.List, file: []const u8, comptime fmt: []const u8, args: anytype, ) Error!void { try diags.addFmt(fmt, args, .{ .file = file }); } // -- tests ------------------------------------------------------------------- const expect = std.testing.expect; const expectEqual = std.testing.expectEqual; const expectEqualStrings = std.testing.expectEqualStrings; const expectError = std.testing.expectError; fn readAll( arena: *std.heap.ArenaAllocator, source: []const u8, options: Options, ) Error![]const Record { const alloc = arena.allocator(); const diags = diag.List.create(alloc) catch return error.OutOfMemory; var reader = try Reader.init(alloc, "bank.csv", source, options, diags); var list: std.ArrayList(Record) = .empty; while (try reader.next()) |record| try list.append(alloc, record); return list.items; } const comma_lf: Options = .{ .delimiter = .comma, .line_ending = .lf }; const comma_crlf: Options = .{ .delimiter = .comma, .line_ending = .crlf }; test "M1-CSV-001: a leading byte-order mark is ignored, not kept" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "\xEF\xBB\xBFa,b\n", comma_lf); try expectEqual(@as(usize, 1), records.len); // It must not survive into the first cell, or an exact header match would // fail against a file that plainly says "a" (M1-CSV-012). try expectEqualStrings("a", records[0].fields[0]); } test "M1-CSV-001: a mark elsewhere in the file is ordinary data" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Only a leading mark is framing; U+FEFF inside a cell is content. const records = try readAll(&arena, "a,\xEF\xBB\xBFb\n", comma_lf); try expectEqualStrings("\xEF\xBB\xBFb", records[0].fields[1]); } test "M1-CSV-001: invalid UTF-8 is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,\xC3\x28\n", comma_lf)); } test "M1-CSV-001: non-ASCII UTF-8 survives unchanged" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "Wä,café\n", comma_lf); try expectEqualStrings("Wä", records[0].fields[0]); try expectEqualStrings("café", records[0].fields[1]); } test "M1-CSV-002: each supported delimiter separates fields" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const by_comma = try readAll(&arena, "a,b\n", comma_lf); try expectEqual(@as(usize, 2), by_comma[0].fields.len); const by_semicolon = try readAll(&arena, "a;b\n", .{ .delimiter = .semicolon, .line_ending = .lf }); try expectEqual(@as(usize, 2), by_semicolon[0].fields.len); const by_tab = try readAll(&arena, "a\tb\n", .{ .delimiter = .tab, .line_ending = .lf }); try expectEqual(@as(usize, 2), by_tab[0].fields.len); // A delimiter that is not the configured one is ordinary data. const literal = try readAll(&arena, "a;b\n", comma_lf); try expectEqual(@as(usize, 1), literal[0].fields.len); try expectEqualStrings("a;b", literal[0].fields[0]); } test "M1-CSV-003: in lf mode any carriage return is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,b\r\n", comma_lf)); try expectError(error.InvalidCsv, readAll(&arena, "a,b\rc\n", comma_lf)); try expectError(error.InvalidCsv, readAll(&arena, "a,\"x\ry\"\n", comma_lf)); } test "M1-CSV-003: in crlf mode a lone LF or CR is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,b\n", comma_crlf)); try expectError(error.InvalidCsv, readAll(&arena, "a,b\rc\r\n", comma_crlf)); try expectError(error.InvalidCsv, readAll(&arena, "a,b\r\nc,d\n", comma_crlf)); } test "M1-AC-018: a lone LF inside a quoted field fails under crlf" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,\"x\ny\"\r\n", comma_crlf)); // The same file with the configured ending throughout is fine. const rows = try readAll(&arena, "a,\"x\r\ny\"\r\n", comma_crlf); try expectEqualStrings("x\r\ny", rows[0].fields[1]); } test "M1-CSV-004: a final record may end at EOF without a line ending" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a,b\nc,d", comma_lf); try expectEqual(@as(usize, 2), records.len); try expectEqualStrings("d", records[1].fields[1]); } test "M1-CSV-004: a trailing line ending yields no extra record" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a,b\nc,d\n", comma_lf); try expectEqual(@as(usize, 2), records.len); } test "M1-CSV-005: a quote opens a field only as its first byte" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a\"b,c\n", comma_lf); try expectEqualStrings("a\"b", records[0].fields[0]); try expectEqualStrings("c", records[0].fields[1]); } test "M1-CSV-006: a quoted field carries delimiters and line breaks" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const with_delimiter = try readAll(&arena, "\"a,b\",c\n", comma_lf); try expectEqual(@as(usize, 2), with_delimiter[0].fields.len); try expectEqualStrings("a,b", with_delimiter[0].fields[0]); const with_break = try readAll(&arena, "\"a\nb\",c\n", comma_lf); try expectEqualStrings("a\nb", with_break[0].fields[0]); } test "M1-CSV-006: a doubled quote is a literal quote" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "\"a\"\"b\"\n", comma_lf); try expectEqualStrings("a\"b", records[0].fields[0]); } test "M1-CSV-006: unclosed quoting is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "\"unterminated\n", comma_lf)); } test "M1-CSV-007: cells keep their exact bytes" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, " padded ,UPPER,\ttab\n", comma_lf); try expectEqualStrings(" padded ", records[0].fields[0]); try expectEqualStrings("UPPER", records[0].fields[1]); try expectEqualStrings("\ttab", records[0].fields[2]); } test "M1-CSV-010: a physically blank record is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, readAll(&arena, "a,b\n\nc,d\n", comma_lf)); try expectError(error.InvalidCsv, readAll(&arena, "\na,b\n", comma_lf)); } test "M1-CSV-010: a record of empty fields is not physically blank" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const separators = try readAll(&arena, ",,\n", comma_lf); try expectEqual(@as(usize, 3), separators[0].fields.len); try expectEqualStrings("", separators[0].fields[0]); // A single quoted empty field is a record, though it decodes to the same // one empty field a blank line would produce. const quoted = try readAll(&arena, "\"\"\n", comma_lf); try expectEqual(@as(usize, 1), quoted[0].fields.len); try expectEqualStrings("", quoted[0].fields[0]); } test "M1-DIAG-001: physical rows are zero-based and count every record" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "a\nb\nc\n", comma_lf); try expectEqual(@as(usize, 0), records[0].physical_row); try expectEqual(@as(usize, 1), records[1].physical_row); try expectEqual(@as(usize, 2), records[2].physical_row); } test "M1-TPL-004: a quoted line break does not advance the physical row" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "\"a\nb\"\nc\n", comma_lf); try expectEqual(@as(usize, 2), records.len); try expectEqual(@as(usize, 0), records[0].physical_row); try expectEqual(@as(usize, 1), records[1].physical_row); } test "M1-CSV-024: a rejection names the source file" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const diags = try diag.List.create(alloc); _ = Reader.init(alloc, "bank/2026.csv", "a,b\r\n", comma_lf, diags) catch {}; try expect(!diags.isEmpty()); try expectEqualStrings("bank/2026.csv", diags.items.items[0].file.?); } test "an empty file yields no records" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const records = try readAll(&arena, "", comma_lf); try expectEqual(@as(usize, 0), records.len); }