Something went wrong. Try again.
A low-level, non-allocating CSV tokenizer for Zig that reads any std.Io.Reader and yields tokens.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520// SPDX-FileCopyrightText: © 2021-2024 @_beho// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
const std = @import("std");const testing = std.testing;const expect = testing.expect;const csv_mod = @import("csv");
fn getTokenizer(reader: *std.Io.Reader, buffer: []u8, config: csv_mod.CsvConfig) !csv_mod.CsvTokenizer { const csv = try csv_mod.CsvTokenizer.init(reader, buffer, config); return csv;}
fn expectToken(comptime expected: csv_mod.CsvToken, maybe_actual: ?csv_mod.CsvToken) !void { if (maybe_actual) |actual| { if (@backingInt(expected) != @backingInt(actual)) { std.log.warn("Expected {t} but is {t}\n", .{ expected, actual }); return error.TestFailed; }
switch (expected) { .field => { try testing.expectEqualStrings(expected.field, actual.field); }, else => {}, } } else { std.log.warn("Expected {t} but is {?t}\n", .{ expected, maybe_actual }); return error.TestFailed; }}
test "Create iterator for file reader" { const data = @embedFile("resources/test-1.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined;
_ = try getTokenizer(&reader, &csv_buf, .{});}
test "Read single simple record from file" { const data = @embedFile("resources/test-1.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "abc" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
const next = csv.next() catch unreachable;
try expect(next == null);}
test "Read multiple simple records from file" { const data = @embedFile("resources/test-2.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "abc" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
try expectToken(csv_mod.CsvToken{ .field = "2" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "def ghc" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
const next = csv.next() catch unreachable;
try expect(next == null);}
test "Read quoted fields" { const data = @embedFile("resources/test-4.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "def ghc" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
try expectToken(csv_mod.CsvToken{ .field = "2" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "abc \"def\"" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
const next = csv.next() catch unreachable;
try expect(next == null);}
test "Second read is necessary to obtain field" { const data = @embedFile("resources/test-read-required-for-field.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
try expectToken(csv_mod.CsvToken{ .field = "12345" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "67890" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
const next = csv.next() catch unreachable;
try expect(next == null);}
test "File is empty" { const data = @embedFile("resources/test-empty.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
const next = csv.next() catch unreachable;
try expect(next == null);}
test "Field is longer than buffer" { const data = @embedFile("resources/test-error-short-buffer.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [8]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
const next = csv.next(); try std.testing.expectError(csv_mod.CsvError.ShortBuffer, next);}
test "Quoted field is longer than buffer" { const data = @embedFile("resources/test-error-short-buffer-quoted.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [8]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
const next = csv.next(); try std.testing.expectError(csv_mod.CsvError.ShortBuffer, next);}
test "Quoted field with double quotes is longer than buffer" { const data = @embedFile("resources/test-error-short-buffer-quoted-with-double.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [8]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
const next = csv.next(); try std.testing.expectError(csv_mod.CsvError.ShortBuffer, next);}
test "Quoted field with double quotes can be read on retry" { const data = @embedFile("resources/test-error-short-buffer-quoted-with-double.csv"); var reader = std.Io.Reader.fixed(data); var csv_buf: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &csv_buf, .{});
try expectToken(csv_mod.CsvToken{ .field = "1234567890\"" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());
const next = csv.next() catch unreachable;
try expect(next == null);}
/// Drive a tokenizer across `data` and assert the entire token stream, so a/// terminator case can be stated as the fields and record breaks it ought to/// produce. A `null` in `expected` stands for a `row_end`.fn expectStream( data: []const u8, config: csv_mod.CsvConfig, comptime buffer_len: usize, expected: []const ?[]const u8,) !void { var reader = std.Io.Reader.fixed(data); var buffer: [buffer_len]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, config);
for (expected) |maybe_field| { const token = (try csv.next()) orelse { std.log.warn("stream ended early, still expecting {?s}\n", .{maybe_field}); return error.TestFailed; };
switch (token) { .field => |value| { const expected_field = maybe_field orelse { std.log.warn("expected row_end but got field {s}\n", .{value}); return error.TestFailed; }; try testing.expectEqualStrings(expected_field, value); }, .row_end => { if (maybe_field) |expected_field| { std.log.warn("expected field {s} but got row_end\n", .{expected_field}); return error.TestFailed; } }, } }
try expect((try csv.next()) == null);}
test "LF terminates a record" { try expectStream("1,abc\n2,def\n", .{}, 64, &.{ "1", "abc", null, "2", "def", null });}
test "CRLF terminates a record" { try expectStream("1,abc\r\n2,def\r\n", .{}, 64, &.{ "1", "abc", null, "2", "def", null });}
test "CR alone terminates a record" { try expectStream("1,abc\r2,def\r", .{}, 64, &.{ "1", "abc", null, "2", "def", null });}
test "CRLF is a single terminator and makes no empty record" { try expectStream("1\r\n2\r\n3\r\n", .{}, 64, &.{ "1", null, "2", null, "3", null });}
test "Terminators may be mixed within one input" { try expectStream( "1,a\n2,b\r\n3,c\r4,d\r\n", .{}, 64, &.{ "1", "a", null, "2", "b", null, "3", "c", null, "4", "d", null }, );}
test "Input ending in a bare CR terminates the last record" { try expectStream("1,abc\r", .{}, 64, &.{ "1", "abc", null });}
test "Empty record under CRLF is one zero-length field" { try expectStream("1\r\n\r\n2\r\n", .{}, 64, &.{ "1", null, "", null, "2", null });}
test "CRLF split across a buffer refill" { const data = "aaa,bbb\r\nccc,ddd\r\neee,fff\r\n"; const expected: []const ?[]const u8 = &.{ "aaa", "bbb", null, "ccc", "ddd", null, "eee", "fff", null, };
// The interesting case is a refill landing between the CR and the LF. // Rather than compute which buffer length does that, run every small one // and require the same answer from each. inline for (.{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 32 }) |buffer_len| { expectStream(data, .{}, buffer_len, expected) catch |err| { std.log.warn("failed with buffer_len={d}\n", .{buffer_len}); return err; }; }}
test "CRLF inside a quoted field is data, not a terminator" { try expectStream( "\"a\r\nb\",c\r\nd\r\n", .{}, 64, &.{ "a\r\nb", "c", null, "d", null }, );}
test "Quoted field followed by CRLF" { try expectStream( "\"a\",\"b\"\r\n\"c\",\"d\"\r\n", .{}, 64, &.{ "a", "b", null, "c", "d", null }, );}
test "Under .byte = LF a carriage return is field data" { try expectStream( "1,abc\r\n2,def\r\n", .{ .row_sep = .{ .byte = '\n' } }, 64, &.{ "1", "abc\r", null, "2", "def\r", null }, );}
test "Under .byte = CR a line feed is field data" { try expectStream( "1,abc\r2,def\r", .{ .row_sep = .{ .byte = '\r' } }, 64, &.{ "1", "abc", null, "2", "def", null }, ); try expectStream( "1,a\nb\r", .{ .row_sep = .{ .byte = '\r' } }, 64, &.{ "1", "a\nb", null }, );}
test "A record separator may be any byte" { try expectStream( "1,abc;2,def;", .{ .row_sep = .{ .byte = ';' } }, 64, &.{ "1", "abc", null, "2", "def", null }, );}
test "Final record need not be terminated" { try expectStream("1,abc\r\n2,def", .{}, 64, &.{ "1", "abc", null, "2", "def", null }); try expectStream("1,abc\n2,def", .{}, 64, &.{ "1", "abc", null, "2", "def", null });}
test "Input of a single unterminated field" { try expectStream("abc", .{}, 64, &.{ "abc", null });}
test "Unterminated record ending in a column separator keeps the empty field" { // `1,` is two fields, exactly as `1,\n` is. try expectStream("1,", .{}, 64, &.{ "1", "", null }); try expectStream("1,\n", .{}, 64, &.{ "1", "", null }); try expectStream("a,b\r\nc,", .{}, 64, &.{ "a", "b", null, "c", "", null });}
test "Unterminated record ending in a quoted field" { try expectStream("1,\"abc\"", .{}, 64, &.{ "1", "abc", null }); try expectStream("\"a\",\"b\"", .{}, 64, &.{ "a", "b", null });}
test "Unterminated final field spanning a buffer refill" { const data = "aaa,bbb\r\nccc,dddddddddddddddddddd"; const expected: []const ?[]const u8 = &.{ "aaa", "bbb", null, "ccc", "dddddddddddddddddddd", null };
inline for (.{ 21, 22, 24, 28, 32, 64 }) |buffer_len| { expectStream(data, .{}, buffer_len, expected) catch |err| { std.log.warn("failed with buffer_len={d}\n", .{buffer_len}); return err; }; }}
test "Unterminated final record under an explicit byte separator" { try expectStream("1,abc\n2,def", .{ .row_sep = .{ .byte = '\n' } }, 64, &.{ "1", "abc", null, "2", "def", null }); try expectStream("1,abc\r2,def", .{ .row_sep = .{ .byte = '\r' } }, 64, &.{ "1", "abc", null, "2", "def", null });}
test "A field longer than the buffer is still a short buffer" { // The unterminated-last-record path must not swallow the genuine error: // here the input has not run out, the buffer has. var reader = std.Io.Reader.fixed("aaaaaaaaaaaaaaaaaaaa,b\n"); var buffer: [8]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{});
try testing.expectError(csv_mod.CsvError.ShortBuffer, csv.next());}
test "Unterminated quoted field with a doubled quote" { try expectStream("\"a\"\"b\"", .{}, 64, &.{ "a\"b", null }); try expectStream("\"\"", .{}, 64, &.{ "", null }); try expectStream("\"a\r\nb\"", .{}, 64, &.{ "a\r\nb", null });}
test "An unclosed quoted field is an error" { var reader = std.Io.Reader.fixed("1,\"abc"); var buffer: [64]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{});
try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try testing.expectError(csv_mod.CsvError.UnclosedQuote, csv.next());}
test "An unclosed quote is told apart from a short buffer" { // The input ends with the field still open, and the buffer has room to // spare. Reporting ShortBuffer here would send the caller looking for a // bigger buffer that cannot help. { var reader = std.Io.Reader.fixed("\"oops,a\r\nb,c\r\n"); var buffer: [256]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{}); try testing.expectError(csv_mod.CsvError.UnclosedQuote, csv.next()); }
// Here the quote does close, but not before the buffer fills. { var reader = std.Io.Reader.fixed("\"aaaaaaaaaaaaaaaaaaaa\"\r\n"); var buffer: [8]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{}); try testing.expectError(csv_mod.CsvError.ShortBuffer, csv.next()); }}
test "An unclosed quote is bounded by the buffer, not the input" { // A single unclosed quote must not swallow the rest of the file, however // much of it there is. const gpa = testing.allocator;
var input: std.ArrayList(u8) = .empty; defer input.deinit(gpa); try input.appendSlice(gpa, "\"oops"); for (0..20000) |i| try input.print(gpa, "row{d},a,b\r\n", .{i});
var reader = std.Io.Reader.fixed(input.items); var buffer: [1024]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{});
try testing.expectError(csv_mod.CsvError.ShortBuffer, csv.next());}
test "A UTF-8 byte order mark is skipped by default" { try expectStream("\xef\xbb\xbfName,Age\r\nx,1\r\n", .{}, 64, &.{ "Name", "Age", null, "x", "1", null });
// Only at the very beginning, and only once. try expectStream("a,\xef\xbb\xbfb\r\n", .{}, 64, &.{ "a", "\xef\xbb\xbfb", null }); try expectStream("\xef\xbb\xbf\xef\xbb\xbfa\r\n", .{}, 64, &.{ "\xef\xbb\xbfa", null });
// A file holding nothing but a mark holds no records. try expectStream("\xef\xbb\xbf", .{}, 64, &.{});
// Bytes that only start like a mark are field data. try expectStream("\xef\xbbX,y\r\n", .{}, 64, &.{ "\xef\xbbX", "y", null });}
test "Skipping the byte order mark can be turned off" { try expectStream( "\xef\xbb\xbfName,Age\r\n", .{ .skip_bom = false }, 64, &.{ "\xef\xbb\xbfName", "Age", null }, );}
test "A byte order mark split across a buffer refill" { // The mark has to be recognized even when it does not arrive at once. inline for (.{ 4, 5, 6, 8, 16, 64 }) |buffer_len| { expectStream("\xef\xbb\xbfab,cd\r\n", .{}, buffer_len, &.{ "ab", "cd", null }) catch |err| { std.log.warn("failed with buffer_len={d}\n", .{buffer_len}); return err; }; }}
test "Strict CRLF: only the pair terminates a record" { const strict: csv_mod.CsvConfig = .{ .row_sep = .crlf };
try expectStream("a,b\r\nc,d\r\n", strict, 64, &.{ "a", "b", null, "c", "d", null }); try expectStream("a,b\r\nc,d", strict, 64, &.{ "a", "b", null, "c", "d", null }); try expectStream("a\r\n\r\nb\r\n", strict, 64, &.{ "a", null, "", null, "b", null });}
test "Strict CRLF: a lone CR or LF is field data" { const strict: csv_mod.CsvConfig = .{ .row_sep = .crlf };
try expectStream("a\nb\r\n", strict, 64, &.{ "a\nb", null }); try expectStream("a\rb\r\n", strict, 64, &.{ "a\rb", null }); try expectStream("a\r", strict, 64, &.{ "a\r", null }); try expectStream("x\r\ry\r\n", strict, 64, &.{ "x\r\ry", null });}
test "Strict CRLF: a CR split across a buffer refill" { const data = "aaa,bbb\r\nccc\rddd\r\neee\r\n"; const expected: []const ?[]const u8 = &.{ "aaa", "bbb", null, "ccc\rddd", null, "eee", null };
// A refill landing between a CR and the byte that decides its meaning is // the whole difficulty of this mode, so try every small buffer. inline for (.{ 9, 10, 11, 12, 13, 14, 16, 24, 64 }) |buffer_len| { expectStream(data, .{ .row_sep = .crlf }, buffer_len, expected) catch |err| { std.log.warn("failed with buffer_len={d}\n", .{buffer_len}); return err; }; }}
test "Strict CRLF: quoted fields carry CR and LF unharmed" { const strict: csv_mod.CsvConfig = .{ .row_sep = .crlf };
try expectStream("\"x\ry\nz\r\nw\"\r\n", strict, 64, &.{ "x\ry\nz\r\nw", null }); try expectStream("\"a\",\"b\"\r\n", strict, 64, &.{ "a", "b", null });}
test "Strict CRLF: a bare CR after a quoted field is an error" { var reader = std.Io.Reader.fixed("\"a\"\rb\r\n"); var buffer: [64]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{ .row_sep = .crlf });
try expectToken(csv_mod.CsvToken{ .field = "a" }, try csv.next()); try testing.expectError(csv_mod.CsvError.NoSeparatorAfterField, csv.next());}
test "A configured quote is honored in place of the default" { const single: csv_mod.CsvConfig = .{ .quote = '\'' };
try expectStream("'a,b',c\n", single, 64, &.{ "a,b", "c", null }); try expectStream("'a''b'\n", single, 64, &.{ "a'b", null }); // The default quote becomes ordinary data once it is not the quote. try expectStream("a\"b,c\n", single, 64, &.{ "a\"b", "c", null }); try expectStream("'a;b';c\n", .{ .quote = '\'', .col_sep = ';' }, 64, &.{ "a;b", "c", null });}
test "A configured quote still rejects what the default would" { var reader = std.Io.Reader.fixed("'a'x\n"); var buffer: [64]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{ .quote = '\'' });
try expectToken(csv_mod.CsvToken{ .field = "a" }, try csv.next()); try testing.expectError(csv_mod.CsvError.NoSeparatorAfterField, csv.next());}
test "Strict CRLF needs one byte more headroom than the buffer alone" { // Deciding whether a CR ends the record takes the CR and the byte behind // it in the buffer together, so `.crlf` wants one byte more than `.any` // does for the same field. It says so with an error rather than // misreading the input. const data = "ccc\rddd\r\neee\r\n"; // longest field is 7 bytes
try expectStream(data, .{ .row_sep = .crlf }, 9, &.{ "ccc\rddd", null, "eee", null });
var reader = std.Io.Reader.fixed(data); var buffer: [8]u8 = undefined; var csv = try getTokenizer(&reader, &buffer, .{ .row_sep = .crlf }); try testing.expectError(csv_mod.CsvError.ShortBuffer, csv.next());}