Something went wrong. Try again.
A CSV parsing library in Zig.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646//! Low-level field-by-field CSV iterator — the foundational primitive//! of this library.//!//! `FieldIterator` is the only module that reads bytes directly from//! the underlying `std.Io.Reader`. It implements a state machine that//! tokenises the input into fields and emits one field per `next()`//! call, along with a flag indicating whether the field is the last in//! its row.//!//! All higher-level parsers in this library are built on top of//! `FieldIterator`, inheriting its parsing-correctness guarantees.//!//! The iterator performs no heap allocation. Field bytes are written//! into a caller-provided `field_buf`; the returned slice is borrowed//! from that buffer and is invalidated by the next call to `next()`.
const std = @import("std");const Io = std.Io;
const Dialect = @import("dialect.zig").Dialect;const Error = @import("error.zig").Error;
/// Internal parser state.const State = enum { /// Initial state; possibly strip a UTF-8 BOM on first call. start, /// At the beginning of a field (just after a delimiter or row end, /// or at the very start of input). field_start, /// Accumulating bytes of an unquoted field. in_unquoted, /// Accumulating bytes of a quoted field; structural bytes are /// preserved as content. in_quoted, /// Just consumed a closing quote of a quoted field; expecting a /// delimiter, a line ending, or EOF. after_quote, /// End of input reached; further `next()` calls return null. eof, /// A malformed-input error was detected; further `next()` calls /// return the same sticky error. failed,};
/// Low-level field-by-field CSV iterator. The foundational primitive/// of this library; every higher tier wraps a `FieldIterator`. Reads/// bytes from a `std.Io.Reader`, accumulates field bytes into a/// caller-provided `field_buf`, and emits one field per `next()`.pub const FieldIterator = struct { reader: *Io.Reader, field_buf: []u8, dialect: Dialect,
// --- internal state; opaque to callers --------------------------
state: State, /// Position within `field_buf` where the next field's bytes will /// begin. Advances past each emitted field. Reset to 0 via /// `resetBuffer()`. This lets a row iterator hold simultaneous /// slices into `field_buf` for every field of the current row. write_offset: usize, /// True if the most recent emitted field ended a row. Used to /// distinguish "input ended mid-field" (yield trailing empty /// field) from "input ended at end of row" (suppress phantom /// trailing empty row). Initialised true so that an empty input /// yields zero rows. last_was_row_end: bool, /// True if any payload byte has been consumed (BOM bytes do not /// count). Used to make BOM-only inputs equivalent to empty /// inputs. bytes_consumed: bool, /// Sticky error after malformed input. Once set, every subsequent /// `next()` returns this error. pending_error: ?Error,
/// Result of a successful `next()` call. pub const Result = struct { /// Slice borrowed from the iterator's `field_buf`. Valid /// until the next call to `resetBuffer()` (or until /// `field_buf` fills up and `FieldTooLong` is returned). /// Row iterators call `resetBuffer()` at the start of each /// `next()`, so for row-iterator clients the field slices /// are valid until the next row-iterator `next()` call. data: []const u8,
/// True if this field is the last field of its row. end_of_row: bool, };
/// Construct an iterator that reads from `reader`, accumulating /// field bytes into `field_buf`. `field_buf` must be large enough /// to hold every concurrently-live field (between calls to /// `resetBuffer()` — in practice, an entire row's worth of field /// bytes when used via a row iterator). Otherwise `next()` will /// return `error.FieldTooLong`. /// /// Returns `error.InvalidDialect` if the dialect is malformed. pub fn init(reader: *Io.Reader, field_buf: []u8, dialect: Dialect) Error!FieldIterator { try dialect.validate(); return .{ .reader = reader, .field_buf = field_buf, .dialect = dialect, .state = .start, .write_offset = 0, .last_was_row_end = true, .bytes_consumed = false, .pending_error = null, }; }
/// Reset the write position within `field_buf` to 0, invalidating /// all previously-returned `Result.data` slices. Row iterators /// call this at the start of each `next()` so all fields of a /// row can coexist in `field_buf` simultaneously. pub fn resetBuffer(self: *FieldIterator) void { self.write_offset = 0; }
/// Yields the next field, or null at end-of-input. /// /// The `Result.data` slice is borrowed from the iterator's /// `field_buf` and is **invalidated by the next call to `next()`**. /// /// On malformed input, returns one of `UnexpectedQuote`, /// `UnexpectedByteAfterQuote`, or `UnterminatedQuotedField`. /// After a malformed-input error, the iterator becomes sticky: /// every subsequent call returns the same error. /// /// If a single field's decoded content would exceed `field_buf`, /// returns `error.FieldTooLong`. pub fn next(self: *FieldIterator) Error!?Result { if (self.pending_error) |e| return e;
if (self.state == .start) { try self.maybeStripBom(); self.state = .field_start; }
if (self.state == .eof) return null;
// Accumulator for the current field's decoded bytes. Index // into `self.field_buf`. // Bytes are written starting at `self.write_offset` so that // previously-emitted fields in the same row remain valid. // `start` snapshots the offset at the beginning of this field; // `len` is the number of decoded bytes written so far. const start = self.write_offset; var len: usize = 0;
while (true) { switch (self.state) { .start, .eof, .failed => unreachable, // handled above
.field_start => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { // End of input at field boundary. self.state = .eof; if (self.bytes_consumed and !self.last_was_row_end) { // Input ended mid-row with no terminator; yield // a final empty field as the row's last field. self.last_was_row_end = true; return Result{ .data = self.field_buf[start..start], .end_of_row = true }; } return null; }, else => |e| return e, };
if (b == self.dialect.quote_char) { _ = try self.takeByte(); self.state = .in_quoted; continue; } if (b == self.dialect.delimiter) { _ = try self.takeByte(); self.last_was_row_end = false; return Result{ .data = self.field_buf[start..start], .end_of_row = false }; } if (b == '\n') { _ = try self.takeByte(); self.last_was_row_end = true; return Result{ .data = self.field_buf[start..start], .end_of_row = true }; } if (b == '\r') { _ = try self.takeByte(); // Consume an optional following \n for CRLF. self.consumeOptionalLf(); self.last_was_row_end = true; return Result{ .data = self.field_buf[start..start], .end_of_row = true }; } // Ordinary byte: enter unquoted accumulation. self.state = .in_unquoted; continue; },
.in_unquoted => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { self.state = .eof; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; }, else => |e| return e, };
if (b == self.dialect.quote_char) { self.pending_error = error.UnexpectedQuote; self.state = .failed; return error.UnexpectedQuote; } if (b == self.dialect.delimiter) { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = false; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = false }; } if (b == '\n') { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } if (b == '\r') { _ = try self.takeByte(); self.consumeOptionalLf(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } // Append byte to field. if (start + len >= self.field_buf.len) { self.pending_error = error.FieldTooLong; self.state = .failed; return error.FieldTooLong; } _ = try self.takeByte(); self.field_buf[start + len] = b; len += 1; },
.in_quoted => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { self.pending_error = error.UnterminatedQuotedField; self.state = .failed; return error.UnterminatedQuotedField; }, else => |e| return e, };
if (b == self.dialect.quote_char) { _ = try self.takeByte(); // Lookahead: doubled quote = literal quote in content. const next_byte = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { // EOF immediately after closing quote. self.state = .after_quote; continue; }, else => |e| return e, }; if (next_byte == self.dialect.quote_char) { _ = try self.takeByte(); if (start + len >= self.field_buf.len) { self.pending_error = error.FieldTooLong; self.state = .failed; return error.FieldTooLong; } self.field_buf[start + len] = self.dialect.quote_char; len += 1; continue; } self.state = .after_quote; continue; } // Any other byte (including delimiter, \n, \r) is // literal content inside a quoted field. if (start + len >= self.field_buf.len) { self.pending_error = error.FieldTooLong; self.state = .failed; return error.FieldTooLong; } _ = try self.takeByte(); self.field_buf[start + len] = b; len += 1; },
.after_quote => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { self.state = .eof; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; }, else => |e| return e, }; if (b == self.dialect.delimiter) { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = false; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = false }; } if (b == '\n') { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } if (b == '\r') { _ = try self.takeByte(); self.consumeOptionalLf(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } self.pending_error = error.UnexpectedByteAfterQuote; self.state = .failed; return error.UnexpectedByteAfterQuote; }, } } }
/// On first call, peek at up to three bytes and discard them iff /// they form a UTF-8 BOM and BOM stripping is enabled. The BOM is /// not counted as a "consumed payload byte". fn maybeStripBom(self: *FieldIterator) Error!void { if (!self.dialect.strip_bom) return; const bom = "\xEF\xBB\xBF"; const peeked = self.reader.peek(3) catch |err| switch (err) { error.EndOfStream => return, // fewer than 3 bytes available; no BOM else => |e| return e, }; if (std.mem.eql(u8, peeked, bom)) { try self.reader.discardAll(3); } }
/// Consume one byte from the reader and mark the iterator as /// having consumed payload data. All payload byte consumption /// should go through this helper so `bytes_consumed` stays in /// sync. fn takeByte(self: *FieldIterator) Error!u8 { const b = try self.reader.takeByte(); self.bytes_consumed = true; return b; }
/// If the next byte is `\n`, consume it (the `\r` half of a CRLF /// pair has already been consumed). Otherwise do nothing. fn consumeOptionalLf(self: *FieldIterator) void { const b = self.reader.peekByte() catch return; if (b == '\n') { // takeByte cannot fail here: peekByte already succeeded // for this byte, so the buffer holds it. _ = self.takeByte() catch unreachable; } }};
// ===== Tests =================================================================
const testing = std.testing;
const Item = struct { data: []u8, eor: bool };
/// Drain `input` through a `FieldIterator` and return the list of/// `(field_bytes, end_of_row)` pairs, or the first error encountered.fn parseAll( allocator: std.mem.Allocator, input: []const u8, dialect: Dialect,) !std.ArrayListUnmanaged(Item) { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, dialect); var out: std.ArrayListUnmanaged(Item) = .empty; errdefer { for (out.items) |item| allocator.free(item.data); out.deinit(allocator); } while (try it.next()) |r| { const copy = try allocator.dupe(u8, r.data); try out.append(allocator, .{ .data = copy, .eor = r.end_of_row }); } return out;}
fn freeAll(allocator: std.mem.Allocator, list: *std.ArrayListUnmanaged(Item)) void { for (list.items) |item| allocator.free(item.data); list.deinit(allocator);}
/// Convenience: assert that `input` yields exactly `expected` fields,/// each tagged with the expected end-of-row flag.fn expectFields( input: []const u8, dialect: Dialect, expected: []const struct { []const u8, bool },) !void { var got = try parseAll(testing.allocator, input, dialect); defer freeAll(testing.allocator, &got); try testing.expectEqual(expected.len, got.items.len); for (expected, got.items) |exp, actual| { try testing.expectEqualStrings(exp[0], actual.data); try testing.expectEqual(exp[1], actual.eor); }}
fn expectParseError( input: []const u8, dialect: Dialect, expected_err: anyerror,) !void { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, dialect); // Drive until we hit the error. while (true) { const r = it.next() catch |e| { try testing.expectEqual(expected_err, e); return; }; if (r == null) return error.TestExpectedErrorButGotEof; }}
// --- Correctness #7,#8,#9: empty / newline / trailing-newline ---------------
test "FieldIterator: empty input yields zero rows (correctness #7)" { try expectFields("", .{}, &.{});}
test "FieldIterator: lone LF yields one empty field (correctness #8)" { try expectFields("\n", .{}, &.{.{ "", true }});}
test "FieldIterator: lone CR yields one empty field (correctness #8)" { try expectFields("\r", .{}, &.{.{ "", true }});}
test "FieldIterator: lone CRLF yields one empty field (correctness #8)" { try expectFields("\r\n", .{}, &.{.{ "", true }});}
test "FieldIterator: trailing newline does not produce phantom row (correctness #9)" { try expectFields("a\n", .{}, &.{.{ "a", true }}); try expectFields("a,b\n", .{}, &.{ .{ "a", false }, .{ "b", true } });}
test "FieldIterator: no trailing newline yields last field with eor=true" { try expectFields("a", .{}, &.{.{ "a", true }}); try expectFields("a,b", .{}, &.{ .{ "a", false }, .{ "b", true } });}
// --- Correctness #6: empty vs missing fields --------------------------------
test "FieldIterator: trailing delimiter produces empty trailing field (correctness #6)" { try expectFields("a,", .{}, &.{ .{ "a", false }, .{ "", true } });}
test "FieldIterator: line of N delimiters produces N+1 empty fields (correctness #6)" { try expectFields(",,", .{}, &.{ .{ "", false }, .{ "", false }, .{ "", true } });}
// --- Correctness #1,#2,#3: multiple rows & line endings ---------------------
test "FieldIterator: two-row CSV with LF terminators (correctness #1,#2)" { try expectFields("a,b\nc,d\n", .{}, &.{ .{ "a", false }, .{ "b", true }, .{ "c", false }, .{ "d", true }, });}
test "FieldIterator: two-row CSV with CRLF terminators (correctness #2)" { try expectFields("a,b\r\nc,d\r\n", .{}, &.{ .{ "a", false }, .{ "b", true }, .{ "c", false }, .{ "d", true }, });}
test "FieldIterator: mixed line endings" { try expectFields("a\nb\r\nc\rd", .{}, &.{ .{ "a", true }, .{ "b", true }, .{ "c", true }, .{ "d", true }, });}
test "FieldIterator: bare CR is a row terminator (correctness #3)" { try expectFields("a\rb", .{}, &.{ .{ "a", true }, .{ "b", true } });}
// --- Correctness #3,#4,#5: quoted fields ------------------------------------
test "FieldIterator: quoted field with embedded delimiter (correctness #3,#4)" { try expectFields("\"a,b\",c", .{}, &.{ .{ "a,b", false }, .{ "c", true } });}
test "FieldIterator: quoted field with embedded LF (correctness #3,#4)" { try expectFields("\"a\nb\"", .{}, &.{.{ "a\nb", true }});}
test "FieldIterator: quoted field with embedded CR and CRLF (correctness #3,#4)" { try expectFields("\"a\rb\",\"x\r\ny\"", .{}, &.{ .{ "a\rb", false }, .{ "x\r\ny", true }, });}
test "FieldIterator: doubled quote inside quoted field is literal quote (correctness #5)" { try expectFields("\"a\"\"b\"", .{}, &.{.{ "a\"b", true }});}
test "FieldIterator: empty quoted field" { try expectFields("\"\"", .{}, &.{.{ "", true }}); try expectFields("\"\",\"\"", .{}, &.{ .{ "", false }, .{ "", true } });}
// --- Correctness #11,#12,#13: malformed input -------------------------------
test "FieldIterator: quote in middle of unquoted field => UnexpectedQuote (correctness #11)" { try expectParseError("a\"b", .{}, error.UnexpectedQuote);}
test "FieldIterator: trailing byte after closing quote => UnexpectedByteAfterQuote (correctness #12)" { try expectParseError("\"a\"b", .{}, error.UnexpectedByteAfterQuote);}
test "FieldIterator: unterminated quoted field => UnterminatedQuotedField (correctness #13)" { try expectParseError("\"abc", .{}, error.UnterminatedQuotedField);}
// --- Correctness #14: sticky error ------------------------------------------
test "FieldIterator: malformed-input error is sticky (correctness #14)" { var reader = Io.Reader.fixed("a\"b"); var field_buf: [16]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); try testing.expectError(error.UnexpectedQuote, it.next()); try testing.expectError(error.UnexpectedQuote, it.next()); try testing.expectError(error.UnexpectedQuote, it.next());}
// --- Story 6 / Dialect ------------------------------------------------------
test "FieldIterator: tab dialect" { try expectFields("a\tb\tc\n", .{ .delimiter = '\t' }, &.{ .{ "a", false }, .{ "b", false }, .{ "c", true }, });}
test "FieldIterator: semicolon dialect" { try expectFields("a;b;c", .{ .delimiter = ';' }, &.{ .{ "a", false }, .{ "b", false }, .{ "c", true }, });}
test "FieldIterator: alternative quote character (apostrophe)" { try expectFields("'a,b','c'", .{ .quote_char = '\'' }, &.{ .{ "a,b", false }, .{ "c", true }, });}
test "FieldIterator: invalid dialect rejected by init" { var reader = Io.Reader.fixed(""); var field_buf: [16]u8 = undefined; try testing.expectError( error.InvalidDialect, FieldIterator.init(&reader, &field_buf, .{ .delimiter = '"', .quote_char = '"' }), );}
// --- FieldTooLong -----------------------------------------------------------
test "FieldIterator: field exactly fills buffer is OK" { var reader = Io.Reader.fixed("abcd"); var field_buf: [4]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); const r = (try it.next()).?; try testing.expectEqualStrings("abcd", r.data); try testing.expectEqual(true, r.end_of_row);}
test "FieldIterator: field one byte too large => FieldTooLong" { var reader = Io.Reader.fixed("abcde"); var field_buf: [4]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); try testing.expectError(error.FieldTooLong, it.next());}
test "FieldIterator: FieldTooLong inside quoted field" { var reader = Io.Reader.fixed("\"abcde\""); var field_buf: [4]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); try testing.expectError(error.FieldTooLong, it.next());}
// --- BOM stripping (correctness #19) ----------------------------------------
test "FieldIterator: BOM stripping on (default) consumes leading BOM" { try expectFields("\xEF\xBB\xBFa,b", .{}, &.{ .{ "a", false }, .{ "b", true } });}
test "FieldIterator: BOM stripping off preserves BOM as field content" { try expectFields( "\xEF\xBB\xBFa,b", .{ .strip_bom = false }, &.{ .{ "\xEF\xBB\xBFa", false }, .{ "b", true } }, );}
test "FieldIterator: BOM-only input yields zero rows" { try expectFields("\xEF\xBB\xBF", .{}, &.{});}
test "FieldIterator: shorter-than-BOM input is not mistaken for a BOM" { try expectFields("\xEF\xBB", .{}, &.{.{ "\xEF\xBB", true }});}
// --- Correctness #17: UTF-8 transparency ------------------------------------
test "FieldIterator: UTF-8 multibyte content passes through unmodified (correctness #17)" { // "caf\u00e9,\u4e2d\u6587\n" — 'café' and '中文' both contain multibyte UTF-8. try expectFields( "caf\xC3\xA9,\xE4\xB8\xAD\xE6\x96\x87\n", .{}, &.{ .{ "caf\xC3\xA9", false }, .{ "\xE4\xB8\xAD\xE6\x96\x87", true } }, );}
test "FieldIterator: UTF-8 content inside quoted field passes through (correctness #17)" { try expectFields( "\"caf\xC3\xA9, \xE4\xB8\xAD\xE6\x96\x87\"", .{}, &.{.{ "caf\xC3\xA9, \xE4\xB8\xAD\xE6\x96\x87", true }}, );}