//! Low-level field-by-field CSV iterator — the foundational primitive //! of this library. //! //! `FieldIterator` is the only module that reads bytes directly from //! the underlying `std.Io.Reader`. It implements a state machine that //! tokenises the input into fields and emits one field per `next()` //! call, along with a flag indicating whether the field is the last in //! its row. //! //! All higher-level parsers in this library are built on top of //! `FieldIterator`, inheriting its parsing-correctness guarantees. //! //! The iterator performs no heap allocation. Field bytes are written //! into a caller-provided `field_buf`; the returned slice is borrowed //! from that buffer and is invalidated by the next call to `next()`. const std = @import("std"); const Io = std.Io; const Dialect = @import("dialect.zig").Dialect; const Error = @import("error.zig").Error; /// Internal parser state. const State = enum { /// Initial state; possibly strip a UTF-8 BOM on first call. start, /// At the beginning of a field (just after a delimiter or row end, /// or at the very start of input). field_start, /// Accumulating bytes of an unquoted field. in_unquoted, /// Accumulating bytes of a quoted field; structural bytes are /// preserved as content. in_quoted, /// Just consumed a closing quote of a quoted field; expecting a /// delimiter, a line ending, or EOF. after_quote, /// End of input reached; further `next()` calls return null. eof, /// A malformed-input error was detected; further `next()` calls /// return the same sticky error. failed, }; /// Low-level field-by-field CSV iterator. The foundational primitive /// of this library; every higher tier wraps a `FieldIterator`. Reads /// bytes from a `std.Io.Reader`, accumulates field bytes into a /// caller-provided `field_buf`, and emits one field per `next()`. pub const FieldIterator = struct { reader: *Io.Reader, field_buf: []u8, dialect: Dialect, // --- internal state; opaque to callers -------------------------- state: State, /// Position within `field_buf` where the next field's bytes will /// begin. Advances past each emitted field. Reset to 0 via /// `resetBuffer()`. This lets a row iterator hold simultaneous /// slices into `field_buf` for every field of the current row. write_offset: usize, /// True if the most recent emitted field ended a row. Used to /// distinguish "input ended mid-field" (yield trailing empty /// field) from "input ended at end of row" (suppress phantom /// trailing empty row). Initialised true so that an empty input /// yields zero rows. last_was_row_end: bool, /// True if any payload byte has been consumed (BOM bytes do not /// count). Used to make BOM-only inputs equivalent to empty /// inputs. bytes_consumed: bool, /// Sticky error after malformed input. Once set, every subsequent /// `next()` returns this error. pending_error: ?Error, /// Result of a successful `next()` call. pub const Result = struct { /// Slice borrowed from the iterator's `field_buf`. Valid /// until the next call to `resetBuffer()` (or until /// `field_buf` fills up and `FieldTooLong` is returned). /// Row iterators call `resetBuffer()` at the start of each /// `next()`, so for row-iterator clients the field slices /// are valid until the next row-iterator `next()` call. data: []const u8, /// True if this field is the last field of its row. end_of_row: bool, }; /// Construct an iterator that reads from `reader`, accumulating /// field bytes into `field_buf`. `field_buf` must be large enough /// to hold every concurrently-live field (between calls to /// `resetBuffer()` — in practice, an entire row's worth of field /// bytes when used via a row iterator). Otherwise `next()` will /// return `error.FieldTooLong`. /// /// Returns `error.InvalidDialect` if the dialect is malformed. pub fn init(reader: *Io.Reader, field_buf: []u8, dialect: Dialect) Error!FieldIterator { try dialect.validate(); return .{ .reader = reader, .field_buf = field_buf, .dialect = dialect, .state = .start, .write_offset = 0, .last_was_row_end = true, .bytes_consumed = false, .pending_error = null, }; } /// Reset the write position within `field_buf` to 0, invalidating /// all previously-returned `Result.data` slices. Row iterators /// call this at the start of each `next()` so all fields of a /// row can coexist in `field_buf` simultaneously. pub fn resetBuffer(self: *FieldIterator) void { self.write_offset = 0; } /// Yields the next field, or null at end-of-input. /// /// The `Result.data` slice is borrowed from the iterator's /// `field_buf` and is **invalidated by the next call to `next()`**. /// /// On malformed input, returns one of `UnexpectedQuote`, /// `UnexpectedByteAfterQuote`, or `UnterminatedQuotedField`. /// After a malformed-input error, the iterator becomes sticky: /// every subsequent call returns the same error. /// /// If a single field's decoded content would exceed `field_buf`, /// returns `error.FieldTooLong`. pub fn next(self: *FieldIterator) Error!?Result { if (self.pending_error) |e| return e; if (self.state == .start) { try self.maybeStripBom(); self.state = .field_start; } if (self.state == .eof) return null; // Accumulator for the current field's decoded bytes. Index // into `self.field_buf`. // Bytes are written starting at `self.write_offset` so that // previously-emitted fields in the same row remain valid. // `start` snapshots the offset at the beginning of this field; // `len` is the number of decoded bytes written so far. const start = self.write_offset; var len: usize = 0; while (true) { switch (self.state) { .start, .eof, .failed => unreachable, // handled above .field_start => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { // End of input at field boundary. self.state = .eof; if (self.bytes_consumed and !self.last_was_row_end) { // Input ended mid-row with no terminator; yield // a final empty field as the row's last field. self.last_was_row_end = true; return Result{ .data = self.field_buf[start..start], .end_of_row = true }; } return null; }, else => |e| return e, }; if (b == self.dialect.quote_char) { _ = try self.takeByte(); self.state = .in_quoted; continue; } if (b == self.dialect.delimiter) { _ = try self.takeByte(); self.last_was_row_end = false; return Result{ .data = self.field_buf[start..start], .end_of_row = false }; } if (b == '\n') { _ = try self.takeByte(); self.last_was_row_end = true; return Result{ .data = self.field_buf[start..start], .end_of_row = true }; } if (b == '\r') { _ = try self.takeByte(); // Consume an optional following \n for CRLF. self.consumeOptionalLf(); self.last_was_row_end = true; return Result{ .data = self.field_buf[start..start], .end_of_row = true }; } // Ordinary byte: enter unquoted accumulation. self.state = .in_unquoted; continue; }, .in_unquoted => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { self.state = .eof; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; }, else => |e| return e, }; if (b == self.dialect.quote_char) { self.pending_error = error.UnexpectedQuote; self.state = .failed; return error.UnexpectedQuote; } if (b == self.dialect.delimiter) { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = false; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = false }; } if (b == '\n') { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } if (b == '\r') { _ = try self.takeByte(); self.consumeOptionalLf(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } // Append byte to field. if (start + len >= self.field_buf.len) { self.pending_error = error.FieldTooLong; self.state = .failed; return error.FieldTooLong; } _ = try self.takeByte(); self.field_buf[start + len] = b; len += 1; }, .in_quoted => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { self.pending_error = error.UnterminatedQuotedField; self.state = .failed; return error.UnterminatedQuotedField; }, else => |e| return e, }; if (b == self.dialect.quote_char) { _ = try self.takeByte(); // Lookahead: doubled quote = literal quote in content. const next_byte = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { // EOF immediately after closing quote. self.state = .after_quote; continue; }, else => |e| return e, }; if (next_byte == self.dialect.quote_char) { _ = try self.takeByte(); if (start + len >= self.field_buf.len) { self.pending_error = error.FieldTooLong; self.state = .failed; return error.FieldTooLong; } self.field_buf[start + len] = self.dialect.quote_char; len += 1; continue; } self.state = .after_quote; continue; } // Any other byte (including delimiter, \n, \r) is // literal content inside a quoted field. if (start + len >= self.field_buf.len) { self.pending_error = error.FieldTooLong; self.state = .failed; return error.FieldTooLong; } _ = try self.takeByte(); self.field_buf[start + len] = b; len += 1; }, .after_quote => { const b = self.reader.peekByte() catch |err| switch (err) { error.EndOfStream => { self.state = .eof; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; }, else => |e| return e, }; if (b == self.dialect.delimiter) { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = false; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = false }; } if (b == '\n') { _ = try self.takeByte(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } if (b == '\r') { _ = try self.takeByte(); self.consumeOptionalLf(); self.state = .field_start; self.last_was_row_end = true; self.write_offset = start + len; return Result{ .data = self.field_buf[start..start + len], .end_of_row = true }; } self.pending_error = error.UnexpectedByteAfterQuote; self.state = .failed; return error.UnexpectedByteAfterQuote; }, } } } /// On first call, peek at up to three bytes and discard them iff /// they form a UTF-8 BOM and BOM stripping is enabled. The BOM is /// not counted as a "consumed payload byte". fn maybeStripBom(self: *FieldIterator) Error!void { if (!self.dialect.strip_bom) return; const bom = "\xEF\xBB\xBF"; const peeked = self.reader.peek(3) catch |err| switch (err) { error.EndOfStream => return, // fewer than 3 bytes available; no BOM else => |e| return e, }; if (std.mem.eql(u8, peeked, bom)) { try self.reader.discardAll(3); } } /// Consume one byte from the reader and mark the iterator as /// having consumed payload data. All payload byte consumption /// should go through this helper so `bytes_consumed` stays in /// sync. fn takeByte(self: *FieldIterator) Error!u8 { const b = try self.reader.takeByte(); self.bytes_consumed = true; return b; } /// If the next byte is `\n`, consume it (the `\r` half of a CRLF /// pair has already been consumed). Otherwise do nothing. fn consumeOptionalLf(self: *FieldIterator) void { const b = self.reader.peekByte() catch return; if (b == '\n') { // takeByte cannot fail here: peekByte already succeeded // for this byte, so the buffer holds it. _ = self.takeByte() catch unreachable; } } }; // ===== Tests ================================================================= const testing = std.testing; const Item = struct { data: []u8, eor: bool }; /// Drain `input` through a `FieldIterator` and return the list of /// `(field_bytes, end_of_row)` pairs, or the first error encountered. fn parseAll( allocator: std.mem.Allocator, input: []const u8, dialect: Dialect, ) !std.ArrayListUnmanaged(Item) { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, dialect); var out: std.ArrayListUnmanaged(Item) = .empty; errdefer { for (out.items) |item| allocator.free(item.data); out.deinit(allocator); } while (try it.next()) |r| { const copy = try allocator.dupe(u8, r.data); try out.append(allocator, .{ .data = copy, .eor = r.end_of_row }); } return out; } fn freeAll(allocator: std.mem.Allocator, list: *std.ArrayListUnmanaged(Item)) void { for (list.items) |item| allocator.free(item.data); list.deinit(allocator); } /// Convenience: assert that `input` yields exactly `expected` fields, /// each tagged with the expected end-of-row flag. fn expectFields( input: []const u8, dialect: Dialect, expected: []const struct { []const u8, bool }, ) !void { var got = try parseAll(testing.allocator, input, dialect); defer freeAll(testing.allocator, &got); try testing.expectEqual(expected.len, got.items.len); for (expected, got.items) |exp, actual| { try testing.expectEqualStrings(exp[0], actual.data); try testing.expectEqual(exp[1], actual.eor); } } fn expectParseError( input: []const u8, dialect: Dialect, expected_err: anyerror, ) !void { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, dialect); // Drive until we hit the error. while (true) { const r = it.next() catch |e| { try testing.expectEqual(expected_err, e); return; }; if (r == null) return error.TestExpectedErrorButGotEof; } } // --- Correctness #7,#8,#9: empty / newline / trailing-newline --------------- test "FieldIterator: empty input yields zero rows (correctness #7)" { try expectFields("", .{}, &.{}); } test "FieldIterator: lone LF yields one empty field (correctness #8)" { try expectFields("\n", .{}, &.{.{ "", true }}); } test "FieldIterator: lone CR yields one empty field (correctness #8)" { try expectFields("\r", .{}, &.{.{ "", true }}); } test "FieldIterator: lone CRLF yields one empty field (correctness #8)" { try expectFields("\r\n", .{}, &.{.{ "", true }}); } test "FieldIterator: trailing newline does not produce phantom row (correctness #9)" { try expectFields("a\n", .{}, &.{.{ "a", true }}); try expectFields("a,b\n", .{}, &.{ .{ "a", false }, .{ "b", true } }); } test "FieldIterator: no trailing newline yields last field with eor=true" { try expectFields("a", .{}, &.{.{ "a", true }}); try expectFields("a,b", .{}, &.{ .{ "a", false }, .{ "b", true } }); } // --- Correctness #6: empty vs missing fields -------------------------------- test "FieldIterator: trailing delimiter produces empty trailing field (correctness #6)" { try expectFields("a,", .{}, &.{ .{ "a", false }, .{ "", true } }); } test "FieldIterator: line of N delimiters produces N+1 empty fields (correctness #6)" { try expectFields(",,", .{}, &.{ .{ "", false }, .{ "", false }, .{ "", true } }); } // --- Correctness #1,#2,#3: multiple rows & line endings --------------------- test "FieldIterator: two-row CSV with LF terminators (correctness #1,#2)" { try expectFields("a,b\nc,d\n", .{}, &.{ .{ "a", false }, .{ "b", true }, .{ "c", false }, .{ "d", true }, }); } test "FieldIterator: two-row CSV with CRLF terminators (correctness #2)" { try expectFields("a,b\r\nc,d\r\n", .{}, &.{ .{ "a", false }, .{ "b", true }, .{ "c", false }, .{ "d", true }, }); } test "FieldIterator: mixed line endings" { try expectFields("a\nb\r\nc\rd", .{}, &.{ .{ "a", true }, .{ "b", true }, .{ "c", true }, .{ "d", true }, }); } test "FieldIterator: bare CR is a row terminator (correctness #3)" { try expectFields("a\rb", .{}, &.{ .{ "a", true }, .{ "b", true } }); } // --- Correctness #3,#4,#5: quoted fields ------------------------------------ test "FieldIterator: quoted field with embedded delimiter (correctness #3,#4)" { try expectFields("\"a,b\",c", .{}, &.{ .{ "a,b", false }, .{ "c", true } }); } test "FieldIterator: quoted field with embedded LF (correctness #3,#4)" { try expectFields("\"a\nb\"", .{}, &.{.{ "a\nb", true }}); } test "FieldIterator: quoted field with embedded CR and CRLF (correctness #3,#4)" { try expectFields("\"a\rb\",\"x\r\ny\"", .{}, &.{ .{ "a\rb", false }, .{ "x\r\ny", true }, }); } test "FieldIterator: doubled quote inside quoted field is literal quote (correctness #5)" { try expectFields("\"a\"\"b\"", .{}, &.{.{ "a\"b", true }}); } test "FieldIterator: empty quoted field" { try expectFields("\"\"", .{}, &.{.{ "", true }}); try expectFields("\"\",\"\"", .{}, &.{ .{ "", false }, .{ "", true } }); } // --- Correctness #11,#12,#13: malformed input ------------------------------- test "FieldIterator: quote in middle of unquoted field => UnexpectedQuote (correctness #11)" { try expectParseError("a\"b", .{}, error.UnexpectedQuote); } test "FieldIterator: trailing byte after closing quote => UnexpectedByteAfterQuote (correctness #12)" { try expectParseError("\"a\"b", .{}, error.UnexpectedByteAfterQuote); } test "FieldIterator: unterminated quoted field => UnterminatedQuotedField (correctness #13)" { try expectParseError("\"abc", .{}, error.UnterminatedQuotedField); } // --- Correctness #14: sticky error ------------------------------------------ test "FieldIterator: malformed-input error is sticky (correctness #14)" { var reader = Io.Reader.fixed("a\"b"); var field_buf: [16]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); try testing.expectError(error.UnexpectedQuote, it.next()); try testing.expectError(error.UnexpectedQuote, it.next()); try testing.expectError(error.UnexpectedQuote, it.next()); } // --- Story 6 / Dialect ------------------------------------------------------ test "FieldIterator: tab dialect" { try expectFields("a\tb\tc\n", .{ .delimiter = '\t' }, &.{ .{ "a", false }, .{ "b", false }, .{ "c", true }, }); } test "FieldIterator: semicolon dialect" { try expectFields("a;b;c", .{ .delimiter = ';' }, &.{ .{ "a", false }, .{ "b", false }, .{ "c", true }, }); } test "FieldIterator: alternative quote character (apostrophe)" { try expectFields("'a,b','c'", .{ .quote_char = '\'' }, &.{ .{ "a,b", false }, .{ "c", true }, }); } test "FieldIterator: invalid dialect rejected by init" { var reader = Io.Reader.fixed(""); var field_buf: [16]u8 = undefined; try testing.expectError( error.InvalidDialect, FieldIterator.init(&reader, &field_buf, .{ .delimiter = '"', .quote_char = '"' }), ); } // --- FieldTooLong ----------------------------------------------------------- test "FieldIterator: field exactly fills buffer is OK" { var reader = Io.Reader.fixed("abcd"); var field_buf: [4]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); const r = (try it.next()).?; try testing.expectEqualStrings("abcd", r.data); try testing.expectEqual(true, r.end_of_row); } test "FieldIterator: field one byte too large => FieldTooLong" { var reader = Io.Reader.fixed("abcde"); var field_buf: [4]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); try testing.expectError(error.FieldTooLong, it.next()); } test "FieldIterator: FieldTooLong inside quoted field" { var reader = Io.Reader.fixed("\"abcde\""); var field_buf: [4]u8 = undefined; var it = try FieldIterator.init(&reader, &field_buf, .{}); try testing.expectError(error.FieldTooLong, it.next()); } // --- BOM stripping (correctness #19) ---------------------------------------- test "FieldIterator: BOM stripping on (default) consumes leading BOM" { try expectFields("\xEF\xBB\xBFa,b", .{}, &.{ .{ "a", false }, .{ "b", true } }); } test "FieldIterator: BOM stripping off preserves BOM as field content" { try expectFields( "\xEF\xBB\xBFa,b", .{ .strip_bom = false }, &.{ .{ "\xEF\xBB\xBFa", false }, .{ "b", true } }, ); } test "FieldIterator: BOM-only input yields zero rows" { try expectFields("\xEF\xBB\xBF", .{}, &.{}); } test "FieldIterator: shorter-than-BOM input is not mistaken for a BOM" { try expectFields("\xEF\xBB", .{}, &.{.{ "\xEF\xBB", true }}); } // --- Correctness #17: UTF-8 transparency ------------------------------------ test "FieldIterator: UTF-8 multibyte content passes through unmodified (correctness #17)" { // "caf\u00e9,\u4e2d\u6587\n" — 'café' and '中文' both contain multibyte UTF-8. try expectFields( "caf\xC3\xA9,\xE4\xB8\xAD\xE6\x96\x87\n", .{}, &.{ .{ "caf\xC3\xA9", false }, .{ "\xE4\xB8\xAD\xE6\x96\x87", true } }, ); } test "FieldIterator: UTF-8 content inside quoted field passes through (correctness #17)" { try expectFields( "\"caf\xC3\xA9, \xE4\xB8\xAD\xE6\x96\x87\"", .{}, &.{.{ "caf\xC3\xA9, \xE4\xB8\xAD\xE6\x96\x87", true }}, ); }