Something went wrong. Try again.
A CSV parsing library in Zig.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367//! Row-by-row CSV iterators.//!//! This module groups the fields emitted by `FieldIterator` into rows//! and yields them as `Row` values. Three storage strategies are//! offered, mirroring the standard library's split between `ArrayList`//! (allocating), `BoundedArray` (comptime-fixed inline), and//! `FixedBufferAllocator` (caller buffer)://!//! - `RowIterator` — allocating, grows on demand. Use when you have an//! allocator and don't want to think about a maximum column count.//! - `BoundedRowIterator(N)` — comptime-fixed capacity, stack-stored.//! Use when no allocator is available and the column count fits a//! known maximum.//! - `FixedBufferRowIterator` — uses a caller-provided//! `[][]const u8` buffer. Use when neither of the above fits.//!//! All three yield the same `Row` value; consumer code can be written//! generically against `Row`.
const std = @import("std");const Io = std.Io;const Allocator = std.mem.Allocator;
const Dialect = @import("dialect.zig").Dialect;const Error = @import("error.zig").Error;const FieldIterator = @import("field_iterator.zig").FieldIterator;
/// Default maximum fields per row for `BoundedRowIterator` callers/// that don't have a specific number in mind. Large enough for the/// vast majority of real-world CSVs (typical max in the wild: ~30/// columns).pub const default_max_fields: usize = 64;
/// A single row produced by any of the row-iterator variants.////// `fields` is borrowed from the iterator's internal row-slot storage,/// and each element is borrowed from the underlying `FieldIterator`'s/// `field_buf`. **Both layers are invalidated by the next call on the/// iterator** — copy any data you need to retain across calls.pub const Row = struct { fields: []const []const u8,
/// Number of fields in this row. pub fn count(self: Row) usize { return self.fields.len; }
/// Field at column index `i`, or null if `i` is out of range. pub fn at(self: Row, i: usize) ?[]const u8 { return if (i < self.fields.len) self.fields[i] else null; }};
// =============================================================================// RowIterator — allocating, grows on demand// =============================================================================
/// Row iterator backed by an allocating, growable list of field/// slots. There is no `RowTooWide` failure mode for this variant —/// width is bounded only by available memory.pub const RowIterator = struct { field_iter: FieldIterator, row_buf: std.ArrayListUnmanaged([]const u8), allocator: Allocator,
/// Construct an iterator. `field_buf` is used by the underlying /// `FieldIterator` for per-field accumulation. `allocator` is used /// only for the growable row-slot list. pub fn init( reader: *Io.Reader, field_buf: []u8, allocator: Allocator, dialect: Dialect, ) Error!RowIterator { return .{ .field_iter = try FieldIterator.init(reader, field_buf, dialect), .row_buf = .empty, .allocator = allocator, }; }
/// Release the row-slot list. Does not free the `field_buf` (which /// the caller owns) or the reader (which the caller also owns). pub fn deinit(self: *RowIterator) void { self.row_buf.deinit(self.allocator); }
/// Yields the next row, or null at end of input. The returned /// `Row.fields` slice and every field slice within it are /// invalidated by the next call to `next()`. pub fn next(self: *RowIterator) Error!?Row { self.row_buf.clearRetainingCapacity(); self.field_iter.resetBuffer(); while (try self.field_iter.next()) |result| { try self.row_buf.append(self.allocator, result.data); if (result.end_of_row) { return Row{ .fields = self.row_buf.items }; } } return null; }};
// =============================================================================// BoundedRowIterator(N) — comptime-fixed inline buffer// =============================================================================
/// Row iterator with a comptime-fixed maximum number of fields per/// row. All storage is internal; no allocator is required.////// Returns `error.RowTooWide` if any row exceeds `max_fields`.pub fn BoundedRowIterator(comptime max_fields: usize) type { return struct { const Self = @This();
field_iter: FieldIterator, row_buf: [max_fields][]const u8 = undefined, row_len: usize = 0,
/// Construct an iterator. `field_buf` is used by the underlying /// `FieldIterator` for per-field accumulation. pub fn init( reader: *Io.Reader, field_buf: []u8, dialect: Dialect, ) Error!Self { return .{ .field_iter = try FieldIterator.init(reader, field_buf, dialect), }; }
/// Yields the next row, or null at end of input. The returned /// `Row.fields` slice and every field slice within it are /// invalidated by the next call to `next()`. pub fn next(self: *Self) Error!?Row { self.row_len = 0; self.field_iter.resetBuffer(); while (try self.field_iter.next()) |result| { if (self.row_len >= max_fields) return error.RowTooWide; self.row_buf[self.row_len] = result.data; self.row_len += 1; if (result.end_of_row) { return Row{ .fields = self.row_buf[0..self.row_len] }; } } return null; } };}
// =============================================================================// FixedBufferRowIterator — caller-provided row buffer// =============================================================================
/// Row iterator that uses a caller-provided `[][]const u8` row/// buffer. Use when neither `RowIterator` (no allocator) nor/// `BoundedRowIterator` (max not known at comptime) fits.////// Returns `error.RowTooWide` if any row exceeds `row_buf.len`.pub const FixedBufferRowIterator = struct { field_iter: FieldIterator, row_buf: [][]const u8, row_len: usize = 0,
/// Construct an iterator. `field_buf` is for per-field /// accumulation; `row_buf` bounds the number of fields per row. pub fn init( reader: *Io.Reader, field_buf: []u8, row_buf: [][]const u8, dialect: Dialect, ) Error!FixedBufferRowIterator { return .{ .field_iter = try FieldIterator.init(reader, field_buf, dialect), .row_buf = row_buf, }; }
/// Yields the next row, or null at end of input. The returned /// `Row.fields` slice and every field slice within it are /// invalidated by the next call to `next()`. pub fn next(self: *FixedBufferRowIterator) Error!?Row { self.row_len = 0; self.field_iter.resetBuffer(); while (try self.field_iter.next()) |result| { if (self.row_len >= self.row_buf.len) return error.RowTooWide; self.row_buf[self.row_len] = result.data; self.row_len += 1; if (result.end_of_row) { return Row{ .fields = self.row_buf[0..self.row_len] }; } } return null; }};
// =============================================================================// Tests// =============================================================================
const testing = std.testing;
/// A row of expected fields used by the shared-corpus tests.const ExpectedRow = []const []const u8;
/// Collect all rows from `iter` (any of the three variants) into a/// newly allocated `[][][]u8`. Each cell is `dupe`d so the result/// outlives the iterator's borrowed buffers.fn collect(allocator: Allocator, iter: anytype) ![][][]u8 { var rows: std.ArrayListUnmanaged([][]u8) = .empty; errdefer { for (rows.items) |row| { for (row) |cell| allocator.free(cell); allocator.free(row); } rows.deinit(allocator); } while (try iter.next()) |row| { const cells = try allocator.alloc([]u8, row.count()); errdefer allocator.free(cells); var i: usize = 0; errdefer for (cells[0..i]) |c| allocator.free(c); while (i < cells.len) : (i += 1) { cells[i] = try allocator.dupe(u8, row.fields[i]); } try rows.append(allocator, cells); } return rows.toOwnedSlice(allocator);}
fn freeRows(allocator: Allocator, rows: [][][]u8) void { for (rows) |row| { for (row) |cell| allocator.free(cell); allocator.free(row); } allocator.free(rows);}
fn expectRowsEqual(expected: []const ExpectedRow, actual: [][][]u8) !void { try testing.expectEqual(expected.len, actual.len); for (expected, actual) |exp_row, act_row| { try testing.expectEqual(exp_row.len, act_row.len); for (exp_row, act_row) |exp_cell, act_cell| { try testing.expectEqualStrings(exp_cell, act_cell); } }}
/// Run the shared corpus through all three iterator variants and/// confirm they all produce `expected`.fn expectAllVariants(input: []const u8, expected: []const ExpectedRow) !void { // --- RowIterator (allocating) ---- { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var it = try RowIterator.init(&reader, &field_buf, testing.allocator, .{}); defer it.deinit(); const got = try collect(testing.allocator, &it); defer freeRows(testing.allocator, got); try expectRowsEqual(expected, got); } // --- BoundedRowIterator(default_max_fields) ---- { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var it = try BoundedRowIterator(default_max_fields).init(&reader, &field_buf, .{}); const got = try collect(testing.allocator, &it); defer freeRows(testing.allocator, got); try expectRowsEqual(expected, got); } // --- FixedBufferRowIterator ---- { var reader = Io.Reader.fixed(input); var field_buf: [4096]u8 = undefined; var row_buf: [64][]const u8 = undefined; var it = try FixedBufferRowIterator.init(&reader, &field_buf, &row_buf, .{}); const got = try collect(testing.allocator, &it); defer freeRows(testing.allocator, got); try expectRowsEqual(expected, got); }}
test "row iterators: empty input yields zero rows" { try expectAllVariants("", &.{});}
test "row iterators: single row" { try expectAllVariants("a,b,c\n", &.{&.{ "a", "b", "c" }});}
test "row iterators: two rows, no trailing newline" { try expectAllVariants("a,b\n1,2", &.{ &.{ "a", "b" }, &.{ "1", "2" } });}
test "row iterators: ragged rows are accepted" { try expectAllVariants("a,b,c\n1\nx,y", &.{ &.{ "a", "b", "c" }, &.{"1"}, &.{ "x", "y" }, });}
test "row iterators: quoted fields with embedded delimiter, newline, doubled quote" { try expectAllVariants( "\"a,b\",\"c\nd\",\"e\"\"f\"\n", &.{&.{ "a,b", "c\nd", "e\"f" }}, );}
test "row iterators: empty fields preserved" { try expectAllVariants(",,\n", &.{&.{ "", "", "" }});}
// --- RowTooWide ---
test "BoundedRowIterator(4) returns RowTooWide on a 5-field row" { var reader = Io.Reader.fixed("a,b,c,d,e\n"); var field_buf: [64]u8 = undefined; var it = try BoundedRowIterator(4).init(&reader, &field_buf, .{}); try testing.expectError(error.RowTooWide, it.next());}
test "FixedBufferRowIterator returns RowTooWide when row_buf overflows" { var reader = Io.Reader.fixed("a,b,c,d,e\n"); var field_buf: [64]u8 = undefined; var row_buf: [4][]const u8 = undefined; var it = try FixedBufferRowIterator.init(&reader, &field_buf, &row_buf, .{}); try testing.expectError(error.RowTooWide, it.next());}
test "RowIterator (allocating) has no RowTooWide failure mode" { var reader = Io.Reader.fixed("a,b,c,d,e,f,g,h\n"); var field_buf: [64]u8 = undefined; var it = try RowIterator.init(&reader, &field_buf, testing.allocator, .{}); defer it.deinit(); const row = (try it.next()).?; try testing.expectEqual(@as(usize, 8), row.count());}
// --- Row.at bounds ---
test "Row.at returns null for out-of-range index" { var reader = Io.Reader.fixed("a,b\n"); var field_buf: [16]u8 = undefined; var it = try BoundedRowIterator(8).init(&reader, &field_buf, .{}); const row = (try it.next()).?; try testing.expectEqualStrings("a", row.at(0).?); try testing.expectEqualStrings("b", row.at(1).?); try testing.expectEqual(@as(?[]const u8, null), row.at(2));}
test "row iterators: many rows through a small field_buf (resetBuffer works)" { // 8-byte field_buf cannot hold the whole input; tests that // `resetBuffer()` recycles the buffer between rows. const input = "abcdef\nghijkl\nmnopqr\n"; var reader = Io.Reader.fixed(input); var field_buf: [8]u8 = undefined; var it = try BoundedRowIterator(2).init(&reader, &field_buf, .{}); const r1 = (try it.next()).?; try testing.expectEqualStrings("abcdef", r1.fields[0]); const r2 = (try it.next()).?; try testing.expectEqualStrings("ghijkl", r2.fields[0]); const r3 = (try it.next()).?; try testing.expectEqualStrings("mnopqr", r3.fields[0]); try testing.expectEqual(@as(?Row, null), try it.next());}