diff --git a/README.md b/README.md index ef97443..12d5766 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,20 @@ something else entirely. Note that `.any` is the default, so a CR that used to survive to the end of a field under the old LF-only behavior is now consumed as part of the terminator. +For the strict reading that RFC 4180 specifies — only CRLF ends a record, and +a CR or an LF on its own is ordinary field data — use `.crlf`: + +```zig +var tokenizer = try csv.CsvTokenizer.init(&reader, &field_buf, .{ + .row_sep = .crlf, +}); +``` + +`.crlf` needs one byte more headroom than the other modes: deciding whether a +CR ends a record takes both the CR and the byte behind it in the buffer at +once, so the buffer must exceed the longest field by two rather than one. It +reports `error.ShortBuffer` rather than misreading the input. + Inside a quoted field none of this applies: CR and LF are data there, and a quoted field may span lines. @@ -157,7 +171,7 @@ quoted field may span lines. | `CsvTokenizer.next()` | Return the next `?CsvToken`, or `null` at end of input. | | `CsvToken` | Tagged union: `.field: []const u8` or `.row_end`. | | `CsvConfig` | `col_sep` (default `,`), `row_sep` (default `.any`), `quote` (default `"`). | -| `RowSeparator` | `.any` to accept CR, LF or CRLF, or `.{ .byte = c }` to require exactly one byte. | +| `RowSeparator` | `.any` to accept CR, LF or CRLF, `.crlf` to require the pair, or `.{ .byte = c }` to require one byte. | | `CsvError` | `ShortBuffer`, `MisplacedQuote`, `NoSeparatorAfterField`. | A `field` slice points into the caller's buffer and is only valid until the @@ -165,7 +179,9 @@ next call to `next()`. Copy it if you need to keep it. ## Behavior and limitations -- Input is read as UTF-8. +- Input is treated as bytes. UTF-8 passes through unharmed, but nothing + validates it, and a field is a slice of the input rather than a sequence of + code points. - Quoted fields may contain the column separator, the row separator, and the quote character itself when doubled (`"He said ""hi"""`). - The column separator and quote are configurable, but **only as single @@ -182,6 +198,35 @@ next call to `next()`. Copy it if you need to keep it. - An unclosed quoted field is an error (`error.ShortBuffer`), as is a field longer than the buffer. +## RFC 4180 + +[RFC 4180](https://www.rfc-editor.org/rfc/rfc4180.txt) describes the format +this implements. Every clause of its section 2 is covered, and the checking is +mechanical rather than a reading: a generator builds documents that satisfy the +RFC's ABNF — records, quoting, `""` escaping, and `TEXTDATA` restricted to +`%x20-21 / %x23-2B / %x2D-7E` — and the fields that come back out are compared +to the ones that went in. 294,590 such documents and 1,845,835 fields match +byte for byte, under both the default configuration and `.crlf`. + +Where this library is deliberately more permissive than the RFC: + +- By default a lone CR or a lone LF also ends a record, where the RFC says + CRLF. This cannot misread a valid RFC document, since `TEXTDATA` admits + neither CR nor LF and so an unquoted field cannot contain one; it only + accepts input the RFC would reject. Use `.crlf` for the strict reading. +- Unquoted fields accept any byte, where `TEXTDATA` is printable ASCII minus + `"` and `,`. Tabs, control bytes and UTF-8 all pass through. +- An empty input yields no tokens. Read literally the ABNF makes an empty file + one record holding one empty field. +- The grammar cannot distinguish a final record holding a single empty field + from the optional trailing CRLF, since both are the same bytes; a trailing + terminator is read as ending the file rather than as starting an empty + record. + +Two things the RFC mentions that a tokenizer is the wrong layer for: the +optional header line is just the first record, and the rule that every record +carry the same number of fields needs a record at a time rather than a token. + ## Development The repository ships a Nix flake with the pinned toolchain: diff --git a/src/main.zig b/src/main.zig index c8d1d41..11894a5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -32,6 +32,12 @@ pub const RowSeparator = union(enum) { /// them. any, + /// Require exactly the two-byte sequence CR LF, as RFC 4180 specifies. + /// A CR that is not followed by an LF, and an LF on its own, are both + /// ordinary field data. This is the strict reading; `.any` is the + /// forgiving one. + crlf, + /// Require exactly this byte. `.byte = '\n'` is the traditional Unix /// terminator and `.byte = '\r'` the classic Mac OS one; under either, /// the other byte is ordinary field data. Any other byte works too, for @@ -114,6 +120,35 @@ pub const CsvReader = struct { return null; } + /// Read up to the first byte in `set`, or to a CR that is immediately + /// followed by an LF. A CR with anything else behind it is data. + /// + /// Returns null when the answer is not yet knowable -- including when a + /// CR is the last byte read, since whether it ends the record depends on + /// a byte that has not arrived. The caller reads more and asks again; at + /// end of input there is nothing more to come and the trailing CR is + /// data, which is what taking the rest of the buffer yields. + pub fn untilCrlf(self: *Self, set: *const TerminalSet) !?[]u8 { + if (!try self.ensureData()) { + return null; + } + + for (self.current, 0..) |c, pos| { + if (c == '\r') { + if (pos + 1 >= self.current.len) return null; + if (self.current[pos + 1] != '\n') continue; + } else if (!set[c]) { + continue; + } + + const s = self.current[0..pos]; + self.current = self.current[pos..]; + return s; + } + + return null; + } + pub fn until(self: *Self, terminators: []const u8) !?[]u8 { if (!try self.ensureData()) { return null; @@ -263,6 +298,9 @@ pub const CsvTokenizer = struct { terminal_chars[len] = b; len += 1; }, + // Neither CR nor LF ends a field on its own here: only the pair + // does, which `untilCrlf` recognizes as it scans. + .crlf => {}, } terminal_chars[len] = config.quote; @@ -291,6 +329,7 @@ pub const CsvTokenizer = struct { inline fn isRowSepStart(self: *const Self, c: u8) bool { return switch (self.config.row_sep) { .any => c == '\r' or c == '\n', + .crlf => c == '\r', .byte => |b| c == b, }; } @@ -315,6 +354,15 @@ pub const CsvTokenizer = struct { } } }, + .crlf => { + // The scan only stops on a CR that an LF follows, so for an + // unquoted field this is already known. After a quoted field + // it is not: there, a CR with no LF behind it is a byte that + // has no business between two fields. + if (!try self.reader.ensureData()) return CsvError.NoSeparatorAfterField; + const lf = (try self.reader.char()) orelse return CsvError.NoSeparatorAfterField; + if (lf != '\n') return CsvError.NoSeparatorAfterField; + }, .byte => {}, } } @@ -405,11 +453,21 @@ pub const CsvTokenizer = struct { unreachable; } + /// Read up to whatever ends an unquoted field, which depends on how + /// records are terminated: under `.crlf` a CR only counts when an LF + /// follows it, so that scan has to look ahead. + inline fn scanField(self: *Self) !?[]u8 { + return switch (self.config.row_sep) { + .crlf => self.reader.untilCrlf(&self.terminal_set), + .any, .byte => self.reader.untilAny(&self.terminal_set), + }; + } + fn parseField(self: *Self) !CsvToken { const first = (try self.reader.peek()).?; if (first != self.config.quote) { - var field = try self.reader.untilAny(&self.terminal_set); + var field = try self.scanField(); while (field == null) { // No terminator among what has been read so far, which means // either that more is coming or that the input has run out. @@ -428,7 +486,7 @@ pub const CsvTokenizer = struct { return CsvError.ShortBuffer; } - field = try self.reader.untilAny(&self.terminal_set); + field = try self.scanField(); } const terminator = (try self.reader.peek()).?; diff --git a/test/csv_tokenizer.zig b/test/csv_tokenizer.zig index db7fe4b..b8990d3 100644 --- a/test/csv_tokenizer.zig +++ b/test/csv_tokenizer.zig @@ -367,6 +367,53 @@ test "An unclosed quoted field is an error" { try testing.expectError(csv_mod.CsvError.ShortBuffer, csv.next()); } +test "Strict CRLF: only the pair terminates a record" { + const strict: csv_mod.CsvConfig = .{ .row_sep = .crlf }; + + try expectStream("a,b\r\nc,d\r\n", strict, 64, &.{ "a", "b", null, "c", "d", null }); + try expectStream("a,b\r\nc,d", strict, 64, &.{ "a", "b", null, "c", "d", null }); + try expectStream("a\r\n\r\nb\r\n", strict, 64, &.{ "a", null, "", null, "b", null }); +} + +test "Strict CRLF: a lone CR or LF is field data" { + const strict: csv_mod.CsvConfig = .{ .row_sep = .crlf }; + + try expectStream("a\nb\r\n", strict, 64, &.{ "a\nb", null }); + try expectStream("a\rb\r\n", strict, 64, &.{ "a\rb", null }); + try expectStream("a\r", strict, 64, &.{ "a\r", null }); + try expectStream("x\r\ry\r\n", strict, 64, &.{ "x\r\ry", null }); +} + +test "Strict CRLF: a CR split across a buffer refill" { + const data = "aaa,bbb\r\nccc\rddd\r\neee\r\n"; + const expected: []const ?[]const u8 = &.{ "aaa", "bbb", null, "ccc\rddd", null, "eee", null }; + + // A refill landing between a CR and the byte that decides its meaning is + // the whole difficulty of this mode, so try every small buffer. + inline for (.{ 9, 10, 11, 12, 13, 14, 16, 24, 64 }) |buffer_len| { + expectStream(data, .{ .row_sep = .crlf }, buffer_len, expected) catch |err| { + std.log.warn("failed with buffer_len={d}\n", .{buffer_len}); + return err; + }; + } +} + +test "Strict CRLF: quoted fields carry CR and LF unharmed" { + const strict: csv_mod.CsvConfig = .{ .row_sep = .crlf }; + + try expectStream("\"x\ry\nz\r\nw\"\r\n", strict, 64, &.{ "x\ry\nz\r\nw", null }); + try expectStream("\"a\",\"b\"\r\n", strict, 64, &.{ "a", "b", null }); +} + +test "Strict CRLF: a bare CR after a quoted field is an error" { + var reader = std.Io.Reader.fixed("\"a\"\rb\r\n"); + var buffer: [64]u8 = undefined; + var csv = try getTokenizer(&reader, &buffer, .{ .row_sep = .crlf }); + + try expectToken(csv_mod.CsvToken{ .field = "a" }, try csv.next()); + try testing.expectError(csv_mod.CsvError.NoSeparatorAfterField, csv.next()); +} + test "A configured quote is honored in place of the default" { const single: csv_mod.CsvConfig = .{ .quote = '\'' }; @@ -385,3 +432,18 @@ test "A configured quote still rejects what the default would" { try expectToken(csv_mod.CsvToken{ .field = "a" }, try csv.next()); try testing.expectError(csv_mod.CsvError.NoSeparatorAfterField, csv.next()); } + +test "Strict CRLF needs one byte more headroom than the buffer alone" { + // Deciding whether a CR ends the record takes the CR and the byte behind + // it in the buffer together, so `.crlf` wants one byte more than `.any` + // does for the same field. It says so with an error rather than + // misreading the input. + const data = "ccc\rddd\r\neee\r\n"; // longest field is 7 bytes + + try expectStream(data, .{ .row_sep = .crlf }, 9, &.{ "ccc\rddd", null, "eee", null }); + + var reader = std.Io.Reader.fixed(data); + var buffer: [8]u8 = undefined; + var csv = try getTokenizer(&reader, &buffer, .{ .row_sep = .crlf }); + try testing.expectError(csv_mod.CsvError.ShortBuffer, csv.next()); +} diff --git a/test/fuzz.zig b/test/fuzz.zig index c504667..ade4a31 100644 --- a/test/fuzz.zig +++ b/test/fuzz.zig @@ -32,7 +32,7 @@ pub const max_input = 4096; pub const col_seps = [_]u8{ ',', ';', '\t', '|' }; /// How many row separator choices `readParams` offers. -pub const row_sep_choices = 3; +pub const row_sep_choices = 4; /// Smallest and largest field buffer, as powers of two. pub const min_buffer_shift = 2; @@ -53,7 +53,8 @@ fn readParams(smith: *Smith, in: []u8) Params { const row_sep: csv.RowSeparator = switch (smith.valueRangeAtMost(u8, 0, row_sep_choices - 1)) { 0 => .any, - 1 => .{ .byte = '\n' }, + 1 => .crlf, + 2 => .{ .byte = '\n' }, else => .{ .byte = '\r' }, }; @@ -128,7 +129,7 @@ fn expectSameTokens(a: []const Token, b: []const Token) !void { /// Write `tokens` back out as CSV that the same configuration will read. fn encode(gpa: Allocator, tokens: []const Token, config: csv.CsvConfig) ![]u8 { const terminator: []const u8 = switch (config.row_sep) { - .any => "\r\n", + .any, .crlf => "\r\n", .byte => |b| &[_]u8{b}, }; @@ -244,10 +245,12 @@ fn reference(gpa: Allocator, data: []const u8, config: csv.CsvConfig) ![]Token { if (data.len == 0) return tokens.toOwnedSlice(gpa); const isTerm = struct { - fn f(c: u8, sep: csv.RowSeparator) bool { + fn f(bytes: []const u8, at: usize, sep: csv.RowSeparator) bool { return switch (sep) { - .any => c == '\r' or c == '\n', - .byte => |b| c == b, + .any => bytes[at] == '\r' or bytes[at] == '\n', + // Only the pair terminates; a lone CR is data. + .crlf => bytes[at] == '\r' and at + 1 < bytes.len and bytes[at + 1] == '\n', + .byte => |b| bytes[at] == b, }; } }.f; @@ -278,11 +281,11 @@ fn reference(gpa: Allocator, data: []const u8, config: csv.CsvConfig) ![]Token { } // Only a separator, a terminator, or the end may follow. - if (i < data.len and data[i] != config.col_sep and !isTerm(data[i], config.row_sep)) { + if (i < data.len and data[i] != config.col_sep and !isTerm(data, i, config.row_sep)) { return error.NoSeparatorAfterField; } } else { - while (i < data.len and data[i] != config.col_sep and !isTerm(data[i], config.row_sep)) { + while (i < data.len and data[i] != config.col_sep and !isTerm(data, i, config.row_sep)) { if (data[i] == config.quote) return error.MisplacedQuote; try field.append(gpa, data[i]); i += 1; @@ -304,7 +307,14 @@ fn reference(gpa: Allocator, data: []const u8, config: csv.CsvConfig) ![]Token { // A record terminator; under `.any`, CR takes a following LF with it. const c = data[i]; i += 1; - if (config.row_sep == .any and c == '\r' and i < data.len and data[i] == '\n') i += 1; + switch (config.row_sep) { + .any => if (c == '\r' and i < data.len and data[i] == '\n') { + i += 1; + }, + // `isTerm` already established that an LF follows. + .crlf => i += 1, + .byte => {}, + } try tokens.append(gpa, .row_end); if (i >= data.len) break; diff --git a/tools/bench.zig b/tools/bench.zig index 9982a06..cb27dd6 100644 --- a/tools/bench.zig +++ b/tools/bench.zig @@ -47,6 +47,7 @@ pub fn main(init: std.process.Init) !void { var rows: usize = 20_000; var col_sep: u8 = ','; var quote: u8 = '"'; + var strict = false; var args = try std.process.Args.Iterator.initAllocator(init.minimal.args, gpa); defer args.deinit(); @@ -62,6 +63,8 @@ pub fn main(init: std.process.Init) !void { col_sep = (args.next() orelse return error.MissingValue)[0]; } else if (std.mem.eql(u8, arg, "--quote")) { quote = (args.next() orelse return error.MissingValue)[0]; + } else if (std.mem.eql(u8, arg, "--strict")) { + strict = true; } else if (std.mem.eql(u8, arg, "--case")) { only = args.next() orelse return error.MissingValue; } else { @@ -73,7 +76,7 @@ pub fn main(init: std.process.Init) !void { // Built from parsed arguments so the optimizer cannot see through it. const config: csv.CsvConfig = .{ .col_sep = col_sep, - .row_sep = .any, + .row_sep = if (strict) .crlf else .any, .quote = quote, };