diff --git a/build.zig b/build.zig index f366795..6aad0eb 100644 --- a/build.zig +++ b/build.zig @@ -8,27 +8,21 @@ pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); - const csv_module = b.addModule("zig-csv", .{ + const csv_module = b.addModule("csv", .{ .root_source_file = b.path("src/main.zig"), }); - const lib = b.addStaticLibrary(.{ - .name = "csv", - .root_source_file = b.path("src/main.zig"), - .optimize = optimize, - .target = target, - }); - b.installArtifact(lib); - - const main_tests = b.addTest(.{ - .name = "csv_tests", - .root_source_file = b.path("test/csv_tokenizer.zig"), - .optimize = optimize, - .target = target, + const csv_tests = b.addTest(.{ + .name = "tests", + .root_module = b.createModule(.{ + .root_source_file = b.path("test/csv_tokenizer.zig"), + .optimize = optimize, + .target = target, + }), }); - main_tests.root_module.addImport("csv", csv_module); + csv_tests.root_module.addImport("csv", csv_module); - const run_test_cmd = b.addRunArtifact(main_tests); + const run_test_cmd = b.addRunArtifact(csv_tests); run_test_cmd.has_side_effects = true; run_test_cmd.step.dependOn(b.getInstallStep()); diff --git a/build.zig.zon b/build.zig.zon index 88d020c..824cdec 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,8 @@ .{ - .name = "zig-csv", - .version = "0.0.1", - .minimum_zig_version = "0.13.0", + .name = .zig_csv, + .version = "0.0.2", + .fingerprint = 0xb51631650614d5d2, + .minimum_zig_version = "0.16.0", .dependencies = .{}, .paths = .{ "build.zig", diff --git a/flake.lock b/flake.lock index 6b7db10..8f2c958 100644 --- a/flake.lock +++ b/flake.lock @@ -1,322 +1,21 @@ { "nodes": { - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { - "inputs": { - "systems": "systems_2" - }, - "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_3": { - "inputs": { - "systems": "systems_3" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_4": { - "inputs": { - "systems": "systems_4" - }, - "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "zls", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "langref": { - "flake": false, - "locked": { - "narHash": "sha256-O6p2tiKD8ZMhSX+DeA/o5hhAvcPkU2J9lFys/r11peY=", - "type": "file", - "url": "https://raw.githubusercontent.com/ziglang/zig/0fb2015fd3422fc1df364995f9782dfe7255eccd/doc/langref.html.in" - }, - "original": { - "type": "file", - "url": "https://raw.githubusercontent.com/ziglang/zig/0fb2015fd3422fc1df364995f9782dfe7255eccd/doc/langref.html.in" - } - }, "nixpkgs": { "locked": { - "lastModified": 1710814491, - "narHash": "sha256-LyI5tOn0F7NM3WyZj3ULRp8i44IqarbY6WTxalD0n4A=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "ff1a94e523ae9fb272e0581f068baee5d1068476", - "type": "github" + "lastModified": 1786384358, + "narHash": "sha256-OETwiymkMwunF+pfL25DzLMF2/YpMcJUjVd9qZ0YdsM=", + "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a", + "type": "tarball", + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1051473.2fcb964de67f/nixexprs.tar.zst" }, "original": { - "owner": "nixos", - "repo": "nixpkgs", - "rev": "ff1a94e523ae9fb272e0581f068baee5d1068476", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1708161998, - "narHash": "sha256-6KnemmUorCvlcAvGziFosAVkrlWZGIc6UNT9GUYr0jQ=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "84d981bae8b5e783b3b548de505b22880559515f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_3": { - "locked": { - "lastModified": 1717696253, - "narHash": "sha256-1+ua0ggXlYYPLTmMl3YeYYsBXDSCqT+Gw3u6l4gvMhA=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "9b5328b7f761a7bbdc0e332ac4cf076a3eedb89b", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", - "type": "github" + "type": "tarball", + "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.zst" } }, "root": { "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs", - "zig": "zig", - "zls": "zls" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_4": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "zig": { - "inputs": { - "flake-compat": "flake-compat", - "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs_2" - }, - "locked": { - "lastModified": 1717848532, - "narHash": "sha256-d+xIUvSTreHl8pAmU1fnmkfDTGQYCn2Rb/zOwByxS2M=", - "owner": "mitchellh", - "repo": "zig-overlay", - "rev": "02fc5cc555fc14fda40c42d7c3250efa43812b43", - "type": "github" - }, - "original": { - "owner": "mitchellh", - "repo": "zig-overlay", - "type": "github" - } - }, - "zig-overlay": { - "inputs": { - "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_4", - "nixpkgs": [ - "zls", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1717848532, - "narHash": "sha256-d+xIUvSTreHl8pAmU1fnmkfDTGQYCn2Rb/zOwByxS2M=", - "owner": "mitchellh", - "repo": "zig-overlay", - "rev": "02fc5cc555fc14fda40c42d7c3250efa43812b43", - "type": "github" - }, - "original": { - "owner": "mitchellh", - "repo": "zig-overlay", - "type": "github" - } - }, - "zls": { - "inputs": { - "flake-utils": "flake-utils_3", - "gitignore": "gitignore", - "langref": "langref", - "nixpkgs": "nixpkgs_3", - "zig-overlay": "zig-overlay" - }, - "locked": { - "lastModified": 1717891507, - "narHash": "sha256-l/Zo1OwdB3js3wXOpgFLozKwq+bdsPySZtKbEbYBb7U=", - "owner": "zigtools", - "repo": "zls", - "rev": "a26718049a8657d4da04c331aeced1697bc7652b", - "type": "github" - }, - "original": { - "owner": "zigtools", - "ref": "refs/tags/0.13.0", - "repo": "zls", - "type": "github" + "nixpkgs": "nixpkgs" } } }, diff --git a/flake.nix b/flake.nix index f7b983d..b1e6524 100644 --- a/flake.nix +++ b/flake.nix @@ -1,46 +1,32 @@ { description = "zig-csv"; - # based on https://github.com/mitchellh/zig-overlay/blob/2c86c36e7fe65faac08bdf85d041cf7b798f8ee8/templates/init/flake.nix inputs = { - # zls at 0.11.0 - nixpkgs.url = "github:nixos/nixpkgs/ff1a94e523ae9fb272e0581f068baee5d1068476"; - flake-utils.url = "github:numtide/flake-utils"; - zig.url = "github:mitchellh/zig-overlay"; - zls.url = "github:zigtools/zls?ref=refs/tags/0.13.0"; + nixpkgs.url = "https://channels.nixos.org/nixos-unstable/nixexprs.tar.zst"; }; - outputs = { - self, - nixpkgs, - flake-utils, - ... - } @ inputs : + outputs = + { + self, + nixpkgs, + ... + }: let - overlays = [ - (final: prev: { - zigpkgs = inputs.zig.packages.${prev.system}; - zlspkgs = inputs.zls.packages.${prev.system}; - }) - ]; - # Our supported systems are the same supported systems as the Zig binaries - systems = builtins.attrNames inputs.zig.packages; + platforms = [ "x86_64-linux" ]; + packages = + system: + import nixpkgs { + inherit system; + }; + forAllSystems = (function: nixpkgs.lib.genAttrs platforms (system: function (packages system))); in - flake-utils.lib.eachSystem systems ( - system: let - pkgs = import nixpkgs { inherit overlays system; }; - in rec { - devShells.default = pkgs.mkShell { - nativeBuildInputs = [ - pkgs.zigpkgs."0.13.0" - ]; - buildInputs = [ - pkgs.zlspkgs.zls - ]; - }; - - # For compatibility with older versions of the `nix` binary - devShell = self.devShells.${system}.default; - } - ); + { + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + nativeBuildInputs = [ + pkgs.zig_0_16 + ]; + }; + }); + }; } diff --git a/src/main.zig b/src/main.zig index 938963e..52a34da 100644 --- a/src/main.zig +++ b/src/main.zig @@ -35,143 +35,140 @@ const QuoteFieldReadResult = struct { contains_quotes: bool, }; -fn CsvReader(comptime Reader: type) type { - - // TODO comptime - return struct { - buffer: []u8, - current: []u8, - - reader: Reader, - all_read: bool = false, - - const Self = @This(); - - pub fn init(reader: Reader, buffer: []u8) Self { - return .{ - .buffer = buffer, - .current = buffer[0..0], - .reader = reader, - }; - } - - inline fn empty(self: *Self) bool { - return self.current.len == 0; +// TODO comptime +pub const CsvReader = struct { + buffer: []u8, + current: []u8, + + reader: *std.Io.Reader, + all_read: bool = false, + + const Self = @This(); + + pub fn init(reader: *std.Io.Reader, buffer: []u8) Self { + return .{ + .buffer = buffer, + .current = buffer[0..0], + .reader = reader, + }; + } + + inline fn empty(self: *Self) bool { + return self.current.len == 0; + } + + pub fn char(self: *Self) !?u8 { + if (!try self.ensureData()) { + return null; } - pub fn char(self: *Self) !?u8 { - if (!try self.ensureData()) { - return null; - } + const c = self.current[0]; + self.current = self.current[1..]; - const c = self.current[0]; - self.current = self.current[1..]; + return c; + } - return c; + pub inline fn peek(self: *Self) !?u8 { + if (!try self.ensureData()) { + return null; } - pub inline fn peek(self: *Self) !?u8 { - if (!try self.ensureData()) { - return null; - } + return self.current[0]; + } - return self.current[0]; + pub fn until(self: *Self, terminators: []const u8) !?[]u8 { + if (!try self.ensureData()) { + return null; } - pub fn until(self: *Self, terminators: []const u8) !?[]u8 { - if (!try self.ensureData()) { - return null; - } - - for (self.current, 0..) |c, pos| { - // TODO inline - for (terminators) |ct| { - if (c == ct) { - const s = self.current[0..pos]; - self.current = self.current[pos..]; - // print("{}|{}", .{ s, self.current }); - return s; - } + for (self.current, 0..) |c, pos| { + // TODO inline + for (terminators) |ct| { + if (c == ct) { + const s = self.current[0..pos]; + self.current = self.current[pos..]; + // print("{}|{}", .{ s, self.current }); + return s; } } + } - // print("ALL_READ: {}\n", .{self.all_read}); + // print("ALL_READ: {}\n", .{self.all_read}); + return null; + } + + pub fn untilClosingQuote(self: *Self, quote: u8) !?QuoteFieldReadResult { + if (!try self.ensureData()) { return null; } - pub fn untilClosingQuote(self: *Self, quote: u8) !?QuoteFieldReadResult { - if (!try self.ensureData()) { - return null; - } - - var idx: usize = 0; - var contains_quotes: bool = false; - while (idx < self.current.len) : (idx += 1) { - const c = self.current[idx]; - // print("IDX QUOTED: {}={c}\n", .{ idx, c }); - if (c == quote) { - // double quotes, shift forward - // print("PEEK {c}\n", .{buffer[idx + 1]}); - if (idx < self.current.len - 1 and self.current[idx + 1] == '"') { - // print("DOUBLE QUOTES\n", .{}); - contains_quotes = true; - idx += 1; - } else { - // print("ALL_READ {}\n", .{self.all_read}); - if (!self.all_read and idx == self.current.len - 1) { - return null; - } + var idx: usize = 0; + var contains_quotes: bool = false; + while (idx < self.current.len) : (idx += 1) { + const c = self.current[idx]; + // print("IDX QUOTED: {}={c}\n", .{ idx, c }); + if (c == quote) { + // double quotes, shift forward + // print("PEEK {c}\n", .{buffer[idx + 1]}); + if (idx < self.current.len - 1 and self.current[idx + 1] == '"') { + // print("DOUBLE QUOTES\n", .{}); + contains_quotes = true; + idx += 1; + } else { + // print("ALL_READ {}\n", .{self.all_read}); + if (!self.all_read and idx == self.current.len - 1) { + return null; + } - const s = self.current[0..idx]; - self.current = self.current[idx..]; + const s = self.current[0..idx]; + self.current = self.current[idx..]; - return QuoteFieldReadResult{ .value = s, .contains_quotes = contains_quotes }; - } + return QuoteFieldReadResult{ .value = s, .contains_quotes = contains_quotes }; } } - - return null; } - /// Tries to read more data from an underlying reader if buffer is not already full. - /// If anything was read returns true, otherwise false. - pub fn read(self: *Self) !bool { - const current_len = self.current.len; + return null; + } - if (current_len == self.buffer.len) { - return false; - } + /// Tries to read more data from an underlying reader if buffer is not already full. + /// If anything was read returns true, otherwise false. + pub fn read(self: *Self) !bool { + const current_len = self.current.len; - if (current_len > 0) { - mem.copyForwards(u8, self.buffer, self.current); - } + if (current_len == self.buffer.len) { + return false; + } - const read_len = try self.reader.read(self.buffer[current_len..]); - // print("READ: current_len={} read_len={}\n", .{ current_len, read_len }); + if (current_len > 0) { + mem.copyForwards(u8, self.buffer, self.current); + } - self.current = self.buffer[0 .. current_len + read_len]; - self.all_read = read_len == 0; + const read_len = try self.reader.readSliceShort(self.buffer[current_len..]); + // print("READ: current_len={} read_len={}\n", .{ current_len, read_len }); - return read_len > 0; - } + self.current = self.buffer[0 .. current_len + read_len]; + self.all_read = read_len == 0; - // Ensures that there are some data in the buffer. Returns false if no data are available - pub inline fn ensureData(self: *Self) !bool { - if (!self.empty()) { - return true; - } + return read_len > 0; + } - if (self.all_read) { - return false; - } + // Ensures that there are some data in the buffer. Returns false if no data are available + pub inline fn ensureData(self: *Self) !bool { + if (!self.empty()) { + return true; + } - return self.read(); + if (self.all_read) { + return false; } - }; -} + + return self.read(); + } +}; /// Tokenizes input from reader into stream of CsvTokens -pub fn CsvTokenizer(comptime Reader: type) type { +pub const CsvTokenizer = struct { const Status = enum { initial, row_start, @@ -180,181 +177,178 @@ pub fn CsvTokenizer(comptime Reader: type) type { row_end, eof, }; + const Self = @This(); - return struct { - const Self = @This(); + config: CsvConfig, + terminal_chars: [3]u8 = undefined, - config: CsvConfig, - terminal_chars: [3]u8 = undefined, + reader: CsvReader, - reader: CsvReader(Reader), + status: Status = .initial, - status: Status = .initial, + pub fn init(reader: *std.Io.Reader, buffer: []u8, config: CsvConfig) !Self { + return Self{ + .config = config, + .terminal_chars = [_]u8{ config.col_sep, config.row_sep, '"' }, + .reader = CsvReader.init(reader, buffer), + }; + } - pub fn init(reader: Reader, buffer: []u8, config: CsvConfig) !Self { - return Self{ - .config = config, - .terminal_chars = [_]u8{ config.col_sep, config.row_sep, '"' }, - .reader = CsvReader(Reader).init(reader, buffer), - }; - } + pub fn next(self: *Self) !?CsvToken { + var next_status: ?Status = self.status; + + // Cannot use anonymous enum literals for Status + // https://github.com/ziglang/zig/issues/4255 - pub fn next(self: *Self) !?CsvToken { - var next_status: ?Status = self.status; + while (next_status) |status| { + // print("STATUS: {}\n", .{self.status}); + next_status = switch (status) { + .initial => if (try self.reader.read()) Status.row_start else Status.eof, + .row_start => if (!try self.reader.ensureData()) Status.eof else Status.field, + .field => blk: { + if (!try self.reader.ensureData()) { + break :blk .row_end; + } - // Cannot use anonymous enum literals for Status - // https://github.com/ziglang/zig/issues/4255 + return try self.parseField(); + }, + .quoted_field_end => blk: { + // read closing quotes + const quote = try self.reader.char(); + assert(quote == self.config.quote); - while (next_status) |status| { - // print("STATUS: {}\n", .{self.status}); - next_status = switch (status) { - .initial => if (try self.reader.read()) Status.row_start else Status.eof, - .row_start => if (!try self.reader.ensureData()) Status.eof else Status.field, - .field => blk: { - if (!try self.reader.ensureData()) { - break :blk .row_end; - } + if (!try self.reader.ensureData()) { + break :blk Status.row_end; + } - return try self.parseField(); - }, - .quoted_field_end => blk: { - // read closing quotes - const quote = try self.reader.char(); - assert(quote == self.config.quote); + const c = (try self.reader.peek()); - if (!try self.reader.ensureData()) { - break :blk Status.row_end; + if (c) |value| { + // print("END: {}\n", .{value}); + if (value == self.config.col_sep) { + // TODO write repro for assert with optional + // const col_sep = try self.reader.char(); + // assert(col_sep == self.config.col_sep); + const col_sep = (try self.reader.char()).?; + assert(col_sep == self.config.col_sep); + + break :blk Status.field; } - const c = (try self.reader.peek()); - - if (c) |value| { - // print("END: {}\n", .{value}); - if (value == self.config.col_sep) { - // TODO write repro for assert with optional - // const col_sep = try self.reader.char(); - // assert(col_sep == self.config.col_sep); - const col_sep = (try self.reader.char()).?; - assert(col_sep == self.config.col_sep); - - break :blk Status.field; - } - - if (value == self.config.row_sep) { - break :blk Status.row_end; - } - - // quote means that it did not fit into buffer and it cannot be analyzed as "" - if (value == self.config.quote) { - return CsvError.ShortBuffer; - } - } else { - break :blk Status.eof; + if (value == self.config.row_sep) { + break :blk Status.row_end; } - return CsvError.NoSeparatorAfterField; - }, - .row_end => { - if (!try self.reader.ensureData()) { - self.status = Status.eof; - return CsvToken{ .row_end = {} }; + // quote means that it did not fit into buffer and it cannot be analyzed as "" + if (value == self.config.quote) { + return CsvError.ShortBuffer; } + } else { + break :blk Status.eof; + } - const rowSep = try self.reader.char(); - assert(rowSep == self.config.row_sep); + return CsvError.NoSeparatorAfterField; + }, + .row_end => { + if (!try self.reader.ensureData()) { + self.status = Status.eof; + return CsvToken{ .row_end = {} }; + } - self.status = Status.row_start; + const rowSep = try self.reader.char(); + assert(rowSep == self.config.row_sep); - return CsvToken{ .row_end = {} }; - }, - .eof => { - return null; - }, - }; + self.status = Status.row_start; - // make the transition and also ensure that next_status is set at this point - self.status = next_status.?; - } + return CsvToken{ .row_end = {} }; + }, + .eof => { + return null; + }, + }; - unreachable; + // make the transition and also ensure that next_status is set at this point + self.status = next_status.?; } - fn parseField(self: *Self) !CsvToken { - const first = (try self.reader.peek()).?; + unreachable; + } - if (first != '"') { - var field = try self.reader.until(&self.terminal_chars); - if (field == null) { - // force read - maybe separator was not read yet - const hasData = try self.reader.read(); - if (!hasData) { - return CsvError.ShortBuffer; - } + fn parseField(self: *Self) !CsvToken { + const first = (try self.reader.peek()).?; - field = try self.reader.until(&self.terminal_chars); - if (field == null) { - return CsvError.ShortBuffer; - } + if (first != '"') { + var field = try self.reader.until(&self.terminal_chars); + if (field == null) { + // force read - maybe separator was not read yet + const hasData = try self.reader.read(); + if (!hasData) { + return CsvError.ShortBuffer; } - const terminator = (try self.reader.peek()).?; - - if (terminator == self.config.col_sep) { - _ = try self.reader.char(); - return CsvToken{ .field = field.? }; + field = try self.reader.until(&self.terminal_chars); + if (field == null) { + return CsvError.ShortBuffer; } + } - if (terminator == self.config.row_sep) { - self.status = .row_end; - return CsvToken{ .field = field.? }; - } + const terminator = (try self.reader.peek()).?; - if (terminator == self.config.quote) { - return CsvError.MisplacedQuote; + if (terminator == self.config.col_sep) { + _ = try self.reader.char(); + return CsvToken{ .field = field.? }; + } + + if (terminator == self.config.row_sep) { + self.status = .row_end; + return CsvToken{ .field = field.? }; + } + + if (terminator == self.config.quote) { + return CsvError.MisplacedQuote; + } + + return CsvError.ShortBuffer; + } else { + // consume opening quote + _ = try self.reader.char(); + var quoted_field = try self.reader.untilClosingQuote(self.config.quote); + if (quoted_field == null) { + // force read - maybe separator was not read yet + const hasData = try self.reader.read(); + if (!hasData) { + return CsvError.ShortBuffer; } - return CsvError.ShortBuffer; - } else { - // consume opening quote - _ = try self.reader.char(); - var quoted_field = try self.reader.untilClosingQuote(self.config.quote); + // this read will fill the buffer + quoted_field = try self.reader.untilClosingQuote(self.config.quote); if (quoted_field == null) { - // force read - maybe separator was not read yet - const hasData = try self.reader.read(); - if (!hasData) { - return CsvError.ShortBuffer; - } - - // this read will fill the buffer - quoted_field = try self.reader.untilClosingQuote(self.config.quote); - if (quoted_field == null) { - return CsvError.ShortBuffer; - } + return CsvError.ShortBuffer; } + } - self.status = .quoted_field_end; + self.status = .quoted_field_end; - const field = quoted_field.?; - if (!field.contains_quotes) { - return CsvToken{ .field = field.value }; - } else { - // walk the field and remove double quotes by shifting bytes - const value = field.value; - var diff: u64 = 0; - var idx: usize = 0; - while (idx < value.len) : (idx += 1) { - const c = value[idx]; - value[idx - diff] = c; - - if (c == self.config.quote) { - diff += 1; - idx += 1; - } + const field = quoted_field.?; + if (!field.contains_quotes) { + return CsvToken{ .field = field.value }; + } else { + // walk the field and remove double quotes by shifting bytes + const value = field.value; + var diff: u64 = 0; + var idx: usize = 0; + while (idx < value.len) : (idx += 1) { + const c = value[idx]; + value[idx - diff] = c; + + if (c == self.config.quote) { + diff += 1; + idx += 1; } - - return CsvToken{ .field = value[0 .. value.len - diff] }; } + + return CsvToken{ .field = value[0 .. value.len - diff] }; } } - }; -} + } +}; diff --git a/test/csv_tokenizer.zig b/test/csv_tokenizer.zig index 7b17371..b45d2e3 100644 --- a/test/csv_tokenizer.zig +++ b/test/csv_tokenizer.zig @@ -8,18 +8,15 @@ const testing = std.testing; const expect = testing.expect; const csv_mod = @import("csv"); -var default_buffer = [_]u8{0} ** 1024; - -fn getTokenizer(file: std.fs.File, buffer: []u8, config: csv_mod.CsvConfig) !csv_mod.CsvTokenizer(std.fs.File.Reader) { - const reader = file.reader(); - const csv = try csv_mod.CsvTokenizer(std.fs.File.Reader).init(reader, buffer, config); +fn getTokenizer(reader: *std.Io.Reader, buffer: []u8, config: csv_mod.CsvConfig) !csv_mod.CsvTokenizer { + const csv = try csv_mod.CsvTokenizer.init(reader, buffer, config); return csv; } fn expectToken(comptime expected: csv_mod.CsvToken, maybe_actual: ?csv_mod.CsvToken) !void { if (maybe_actual) |actual| { if (@intFromEnum(expected) != @intFromEnum(actual)) { - std.log.warn("Expected {?} but is {?}\n", .{ expected, actual }); + std.log.warn("Expected {t} but is {t}\n", .{ expected, actual }); return error.TestFailed; } @@ -30,22 +27,24 @@ fn expectToken(comptime expected: csv_mod.CsvToken, maybe_actual: ?csv_mod.CsvTo else => {}, } } else { - std.log.warn("Expected {?} but is {?}\n", .{ expected, maybe_actual }); + std.log.warn("Expected {t} but is {?t}\n", .{ expected, maybe_actual }); return error.TestFailed; } } test "Create iterator for file reader" { - const file = try std.fs.cwd().openFile("test/resources/test-1.csv", .{}); - defer file.close(); + const data = @embedFile("resources/test-1.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; - _ = try getTokenizer(file, &default_buffer, .{}); + _ = try getTokenizer(&reader, &csv_buf, .{}); } test "Read single simple record from file" { - const file = try std.fs.cwd().openFile("test/resources/test-1.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, &default_buffer, .{}); + const data = @embedFile("resources/test-1.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "abc" }, try csv.next()); @@ -57,9 +56,10 @@ test "Read single simple record from file" { } test "Read multiple simple records from file" { - const file = try std.fs.cwd().openFile("test/resources/test-2.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, &default_buffer, .{}); + const data = @embedFile("resources/test-2.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "abc" }, try csv.next()); @@ -75,9 +75,10 @@ test "Read multiple simple records from file" { } test "Read quoted fields" { - const file = try std.fs.cwd().openFile("test/resources/test-4.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, &default_buffer, .{}); + const data = @embedFile("resources/test-4.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); try expectToken(csv_mod.CsvToken{ .field = "1" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "def ghc" }, try csv.next()); @@ -93,9 +94,10 @@ test "Read quoted fields" { } test "Second read is necessary to obtain field" { - const file = try std.fs.cwd().openFile("test/resources/test-read-required-for-field.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, default_buffer[0..6], .{}); + const data = @embedFile("resources/test-read-required-for-field.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); try expectToken(csv_mod.CsvToken{ .field = "12345" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .field = "67890" }, try csv.next()); @@ -107,9 +109,10 @@ test "Second read is necessary to obtain field" { } test "File is empty" { - const file = try std.fs.cwd().openFile("test/resources/test-empty.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, &default_buffer, .{}); + const data = @embedFile("resources/test-empty.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); const next = csv.next() catch unreachable; @@ -117,36 +120,40 @@ test "File is empty" { } test "Field is longer than buffer" { - const file = try std.fs.cwd().openFile("test/resources/test-error-short-buffer.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, default_buffer[0..9], .{}); + const data = @embedFile("resources/test-error-short-buffer.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [8]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); const next = csv.next(); try std.testing.expectError(csv_mod.CsvError.ShortBuffer, next); } test "Quoted field is longer than buffer" { - const file = try std.fs.cwd().openFile("test/resources/test-error-short-buffer-quoted.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, default_buffer[0..10], .{}); + const data = @embedFile("resources/test-error-short-buffer-quoted.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [8]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); const next = csv.next(); try std.testing.expectError(csv_mod.CsvError.ShortBuffer, next); } test "Quoted field with double quotes is longer than buffer" { - const file = try std.fs.cwd().openFile("test/resources/test-error-short-buffer-quoted-with-double.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, default_buffer[0..11], .{}); + const data = @embedFile("resources/test-error-short-buffer-quoted-with-double.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [8]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); const next = csv.next(); try std.testing.expectError(csv_mod.CsvError.ShortBuffer, next); } test "Quoted field with double quotes can be read on retry" { - const file = try std.fs.cwd().openFile("test/resources/test-error-short-buffer-quoted-with-double.csv", .{}); - defer file.close(); - var csv = try getTokenizer(file, default_buffer[0..14], .{}); + const data = @embedFile("resources/test-error-short-buffer-quoted-with-double.csv"); + var reader = std.Io.Reader.fixed(data); + var csv_buf: [1024]u8 = undefined; + var csv = try getTokenizer(&reader, &csv_buf, .{}); try expectToken(csv_mod.CsvToken{ .field = "1234567890\"" }, try csv.next()); try expectToken(csv_mod.CsvToken{ .row_end = {} }, try csv.next());