//! Preamble, header and alias resolution (M1-CSV-008, 009, 011..014, 023). //! //! Sits on the record stream from reader.zig and decides which records are //! data, how wide they must be, and which column each alias names. //! //! The width rules differ by mode and the difference is deliberate. In header //! mode a data row must match the header exactly, because a differing width //! means the file is not what the profile describes (M1-CSV-013). In //! headerless mode a row need only reach the highest aliased index, and //! anything past it is ignored (M1-CSV-014) -- there is no header to disagree //! with, so trailing columns are simply not this profile's business. const std = @import("std"); const diag = @import("../diag.zig"); const Lexer = @import("../expr/lexer.zig").Lexer; const reader = @import("reader.zig"); const template = @import("../template.zig"); pub const Error = error{ InvalidCsv, InvalidConfig, OutOfMemory }; /// M1-CSV-011: exactly one of these selects a column. pub const Selector = union(enum) { header: []const u8, index: usize, }; pub const FieldSpec = struct { alias: []const u8, selector: Selector, }; pub const Options = struct { fields: []const FieldSpec, /// M1-CSV-009. header: bool, reader: reader.Options, /// M1-CSV-008. skip_records: usize, }; /// Where one alias landed in a particular file, with how the profile asked /// for it, so the interface can say both. pub const Column = struct { field: FieldSpec, index: usize, }; pub const Layout = struct { /// Resolved columns in profile declaration order. columns: []const Column, /// The exact width a data row must have in header mode (M1-CSV-013); /// null when headerless. header_width: ?usize, /// One past the highest aliased index (M1-CSV-014). minimum_width: usize, pub fn indexOf(self: Layout, alias: []const u8) ?usize { for (self.columns) |column| { if (std.mem.eql(u8, column.field.alias, alias)) return column.index; } return null; } }; pub const DataRow = struct { /// Every cell of the record, including ones no alias selects: the /// interface searches them all (M1-UI-006). cells: []const []const u8, physical_row: usize, pub fn value(self: DataRow, index: usize) []const u8 { return self.cells[index]; } }; pub const Rows = struct { arena: std.mem.Allocator, diags: *diag.List, file: []const u8, header_mode: bool, layout: Layout, records: reader.Reader, pub fn init( arena: std.mem.Allocator, file: []const u8, source: []const u8, options: Options, diags: *diag.List, ) Error!Rows { try validateAliases(arena, file, options.fields, diags); var records = try reader.Reader.init(arena, file, source, options.reader, diags); // M1-CSV-008: the preamble is discarded, but the reader has already // counted it, so physical rows stay aligned with the file. var skipped: usize = 0; while (skipped < options.skip_records) : (skipped += 1) { if ((try records.next()) == null) { try diags.add(.{ .message = "file ends before the configured preamble does", .file = file, }); return error.InvalidCsv; } } const layout = if (options.header) try resolveWithHeader(arena, file, &records, options.fields, diags) else try resolveHeaderless(arena, file, options.fields, diags); return .{ .arena = arena, .file = file, .diags = diags, .header_mode = options.header, .records = records, .layout = layout, }; } pub fn next(self: *Rows) Error!?DataRow { const record = (try self.records.next()) orelse return null; if (self.layout.header_width) |width| { // M1-CSV-013: exactly the header's width, neither more nor less. if (record.fields.len != width) { try self.diags.addRow( self.file, record.physical_row, "row has {d} fields but the header has {d}", .{ record.fields.len, width }, ); return error.InvalidCsv; } } else if (record.fields.len < self.layout.minimum_width) { // M1-CSV-014: must reach the highest aliased index. try self.diags.addRow( self.file, record.physical_row, "row has {d} fields but an alias selects index {d}", .{ record.fields.len, self.layout.minimum_width - 1 }, ); return error.InvalidCsv; } return .{ .physical_row = record.physical_row, .cells = record.fields }; } }; /// M1-CSV-011 and M1-RULE-014. fn validateAliases( arena: std.mem.Allocator, file: []const u8, fields: []const FieldSpec, diags: *diag.List, ) Error!void { if (fields.len == 0) { try reportConfig(diags, file, "profile.csv.fields", "at least one field is required", .{}); return error.InvalidConfig; } for (fields, 0..) |field, position| { const path = try arena.print("profile.csv.fields.{s}", .{field.alias}); if (!Lexer.isIdentifier(field.alias)) { try reportConfig(diags, file, path, "'{s}' is not a valid alias", .{field.alias}); return error.InvalidConfig; } // M1-RULE-014: the expression language reserves these, so a field // named after one could never be referenced. if (Lexer.isKeyword(field.alias)) { try reportConfig(diags, file, path, "'{s}' is a reserved keyword", .{field.alias}); return error.InvalidConfig; } // M1-TPL-006: likewise a template built-in, which would otherwise be // shadowed and unreachable. if (template.isReserved(field.alias)) { try reportConfig(diags, file, path, "'{s}' is reserved for a template placeholder", .{field.alias}); return error.InvalidConfig; } for (fields[0..position]) |earlier| { if (std.mem.eql(u8, earlier.alias, field.alias)) { try reportConfig(diags, file, path, "alias '{s}' is declared twice", .{field.alias}); return error.InvalidConfig; } } } } fn resolveWithHeader( arena: std.mem.Allocator, file: []const u8, records: *reader.Reader, fields: []const FieldSpec, diags: *diag.List, ) Error!Layout { // M1-CSV-009: the header must be there. const header = (try records.next()) orelse { try diags.add(.{ .message = "header mode is configured but the file has no header record", .file = file }); return error.InvalidCsv; }; // M1-CSV-013: uniqueness covers every cell, including ones no alias // selects, because a duplicate makes the file ambiguous whether or not // this profile happens to look at it. for (header.fields, 0..) |cell, position| { for (header.fields[0..position]) |earlier| { if (std.mem.eql(u8, earlier, cell)) { try diags.addRow(file, header.physical_row, "header cell '{s}' appears more than once", .{cell}); return error.InvalidCsv; } } } const columns = try arena.alloc(Column, fields.len); var highest: usize = 0; for (fields, 0..) |field, position| { const index = switch (field.selector) { .header => |name| findHeader(header.fields, name) orelse { try diags.addRow(file, header.physical_row, "no header cell matches '{s}'", .{name}); return error.InvalidCsv; }, // M1-CSV-012: an index selector is allowed here too, but must land // inside the header. .index => |index| blk: { if (index >= header.fields.len) { try diags.addRow( file, header.physical_row, "alias '{s}' selects index {d} but the header has {d} columns", .{ field.alias, index, header.fields.len }, ); return error.InvalidCsv; } break :blk index; }, }; columns[position] = .{ .field = field, .index = index }; highest = @max(highest, index); } return .{ .columns = columns, .header_width = header.fields.len, .minimum_width = highest + 1, }; } fn resolveHeaderless( arena: std.mem.Allocator, file: []const u8, fields: []const FieldSpec, diags: *diag.List, ) Error!Layout { const columns = try arena.alloc(Column, fields.len); var highest: usize = 0; for (fields, 0..) |field, position| { const index = switch (field.selector) { // M1-CSV-012: there is no header to match against. .header => |name| { const path = try arena.print("profile.csv.fields.{s}", .{field.alias}); try reportConfig( diags, file, path, "header selector '{s}' needs header mode", .{name}, ); return error.InvalidConfig; }, .index => |index| index, }; columns[position] = .{ .field = field, .index = index }; highest = @max(highest, index); } return .{ .columns = columns, .header_width = null, .minimum_width = highest + 1 }; } /// M1-CSV-012: the exact decoded cell, so case and spacing matter. fn findHeader(cells: []const []const u8, name: []const u8) ?usize { for (cells, 0..) |cell, index| { if (std.mem.eql(u8, cell, name)) return index; } return null; } fn reportConfig( diags: *diag.List, file: []const u8, path: []const u8, comptime fmt: []const u8, args: anytype, ) Error!void { try diags.addConfig(.{ .file = file, .key_path = path }, fmt, args); } // -- tests ------------------------------------------------------------------- const expect = std.testing.expect; const expectEqual = std.testing.expectEqual; const expectEqualStrings = std.testing.expectEqualStrings; const expectError = std.testing.expectError; fn optionsFor(skip: usize, header: bool, fields: []const FieldSpec) Options { return .{ .reader = .{ .delimiter = .comma, .line_ending = .lf }, .skip_records = skip, .header = header, .fields = fields, }; } fn openRows( arena: *std.heap.ArenaAllocator, source: []const u8, options: Options, ) Error!Rows { const alloc = arena.allocator(); const diags = diag.List.create(alloc) catch return error.OutOfMemory; return Rows.init(alloc, "bank.csv", source, options, diags); } fn collect( arena: *std.heap.ArenaAllocator, source: []const u8, options: Options, ) Error![]const DataRow { const alloc = arena.allocator(); var rows = try openRows(arena, source, options); var list: std.ArrayList(DataRow) = .empty; while (try rows.next()) |row| try list.append(alloc, row); return list.items; } const by_index: []const FieldSpec = &.{ .{ .alias = "date", .selector = .{ .index = 0 } }, .{ .alias = "amount", .selector = .{ .index = 1 } }, }; const by_header: []const FieldSpec = &.{ .{ .alias = "date", .selector = .{ .header = "Datum" } }, .{ .alias = "amount", .selector = .{ .header = "Betrag" } }, }; test "M1-CSV-008: skipped records are discarded but still counted" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "junk\nmore junk\na,1\nb,2\n", optionsFor(2, false, by_index)); try expectEqual(@as(usize, 2), rows.len); // Physical rows keep counting through the preamble (M1-TPL-004). try expectEqual(@as(usize, 2), rows[0].physical_row); try expectEqual(@as(usize, 3), rows[1].physical_row); } test "M1-CSV-009: in header mode the record after the preamble is the header" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "junk\nDatum,Betrag\na,1\n", optionsFor(1, true, by_header)); try expectEqual(@as(usize, 1), rows.len); try expectEqual(@as(usize, 2), rows[0].physical_row); try expectEqualStrings("a", rows[0].cells[0]); } test "M1-CSV-009: in headerless mode every record after the preamble is data" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "a,1\nb,2\n", optionsFor(0, false, by_index)); try expectEqual(@as(usize, 2), rows.len); try expectEqual(@as(usize, 0), rows[0].physical_row); } test "M1-CSV-009: header mode requires the header record to exist" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, openRows(&arena, "junk\n", optionsFor(1, true, by_header))); try expectError(error.InvalidCsv, openRows(&arena, "", optionsFor(0, true, by_header))); } test "M1-CSV-023: zero data rows is valid" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const with_header = try collect(&arena, "Datum,Betrag\n", optionsFor(0, true, by_header)); try expectEqual(@as(usize, 0), with_header.len); const headerless = try collect(&arena, "", optionsFor(0, false, by_index)); try expectEqual(@as(usize, 0), headerless.len); } test "M1-CSV-011: the fields mapping must not be empty" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidConfig, openRows(&arena, "a,1\n", optionsFor(0, false, &.{}))); } test "M1-CSV-011: an alias must use the identifier grammar" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); for ([_][]const u8{ "", "1st", "a-b", "a b", "wä" }) |bad| { const fields: []const FieldSpec = &.{.{ .alias = bad, .selector = .{ .index = 0 } }}; try expectError(error.InvalidConfig, openRows(&arena, "a\n", optionsFor(0, false, fields))); } } test "M1-RULE-014: a keyword may not be a field alias" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); for ([_][]const u8{ "and", "or", "not" }) |keyword| { const fields: []const FieldSpec = &.{.{ .alias = keyword, .selector = .{ .index = 0 } }}; try expectError(error.InvalidConfig, openRows(&arena, "a\n", optionsFor(0, false, fields))); } // One merely containing a keyword is fine. const accepted: []const FieldSpec = &.{.{ .alias = "android", .selector = .{ .index = 0 } }}; _ = try openRows(&arena, "a\n", optionsFor(0, false, accepted)); } test "M1-AC-015: an alias named after a template built-in is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); for ([_][]const u8{ "sourceFilename", "physicalRow", "ruleName" }) |reserved| { const spec: []const FieldSpec = &.{.{ .alias = reserved, .selector = .{ .index = 0 } }}; try expectError(error.InvalidConfig, openRows(&arena, "a\n", optionsFor(0, false, spec))); } } test "M1-CSV-011: duplicate aliases are rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const fields: []const FieldSpec = &.{ .{ .alias = "a", .selector = .{ .index = 0 } }, .{ .alias = "a", .selector = .{ .index = 1 } }, }; try expectError(error.InvalidConfig, openRows(&arena, "x,y\n", optionsFor(0, false, fields))); } test "M1-CSV-012: an index selector works in either mode" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const headerless = try collect(&arena, "a,1\n", optionsFor(0, false, by_index)); try expectEqualStrings("1", headerless[0].value(1)); var rows = try openRows(&arena, "Datum,Betrag\na,1\n", optionsFor(0, true, by_index)); const row = (try rows.next()).?; try expectEqualStrings("1", row.value(rows.layout.indexOf("amount").?)); } test "M1-CSV-012: a header selector is rejected in headerless mode" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidConfig, openRows(&arena, "a,1\n", optionsFor(0, false, by_header))); } test "M1-CSV-012: a header selector matches the exact decoded cell" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Case and surrounding space are part of the cell. try expectError(error.InvalidCsv, openRows(&arena, "datum,Betrag\na,1\n", optionsFor(0, true, by_header))); try expectError(error.InvalidCsv, openRows(&arena, " Datum,Betrag\na,1\n", optionsFor(0, true, by_header))); } test "M1-CSV-012: a selected index must be within the header width" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const fields: []const FieldSpec = &.{.{ .alias = "far", .selector = .{ .index = 5 } }}; try expectError(error.InvalidCsv, openRows(&arena, "A,B\na,b\n", optionsFor(0, true, fields))); } test "M1-CSV-013: every header cell must be unique, selected or not" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // The duplicate is in columns no alias mentions, and still fails. const fields: []const FieldSpec = &.{.{ .alias = "date", .selector = .{ .header = "Datum" } }}; try expectError( error.InvalidCsv, openRows(&arena, "Datum,Extra,Extra\na,b,c\n", optionsFor(0, true, fields)), ); } test "M1-CSV-013: a header selector that matches nothing is rejected" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, openRows(&arena, "Datum,Other\na,1\n", optionsFor(0, true, by_header))); } test "M1-CSV-013: a data row must have exactly the header's width" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, collect(&arena, "Datum,Betrag\na\n", optionsFor(0, true, by_header))); try expectError(error.InvalidCsv, collect(&arena, "Datum,Betrag\na,1,extra\n", optionsFor(0, true, by_header))); } test "M1-CSV-014: headerless rows must reach the highest aliased index" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); try expectError(error.InvalidCsv, collect(&arena, "a\n", optionsFor(0, false, by_index))); } test "M1-CSV-014: headerless extra trailing fields are ignored" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const rows = try collect(&arena, "a,1,ignored,also\n", optionsFor(0, false, by_index)); try expectEqual(@as(usize, 1), rows.len); try expectEqualStrings("1", rows[0].value(1)); } test "M1-AC-019: the header and width scenarios end to end" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Duplicate header fails. try expectError(error.InvalidCsv, openRows(&arena, "A,A\n1,2\n", optionsFor(0, true, &.{ .{ .alias = "a", .selector = .{ .header = "A" } }, }))); // Width mismatch fails. try expectError(error.InvalidCsv, collect(&arena, "A,B\n1\n", optionsFor(0, true, &.{ .{ .alias = "a", .selector = .{ .header = "A" } }, }))); // Headerless: extras ignored, missing index fails. const rows = try collect(&arena, "1,2,3\n", optionsFor(0, false, &.{ .{ .alias = "a", .selector = .{ .index = 1 } }, })); try expectEqualStrings("2", rows[0].value(1)); try expectError(error.InvalidCsv, collect(&arena, "1\n", optionsFor(0, false, &.{ .{ .alias = "a", .selector = .{ .index = 1 } }, }))); } test "M1-DIAG-001: a width failure names the file and physical row" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); const diags = try diag.List.create(alloc); var rows = try Rows.init(alloc, "bank/2026.csv", "A,B\n1,2\n3\n", .{ .reader = .{ .delimiter = .comma, .line_ending = .lf }, .skip_records = 0, .header = true, .fields = &.{.{ .alias = "a", .selector = .{ .header = "A" } }}, }, diags); _ = rows.next() catch {}; _ = rows.next() catch {}; try expect(!diags.isEmpty()); const last = diags.items.items[diags.items.items.len - 1]; try expectEqualStrings("bank/2026.csv", last.file.?); try expectEqual(@as(usize, 2), last.physical_row.?); }