const std = @import("std"); const tools = @import("../tools.zig"); const schema = @import("schema.zig"); pub const definition = .{ .name = "read", .description = \\Read files. Each file is returned as a block: a line, then one line per source line formatted LINE| text, then \\. start and end are 1-based inclusive line numbers. \\Output is capped at 2000 lines or 50 KB per call and then names the line to \\continue from and how many of the next files were not read. A line that \\cannot fit in one call is cut. A missing, unreadable or binary file gives \\ and the other files are still returned. , .Args = Args, .execute = execute, }; pub const FileSpec = struct { path: schema.String(.{ .description = "File path, relative to the working directory or absolute." }), start: ?schema.Integer(.{ .description = "First line, 1-based and inclusive.", .min = 1 }) = null, end: ?schema.Integer(.{ .description = "Last line, 1-based and inclusive.", .min = 1 }) = null, }; pub const Args = struct { files: schema.Array(FileSpec, .{ .description = "Files to read.", .min_items = 1 }), }; pub const Details = struct { files: usize, lines: usize, }; pub const Output = struct { /// The text the model sees. text: []const u8, details: Details, }; pub fn execute(ctx: tools.Context, arena: std.mem.Allocator, args: Args) !Output { const cwd = std.Io.Dir.cwd(); const buffer = try arena.alloc(u8, tools.max_output_bytes); var out: std.ArrayList(u8) = .empty; var lines: usize = 0; var files_read: usize = 0; var capped: ?struct { path: []const u8, line: i64, cut: ?i64 = null, more: bool = true } = null; var unread: usize = 0; for (args.files.items, 0..) |spec, index| { if (capped != null) { unread = args.files.items.len - index; break; } const path = try escape(arena, spec.path.value); const first: i64 = if (spec.start) |start| start.value else 1; const last: i64 = if (spec.end) |end| end.value else std.math.maxInt(i64); if (first > last) { try out.print(arena, "\n", .{ path, first, last }); continue; } // Opening a FIFO for reading blocks until it has a writer. const stat = cwd.statFile(ctx.io, spec.path.value, .{}) catch |err| { try out.print(arena, "\n", .{ path, @errorName(err) }); continue; }; if (stat.kind != .file) { try out.print(arena, "\n", .{path}); continue; } const file = cwd.openFile(ctx.io, spec.path.value, .{}) catch |err| { try out.print(arena, "\n", .{ path, @errorName(err) }); continue; }; defer file.close(ctx.io); var reader = file.reader(ctx.io, buffer); const in = &reader.interface; in.fill(8192) catch {}; if (reader.err == null and std.mem.indexOfScalar(u8, in.buffered()[0..@min(8192, in.bufferedLen())], 0) != null) { try out.print(arena, "\n", .{path}); continue; } var body: std.ArrayList(u8) = .empty; var file_lines: usize = 0; var last_emitted: i64 = first; var line: i64 = 1; var ended = false; while (line <= last) : (line += 1) { const next = (peekLine(in) catch break) orelse { ended = true; break; }; if (line >= first) { const text = next.text; const room = reserve(path, first, line, args.files.items.len); const used = out.items.len + body.items.len + room; const full = lines == tools.max_output_lines or used + text.len > tools.max_output_bytes; var end = text.len; if (full) { // Cut only a line that would not fit in a call of its own. const alone = text.len + room <= tools.max_output_bytes; end = tools.max_output_bytes -| used; if (lines > 0 or alone or end < 1024) { capped = .{ .path = path, .line = line }; break; } for (0..3) |_| { if (text[end] & 0xC0 != 0x80) { break; } end -= 1; } } try body.print(arena, "{d}| {s}\n", .{ line, text[0..end] }); lines += 1; file_lines += 1; last_emitted = line; if (full) { takeLine(in, next) catch {}; const more = line < last and if (in.peekByte()) |_| true else |_| false; capped = .{ .path = path, .line = line + 1, .cut = line, .more = more }; break; } } takeLine(in, next) catch break; } if (reader.err) |err| { try out.print(arena, "\n", .{ path, @errorName(err) }); continue; } if (file_lines == 0 and capped != null) { unread = args.files.items.len - index - 1; break; } if (file_lines == 0 and ended and first > 1) { try out.print(arena, "\n", .{ path, first, line - 1 }); continue; } files_read += 1; try out.print(arena, "\n{s}\n", .{ path, first, last_emitted, body.items }); } if (capped) |cap| { try out.print(arena, "[output capped in {s}", .{cap.path}); if (cap.cut) |cut| { try out.print(arena, "; line {d} was cut", .{cut}); } if (cap.more) { try out.print(arena, "; continue with start={d}", .{cap.line}); } if (unread > 0) { try out.print(arena, "; the next {d} file(s) were not read", .{unread}); } try out.appendSlice(arena, "]\n"); } return .{ .text = out.items, .details = .{ .files = files_read, .lines = lines }, }; } /// Room for one line prefix, the file tags and the cap hint. fn reserve(path: []const u8, first: i64, line: i64, files: usize) usize { return std.fmt.count("{d}| \n\n\n" ++ "[output capped in {s}; line {d} was cut; continue with start={d}; the next {d} file(s) were not read]\n", .{ line, path, first, line, path, line, line + 1, files }); } /// The path with the characters that end or break an attribute value replaced by XML entities. fn escape(arena: std.mem.Allocator, path: []const u8) ![]const u8 { if (std.mem.indexOfAny(u8, path, "&\"<>\n") == null) { return path; } var out: std.ArrayList(u8) = .empty; for (path) |byte| try out.appendSlice(arena, switch (byte) { '&' => "&", '"' => """, '<' => "<", '>' => ">", '\n' => " ", else => &.{byte}, }); return out.items; } const Line = struct { /// The line without its newline, or as much of it as the buffer holds. text: []const u8, newline: bool, }; /// The next line, without consuming it; null at the end of the file. fn peekLine(in: *std.Io.Reader) error{ReadFailed}!?Line { if (in.peekDelimiterInclusive('\n')) |line| { return .{ .text = line[0 .. line.len - 1], .newline = true }; } else |err| switch (err) { error.EndOfStream, error.StreamTooLong => {}, error.ReadFailed => return error.ReadFailed, } const rest = in.buffered(); return if (rest.len == 0) null else .{ .text = rest, .newline = false }; } /// Consumes a line from `peekLine`, newline included. fn takeLine(in: *std.Io.Reader, line: Line) error{ReadFailed}!void { in.toss(line.text.len + @intFromBool(line.newline)); if (line.newline) { return; } _ = try in.discardDelimiterExclusive('\n'); in.toss(@intFromBool(in.bufferedLen() > 0)); } const testing = std.testing; test "read returns one block with line numbers and shows it expanded" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("hello.txt", "one\ntwo\nthree\n"); const result = dir.call("read", .{ .files = &.{.{ .path = path }} }); try testing.expect(!result.is_error); const expected = try std.fmt.allocPrint( dir.arena(), "\n" ++ "1| one\n" ++ "2| two\n" ++ "3| three\n" ++ "\n", .{path}, ); try testing.expectEqualStrings(expected, result.text); try testing.expectEqualStrings(expected, try tools.content.plain(dir.arena(), result.expanded)); try testing.expectEqualStrings("1 file(s), 3 lines", try tools.content.plain(dir.arena(), result.collapsed)); } test "read returns a line range" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("f.txt", "a\nbb\nccc\ndddd\neeeee\n"); const result = dir.call("read", .{ .files = &.{.{ .path = path, .start = @as(u32, 2), .end = @as(u32, 3) }} }); try testing.expect(!result.is_error); const expected = try std.fmt.allocPrint( dir.arena(), "\n" ++ "2| bb\n" ++ "3| ccc\n" ++ "\n", .{path}, ); try testing.expectEqualStrings(expected, result.text); } test "read returns several files in one call" { var dir = try tools.TestDir.init(); defer dir.deinit(); const first = try dir.write("x.txt", "alpha\nbeta\n"); const second = try dir.write("y.txt", "gamma\n"); const result = dir.call("read", .{ .files = &.{ .{ .path = first }, .{ .path = second } } }); try testing.expect(!result.is_error); const expected = try std.fmt.allocPrint( dir.arena(), "\n" ++ "1| alpha\n" ++ "2| beta\n" ++ "\n" ++ "\n" ++ "1| gamma\n" ++ "\n", .{ first, second }, ); try testing.expectEqualStrings(expected, result.text); try testing.expectEqualStrings("2 file(s), 3 lines", try tools.content.plain(dir.arena(), result.collapsed)); } test "read returns a last line without a newline" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("f.txt", "hello\nworld"); const result = dir.call("read", .{ .files = &.{.{ .path = path }} }); try testing.expect(!result.is_error); const expected = try std.fmt.allocPrint( dir.arena(), "\n" ++ "1| hello\n" ++ "2| world\n" ++ "\n", .{path}, ); try testing.expectEqualStrings(expected, result.text); } test "read cap names the line to continue from" { var dir = try tools.TestDir.init(); defer dir.deinit(); var content: std.ArrayList(u8) = .empty; for (1..2501) |i| { try content.appendSlice(dir.arena(), try std.fmt.allocPrint(dir.arena(), "L{d}\n", .{i})); } const path = try dir.write("big.txt", content.items); const capped = dir.call("read", .{ .files = &.{.{ .path = path }} }); try testing.expect(!capped.is_error); const header = try std.fmt.allocPrint(dir.arena(), "\n1| L1\n", .{path}); try testing.expect(std.mem.startsWith(u8, capped.text, header)); const hint = try std.fmt.allocPrint(dir.arena(), "[output capped in {s}; continue with start=2001]\n", .{path}); try testing.expect(std.mem.endsWith(u8, capped.text, hint)); const rest = dir.call("read", .{ .files = &.{.{ .path = path, .start = @as(u32, 2001) }} }); try testing.expect(!rest.is_error); const rest_header = try std.fmt.allocPrint(dir.arena(), "\n", .{path}); try testing.expect(std.mem.startsWith(u8, rest.text, rest_header)); try testing.expect(std.mem.indexOf(u8, rest.text, "[output capped") == null); try testing.expect(std.mem.endsWith(u8, rest.text, "\n\n")); } test "read cuts a line longer than the output cap at a character boundary" { var dir = try tools.TestDir.init(); defer dir.deinit(); // One ASCII byte first, so the cut can land inside a two-byte character. const long = try dir.arena().alloc(u8, 2 * tools.max_output_bytes + 1); long[0] = 'x'; for (0..tools.max_output_bytes) |i| long[1 + 2 * i ..][0..2].* = "\u{e9}".*; const path = try dir.write("long.txt", try std.mem.concat(dir.arena(), u8, &.{ "a\n", long, "\nshort\n" })); const result = dir.call("read", .{ .files = &.{.{ .path = path, .start = @as(u32, 2) }} }); try testing.expect(!result.is_error); try testing.expect(result.text.len <= tools.max_output_bytes); const header = try std.fmt.allocPrint(dir.arena(), "\n2| x", .{path}); try testing.expectStringStartsWith(result.text, header); const cut_line = result.text[header.len..std.mem.indexOf(u8, result.text, "\n").?]; try testing.expect(std.unicode.utf8ValidateSlice(cut_line)); const hint = try std.fmt.allocPrint(dir.arena(), "[output capped in {s}; line 2 was cut; continue with start=3]\n", .{path}); try testing.expectStringEndsWith(result.text, hint); } test "a cut last line ends the output without a line to continue from" { var dir = try tools.TestDir.init(); defer dir.deinit(); const long = try dir.arena().alloc(u8, 2 * tools.max_output_bytes); @memset(long, 'x'); const path = try dir.write("long.txt", long); const result = dir.call("read", .{ .files = &.{.{ .path = path }} }); try testing.expect(!result.is_error); const hint = try std.fmt.allocPrint(dir.arena(), "[output capped in {s}; line 1 was cut]\n", .{path}); try testing.expectStringEndsWith(result.text, hint); } test "a capped call says how many files it did not read" { var dir = try tools.TestDir.init(); defer dir.deinit(); const long = try dir.arena().alloc(u8, 2 * tools.max_output_bytes); @memset(long, 'x'); const first = try dir.write("long.txt", long); const second = try dir.write("b.txt", "b\n"); const result = dir.call("read", .{ .files = &.{ .{ .path = first }, .{ .path = second }, .{ .path = second } } }); try testing.expect(!result.is_error); const hint = try std.fmt.allocPrint(dir.arena(), "[output capped in {s}; line 1 was cut; the next 2 file(s) were not read]\n", .{first}); try testing.expectStringEndsWith(result.text, hint); } test "a start after the end gives an error block" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("f.txt", "a\nb\nc\nd\n"); const result = dir.call("read", .{ .files = &.{.{ .path = path, .start = @as(u32, 3), .end = @as(u32, 2) }} }); try testing.expect(!result.is_error); try testing.expectEqualStrings(try std.fmt.allocPrint(dir.arena(), "\n", .{path}), result.text); } test "the output stays within the cap with long line numbers" { var dir = try tools.TestDir.init(); defer dir.deinit(); const empty: [1_000_000]u8 = @splat('\n'); const long: [2 * tools.max_output_bytes]u8 = @splat('x'); const path = try dir.write("f.txt", &empty ++ &long ++ "\nx\n"); const result = dir.call("read", .{ .files = &.{ .{ .path = path, .start = @as(u32, 1_000_001) }, .{ .path = path } } }); try testing.expect(!result.is_error); try testing.expectStringEndsWith(result.text, "; line 1000001 was cut; continue with start=1000002; the next 1 file(s) were not read]\n"); try testing.expect(result.text.len <= tools.max_output_bytes); } test "read escapes the path in the block" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("a\"b&c.txt", "x\n"); const result = dir.call("read", .{ .files = &.{.{ .path = path }} }); try testing.expectStringStartsWith(result.text, try std.fmt.allocPrint(dir.arena(), "\n\n", .{ fifo, dir.root }); try testing.expectEqualStrings(expected, result.text); } test "read returns lines past the end of a file larger than the edit limit" { var dir = try tools.TestDir.init(); defer dir.deinit(); const row: [1023]u8 = @splat('r'); const rows = tools.max_file_bytes / 1024 + 2; const file = try dir.tmp.dir.createFile(testing.io, "big.txt", .{}); var buffer: [64 * 1024]u8 = undefined; var writer = file.writer(testing.io, &buffer); for (0..rows) |_| try writer.interface.writeAll(&row ++ "\n"); try writer.interface.flush(); file.close(testing.io); const path = try dir.path("big.txt"); const result = dir.call("read", .{ .files = &.{.{ .path = path, .start = @as(u32, rows) }} }); const expected = try std.fmt.allocPrint(dir.arena(), "\n{d}| {s}\n\n", .{ path, rows, rows, rows, &row }); try testing.expectEqualStrings(expected, result.text); } test "the output stays within the cap when the path is long" { var dir = try tools.TestDir.init(); defer dir.deinit(); const name: [200]u8 = @splat('d'); const row: [99]u8 = @splat('y'); var content: std.ArrayList(u8) = .empty; for (0..1000) |_| try content.appendSlice(dir.arena(), &row ++ "\n"); const path = try dir.write(&name ++ "/" ++ &name ++ "/" ++ &name ++ "/f.txt", content.items); const result = dir.call("read", .{ .files = &.{ .{ .path = path }, .{ .path = path, .start = @as(u32, 500) } } }); try testing.expect(!result.is_error); try testing.expect(std.mem.indexOf(u8, result.text, "[output capped in ") != null); try testing.expect(result.text.len <= tools.max_output_bytes); } test "read reports a start past the end of the file" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("f.txt", "a\nb\nc\n"); const result = dir.call("read", .{ .files = &.{.{ .path = path, .start = @as(u32, 4) }} }); try testing.expect(!result.is_error); const block = try std.fmt.allocPrint(dir.arena(), "\n", .{path}); try testing.expectEqualStrings(block, result.text); try testing.expectEqualStrings("0 file(s), 0 lines", try tools.content.plain(dir.arena(), result.collapsed)); } test "read keeps good files when one file is missing" { var dir = try tools.TestDir.init(); defer dir.deinit(); const good = try dir.write("good.txt", "ok\n"); const missing = try dir.path("missing.txt"); const result = dir.call("read", .{ .files = &.{ .{ .path = missing }, .{ .path = good } } }); try testing.expect(!result.is_error); const error_block = try std.fmt.allocPrint(dir.arena(), "\n", .{missing}); try testing.expectStringStartsWith(result.text, error_block); const good_block = try std.fmt.allocPrint(dir.arena(), "\n1| ok\n\n", .{good}); try testing.expect(std.mem.indexOf(u8, result.text, good_block) != null); try testing.expectEqualStrings("1 file(s), 1 lines", try tools.content.plain(dir.arena(), result.collapsed)); } test "read reports a binary file" { var dir = try tools.TestDir.init(); defer dir.deinit(); const path = try dir.write("bin.dat", "abc\x00def\n"); const result = dir.call("read", .{ .files = &.{.{ .path = path }} }); const block = try std.fmt.allocPrint(dir.arena(), "\n", .{path}); try testing.expect(std.mem.startsWith(u8, result.text, block)); } /// The paths with their line ranges. pub fn renderCall(arena: std.mem.Allocator, args: Args) std.mem.Allocator.Error!tools.content.Content { var out: std.ArrayList(u8) = .empty; for (args.files.items, 0..) |file, index| { if (index > 0) { try out.appendSlice(arena, ", "); } try out.appendSlice(arena, file.path.value); if (file.start) |start| { try out.print(arena, ":{d}", .{start.value}); if (file.end) |end| { try out.print(arena, "-{d}", .{end.value}); } } } return tools.content.callLine(arena, definition.name, out.items); } /// How much was read. pub fn renderResult(arena: std.mem.Allocator, output: Output) std.mem.Allocator.Error!tools.Rendered { const line = try std.fmt.allocPrint(arena, "{d} file(s), {d} lines", .{ output.details.files, output.details.lines }); return .{ .collapsed = try tools.content.text(arena, line), .expanded = try tools.content.text(arena, output.text), }; }