Something went wrong. Try again.
An HTTP library for Zig 0.17, server and client, HTTP/1.0 through HTTP/3, with the protocol in a module that owns no socket.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! Content codings, and choosing one the peer will accept.//!//! §8.4.1 of RFC 9110 names the codings and §12.5.3 says how a client asks for//! one. Both halves are here, and neither does any I/O: a content coding is a//! transformation on a byte stream, which in this standard library is a//! `Reader` over a `Reader` and a `Writer` over a `Writer`. That is why this//! sits in the sans-I/O module beside the framing rather than in a layer above//! it — and it is the reason a coding applies identically over HTTP/1.1,//! HTTP/2 and HTTP/3, none of which it knows about.//!//! A content coding is not a transfer coding. §8.4 makes it part of the//! *representation*: it travels end to end, a cache stores it coded, and//! `Content-Length` counts the coded octets. That is the whole reason a//! `Content-Encoding: gzip` request body reaches a handler still compressed,//! which the README argues for at more length — deciding to decode it is the//! handler's business, and this module is what it decodes with.
const std = @import("std");const Reader = std.Io.Reader;const Writer = std.Io.Writer;const flate = std.compress.flate;
const token = @import("token.zig");
/// The codings this library has a name for.////// The ones it cannot perform are named rather than left out, because a/// `Content-Encoding` that cannot be decoded has to be told apart from one/// nobody has heard of. The first is a representation to hand over as it/// arrived; the second is a message whose sender and recipient disagree about/// what it contains, which is worth refusing.pub const Coding = enum { /// §8.4.1.1: no coding at all, and the one that needs no asking for. identity, /// §8.4.1.3. gzip, /// §8.4.1.2. The zlib format of RFC 1950, which is *not* what most /// senders that say `deflate` actually send — a fair number send raw /// RFC 1951 — and telling those apart means sniffing the first octets. /// Named, not performed. deflate, /// §8.4.1.4's `compress`, which is LZW and long dead. compress, /// RFC 7932 Brotli. br, /// RFC 8878 Zstandard. zstd,
/// The coding's registered name, as it goes in `Content-Encoding` and /// `Accept-Encoding`. Always the current spelling, never `x-gzip`. pub fn name(c: Coding) []const u8 { return switch (c) { .identity => "identity", .gzip => "gzip", .deflate => "deflate", .compress => "compress", .br => "br", .zstd => "zstd", }; }
test name { try testing.expectEqualStrings("gzip", Coding.gzip.name()); try testing.expectEqualStrings("br", Coding.br.name()); }
/// §8.4.1: a coding name is a token and is matched case-insensitively. /// `x-gzip` and `x-compress` are the historical spellings §8.4.1 still /// lists as equivalents, so they are accepted rather than treated as /// codings nobody has heard of. Optional whitespace around the name is /// ignored; anything else unrecognized is null. pub fn parse(text: []const u8) ?Coding { const trimmed = token.trimOws(text); if (token.eqlIgnoreCase(trimmed, "identity")) return .identity; if (token.eqlIgnoreCase(trimmed, "gzip")) return .gzip; if (token.eqlIgnoreCase(trimmed, "x-gzip")) return .gzip; if (token.eqlIgnoreCase(trimmed, "deflate")) return .deflate; if (token.eqlIgnoreCase(trimmed, "compress")) return .compress; if (token.eqlIgnoreCase(trimmed, "x-compress")) return .compress; if (token.eqlIgnoreCase(trimmed, "br")) return .br; if (token.eqlIgnoreCase(trimmed, "zstd")) return .zstd; return null; }
test parse { try testing.expectEqual(Coding.gzip, Coding.parse("GZip").?); try testing.expectEqual(Coding.gzip, Coding.parse("x-gzip").?); try testing.expectEqual(Coding.zstd, Coding.parse(" zstd ").?); try testing.expect(Coding.parse("lzma") == null); }
/// Whether this library can undo it. pub fn canDecode(c: Coding) bool { return switch (c) { .identity, .gzip, .zstd => true, .deflate, .compress, .br => false, }; }
test canDecode { try testing.expect(Coding.gzip.canDecode()); try testing.expect(Coding.zstd.canDecode()); // Named, so it can be recognized, but not performed. try testing.expect(!Coding.br.canDecode()); }
/// Whether this library can apply it. /// /// Deliberately not the same set, and zstd is the reason the two had to be /// separated: `std.compress.zstd` ships a decoder and no encoder, so this /// library can read what a peer sends and cannot produce it. A client may /// therefore ask for zstd and undo the answer; a server may not offer it. /// One predicate meaning "supported" could only have been wrong about one /// of those. pub fn canEncode(c: Coding) bool { return switch (c) { .identity, .gzip => true, .deflate, .compress, .br, .zstd => false, }; }
test canEncode { try testing.expect(Coding.gzip.canEncode()); // Decodable, and still not something this library can produce. try testing.expect(!Coding.zstd.canEncode()); }};
/// A quality value in thousandths, which is exactly the precision §12.5.3/// allows: "a weight is a value between 0 and 1 with at most three digits/// after the decimal point". Integers because a preference compared with/// floating point is a preference that can compare unequal to itself.pub const Quality = u16;
/// The highest weight, and what an element with no weight at all carries.pub const highest: Quality = 1000;
/// §12.5.3's weight, or `highest` when the element carries none.////// A weight this cannot read is treated as absent rather than as zero. The/// grammar allows only `q=`, so anything else is a sender doing something the/// specification does not describe — and reading that as "not acceptable"/// would refuse a peer over a parameter nobody meant as a refusal, where/// reading it as "no preference stated" merely ignores it.fn weightOf(element: []const u8) Quality { var rest = element; while (std.mem.indexOfScalar(u8, rest, ';')) |at| { const parameter = token.trimOws(rest[at + 1 ..]); rest = parameter; const eq = std.mem.indexOfScalar(u8, parameter, '=') orelse continue; if (!token.eqlIgnoreCase(token.trimOws(parameter[0..eq]), "q")) continue; var value = token.trimOws(parameter[eq + 1 ..]); if (std.mem.indexOfScalar(u8, value, ';')) |end| value = token.trimOws(value[0..end]); return parseQuality(value) orelse highest; } return highest;}
/// `0`, `1`, `0.5`, `1.000` — and nothing else. §12.5.3's grammar is/// `( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] )`, so `1.5` is not a/// weight and neither is `.5`.fn parseQuality(text: []const u8) ?Quality { if (text.len == 0) return null; const whole = text[0]; if (whole != '0' and whole != '1') return null; var thousandths: Quality = if (whole == '1') highest else 0; if (text.len == 1) return thousandths; if (text[1] != '.') return null; const digits = text[2..]; if (digits.len > 3) return null; var scale: Quality = 100; for (digits) |d| { if (!std.ascii.isDigit(d)) return null; const n: Quality = d - '0'; if (whole == '1') { // §12.5.3 allows only zeroes after `1.`, and `1.001` is not a // weight that means anything above the maximum. if (n != 0) return null; } else { thousandths += n * scale; } scale /= 10; } return thousandths;}
/// The weight a list such as an `Accept-Encoding` field value gives `name`,/// or null when the list does not name it at all — which is not the same as/// a weight of zero, since an unnamed coding falls to `*` and a coding/// weighted zero is refused outright.////// Names are compared without regard to case, as §8.4.1 says content codings/// are, and `*` is a name like any other here: asking for it is how a caller/// learns what the wildcard says. Where a list names something twice, the/// last mention is the one that counts, as in `selectEncoding`.pub fn statedWeight(field_value: []const u8, name: []const u8) ?Quality { var stated: ?Quality = null; var it = token.list(field_value); while (it.next()) |element| { const before_weight = element[0 .. std.mem.indexOfScalar(u8, element, ';') orelse element.len]; if (token.eqlIgnoreCase(token.trimOws(before_weight), name)) stated = weightOf(element); } return stated;}
test statedWeight { const field = "GZIP;q=0.8, br, *;q=0"; try testing.expectEqual(@as(?Quality, 800), statedWeight(field, "gzip")); // No weight is the highest one. try testing.expectEqual(@as(?Quality, highest), statedWeight(field, "br")); try testing.expectEqual(@as(?Quality, 0), statedWeight(field, "*")); // Not named is not refused: that is for `*` to decide. try testing.expectEqual(@as(?Quality, null), statedWeight(field, "zstd"));}
/// Chooses a coding from what a peer said it would take.////// `offered` is in this end's order of preference, so a caller that would/// rather send gzip than nothing passes `&.{ .gzip, .identity }`. Ties in the/// peer's weights are broken by that order, which is the only thing that can/// break them: §12.5.3 gives a client a way to say what it prefers and no way/// to say what it is indifferent about.////// Null means the peer refused everything on offer, which §12.5.3 says a/// server "MAY" answer with a 406 — and which it can only mean deliberately,/// since identity is acceptable unless a client goes out of its way to refuse/// it.pub fn selectEncoding(field_value: ?[]const u8, offered: []const Coding) ?Coding { // §12.5.3: "If no Accept-Encoding header field is in the request, any // content coding is considered acceptable by the user agent." Acceptable // is not the same as wanted, and this returns what to *send*: a client // that did not ask gets identity, because a coding applied to a client // that never mentioned one is how a response becomes unreadable to // something in the middle that also never mentioned one. const list = field_value orelse return firstOf(offered, .identity);
// "An Accept-Encoding header field with a combined field value that is // empty implies that the user agent does not want any content coding in // response." if (token.trimOws(list).len == 0) return firstOf(offered, .identity);
var star: ?Quality = null; var best: ?Coding = null; var best_weight: Quality = 0;
for (offered) |candidate| { var stated: ?Quality = null;
var it = token.list(list); while (it.next()) |element| { const before_weight = element[0 .. std.mem.indexOfScalar(u8, element, ';') orelse element.len]; const name = token.trimOws(before_weight); if (std.mem.eql(u8, name, "*")) { if (star == null) star = weightOf(element); continue; } const coding = Coding.parse(name) orelse continue; if (coding != candidate) continue; stated = weightOf(element); }
// What the client did not name, the wildcard speaks for; and what // neither names is a coding it never asked for. // // Except identity, which §12.5.3 says "is always acceptable, unless // specifically refused". Acceptable is not preferred, and the // difference decides every ordinary request: a client sending // `gzip;q=0.9` and never mentioning identity plainly wants gzip, so // identity's unstated weight has to sit *below* anything the client // troubled to ask for rather than at the top. The lowest weight there // is, then — enough to be chosen when nothing else may be, and never // enough to beat a coding that was actually requested. const weight = stated orelse star orelse (if (candidate == .identity) @as(Quality, 1) else @as(Quality, 0));
if (weight > best_weight) { best = candidate; best_weight = weight; } } return best;}
/// The first of `offered` that is `wanted`, or null.fn firstOf(offered: []const Coding, wanted: Coding) ?Coding { for (offered) |c| if (c == wanted) return c; return null;}
/// Reads a `Content-Encoding` field and says what has to be undone.////// §8.4: "If one or more encodings have been applied to a representation, the/// sender that applied the encodings MUST generate a Content-Encoding header/// field that lists the content codings in the order in which they were/// applied." So undoing them means walking the list backwards, and the last/// element is the first thing to undo.////// The common case by a very long way is one coding, which is why this reports/// the shape rather than a list: `identity` for a field that is absent or says/// nothing, one `Coding` for a single coding, and `.unsupported` for anything/// this library cannot undo on its own — several codings included, since/// applying gzip twice is legal, vanishingly rare, and not worth a decoder/// stack that would exist for nobody.pub const Applied = union(enum) { /// Nothing to undo. identity, /// Exactly one coding, and this library can undo it. one: Coding, /// Something this library will not undo: a coding it cannot perform, or /// more than one of them. The representation is the caller's to hand over /// as it arrived or to refuse. unsupported,};
/// Reads a `Content-Encoding` field value, or null for a message that had/// none, into the `Applied` that says what undoing it takes.pub fn applied(field_value: ?[]const u8) Applied { const list = field_value orelse return .identity; var found: ?Coding = null; var it = token.list(list); while (it.next()) |element| { const name = token.trimOws(element); if (name.len == 0) continue; const coding = Coding.parse(name) orelse return .unsupported; // §8.4.1.1: `identity` in a `Content-Encoding` is a coding that does // nothing, so it neither counts as a coding to undo nor makes the // field unreadable. if (coding == .identity) continue; if (found != null) return .unsupported; if (!coding.canDecode()) return .unsupported; found = coding; } return if (found) |c| .{ .one = c } else .identity;}
// -- the codec itself -------------------------------------------------------//// std's flate already is what this library wants -- a `Reader` over a `Reader`// and a `Writer` over a `Writer` -- so it is re-exported rather than wrapped.// What is worth adding is the two numbers a caller has to get right, because// both are asserted rather than reported: a window smaller than// `window_len` panics inside the compressor, and so does an output buffer of// eight octets or fewer. That is the same shape of trap as the TLS handshake// buffers, and it is worth naming in one place rather than being discovered.
/// The window either direction needs, and the assertion both `Compress.init`/// and `Decompress.init` make about the buffer they are handed. 64 KiB, which/// is per stream and not per connection: a server compressing many responses/// at once pays this for each, which is the number to multiply before turning/// compression on for everything.pub const window_len = flate.max_window_len;
/// The smallest output buffer `Compress.init` will accept. It asserts/// `output.buffer.len > 8`.pub const min_output_buffer_len = 9;
pub const Compressor = flate.Compress;pub const Decompressor = flate.Decompress;
/// zstd's decoder, which is all of zstd this library has: §8.4.1 of RFC 9110/// registers the coding and `std.compress.zstd` implements one direction of/// it.pub const ZstdDecompressor = std.compress.zstd.Decompress;
/// The window a zstd stream may refer back into.////// This is the number to look at before turning zstd on anywhere that decodes/// what a peer chose to send. §3.1.1.1.2 of RFC 8878 lets a *sender* pick the/// window and a decoder that will not allocate it has to refuse the stream, so/// this is a ceiling on what will be accepted rather than a buffer that is/// always full. std's default is eight megabytes, which is 128 times what gzip/// wants, and it is per stream: a server decoding request bodies pays it for/// every concurrent one.pub const zstd_window_len: u32 = std.compress.zstd.default_window_len;
/// How large a buffer `Decoder.init` wants for `c`, and zero for a coding it/// cannot decode.pub fn decodeBufferLen(c: Coding) usize { return switch (c) { .gzip => window_len, // §3.1.1.1.2 again: the window, and a block on top of it, which is // what `Decompress.init` asserts. .zstd => zstd_window_len + std.compress.zstd.block_size_max, .identity, .deflate, .compress, .br => 0, };}
/// A decoder for any coding this library can undo, as one type.////// The two std decompressors are different types that both happen to expose a/// `reader`, so something has to name the choice — and a caller that holds one/// of these allocates once and asks for a reader, rather than switching on the/// coding at every use. It must not move once `reader` has been taken: what/// comes back points inside it.pub const Decoder = union(enum) { gzip: Decompressor, zstd: ZstdDecompressor,
/// A decoder for `c` reading coded octets from `input`. /// /// `buffer` must be `decodeBufferLen(c)` octets, and it and `input` must /// outlive the decoder. `error.Unsupported` is any coding `canDecode` /// says no to, and `identity`, which has nothing to undo. pub fn init(c: Coding, input: *Reader, buffer: []u8) error{Unsupported}!Decoder { return switch (c) { .gzip => .{ .gzip = gzipDecompressor(input, buffer) }, .zstd => .{ .zstd = ZstdDecompressor.init(input, buffer, .{ .window_len = zstd_window_len, // Off, and not by choice. §3.1.1 of RFC 8878 makes a // frame's content checksum optional and cheap to verify, // and `zstd` writes one by default — but verifying it in // Zig 0.17.0 reaches `@panic("TODO all those bytes // written needed to go through the hasher too")` in // `std/compress/zstd/Decompress.zig`, which is an // unimplemented path rather than a rejected frame. // Handing a peer's octets to a panic is not a trade worth // making, so this stays false until the toolchain moves. // The decode is unaffected and a corrupt frame still // fails on its own structure; what is lost is the last // check that what came out is what went in. .verify_checksum = false, }), }, .identity, .deflate, .compress, .br => error.Unsupported, }; }
test init { var source: Reader = .fixed(""); var window: [window_len]u8 = undefined; const d = try Decoder.init(.gzip, &source, &window); try testing.expect(d == .gzip); try testing.expectError(error.Unsupported, Decoder.init(.br, &source, &.{})); }
/// The decoded octets. It points inside `d`, which therefore must not move /// while the reader is in use. pub fn reader(d: *Decoder) *Reader { return switch (d.*) { .gzip => |*g| &g.reader, .zstd => |*z| &z.reader, }; }
test reader { // An empty gzip member: header, an empty final block, and a footer. const coded = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; var source: Reader = .fixed(&coded); var window: [window_len]u8 = undefined; var d = try Decoder.init(.gzip, &source, &window); const plain = try d.reader().allocRemaining(testing.allocator, .limited(1024)); defer testing.allocator.free(plain); try testing.expectEqual(@as(usize, 0), plain.len); }};
/// A gzip compressor writing into `output`. `buffer` must be at least/// `window_len`, and `output`'s own buffer more than eight octets.////// The stream has to be ended with `Compressor.finish`, which writes §2.2 of/// RFC 1952's final block and the footer carrying the CRC and the/// uncompressed size. A `flush` alone is not that: it moves what is buffered/// and leaves the member unfinished, and what the peer gets then is a body/// that stops mid-stream — `error.EndOfStream` to a reader that checks, and a/// truncated representation to one that does not.pub fn gzipCompressor( output: *Writer, buffer: []u8, level: Compressor.Options,) Writer.Error!Compressor { return Compressor.init(output, buffer, .gzip, level);}
/// A gzip decompressor reading from `input`.////// `buffer` may be empty, which puts std's decompressor in its direct mode and/// costs no window of its own — the right choice when the caller is streaming/// through to somewhere else, since it then uses the destination writer's/// buffer as the window. Only `stream` works in that mode: reading into a/// slice, `readSliceShort` and the like, overflows an integer inside std and/// panics. Otherwise `buffer` must be at least `window_len`.pub fn gzipDecompressor(input: *Reader, buffer: []u8) Decompressor { return Decompressor.init(input, .gzip, buffer);}
/// Why undoing a coding is opt-in everywhere here.////// A coding is the sender's choice of how many octets the receiver spends, and/// it is undone by std's decoders, which are as robust as the toolchain makes/// them. Zig 0.16.0's flate decoder was not: `tossBitsShort` added/// `consumed_bits` where it had to subtract them, and twenty-five octets of/// malformed gzip -- found by this library's `coded-bodies` fuzz target --/// were enough to panic the process. ziglang/zig#35815 fixed that for 0.17.0,/// which this library now requires, and the octets that found it are a test/// below that has to come back `Malformed`.////// What it means for a caller:////// * A **client** decoding a response trusts the server it chose to talk to./// * A **server** decoding a request body trusts nobody, which is why/// `server.compression.RequestDecoding` is null by default: whatever the/// decoder does with hostile input, anybody who can reach the port chooses/// that input.////// zstd's content checksum stays unverified for the same reason -- see/// `Decoder.init`.////// Compressing is unaffected: the encoder is a different path and reads only/// what this end produced.pub const UndoError = error{ OutOfMemory, /// The octets are not a valid stream of that coding. Malformed, /// The decoded representation is larger than the caller allowed. /// /// §8.4 of RFC 9110 puts no bound on what a coding may expand to, and /// nothing in a gzip member says up front how big it decodes to — the /// size in its footer is eight octets a sender chooses and is checked /// only at the end, long after the memory has been spent. A few hundred /// octets of zeroes decode to gigabytes, which is why this bound is a /// parameter rather than a default: every caller has a number and none of /// them is this module's to pick. TooLarge, /// A coding this library cannot undo. Unsupported,};
/// Undoes one coding on a representation already gathered whole.////// The streaming case wraps a reader instead — see `Decoder` — and is what the/// HTTP/1.x client and the server's request decoding do. This is for the two multiplexed versions,/// where a stream's octets arrive interleaved with every other stream's, so/// there is no single reader to wrap and the body is accumulated before/// anything can be done to it.////// `window` must be `decodeBufferLen(what)` octets — which differs by coding,/// and by two orders of magnitude between the two this library can undo. What/// comes back is the caller's to free.pub fn undo( gpa: std.mem.Allocator, coded: []const u8, what: Coding, window: []u8, limit: usize,) UndoError![]u8 { if (what == .identity) return gpa.dupe(u8, coded); var source: Reader = .fixed(coded); var d = Decoder.init(what, &source, window) catch return error.Unsupported; return d.reader().allocRemaining(gpa, .limited(limit)) catch |err| switch (err) { error.OutOfMemory => error.OutOfMemory, error.StreamTooLong => error.TooLarge, else => error.Malformed, };}
const testing = std.testing;
test "RFC 9110 12.5.3: a weight is read, and only in the shape given" { try testing.expectEqual(@as(?Quality, 0), parseQuality("0")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1")); try testing.expectEqual(@as(?Quality, 500), parseQuality("0.5")); try testing.expectEqual(@as(?Quality, 1), parseQuality("0.001")); try testing.expectEqual(@as(?Quality, 0), parseQuality("0.000")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1.0")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1.000"));
// The grammar is `( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] )`, // so each of these is not a weight. try testing.expectEqual(@as(?Quality, null), parseQuality(".5")); try testing.expectEqual(@as(?Quality, null), parseQuality("1.5")); try testing.expectEqual(@as(?Quality, null), parseQuality("0.1234")); try testing.expectEqual(@as(?Quality, null), parseQuality("2")); try testing.expectEqual(@as(?Quality, null), parseQuality("")); try testing.expectEqual(@as(?Quality, null), parseQuality("0,5"));}
test "RFC 9110 12.5.3: choosing a coding a client will take" { const both = [_]Coding{ .gzip, .identity };
// The ordinary case, and the one every browser sends. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip, deflate, br", &both));
// No field at all means any coding is *acceptable*, and identity is what // gets sent: a coding nobody asked for is one something in the middle did // not ask for either. try testing.expectEqual(@as(?Coding, .identity), selectEncoding(null, &both)); // An empty field value is the client saying it wants no coding. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("", &both));
// A weight of zero is a refusal and not a low preference. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("gzip;q=0", &both)); // Barely wanted still beats never asked for: identity's unstated weight // is the lowest there is, so any coding the client troubled to request // outranks it. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip;q=0.001", &both)); try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip;q=0.9", &both));
// Preference is read, not guessed: the client would rather have identity. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("gzip;q=0.3, identity;q=0.9", &both));
// The wildcard stands for everything not named. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("*", &both)); try testing.expectEqual(@as(?Coding, .identity), selectEncoding("*;q=0, identity", &both));
// Refusing everything, which §12.5.3 permits and a server answers with a // 406 if it has nothing left to send. try testing.expectEqual(@as(?Coding, null), selectEncoding("*;q=0", &both)); try testing.expectEqual(@as(?Coding, null), selectEncoding("identity;q=0, gzip;q=0", &both));
// A coding this library cannot perform is not chosen however much the // client wants it, because `offered` is what the caller can actually do. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("br;q=1.0, identity;q=0.1", &both));
// Case and whitespace are not significant; §8.4.1 matches names // case-insensitively and §5.6.1 allows the spaces. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding(" GZIP ; Q=1.0 ", &both)); // `x-gzip` is the spelling §8.4.1 still lists as equivalent. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("x-gzip", &both));
// A caller offering nothing gets nothing, rather than identity by // accident. try testing.expectEqual(@as(?Coding, null), selectEncoding("gzip", &.{}));}
test "RFC 9110 8.4: what a Content-Encoding says has to be undone" { try testing.expectEqual(Applied.identity, applied(null)); try testing.expectEqual(Applied.identity, applied("")); // §8.4.1.1: `identity` there is a coding that does nothing. try testing.expectEqual(Applied.identity, applied("identity")); try testing.expectEqual(Applied{ .one = .gzip }, applied("gzip")); try testing.expectEqual(Applied{ .one = .gzip }, applied("identity, gzip"));
// Anything this library will not undo on its own, which the caller hands // over as it arrived or refuses. try testing.expectEqual(Applied.unsupported, applied("br")); try testing.expectEqual(Applied.unsupported, applied("gzip, gzip")); try testing.expectEqual(Applied.unsupported, applied("gzip, br")); try testing.expectEqual(Applied.unsupported, applied("something-invented"));}
test "RFC 9110 8.4: undoing a coding is bounded, because a coding is not" { const gpa = testing.allocator; var window: [window_len]u8 = undefined;
// A member that decodes to far more than it weighs, which is the whole // shape of the attack: the cost of making it is nothing and the cost of // undoing it is whatever the recipient allows. const zeroes: [200_000]u8 = @splat(0); var coded_buf: [4096]u8 = undefined; var out: std.Io.Writer = .fixed(&coded_buf); var w = try gzipCompressor(&out, &window, .default); try w.writer.writeAll(&zeroes); try w.finish(); const bomb = out.buffered(); try testing.expect(bomb.len < 1000);
// Allowed, and it decodes. const roomy = try undo(gpa, bomb, .gzip, &window, 1 << 20); defer gpa.free(roomy); try testing.expectEqualSlices(u8, &zeroes, roomy);
// Refused, rather than allocated and then regretted. Nothing in the // member says how large it is until it has already been decoded. try testing.expectError(error.TooLarge, undo(gpa, bomb, .gzip, &window, 1024));
// Not a gzip stream at all. try testing.expectError(error.Malformed, undo(gpa, "not gzip", .gzip, &window, 1 << 20));
// Identity copies, so that a caller has one thing to free either way. const same = try undo(gpa, "unchanged", .identity, &window, 1 << 20); defer gpa.free(same); try testing.expectEqualStrings("unchanged", same);
try testing.expectError(error.Unsupported, undo(gpa, "", .br, &window, 1 << 20));}
test "the twenty-five octets that panicked Zig 0.16.0's flate decoder" { const gpa = testing.allocator; var window: [window_len]u8 = undefined;
// A gzip header whose `FLG` is 0xfe, then a dynamic block that ends // mid-bit-stream while its code lengths are being read. On 0.16.0 this // underflowed a `usize` in `peekBitsEnding`; on 0.17.0 it is a stream // that ends too soon, which is all it ever was. const octets = [_]u8{ 0x1f, 0x8b, 0x08, 0xfe, 0x64, 0x00, 0x00, 0xd2, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x0d, 0x73, 0x32, 0x62, 0x63, }; try testing.expectError(error.Malformed, undo(gpa, &octets, .gzip, &window, 1 << 20));}
test "RFC 8878: a zstd frame this library could not have produced" { // The point of a fixed vector rather than a round trip: `std.compress.zstd` // is a decoder and nothing here is an encoder, so there is no way to // produce this and no temptation to test the decoder against itself. These // octets came from libzstd 1.5.7 — Facebook's own implementation — and are // 1056 octets of repeated text in 77. const frame = [_]u8{ 0x28, 0xb5, 0x2f, 0xfd, 0x64, 0x20, 0x03, 0xfd, 0x01, 0x00, 0x22, 0x44, 0x0d, 0x11, 0xb0, 0xeb, 0x20, 0x4b, 0x84, 0x2d, 0x02, 0x96, 0x34, 0xc5, 0x98, 0x24, 0xe0, 0xa2, 0xc9, 0x42, 0x04, 0x0e, 0x4f, 0xe2, 0xae, 0xbb, 0x9c, 0x9b, 0xc7, 0x46, 0xa1, 0xd3, 0x54, 0x79, 0xb5, 0x91, 0x7c, 0x16, 0x4e, 0xba, 0xc5, 0xe0, 0x8d, 0x19, 0x0e, 0x3f, 0xaf, 0x24, 0xef, 0x4c, 0x62, 0x8b, 0xdf, 0x58, 0xb4, 0x08, 0x01, 0x00, 0xc2, 0xf6, 0xa2, 0x0a, 0x51, 0xa6, 0x6a, 0x4c, 0xe4, }; const expected = repeat("a zstd frame decoded by std, from a vector nothing here produced. ", 16);
// §2.2 of RFC 8878: a frame begins with this magic number. Checking it // keeps "it decoded" from also being true of some other container. try testing.expectEqual(@as(u8, 0x28), frame[0]); try testing.expectEqual(@as(u8, 0xb5), frame[1]);
const gpa = testing.allocator; const window = try gpa.alloc(u8, decodeBufferLen(.zstd)); defer gpa.free(window);
const plain = try undo(gpa, &frame, .zstd, window, 1 << 20); defer gpa.free(plain); try testing.expectEqualStrings(expected, plain);
// Bounded the same way gzip is, and for the same reason: §3.1.1 puts the // decoded size in the frame header as something the *sender* wrote. try testing.expectError(error.TooLarge, undo(gpa, &frame, .zstd, window, 100)); // And not a zstd frame at all. try testing.expectError(error.Malformed, undo(gpa, "not zstd", .zstd, window, 1 << 20));}
test "the two questions a coding answers are not the same question" { // zstd is why `supported` had to become two predicates: this library can // read it and cannot write it, so one answer would have been wrong about // one direction. A client may ask for it; a server may not offer it. try testing.expect(Coding.zstd.canDecode()); try testing.expect(!Coding.zstd.canEncode()); try testing.expect(Coding.gzip.canDecode()); try testing.expect(Coding.gzip.canEncode()); // identity is both, trivially, being nothing at all. try testing.expect(Coding.identity.canDecode()); try testing.expect(Coding.identity.canEncode()); // And the ones this library only knows the name of. for ([_]Coding{ .deflate, .compress, .br }) |c| { try testing.expect(!c.canDecode()); try testing.expect(!c.canEncode()); }
// So a `Content-Encoding: zstd` is now something to undo rather than // something to hand over coded. try testing.expectEqual(Applied{ .one = .zstd }, applied("zstd"));
// The window is the cost, and the difference is the thing to notice. try testing.expectEqual(window_len, decodeBufferLen(.gzip)); try testing.expect(decodeBufferLen(.zstd) > 100 * decodeBufferLen(.gzip));}
test "a gzip stream survives being written and read again" { // The codec is std's, so what this pins is the two buffer sizes above and // the container being gzip rather than zlib or raw -- which is the one // thing a caller of `flate` can get wrong without any error saying so, // because all three produce a perfectly good stream of the wrong kind. const original = repeat("the same sentence, repeated, is what a compressor is for. ", 16);
// A fixed buffer and not an `Allocating`: that one starts with a buffer of // no octets at all, and `Compress.init` asserts more than eight. Which is // exactly the trap `min_output_buffer_len` exists to name, met here on the // first attempt at using this module. var out_buf: [4096]u8 = undefined; var compressed: std.Io.Writer = .fixed(&out_buf);
var window: [window_len]u8 = undefined; var c = try gzipCompressor(&compressed, &window, .default); try c.writer.writeAll(original); // `finish` and not `flush`: §2.2 of RFC 1952 ends a gzip member with a // final deflate block and an eight-octet footer carrying the CRC and the // uncompressed size, and only `finish` writes them. A flushed-but-unended // stream decompresses to `error.EndOfStream`, which is what this test did // when it was written the obvious way. try c.finish();
const coded = compressed.buffered(); // §2.3.1 of RFC 1952: a gzip member starts with these two octets and then // the deflate method. Checking them is how "it round-tripped" stops also // being true of the raw and zlib containers. try testing.expect(coded.len > 2); try testing.expectEqual(@as(u8, 0x1f), coded[0]); try testing.expectEqual(@as(u8, 0x8b), coded[1]); try testing.expect(coded.len < original.len);
var source: std.Io.Reader = .fixed(coded); var out_window: [window_len]u8 = undefined; var d = gzipDecompressor(&source, &out_window); const back = try d.reader.allocRemaining(testing.allocator, .limited(1 << 20)); defer testing.allocator.free(back); try testing.expectEqualStrings(original, back);}
test selectEncoding { // What a server that can send gzip or nothing passes, in its preference. const offered = [_]Coding{ .gzip, .identity }; try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip, br", &offered)); // A client that never asked gets the representation as it is. try testing.expectEqual(@as(?Coding, .identity), selectEncoding(null, &offered)); // And one that refused everything on offer gets nothing: a 406. try testing.expectEqual(@as(?Coding, null), selectEncoding("br, *;q=0", &offered));}
test applied { try testing.expectEqual(Applied.identity, applied(null)); try testing.expectEqual(Applied{ .one = .gzip }, applied("gzip")); // Brotli is a coding this library has a name for and cannot undo. try testing.expectEqual(Applied.unsupported, applied("br"));}
test decodeBufferLen { try testing.expectEqual(window_len, decodeBufferLen(.gzip)); try testing.expectEqual(@as(usize, 0), decodeBufferLen(.identity)); // zstd's window is the sender's to choose, and this is the most accepted. try testing.expect(decodeBufferLen(.zstd) >= zstd_window_len);}
test gzipCompressor { var out_buf: [256]u8 = undefined; var out: Writer = .fixed(&out_buf); var window: [window_len]u8 = undefined;
var c = try gzipCompressor(&out, &window, .default); try c.writer.writeAll("hello, hello, hello"); // Only `finish` writes the footer that makes it a whole gzip member. try c.finish();
const coded = out.buffered(); try testing.expectEqual(@as(u8, 0x1f), coded[0]); try testing.expectEqual(@as(u8, 0x8b), coded[1]);}
test gzipDecompressor { // An empty gzip member: header, an empty final block, and a footer. const coded = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; var source: Reader = .fixed(&coded); var window: [window_len]u8 = undefined; var d = gzipDecompressor(&source, &window); var plain: [16]u8 = undefined; try testing.expectEqual(@as(usize, 0), try d.reader.readSliceShort(&plain));}
test undo { const gpa = testing.allocator; var window: [window_len]u8 = undefined;
var coded_buf: [256]u8 = undefined; var out: Writer = .fixed(&coded_buf); var c = try gzipCompressor(&out, &window, .default); try c.writer.writeAll("a body gathered whole"); try c.finish();
const plain = try undo(gpa, out.buffered(), .gzip, &window, 1024); defer gpa.free(plain); try testing.expectEqualStrings("a body gathered whole", plain);
// The limit is on what it decodes to, not on what arrived. try testing.expectError(error.TooLarge, undo(gpa, out.buffered(), .gzip, &window, 4));}
/// `s` said `n` times over: what `s ** n` spelled before Zig 0.17 took the/// operator away.inline fn repeat(comptime s: []const u8, comptime n: usize) *const [s.len * n]u8 { comptime { const rows: [n][s.len]u8 = @splat(s[0..s.len].*); const flat: [s.len * n]u8 = @bitCast(rows); return ♭ }}