// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! Content codings, and choosing one the peer will accept. //! //! §8.4.1 of RFC 9110 names the codings and §12.5.3 says how a client asks for //! one. Both halves are here, and neither does any I/O: a content coding is a //! transformation on a byte stream, which in this standard library is a //! `Reader` over a `Reader` and a `Writer` over a `Writer`. That is why this //! sits in the sans-I/O module beside the framing rather than in a layer above //! it — and it is the reason a coding applies identically over HTTP/1.1, //! HTTP/2 and HTTP/3, none of which it knows about. //! //! A content coding is not a transfer coding. §8.4 makes it part of the //! *representation*: it travels end to end, a cache stores it coded, and //! `Content-Length` counts the coded octets. That is the whole reason a //! `Content-Encoding: gzip` request body reaches a handler still compressed, //! which the README argues for at more length — deciding to decode it is the //! handler's business, and this module is what it decodes with. const std = @import("std"); const Reader = std.Io.Reader; const Writer = std.Io.Writer; const flate = std.compress.flate; const token = @import("token.zig"); /// The codings this library has a name for. /// /// The ones it cannot perform are named rather than left out, because a /// `Content-Encoding` that cannot be decoded has to be told apart from one /// nobody has heard of. The first is a representation to hand over as it /// arrived; the second is a message whose sender and recipient disagree about /// what it contains, which is worth refusing. pub const Coding = enum { /// §8.4.1.1: no coding at all, and the one that needs no asking for. identity, /// §8.4.1.3. gzip, /// §8.4.1.2. The zlib format of RFC 1950, which is *not* what most /// senders that say `deflate` actually send — a fair number send raw /// RFC 1951 — and telling those apart means sniffing the first octets. /// Named, not performed. deflate, /// §8.4.1.4's `compress`, which is LZW and long dead. compress, /// RFC 7932 Brotli. br, /// RFC 8878 Zstandard. zstd, /// The coding's registered name, as it goes in `Content-Encoding` and /// `Accept-Encoding`. Always the current spelling, never `x-gzip`. pub fn name(c: Coding) []const u8 { return switch (c) { .identity => "identity", .gzip => "gzip", .deflate => "deflate", .compress => "compress", .br => "br", .zstd => "zstd", }; } test name { try testing.expectEqualStrings("gzip", Coding.gzip.name()); try testing.expectEqualStrings("br", Coding.br.name()); } /// §8.4.1: a coding name is a token and is matched case-insensitively. /// `x-gzip` and `x-compress` are the historical spellings §8.4.1 still /// lists as equivalents, so they are accepted rather than treated as /// codings nobody has heard of. Optional whitespace around the name is /// ignored; anything else unrecognized is null. pub fn parse(text: []const u8) ?Coding { const trimmed = token.trimOws(text); if (token.eqlIgnoreCase(trimmed, "identity")) return .identity; if (token.eqlIgnoreCase(trimmed, "gzip")) return .gzip; if (token.eqlIgnoreCase(trimmed, "x-gzip")) return .gzip; if (token.eqlIgnoreCase(trimmed, "deflate")) return .deflate; if (token.eqlIgnoreCase(trimmed, "compress")) return .compress; if (token.eqlIgnoreCase(trimmed, "x-compress")) return .compress; if (token.eqlIgnoreCase(trimmed, "br")) return .br; if (token.eqlIgnoreCase(trimmed, "zstd")) return .zstd; return null; } test parse { try testing.expectEqual(Coding.gzip, Coding.parse("GZip").?); try testing.expectEqual(Coding.gzip, Coding.parse("x-gzip").?); try testing.expectEqual(Coding.zstd, Coding.parse(" zstd ").?); try testing.expect(Coding.parse("lzma") == null); } /// Whether this library can undo it. pub fn canDecode(c: Coding) bool { return switch (c) { .identity, .gzip, .zstd => true, .deflate, .compress, .br => false, }; } test canDecode { try testing.expect(Coding.gzip.canDecode()); try testing.expect(Coding.zstd.canDecode()); // Named, so it can be recognized, but not performed. try testing.expect(!Coding.br.canDecode()); } /// Whether this library can apply it. /// /// Deliberately not the same set, and zstd is the reason the two had to be /// separated: `std.compress.zstd` ships a decoder and no encoder, so this /// library can read what a peer sends and cannot produce it. A client may /// therefore ask for zstd and undo the answer; a server may not offer it. /// One predicate meaning "supported" could only have been wrong about one /// of those. pub fn canEncode(c: Coding) bool { return switch (c) { .identity, .gzip => true, .deflate, .compress, .br, .zstd => false, }; } test canEncode { try testing.expect(Coding.gzip.canEncode()); // Decodable, and still not something this library can produce. try testing.expect(!Coding.zstd.canEncode()); } }; /// A quality value in thousandths, which is exactly the precision §12.5.3 /// allows: "a weight is a value between 0 and 1 with at most three digits /// after the decimal point". Integers because a preference compared with /// floating point is a preference that can compare unequal to itself. pub const Quality = u16; /// The highest weight, and what an element with no weight at all carries. pub const highest: Quality = 1000; /// §12.5.3's weight, or `highest` when the element carries none. /// /// A weight this cannot read is treated as absent rather than as zero. The /// grammar allows only `q=`, so anything else is a sender doing something the /// specification does not describe — and reading that as "not acceptable" /// would refuse a peer over a parameter nobody meant as a refusal, where /// reading it as "no preference stated" merely ignores it. fn weightOf(element: []const u8) Quality { var rest = element; while (std.mem.indexOfScalar(u8, rest, ';')) |at| { const parameter = token.trimOws(rest[at + 1 ..]); rest = parameter; const eq = std.mem.indexOfScalar(u8, parameter, '=') orelse continue; if (!token.eqlIgnoreCase(token.trimOws(parameter[0..eq]), "q")) continue; var value = token.trimOws(parameter[eq + 1 ..]); if (std.mem.indexOfScalar(u8, value, ';')) |end| value = token.trimOws(value[0..end]); return parseQuality(value) orelse highest; } return highest; } /// `0`, `1`, `0.5`, `1.000` — and nothing else. §12.5.3's grammar is /// `( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] )`, so `1.5` is not a /// weight and neither is `.5`. fn parseQuality(text: []const u8) ?Quality { if (text.len == 0) return null; const whole = text[0]; if (whole != '0' and whole != '1') return null; var thousandths: Quality = if (whole == '1') highest else 0; if (text.len == 1) return thousandths; if (text[1] != '.') return null; const digits = text[2..]; if (digits.len > 3) return null; var scale: Quality = 100; for (digits) |d| { if (!std.ascii.isDigit(d)) return null; const n: Quality = d - '0'; if (whole == '1') { // §12.5.3 allows only zeroes after `1.`, and `1.001` is not a // weight that means anything above the maximum. if (n != 0) return null; } else { thousandths += n * scale; } scale /= 10; } return thousandths; } /// The weight a list such as an `Accept-Encoding` field value gives `name`, /// or null when the list does not name it at all — which is not the same as /// a weight of zero, since an unnamed coding falls to `*` and a coding /// weighted zero is refused outright. /// /// Names are compared without regard to case, as §8.4.1 says content codings /// are, and `*` is a name like any other here: asking for it is how a caller /// learns what the wildcard says. Where a list names something twice, the /// last mention is the one that counts, as in `selectEncoding`. pub fn statedWeight(field_value: []const u8, name: []const u8) ?Quality { var stated: ?Quality = null; var it = token.list(field_value); while (it.next()) |element| { const before_weight = element[0 .. std.mem.indexOfScalar(u8, element, ';') orelse element.len]; if (token.eqlIgnoreCase(token.trimOws(before_weight), name)) stated = weightOf(element); } return stated; } test statedWeight { const field = "GZIP;q=0.8, br, *;q=0"; try testing.expectEqual(@as(?Quality, 800), statedWeight(field, "gzip")); // No weight is the highest one. try testing.expectEqual(@as(?Quality, highest), statedWeight(field, "br")); try testing.expectEqual(@as(?Quality, 0), statedWeight(field, "*")); // Not named is not refused: that is for `*` to decide. try testing.expectEqual(@as(?Quality, null), statedWeight(field, "zstd")); } /// Chooses a coding from what a peer said it would take. /// /// `offered` is in this end's order of preference, so a caller that would /// rather send gzip than nothing passes `&.{ .gzip, .identity }`. Ties in the /// peer's weights are broken by that order, which is the only thing that can /// break them: §12.5.3 gives a client a way to say what it prefers and no way /// to say what it is indifferent about. /// /// Null means the peer refused everything on offer, which §12.5.3 says a /// server "MAY" answer with a 406 — and which it can only mean deliberately, /// since identity is acceptable unless a client goes out of its way to refuse /// it. pub fn selectEncoding(field_value: ?[]const u8, offered: []const Coding) ?Coding { // §12.5.3: "If no Accept-Encoding header field is in the request, any // content coding is considered acceptable by the user agent." Acceptable // is not the same as wanted, and this returns what to *send*: a client // that did not ask gets identity, because a coding applied to a client // that never mentioned one is how a response becomes unreadable to // something in the middle that also never mentioned one. const list = field_value orelse return firstOf(offered, .identity); // "An Accept-Encoding header field with a combined field value that is // empty implies that the user agent does not want any content coding in // response." if (token.trimOws(list).len == 0) return firstOf(offered, .identity); var star: ?Quality = null; var best: ?Coding = null; var best_weight: Quality = 0; for (offered) |candidate| { var stated: ?Quality = null; var it = token.list(list); while (it.next()) |element| { const before_weight = element[0 .. std.mem.indexOfScalar(u8, element, ';') orelse element.len]; const name = token.trimOws(before_weight); if (std.mem.eql(u8, name, "*")) { if (star == null) star = weightOf(element); continue; } const coding = Coding.parse(name) orelse continue; if (coding != candidate) continue; stated = weightOf(element); } // What the client did not name, the wildcard speaks for; and what // neither names is a coding it never asked for. // // Except identity, which §12.5.3 says "is always acceptable, unless // specifically refused". Acceptable is not preferred, and the // difference decides every ordinary request: a client sending // `gzip;q=0.9` and never mentioning identity plainly wants gzip, so // identity's unstated weight has to sit *below* anything the client // troubled to ask for rather than at the top. The lowest weight there // is, then — enough to be chosen when nothing else may be, and never // enough to beat a coding that was actually requested. const weight = stated orelse star orelse (if (candidate == .identity) @as(Quality, 1) else @as(Quality, 0)); if (weight > best_weight) { best = candidate; best_weight = weight; } } return best; } /// The first of `offered` that is `wanted`, or null. fn firstOf(offered: []const Coding, wanted: Coding) ?Coding { for (offered) |c| if (c == wanted) return c; return null; } /// Reads a `Content-Encoding` field and says what has to be undone. /// /// §8.4: "If one or more encodings have been applied to a representation, the /// sender that applied the encodings MUST generate a Content-Encoding header /// field that lists the content codings in the order in which they were /// applied." So undoing them means walking the list backwards, and the last /// element is the first thing to undo. /// /// The common case by a very long way is one coding, which is why this reports /// the shape rather than a list: `identity` for a field that is absent or says /// nothing, one `Coding` for a single coding, and `.unsupported` for anything /// this library cannot undo on its own — several codings included, since /// applying gzip twice is legal, vanishingly rare, and not worth a decoder /// stack that would exist for nobody. pub const Applied = union(enum) { /// Nothing to undo. identity, /// Exactly one coding, and this library can undo it. one: Coding, /// Something this library will not undo: a coding it cannot perform, or /// more than one of them. The representation is the caller's to hand over /// as it arrived or to refuse. unsupported, }; /// Reads a `Content-Encoding` field value, or null for a message that had /// none, into the `Applied` that says what undoing it takes. pub fn applied(field_value: ?[]const u8) Applied { const list = field_value orelse return .identity; var found: ?Coding = null; var it = token.list(list); while (it.next()) |element| { const name = token.trimOws(element); if (name.len == 0) continue; const coding = Coding.parse(name) orelse return .unsupported; // §8.4.1.1: `identity` in a `Content-Encoding` is a coding that does // nothing, so it neither counts as a coding to undo nor makes the // field unreadable. if (coding == .identity) continue; if (found != null) return .unsupported; if (!coding.canDecode()) return .unsupported; found = coding; } return if (found) |c| .{ .one = c } else .identity; } // -- the codec itself ------------------------------------------------------- // // std's flate already is what this library wants -- a `Reader` over a `Reader` // and a `Writer` over a `Writer` -- so it is re-exported rather than wrapped. // What is worth adding is the two numbers a caller has to get right, because // both are asserted rather than reported: a window smaller than // `window_len` panics inside the compressor, and so does an output buffer of // eight octets or fewer. That is the same shape of trap as the TLS handshake // buffers, and it is worth naming in one place rather than being discovered. /// The window either direction needs, and the assertion both `Compress.init` /// and `Decompress.init` make about the buffer they are handed. 64 KiB, which /// is per stream and not per connection: a server compressing many responses /// at once pays this for each, which is the number to multiply before turning /// compression on for everything. pub const window_len = flate.max_window_len; /// The smallest output buffer `Compress.init` will accept. It asserts /// `output.buffer.len > 8`. pub const min_output_buffer_len = 9; pub const Compressor = flate.Compress; pub const Decompressor = flate.Decompress; /// zstd's decoder, which is all of zstd this library has: §8.4.1 of RFC 9110 /// registers the coding and `std.compress.zstd` implements one direction of /// it. pub const ZstdDecompressor = std.compress.zstd.Decompress; /// The window a zstd stream may refer back into. /// /// This is the number to look at before turning zstd on anywhere that decodes /// what a peer chose to send. §3.1.1.1.2 of RFC 8878 lets a *sender* pick the /// window and a decoder that will not allocate it has to refuse the stream, so /// this is a ceiling on what will be accepted rather than a buffer that is /// always full. std's default is eight megabytes, which is 128 times what gzip /// wants, and it is per stream: a server decoding request bodies pays it for /// every concurrent one. pub const zstd_window_len: u32 = std.compress.zstd.default_window_len; /// How large a buffer `Decoder.init` wants for `c`, and zero for a coding it /// cannot decode. pub fn decodeBufferLen(c: Coding) usize { return switch (c) { .gzip => window_len, // §3.1.1.1.2 again: the window, and a block on top of it, which is // what `Decompress.init` asserts. .zstd => zstd_window_len + std.compress.zstd.block_size_max, .identity, .deflate, .compress, .br => 0, }; } /// A decoder for any coding this library can undo, as one type. /// /// The two std decompressors are different types that both happen to expose a /// `reader`, so something has to name the choice — and a caller that holds one /// of these allocates once and asks for a reader, rather than switching on the /// coding at every use. It must not move once `reader` has been taken: what /// comes back points inside it. pub const Decoder = union(enum) { gzip: Decompressor, zstd: ZstdDecompressor, /// A decoder for `c` reading coded octets from `input`. /// /// `buffer` must be `decodeBufferLen(c)` octets, and it and `input` must /// outlive the decoder. `error.Unsupported` is any coding `canDecode` /// says no to, and `identity`, which has nothing to undo. pub fn init(c: Coding, input: *Reader, buffer: []u8) error{Unsupported}!Decoder { return switch (c) { .gzip => .{ .gzip = gzipDecompressor(input, buffer) }, .zstd => .{ .zstd = ZstdDecompressor.init(input, buffer, .{ .window_len = zstd_window_len, // Off, and not by choice. §3.1.1 of RFC 8878 makes a // frame's content checksum optional and cheap to verify, // and `zstd` writes one by default — but verifying it in // Zig 0.17.0 reaches `@panic("TODO all those bytes // written needed to go through the hasher too")` in // `std/compress/zstd/Decompress.zig`, which is an // unimplemented path rather than a rejected frame. // Handing a peer's octets to a panic is not a trade worth // making, so this stays false until the toolchain moves. // The decode is unaffected and a corrupt frame still // fails on its own structure; what is lost is the last // check that what came out is what went in. .verify_checksum = false, }), }, .identity, .deflate, .compress, .br => error.Unsupported, }; } test init { var source: Reader = .fixed(""); var window: [window_len]u8 = undefined; const d = try Decoder.init(.gzip, &source, &window); try testing.expect(d == .gzip); try testing.expectError(error.Unsupported, Decoder.init(.br, &source, &.{})); } /// The decoded octets. It points inside `d`, which therefore must not move /// while the reader is in use. pub fn reader(d: *Decoder) *Reader { return switch (d.*) { .gzip => |*g| &g.reader, .zstd => |*z| &z.reader, }; } test reader { // An empty gzip member: header, an empty final block, and a footer. const coded = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; var source: Reader = .fixed(&coded); var window: [window_len]u8 = undefined; var d = try Decoder.init(.gzip, &source, &window); const plain = try d.reader().allocRemaining(testing.allocator, .limited(1024)); defer testing.allocator.free(plain); try testing.expectEqual(@as(usize, 0), plain.len); } }; /// A gzip compressor writing into `output`. `buffer` must be at least /// `window_len`, and `output`'s own buffer more than eight octets. /// /// The stream has to be ended with `Compressor.finish`, which writes §2.2 of /// RFC 1952's final block and the footer carrying the CRC and the /// uncompressed size. A `flush` alone is not that: it moves what is buffered /// and leaves the member unfinished, and what the peer gets then is a body /// that stops mid-stream — `error.EndOfStream` to a reader that checks, and a /// truncated representation to one that does not. pub fn gzipCompressor( output: *Writer, buffer: []u8, level: Compressor.Options, ) Writer.Error!Compressor { return Compressor.init(output, buffer, .gzip, level); } /// A gzip decompressor reading from `input`. /// /// `buffer` may be empty, which puts std's decompressor in its direct mode and /// costs no window of its own — the right choice when the caller is streaming /// through to somewhere else, since it then uses the destination writer's /// buffer as the window. Only `stream` works in that mode: reading into a /// slice, `readSliceShort` and the like, overflows an integer inside std and /// panics. Otherwise `buffer` must be at least `window_len`. pub fn gzipDecompressor(input: *Reader, buffer: []u8) Decompressor { return Decompressor.init(input, .gzip, buffer); } /// Why undoing a coding is opt-in everywhere here. /// /// A coding is the sender's choice of how many octets the receiver spends, and /// it is undone by std's decoders, which are as robust as the toolchain makes /// them. Zig 0.16.0's flate decoder was not: `tossBitsShort` added /// `consumed_bits` where it had to subtract them, and twenty-five octets of /// malformed gzip -- found by this library's `coded-bodies` fuzz target -- /// were enough to panic the process. ziglang/zig#35815 fixed that for 0.17.0, /// which this library now requires, and the octets that found it are a test /// below that has to come back `Malformed`. /// /// What it means for a caller: /// /// * A **client** decoding a response trusts the server it chose to talk to. /// * A **server** decoding a request body trusts nobody, which is why /// `server.compression.RequestDecoding` is null by default: whatever the /// decoder does with hostile input, anybody who can reach the port chooses /// that input. /// /// zstd's content checksum stays unverified for the same reason -- see /// `Decoder.init`. /// /// Compressing is unaffected: the encoder is a different path and reads only /// what this end produced. pub const UndoError = error{ OutOfMemory, /// The octets are not a valid stream of that coding. Malformed, /// The decoded representation is larger than the caller allowed. /// /// §8.4 of RFC 9110 puts no bound on what a coding may expand to, and /// nothing in a gzip member says up front how big it decodes to — the /// size in its footer is eight octets a sender chooses and is checked /// only at the end, long after the memory has been spent. A few hundred /// octets of zeroes decode to gigabytes, which is why this bound is a /// parameter rather than a default: every caller has a number and none of /// them is this module's to pick. TooLarge, /// A coding this library cannot undo. Unsupported, }; /// Undoes one coding on a representation already gathered whole. /// /// The streaming case wraps a reader instead — see `Decoder` — and is what the /// HTTP/1.x client and the server's request decoding do. This is for the two multiplexed versions, /// where a stream's octets arrive interleaved with every other stream's, so /// there is no single reader to wrap and the body is accumulated before /// anything can be done to it. /// /// `window` must be `decodeBufferLen(what)` octets — which differs by coding, /// and by two orders of magnitude between the two this library can undo. What /// comes back is the caller's to free. pub fn undo( gpa: std.mem.Allocator, coded: []const u8, what: Coding, window: []u8, limit: usize, ) UndoError![]u8 { if (what == .identity) return gpa.dupe(u8, coded); var source: Reader = .fixed(coded); var d = Decoder.init(what, &source, window) catch return error.Unsupported; return d.reader().allocRemaining(gpa, .limited(limit)) catch |err| switch (err) { error.OutOfMemory => error.OutOfMemory, error.StreamTooLong => error.TooLarge, else => error.Malformed, }; } const testing = std.testing; test "RFC 9110 12.5.3: a weight is read, and only in the shape given" { try testing.expectEqual(@as(?Quality, 0), parseQuality("0")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1")); try testing.expectEqual(@as(?Quality, 500), parseQuality("0.5")); try testing.expectEqual(@as(?Quality, 1), parseQuality("0.001")); try testing.expectEqual(@as(?Quality, 0), parseQuality("0.000")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1.0")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1.000")); // The grammar is `( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] )`, // so each of these is not a weight. try testing.expectEqual(@as(?Quality, null), parseQuality(".5")); try testing.expectEqual(@as(?Quality, null), parseQuality("1.5")); try testing.expectEqual(@as(?Quality, null), parseQuality("0.1234")); try testing.expectEqual(@as(?Quality, null), parseQuality("2")); try testing.expectEqual(@as(?Quality, null), parseQuality("")); try testing.expectEqual(@as(?Quality, null), parseQuality("0,5")); } test "RFC 9110 12.5.3: choosing a coding a client will take" { const both = [_]Coding{ .gzip, .identity }; // The ordinary case, and the one every browser sends. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip, deflate, br", &both)); // No field at all means any coding is *acceptable*, and identity is what // gets sent: a coding nobody asked for is one something in the middle did // not ask for either. try testing.expectEqual(@as(?Coding, .identity), selectEncoding(null, &both)); // An empty field value is the client saying it wants no coding. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("", &both)); // A weight of zero is a refusal and not a low preference. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("gzip;q=0", &both)); // Barely wanted still beats never asked for: identity's unstated weight // is the lowest there is, so any coding the client troubled to request // outranks it. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip;q=0.001", &both)); try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip;q=0.9", &both)); // Preference is read, not guessed: the client would rather have identity. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("gzip;q=0.3, identity;q=0.9", &both)); // The wildcard stands for everything not named. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("*", &both)); try testing.expectEqual(@as(?Coding, .identity), selectEncoding("*;q=0, identity", &both)); // Refusing everything, which §12.5.3 permits and a server answers with a // 406 if it has nothing left to send. try testing.expectEqual(@as(?Coding, null), selectEncoding("*;q=0", &both)); try testing.expectEqual(@as(?Coding, null), selectEncoding("identity;q=0, gzip;q=0", &both)); // A coding this library cannot perform is not chosen however much the // client wants it, because `offered` is what the caller can actually do. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("br;q=1.0, identity;q=0.1", &both)); // Case and whitespace are not significant; §8.4.1 matches names // case-insensitively and §5.6.1 allows the spaces. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding(" GZIP ; Q=1.0 ", &both)); // `x-gzip` is the spelling §8.4.1 still lists as equivalent. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("x-gzip", &both)); // A caller offering nothing gets nothing, rather than identity by // accident. try testing.expectEqual(@as(?Coding, null), selectEncoding("gzip", &.{})); } test "RFC 9110 8.4: what a Content-Encoding says has to be undone" { try testing.expectEqual(Applied.identity, applied(null)); try testing.expectEqual(Applied.identity, applied("")); // §8.4.1.1: `identity` there is a coding that does nothing. try testing.expectEqual(Applied.identity, applied("identity")); try testing.expectEqual(Applied{ .one = .gzip }, applied("gzip")); try testing.expectEqual(Applied{ .one = .gzip }, applied("identity, gzip")); // Anything this library will not undo on its own, which the caller hands // over as it arrived or refuses. try testing.expectEqual(Applied.unsupported, applied("br")); try testing.expectEqual(Applied.unsupported, applied("gzip, gzip")); try testing.expectEqual(Applied.unsupported, applied("gzip, br")); try testing.expectEqual(Applied.unsupported, applied("something-invented")); } test "RFC 9110 8.4: undoing a coding is bounded, because a coding is not" { const gpa = testing.allocator; var window: [window_len]u8 = undefined; // A member that decodes to far more than it weighs, which is the whole // shape of the attack: the cost of making it is nothing and the cost of // undoing it is whatever the recipient allows. const zeroes: [200_000]u8 = @splat(0); var coded_buf: [4096]u8 = undefined; var out: std.Io.Writer = .fixed(&coded_buf); var w = try gzipCompressor(&out, &window, .default); try w.writer.writeAll(&zeroes); try w.finish(); const bomb = out.buffered(); try testing.expect(bomb.len < 1000); // Allowed, and it decodes. const roomy = try undo(gpa, bomb, .gzip, &window, 1 << 20); defer gpa.free(roomy); try testing.expectEqualSlices(u8, &zeroes, roomy); // Refused, rather than allocated and then regretted. Nothing in the // member says how large it is until it has already been decoded. try testing.expectError(error.TooLarge, undo(gpa, bomb, .gzip, &window, 1024)); // Not a gzip stream at all. try testing.expectError(error.Malformed, undo(gpa, "not gzip", .gzip, &window, 1 << 20)); // Identity copies, so that a caller has one thing to free either way. const same = try undo(gpa, "unchanged", .identity, &window, 1 << 20); defer gpa.free(same); try testing.expectEqualStrings("unchanged", same); try testing.expectError(error.Unsupported, undo(gpa, "", .br, &window, 1 << 20)); } test "the twenty-five octets that panicked Zig 0.16.0's flate decoder" { const gpa = testing.allocator; var window: [window_len]u8 = undefined; // A gzip header whose `FLG` is 0xfe, then a dynamic block that ends // mid-bit-stream while its code lengths are being read. On 0.16.0 this // underflowed a `usize` in `peekBitsEnding`; on 0.17.0 it is a stream // that ends too soon, which is all it ever was. const octets = [_]u8{ 0x1f, 0x8b, 0x08, 0xfe, 0x64, 0x00, 0x00, 0xd2, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x0d, 0x73, 0x32, 0x62, 0x63, }; try testing.expectError(error.Malformed, undo(gpa, &octets, .gzip, &window, 1 << 20)); } test "RFC 8878: a zstd frame this library could not have produced" { // The point of a fixed vector rather than a round trip: `std.compress.zstd` // is a decoder and nothing here is an encoder, so there is no way to // produce this and no temptation to test the decoder against itself. These // octets came from libzstd 1.5.7 — Facebook's own implementation — and are // 1056 octets of repeated text in 77. const frame = [_]u8{ 0x28, 0xb5, 0x2f, 0xfd, 0x64, 0x20, 0x03, 0xfd, 0x01, 0x00, 0x22, 0x44, 0x0d, 0x11, 0xb0, 0xeb, 0x20, 0x4b, 0x84, 0x2d, 0x02, 0x96, 0x34, 0xc5, 0x98, 0x24, 0xe0, 0xa2, 0xc9, 0x42, 0x04, 0x0e, 0x4f, 0xe2, 0xae, 0xbb, 0x9c, 0x9b, 0xc7, 0x46, 0xa1, 0xd3, 0x54, 0x79, 0xb5, 0x91, 0x7c, 0x16, 0x4e, 0xba, 0xc5, 0xe0, 0x8d, 0x19, 0x0e, 0x3f, 0xaf, 0x24, 0xef, 0x4c, 0x62, 0x8b, 0xdf, 0x58, 0xb4, 0x08, 0x01, 0x00, 0xc2, 0xf6, 0xa2, 0x0a, 0x51, 0xa6, 0x6a, 0x4c, 0xe4, }; const expected = repeat("a zstd frame decoded by std, from a vector nothing here produced. ", 16); // §2.2 of RFC 8878: a frame begins with this magic number. Checking it // keeps "it decoded" from also being true of some other container. try testing.expectEqual(@as(u8, 0x28), frame[0]); try testing.expectEqual(@as(u8, 0xb5), frame[1]); const gpa = testing.allocator; const window = try gpa.alloc(u8, decodeBufferLen(.zstd)); defer gpa.free(window); const plain = try undo(gpa, &frame, .zstd, window, 1 << 20); defer gpa.free(plain); try testing.expectEqualStrings(expected, plain); // Bounded the same way gzip is, and for the same reason: §3.1.1 puts the // decoded size in the frame header as something the *sender* wrote. try testing.expectError(error.TooLarge, undo(gpa, &frame, .zstd, window, 100)); // And not a zstd frame at all. try testing.expectError(error.Malformed, undo(gpa, "not zstd", .zstd, window, 1 << 20)); } test "the two questions a coding answers are not the same question" { // zstd is why `supported` had to become two predicates: this library can // read it and cannot write it, so one answer would have been wrong about // one direction. A client may ask for it; a server may not offer it. try testing.expect(Coding.zstd.canDecode()); try testing.expect(!Coding.zstd.canEncode()); try testing.expect(Coding.gzip.canDecode()); try testing.expect(Coding.gzip.canEncode()); // identity is both, trivially, being nothing at all. try testing.expect(Coding.identity.canDecode()); try testing.expect(Coding.identity.canEncode()); // And the ones this library only knows the name of. for ([_]Coding{ .deflate, .compress, .br }) |c| { try testing.expect(!c.canDecode()); try testing.expect(!c.canEncode()); } // So a `Content-Encoding: zstd` is now something to undo rather than // something to hand over coded. try testing.expectEqual(Applied{ .one = .zstd }, applied("zstd")); // The window is the cost, and the difference is the thing to notice. try testing.expectEqual(window_len, decodeBufferLen(.gzip)); try testing.expect(decodeBufferLen(.zstd) > 100 * decodeBufferLen(.gzip)); } test "a gzip stream survives being written and read again" { // The codec is std's, so what this pins is the two buffer sizes above and // the container being gzip rather than zlib or raw -- which is the one // thing a caller of `flate` can get wrong without any error saying so, // because all three produce a perfectly good stream of the wrong kind. const original = repeat("the same sentence, repeated, is what a compressor is for. ", 16); // A fixed buffer and not an `Allocating`: that one starts with a buffer of // no octets at all, and `Compress.init` asserts more than eight. Which is // exactly the trap `min_output_buffer_len` exists to name, met here on the // first attempt at using this module. var out_buf: [4096]u8 = undefined; var compressed: std.Io.Writer = .fixed(&out_buf); var window: [window_len]u8 = undefined; var c = try gzipCompressor(&compressed, &window, .default); try c.writer.writeAll(original); // `finish` and not `flush`: §2.2 of RFC 1952 ends a gzip member with a // final deflate block and an eight-octet footer carrying the CRC and the // uncompressed size, and only `finish` writes them. A flushed-but-unended // stream decompresses to `error.EndOfStream`, which is what this test did // when it was written the obvious way. try c.finish(); const coded = compressed.buffered(); // §2.3.1 of RFC 1952: a gzip member starts with these two octets and then // the deflate method. Checking them is how "it round-tripped" stops also // being true of the raw and zlib containers. try testing.expect(coded.len > 2); try testing.expectEqual(@as(u8, 0x1f), coded[0]); try testing.expectEqual(@as(u8, 0x8b), coded[1]); try testing.expect(coded.len < original.len); var source: std.Io.Reader = .fixed(coded); var out_window: [window_len]u8 = undefined; var d = gzipDecompressor(&source, &out_window); const back = try d.reader.allocRemaining(testing.allocator, .limited(1 << 20)); defer testing.allocator.free(back); try testing.expectEqualStrings(original, back); } test selectEncoding { // What a server that can send gzip or nothing passes, in its preference. const offered = [_]Coding{ .gzip, .identity }; try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip, br", &offered)); // A client that never asked gets the representation as it is. try testing.expectEqual(@as(?Coding, .identity), selectEncoding(null, &offered)); // And one that refused everything on offer gets nothing: a 406. try testing.expectEqual(@as(?Coding, null), selectEncoding("br, *;q=0", &offered)); } test applied { try testing.expectEqual(Applied.identity, applied(null)); try testing.expectEqual(Applied{ .one = .gzip }, applied("gzip")); // Brotli is a coding this library has a name for and cannot undo. try testing.expectEqual(Applied.unsupported, applied("br")); } test decodeBufferLen { try testing.expectEqual(window_len, decodeBufferLen(.gzip)); try testing.expectEqual(@as(usize, 0), decodeBufferLen(.identity)); // zstd's window is the sender's to choose, and this is the most accepted. try testing.expect(decodeBufferLen(.zstd) >= zstd_window_len); } test gzipCompressor { var out_buf: [256]u8 = undefined; var out: Writer = .fixed(&out_buf); var window: [window_len]u8 = undefined; var c = try gzipCompressor(&out, &window, .default); try c.writer.writeAll("hello, hello, hello"); // Only `finish` writes the footer that makes it a whole gzip member. try c.finish(); const coded = out.buffered(); try testing.expectEqual(@as(u8, 0x1f), coded[0]); try testing.expectEqual(@as(u8, 0x8b), coded[1]); } test gzipDecompressor { // An empty gzip member: header, an empty final block, and a footer. const coded = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; var source: Reader = .fixed(&coded); var window: [window_len]u8 = undefined; var d = gzipDecompressor(&source, &window); var plain: [16]u8 = undefined; try testing.expectEqual(@as(usize, 0), try d.reader.readSliceShort(&plain)); } test undo { const gpa = testing.allocator; var window: [window_len]u8 = undefined; var coded_buf: [256]u8 = undefined; var out: Writer = .fixed(&coded_buf); var c = try gzipCompressor(&out, &window, .default); try c.writer.writeAll("a body gathered whole"); try c.finish(); const plain = try undo(gpa, out.buffered(), .gzip, &window, 1024); defer gpa.free(plain); try testing.expectEqualStrings("a body gathered whole", plain); // The limit is on what it decodes to, not on what arrived. try testing.expectError(error.TooLarge, undo(gpa, out.buffered(), .gzip, &window, 4)); } /// `s` said `n` times over: what `s ** n` spelled before Zig 0.17 took the /// operator away. inline fn repeat(comptime s: []const u8, comptime n: usize) *const [s.len * n]u8 { comptime { const rows: [n][s.len]u8 = @splat(s[0..s.len].*); const flat: [s.len * n]u8 = @bitCast(rows); return ♭ } }