// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! Content codings, and choosing one the peer will accept. //! //! §8.4.1 of RFC 9110 names the codings and §12.5.3 says how a client asks for //! one. Both halves are here, and neither does any I/O: a content coding is a //! transformation on a byte stream, which in this standard library is a //! `Reader` over a `Reader` and a `Writer` over a `Writer`. That is why this //! sits in the sans-I/O module beside the framing rather than in a layer above //! it — and it is the reason a coding applies identically over HTTP/1.1, //! HTTP/2 and HTTP/3, none of which it knows about. //! //! A content coding is not a transfer coding. §8.4 makes it part of the //! *representation*: it travels end to end, a cache stores it coded, and //! `Content-Length` counts the coded octets. That is the whole reason a //! `Content-Encoding: gzip` request body reaches a handler still compressed, //! which the README argues for at more length — deciding to decode it is the //! handler's business, and this module is what it decodes with. const std = @import("std"); const Reader = std.Io.Reader; const Writer = std.Io.Writer; const flate = std.compress.flate; const token = @import("token.zig"); /// The codings this library has a name for. /// /// The ones it cannot perform are named rather than left out, because a /// `Content-Encoding` that cannot be decoded has to be told apart from one /// nobody has heard of. The first is a representation to hand over as it /// arrived; the second is a message whose sender and recipient disagree about /// what it contains, which is worth refusing. pub const Coding = enum { /// §8.4.1.1: no coding at all, and the one that needs no asking for. identity, /// §8.4.1.3. gzip, /// §8.4.1.2. The zlib format of RFC 1950, which is *not* what most /// senders that say `deflate` actually send — a fair number send raw /// RFC 1951 — and telling those apart means sniffing the first octets. /// Named, not performed. deflate, /// §8.4.1.4's `compress`, which is LZW and long dead. compress, /// RFC 7932 Brotli. br, /// RFC 8878 Zstandard. zstd, pub fn name(c: Coding) []const u8 { return switch (c) { .identity => "identity", .gzip => "gzip", .deflate => "deflate", .compress => "compress", .br => "br", .zstd => "zstd", }; } /// §8.4.1: a coding name is a token and is matched case-insensitively. /// `x-gzip` and `x-compress` are the historical spellings §8.4.1 still /// lists as equivalents, so they are accepted rather than treated as /// codings nobody has heard of. pub fn parse(text: []const u8) ?Coding { const trimmed = token.trimOws(text); if (token.eqlIgnoreCase(trimmed, "identity")) return .identity; if (token.eqlIgnoreCase(trimmed, "gzip")) return .gzip; if (token.eqlIgnoreCase(trimmed, "x-gzip")) return .gzip; if (token.eqlIgnoreCase(trimmed, "deflate")) return .deflate; if (token.eqlIgnoreCase(trimmed, "compress")) return .compress; if (token.eqlIgnoreCase(trimmed, "x-compress")) return .compress; if (token.eqlIgnoreCase(trimmed, "br")) return .br; if (token.eqlIgnoreCase(trimmed, "zstd")) return .zstd; return null; } /// Whether this library can undo it. pub fn canDecode(c: Coding) bool { return switch (c) { .identity, .gzip, .zstd => true, .deflate, .compress, .br => false, }; } /// Whether this library can apply it. /// /// Deliberately not the same set, and zstd is the reason the two had to be /// separated: `std.compress.zstd` ships a decoder and no encoder, so this /// library can read what a peer sends and cannot produce it. A client may /// therefore ask for zstd and undo the answer; a server may not offer it. /// One predicate meaning "supported" could only have been wrong about one /// of those. pub fn canEncode(c: Coding) bool { return switch (c) { .identity, .gzip => true, .deflate, .compress, .br, .zstd => false, }; } }; /// A quality value in thousandths, which is exactly the precision §12.5.3 /// allows: "a weight is a value between 0 and 1 with at most three digits /// after the decimal point". Integers because a preference compared with /// floating point is a preference that can compare unequal to itself. pub const Quality = u16; /// The highest weight, and what an element with no weight at all carries. pub const highest: Quality = 1000; /// §12.5.3's weight, or `highest` when the element carries none. /// /// A weight this cannot read is treated as absent rather than as zero. The /// grammar allows only `q=`, so anything else is a sender doing something the /// specification does not describe — and reading that as "not acceptable" /// would refuse a peer over a parameter nobody meant as a refusal, where /// reading it as "no preference stated" merely ignores it. fn weightOf(element: []const u8) Quality { var rest = element; while (std.mem.indexOfScalar(u8, rest, ';')) |at| { const parameter = token.trimOws(rest[at + 1 ..]); rest = parameter; const eq = std.mem.indexOfScalar(u8, parameter, '=') orelse continue; if (!token.eqlIgnoreCase(token.trimOws(parameter[0..eq]), "q")) continue; var value = token.trimOws(parameter[eq + 1 ..]); if (std.mem.indexOfScalar(u8, value, ';')) |end| value = token.trimOws(value[0..end]); return parseQuality(value) orelse highest; } return highest; } /// `0`, `1`, `0.5`, `1.000` — and nothing else. §12.5.3's grammar is /// `( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] )`, so `1.5` is not a /// weight and neither is `.5`. fn parseQuality(text: []const u8) ?Quality { if (text.len == 0) return null; const whole = text[0]; if (whole != '0' and whole != '1') return null; var thousandths: Quality = if (whole == '1') highest else 0; if (text.len == 1) return thousandths; if (text[1] != '.') return null; const digits = text[2..]; if (digits.len > 3) return null; var scale: Quality = 100; for (digits) |d| { if (!std.ascii.isDigit(d)) return null; const n: Quality = d - '0'; if (whole == '1') { // §12.5.3 allows only zeroes after `1.`, and `1.001` is not a // weight that means anything above the maximum. if (n != 0) return null; } else { thousandths += n * scale; } scale /= 10; } return thousandths; } /// Chooses a coding from what a peer said it would take. /// /// `offered` is in this end's order of preference, so a caller that would /// rather send gzip than nothing passes `&.{ .gzip, .identity }`. Ties in the /// peer's weights are broken by that order, which is the only thing that can /// break them: §12.5.3 gives a client a way to say what it prefers and no way /// to say what it is indifferent about. /// /// Null means the peer refused everything on offer, which §12.5.3 says a /// server "MAY" answer with a 406 — and which it can only mean deliberately, /// since identity is acceptable unless a client goes out of its way to refuse /// it. pub fn selectEncoding(field_value: ?[]const u8, offered: []const Coding) ?Coding { // §12.5.3: "If no Accept-Encoding header field is in the request, any // content coding is considered acceptable by the user agent." Acceptable // is not the same as wanted, and this returns what to *send*: a client // that did not ask gets identity, because a coding applied to a client // that never mentioned one is how a response becomes unreadable to // something in the middle that also never mentioned one. const list = field_value orelse return firstOf(offered, .identity); // "An Accept-Encoding header field with a combined field value that is // empty implies that the user agent does not want any content coding in // response." if (token.trimOws(list).len == 0) return firstOf(offered, .identity); var star: ?Quality = null; var best: ?Coding = null; var best_weight: Quality = 0; for (offered) |candidate| { var stated: ?Quality = null; var it = token.list(list); while (it.next()) |element| { const before_weight = element[0 .. std.mem.indexOfScalar(u8, element, ';') orelse element.len]; const name = token.trimOws(before_weight); if (std.mem.eql(u8, name, "*")) { if (star == null) star = weightOf(element); continue; } const coding = Coding.parse(name) orelse continue; if (coding != candidate) continue; stated = weightOf(element); } // What the client did not name, the wildcard speaks for; and what // neither names is a coding it never asked for. // // Except identity, which §12.5.3 says "is always acceptable, unless // specifically refused". Acceptable is not preferred, and the // difference decides every ordinary request: a client sending // `gzip;q=0.9` and never mentioning identity plainly wants gzip, so // identity's unstated weight has to sit *below* anything the client // troubled to ask for rather than at the top. The lowest weight there // is, then — enough to be chosen when nothing else may be, and never // enough to beat a coding that was actually requested. const weight = stated orelse star orelse (if (candidate == .identity) @as(Quality, 1) else @as(Quality, 0)); if (weight > best_weight) { best = candidate; best_weight = weight; } } return best; } /// The first of `offered` that is `wanted`, or null. fn firstOf(offered: []const Coding, wanted: Coding) ?Coding { for (offered) |c| if (c == wanted) return c; return null; } /// Reads a `Content-Encoding` field and says what has to be undone. /// /// §8.4: "If one or more encodings have been applied to a representation, the /// sender that applied the encodings MUST generate a Content-Encoding header /// field that lists the content codings in the order in which they were /// applied." So undoing them means walking the list backwards, and the last /// element is the first thing to undo. /// /// The common case by a very long way is one coding, which is why this reports /// the shape rather than a list: `identity` for a field that is absent or says /// nothing, one `Coding` for a single coding, and `.unsupported` for anything /// this library cannot undo on its own — several codings included, since /// applying gzip twice is legal, vanishingly rare, and not worth a decoder /// stack that would exist for nobody. pub const Applied = union(enum) { /// Nothing to undo. identity, /// Exactly one coding, and this library can undo it. one: Coding, /// Something this library will not undo: a coding it cannot perform, or /// more than one of them. The representation is the caller's to hand over /// as it arrived or to refuse. unsupported, }; pub fn applied(field_value: ?[]const u8) Applied { const list = field_value orelse return .identity; var found: ?Coding = null; var it = token.list(list); while (it.next()) |element| { const name = token.trimOws(element); if (name.len == 0) continue; const coding = Coding.parse(name) orelse return .unsupported; // §8.4.1.1: `identity` in a `Content-Encoding` is a coding that does // nothing, so it neither counts as a coding to undo nor makes the // field unreadable. if (coding == .identity) continue; if (found != null) return .unsupported; if (!coding.canDecode()) return .unsupported; found = coding; } return if (found) |c| .{ .one = c } else .identity; } // -- the codec itself ------------------------------------------------------- // // std's flate already is what this library wants -- a `Reader` over a `Reader` // and a `Writer` over a `Writer` -- so it is re-exported rather than wrapped. // What is worth adding is the two numbers a caller has to get right, because // both are asserted rather than reported: a window smaller than // `window_len` panics inside the compressor, and so does an output buffer of // eight octets or fewer. That is the same shape of trap as the TLS handshake // buffers, and it is worth naming in one place rather than being discovered. /// The window either direction needs, and the assertion both `Compress.init` /// and `Decompress.init` make about the buffer they are handed. 64 KiB, which /// is per stream and not per connection: a server compressing many responses /// at once pays this for each, which is the number to multiply before turning /// compression on for everything. pub const window_len = flate.max_window_len; /// The smallest output buffer `Compress.init` will accept. It asserts /// `output.buffer.len > 8`. pub const min_output_buffer_len = 9; pub const Compressor = flate.Compress; pub const Decompressor = flate.Decompress; /// zstd's decoder, which is all of zstd this library has: §8.4.1 of RFC 9110 /// registers the coding and `std.compress.zstd` implements one direction of /// it. pub const ZstdDecompressor = std.compress.zstd.Decompress; /// The window a zstd stream may refer back into. /// /// This is the number to look at before turning zstd on anywhere that decodes /// what a peer chose to send. §3.1.1.1.2 of RFC 8878 lets a *sender* pick the /// window and a decoder that will not allocate it has to refuse the stream, so /// this is a ceiling on what will be accepted rather than a buffer that is /// always full. std's default is eight megabytes, which is 128 times what gzip /// wants, and it is per stream: a server decoding request bodies pays it for /// every concurrent one. pub const zstd_window_len: u32 = std.compress.zstd.default_window_len; /// How large a buffer `Decoder.init` wants for `c`. pub fn decodeBufferLen(c: Coding) usize { return switch (c) { .gzip => window_len, // §3.1.1.1.2 again: the window, and a block on top of it, which is // what `Decompress.init` asserts. .zstd => zstd_window_len + std.compress.zstd.block_size_max, .identity, .deflate, .compress, .br => 0, }; } /// A decoder for any coding this library can undo, as one type. /// /// The two std decompressors are different types that both happen to expose a /// `reader`, so something has to name the choice — and a caller that holds one /// of these allocates once and asks for a reader, rather than switching on the /// coding at every use. It must not move once `reader` has been taken: what /// comes back points inside it. pub const Decoder = union(enum) { gzip: Decompressor, zstd: ZstdDecompressor, /// `buffer` must be `decodeBufferLen(c)` octets. pub fn init(c: Coding, input: *Reader, buffer: []u8) error{Unsupported}!Decoder { return switch (c) { .gzip => .{ .gzip = gzipDecompressor(input, buffer) }, .zstd => .{ .zstd = ZstdDecompressor.init(input, buffer, .{ .window_len = zstd_window_len, // Off, and not by choice. §3.1.1 of RFC 8878 makes a // frame's content checksum optional and cheap to verify, // and `zstd` writes one by default — but verifying it in // Zig 0.16.0 reaches `@panic("TODO all those bytes // written needed to go through the hasher too")` in // `std/compress/zstd/Decompress.zig`, which is an // unimplemented path rather than a rejected frame. // Handing a peer's octets to a panic is not a trade worth // making, so this stays false until the toolchain moves. // The decode is unaffected and a corrupt frame still // fails on its own structure; what is lost is the last // check that what came out is what went in. .verify_checksum = false, }), }, .identity, .deflate, .compress, .br => error.Unsupported, }; } pub fn reader(d: *Decoder) *Reader { return switch (d.*) { .gzip => |*g| &g.reader, .zstd => |*z| &z.reader, }; } }; /// A gzip compressor writing into `output`. `buffer` must be at least /// `window_len`, and `output`'s own buffer more than eight octets. /// /// The stream has to be ended with `Compressor.finish`, which writes §2.2 of /// RFC 1952's final block and the footer carrying the CRC and the /// uncompressed size. A `flush` alone is not that: it moves what is buffered /// and leaves the member unfinished, and what the peer gets then is a body /// that stops mid-stream — `error.EndOfStream` to a reader that checks, and a /// truncated representation to one that does not. pub fn gzipCompressor( output: *Writer, buffer: []u8, level: Compressor.Options, ) Writer.Error!Compressor { return Compressor.init(output, buffer, .gzip, level); } /// A gzip decompressor reading from `input`. /// /// `buffer` may be empty, which puts std's decompressor in its direct mode and /// costs no window at all — the right choice when the caller is streaming /// through to somewhere else. Otherwise it must be at least `window_len`. pub fn gzipDecompressor(input: *Reader, buffer: []u8) Decompressor { return Decompressor.init(input, .gzip, buffer); } /// A warning that belongs with every one of these, for as long as this /// library is built with Zig 0.16.0. /// /// **That toolchain's flate decoder panics on some malformed gzip members.** /// Not an error — a panic, which takes the process with it and cannot be /// caught. `tossBitsShort` adds `consumed_bits` where it must subtract them, /// so it reports more bits available than remain; the decoder carries on and /// underflows a `usize` afterwards in `peekBitsEnding`, on a subtraction that /// is itself correct. Twenty-five octets are enough, and this library's /// `coded-bodies` fuzz target met it within a quarter of an hour of existing. /// /// Fixed upstream by ziglang/zig#35815, merged 2026-06-17 for 0.17.0 — after /// 0.16.0 had shipped, which is why it is still here. `flake.nix` backports /// that exact change for the devshell, so the tests and the fuzzer do not meet /// it; `package.nix` builds with nixpkgs' unpatched compiler, so a binary /// built from this repository is exposed exactly as any other program on this /// toolchain is. The warning goes away when the pin moves past the fix, and /// `--replace-fail` in the flake is what will say so. /// /// What it means for a caller. Undoing a coding is opt-in everywhere here, and /// until the pin moves that default is load-bearing rather than merely /// conservative: /// /// * A **client** decoding a response trusts the server it chose to talk to. /// The exposure is a server that can crash a client which asked it for gzip, /// which is a fair trade for a client that wanted the coding. /// * A **server** decoding a request body trusts nobody. Twenty-five octets /// from anybody who can reach the port is a denial of service on the whole /// process. Do not enable `server.compression.RequestDecoding` on an /// untrusted network until this library is built with a Zig that has the /// fix. /// /// Compressing is unaffected: the encoder is a different path and reads only /// what this end produced. pub const UndoError = error{ OutOfMemory, /// The octets are not a valid stream of that coding. Malformed, /// The decoded representation is larger than the caller allowed. /// /// §8.4 of RFC 9110 puts no bound on what a coding may expand to, and /// nothing in a gzip member says up front how big it decodes to — the /// size in its footer is eight octets a sender chooses and is checked /// only at the end, long after the memory has been spent. A few hundred /// octets of zeroes decode to gigabytes, which is why this bound is a /// parameter rather than a default: every caller has a number and none of /// them is this module's to pick. TooLarge, /// A coding this library cannot undo. Unsupported, }; /// Undoes one coding on a representation already gathered whole. /// /// The streaming case wraps a reader instead — see `gzipDecompressor` — and is /// what the HTTP/1.x driver does. This is for the two multiplexed versions, /// where a stream's octets arrive interleaved with every other stream's, so /// there is no single reader to wrap and the body is accumulated before /// anything can be done to it. /// /// `window` must be `decodeBufferLen(what)` octets — which differs by coding, /// and by two orders of magnitude between the two this library can undo. What /// comes back is the caller's to free. pub fn undo( gpa: std.mem.Allocator, coded: []const u8, what: Coding, window: []u8, limit: usize, ) UndoError![]u8 { if (what == .identity) return gpa.dupe(u8, coded); var source: Reader = .fixed(coded); var d = Decoder.init(what, &source, window) catch return error.Unsupported; return d.reader().allocRemaining(gpa, .limited(limit)) catch |err| switch (err) { error.OutOfMemory => error.OutOfMemory, error.StreamTooLong => error.TooLarge, else => error.Malformed, }; } const testing = std.testing; test "RFC 9110 12.5.3: a weight is read, and only in the shape given" { try testing.expectEqual(@as(?Quality, 0), parseQuality("0")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1")); try testing.expectEqual(@as(?Quality, 500), parseQuality("0.5")); try testing.expectEqual(@as(?Quality, 1), parseQuality("0.001")); try testing.expectEqual(@as(?Quality, 0), parseQuality("0.000")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1.0")); try testing.expectEqual(@as(?Quality, 1000), parseQuality("1.000")); // The grammar is `( "0" [ "." 0*3DIGIT ] ) / ( "1" [ "." 0*3("0") ] )`, // so each of these is not a weight. try testing.expectEqual(@as(?Quality, null), parseQuality(".5")); try testing.expectEqual(@as(?Quality, null), parseQuality("1.5")); try testing.expectEqual(@as(?Quality, null), parseQuality("0.1234")); try testing.expectEqual(@as(?Quality, null), parseQuality("2")); try testing.expectEqual(@as(?Quality, null), parseQuality("")); try testing.expectEqual(@as(?Quality, null), parseQuality("0,5")); } test "RFC 9110 12.5.3: choosing a coding a client will take" { const both = [_]Coding{ .gzip, .identity }; // The ordinary case, and the one every browser sends. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip, deflate, br", &both)); // No field at all means any coding is *acceptable*, and identity is what // gets sent: a coding nobody asked for is one something in the middle did // not ask for either. try testing.expectEqual(@as(?Coding, .identity), selectEncoding(null, &both)); // An empty field value is the client saying it wants no coding. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("", &both)); // A weight of zero is a refusal and not a low preference. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("gzip;q=0", &both)); // Barely wanted still beats never asked for: identity's unstated weight // is the lowest there is, so any coding the client troubled to request // outranks it. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip;q=0.001", &both)); try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("gzip;q=0.9", &both)); // Preference is read, not guessed: the client would rather have identity. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("gzip;q=0.3, identity;q=0.9", &both)); // The wildcard stands for everything not named. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("*", &both)); try testing.expectEqual(@as(?Coding, .identity), selectEncoding("*;q=0, identity", &both)); // Refusing everything, which §12.5.3 permits and a server answers with a // 406 if it has nothing left to send. try testing.expectEqual(@as(?Coding, null), selectEncoding("*;q=0", &both)); try testing.expectEqual(@as(?Coding, null), selectEncoding("identity;q=0, gzip;q=0", &both)); // A coding this library cannot perform is not chosen however much the // client wants it, because `offered` is what the caller can actually do. try testing.expectEqual(@as(?Coding, .identity), selectEncoding("br;q=1.0, identity;q=0.1", &both)); // Case and whitespace are not significant; §8.4.1 matches names // case-insensitively and §5.6.1 allows the spaces. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding(" GZIP ; Q=1.0 ", &both)); // `x-gzip` is the spelling §8.4.1 still lists as equivalent. try testing.expectEqual(@as(?Coding, .gzip), selectEncoding("x-gzip", &both)); // A caller offering nothing gets nothing, rather than identity by // accident. try testing.expectEqual(@as(?Coding, null), selectEncoding("gzip", &.{})); } test "RFC 9110 8.4: what a Content-Encoding says has to be undone" { try testing.expectEqual(Applied.identity, applied(null)); try testing.expectEqual(Applied.identity, applied("")); // §8.4.1.1: `identity` there is a coding that does nothing. try testing.expectEqual(Applied.identity, applied("identity")); try testing.expectEqual(Applied{ .one = .gzip }, applied("gzip")); try testing.expectEqual(Applied{ .one = .gzip }, applied("identity, gzip")); // Anything this library will not undo on its own, which the caller hands // over as it arrived or refuses. try testing.expectEqual(Applied.unsupported, applied("br")); try testing.expectEqual(Applied.unsupported, applied("gzip, gzip")); try testing.expectEqual(Applied.unsupported, applied("gzip, br")); try testing.expectEqual(Applied.unsupported, applied("something-invented")); } test "RFC 9110 8.4: undoing a coding is bounded, because a coding is not" { const gpa = testing.allocator; var window: [window_len]u8 = undefined; // A member that decodes to far more than it weighs, which is the whole // shape of the attack: the cost of making it is nothing and the cost of // undoing it is whatever the recipient allows. const zeroes = [_]u8{0} ** 200_000; var coded_buf: [4096]u8 = undefined; var out: std.Io.Writer = .fixed(&coded_buf); var w = try gzipCompressor(&out, &window, .default); try w.writer.writeAll(&zeroes); try w.finish(); const bomb = out.buffered(); try testing.expect(bomb.len < 1000); // Allowed, and it decodes. const roomy = try undo(gpa, bomb, .gzip, &window, 1 << 20); defer gpa.free(roomy); try testing.expectEqualSlices(u8, &zeroes, roomy); // Refused, rather than allocated and then regretted. Nothing in the // member says how large it is until it has already been decoded. try testing.expectError(error.TooLarge, undo(gpa, bomb, .gzip, &window, 1024)); // Not a gzip stream at all. try testing.expectError(error.Malformed, undo(gpa, "not gzip", .gzip, &window, 1 << 20)); // Identity copies, so that a caller has one thing to free either way. const same = try undo(gpa, "unchanged", .identity, &window, 1 << 20); defer gpa.free(same); try testing.expectEqualStrings("unchanged", same); try testing.expectError(error.Unsupported, undo(gpa, "", .br, &window, 1 << 20)); } test "RFC 8878: a zstd frame this library could not have produced" { // The point of a fixed vector rather than a round trip: `std.compress.zstd` // is a decoder and nothing here is an encoder, so there is no way to // produce this and no temptation to test the decoder against itself. These // octets came from libzstd 1.5.7 — Facebook's own implementation — and are // 1056 octets of repeated text in 77. const frame = [_]u8{ 0x28, 0xb5, 0x2f, 0xfd, 0x64, 0x20, 0x03, 0xfd, 0x01, 0x00, 0x22, 0x44, 0x0d, 0x11, 0xb0, 0xeb, 0x20, 0x4b, 0x84, 0x2d, 0x02, 0x96, 0x34, 0xc5, 0x98, 0x24, 0xe0, 0xa2, 0xc9, 0x42, 0x04, 0x0e, 0x4f, 0xe2, 0xae, 0xbb, 0x9c, 0x9b, 0xc7, 0x46, 0xa1, 0xd3, 0x54, 0x79, 0xb5, 0x91, 0x7c, 0x16, 0x4e, 0xba, 0xc5, 0xe0, 0x8d, 0x19, 0x0e, 0x3f, 0xaf, 0x24, 0xef, 0x4c, 0x62, 0x8b, 0xdf, 0x58, 0xb4, 0x08, 0x01, 0x00, 0xc2, 0xf6, 0xa2, 0x0a, 0x51, 0xa6, 0x6a, 0x4c, 0xe4, }; const expected = "a zstd frame decoded by std, from a vector nothing here produced. " ** 16; // §2.2 of RFC 8878: a frame begins with this magic number. Checking it // keeps "it decoded" from also being true of some other container. try testing.expectEqual(@as(u8, 0x28), frame[0]); try testing.expectEqual(@as(u8, 0xb5), frame[1]); const gpa = testing.allocator; const window = try gpa.alloc(u8, decodeBufferLen(.zstd)); defer gpa.free(window); const plain = try undo(gpa, &frame, .zstd, window, 1 << 20); defer gpa.free(plain); try testing.expectEqualStrings(expected, plain); // Bounded the same way gzip is, and for the same reason: §3.1.1 puts the // decoded size in the frame header as something the *sender* wrote. try testing.expectError(error.TooLarge, undo(gpa, &frame, .zstd, window, 100)); // And not a zstd frame at all. try testing.expectError(error.Malformed, undo(gpa, "not zstd", .zstd, window, 1 << 20)); } test "the two questions a coding answers are not the same question" { // zstd is why `supported` had to become two predicates: this library can // read it and cannot write it, so one answer would have been wrong about // one direction. A client may ask for it; a server may not offer it. try testing.expect(Coding.zstd.canDecode()); try testing.expect(!Coding.zstd.canEncode()); try testing.expect(Coding.gzip.canDecode()); try testing.expect(Coding.gzip.canEncode()); // identity is both, trivially, being nothing at all. try testing.expect(Coding.identity.canDecode()); try testing.expect(Coding.identity.canEncode()); // And the ones this library only knows the name of. for ([_]Coding{ .deflate, .compress, .br }) |c| { try testing.expect(!c.canDecode()); try testing.expect(!c.canEncode()); } // So a `Content-Encoding: zstd` is now something to undo rather than // something to hand over coded. try testing.expectEqual(Applied{ .one = .zstd }, applied("zstd")); // The window is the cost, and the difference is the thing to notice. try testing.expectEqual(window_len, decodeBufferLen(.gzip)); try testing.expect(decodeBufferLen(.zstd) > 100 * decodeBufferLen(.gzip)); } test "a gzip stream survives being written and read again" { // The codec is std's, so what this pins is the two buffer sizes above and // the container being gzip rather than zlib or raw -- which is the one // thing a caller of `flate` can get wrong without any error saying so, // because all three produce a perfectly good stream of the wrong kind. const original = "the same sentence, repeated, is what a compressor is for. " ** 16; // A fixed buffer and not an `Allocating`: that one starts with a buffer of // no octets at all, and `Compress.init` asserts more than eight. Which is // exactly the trap `min_output_buffer_len` exists to name, met here on the // first attempt at using this module. var out_buf: [4096]u8 = undefined; var compressed: std.Io.Writer = .fixed(&out_buf); var window: [window_len]u8 = undefined; var c = try gzipCompressor(&compressed, &window, .default); try c.writer.writeAll(original); // `finish` and not `flush`: §2.2 of RFC 1952 ends a gzip member with a // final deflate block and an eight-octet footer carrying the CRC and the // uncompressed size, and only `finish` writes them. A flushed-but-unended // stream decompresses to `error.EndOfStream`, which is what this test did // when it was written the obvious way. try c.finish(); const coded = compressed.buffered(); // §2.3.1 of RFC 1952: a gzip member starts with these two octets and then // the deflate method. Checking them is how "it round-tripped" stops also // being true of the raw and zlib containers. try testing.expect(coded.len > 2); try testing.expectEqual(@as(u8, 0x1f), coded[0]); try testing.expectEqual(@as(u8, 0x8b), coded[1]); try testing.expect(coded.len < original.len); var source: std.Io.Reader = .fixed(coded); var out_window: [window_len]u8 = undefined; var d = gzipDecompressor(&source, &out_window); const back = try d.reader.allocRemaining(testing.allocator, .limited(1 << 20)); defer testing.allocator.free(back); try testing.expectEqualStrings(original, back); }