From 1ce4a238a49cdb33e9df73c6e48c84e622bc46ff Mon Sep 17 00:00:00 2001 From: "Jeffrey C. Ollie" Date: Sat, 5 Sep 2026 19:13:24 -0500 Subject: [PATCH] Fuzz the Layer III encoder, and test what structure cannot show The target added with the encoder checked that its output was well formed and nothing else, which is exactly the shape of assurance the Layer III fault defeated: those frames had correct sizes, parseable side information, exact Huffman data and the right number of finite samples, and carried noise. A contract about structure says nothing about whether the structure still means anything. So the target now also encodes a tone and insists it comes back. Not at any particular quality -- a signal-to-noise ratio is the wrong measure for a coder that deliberately discards what cannot be heard, and Layer II at 192 kbit/s in stereo has only about 13 dB of ceiling to begin with, so a bar set near what a good encode reaches fails on settings that are behaving perfectly. A 300,000 round soak made that point by finding a legitimate 7.89 dB case against a bar of 10. The bar is now four decibels and means what it says: the signal is still there rather than destroyed, and destruction reads as a fraction of one. Three gaps in the target itself, each found by looking rather than guessing: The tone restarted at phase zero every frame, so a run of them was a sequence of discontinuities rather than a signal, and nothing could have been measured across it. It is now continuous, and a fundamental with two harmonics rather than a bare sine -- one sine occupies a single scalefactor band and leaves the loops nothing to trade off, so an encoder can be badly wrong and still get it right. The shape and the layer were advancing together. With six shapes and the layer alternating by round parity, the two were locked: even-numbered shapes only ever met Layer II and odd-numbered ones only Layer III, so half the pairings never occurred and the tone never met Layer III at all. The layer now advances once per pass over the shapes. And the frames were being generated one at a time into the same buffer, so the input was gone by the time the output arrived. The whole run is built first, which is what makes comparing them possible. The sharp guards for the original fault live beside the code instead, where they can aim at it: one checks that a quantizer step too small to code is refused rather than clamped, and one rebuilds a granule from what the loops chose and insists it still resembles the spectrum it came from. Both fail when the clamp is put back -- verified by putting it back -- and the fuzz target does not, because its tone is an easier thing to code than the signal that first exposed it. Breadth and aim are different jobs and it is worth being clear about which is which. Campaigns run clean: 300,000 rounds in ReleaseSafe and 30,000 in Debug. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Aj95dBZhkUtaGfEW8ZXbHx --- README.md | 21 +++++- src/encode3.zig | 88 +++++++++++++++++++++++ src/fuzz_test.zig | 176 ++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 263 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 07ca8ce..4adde4f 100644 --- a/README.md +++ b/README.md @@ -327,10 +327,27 @@ must be the size its header claims, must parse back as the scale information it was written as, must decode, and must give back the right number of finite samples — and silence in must be silence out exactly. +That contract is not enough on its own, and the Layer III encoder proved it. +Its worst fault produced frames that satisfied every clause of it: correct +sizes, parseable side information, exact Huffman data, the right number of +finite samples — carrying noise. Structure can be perfect while the thing the +structure describes is meaningless. So the target also encodes a tone and +insists it comes back: not at any particular quality, since a signal-to-noise +ratio is the wrong measure for a coder that deliberately discards what cannot +be heard, but far enough above nothing to tell a coded signal from a destroyed +one. + +The sharper guards for that fault live beside the code instead, where they can +aim: one checks that a quantizer step too small to code is refused rather than +clamped, and one rebuilds a granule from what the loops chose and insists it +still resembles the spectrum it was made from. Both fail if the clamp comes +back; the fuzz target, whose tone is an easier thing to code than the signal +that first exposed it, does not. Breadth and aim are different jobs. + Its input is not just noise, either. Uniform noise lights every subband at a similar level, so the allocation loop takes the same path every time; the -target walks six shapes instead, adding silence, tones, samples well past -full scale, sparse impulses and direct current, which are what reach the +target walks six shapes instead, adding silence, harmonic tones, samples well +past full scale, sparse impulses and direct current, which are what reach the corners where subbands are given everything or nothing. What the loop decided is then read back out of the encoder's own output with the decoder's parser, which is both a check that the two agree about the layout and the only way to diff --git a/src/encode3.zig b/src/encode3.zig index 33f7b01..7a38824 100644 --- a/src/encode3.zig +++ b/src/encode3.zig @@ -815,3 +815,91 @@ test "what is encoded decodes back to something like the input" { return err; }; } + +test "a granule given plenty of bits comes back close to what went in" { + // The guard on the fault that cost the most to find. Every structural + // check passed while the encoder was producing noise: the frames were + // well formed, the side information parsed, the Huffman data read back + // exactly, and the arithmetic was self consistent. What was wrong was + // that the granule no longer stood for the spectrum it was made from, and + // the only way to see that is to rebuild it and look. + // + // The quantizer had been clamping values too large for any table to name. + // That did not make the bit count wrong in a way any assertion caught -- + // it made it a lie, so a granule of saturated lines fit a budget it should + // never have fit, and the rate loop settled on a step far too small. + var encoder = try Encoder.init(.{ .sample_rate = 44100, .channels = 1, .bitrate = 256_000 }); + var prng = std.Random.DefaultPrng.init(0x51ee); + const random = prng.random(); + + const Shape = enum { flat, decaying, spikes }; + for (std.enums.values(Shape)) |shape| { + var spectrum: [granule_len]f32 = undefined; + switch (shape) { + .flat => for (&spectrum) |*value| { + value.* = random.float(f32) - 0.5; + }, + .decaying => for (&spectrum, 0..) |*value, k| { + const decay = std.math.pow(f32, 0.995, @floatFromInt(k)); + value.* = (random.float(f32) - 0.5) * decay; + }, + // A wide dynamic range is what exposed the fault: a few lines far + // above the rest saturate on their own while everything around + // them quantizes to nothing. + .spikes => { + for (&spectrum) |*value| value.* = (random.float(f32) - 0.5) * 1e-4; + for ([_]usize{ 0, 3, 40, 200 }) |k| spectrum[k] = 30.0; + }, + } + + // Thresholds far below the signal, so the distortion loop pushes the + // scalefactors as hard as it can -- which is the state the fault + // showed up in. + var thresholds: [long_bands]f32 = @splat(1e-20); + const coded = encoder.codeGranule(&spectrum, &thresholds, 4000); + + var signal: f64 = 0; + var noise: f64 = 0; + for (0..long_bands + 1) |band| { + const from = encoder.starts[band]; + const to = if (band < long_bands) encoder.starts[band + 1] else granule_len; + const factor = if (band < long_bands) coded.scalefac[band] else 0; + const gain = exp4(Encoder.exponentOf(coded.global_gain, factor)); + for (from..@min(to, granule_len)) |k| { + const size = std.math.pow(f32, @floatFromInt(@abs(coded.quantized[k])), 4.0 / 3.0); + const back = if (coded.quantized[k] < 0) -size * gain else size * gain; + signal += @as(f64, spectrum[k]) * spectrum[k]; + noise += @as(f64, back - spectrum[k]) * (back - spectrum[k]); + } + } + const snr = 10 * std.math.log10(signal / @max(noise, 1e-30)); + testing.expect(snr > 15) catch |err| { + std.debug.print("{t} spectrum rebuilt at {d:.2} dB from 4000 bits\n", .{ shape, snr }); + return err; + }; + } +} + +test "a step too small to code is refused rather than clamped" { + var encoder = try Encoder.init(.{ .sample_rate = 44100, .channels = 1, .bitrate = 128_000 }); + var spectrum: [granule_len]f32 = @splat(0); + spectrum[0] = 1.0; + var coded: Coded = .{}; + + // Gain 210 is the point where the exponent is zero, so the quantized + // value is about one and everything is codeable. + try testing.expect(encoder.quantize( + &spectrum, + &@as([long_bands]u8, @splat(0)), + 210, + &coded, + )); + // Far below that the value cannot be named by any table, and saying so is + // the whole point: the caller has to learn that its step is too small. + try testing.expect(!encoder.quantize( + &spectrum, + &@as([long_bands]u8, @splat(0)), + 0, + &coded, + )); +} diff --git a/src/fuzz_test.zig b/src/fuzz_test.zig index b3f02ba..842fd44 100644 --- a/src/fuzz_test.zig +++ b/src/fuzz_test.zig @@ -1107,6 +1107,14 @@ const EncoderCoverage = struct { shapes: u8 = 0, mono: usize = 0, stereo: usize = 0, + /// Rounds where the input was a tone with bits enough that the signal had + /// to survive being coded, and did. + /// + /// Everything else here is structural, and structure is not enough: the + /// worst fault this encoder has had produced frames that were correct in + /// every measurable way and carried noise. Only listening to the output + /// catches that. + fidelity: usize = 0, fn add(self: *EncoderCoverage, other: EncoderCoverage) void { self.frames += other.frames; @@ -1118,6 +1126,7 @@ const EncoderCoverage = struct { self.shapes |= other.shapes; self.mono += other.mono; self.stereo += other.stereo; + self.fidelity += other.fidelity; } }; @@ -1141,7 +1150,14 @@ const Shape = enum { direct_current, }; -fn fillShape(choices: Choices, shape: Shape, rate: u32, channels: usize, pcm: []f32) void { +fn fillShape( + choices: Choices, + shape: Shape, + rate: u32, + channels: usize, + at: usize, + pcm: []f32, +) void { switch (shape) { .silence => @memset(pcm, 0), .noise => for (pcm) |*value| { @@ -1161,18 +1177,34 @@ fn fillShape(choices: Choices, shape: Shape, rate: u32, channels: usize, pcm: [] @memset(pcm, 0); const count = choices.int(u8, "enc_impulses", 1, 8); for (0..count) |_| { - const at = choices.int(u16, "enc_impulse_at", 0, @intCast(pcm.len - 1)); - pcm[at] = if (choices.boolean("enc_impulse_sign")) 0.95 else -0.95; + const where = choices.int(u16, "enc_impulse_at", 0, @intCast(pcm.len - 1)); + pcm[where] = if (choices.boolean("enc_impulse_sign")) 0.95 else -0.95; } }, .tone => { - const hz = @as(f64, @floatFromInt(choices.int(u16, "enc_hz", 20, 20000))); - const amplitude = @as(f64, @floatFromInt(choices.int(u8, "enc_amp", 1, 200))) / 200.0; + // Loud, and in the middle of the band where hearing is sharpest. + // The point of this shape is that its survival can be insisted + // on, and a tone that is very quiet or very high need not survive + // at all -- a model that spends bits on either is a bad model, + // not a good one. Those extremes are covered structurally by the + // other shapes; 8 kHz is under the lowest supported Nyquist. + const hz = @as(f64, @floatFromInt(choices.int(u16, "enc_hz", 100, 2000))); + const amplitude = @as(f64, @floatFromInt(choices.int(u8, "enc_amp", 50, 160))) / 200.0; for (0..pcm.len / channels) |n| { - const phase = 2.0 * std.math.pi * hz * @as(f64, @floatFromInt(n)) / + // Counted from the start of the whole run, not of this frame, + // so that the tone is one continuous signal rather than one + // that jumps back to its start every frame. + const t = 2.0 * std.math.pi * @as(f64, @floatFromInt(at + n)) / @as(f64, @floatFromInt(rate)); - const value: f32 = @floatCast(amplitude * @sin(phase)); - for (0..channels) |ch| pcm[n * channels + ch] = value; + // A fundamental and two harmonics rather than a bare sine. + // One sine is an unusually easy thing to code -- it occupies + // a single scalefactor band and leaves the loops nothing to + // trade off -- and an encoder can be badly wrong while still + // getting it right. + const value = amplitude * (0.6 * @sin(hz * t) + + 0.3 * @sin(2.0 * hz * t) + + 0.1 * @sin(3.0 * hz * t)); + for (0..channels) |ch| pcm[n * channels + ch] = @floatCast(value); } }, } @@ -1205,7 +1237,23 @@ fn checkEncoder( encode12.bitRates(rate, &offered) else encode3.bitRates(rate, &offered)) catch return .{}; - const bitrate = available[choices.int(u8, "enc_bitrate", 0, @intCast(available.len - 1))]; + // A tone is the one shape whose survival can be measured, so those rounds + // are given the most bits the format offers and run long enough to + // measure: the point of them is the fidelity check at the end, and a + // round that cannot reach it is a round that tests nothing new. + const bitrate = if (shape == .tone) + // The upper half rather than the very top: plenty for a tone to + // survive, but not so much that the loops never have to choose. + available[ + choices.int( + u8, + "enc_bitrate", + @intCast(available.len / 2), + @intCast(available.len - 1), + ) + ] + else + available[choices.int(u8, "enc_bitrate", 0, @intCast(available.len - 1))]; var coverage: EncoderCoverage = .{}; if (layer == 3) return try checkEncoder3(choices, gpa, shape, rate, channels, bitrate); @@ -1220,16 +1268,26 @@ fn checkEncoder( coverage.shapes |= @as(u8, 1) << @intCast(@intFromEnum(shape)); - const frames = choices.int(u8, "enc_frames", 1, 4); - const pcm = try gpa.alloc(f32, encode12.frame_samples * @as(usize, channels)); + // Six frames rather than one, so that a tone has run long enough for + // the filterbanks to fill and the fidelity check below to mean something. + const frames = if (shape == .tone) 6 else choices.int(u8, "enc_frames", 1, 6); + const per_frame = encode12.frame_samples * @as(usize, channels); + const pcm = try gpa.alloc(f32, per_frame * frames); defer gpa.free(pcm); var stream: std.ArrayList(u8) = .empty; defer stream.deinit(gpa); var frame: [encode12.max_frame_bytes]u8 = undefined; - for (0..frames) |_| { - fillShape(choices, shape, rate, channels, pcm); - const size = try encoder.encodeFrame(pcm, &frame); + for (0..frames) |f| { + fillShape( + choices, + shape, + rate, + channels, + f * encode12.frame_samples, + pcm[f * per_frame ..][0..per_frame], + ); + const size = try encoder.encodeFrame(pcm[f * per_frame ..][0..per_frame], &frame); coverage.frames += 1; // The header has to describe the frame that carries it. @@ -1282,9 +1340,59 @@ fn checkEncoder( if (shape == .silence) { for (audio.samples) |sample| try std.testing.expectEqual(@as(f32, 0), sample); } + // And a tone given bits enough has to come back out as that tone. + if (shape == .tone) { + if (fidelityOf(pcm, audio.samples, channels, 481)) |snr| { + coverage.fidelity += 1; + std.testing.expect(snr > fidelity_floor) catch |err| { + std.debug.print("layer 2 tone survived at only {d:.2} dB\n", .{snr}); + return err; + }; + } + } return coverage; } +/// The bar a coded tone has to clear, in decibels. +/// +/// This is a "the signal is still there" check and not a quality one. How +/// good the encoders sound is measured by `tools/check-encoder.py` against +/// the ceiling each layer's budget sets, which for Layer II at 192 kbit/s in +/// stereo is only about 13 dB to begin with -- so a bar set anywhere near +/// what a good encode reaches would fail on settings that are behaving +/// perfectly. What this catches is the signal being destroyed rather than +/// merely coded coarsely, and destruction reads as a fraction of a decibel. +const fidelity_floor = 4.0; + +/// How much of a tone survived being encoded and decoded again, in decibels, +/// or null when the run was too short or too poorly resourced to say. +/// +/// The filterbanks delay the signal, and Layer III's transform delays it by a +/// granule more; both are fixed and known, so the comparison is against the +/// input shifted by that much. Only a tone is worth checking this way -- the +/// other shapes are noise, impulses or silence, where a signal-to-noise ratio +/// against the input says little. +fn fidelityOf( + input: []const f32, + decoded: []const f32, + channels: usize, + delay: usize, +) ?f64 { + const skip = (delay + 2400) * channels; + if (decoded.len <= skip or input.len <= skip) return null; + var signal: f64 = 0; + var noise: f64 = 0; + var n = skip; + while (n < @min(decoded.len, input.len)) : (n += 1) { + const want = input[n - delay * channels]; + const got = decoded[n]; + signal += @as(f64, want) * want; + noise += @as(f64, got - want) * (got - want); + } + if (signal == 0) return null; + return 10 * std.math.log10(signal / @max(noise, 1e-30)); +} + /// Encode arbitrary samples as Layer III and check what comes back. /// /// There is no allocation field to read back here the way there is for Layer @@ -1310,16 +1418,26 @@ fn checkEncoder3( if (channels == 1) coverage.mono += 1 else coverage.stereo += 1; coverage.shapes |= @as(u8, 1) << @intCast(@intFromEnum(shape)); - const frames = choices.int(u8, "enc_frames", 1, 4); - const pcm = try gpa.alloc(f32, encode3.frame_samples * @as(usize, channels)); + // Six frames rather than one, so that a tone has run long enough for + // the filterbanks to fill and the fidelity check below to mean something. + const frames = if (shape == .tone) 6 else choices.int(u8, "enc_frames", 1, 6); + const per_frame = encode3.frame_samples * @as(usize, channels); + const pcm = try gpa.alloc(f32, per_frame * frames); defer gpa.free(pcm); var stream: std.ArrayList(u8) = .empty; defer stream.deinit(gpa); var frame: [encode3.max_frame_bytes]u8 = undefined; - for (0..frames) |_| { - fillShape(choices, shape, rate, channels, pcm); - const size = try encoder.encodeFrame(pcm, &frame); + for (0..frames) |f| { + fillShape( + choices, + shape, + rate, + channels, + f * encode3.frame_samples, + pcm[f * per_frame ..][0..per_frame], + ); + const size = try encoder.encodeFrame(pcm[f * per_frame ..][0..per_frame], &frame); coverage.frames += 1; const header = Header.parse(frame[0..4].*) orelse return error.NotAHeader; @@ -1366,6 +1484,15 @@ fn checkEncoder3( if (shape == .silence) { for (audio.samples) |sample| try std.testing.expectEqual(@as(f32, 0), sample); } + if (shape == .tone) { + if (fidelityOf(pcm, audio.samples, channels, 481 + 576)) |snr| { + coverage.fidelity += 1; + std.testing.expect(snr > fidelity_floor) catch |err| { + std.debug.print("layer 3 tone survived at only {d:.2} dB\n", .{snr}); + return err; + }; + } + } return coverage; } @@ -1428,12 +1555,16 @@ test "encoded streams decode back to the shape they claim" { // far heavier than the others and runs proportionally fewer times. The // shapes are walked in turn rather than drawn, so that even the shortest // run tries all of them; the fuzzer's own entry point still chooses. + // The layer advances once per pass over the shapes rather than once per + // round. Alternating both together locks them to each other whenever the + // shape count is even -- which it is -- so half the pairings never come + // up at all, and the tone never met Layer III. for (0..@max(rounds / 40, shapes.len * 2)) |round| { coverage.add(try checkEncoder( .{ .random = prng.random() }, std.testing.allocator, shapes[round % shapes.len], - if (round % 2 == 0) 2 else 3, + if ((round / shapes.len) % 2 == 0) 2 else 3, )); } @@ -1450,6 +1581,11 @@ test "encoded streams decode back to the shape they claim" { try std.testing.expect(coverage.grouped > taken); try std.testing.expect(coverage.silent > taken); try std.testing.expect(coverage.maxed > taken / 4); + // Structure is not enough on its own: the encoder's worst fault made + // frames that were correct in every measurable way and carried noise. + // One shape in six is the tone, and each shape meets both layers, so a + // run that walks the shapes twice reaches this at least twice. + try std.testing.expect(coverage.fidelity > taken / 12); // All four scalefactor selectors, and every shape of input. try std.testing.expectEqual(@as(u8, 0b1111), coverage.selectors & 0b1111); const every_shape = (@as(u8, 1) << @intCast(std.enums.values(Shape).len)) - 1; -- knotmirror 0.0.1