// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! Byte order marks. //! //! The standard keeps these out of the encodings themselves -- UTF-8's //! decoder has nothing to say about `EF BB BF`, and neither UTF-16 decoder //! knows what `FE FF` means -- and puts them in the wrapper that chooses an //! encoding instead. That is deliberate: a byte order mark is more reliable //! than a label, having been written by whatever produced the bytes rather //! than by whatever is serving them, so it wins. In an HTTP context that //! contradicts the semantics of the `Content-Type` header, and the standard //! says so and does it anyway, because deployed content leaves no choice. const std = @import("std"); const Encoding = @import("encoding.zig").Encoding; /// The standard's "BOM sniff": the encoding a leading byte order mark names, /// or null if there is not one. /// /// A short input is fine. Two bytes can still carry a UTF-16 mark, and one /// byte can carry nothing. pub fn sniff(bytes: []const u8) ?Encoding { if (std.mem.startsWith(u8, bytes, &.{ 0xef, 0xbb, 0xbf })) return .utf8; if (std.mem.startsWith(u8, bytes, &.{ 0xfe, 0xff })) return .utf16be; if (std.mem.startsWith(u8, bytes, &.{ 0xff, 0xfe })) return .utf16le; return null; } /// How many bytes the mark for an encoding takes, so that a caller can step /// over it. Three for UTF-8, two for UTF-16, and zero for everything else -- /// no other encoding has one. pub fn length(encoding: Encoding) usize { return switch (encoding) { .utf8 => 3, .utf16be, .utf16le => 2, else => 0, }; } /// The input with a leading byte order mark removed, and the encoding it /// named. When there is no mark, the input unchanged and `fallback`. /// /// This is the first half of the standard's "decode" hook; `charset.decode` /// is the whole of it. pub fn strip(bytes: []const u8, fallback: Encoding) struct { Encoding, []const u8 } { const encoding = sniff(bytes) orelse return .{ fallback, bytes }; return .{ encoding, bytes[length(encoding)..] }; } test "sniff finds each mark and nothing else" { try std.testing.expectEqual(Encoding.utf8, sniff("\xef\xbb\xbfhello").?); try std.testing.expectEqual(Encoding.utf16be, sniff("\xfe\xff\x00a").?); try std.testing.expectEqual(Encoding.utf16le, sniff("\xff\xfea\x00").?); try std.testing.expect(sniff("hello") == null); try std.testing.expect(sniff("") == null); try std.testing.expect(sniff("\xef\xbb") == null); try std.testing.expect(sniff("\xfe") == null); } test "strip removes the mark it found and nothing it did not" { { const encoding, const rest = strip("\xef\xbb\xbfhi", .windows_1252); try std.testing.expectEqual(Encoding.utf8, encoding); try std.testing.expectEqualStrings("hi", rest); } { const encoding, const rest = strip("hi", .windows_1252); try std.testing.expectEqual(Encoding.windows_1252, encoding); try std.testing.expectEqualStrings("hi", rest); } } test "a mark outranks the label, which is the point of the exercise" { // Bytes served as windows-1252 that begin with a UTF-8 mark are UTF-8. const encoding, _ = strip("\xef\xbb\xbf", .shift_jis); try std.testing.expectEqual(Encoding.utf8, encoding); }