Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! Byte order marks.//!//! The standard keeps these out of the encodings themselves -- UTF-8's//! decoder has nothing to say about `EF BB BF`, and neither UTF-16 decoder//! knows what `FE FF` means -- and puts them in the wrapper that chooses an//! encoding instead. That is deliberate: a byte order mark is more reliable//! than a label, having been written by whatever produced the bytes rather//! than by whatever is serving them, so it wins. In an HTTP context that//! contradicts the semantics of the `Content-Type` header, and the standard//! says so and does it anyway, because deployed content leaves no choice.
const std = @import("std");
const Encoding = @import("encoding.zig").Encoding;
/// The standard's "BOM sniff": the encoding a leading byte order mark names,/// or null if there is not one.////// A short input is fine. Two bytes can still carry a UTF-16 mark, and one/// byte can carry nothing.pub fn sniff(bytes: []const u8) ?Encoding { if (std.mem.startsWith(u8, bytes, &.{ 0xef, 0xbb, 0xbf })) return .utf8; if (std.mem.startsWith(u8, bytes, &.{ 0xfe, 0xff })) return .utf16be; if (std.mem.startsWith(u8, bytes, &.{ 0xff, 0xfe })) return .utf16le; return null;}
/// How many bytes the mark for an encoding takes, so that a caller can step/// over it. Three for UTF-8, two for UTF-16, and zero for everything else --/// no other encoding has one.pub fn length(encoding: Encoding) usize { return switch (encoding) { .utf8 => 3, .utf16be, .utf16le => 2, else => 0, };}
/// The input with a leading byte order mark removed, and the encoding it/// named. When there is no mark, the input unchanged and `fallback`.////// This is the first half of the standard's "decode" hook; `charset.decode`/// is the whole of it.pub fn strip(bytes: []const u8, fallback: Encoding) struct { Encoding, []const u8 } { const encoding = sniff(bytes) orelse return .{ fallback, bytes }; return .{ encoding, bytes[length(encoding)..] };}
test "sniff finds each mark and nothing else" { try std.testing.expectEqual(Encoding.utf8, sniff("\xef\xbb\xbfhello").?); try std.testing.expectEqual(Encoding.utf16be, sniff("\xfe\xff\x00a").?); try std.testing.expectEqual(Encoding.utf16le, sniff("\xff\xfea\x00").?); try std.testing.expect(sniff("hello") == null); try std.testing.expect(sniff("") == null); try std.testing.expect(sniff("\xef\xbb") == null); try std.testing.expect(sniff("\xfe") == null);}
test "strip removes the mark it found and nothing it did not" { { const encoding, const rest = strip("\xef\xbb\xbfhi", .windows_1252); try std.testing.expectEqual(Encoding.utf8, encoding); try std.testing.expectEqualStrings("hi", rest); } { const encoding, const rest = strip("hi", .windows_1252); try std.testing.expectEqual(Encoding.windows_1252, encoding); try std.testing.expectEqualStrings("hi", rest); }}
test "a mark outranks the label, which is the point of the exercise" { // Bytes served as windows-1252 that begin with a UTF-8 mark are UTF-8. const encoding, _ = strip("\xef\xbb\xbf", .shift_jis); try std.testing.expectEqual(Encoding.utf8, encoding);}