sync

.gitmodules

···
+[submodule "html5lib-tests"]
+	path = html5lib-tests
+	url = https://github.com/html5lib/html5lib-tests

dune-project

···
   (bytesrw (>= 0.3.0))
   (uutf (>= 1.0.0))
   (re (>= 1.10.0))
-  (yojson (and :build (>= 2.0.0)))))
+  (jsont (and :with-test (>= 0.2.0)))))

html5rw.opam

···
   "bytesrw" {>= "0.3.0"}
   "uutf" {>= "1.0.0"}
   "re" {>= "1.10.0"}
-  "yojson" {build & >= "2.0.0"}
+  "jsont" {with-test & >= "0.2.0"}
   "odoc" {with-doc}
 ]
 build: [

lib/parser/tree_builder.ml

···
 
 and process_foreign_content t token =
   match token with
-  | Token.Character "\x00" ->
+  | Token.Character data when String.contains data '\x00' ->
+    (* Replace NUL characters with U+FFFD replacement character *)
     parse_error t "unexpected-null-character";
-    insert_character t "\xEF\xBF\xBD"
+    let buf = Buffer.create (String.length data) in
+    let has_non_ws_non_nul = ref false in
+    String.iter (fun c ->
+      if c = '\x00' then Buffer.add_string buf "\xEF\xBF\xBD"
+      else begin
+        Buffer.add_char buf c;
+        if not (c = ' ' || c = '\t' || c = '\n' || c = '\x0C' || c = '\r') then
+          has_non_ws_non_nul := true
+      end
+    ) data;
+    let replaced = Buffer.contents buf in
+    insert_character t replaced;
+    (* Only set frameset_ok to false if there's actual non-whitespace non-NUL content *)
+    if !has_non_ws_non_nul then t.frameset_ok <- false
   | Token.Character data when is_whitespace data ->
     insert_character t data
   | Token.Character data ->

lib/tokenizer/stream.ml

···
   mutable column : int;
   (* Track if we just saw CR (for CR/LF normalization) *)
   mutable last_was_cr : bool;
+  (* Track if we need to skip the next LF from raw stream (set after peek of CR) *)
+  mutable skip_next_lf : bool;
 }
 
 (* Create a stream from a Bytes.Reader.t *)
···
     line = 1;
     column = 0;
     last_was_cr = false;
+    skip_next_lf = false;
   }
 
 (* Create a stream from a string - discouraged, prefer create_from_reader *)
···
     None
   | Some '\r' ->
     t.last_was_cr <- true;
-    Some '\n'  (* CR becomes LF *)
+    (* Immediately consume following LF if present (CRLF -> single LF) *)
+    (match read_raw_char t with
+     | Some '\n' -> ()  (* Consume the LF that follows CR *)
+     | Some c -> push_back_char t c  (* Put non-LF char back *)
+     | None -> ());
+    Some '\n'  (* CR (or CRLF) becomes single LF *)
   | Some '\n' when t.last_was_cr ->
     (* Skip LF after CR - it was already converted *)
     t.last_was_cr <- false;
···
      Bytes.Slice.is_eod next)))
 
 let peek t =
+  (* Save last_was_cr state before reading *)
+  let saved_last_was_cr = t.last_was_cr in
   match read_normalized_char t with
   | None -> None
   | Some c ->
     push_back_char t c;
-    (* Undo last_was_cr if we pushed back a CR-converted LF *)
-    if c = '\n' then t.last_was_cr <- false;
+    (* Restore the last_was_cr state so advance handles CR/LF correctly *)
+    t.last_was_cr <- saved_last_was_cr;
     Some c
 
 (* Read n characters into a list, returns (chars_read, all_read_successfully) *)

lib/tokenizer/tokenizer.ml

···
   let emit_current_tag () =
     finish_attribute t;
     let name = Buffer.contents t.current_tag_name in
+    let attrs = List.rev t.current_attrs in
+    (* Check for end tag with attributes or self-closing flag *)
+    if t.current_tag_kind = Token.End then begin
+      if attrs <> [] then
+        error t "end-tag-with-attributes";
+      if t.current_tag_self_closing then
+        error t "end-tag-with-trailing-solidus"
+    end;
     let tag = {
       Token.kind = t.current_tag_kind;
       name;
-      attrs = List.rev t.current_attrs;
+      attrs;
       self_closing = t.current_tag_self_closing;
     } in
     if t.current_tag_kind = Token.Start then
···
 
   let emit_current_comment () =
     emit (Token.Comment (Buffer.contents t.current_comment))
+  in
+
+  (* Check for control characters and emit error if needed *)
+  let check_control_char c =
+    let code = Char.code c in
+    (* Control chars: U+0001-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F *)
+    (* Allowed: U+0009 (tab), U+000A (LF), U+000C (FF), U+000D (CR) *)
+    if (code >= 0x01 && code <= 0x08) ||
+       code = 0x0B ||
+       (code >= 0x0E && code <= 0x1F) ||
+       (code >= 0x7F && code <= 0x9F) then
+      error t "control-character-in-input-stream"
+  in
+
+  (* Emit char with control character check *)
+  let emit_char_checked c =
+    check_control_char c;
+    emit_char t c
   in
 
   let rec process_state () =
···
       flush_code_points_consumed_as_char_ref t;
       t.state <- t.return_state;
       handle_eof ()
-    | State.Named_character_reference
-    | State.Numeric_character_reference
+    | State.Named_character_reference ->
+      flush_code_points_consumed_as_char_ref t;
+      t.state <- t.return_state;
+      handle_eof ()
+    | State.Numeric_character_reference ->
+      (* At EOF with just "&#" - no digits follow *)
+      error t "absence-of-digits-in-numeric-character-reference";
+      flush_code_points_consumed_as_char_ref t;
+      t.state <- t.return_state;
+      handle_eof ()
     | State.Hexadecimal_character_reference_start
-    | State.Decimal_character_reference_start
-    | State.Numeric_character_reference_end ->
+    | State.Decimal_character_reference_start ->
+      error t "absence-of-digits-in-numeric-character-reference";
       flush_code_points_consumed_as_char_ref t;
       t.state <- t.return_state;
+      handle_eof ()
+    | State.Numeric_character_reference_end ->
+      (* We have collected digits, just need to finalize the character reference *)
+      step ();
       handle_eof ()
     | State.Ambiguous_ampersand ->
       (* Buffer was already flushed when entering this state, just transition *)
···
       error t "unexpected-null-character";
       ignore (S.process t.sink (Token.Character "\x00"))
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_rcdata () =
···
       error t "unexpected-null-character";
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_rawtext () =
···
       error t "unexpected-null-character";
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data () =
···
       error t "unexpected-null-character";
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_plaintext () =
···
       error t "unexpected-null-character";
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_tag_open () =
···
       error t "unexpected-null-character";
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data_escaped_dash () =
···
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
       t.state <- State.Script_data_escaped;
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data_escaped_dash_dash () =
···
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
       t.state <- State.Script_data_escaped;
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data_escaped_less_than_sign () =
···
       error t "unexpected-null-character";
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data_double_escaped_dash () =
···
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
       t.state <- State.Script_data_double_escaped;
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data_double_escaped_dash_dash () =
···
       emit_str t "\xEF\xBF\xBD"
     | Some c ->
       t.state <- State.Script_data_double_escaped;
-      emit_char t c
+      emit_char_checked c
     | None -> ()
 
   and state_script_data_double_escaped_less_than_sign () =
···
     match Stream.consume t.stream with
     | Some ']' ->
       t.state <- State.Cdata_section_bracket
-    | Some '\x00' ->
-      error t "unexpected-null-character";
-      emit_str t "\xEF\xBF\xBD"
     | Some c ->
+      (* CDATA section emits all characters as-is, including NUL *)
       emit_char t c
     | None -> ()
 
···
         t.state <- t.return_state
       end
     | None ->
-      (* No match - check if we should report ambiguous ampersand *)
+      (* No match - check if we should report unknown-named-character-reference *)
       if String.length entity_name > 0 then begin
-        t.state <- State.Ambiguous_ampersand;
-        (* Reset position - we need to emit the ampersand and chars *)
-        flush_code_points_consumed_as_char_ref t
+        (* If we have a semicolon, it's definitely an unknown named character reference *)
+        if has_semicolon then
+          error t "unknown-named-character-reference";
+        (* Emit all the chars we consumed *)
+        flush_code_points_consumed_as_char_ref t;
+        t.state <- t.return_state
       end else begin
         flush_code_points_consumed_as_char_ref t;
         t.state <- t.return_state

test/dune

···
  (name test_html5lib)
  (libraries bytesrw html5rw.parser html5rw.dom))
 
+(rule
+ (alias runtest)
+ (deps
+  (glob_files ../html5lib-tests/tree-construction/*.dat))
+ (action
+  (run %{exe:test_html5lib.exe} ../html5lib-tests/tree-construction)))
+
+(executable
+ (name test_tokenizer)
+ (libraries bytesrw html5rw.tokenizer jsont jsont.bytesrw))
+
+(rule
+ (alias runtest)
+ (deps
+  (glob_files ../html5lib-tests/tokenizer/*.test))
+ (action
+  (run %{exe:test_tokenizer.exe} ../html5lib-tests/tokenizer)))
+
+(executable
+ (name test_encoding)
+ (libraries html5rw.encoding))
+
+(rule
+ (alias runtest)
+ (deps
+  (glob_files ../html5lib-tests/encoding/*.dat))
+ (action
+  (run %{exe:test_encoding.exe} ../html5lib-tests/encoding)))
+
+(executable
+ (name test_serializer)
+ (libraries html5rw.dom jsont jsont.bytesrw))
+
+(rule
+ (alias runtest)
+ (deps
+  (glob_files ../html5lib-tests/serializer/*.test))
+ (action
+  (run %{exe:test_serializer.exe} ../html5lib-tests/serializer)))
+
 (executable
  (name debug_fragment)
  (libraries bytesrw html5rw.parser html5rw.dom))

test/test_encoding.ml

···
+(* Test runner for html5lib-tests encoding tests *)
+
+module Encoding = Html5rw_encoding
+
+type test_case = {
+  input : string;
+  expected_encoding : string;
+}
+
+(* Normalize encoding name for comparison *)
+let normalize_encoding_name s =
+  String.lowercase_ascii (String.trim s)
+
+(* Convert our encoding type to canonical test name *)
+let encoding_to_test_name = function
+  | Encoding.Utf8 -> "utf-8"
+  | Encoding.Utf16le -> "utf-16le"
+  | Encoding.Utf16be -> "utf-16be"
+  | Encoding.Windows_1252 -> "windows-1252"
+  | Encoding.Iso_8859_2 -> "iso-8859-2"
+  | Encoding.Euc_jp -> "euc-jp"
+
+(* Parse a single test case from lines *)
+let parse_test_case lines =
+  let rec parse acc = function
+    | [] -> acc
+    | line :: rest when String.length line > 0 && line.[0] = '#' ->
+      let section = String.trim line in
+      let content, remaining = collect_section rest in
+      parse ((section, content) :: acc) remaining
+    | _ :: rest -> parse acc rest
+  and collect_section lines =
+    let rec loop acc = function
+      | [] -> (List.rev acc, [])
+      | line :: rest when String.length line > 0 && line.[0] = '#' ->
+        (List.rev acc, line :: rest)
+      | line :: rest -> loop (line :: acc) rest
+    in
+    loop [] lines
+  in
+  let sections = parse [] lines in
+
+  let get_section name =
+    match List.assoc_opt name sections with
+    | Some lines -> String.concat "\n" lines
+    | None -> ""
+  in
+
+  let data = get_section "#data" in
+  let encoding = get_section "#encoding" in
+
+  { input = data; expected_encoding = String.trim encoding }
+
+(* Parse a .dat file into test cases *)
+let parse_dat_file content =
+  let lines = String.split_on_char '\n' content in
+  (* Split on empty lines followed by #data *)
+  let rec split_tests current acc = function
+    | [] ->
+      if current = [] then List.rev acc
+      else List.rev (List.rev current :: acc)
+    | "" :: "#data" :: rest ->
+      let new_acc = if current = [] then acc else (List.rev current :: acc) in
+      split_tests ["#data"] new_acc rest
+    | line :: rest ->
+      split_tests (line :: current) acc rest
+  in
+  let test_groups = split_tests [] [] lines in
+  List.filter_map (fun lines ->
+    if List.exists (fun l -> l = "#data") lines then
+      Some (parse_test_case lines)
+    else None
+  ) test_groups
+
+(* Run a single encoding test *)
+let run_test test =
+  try
+    (* Detect encoding from the input bytes *)
+    let (_, detected_encoding) = Encoding.decode (Bytes.of_string test.input) () in
+    let detected_name = encoding_to_test_name detected_encoding in
+    let expected_name = normalize_encoding_name test.expected_encoding in
+
+    (* Compare - allow some flexibility in naming *)
+    let match_encoding det exp =
+      det = exp ||
+      (det = "windows-1252" && (exp = "windows-1252" || exp = "cp1252" || exp = "iso-8859-1")) ||
+      (det = "iso-8859-2" && (exp = "iso-8859-2" || exp = "iso8859-2" || exp = "latin2")) ||
+      (det = "utf-8" && (exp = "utf-8" || exp = "utf8")) ||
+      (det = "euc-jp" && (exp = "euc-jp" || exp = "eucjp"))
+    in
+
+    (match_encoding detected_name expected_name, detected_name, expected_name)
+  with e ->
+    (false, Printf.sprintf "EXCEPTION: %s" (Printexc.to_string e), test.expected_encoding)
+
+(* Run all tests in a file *)
+let run_file path =
+  let ic = open_in path in
+  let content = really_input_string ic (in_channel_length ic) in
+  close_in ic;
+
+  let tests = parse_dat_file content in
+  let filename = Filename.basename path in
+
+  let passed = ref 0 in
+  let failed = ref 0 in
+  let errors = ref [] in
+
+  List.iteri (fun i test ->
+    if String.trim test.expected_encoding = "" then
+      (* Skip tests without expected encoding *)
+      ()
+    else begin
+      let (success, detected, expected) = run_test test in
+      if success then
+        incr passed
+      else begin
+        incr failed;
+        if List.length !errors < 5 then
+          errors := (i + 1, test.input, detected, expected) :: !errors
+      end
+    end
+  ) tests;
+
+  (!passed, !failed, List.rev !errors, filename)
+
+let () =
+  let test_dir = Sys.argv.(1) in
+  let files = Sys.readdir test_dir |> Array.to_list in
+  let dat_files = List.filter (fun f ->
+    Filename.check_suffix f ".dat" &&
+    not (String.contains f '/')
+  ) files in
+
+  let total_passed = ref 0 in
+  let total_failed = ref 0 in
+  let all_errors = ref [] in
+
+  List.iter (fun file ->
+    let path = Filename.concat test_dir file in
+    if Sys.is_directory path then () else begin
+      let (passed, failed, errors, filename) = run_file path in
+      total_passed := !total_passed + passed;
+      total_failed := !total_failed + failed;
+      if errors <> [] then
+        all_errors := (filename, errors) :: !all_errors;
+      Printf.printf "%s: %d passed, %d failed\n" filename passed failed
+    end
+  ) (List.sort String.compare dat_files);
+
+  Printf.printf "\n=== Summary ===\n";
+  Printf.printf "Total: %d passed, %d failed\n" !total_passed !total_failed;
+
+  if !all_errors <> [] then begin
+    Printf.printf "\n=== First failures ===\n";
+    List.iter (fun (filename, errors) ->
+      List.iter (fun (test_num, input, detected, expected) ->
+        Printf.printf "\n--- %s test %d ---\n" filename test_num;
+        Printf.printf "Input (first 200 chars): %s\n"
+          (String.escaped (String.sub input 0 (min 200 (String.length input))));
+        Printf.printf "Expected encoding: %s\n" expected;
+        Printf.printf "Detected encoding: %s\n" detected
+      ) errors
+    ) (List.rev !all_errors)
+  end;
+
+  exit (if !total_failed > 0 then 1 else 0)

test/test_serializer.ml

···
+(* Test runner for html5lib-tests serializer tests *)
+
+module Dom = Html5rw_dom
+
+(* Extract values from JSON *)
+let json_string = function
+  | Jsont.String (s, _) -> s
+  | _ -> failwith "Expected string"
+
+let json_string_opt = function
+  | Jsont.Null _ -> None
+  | Jsont.String (s, _) -> Some s
+  | _ -> failwith "Expected string or null"
+
+let json_array = function
+  | Jsont.Array (arr, _) -> arr
+  | _ -> failwith "Expected array"
+
+let json_object = function
+  | Jsont.Object (obj, _) -> obj
+  | _ -> failwith "Expected object"
+
+let json_mem name obj =
+  match List.find_opt (fun ((n, _), _) -> n = name) obj with
+  | Some (_, v) -> Some v
+  | None -> None
+
+let json_mem_exn name obj =
+  match json_mem name obj with
+  | Some v -> v
+  | None -> failwith ("Missing member: " ^ name)
+
+(* Test case *)
+type test_case = {
+  description : string;
+  input : Jsont.json list;
+  expected : string list;
+}
+
+let parse_test_case json =
+  let obj = json_object json in
+  let description = json_string (json_mem_exn "description" obj) in
+  let input = json_array (json_mem_exn "input" obj) in
+  let expected = List.map json_string (json_array (json_mem_exn "expected" obj)) in
+  { description; input; expected }
+
+(* Build a DOM node from test input token *)
+let build_node_from_token token =
+  let arr = json_array token in
+  match arr with
+  | [] -> None
+  | type_json :: rest ->
+    let token_type = json_string type_json in
+    match token_type, rest with
+    | "StartTag", [_ns_json; name_json; attrs_json] ->
+      let name = json_string name_json in
+      let attrs_list = json_array attrs_json in
+      let attrs = List.map (fun attr_json ->
+        let attr_obj = json_object attr_json in
+        let attr_name = json_string (json_mem_exn "name" attr_obj) in
+        let value = json_string (json_mem_exn "value" attr_obj) in
+        (attr_name, value)
+      ) attrs_list in
+      Some (Dom.create_element name ~attrs ())
+
+    | "StartTag", [name_json; attrs_json] ->
+      let name = json_string name_json in
+      let attrs_obj = json_object attrs_json in
+      let attrs = List.map (fun ((n, _), v) -> (n, json_string v)) attrs_obj in
+      Some (Dom.create_element name ~attrs ())
+
+    | "EmptyTag", [name_json; attrs_json] ->
+      let name = json_string name_json in
+      let attrs_obj = json_object attrs_json in
+      let attrs = List.map (fun ((n, _), v) -> (n, json_string v)) attrs_obj in
+      Some (Dom.create_element name ~attrs ())
+
+    | "Characters", [text_json] ->
+      let text = json_string text_json in
+      Some (Dom.create_text text)
+
+    | "Comment", [text_json] ->
+      let text = json_string text_json in
+      Some (Dom.create_comment text)
+
+    | "Doctype", [name_json] ->
+      let name = json_string name_json in
+      Some (Dom.create_doctype ~name ())
+
+    | "Doctype", [name_json; public_json] ->
+      let name = json_string name_json in
+      let public_id = json_string_opt public_json in
+      (match public_id with
+       | Some pub -> Some (Dom.create_doctype ~name ~public_id:pub ())
+       | None -> Some (Dom.create_doctype ~name ()))
+
+    | "Doctype", [name_json; public_json; system_json] ->
+      let name = json_string name_json in
+      let public_id = json_string_opt public_json in
+      let system_id = json_string_opt system_json in
+      (match public_id, system_id with
+       | Some pub, Some sys -> Some (Dom.create_doctype ~name ~public_id:pub ~system_id:sys ())
+       | Some pub, None -> Some (Dom.create_doctype ~name ~public_id:pub ())
+       | None, Some sys -> Some (Dom.create_doctype ~name ~system_id:sys ())
+       | None, None -> Some (Dom.create_doctype ~name ()))
+
+    | _ -> None
+
+(* Serialize a single node to HTML (simplified, matches test expectations) *)
+let escape_text text =
+  let buf = Buffer.create (String.length text) in
+  String.iter (fun c ->
+    match c with
+    | '&' -> Buffer.add_string buf "&"
+    | '<' -> Buffer.add_string buf "&lt;"
+    | '>' -> Buffer.add_string buf "&gt;"
+    | c -> Buffer.add_char buf c
+  ) text;
+  Buffer.contents buf
+
+let can_unquote_attr_value value =
+  if String.length value = 0 then false
+  else
+    let valid = ref true in
+    String.iter (fun c ->
+      if c = '>' || c = '"' || c = '\'' || c = '=' || c = '`' ||
+         c = ' ' || c = '\t' || c = '\n' || c = '\x0C' || c = '\r' then
+        valid := false
+    ) value;
+    !valid
+
+let choose_quote value =
+  if String.contains value '"' && not (String.contains value '\'') then '\''
+  else '"'
+
+let escape_attr_value value quote_char =
+  let buf = Buffer.create (String.length value) in
+  String.iter (fun c ->
+    match c with
+    | '&' -> Buffer.add_string buf "&amp;"
+    | '"' when quote_char = '"' -> Buffer.add_string buf "&quot;"
+    | c -> Buffer.add_char buf c
+  ) value;
+  Buffer.contents buf
+
+let serialize_node node =
+  match node.Dom.name with
+  | "#text" ->
+    (* Check if parent is a raw text element *)
+    escape_text node.Dom.data
+  | "#comment" ->
+    "<!--" ^ node.Dom.data ^ "-->"
+  | "!doctype" ->
+    let buf = Buffer.create 64 in
+    Buffer.add_string buf "<!DOCTYPE ";
+    (match node.Dom.doctype with
+     | Some dt ->
+       Buffer.add_string buf (Option.value ~default:"html" dt.Dom.name);
+       (match dt.Dom.public_id with
+        | Some pub when pub <> "" ->
+          Buffer.add_string buf " PUBLIC \"";
+          Buffer.add_string buf pub;
+          Buffer.add_char buf '"';
+          (match dt.Dom.system_id with
+           | Some sys ->
+             Buffer.add_string buf " \"";
+             Buffer.add_string buf sys;
+             Buffer.add_char buf '"'
+           | None -> ())
+        | _ ->
+          match dt.Dom.system_id with
+          | Some sys when sys <> "" ->
+            Buffer.add_string buf " SYSTEM \"";
+            Buffer.add_string buf sys;
+            Buffer.add_char buf '"'
+          | _ -> ())
+     | None -> Buffer.add_string buf "html");
+    Buffer.add_char buf '>';
+    Buffer.contents buf
+  | _ ->
+    (* Element *)
+    let buf = Buffer.create 64 in
+    Buffer.add_char buf '<';
+    Buffer.add_string buf node.Dom.name;
+    List.iter (fun (key, value) ->
+      Buffer.add_char buf ' ';
+      Buffer.add_string buf key;
+      if can_unquote_attr_value value then begin
+        Buffer.add_char buf '=';
+        Buffer.add_string buf value
+      end else begin
+        let quote = choose_quote value in
+        Buffer.add_char buf '=';
+        Buffer.add_char buf quote;
+        Buffer.add_string buf (escape_attr_value value quote);
+        Buffer.add_char buf quote
+      end
+    ) node.Dom.attrs;
+    Buffer.add_char buf '>';
+    Buffer.contents buf
+
+(* Run a single test *)
+let run_test test =
+  try
+    (* Build nodes from input tokens *)
+    let nodes = List.filter_map build_node_from_token test.input in
+
+    (* Serialize *)
+    let serialized = String.concat "" (List.map serialize_node nodes) in
+
+    (* Check if it matches any expected output *)
+    let matches = List.exists (fun exp -> serialized = exp) test.expected in
+
+    (matches, serialized, test.expected)
+  with e ->
+    (false, Printf.sprintf "EXCEPTION: %s" (Printexc.to_string e), test.expected)
+
+(* Run all tests in a file *)
+let run_file path =
+  let content =
+    let ic = open_in path in
+    let n = in_channel_length ic in
+    let s = really_input_string ic n in
+    close_in ic;
+    s
+  in
+
+  let json = match Jsont_bytesrw.decode_string Jsont.json content with
+    | Ok j -> j
+    | Error e -> failwith (Printf.sprintf "JSON parse error in %s: %s" path e)
+  in
+
+  let obj = json_object json in
+  let tests_json = match json_mem "tests" obj with
+    | Some t -> json_array t
+    | None -> []
+  in
+
+  let filename = Filename.basename path in
+  let passed = ref 0 in
+  let failed = ref 0 in
+  let first_failures = ref [] in
+
+  List.iteri (fun i test_json ->
+    try
+      let test = parse_test_case test_json in
+      let (success, actual, expected) = run_test test in
+
+      if success then
+        incr passed
+      else begin
+        incr failed;
+        if List.length !first_failures < 3 then
+          first_failures := (i + 1, test.description, actual, expected) :: !first_failures
+      end
+    with e ->
+      incr failed;
+      Printf.eprintf "Exception parsing test %d: %s\n" (i + 1) (Printexc.to_string e)
+  ) tests_json;
+
+  (!passed, !failed, List.rev !first_failures, filename)
+
+let () =
+  let test_dir = Sys.argv.(1) in
+  let files = Sys.readdir test_dir |> Array.to_list in
+  let test_files = List.filter (fun f -> Filename.check_suffix f ".test") files in
+
+  let total_passed = ref 0 in
+  let total_failed = ref 0 in
+  let all_failures = ref [] in
+
+  List.iter (fun file ->
+    let path = Filename.concat test_dir file in
+    let (passed, failed, failures, filename) = run_file path in
+    total_passed := !total_passed + passed;
+    total_failed := !total_failed + failed;
+    if failures <> [] then
+      all_failures := (filename, failures) :: !all_failures;
+    Printf.printf "%s: %d passed, %d failed\n" filename passed failed
+  ) (List.sort String.compare test_files);
+
+  Printf.printf "\n=== Summary ===\n";
+  Printf.printf "Total: %d passed, %d failed\n" !total_passed !total_failed;
+
+  if !all_failures <> [] then begin
+    Printf.printf "\n=== First failures ===\n";
+    List.iter (fun (filename, failures) ->
+      List.iter (fun (test_num, desc, actual, expected) ->
+        Printf.printf "\n--- %s test %d ---\n" filename test_num;
+        Printf.printf "Description: %s\n" desc;
+        Printf.printf "Expected: %s\n" (String.concat " | " expected);
+        Printf.printf "Actual:   %s\n" actual
+      ) failures
+    ) (List.rev !all_failures)
+  end;
+
+  exit (if !total_failed > 0 then 1 else 0)

test/test_tokenizer.ml

···
+(* Test runner for html5lib-tests tokenizer tests *)
+
+open Bytesrw
+
+module Tokenizer = Html5rw_tokenizer
+
+(* Token collector sink - collects all tokens into a list *)
+module TokenCollector = struct
+  type t = {
+    mutable tokens : Tokenizer.Token.t list;
+  }
+
+  let create () = { tokens = [] }
+
+  let process t token =
+    t.tokens <- token :: t.tokens;
+    `Continue
+
+  let adjusted_current_node_in_html_namespace _ = true
+
+  let get_tokens t = List.rev t.tokens
+end
+
+(* Test case representation *)
+type test_error = {
+  code : string;
+  line : int;
+  col : int;
+}
+
+type test_case = {
+  description : string;
+  input : string;
+  output : Jsont.json list;
+  errors : test_error list;
+  initial_states : string list;
+  last_start_tag : string option;
+  double_escaped : bool;
+}
+
+(* Unescape double-escaped strings from tests *)
+let unescape_double s =
+  let b = Buffer.create (String.length s) in
+  let i = ref 0 in
+  while !i < String.length s do
+    if !i + 1 < String.length s && s.[!i] = '\\' then begin
+      match s.[!i + 1] with
+      | 'u' when !i + 5 < String.length s ->
+        let hex = String.sub s (!i + 2) 4 in
+        (try
+          let code = int_of_string ("0x" ^ hex) in
+          if code < 128 then Buffer.add_char b (Char.chr code)
+          else begin
+            (* UTF-8 encode *)
+            if code < 0x800 then begin
+              Buffer.add_char b (Char.chr (0xC0 lor (code lsr 6)));
+              Buffer.add_char b (Char.chr (0x80 lor (code land 0x3F)))
+            end else begin
+              Buffer.add_char b (Char.chr (0xE0 lor (code lsr 12)));
+              Buffer.add_char b (Char.chr (0x80 lor ((code lsr 6) land 0x3F)));
+              Buffer.add_char b (Char.chr (0x80 lor (code land 0x3F)))
+            end
+          end;
+          i := !i + 6
+        with _ ->
+          Buffer.add_char b s.[!i];
+          incr i)
+      | 'n' -> Buffer.add_char b '\n'; i := !i + 2
+      | 'r' -> Buffer.add_char b '\r'; i := !i + 2
+      | 't' -> Buffer.add_char b '\t'; i := !i + 2
+      | '\\' -> Buffer.add_char b '\\'; i := !i + 2
+      | _ -> Buffer.add_char b s.[!i]; incr i
+    end else begin
+      Buffer.add_char b s.[!i];
+      incr i
+    end
+  done;
+  Buffer.contents b
+
+(* Extract string from JSON node *)
+let json_string = function
+  | Jsont.String (s, _) -> s
+  | _ -> failwith "Expected string"
+
+let json_bool = function
+  | Jsont.Bool (b, _) -> b
+  | _ -> failwith "Expected bool"
+
+let json_int = function
+  | Jsont.Number (n, _) -> int_of_float n
+  | _ -> failwith "Expected number"
+
+let json_array = function
+  | Jsont.Array (arr, _) -> arr
+  | _ -> failwith "Expected array"
+
+let json_object = function
+  | Jsont.Object (obj, _) -> obj
+  | _ -> failwith "Expected object"
+
+let json_mem name obj =
+  match List.find_opt (fun ((n, _), _) -> n = name) obj with
+  | Some (_, v) -> Some v
+  | None -> None
+
+let json_mem_exn name obj =
+  match json_mem name obj with
+  | Some v -> v
+  | None -> failwith ("Missing member: " ^ name)
+
+(* Parse test error from JSON *)
+let parse_test_error json =
+  let obj = json_object json in
+  {
+    code = json_string (json_mem_exn "code" obj);
+    line = json_int (json_mem_exn "line" obj);
+    col = json_int (json_mem_exn "col" obj);
+  }
+
+(* Parse a single test case from JSON *)
+let parse_test_case json =
+  let obj = json_object json in
+  let description = json_string (json_mem_exn "description" obj) in
+  let input = json_string (json_mem_exn "input" obj) in
+  let output = json_array (json_mem_exn "output" obj) in
+  let errors = match json_mem "errors" obj with
+    | Some e -> List.map parse_test_error (json_array e)
+    | None -> []
+  in
+  let initial_states = match json_mem "initialStates" obj with
+    | Some s -> List.map json_string (json_array s)
+    | None -> ["Data state"]
+  in
+  let last_start_tag = match json_mem "lastStartTag" obj with
+    | Some s -> Some (json_string s)
+    | None -> None
+  in
+  let double_escaped = match json_mem "doubleEscaped" obj with
+    | Some b -> json_bool b
+    | None -> false
+  in
+  { description; input; output; errors; initial_states; last_start_tag; double_escaped }
+
+(* Convert state name to State.t *)
+let state_of_string = function
+  | "Data state" -> Tokenizer.State.Data
+  | "PLAINTEXT state" -> Tokenizer.State.Plaintext
+  | "RCDATA state" -> Tokenizer.State.Rcdata
+  | "RAWTEXT state" -> Tokenizer.State.Rawtext
+  | "Script data state" -> Tokenizer.State.Script_data
+  | "CDATA section state" -> Tokenizer.State.Cdata_section
+  | s -> failwith ("Unknown state: " ^ s)
+
+(* Convert our token to test format for comparison *)
+let token_to_test_json (tok : Tokenizer.Token.t) : Jsont.json list =
+  let str s = Jsont.String (s, Jsont.Meta.none) in
+  let arr l = Jsont.Array (l, Jsont.Meta.none) in
+  match tok with
+  | Tokenizer.Token.Doctype { name; public_id; system_id; force_quirks } ->
+    let name_json = match name with
+      | Some n -> str n
+      | None -> Jsont.Null ((), Jsont.Meta.none)
+    in
+    let public_json = match public_id with
+      | Some p -> str p
+      | None -> Jsont.Null ((), Jsont.Meta.none)
+    in
+    let system_json = match system_id with
+      | Some s -> str s
+      | None -> Jsont.Null ((), Jsont.Meta.none)
+    in
+    let correctness = Jsont.Bool (not force_quirks, Jsont.Meta.none) in
+    [arr [str "DOCTYPE"; name_json; public_json; system_json; correctness]]
+  | Tokenizer.Token.Tag { kind = Start; name; attrs; self_closing } ->
+    let attrs_obj = Jsont.Object (
+      List.map (fun (n, v) -> ((n, Jsont.Meta.none), str v)) (List.rev attrs),
+      Jsont.Meta.none
+    ) in
+    if self_closing then
+      [arr [str "StartTag"; str name; attrs_obj; Jsont.Bool (true, Jsont.Meta.none)]]
+    else
+      [arr [str "StartTag"; str name; attrs_obj]]
+  | Tokenizer.Token.Tag { kind = End; name; _ } ->
+    [arr [str "EndTag"; str name]]
+  | Tokenizer.Token.Comment data ->
+    [arr [str "Comment"; str data]]
+  | Tokenizer.Token.Character data ->
+    (* Split into individual characters for comparison - but actually
+       the tests expect consecutive characters to be merged *)
+    [arr [str "Character"; str data]]
+  | Tokenizer.Token.EOF -> []
+
+(* Compare JSON values for equality *)
+let rec json_equal a b =
+  match a, b with
+  | Jsont.Null _, Jsont.Null _ -> true
+  | Jsont.Bool (a, _), Jsont.Bool (b, _) -> a = b
+  | Jsont.Number (a, _), Jsont.Number (b, _) -> a = b
+  | Jsont.String (a, _), Jsont.String (b, _) -> a = b
+  | Jsont.Array (a, _), Jsont.Array (b, _) ->
+    List.length a = List.length b &&
+    List.for_all2 json_equal a b
+  | Jsont.Object (a, _), Jsont.Object (b, _) ->
+    let a_sorted = List.sort (fun ((n1, _), _) ((n2, _), _) -> String.compare n1 n2) a in
+    let b_sorted = List.sort (fun ((n1, _), _) ((n2, _), _) -> String.compare n1 n2) b in
+    List.length a_sorted = List.length b_sorted &&
+    List.for_all2 (fun ((n1, _), v1) ((n2, _), v2) -> n1 = n2 && json_equal v1 v2) a_sorted b_sorted
+  | _ -> false
+
+(* Merge consecutive Character tokens *)
+let merge_character_tokens tokens =
+  let rec loop acc = function
+    | [] -> List.rev acc
+    | Tokenizer.Token.Character s1 :: Tokenizer.Token.Character s2 :: rest ->
+      loop acc (Tokenizer.Token.Character (s1 ^ s2) :: rest)
+    | tok :: rest -> loop (tok :: acc) rest
+  in
+  loop [] tokens
+
+(* Run a single test *)
+let run_test test initial_state =
+  let input = if test.double_escaped then unescape_double test.input else test.input in
+
+  let collector = TokenCollector.create () in
+  let tokenizer = Tokenizer.create (module TokenCollector) collector ~collect_errors:true () in
+
+  (* Set initial state *)
+  Tokenizer.set_state tokenizer initial_state;
+
+  (* Set last start tag if specified *)
+  (match test.last_start_tag with
+   | Some tag -> Tokenizer.set_last_start_tag tokenizer tag
+   | None -> ());
+
+  (* Run tokenizer *)
+  let reader = Bytes.Reader.of_string input in
+  Tokenizer.run tokenizer (module TokenCollector) reader;
+
+  (* Get results *)
+  let tokens = merge_character_tokens (TokenCollector.get_tokens collector) in
+  let actual_tokens = List.concat_map token_to_test_json tokens in
+
+  (* Unescape expected output if double_escaped *)
+  let expected_output = if test.double_escaped then
+    let rec unescape_json = function
+      | Jsont.String (s, m) -> Jsont.String (unescape_double s, m)
+      | Jsont.Array (arr, m) -> Jsont.Array (List.map unescape_json arr, m)
+      | Jsont.Object (obj, m) ->
+        Jsont.Object (List.map (fun (n, v) -> (n, unescape_json v)) obj, m)
+      | other -> other
+    in
+    List.map unescape_json test.output
+  else test.output
+  in
+
+  (* Merge consecutive Character tokens in expected output too *)
+  let rec merge_expected = function
+    | [] -> []
+    | [x] -> [x]
+    | Jsont.Array ([Jsont.String ("Character", _); Jsont.String (s1, m1)], am1) ::
+      Jsont.Array ([Jsont.String ("Character", _); Jsont.String (s2, _)], _) ::
+      rest ->
+      merge_expected (Jsont.Array ([Jsont.String ("Character", Jsont.Meta.none); Jsont.String (s1 ^ s2, m1)], am1) :: rest)
+    | x :: rest -> x :: merge_expected rest
+  in
+  let expected = merge_expected expected_output in
+
+  (* Compare *)
+  let tokens_match =
+    List.length actual_tokens = List.length expected &&
+    List.for_all2 json_equal actual_tokens expected
+  in
+
+  let actual_errors = Tokenizer.get_errors tokenizer in
+  let errors_count_match = List.length actual_errors = List.length test.errors in
+
+  (tokens_match && errors_count_match, actual_tokens, expected, actual_errors, test.errors)
+
+(* Format JSON for display *)
+let rec json_to_string = function
+  | Jsont.Null _ -> "null"
+  | Jsont.Bool (b, _) -> string_of_bool b
+  | Jsont.Number (n, _) -> Printf.sprintf "%g" n
+  | Jsont.String (s, _) -> Printf.sprintf "%S" s
+  | Jsont.Array (arr, _) ->
+    "[" ^ String.concat ", " (List.map json_to_string arr) ^ "]"
+  | Jsont.Object (obj, _) ->
+    "{" ^ String.concat ", " (List.map (fun ((n, _), v) -> Printf.sprintf "%S: %s" n (json_to_string v)) obj) ^ "}"
+
+(* Run all tests in a file *)
+let run_file path =
+  let content =
+    let ic = open_in path in
+    let n = in_channel_length ic in
+    let s = really_input_string ic n in
+    close_in ic;
+    s
+  in
+
+  (* Parse JSON *)
+  let json = match Jsont_bytesrw.decode_string Jsont.json content with
+    | Ok j -> j
+    | Error e -> failwith (Printf.sprintf "JSON parse error in %s: %s" path e)
+  in
+
+  let obj = json_object json in
+
+  (* Handle both {"tests": [...]} and {"xmlViolationTests": [...], "tests": [...]} formats *)
+  let test_arrays =
+    let tests = match json_mem "tests" obj with
+      | Some t -> json_array t
+      | None -> []
+    in
+    let xml_tests = match json_mem "xmlViolationTests" obj with
+      | Some t -> json_array t
+      | None -> []
+    in
+    tests @ xml_tests
+  in
+
+  let filename = Filename.basename path in
+  let passed = ref 0 in
+  let failed = ref 0 in
+  let first_failures = ref [] in
+
+  List.iteri (fun i test_json ->
+    let test = parse_test_case test_json in
+
+    (* Run for each initial state *)
+    List.iter (fun state_name ->
+      try
+        let state = state_of_string state_name in
+        let (success, actual, expected, actual_errors, expected_errors) = run_test test state in
+
+        if success then
+          incr passed
+        else begin
+          incr failed;
+          if List.length !first_failures < 3 then
+            first_failures := (i + 1, test.description, state_name, actual, expected, actual_errors, expected_errors) :: !first_failures
+        end
+      with e ->
+        incr failed;
+        if List.length !first_failures < 3 then
+          first_failures := (i + 1, test.description, state_name, [], [], [], []) :: !first_failures;
+        Printf.eprintf "Exception in test %d (%s): %s\n" (i + 1) test.description (Printexc.to_string e)
+    ) test.initial_states
+  ) test_arrays;
+
+  (!passed, !failed, List.rev !first_failures, filename)
+
+let () =
+  let test_dir = Sys.argv.(1) in
+  let files = Sys.readdir test_dir |> Array.to_list in
+  let test_files = List.filter (fun f -> Filename.check_suffix f ".test") files in
+
+  let total_passed = ref 0 in
+  let total_failed = ref 0 in
+  let all_failures = ref [] in
+
+  List.iter (fun file ->
+    let path = Filename.concat test_dir file in
+    let (passed, failed, failures, filename) = run_file path in
+    total_passed := !total_passed + passed;
+    total_failed := !total_failed + failed;
+    if failures <> [] then
+      all_failures := (filename, failures) :: !all_failures;
+    Printf.printf "%s: %d passed, %d failed\n" filename passed failed
+  ) (List.sort String.compare test_files);
+
+  Printf.printf "\n=== Summary ===\n";
+  Printf.printf "Total: %d passed, %d failed\n" !total_passed !total_failed;
+
+  if !all_failures <> [] then begin
+    Printf.printf "\n=== First failures ===\n";
+    List.iter (fun (filename, failures) ->
+      List.iter (fun (test_num, desc, state, actual, expected, actual_errs, expected_errs) ->
+        Printf.printf "\n--- %s test %d (%s) in %s ---\n" filename test_num state desc;
+        Printf.printf "Expected tokens: [%s]\n" (String.concat "; " (List.map json_to_string expected));
+        Printf.printf "Actual tokens:   [%s]\n" (String.concat "; " (List.map json_to_string actual));
+        Printf.printf "Expected %d errors, got %d\n" (List.length expected_errs) (List.length actual_errs)
+      ) failures
+    ) (List.rev !all_failures)
+  end;
+
+  exit (if !total_failed > 0 then 1 else 0)