fix

lib/check/error_code.ml

···
       Printf.sprintf "Element %s is missing one or more of the following attributes: [%s]."
         (q element) attrs_str
   | `Attr (`Bad_value (`Elem element, `Attr attr, `Value value, `Reason reason)) ->
-      Printf.sprintf "Bad value %s for attribute %s on element %s: %s"
-        (q value) (q attr) (q element) reason
+      if reason = "" then
+        Printf.sprintf "Bad value %s for attribute %s on element %s."
+          (q value) (q attr) (q element)
+      else
+        Printf.sprintf "Bad value %s for attribute %s on element %s: %s"
+          (q value) (q attr) (q element) reason
   | `Attr (`Bad_value_generic (`Message message)) -> message
   | `Attr (`Duplicate_id (`Id id)) ->
       Printf.sprintf "Duplicate ID %s." (q id)

test/test_validator.ml

···
   } in
   Report.generate_report report output_path
 
-let () =
-  (* Parse command line arguments *)
-  let args = Array.to_list Sys.argv |> List.tl in
-  let is_strict = List.mem "--strict" args in
-  let non_flag_args = List.filter (fun s -> not (String.length s > 0 && s.[0] = '-')) args in
-  let tests_dir = match non_flag_args with x :: _ -> x | [] -> "validator/tests" in
-  let report_path = match non_flag_args with _ :: x :: _ -> x | _ -> "test_validator_report.html" in
-
-  (* Apply strict mode if requested - use exact_message which requires exact text but not typed codes *)
-  if is_strict then begin
-    strictness := Expected_message.exact_message;
-    Printf.printf "Running in STRICT mode (exact message matching required)\n%!"
-  end;
-
-  Printf.printf "Loading messages.json...\n%!";
-  let messages_path = Filename.concat tests_dir "messages.json" in
-  let messages = Validator_messages.load messages_path in
-  Printf.printf "Loaded %d expected messages\n%!" (Validator_messages.count messages);
-
-  Printf.printf "Discovering test files...\n%!";
-  let tests = discover_tests tests_dir in
-  Printf.printf "Found %d test files\n%!" (List.length tests);
-
-  Printf.printf "Running tests...\n%!";
+(** Run tests with a given strictness and return results *)
+let run_all_tests ~mode_name ~strictness_setting messages tests =
+  strictness := strictness_setting;
+  Printf.printf "\n=== Running in %s mode ===\n%!" mode_name;
   let total = List.length tests in
   let results = List.mapi (fun i test ->
     Printf.printf "\r[%d/%d] %s%!" (i + 1) total test.relative_path;
     run_test messages test
   ) tests in
   Printf.printf "\n%!";
+  results
 
-  (* Print failing isvalid tests *)
+(** Print failures for a test run *)
+let print_failures mode_name results =
+  Printf.printf "\n--- %s mode results ---\n" mode_name;
+
   let failing_isvalid = List.filter (fun r ->
     r.file.expected = Valid && not r.passed
   ) results in
   if failing_isvalid <> [] then begin
-    Printf.printf "\n=== Failing isvalid tests ===\n";
+    Printf.printf "Failing isvalid tests:\n";
     List.iter (fun r ->
-      Printf.printf "%s: %s\n" r.file.relative_path r.details
+      Printf.printf "  %s: %s\n" r.file.relative_path r.details
     ) failing_isvalid
   end;
 
-  (* Print failing haswarn tests *)
   let failing_haswarn = List.filter (fun r ->
     r.file.expected = HasWarning && not r.passed
   ) results in
   if failing_haswarn <> [] then begin
-    Printf.printf "\n=== Failing haswarn tests ===\n";
+    Printf.printf "Failing haswarn tests:\n";
     List.iter (fun r ->
-      Printf.printf "%s\n" r.file.relative_path
+      Printf.printf "  %s\n" r.file.relative_path
     ) failing_haswarn
   end;
 
-  (* Print failing novalid tests *)
   let failing_novalid = List.filter (fun r ->
     r.file.expected = Invalid && not r.passed
   ) results in
   if failing_novalid <> [] then begin
-    Printf.printf "\n=== Failing novalid tests (first 50) ===\n";
+    Printf.printf "Failing novalid tests (first 20):\n";
     List.iteri (fun i r ->
-      if i < 50 then Printf.printf "%s\n" r.file.relative_path
+      if i < 20 then Printf.printf "  %s\n" r.file.relative_path
     ) failing_novalid
   end;
 
-  print_summary results;
-  generate_html_report results report_path;
+  let passed = List.filter (fun r -> r.passed) results |> List.length in
+  let total = List.length results in
+  Printf.printf "%s: %d/%d passed (%.1f%%)\n%!" mode_name passed total
+    (100.0 *. float_of_int passed /. float_of_int total)
+
+(** Generate combined HTML report for both modes *)
+let generate_combined_html_report ~lenient_results ~strict_results output_path =
+  (* Helper to build file results from a set of results *)
+  let build_file_results results =
+    let by_category = group_by_category results in
+    List.map (fun (category, tests) ->
+      let passed_count = List.filter (fun r -> r.passed) tests |> List.length in
+      let failed_count = List.length tests - passed_count in
+      let test_results = List.mapi (fun i r ->
+        let outcome_str = match r.file.expected with
+          | Valid -> "isvalid"
+          | Invalid -> "novalid"
+          | HasWarning -> "haswarn"
+          | Unknown -> "unknown"
+        in
+        let description = Printf.sprintf "[%s] %s" outcome_str (Filename.basename r.file.relative_path) in
+        let expected = match r.expected_message with
+          | Some m -> m
+          | None -> match r.file.expected with
+            | Valid -> "(should produce no errors or warnings)"
+            | Invalid -> "(should produce at least one error)"
+            | HasWarning -> "(should produce at least one warning)"
+            | Unknown -> "(unknown test type)"
+        in
+        let actual_str =
+          let errors = if r.actual_errors = [] then ""
+            else "Errors:\n  • " ^ String.concat "\n  • " r.actual_errors in
+          let warnings = if r.actual_warnings = [] then ""
+            else "Warnings:\n  • " ^ String.concat "\n  • " r.actual_warnings in
+          let infos = if r.actual_infos = [] then ""
+            else "Info:\n  • " ^ String.concat "\n  • " r.actual_infos in
+          if errors = "" && warnings = "" && infos = "" then "(no messages produced)"
+          else String.trim (errors ^ (if errors <> "" && warnings <> "" then "\n\n" else "") ^
+                           warnings ^ (if (errors <> "" || warnings <> "") && infos <> "" then "\n\n" else "") ^
+                           infos)
+        in
+        let match_quality_str = match r.match_quality with
+          | Some q -> Expected_message.match_quality_to_string q
+          | None -> "N/A"
+        in
+        Report.{
+          test_num = i + 1;
+          description;
+          input = r.file.relative_path;
+          expected;
+          actual = actual_str;
+          success = r.passed;
+          details = [
+            ("Result", r.details);
+            ("Match Quality", match_quality_str);
+          ];
+          raw_test_data = read_html_source r.file.path;
+        }
+      ) tests in
+      Report.{
+        filename = category;
+        test_type = "HTML5 Validator";
+        passed_count;
+        failed_count;
+        tests = test_results;
+      }
+    ) by_category
+  in
 
-  let failed_count = List.filter (fun r -> not r.passed) results |> List.length in
-  exit (if failed_count > 0 then 1 else 0)
+  let compute_stats results mode_name =
+    let total_passed = List.filter (fun r -> r.passed) results |> List.length in
+    let total_failed = List.length results - total_passed in
+    let count_quality q = List.filter (fun r ->
+      match r.match_quality with Some mq -> mq = q | None -> false
+    ) results |> List.length in
+    let match_quality_stats : Report.match_quality_stats = {
+      exact_matches = count_quality Expected_message.Exact_match;
+      code_matches = count_quality Expected_message.Code_match;
+      message_matches = count_quality Expected_message.Message_match;
+      substring_matches = count_quality Expected_message.Substring_match;
+      severity_mismatches = count_quality Expected_message.Severity_mismatch;
+      no_matches = count_quality Expected_message.No_match;
+      not_applicable = List.filter (fun r -> r.match_quality = None) results |> List.length;
+    } in
+    let isvalid_results = List.filter (fun r -> r.file.expected = Valid) results in
+    let novalid_results = List.filter (fun r -> r.file.expected = Invalid) results in
+    let haswarn_results = List.filter (fun r -> r.file.expected = HasWarning) results in
+    let count_passed rs = List.filter (fun r -> r.passed) rs |> List.length in
+    let test_type_stats : Report.test_type_stats = {
+      isvalid_passed = count_passed isvalid_results;
+      isvalid_total = List.length isvalid_results;
+      novalid_passed = count_passed novalid_results;
+      novalid_total = List.length novalid_results;
+      haswarn_passed = count_passed haswarn_results;
+      haswarn_total = List.length haswarn_results;
+    } in
+    (total_passed, total_failed, match_quality_stats, test_type_stats, mode_name)
+  in
+
+  let lenient_stats = compute_stats lenient_results "lenient" in
+  let strict_stats = compute_stats strict_results "strict" in
+
+  (* Use strict results for the main report, but include both in description *)
+  let (strict_passed, strict_failed, strict_mq, strict_tt, _) = strict_stats in
+  let (lenient_passed, _lenient_failed, _, _, _) = lenient_stats in
+
+  let now = Unix.gettimeofday () in
+  let tm = Unix.localtime now in
+  let timestamp = Printf.sprintf "%04d-%02d-%02d %02d:%02d:%02d"
+    (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday
+    tm.Unix.tm_hour tm.Unix.tm_min tm.Unix.tm_sec in
+
+  let total = List.length strict_results in
+  let description = Printf.sprintf
+    "Tests from the Nu HTML Validator (W3C's official HTML checker). \
+     Tests validate HTML5 conformance including element nesting, required attributes, \
+     ARIA roles, obsolete elements, and more.\n\n\
+     LENIENT mode: %d/%d passed (%.1f%%) - allows substring matching\n\
+     STRICT mode: %d/%d passed (%.1f%%) - requires exact message matching"
+    lenient_passed total (100.0 *. float_of_int lenient_passed /. float_of_int total)
+    strict_passed total (100.0 *. float_of_int strict_passed /. float_of_int total)
+  in
+
+  let report : Report.report = {
+    title = "Nu HTML Validator Tests (Lenient + Strict)";
+    test_type = "validator";
+    description;
+    files = build_file_results strict_results;  (* Show strict results in detail *)
+    total_passed = strict_passed;
+    total_failed = strict_failed;
+    match_quality = Some strict_mq;
+    test_type_breakdown = Some strict_tt;
+    strictness_mode = Some (Printf.sprintf "BOTH (Lenient: %d/%d, Strict: %d/%d)"
+      lenient_passed total strict_passed total);
+    run_timestamp = Some timestamp;
+  } in
+  Report.generate_report report output_path
+
+let () =
+  (* Parse command line arguments *)
+  let args = Array.to_list Sys.argv |> List.tl in
+  let is_strict = List.mem "--strict" args in
+  let is_both = List.mem "--both" args in
+  let non_flag_args = List.filter (fun s -> not (String.length s > 0 && s.[0] = '-')) args in
+  let tests_dir = match non_flag_args with x :: _ -> x | [] -> "validator/tests" in
+  let report_path = match non_flag_args with _ :: x :: _ -> x | _ -> "test_validator_report.html" in
+
+  Printf.printf "Loading messages.json...\n%!";
+  let messages_path = Filename.concat tests_dir "messages.json" in
+  let messages = Validator_messages.load messages_path in
+  Printf.printf "Loaded %d expected messages\n%!" (Validator_messages.count messages);
+
+  Printf.printf "Discovering test files...\n%!";
+  let tests = discover_tests tests_dir in
+  Printf.printf "Found %d test files\n%!" (List.length tests);
+
+  if is_both then begin
+    (* Run both modes *)
+    let lenient_results = run_all_tests ~mode_name:"LENIENT"
+      ~strictness_setting:Expected_message.lenient messages tests in
+    let strict_results = run_all_tests ~mode_name:"STRICT"
+      ~strictness_setting:Expected_message.exact_message messages tests in
+
+    print_failures "LENIENT" lenient_results;
+    print_failures "STRICT" strict_results;
+
+    Printf.printf "\n=== Summary ===\n";
+    let lenient_passed = List.filter (fun r -> r.passed) lenient_results |> List.length in
+    let strict_passed = List.filter (fun r -> r.passed) strict_results |> List.length in
+    let total = List.length tests in
+    Printf.printf "LENIENT: %d/%d (%.1f%%)\n" lenient_passed total
+      (100.0 *. float_of_int lenient_passed /. float_of_int total);
+    Printf.printf "STRICT:  %d/%d (%.1f%%)\n" strict_passed total
+      (100.0 *. float_of_int strict_passed /. float_of_int total);
+
+    generate_combined_html_report ~lenient_results ~strict_results report_path;
+    Printf.printf "\nHTML report written to: %s\n" report_path;
+
+    (* Exit with error if strict mode has failures *)
+    let strict_failed = List.filter (fun r -> not r.passed) strict_results |> List.length in
+    exit (if strict_failed > 0 then 1 else 0)
+  end else begin
+    (* Single mode (original behavior) *)
+    if is_strict then begin
+      strictness := Expected_message.exact_message;
+      Printf.printf "Running in STRICT mode (exact message matching required)\n%!"
+    end;
+
+    Printf.printf "Running tests...\n%!";
+    let total = List.length tests in
+    let results = List.mapi (fun i test ->
+      Printf.printf "\r[%d/%d] %s%!" (i + 1) total test.relative_path;
+      run_test messages test
+    ) tests in
+    Printf.printf "\n%!";
+
+    (* Print failing isvalid tests *)
+    let failing_isvalid = List.filter (fun r ->
+      r.file.expected = Valid && not r.passed
+    ) results in
+    if failing_isvalid <> [] then begin
+      Printf.printf "\n=== Failing isvalid tests ===\n";
+      List.iter (fun r ->
+        Printf.printf "%s: %s\n" r.file.relative_path r.details
+      ) failing_isvalid
+    end;
+
+    (* Print failing haswarn tests *)
+    let failing_haswarn = List.filter (fun r ->
+      r.file.expected = HasWarning && not r.passed
+    ) results in
+    if failing_haswarn <> [] then begin
+      Printf.printf "\n=== Failing haswarn tests ===\n";
+      List.iter (fun r ->
+        Printf.printf "%s\n" r.file.relative_path
+      ) failing_haswarn
+    end;
+
+    (* Print failing novalid tests *)
+    let failing_novalid = List.filter (fun r ->
+      r.file.expected = Invalid && not r.passed
+    ) results in
+    if failing_novalid <> [] then begin
+      Printf.printf "\n=== Failing novalid tests (first 50) ===\n";
+      List.iteri (fun i r ->
+        if i < 50 then Printf.printf "%s\n" r.file.relative_path
+      ) failing_novalid
+    end;
+
+    print_summary results;
+    generate_html_report results report_path;
+
+    let failed_count = List.filter (fun r -> not r.passed) results |> List.length in
+    exit (if failed_count > 0 then 1 else 0)
+  end