//! Test262 test harness. //! //! Walks the Test262 test suite and runs each test case against our JavaScript //! engine. Reports pass/fail/skip counts grouped by category. //! //! Run with: `cargo test -p we-js --test test262 -- --nocapture` /// Workspace root relative to the crate directory. const WORKSPACE_ROOT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../"); /// Minimal JS preamble that defines the Test262 harness helpers using only /// features our engine supports. This replaces the standard `sta.js` and /// `assert.js` harness files which require built-ins we don't have yet. const HARNESS_PREAMBLE: &str = r#" function Test262Error(message) { return "Test262Error: " + message; } function $DONOTEVALUATE() { throw "Test262: This statement should not be evaluated."; } function assert(mustBeTrue, message) { if (mustBeTrue === true) { return; } if (message === undefined) { message = "Expected true but got " + mustBeTrue; } throw message; } assert._isSameValue = function(a, b) { if (a === b) { if (a !== 0) { return true; } return 1 / a === 1 / b; } if (a !== a && b !== b) { return true; } return false; }; assert.sameValue = function(actual, expected, message) { if (assert._isSameValue(actual, expected)) { return; } if (message === undefined) { message = ""; } else { message = message + " "; } message = message + "Expected SameValue(" + actual + ", " + expected + ") to be true"; throw message; }; assert.notSameValue = function(actual, unexpected, message) { if (!assert._isSameValue(actual, unexpected)) { return; } if (message === undefined) { message = ""; } else { message = message + " "; } message = message + "Expected not SameValue(" + actual + ", " + unexpected + ")"; throw message; }; assert.throws = function(expectedErrorConstructor, func, message) { if (typeof func !== "function") { throw "assert.throws requires a function argument"; } var threw = false; try { func(); } catch (e) { threw = true; } if (!threw) { if (message === undefined) { message = "Expected an exception to be thrown"; } throw message; } }; function print() {} "#; /// Features that our engine does not yet support. Tests requiring any of these /// are skipped rather than counted as failures. const UNSUPPORTED_FEATURES: &[&str] = &[ // Type system extensions "BigInt", "Symbol", "Symbol.asyncIterator", "Symbol.hasInstance", "Symbol.isConcatSpreadable", "Symbol.iterator", "Symbol.match", "Symbol.matchAll", "Symbol.replace", "Symbol.search", "Symbol.species", "Symbol.split", "Symbol.toPrimitive", "Symbol.toStringTag", "Symbol.unscopables", // Collections and buffers "ArrayBuffer", "DataView", "Float16Array", "Float32Array", "Float64Array", "Int8Array", "Int16Array", "Int32Array", "SharedArrayBuffer", "TypedArray", "Uint8Array", "Uint8ClampedArray", "Uint16Array", "Uint32Array", "WeakMap", "WeakRef", "WeakSet", "FinalizationRegistry", // Async "async-functions", "async-iteration", "top-level-await", // Generators and iterators "async-generators", // Modules "import-assertions", "import-attributes", "dynamic-import", "import.meta", // Proxy and Reflect "Proxy", "Reflect", "Reflect.construct", "Reflect.set", "Reflect.setPrototypeOf", // Regex features "regexp-dotall", "regexp-lookbehind", "regexp-named-groups", "regexp-unicode-property-escapes", "regexp-v-flag", "regexp-match-indices", "regexp-duplicate-named-groups", "regexp-modifiers", // Modern syntax "class-fields-private", "class-fields-private-in", "class-fields-public", "class-methods-private", "class-static-block", "class-static-fields-private", "class-static-fields-public", "class-static-methods-private", "decorators", "hashbang", // Intl "Intl-enumeration", "Intl.DateTimeFormat", "Intl.DisplayNames", "Intl.ListFormat", "Intl.Locale", "Intl.NumberFormat", "Intl.PluralRules", "Intl.RelativeTimeFormat", "Intl.Segmenter", // Built-in methods we don't have "Array.fromAsync", "Array.prototype.at", "Array.prototype.flat", "Array.prototype.flatMap", "Array.prototype.includes", "Array.prototype.values", "ArrayBuffer.prototype.transfer", "Object.fromEntries", "Object.hasOwn", "Object.is", "Promise.allSettled", "Promise.any", "Promise.prototype.finally", "String.fromCodePoint", "String.prototype.at", "String.prototype.endsWith", "String.prototype.includes", "String.prototype.matchAll", "String.prototype.replaceAll", "String.prototype.trimEnd", "String.prototype.trimStart", "String.prototype.isWellFormed", "String.prototype.toWellFormed", // Other "Atomics", "Atomics.waitAsync", "cleanupSome", "coalesce-expression", "cross-realm", "error-cause", "explicit-resource-management", "for-in-order", "globalThis", "json-modules", "json-parse-with-source", "json-superset", "legacy-regexp", "logical-assignment-operators", "numeric-separator-literal", "optional-catch-binding", "optional-chaining", "resizable-arraybuffer", "ShadowRealm", "string-trimming", "super", "tail-call-optimization", "template", "u180e", "well-formed-json-stringify", "__getter__", "__setter__", "__proto__", // Iterator helpers "iterator-helpers", "set-methods", "change-array-by-copy", "symbols-as-weakmap-keys", "Temporal", "Array.prototype.group", "Math.sumPrecise", "Disposable", "using", ]; /// Harness include files that we can handle (we supply our own preamble). /// Tests requiring other includes are skipped. const SUPPORTED_INCLUDES: &[&str] = &[ "sta.js", "assert.js", "compareArray.js", "propertyHelper.js", ]; /// Metadata extracted from a Test262 test file's YAML frontmatter. struct TestMeta { /// If true, the test expects a parse/early error. negative_phase_parse: bool, /// If true, the test expects a runtime error. negative_phase_runtime: bool, /// The expected error type for negative tests (e.g. "SyntaxError"). _negative_type: Option, /// If true, this is an async test. is_async: bool, /// If true, this test should be run as a module. is_module: bool, /// If true, skip the harness preamble. is_raw: bool, /// Required features. features: Vec, /// Required harness includes. includes: Vec, } impl TestMeta { fn should_skip(&self) -> Option<&'static str> { if self.is_async { return Some("async"); } if self.is_module { return Some("module"); } // Skip tests requiring unsupported features. for feat in &self.features { if UNSUPPORTED_FEATURES.contains(&feat.as_str()) { return Some("unsupported feature"); } } // Skip tests requiring harness includes we can't provide. for inc in &self.includes { if !SUPPORTED_INCLUDES.contains(&inc.as_str()) { return Some("unsupported include"); } } None } } /// Parse the YAML-ish frontmatter from a Test262 test file. /// /// The frontmatter is between `/*---` and `---*/`. fn parse_frontmatter(source: &str) -> TestMeta { let mut meta = TestMeta { negative_phase_parse: false, negative_phase_runtime: false, _negative_type: None, is_async: false, is_module: false, is_raw: false, features: Vec::new(), includes: Vec::new(), }; let start = match source.find("/*---") { Some(i) => i + 5, None => return meta, }; let end = match source[start..].find("---*/") { Some(i) => start + i, None => return meta, }; let yaml = &source[start..end]; // Very simple line-by-line YAML extraction. let mut in_negative = false; let mut in_features = false; let mut in_includes = false; let mut in_flags = false; for line in yaml.lines() { let trimmed = line.trim(); // Detect top-level keys (not indented or with specific indent). if !trimmed.is_empty() && !trimmed.starts_with('-') && !line.starts_with(' ') { in_negative = false; in_features = false; in_includes = false; in_flags = false; } if trimmed.starts_with("negative:") { in_negative = true; continue; } if trimmed.starts_with("features:") { in_features = true; // Check for inline list: features: [a, b] if let Some(rest) = trimmed.strip_prefix("features:") { let rest = rest.trim(); if rest.starts_with('[') && rest.ends_with(']') { let inner = &rest[1..rest.len() - 1]; for item in inner.split(',') { let item = item.trim(); if !item.is_empty() { meta.features.push(item.to_string()); } } in_features = false; } } continue; } if trimmed.starts_with("includes:") { in_includes = true; if let Some(rest) = trimmed.strip_prefix("includes:") { let rest = rest.trim(); if rest.starts_with('[') && rest.ends_with(']') { let inner = &rest[1..rest.len() - 1]; for item in inner.split(',') { let item = item.trim(); if !item.is_empty() { meta.includes.push(item.to_string()); } } in_includes = false; } } continue; } if trimmed.starts_with("flags:") { in_flags = true; if let Some(rest) = trimmed.strip_prefix("flags:") { let rest = rest.trim(); if rest.starts_with('[') && rest.ends_with(']') { let inner = &rest[1..rest.len() - 1]; for item in inner.split(',') { let flag = item.trim(); match flag { "async" => meta.is_async = true, "module" => meta.is_module = true, "raw" => meta.is_raw = true, _ => {} } } in_flags = false; } } continue; } // Handle list items under current key. if let Some(item) = trimmed.strip_prefix("- ") { if in_features { meta.features.push(item.to_string()); } else if in_includes { meta.includes.push(item.to_string()); } else if in_flags { match item { "async" => meta.is_async = true, "module" => meta.is_module = true, "raw" => meta.is_raw = true, _ => {} } } continue; } // Handle sub-keys under negative. if in_negative { if let Some(rest) = trimmed.strip_prefix("phase:") { let phase = rest.trim(); match phase { "parse" | "early" => meta.negative_phase_parse = true, "runtime" | "resolution" => meta.negative_phase_runtime = true, _ => {} } } if let Some(rest) = trimmed.strip_prefix("type:") { meta._negative_type = Some(rest.trim().to_string()); } } } meta } /// Recursively collect all `.js` test files under a directory. fn collect_test_files(dir: &std::path::Path, files: &mut Vec) { let entries = match std::fs::read_dir(dir) { Ok(e) => e, Err(_) => return, }; let mut entries: Vec<_> = entries.filter_map(|e| e.ok()).collect(); entries.sort_by_key(|e| e.file_name()); for entry in entries { let path = entry.path(); if path.is_dir() { collect_test_files(&path, files); } else if path.extension().map_or(false, |e| e == "js") { // Skip _FIXTURE files (test helpers, not tests themselves). let name = path.file_name().unwrap().to_string_lossy(); if !name.contains("_FIXTURE") { files.push(path); } } } } /// Result of running a single test. #[allow(dead_code)] enum TestResult { Pass, Fail(String), Skip(String), } /// Maximum instructions per test (prevents infinite loops). const INSTRUCTION_LIMIT: u64 = 1_000_000; /// Run a single Test262 test file. Uses `catch_unwind` to handle compiler/VM /// panics gracefully so a single broken test doesn't crash the whole suite. fn run_test(path: &std::path::Path) -> TestResult { let source = match std::fs::read_to_string(path) { Ok(s) => s, Err(e) => return TestResult::Skip(format!("read error: {e}")), }; let meta = parse_frontmatter(&source); if let Some(reason) = meta.should_skip() { return TestResult::Skip(reason.to_string()); } // Wrap execution in catch_unwind to survive compiler/VM panics. let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { run_test_inner(&source, &meta) })); match result { Ok(r) => r, Err(_) => TestResult::Fail("panic".into()), } } /// Install a silent panic hook to avoid thousands of lines of panic output /// when tests trigger compiler/VM bugs. fn set_silent_panic_hook() { std::panic::set_hook(Box::new(|_| {})); } /// Restore the default panic hook. fn restore_panic_hook() { let _ = std::panic::take_hook(); } fn run_test_inner(source: &str, meta: &TestMeta) -> TestResult { let limit = Some(INSTRUCTION_LIMIT); // Negative parse tests: we expect parsing to fail. if meta.negative_phase_parse { match we_js::evaluate(source) { Err(_) => return TestResult::Pass, Ok(_) => return TestResult::Fail("expected parse error but succeeded".into()), } } // Negative runtime tests: we expect a runtime error. if meta.negative_phase_runtime { let preamble = if meta.is_raw { "" } else { HARNESS_PREAMBLE }; match we_js::evaluate_with_preamble_limited(preamble, source, limit) { Err(we_js::JsError::RuntimeError(_)) => return TestResult::Pass, Err(we_js::JsError::SyntaxError(_)) => { // Parse error for a runtime-negative test; count as pass since // stricter parsing is acceptable. return TestResult::Pass; } Err(_) => return TestResult::Pass, Ok(()) => return TestResult::Fail("expected runtime error but succeeded".into()), } } // Positive tests: should parse and run without errors. if meta.is_raw { match we_js::evaluate(source) { Ok(_) => TestResult::Pass, Err(e) => TestResult::Fail(format!("{e}")), } } else { match we_js::evaluate_with_preamble_limited(HARNESS_PREAMBLE, source, limit) { Ok(()) => TestResult::Pass, Err(e) => TestResult::Fail(format!("{e}")), } } } /// Test category statistics. struct CategoryStats { pass: usize, fail: usize, skip: usize, } impl CategoryStats { fn new() -> Self { Self { pass: 0, fail: 0, skip: 0, } } fn total(&self) -> usize { self.pass + self.fail + self.skip } fn pass_rate(&self) -> f64 { let executed = self.pass + self.fail; if executed == 0 { 0.0 } else { (self.pass as f64 / executed as f64) * 100.0 } } } /// Verify the harness preamble compiles and works correctly. #[test] fn test262_harness_preamble() { // Preamble should evaluate without errors. we_js::evaluate(HARNESS_PREAMBLE).expect("harness preamble should evaluate cleanly"); // assert(true) should pass. we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert(true);") .expect("assert(true) should pass"); // assert(false) should throw. assert!( we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert(false);").is_err(), "assert(false) should throw" ); // assert.sameValue with equal values should pass. we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert.sameValue(1, 1);") .expect("assert.sameValue(1, 1) should pass"); // assert.sameValue with unequal values should throw. assert!( we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert.sameValue(1, 2);").is_err(), "assert.sameValue(1, 2) should throw" ); // assert.notSameValue with unequal values should pass. we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert.notSameValue(1, 2);") .expect("assert.notSameValue(1, 2) should pass"); // assert.throws should pass when function throws. we_js::evaluate_with_preamble( HARNESS_PREAMBLE, r#"assert.throws(null, function() { throw "err"; });"#, ) .expect("assert.throws should pass when function throws"); // assert.throws should fail when function doesn't throw. assert!( we_js::evaluate_with_preamble(HARNESS_PREAMBLE, r#"assert.throws(null, function() { });"#,) .is_err(), "assert.throws should fail when function doesn't throw" ); } #[test] fn test262_language_tests() { // Run in a thread with a large stack to avoid stack overflows in debug mode. // Some test262 tests trigger deep recursion in the parser/compiler/VM. let builder = std::thread::Builder::new() .name("test262".into()) .stack_size(32 * 1024 * 1024); let handle = builder .spawn(test262_language_tests_inner) .expect("failed to spawn test262 thread"); handle.join().expect("test262 thread panicked"); } fn test262_language_tests_inner() { let test_dir = std::path::PathBuf::from(WORKSPACE_ROOT).join("tests/test262/test/language"); if !test_dir.exists() { eprintln!( "test262 submodule not checked out at {}", test_dir.display() ); eprintln!("Run: git submodule update --init tests/test262"); return; } let mut files = Vec::new(); collect_test_files(&test_dir, &mut files); let mut total = CategoryStats::new(); let mut groups: Vec<(String, CategoryStats)> = Vec::new(); let mut current_group = String::new(); // Suppress panic output — many tests trigger pre-existing compiler bugs // (register allocation) which panic and are caught by catch_unwind. set_silent_panic_hook(); eprintln!("\n=== Test262 Language Tests ===\n"); for path in &files { // Determine the top-level group (e.g. "expressions", "literals"). let rel = path.strip_prefix(&test_dir).unwrap_or(path); let group = rel .components() .next() .map(|c| c.as_os_str().to_string_lossy().to_string()) .unwrap_or_default(); if group != current_group { current_group = group.clone(); groups.push((group, CategoryStats::new())); } let stats = &mut groups.last_mut().unwrap().1; match run_test(path) { TestResult::Pass => { stats.pass += 1; total.pass += 1; } TestResult::Fail(_) => { stats.fail += 1; total.fail += 1; } TestResult::Skip(_) => { stats.skip += 1; total.skip += 1; } } } // Print results grouped by category. for (name, stats) in &groups { if stats.total() > 0 { eprintln!( " {:<30} {:>4} pass {:>4} fail {:>4} skip ({:.0}% of executed)", name, stats.pass, stats.fail, stats.skip, stats.pass_rate() ); } } eprintln!(); eprintln!( " {:<30} {:>4} pass {:>4} fail {:>4} skip ({:.0}% of executed)", "TOTAL", total.pass, total.fail, total.skip, total.pass_rate() ); eprintln!( " {} total tests, {} executed", total.total(), total.pass + total.fail ); eprintln!(); // Restore default panic hook. restore_panic_hook(); // The test passes as long as the harness itself works. We track pass rate // for progress monitoring but don't assert a minimum threshold yet. // As more built-ins are implemented, the pass rate will increase. assert!( total.pass > 0, "Expected at least some Test262 tests to pass" ); }