we / crates / js / tests / test262.rs

//! Test262 test harness.
//!
//! Walks the Test262 test suite and runs each test case against our JavaScript
//! engine. Reports pass/fail/skip counts grouped by category.
//!
//! Run with: `cargo test -p we-js --test test262 -- --nocapture`

/// Workspace root relative to the crate directory.
const WORKSPACE_ROOT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../");

/// Minimal JS preamble that defines the Test262 harness helpers using only
/// features our engine supports. This replaces the standard `sta.js` and
/// `assert.js` harness files which require built-ins we don't have yet.
const HARNESS_PREAMBLE: &str = r#"
function Test262Error(message) {
    return "Test262Error: " + message;
}

function $DONOTEVALUATE() {
    throw "Test262: This statement should not be evaluated.";
}

function assert(mustBeTrue, message) {
    if (mustBeTrue === true) {
        return;
    }
    if (message === undefined) {
        message = "Expected true but got " + mustBeTrue;
    }
    throw message;
}

assert._isSameValue = function(a, b) {
    if (a === b) {
        if (a !== 0) { return true; }
        return 1 / a === 1 / b;
    }
    if (a !== a && b !== b) { return true; }
    return false;
};

assert.sameValue = function(actual, expected, message) {
    if (assert._isSameValue(actual, expected)) {
        return;
    }
    if (message === undefined) {
        message = "";
    } else {
        message = message + " ";
    }
    message = message + "Expected SameValue(" + actual + ", " + expected + ") to be true";
    throw message;
};

assert.notSameValue = function(actual, unexpected, message) {
    if (!assert._isSameValue(actual, unexpected)) {
        return;
    }
    if (message === undefined) {
        message = "";
    } else {
        message = message + " ";
    }
    message = message + "Expected not SameValue(" + actual + ", " + unexpected + ")";
    throw message;
};

assert.throws = function(expectedErrorConstructor, func, message) {
    if (typeof func !== "function") {
        throw "assert.throws requires a function argument";
    }
    var threw = false;
    try {
        func();
    } catch (e) {
        threw = true;
    }
    if (!threw) {
        if (message === undefined) {
            message = "Expected an exception to be thrown";
        }
        throw message;
    }
};

function print() {}
"#;

/// Features that our engine does not yet support. Tests requiring any of these
/// are skipped rather than counted as failures.
const UNSUPPORTED_FEATURES: &[&str] = &[
    // Type system extensions
    "BigInt",
    "Symbol",
    "Symbol.asyncIterator",
    "Symbol.hasInstance",
    "Symbol.isConcatSpreadable",
    "Symbol.iterator",
    "Symbol.match",
    "Symbol.matchAll",
    "Symbol.replace",
    "Symbol.search",
    "Symbol.species",
    "Symbol.split",
    "Symbol.toPrimitive",
    "Symbol.toStringTag",
    "Symbol.unscopables",
    // Collections and buffers
    "ArrayBuffer",
    "DataView",
    "Float16Array",
    "Float32Array",
    "Float64Array",
    "Int8Array",
    "Int16Array",
    "Int32Array",
    "Map",
    "Set",
    "SharedArrayBuffer",
    "TypedArray",
    "Uint8Array",
    "Uint8ClampedArray",
    "Uint16Array",
    "Uint32Array",
    "WeakMap",
    "WeakRef",
    "WeakSet",
    "FinalizationRegistry",
    // Async
    "Promise",
    "async-functions",
    "async-iteration",
    "top-level-await",
    // Generators and iterators
    "generators",
    "async-generators",
    // Modules
    "import-assertions",
    "import-attributes",
    "dynamic-import",
    "import.meta",
    // Proxy and Reflect
    "Proxy",
    "Reflect",
    "Reflect.construct",
    "Reflect.set",
    "Reflect.setPrototypeOf",
    // Regex features
    "regexp-dotall",
    "regexp-lookbehind",
    "regexp-named-groups",
    "regexp-unicode-property-escapes",
    "regexp-v-flag",
    "regexp-match-indices",
    "regexp-duplicate-named-groups",
    "regexp-modifiers",
    // Modern syntax
    "class-fields-private",
    "class-fields-private-in",
    "class-fields-public",
    "class-methods-private",
    "class-static-block",
    "class-static-fields-private",
    "class-static-fields-public",
    "class-static-methods-private",
    "decorators",
    "hashbang",
    // Intl
    "Intl-enumeration",
    "Intl.DateTimeFormat",
    "Intl.DisplayNames",
    "Intl.ListFormat",
    "Intl.Locale",
    "Intl.NumberFormat",
    "Intl.PluralRules",
    "Intl.RelativeTimeFormat",
    "Intl.Segmenter",
    // Built-in methods we don't have
    "Array.fromAsync",
    "Array.prototype.at",
    "Array.prototype.flat",
    "Array.prototype.flatMap",
    "Array.prototype.includes",
    "Array.prototype.values",
    "ArrayBuffer.prototype.transfer",
    "Object.fromEntries",
    "Object.hasOwn",
    "Object.is",
    "Promise.allSettled",
    "Promise.any",
    "Promise.prototype.finally",
    "String.fromCodePoint",
    "String.prototype.at",
    "String.prototype.endsWith",
    "String.prototype.includes",
    "String.prototype.matchAll",
    "String.prototype.replaceAll",
    "String.prototype.trimEnd",
    "String.prototype.trimStart",
    "String.prototype.isWellFormed",
    "String.prototype.toWellFormed",
    // Other
    "Atomics",
    "Atomics.waitAsync",
    "cleanupSome",
    "coalesce-expression",
    "cross-realm",
    "error-cause",
    "explicit-resource-management",
    "for-in-order",
    "globalThis",
    "json-modules",
    "json-parse-with-source",
    "json-superset",
    "legacy-regexp",
    "logical-assignment-operators",
    "numeric-separator-literal",
    "optional-catch-binding",
    "optional-chaining",
    "resizable-arraybuffer",
    "ShadowRealm",
    "string-trimming",
    "super",
    "tail-call-optimization",
    "template",
    "u180e",
    "well-formed-json-stringify",
    "__getter__",
    "__setter__",
    "__proto__",
    // Iterator helpers
    "iterator-helpers",
    "set-methods",
    "change-array-by-copy",
    "symbols-as-weakmap-keys",
    "Temporal",
    "Array.prototype.group",
    "Math.sumPrecise",
    "Disposable",
    "using",
];

/// Harness include files that we can handle (we supply our own preamble).
/// Tests requiring other includes are skipped.
const SUPPORTED_INCLUDES: &[&str] = &[
    "sta.js",
    "assert.js",
    "compareArray.js",
    "propertyHelper.js",
];

/// Metadata extracted from a Test262 test file's YAML frontmatter.
struct TestMeta {
    /// If true, the test expects a parse/early error.
    negative_phase_parse: bool,
    /// If true, the test expects a runtime error.
    negative_phase_runtime: bool,
    /// The expected error type for negative tests (e.g. "SyntaxError").
    _negative_type: Option<String>,
    /// If true, this is an async test.
    is_async: bool,
    /// If true, this test should be run as a module.
    is_module: bool,
    /// If true, skip the harness preamble.
    is_raw: bool,
    /// Required features.
    features: Vec<String>,
    /// Required harness includes.
    includes: Vec<String>,
}

impl TestMeta {
    fn should_skip(&self) -> Option<&'static str> {
        if self.is_async {
            return Some("async");
        }
        if self.is_module {
            return Some("module");
        }
        // Skip tests requiring unsupported features.
        for feat in &self.features {
            if UNSUPPORTED_FEATURES.contains(&feat.as_str()) {
                return Some("unsupported feature");
            }
        }
        // Skip tests requiring harness includes we can't provide.
        for inc in &self.includes {
            if !SUPPORTED_INCLUDES.contains(&inc.as_str()) {
                return Some("unsupported include");
            }
        }
        None
    }
}

/// Parse the YAML-ish frontmatter from a Test262 test file.
///
/// The frontmatter is between `/*---` and `---*/`.
fn parse_frontmatter(source: &str) -> TestMeta {
    let mut meta = TestMeta {
        negative_phase_parse: false,
        negative_phase_runtime: false,
        _negative_type: None,
        is_async: false,
        is_module: false,
        is_raw: false,
        features: Vec::new(),
        includes: Vec::new(),
    };

    let start = match source.find("/*---") {
        Some(i) => i + 5,
        None => return meta,
    };
    let end = match source[start..].find("---*/") {
        Some(i) => start + i,
        None => return meta,
    };
    let yaml = &source[start..end];

    // Very simple line-by-line YAML extraction.
    let mut in_negative = false;
    let mut in_features = false;
    let mut in_includes = false;
    let mut in_flags = false;

    for line in yaml.lines() {
        let trimmed = line.trim();

        // Detect top-level keys (not indented or with specific indent).
        if !trimmed.is_empty() && !trimmed.starts_with('-') && !line.starts_with(' ') {
            in_negative = false;
            in_features = false;
            in_includes = false;
            in_flags = false;
        }

        if trimmed.starts_with("negative:") {
            in_negative = true;
            continue;
        }
        if trimmed.starts_with("features:") {
            in_features = true;
            // Check for inline list: features: [a, b]
            if let Some(rest) = trimmed.strip_prefix("features:") {
                let rest = rest.trim();
                if rest.starts_with('[') && rest.ends_with(']') {
                    let inner = &rest[1..rest.len() - 1];
                    for item in inner.split(',') {
                        let item = item.trim();
                        if !item.is_empty() {
                            meta.features.push(item.to_string());
                        }
                    }
                    in_features = false;
                }
            }
            continue;
        }
        if trimmed.starts_with("includes:") {
            in_includes = true;
            if let Some(rest) = trimmed.strip_prefix("includes:") {
                let rest = rest.trim();
                if rest.starts_with('[') && rest.ends_with(']') {
                    let inner = &rest[1..rest.len() - 1];
                    for item in inner.split(',') {
                        let item = item.trim();
                        if !item.is_empty() {
                            meta.includes.push(item.to_string());
                        }
                    }
                    in_includes = false;
                }
            }
            continue;
        }
        if trimmed.starts_with("flags:") {
            in_flags = true;
            if let Some(rest) = trimmed.strip_prefix("flags:") {
                let rest = rest.trim();
                if rest.starts_with('[') && rest.ends_with(']') {
                    let inner = &rest[1..rest.len() - 1];
                    for item in inner.split(',') {
                        let flag = item.trim();
                        match flag {
                            "async" => meta.is_async = true,
                            "module" => meta.is_module = true,
                            "raw" => meta.is_raw = true,
                            _ => {}
                        }
                    }
                    in_flags = false;
                }
            }
            continue;
        }

        // Handle list items under current key.
        if let Some(item) = trimmed.strip_prefix("- ") {
            if in_features {
                meta.features.push(item.to_string());
            } else if in_includes {
                meta.includes.push(item.to_string());
            } else if in_flags {
                match item {
                    "async" => meta.is_async = true,
                    "module" => meta.is_module = true,
                    "raw" => meta.is_raw = true,
                    _ => {}
                }
            }
            continue;
        }

        // Handle sub-keys under negative.
        if in_negative {
            if let Some(rest) = trimmed.strip_prefix("phase:") {
                let phase = rest.trim();
                match phase {
                    "parse" | "early" => meta.negative_phase_parse = true,
                    "runtime" | "resolution" => meta.negative_phase_runtime = true,
                    _ => {}
                }
            }
            if let Some(rest) = trimmed.strip_prefix("type:") {
                meta._negative_type = Some(rest.trim().to_string());
            }
        }
    }

    meta
}

/// Recursively collect all `.js` test files under a directory.
fn collect_test_files(dir: &std::path::Path, files: &mut Vec<std::path::PathBuf>) {
    let entries = match std::fs::read_dir(dir) {
        Ok(e) => e,
        Err(_) => return,
    };
    let mut entries: Vec<_> = entries.filter_map(|e| e.ok()).collect();
    entries.sort_by_key(|e| e.file_name());

    for entry in entries {
        let path = entry.path();
        if path.is_dir() {
            collect_test_files(&path, files);
        } else if path.extension().map_or(false, |e| e == "js") {
            // Skip _FIXTURE files (test helpers, not tests themselves).
            let name = path.file_name().unwrap().to_string_lossy();
            if !name.contains("_FIXTURE") {
                files.push(path);
            }
        }
    }
}

/// Result of running a single test.
#[allow(dead_code)]
enum TestResult {
    Pass,
    Fail(String),
    Skip(String),
}

/// Maximum instructions per test (prevents infinite loops).
const INSTRUCTION_LIMIT: u64 = 1_000_000;

/// Run a single Test262 test file. Uses `catch_unwind` to handle compiler/VM
/// panics gracefully so a single broken test doesn't crash the whole suite.
fn run_test(path: &std::path::Path) -> TestResult {
    let source = match std::fs::read_to_string(path) {
        Ok(s) => s,
        Err(e) => return TestResult::Skip(format!("read error: {e}")),
    };

    let meta = parse_frontmatter(&source);

    if let Some(reason) = meta.should_skip() {
        return TestResult::Skip(reason.to_string());
    }

    // Wrap execution in catch_unwind to survive compiler/VM panics.
    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
        run_test_inner(&source, &meta)
    }));

    match result {
        Ok(r) => r,
        Err(_) => TestResult::Fail("panic".into()),
    }
}

/// Install a silent panic hook to avoid thousands of lines of panic output
/// when tests trigger compiler/VM bugs.
fn set_silent_panic_hook() {
    std::panic::set_hook(Box::new(|_| {}));
}

/// Restore the default panic hook.
fn restore_panic_hook() {
    let _ = std::panic::take_hook();
}

fn run_test_inner(source: &str, meta: &TestMeta) -> TestResult {
    let limit = Some(INSTRUCTION_LIMIT);

    // Negative parse tests: we expect parsing to fail.
    if meta.negative_phase_parse {
        match we_js::evaluate(source) {
            Err(_) => return TestResult::Pass,
            Ok(_) => return TestResult::Fail("expected parse error but succeeded".into()),
        }
    }

    // Negative runtime tests: we expect a runtime error.
    if meta.negative_phase_runtime {
        let preamble = if meta.is_raw { "" } else { HARNESS_PREAMBLE };
        match we_js::evaluate_with_preamble_limited(preamble, source, limit) {
            Err(we_js::JsError::RuntimeError(_)) => return TestResult::Pass,
            Err(we_js::JsError::SyntaxError(_)) => {
                // Parse error for a runtime-negative test; count as pass since
                // stricter parsing is acceptable.
                return TestResult::Pass;
            }
            Err(_) => return TestResult::Pass,
            Ok(()) => return TestResult::Fail("expected runtime error but succeeded".into()),
        }
    }

    // Positive tests: should parse and run without errors.
    if meta.is_raw {
        match we_js::evaluate(source) {
            Ok(_) => TestResult::Pass,
            Err(e) => TestResult::Fail(format!("{e}")),
        }
    } else {
        match we_js::evaluate_with_preamble_limited(HARNESS_PREAMBLE, source, limit) {
            Ok(()) => TestResult::Pass,
            Err(e) => TestResult::Fail(format!("{e}")),
        }
    }
}

/// Test category statistics.
struct CategoryStats {
    pass: usize,
    fail: usize,
    skip: usize,
}

impl CategoryStats {
    fn new() -> Self {
        Self {
            pass: 0,
            fail: 0,
            skip: 0,
        }
    }

    fn total(&self) -> usize {
        self.pass + self.fail + self.skip
    }

    fn pass_rate(&self) -> f64 {
        let executed = self.pass + self.fail;
        if executed == 0 {
            0.0
        } else {
            (self.pass as f64 / executed as f64) * 100.0
        }
    }
}

/// Verify the harness preamble compiles and works correctly.
#[test]
fn test262_harness_preamble() {
    // Preamble should evaluate without errors.
    we_js::evaluate(HARNESS_PREAMBLE).expect("harness preamble should evaluate cleanly");

    // assert(true) should pass.
    we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert(true);")
        .expect("assert(true) should pass");

    // assert(false) should throw.
    assert!(
        we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert(false);").is_err(),
        "assert(false) should throw"
    );

    // assert.sameValue with equal values should pass.
    we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert.sameValue(1, 1);")
        .expect("assert.sameValue(1, 1) should pass");

    // assert.sameValue with unequal values should throw.
    assert!(
        we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert.sameValue(1, 2);").is_err(),
        "assert.sameValue(1, 2) should throw"
    );

    // assert.notSameValue with unequal values should pass.
    we_js::evaluate_with_preamble(HARNESS_PREAMBLE, "assert.notSameValue(1, 2);")
        .expect("assert.notSameValue(1, 2) should pass");

    // assert.throws should pass when function throws.
    we_js::evaluate_with_preamble(
        HARNESS_PREAMBLE,
        r#"assert.throws(null, function() { throw "err"; });"#,
    )
    .expect("assert.throws should pass when function throws");

    // assert.throws should fail when function doesn't throw.
    assert!(
        we_js::evaluate_with_preamble(HARNESS_PREAMBLE, r#"assert.throws(null, function() { });"#,)
            .is_err(),
        "assert.throws should fail when function doesn't throw"
    );
}

#[test]
fn test262_language_tests() {
    // Run in a thread with a large stack to avoid stack overflows in debug mode.
    // Some test262 tests trigger deep recursion in the parser/compiler/VM.
    let builder = std::thread::Builder::new()
        .name("test262".into())
        .stack_size(32 * 1024 * 1024);
    let handle = builder
        .spawn(test262_language_tests_inner)
        .expect("failed to spawn test262 thread");
    handle.join().expect("test262 thread panicked");
}

fn test262_language_tests_inner() {
    let test_dir = std::path::PathBuf::from(WORKSPACE_ROOT).join("tests/test262/test/language");

    if !test_dir.exists() {
        eprintln!(
            "test262 submodule not checked out at {}",
            test_dir.display()
        );
        eprintln!("Run: git submodule update --init tests/test262");
        return;
    }

    let mut files = Vec::new();
    collect_test_files(&test_dir, &mut files);

    let mut total = CategoryStats::new();
    let mut groups: Vec<(String, CategoryStats)> = Vec::new();
    let mut current_group = String::new();

    // Suppress panic output — many tests trigger pre-existing compiler bugs
    // (register allocation) which panic and are caught by catch_unwind.
    set_silent_panic_hook();

    eprintln!("\n=== Test262 Language Tests ===\n");

    for path in &files {
        // Determine the top-level group (e.g. "expressions", "literals").
        let rel = path.strip_prefix(&test_dir).unwrap_or(path);
        let group = rel
            .components()
            .next()
            .map(|c| c.as_os_str().to_string_lossy().to_string())
            .unwrap_or_default();

        if group != current_group {
            current_group = group.clone();
            groups.push((group, CategoryStats::new()));
        }

        let stats = &mut groups.last_mut().unwrap().1;

        match run_test(path) {
            TestResult::Pass => {
                stats.pass += 1;
                total.pass += 1;
            }
            TestResult::Fail(_) => {
                stats.fail += 1;
                total.fail += 1;
            }
            TestResult::Skip(_) => {
                stats.skip += 1;
                total.skip += 1;
            }
        }
    }

    // Print results grouped by category.
    for (name, stats) in &groups {
        if stats.total() > 0 {
            eprintln!(
                "  {:<30} {:>4} pass  {:>4} fail  {:>4} skip  ({:.0}% of executed)",
                name,
                stats.pass,
                stats.fail,
                stats.skip,
                stats.pass_rate()
            );
        }
    }

    eprintln!();
    eprintln!(
        "  {:<30} {:>4} pass  {:>4} fail  {:>4} skip  ({:.0}% of executed)",
        "TOTAL",
        total.pass,
        total.fail,
        total.skip,
        total.pass_rate()
    );
    eprintln!(
        "  {} total tests, {} executed",
        total.total(),
        total.pass + total.fail
    );
    eprintln!();

    // Restore default panic hook.
    restore_panic_hook();

    // The test passes as long as the harness itself works. We track pass rate
    // for progress monitoring but don't assert a minimum threshold yet.
    // As more built-ins are implemented, the pass rate will increase.
    assert!(
        total.pass > 0,
        "Expected at least some Test262 tests to pass"
    );
}