we / crates / js / tests / test262.rs

//! Test262 test harness.
//!
//! Walks the Test262 test suite and runs each test case against our JavaScript
//! engine. Reports pass/fail/skip counts.
//!
//! Run with: `cargo test -p we-js --test test262`

/// Workspace root relative to the crate directory.
const WORKSPACE_ROOT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../");

/// Metadata extracted from a Test262 test file's YAML frontmatter.
struct TestMeta {
    /// If true, the test expects a parse/early error.
    negative_phase_parse: bool,
    /// If true, the test expects a runtime error.
    negative_phase_runtime: bool,
    /// The expected error type for negative tests (e.g. "SyntaxError").
    negative_type: Option<String>,
    /// If true, this is an async test.
    is_async: bool,
    /// If true, this test should be run as a module.
    is_module: bool,
    /// If true, skip the harness preamble.
    is_raw: bool,
    /// Required features.
    features: Vec<String>,
    /// Required harness includes.
    includes: Vec<String>,
}

impl TestMeta {
    fn should_skip(&self) -> bool {
        // Skip async tests and module tests for now.
        self.is_async || self.is_module
    }
}

/// Parse the YAML-ish frontmatter from a Test262 test file.
///
/// The frontmatter is between `/*---` and `---*/`.
fn parse_frontmatter(source: &str) -> TestMeta {
    let mut meta = TestMeta {
        negative_phase_parse: false,
        negative_phase_runtime: false,
        negative_type: None,
        is_async: false,
        is_module: false,
        is_raw: false,
        features: Vec::new(),
        includes: Vec::new(),
    };

    let start = match source.find("/*---") {
        Some(i) => i + 5,
        None => return meta,
    };
    let end = match source[start..].find("---*/") {
        Some(i) => start + i,
        None => return meta,
    };
    let yaml = &source[start..end];

    // Very simple line-by-line YAML extraction.
    let mut in_negative = false;
    let mut in_features = false;
    let mut in_includes = false;
    let mut in_flags = false;

    for line in yaml.lines() {
        let trimmed = line.trim();

        // Detect top-level keys (not indented or with specific indent).
        if !trimmed.is_empty() && !trimmed.starts_with('-') && !line.starts_with(' ') {
            in_negative = false;
            in_features = false;
            in_includes = false;
            in_flags = false;
        }

        if trimmed.starts_with("negative:") {
            in_negative = true;
            continue;
        }
        if trimmed.starts_with("features:") {
            in_features = true;
            // Check for inline list: features: [a, b]
            if let Some(rest) = trimmed.strip_prefix("features:") {
                let rest = rest.trim();
                if rest.starts_with('[') && rest.ends_with(']') {
                    let inner = &rest[1..rest.len() - 1];
                    for item in inner.split(',') {
                        let item = item.trim();
                        if !item.is_empty() {
                            meta.features.push(item.to_string());
                        }
                    }
                    in_features = false;
                }
            }
            continue;
        }
        if trimmed.starts_with("includes:") {
            in_includes = true;
            if let Some(rest) = trimmed.strip_prefix("includes:") {
                let rest = rest.trim();
                if rest.starts_with('[') && rest.ends_with(']') {
                    let inner = &rest[1..rest.len() - 1];
                    for item in inner.split(',') {
                        let item = item.trim();
                        if !item.is_empty() {
                            meta.includes.push(item.to_string());
                        }
                    }
                    in_includes = false;
                }
            }
            continue;
        }
        if trimmed.starts_with("flags:") {
            in_flags = true;
            if let Some(rest) = trimmed.strip_prefix("flags:") {
                let rest = rest.trim();
                if rest.starts_with('[') && rest.ends_with(']') {
                    let inner = &rest[1..rest.len() - 1];
                    for item in inner.split(',') {
                        let flag = item.trim();
                        match flag {
                            "async" => meta.is_async = true,
                            "module" => meta.is_module = true,
                            "raw" => meta.is_raw = true,
                            _ => {}
                        }
                    }
                    in_flags = false;
                }
            }
            continue;
        }

        // Handle list items under current key.
        if let Some(item) = trimmed.strip_prefix("- ") {
            if in_features {
                meta.features.push(item.to_string());
            } else if in_includes {
                meta.includes.push(item.to_string());
            } else if in_flags {
                match item {
                    "async" => meta.is_async = true,
                    "module" => meta.is_module = true,
                    "raw" => meta.is_raw = true,
                    _ => {}
                }
            }
            continue;
        }

        // Handle sub-keys under negative.
        if in_negative {
            if let Some(rest) = trimmed.strip_prefix("phase:") {
                let phase = rest.trim();
                match phase {
                    "parse" | "early" => meta.negative_phase_parse = true,
                    "runtime" | "resolution" => meta.negative_phase_runtime = true,
                    _ => {}
                }
            }
            if let Some(rest) = trimmed.strip_prefix("type:") {
                meta.negative_type = Some(rest.trim().to_string());
            }
        }
    }

    meta
}

/// Recursively collect all `.js` test files under a directory.
fn collect_test_files(dir: &std::path::Path, files: &mut Vec<std::path::PathBuf>) {
    let entries = match std::fs::read_dir(dir) {
        Ok(e) => e,
        Err(_) => return,
    };
    let mut entries: Vec<_> = entries.filter_map(|e| e.ok()).collect();
    entries.sort_by_key(|e| e.file_name());

    for entry in entries {
        let path = entry.path();
        if path.is_dir() {
            collect_test_files(&path, files);
        } else if path.extension().map_or(false, |e| e == "js") {
            // Skip _FIXTURE files (test helpers, not tests themselves).
            let name = path.file_name().unwrap().to_string_lossy();
            if !name.contains("_FIXTURE") {
                files.push(path);
            }
        }
    }
}

/// Run a single Test262 test file. Returns (pass, fail, skip).
fn run_test(path: &std::path::Path) -> (usize, usize, usize) {
    let source = match std::fs::read_to_string(path) {
        Ok(s) => s,
        Err(_) => return (0, 0, 1),
    };

    let meta = parse_frontmatter(&source);

    if meta.should_skip() {
        return (0, 0, 1);
    }

    // For negative parse tests, if our evaluate returns an error, that's a pass.
    // For positive tests, evaluate should succeed (return Ok).
    let result = we_js::evaluate(&source);

    if meta.negative_phase_parse {
        // We expect a parse error. If our engine returns any error, count as pass.
        match result {
            Err(_) => (1, 0, 0),
            Ok(()) => (0, 1, 0),
        }
    } else {
        // We expect success.
        match result {
            Ok(()) => (1, 0, 0),
            Err(_) => (0, 1, 0),
        }
    }
}

#[test]
fn test262_language_tests() {
    let test_dir = std::path::PathBuf::from(WORKSPACE_ROOT).join("tests/test262/test/language");

    if !test_dir.exists() {
        eprintln!(
            "test262 submodule not checked out at {}",
            test_dir.display()
        );
        eprintln!("Run: git submodule update --init tests/test262");
        return;
    }

    let mut files = Vec::new();
    collect_test_files(&test_dir, &mut files);

    let mut total_pass = 0;
    let mut total_fail = 0;
    let mut total_skip = 0;

    // Group results by top-level subdirectory for reporting.
    let mut current_group = String::new();
    let mut group_pass = 0;
    let mut group_fail = 0;
    let mut group_skip = 0;

    for path in &files {
        // Determine the top-level group (e.g. "expressions", "literals").
        let rel = path.strip_prefix(&test_dir).unwrap_or(path);
        let group = rel
            .components()
            .next()
            .map(|c| c.as_os_str().to_string_lossy().to_string())
            .unwrap_or_default();

        if group != current_group {
            if !current_group.is_empty() {
                eprintln!(
                    "  {}: {} pass, {} fail, {} skip",
                    current_group, group_pass, group_fail, group_skip
                );
            }
            current_group = group;
            group_pass = 0;
            group_fail = 0;
            group_skip = 0;
        }

        let (p, f, s) = run_test(path);
        group_pass += p;
        group_fail += f;
        group_skip += s;
        total_pass += p;
        total_fail += f;
        total_skip += s;
    }

    // Print last group.
    if !current_group.is_empty() {
        eprintln!(
            "  {}: {} pass, {} fail, {} skip",
            current_group, group_pass, group_fail, group_skip
        );
    }

    eprintln!();
    eprintln!(
        "Test262 language totals: {} pass, {} fail, {} skip ({} total)",
        total_pass,
        total_fail,
        total_skip,
        total_pass + total_fail + total_skip
    );
}