// Downlevel-mode correctness sweep over the pinned OSS corpus (vendor/oss/), // one test/downlevel.mjs --dir run per project against a shared known-failure baseline // (test/baselines/downlevel-oss.txt), restricted to files with non-erasable syntax (the // only ones downlevel mode down-levels anything in). Same ratchet semantics as // pnpm test:downlevel:typescript: a failure not already in the baseline fails the run. // // Usage: node scripts/oss-downlevel-sweep.mjs [--write-baseline] [--minutes N] import { execFileSync } from "node:child_process"; import path from "node:path"; import fs from "node:fs"; import { projects, ossRoot } from "./oss-corpus.mjs"; const root = path.resolve(import.meta.dirname, ".."); const baseline = path.join(root, "test/baselines/downlevel-oss.txt"); const write = process.argv.includes("--write-baseline"); const minutesArg = process.argv.indexOf("--minutes"); const perProjectMinutes = minutesArg >= 0 ? Number(process.argv[minutesArg + 1]) : 4; // Syntactic over-approximation (same technique as test:downlevel:typescript's --grep): a raw // text match, not an AST walk, so it can keep a few files with none of these as // comments/strings and occasionally miss an edge case, but it's cheap and shared // with the rest of the project's downlevel tooling. Covers all five non-erasable // constructs downlevel mode down-levels: enum, namespace/module, parameter properties // (public/private/protected/readonly/override on a constructor parameter), and both // `import x = ...` and `export = ...`. const GREP = "enum|namespace|module |public |private |protected |readonly |override |import\\s+[A-Za-z_$][\\w$]*\\s*=|export\\s*=\\s*"; const ps = projects(); if (ps.length === 0) { console.log("oss-downlevel-sweep: no projects under vendor/oss (run ./scripts/oss.sh first)"); process.exit(1); } let totals = {}; let totalNew = 0; const rows = []; const allFailingPaths = []; const allFailDetails = []; // Written after every project, not just at the end: a 17-project sweep over // vscode-sized repos can run long, and "prefer incremental, sharded runs that // write results to files you can read back" (project rule) means a reader // shouldn't have to wait for the whole sweep to see what's done so far. const progressFile = path.join(root, "corpus/oss-downlevel-progress.json"); const failDetailsFile = path.join(root, "corpus/oss-downlevel-fail-details.jsonl"); function writeProgress() { fs.writeFileSync(progressFile, JSON.stringify({ done: rows.map((r) => r.name), rows, totals }, null, 2)); fs.writeFileSync(failDetailsFile, allFailDetails.map(([project, p, detail]) => JSON.stringify({ project, path: p, detail })).join("\n") + (allFailDetails.length ? "\n" : "")); } // vscode's working tree is an order of magnitude bigger than the next-largest // project (corpus/oss-nonerasable.json: 11,265 files, 4,301 with non-erasable // syntax, vs. angular's 4,713/718); the flat per-project budget would either // starve it or blow the whole sweep's wall clock, so it gets its own larger // allowance. const BUDGET_OVERRIDES = { vscode: perProjectMinutes * 4 }; for (const name of ps) { const dir = path.join(ossRoot, name); const minutes = BUDGET_OVERRIDES[name] ?? perProjectMinutes; const args = [ path.join(root, "test/downlevel.mjs"), "--dir", dir, "--grep", GREP, "--minutes", String(minutes), ]; let out; try { out = execFileSync(process.execPath, args, { cwd: root, maxBuffer: 1 << 29, env: { ...process.env, SHOW: "1000000", SHOW_OXIDASE: "" } }).toString(); } catch (e) { out = (e.stdout ?? "").toString(); } const bucketsLine = /downlevel --dir: (\{.*\})/.exec(out); if (!bucketsLine) throw new Error(`oss-downlevel-sweep: test/downlevel.mjs printed no summary for ${name}:\n${out.slice(-2000)}`); const buckets = JSON.parse(bucketsLine[1]); const oxLine = /oxidase: (\d+)\/(\d+) agree/.exec(out); for (const line of out.split("\n")) { const fm = /^FAIL (.+?): (.*)$/.exec(line); if (fm) { allFailingPaths.push(fm[1]); allFailDetails.push([name, fm[1], fm[2]]); } } for (const [k, v] of Object.entries(buckets)) totals[k] = (totals[k] ?? 0) + v; const fail = buckets.fail ?? 0; const judged = Object.values(buckets).reduce((a, b) => a + b, 0); rows.push({ name, buckets, oxidase: oxLine ? `${oxLine[1]}/${oxLine[2]}` : null }); console.log(`${name.padEnd(18)} ${JSON.stringify(buckets)}${oxLine ? ` oxidase ${oxLine[1]}/${oxLine[2]}` : ""}`); writeProgress(); } console.log(`\noss-downlevel-sweep totals: ${JSON.stringify(totals)}`); if (write) { const unique = [...new Set(allFailingPaths.map((p) => path.relative(root, path.resolve(p))))].sort(); fs.writeFileSync(baseline, unique.length ? unique.join("\n") + "\n" : ""); console.log(`oss-downlevel-sweep: wrote ${unique.length} known failures to test/baselines/downlevel-oss.txt`); process.exit(0); } const known = new Set(fs.existsSync(baseline) ? fs.readFileSync(baseline, "utf8").split("\n").filter(Boolean) : []); const failingNow = new Set(allFailingPaths.map((p) => path.relative(root, path.resolve(p)))); const fresh = [...failingNow].filter((p) => !known.has(p)); const fixed = [...known].filter((p) => !failingNow.has(p)); console.log(`baseline test/baselines/downlevel-oss.txt: ${known.size} known, ${fresh.length} new, ${fixed.length} now passing${fixed.length ? " (shrink the baseline)" : ""}`); if (fresh.length) console.log(`fresh failures:\n ${fresh.join("\n ")}`); process.exitCode = fresh.length ? 1 : 0;