Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317// Fuzzer for non-ASCII correctness in engine/. Injects non-ASCII at many lexical sites (comments, string and// template contents, identifiers, JSDoc) into corpus inputs and checks the same contract test/erase.mjs// checks: positional UTF-16 preservation, JS AST equality with ts-blank-space, error-position equality, and// streaming equivalence at several chunk sizes including tiny ones. Deterministic by seed; prints a minimized// failing input on the first failure (or every failure with --all).//// Usage: node test/fuzz-unicode.mjs [count] [seed]// TYPESTRIP_ERASE_WASM=build/erase.wasm node test/fuzz-unicode.mjs 50000 1import fs from "node:fs";import path from "node:path";import ts from "typescript";import { blankSourceFile } from "../vendor/ts-blank-space/out/index.js";import { transformSync } from "../src/index.js";import { cases } from "./corpus.mjs";
const root = path.resolve(import.meta.dirname, "..");const SK = ts.SyntaxKind;const ALLOWED = new Set([" ", ";", "(", ")"]);const COUNT = Number(process.argv[2] ?? 50000);const SEED = Number(process.argv[3] ?? 1);const SHOW_ALL = process.argv.includes("--all");
// ---------------------------------------------------------------------------// Deterministic PRNG (xorshift32), seeded.
function makeRng(seed) { let s = seed >>> 0 || 1; return function rng() { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s >>>= 0; s ^= s << 5; s >>>= 0; return s / 0x100000000; };}
function pick(rng, arr) { return arr[Math.floor(rng() * arr.length)];}
// ---------------------------------------------------------------------------// Character pools. Codepoints as strings, grouped by UTF-8 width and kind.
const TWO_BYTE = ["é", "ñ", "σ", "©", "Д"]; // é ñ σ © Дconst THREE_BYTE = ["中", "文", "あ", "☃", "❤"]; // 中 文 あ ☃ ❤const FOUR_BYTE = ["\u{1f600}", "\u{1f984}", "\u{10000}", "\u{1d4b3}", "\u{1f1fa}\u{1f1f8}"]; // emoji, unicorn, 𐀀, 𝒳, flag pairconst ZWJ_SEQS = ["\u{1f468}\u{1f469}\u{1f467}", "\u{1f44d}", "ab", "ab"]; // family, thumbs-up+ZWJ, ZWJ/ZWNJ in plain textconst LINE_SEP = ["
", "
"]; // LS, PSconst NBSP = " ";const FEFF = "";// Unicode letters valid as IdentifierStart/Part (avoid digits-only or combining marks that need a base).const ID_CHARS = ["é", "σ", "中", "Д", "\u{1d4b3}"];
function loneSurrogateHigh() { return "\ud83d"; } // high surrogate alone (no low pair follows)function loneSurrogateLow() { return "\udc00"; } // low surrogate alone (no high pair precedes)
// ---------------------------------------------------------------------------// Site discovery via the TypeScript scanner (skipTrivia=false to see comments individually).
function findSites(src) { const sites = { commentLine: [], commentBlock: [], jsdoc: [], string: [], template: [], identifier: [], trivia: [] }; const scanner = ts.createScanner(ts.ScriptTarget.ESNext, false, ts.LanguageVariant.Standard, src); scanner.setOnError(() => {}); let prevEnd = 0; while (true) { const kind = scanner.scan(); if (kind === SK.EndOfFileToken) break; const start = scanner.getTokenPos(); const end = scanner.getTextPos(); if (kind === SK.SingleLineCommentTrivia) sites.commentLine.push([start, end]); else if (kind === SK.MultiLineCommentTrivia) { if (src.slice(start, start + 3) === "/**") sites.jsdoc.push([start, end]); else sites.commentBlock.push([start, end]); } else if (kind === SK.WhitespaceTrivia || kind === SK.NewLineTrivia) sites.trivia.push([start, end]); else if ( kind === SK.StringLiteral || kind === SK.NoSubstitutionTemplateLiteral || kind === SK.TemplateHead || kind === SK.TemplateMiddle || kind === SK.TemplateTail ) { // Exclude the delimiter bytes (quotes, backtick, ${, }). const text = src.slice(start, end); let a = start + 1, b = end - 1; if (kind === SK.TemplateHead) b = end - 2; // ends in ${ if (kind === SK.TemplateMiddle) { a = start + 1; b = end - 2; } if (kind === SK.TemplateTail) a = start + 1; if (b > a) sites.string.push([a, b]); if (kind !== SK.StringLiteral) sites.template.push([a, b]); } else if (kind === SK.Identifier) { sites.identifier.push([start, end]); } prevEnd = end; } return sites;}
// ---------------------------------------------------------------------------// Mutators: given src and a site range, return mutated src or null if not applicable.
function insertAt(src, pos, text) { return src.slice(0, pos) + text + src.slice(pos);}
function midpoint(rng, a, b) { if (b <= a) return a; return a + 1 + Math.floor(rng() * (b - a - 1 >= 0 ? b - a : 1));}
const MUTATORS = [ // comment-line: no raw newline, no U+2028/U+2029 (both end a single-line comment). { site: "commentLine", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, NBSP, FEFF, ...ZWJ_SEQS]) }, // comment-block / jsdoc: anything except the literal "*/" sequence. { site: "commentBlock", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, ...LINE_SEP, NBSP, FEFF, ...ZWJ_SEQS, loneSurrogateHigh(), loneSurrogateLow()]) }, { site: "jsdoc", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, ...LINE_SEP, NBSP, FEFF, ...ZWJ_SEQS]) }, // string contents: avoid the literal's own quote char, which the caller enforces by range choice. { site: "string", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, NBSP, FEFF, ...ZWJ_SEQS, loneSurrogateHigh(), loneSurrogateLow()]) }, // template contents: same, plus line separators are legal inside templates (they're real newlines there). { site: "template", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, ...LINE_SEP, NBSP, FEFF, ...ZWJ_SEQS, loneSurrogateHigh(), loneSurrogateLow()]) }, // identifiers: append a valid ID continuation char (letter, or ZWJ/ZWNJ which the grammar special-cases). { site: "identifier", gen: (rng) => pick(rng, [...ID_CHARS, "", ""]), append: true }, // trivia: BOM-like U+FEFF or other whitespace mid-file, between tokens. { site: "trivia", gen: (rng) => pick(rng, [FEFF, NBSP, ...LINE_SEP]) },];
function mutateOnce(src, rng) { const sites = findSites(src); const available = MUTATORS.filter((m) => sites[m.site].length > 0); if (!available.length) return null; const m = pick(rng, available); const range = pick(rng, sites[m.site]); const text = m.gen(rng); if (m.append) return insertAt(src, range[1], text); const pos = midpoint(rng, range[0], range[1]); return insertAt(src, pos, text);}
function mutate(src, rng, times) { let out = src; for (let i = 0; i < times; i++) { const next = mutateOnce(out, rng); if (next === null) break; out = next; } return out;}
// ---------------------------------------------------------------------------// The same contract test/erase.mjs checks.
function jsAst(code) { const sf = ts.createSourceFile("out.js", code, ts.ScriptTarget.ESNext, false, ts.ScriptKind.JS); const out = []; (function visit(n) { if (n.kind === SK.EmptyStatement || n.kind === SK.SemicolonClassElement) return; out.push(n.kind); if (ts.isIdentifier(n) || ts.isPrivateIdentifier(n) || ts.isLiteralExpression(n)) out.push(JSON.stringify(n.text)); ts.forEachChild(n, visit); out.push(-1); })(sf); return { ok: sf.parseDiagnostics.length === 0, ast: out.join(",") };}
function positional(src, out) { if (out.length !== src.length) return `length ${out.length} != input ${src.length}`; for (let i = 0; i < src.length; i++) { const a = src[i], b = out[i]; if (a === b) continue; if (a === "\n" || a === "\r") return `line terminator changed at ${i}`; if (!ALLOWED.has(b)) return `invented ${JSON.stringify(b)} at ${i}`; } return null;}
function* parts(src, n) { for (let i = 0; i < src.length; i += n) yield src.slice(i, i + n);}function chunkedFixed(src, n) { let out = ""; for (const piece of transformSync(parts(src, n), { onError() {} })) out += piece; return out;}
const CHUNK_SIZES = [1, 2, 3, 7, 61, 509];
function check(src) { let sf; try { sf = ts.createSourceFile("input.ts", src, ts.ScriptTarget.ESNext, false, ts.ScriptKind.TS); } catch (e) { return null; // TS itself rejects it; not a valid fuzz input. } if (sf.parseDiagnostics.length) return null; const wantErrors = []; let want; try { want = blankSourceFile(sf, (n) => wantErrors.push(n.getStart(sf))); } catch { return null; // the oracle itself can't handle it; skip. } const gotErrors = []; let got; try { got = transformSync(src, { onError: (e) => gotErrors.push(e.start) }); } catch (e) { return `threw whole: ${e.message}`; } const posProblem = positional(src, got); if (posProblem) return posProblem; const w = jsAst(want), g = jsAst(got); if (w.ok && !g.ok) return "JS parse failed where ts-blank-space's output parses"; if (w.ast !== g.ast) return "JS AST differs from ts-blank-space's output"; if (wantErrors.join() !== gotErrors.join()) return `errors want [${wantErrors}] got [${gotErrors}]`; for (const n of CHUNK_SIZES) { let streamed; try { streamed = chunkedFixed(src, n); } catch (e) { return `threw chunked(${n}): ${e.message}`; } if (streamed !== got) return `streamed output differs from whole-input output at chunk ${n}`; } return null;}
// ---------------------------------------------------------------------------// Minimization: ddmin over lines, then over token-ish runs, keeping the same failure.
function failsSame(src, wantKind) { const problem = check(src); if (!problem) return false; if (!wantKind) return true; return problem.split(" ").slice(0, 2).join(" ") === wantKind;}
function shrink(parts, join, wantKind) { let n = 2; while (parts.length >= 2) { const size = Math.ceil(parts.length / n); let reduced = false; for (let i = 0; i < parts.length; i += size) { const cand = parts.slice(0, i).concat(parts.slice(i + size)); if (cand.length && failsSame(cand.join(join), wantKind)) { parts = cand; n = Math.max(n - 1, 2); reduced = true; break; } } if (!reduced) { if (n >= parts.length) break; n = Math.min(n * 2, parts.length); } } return parts;}
function minimize(src) { const problem = check(src); if (!problem) return src; const wantKind = problem.split(" ").slice(0, 2).join(" "); let out = shrink(src.split("\n"), "\n", wantKind).join("\n"); out = shrink(out.match(/\s+|\w+|./gs) ?? [], "", wantKind).join(""); return out;}
// ---------------------------------------------------------------------------// Main loop.
const corpus = [...cases()];if (!corpus.length) { console.error("fuzz-unicode: empty corpus"); process.exit(1);}
const rng = makeRng(SEED);let tested = 0;let failures = 0;const seenKinds = new Set();
for (let i = 0; i < COUNT; i++) { const base = pick(rng, corpus).src; if (!base) continue; const times = 1 + Math.floor(rng() * 3); let mutant; try { mutant = mutate(base, rng, times); } catch { continue; } if (mutant === base) continue; tested++; let problem; try { problem = check(mutant); } catch (e) { problem = `fuzzer crashed: ${e.message}`; } if (!problem) continue; failures++; const kind = problem.split(" ").slice(0, 2).join(" "); if (!SHOW_ALL && seenKinds.has(kind)) continue; seenKinds.add(kind); console.log(`\nFAIL (mutant ${i}, seed ${SEED}): ${problem}`); if (process.env.FUZZ_DEBUG) fs.writeFileSync(`${process.env.FUZZ_DEBUG}/fuzz-raw-${i}.txt`, mutant, "utf8"); let minimized; try { minimized = minimize(mutant); } catch { minimized = mutant; } console.log("minimized:", JSON.stringify(minimized)); if (!SHOW_ALL) break;}
console.log(`\nfuzz-unicode: ${tested} mutants tested, ${failures} failed, seed ${SEED}`);process.exitCode = failures ? 1 : 0;