// Fuzzer for non-ASCII correctness in engine/. Injects non-ASCII at many lexical sites (comments, string and // template contents, identifiers, JSDoc) into corpus inputs and checks the same contract test/erase.mjs // checks: positional UTF-16 preservation, JS AST equality with ts-blank-space, error-position equality, and // streaming equivalence at several chunk sizes including tiny ones. Deterministic by seed; prints a minimized // failing input on the first failure (or every failure with --all). // // Usage: node test/fuzz-unicode.mjs [count] [seed] // TYPESTRIP_ERASE_WASM=build/erase.wasm node test/fuzz-unicode.mjs 50000 1 import fs from "node:fs"; import path from "node:path"; import ts from "typescript"; import { blankSourceFile } from "../vendor/ts-blank-space/out/index.js"; import { transformSync } from "../src/index.js"; import { cases } from "./corpus.mjs"; const root = path.resolve(import.meta.dirname, ".."); const SK = ts.SyntaxKind; const ALLOWED = new Set([" ", ";", "(", ")"]); const COUNT = Number(process.argv[2] ?? 50000); const SEED = Number(process.argv[3] ?? 1); const SHOW_ALL = process.argv.includes("--all"); // --------------------------------------------------------------------------- // Deterministic PRNG (xorshift32), seeded. function makeRng(seed) { let s = seed >>> 0 || 1; return function rng() { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s >>>= 0; s ^= s << 5; s >>>= 0; return s / 0x100000000; }; } function pick(rng, arr) { return arr[Math.floor(rng() * arr.length)]; } // --------------------------------------------------------------------------- // Character pools. Codepoints as strings, grouped by UTF-8 width and kind. const TWO_BYTE = ["é", "ñ", "σ", "©", "Д"]; // é ñ σ © Д const THREE_BYTE = ["中", "文", "あ", "☃", "❤"]; // 中 文 あ ☃ ❤ const FOUR_BYTE = ["\u{1f600}", "\u{1f984}", "\u{10000}", "\u{1d4b3}", "\u{1f1fa}\u{1f1f8}"]; // emoji, unicorn, 𐀀, 𝒳, flag pair const ZWJ_SEQS = ["\u{1f468}‍\u{1f469}‍\u{1f467}", "\u{1f44d}‍", "a‍b", "a‌b"]; // family, thumbs-up+ZWJ, ZWJ/ZWNJ in plain text const LINE_SEP = ["
", "
"]; // LS, PS const NBSP = " "; const FEFF = ""; // Unicode letters valid as IdentifierStart/Part (avoid digits-only or combining marks that need a base). const ID_CHARS = ["é", "σ", "中", "Д", "\u{1d4b3}"]; function loneSurrogateHigh() { return "\ud83d"; } // high surrogate alone (no low pair follows) function loneSurrogateLow() { return "\udc00"; } // low surrogate alone (no high pair precedes) // --------------------------------------------------------------------------- // Site discovery via the TypeScript scanner (skipTrivia=false to see comments individually). function findSites(src) { const sites = { commentLine: [], commentBlock: [], jsdoc: [], string: [], template: [], identifier: [], trivia: [] }; const scanner = ts.createScanner(ts.ScriptTarget.ESNext, false, ts.LanguageVariant.Standard, src); scanner.setOnError(() => {}); let prevEnd = 0; while (true) { const kind = scanner.scan(); if (kind === SK.EndOfFileToken) break; const start = scanner.getTokenPos(); const end = scanner.getTextPos(); if (kind === SK.SingleLineCommentTrivia) sites.commentLine.push([start, end]); else if (kind === SK.MultiLineCommentTrivia) { if (src.slice(start, start + 3) === "/**") sites.jsdoc.push([start, end]); else sites.commentBlock.push([start, end]); } else if (kind === SK.WhitespaceTrivia || kind === SK.NewLineTrivia) sites.trivia.push([start, end]); else if ( kind === SK.StringLiteral || kind === SK.NoSubstitutionTemplateLiteral || kind === SK.TemplateHead || kind === SK.TemplateMiddle || kind === SK.TemplateTail ) { // Exclude the delimiter bytes (quotes, backtick, ${, }). const text = src.slice(start, end); let a = start + 1, b = end - 1; if (kind === SK.TemplateHead) b = end - 2; // ends in ${ if (kind === SK.TemplateMiddle) { a = start + 1; b = end - 2; } if (kind === SK.TemplateTail) a = start + 1; if (b > a) sites.string.push([a, b]); if (kind !== SK.StringLiteral) sites.template.push([a, b]); } else if (kind === SK.Identifier) { sites.identifier.push([start, end]); } prevEnd = end; } return sites; } // --------------------------------------------------------------------------- // Mutators: given src and a site range, return mutated src or null if not applicable. function insertAt(src, pos, text) { return src.slice(0, pos) + text + src.slice(pos); } function midpoint(rng, a, b) { if (b <= a) return a; return a + 1 + Math.floor(rng() * (b - a - 1 >= 0 ? b - a : 1)); } const MUTATORS = [ // comment-line: no raw newline, no U+2028/U+2029 (both end a single-line comment). { site: "commentLine", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, NBSP, FEFF, ...ZWJ_SEQS]) }, // comment-block / jsdoc: anything except the literal "*/" sequence. { site: "commentBlock", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, ...LINE_SEP, NBSP, FEFF, ...ZWJ_SEQS, loneSurrogateHigh(), loneSurrogateLow()]) }, { site: "jsdoc", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, ...LINE_SEP, NBSP, FEFF, ...ZWJ_SEQS]) }, // string contents: avoid the literal's own quote char, which the caller enforces by range choice. { site: "string", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, NBSP, FEFF, ...ZWJ_SEQS, loneSurrogateHigh(), loneSurrogateLow()]) }, // template contents: same, plus line separators are legal inside templates (they're real newlines there). { site: "template", gen: (rng) => pick(rng, [...TWO_BYTE, ...THREE_BYTE, ...FOUR_BYTE, ...LINE_SEP, NBSP, FEFF, ...ZWJ_SEQS, loneSurrogateHigh(), loneSurrogateLow()]) }, // identifiers: append a valid ID continuation char (letter, or ZWJ/ZWNJ which the grammar special-cases). { site: "identifier", gen: (rng) => pick(rng, [...ID_CHARS, "‍", "‌"]), append: true }, // trivia: BOM-like U+FEFF or other whitespace mid-file, between tokens. { site: "trivia", gen: (rng) => pick(rng, [FEFF, NBSP, ...LINE_SEP]) }, ]; function mutateOnce(src, rng) { const sites = findSites(src); const available = MUTATORS.filter((m) => sites[m.site].length > 0); if (!available.length) return null; const m = pick(rng, available); const range = pick(rng, sites[m.site]); const text = m.gen(rng); if (m.append) return insertAt(src, range[1], text); const pos = midpoint(rng, range[0], range[1]); return insertAt(src, pos, text); } function mutate(src, rng, times) { let out = src; for (let i = 0; i < times; i++) { const next = mutateOnce(out, rng); if (next === null) break; out = next; } return out; } // --------------------------------------------------------------------------- // The same contract test/erase.mjs checks. function jsAst(code) { const sf = ts.createSourceFile("out.js", code, ts.ScriptTarget.ESNext, false, ts.ScriptKind.JS); const out = []; (function visit(n) { if (n.kind === SK.EmptyStatement || n.kind === SK.SemicolonClassElement) return; out.push(n.kind); if (ts.isIdentifier(n) || ts.isPrivateIdentifier(n) || ts.isLiteralExpression(n)) out.push(JSON.stringify(n.text)); ts.forEachChild(n, visit); out.push(-1); })(sf); return { ok: sf.parseDiagnostics.length === 0, ast: out.join(",") }; } function positional(src, out) { if (out.length !== src.length) return `length ${out.length} != input ${src.length}`; for (let i = 0; i < src.length; i++) { const a = src[i], b = out[i]; if (a === b) continue; if (a === "\n" || a === "\r") return `line terminator changed at ${i}`; if (!ALLOWED.has(b)) return `invented ${JSON.stringify(b)} at ${i}`; } return null; } function* parts(src, n) { for (let i = 0; i < src.length; i += n) yield src.slice(i, i + n); } function chunkedFixed(src, n) { let out = ""; for (const piece of transformSync(parts(src, n), { onError() {} })) out += piece; return out; } const CHUNK_SIZES = [1, 2, 3, 7, 61, 509]; function check(src) { let sf; try { sf = ts.createSourceFile("input.ts", src, ts.ScriptTarget.ESNext, false, ts.ScriptKind.TS); } catch (e) { return null; // TS itself rejects it; not a valid fuzz input. } if (sf.parseDiagnostics.length) return null; const wantErrors = []; let want; try { want = blankSourceFile(sf, (n) => wantErrors.push(n.getStart(sf))); } catch { return null; // the oracle itself can't handle it; skip. } const gotErrors = []; let got; try { got = transformSync(src, { onError: (e) => gotErrors.push(e.start) }); } catch (e) { return `threw whole: ${e.message}`; } const posProblem = positional(src, got); if (posProblem) return posProblem; const w = jsAst(want), g = jsAst(got); if (w.ok && !g.ok) return "JS parse failed where ts-blank-space's output parses"; if (w.ast !== g.ast) return "JS AST differs from ts-blank-space's output"; if (wantErrors.join() !== gotErrors.join()) return `errors want [${wantErrors}] got [${gotErrors}]`; for (const n of CHUNK_SIZES) { let streamed; try { streamed = chunkedFixed(src, n); } catch (e) { return `threw chunked(${n}): ${e.message}`; } if (streamed !== got) return `streamed output differs from whole-input output at chunk ${n}`; } return null; } // --------------------------------------------------------------------------- // Minimization: ddmin over lines, then over token-ish runs, keeping the same failure. function failsSame(src, wantKind) { const problem = check(src); if (!problem) return false; if (!wantKind) return true; return problem.split(" ").slice(0, 2).join(" ") === wantKind; } function shrink(parts, join, wantKind) { let n = 2; while (parts.length >= 2) { const size = Math.ceil(parts.length / n); let reduced = false; for (let i = 0; i < parts.length; i += size) { const cand = parts.slice(0, i).concat(parts.slice(i + size)); if (cand.length && failsSame(cand.join(join), wantKind)) { parts = cand; n = Math.max(n - 1, 2); reduced = true; break; } } if (!reduced) { if (n >= parts.length) break; n = Math.min(n * 2, parts.length); } } return parts; } function minimize(src) { const problem = check(src); if (!problem) return src; const wantKind = problem.split(" ").slice(0, 2).join(" "); let out = shrink(src.split("\n"), "\n", wantKind).join("\n"); out = shrink(out.match(/\s+|\w+|./gs) ?? [], "", wantKind).join(""); return out; } // --------------------------------------------------------------------------- // Main loop. const corpus = [...cases()]; if (!corpus.length) { console.error("fuzz-unicode: empty corpus"); process.exit(1); } const rng = makeRng(SEED); let tested = 0; let failures = 0; const seenKinds = new Set(); for (let i = 0; i < COUNT; i++) { const base = pick(rng, corpus).src; if (!base) continue; const times = 1 + Math.floor(rng() * 3); let mutant; try { mutant = mutate(base, rng, times); } catch { continue; } if (mutant === base) continue; tested++; let problem; try { problem = check(mutant); } catch (e) { problem = `fuzzer crashed: ${e.message}`; } if (!problem) continue; failures++; const kind = problem.split(" ").slice(0, 2).join(" "); if (!SHOW_ALL && seenKinds.has(kind)) continue; seenKinds.add(kind); console.log(`\nFAIL (mutant ${i}, seed ${SEED}): ${problem}`); if (process.env.FUZZ_DEBUG) fs.writeFileSync(`${process.env.FUZZ_DEBUG}/fuzz-raw-${i}.txt`, mutant, "utf8"); let minimized; try { minimized = minimize(mutant); } catch { minimized = mutant; } console.log("minimized:", JSON.stringify(minimized)); if (!SHOW_ALL) break; } console.log(`\nfuzz-unicode: ${tested} mutants tested, ${failures} failed, seed ${SEED}`); process.exitCode = failures ? 1 : 0;