// Build a JSON array of sources for test/bytediff.mjs or bench/throughput.mjs from the // pinned OSS corpus (vendor/oss/). // // Usage: // node scripts/oss-sample.mjs bytediff [project] -> corpus/oss-bytediff-input.json // all parse-clean .ts/.mts/.cts files (whole corpus, or one project). // node scripts/oss-sample.mjs throughput [--mb=20] [--seed=N] // a seeded ~20MB sample of non-.d.ts (runtime) parse-clean files, drawn evenly // across projects so one large repo can't dominate -> corpus/oss-throughput-sample.json import fs from "node:fs"; import path from "node:path"; import { projects, ossRoot, walkFiles, parseClean } from "./oss-corpus.mjs"; const root = path.resolve(import.meta.dirname, ".."); const mode = process.argv[2]; const arg3 = process.argv[3] && !process.argv[3].startsWith("--") ? process.argv[3] : undefined; const mb = Number((process.argv.find((a) => a.startsWith("--mb=")) ?? "--mb=20").slice(5)); const seed0 = Number((process.argv.find((a) => a.startsWith("--seed=")) ?? "--seed=1").slice(7)); function mulberry32(a) { return () => { a |= 0; a = (a + 0x6d2b79f5) | 0; let t = Math.imul(a ^ (a >>> 15), 1 | a); t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; } if (mode === "bytediff") { const ps = arg3 ? [arg3] : projects(); const out = []; for (const name of ps) { const dir = path.join(ossRoot, name); if (!fs.existsSync(dir)) continue; for (const f of walkFiles(dir)) { const c = parseClean(f); if (c) out.push(c.src); } } const dest = path.join(root, "corpus/oss-bytediff-input.json"); fs.writeFileSync(dest, JSON.stringify(out)); console.log(`oss-sample: ${out.length} parse-clean files -> ${dest}`); } else if (mode === "throughput") { const rand = mulberry32(seed0); const ps = projects(); const perProject = []; for (const name of ps) { const dir = path.join(ossRoot, name); if (!fs.existsSync(dir)) continue; const files = [...walkFiles(dir)].filter((f) => !f.endsWith(".d.ts")); // Fisher-Yates shuffle with the seeded RNG so the sample is reproducible. for (let i = files.length - 1; i > 0; i--) { const j = Math.floor(rand() * (i + 1)); [files[i], files[j]] = [files[j], files[i]]; } perProject.push({ name, files }); } const budget = mb * 1e6; let used = 0; const out = []; // Round-robin across projects so the sample isn't dominated by whichever repo sorts first. let idx = 0, remaining = perProject.map((p) => p.files.length).reduce((a, b) => a + b, 0); while (used < budget && remaining > 0) { const p = perProject[idx % perProject.length]; idx++; if (p.files.length === 0) { remaining = perProject.reduce((a, b) => a + b.files.length, 0); if (remaining === 0) break; continue; } const f = p.files.pop(); remaining--; const c = parseClean(f); if (!c) continue; out.push(c.src); used += Buffer.byteLength(c.src); } const dest = path.join(root, "corpus/oss-throughput-sample.json"); fs.writeFileSync(dest, JSON.stringify(out)); console.log(`oss-sample: ${out.length} files, ${(used / 1e6).toFixed(1)}MB -> ${dest}`); } else { console.error("usage: node scripts/oss-sample.mjs [project] [--mb=20] [--seed=1]"); process.exit(1); }