Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081// Build a JSON array of sources for test/bytediff.mjs or bench/throughput.mjs from the// pinned OSS corpus (vendor/oss/<project>).//// Usage:// node scripts/oss-sample.mjs bytediff [project] -> corpus/oss-bytediff-input.json// all parse-clean .ts/.mts/.cts files (whole corpus, or one project).// node scripts/oss-sample.mjs throughput [--mb=20] [--seed=N]// a seeded ~20MB sample of non-.d.ts (runtime) parse-clean files, drawn evenly// across projects so one large repo can't dominate -> corpus/oss-throughput-sample.jsonimport fs from "node:fs";import path from "node:path";import { projects, ossRoot, walkFiles, parseClean } from "./oss-corpus.mjs";
const root = path.resolve(import.meta.dirname, "..");const mode = process.argv[2];const arg3 = process.argv[3] && !process.argv[3].startsWith("--") ? process.argv[3] : undefined;const mb = Number((process.argv.find((a) => a.startsWith("--mb=")) ?? "--mb=20").slice(5));const seed0 = Number((process.argv.find((a) => a.startsWith("--seed=")) ?? "--seed=1").slice(7));
function mulberry32(a) { return () => { a |= 0; a = (a + 0x6d2b79f5) | 0; let t = Math.imul(a ^ (a >>> 15), 1 | a); t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };}
if (mode === "bytediff") { const ps = arg3 ? [arg3] : projects(); const out = []; for (const name of ps) { const dir = path.join(ossRoot, name); if (!fs.existsSync(dir)) continue; for (const f of walkFiles(dir)) { const c = parseClean(f); if (c) out.push(c.src); } } const dest = path.join(root, "corpus/oss-bytediff-input.json"); fs.writeFileSync(dest, JSON.stringify(out)); console.log(`oss-sample: ${out.length} parse-clean files -> ${dest}`);} else if (mode === "throughput") { const rand = mulberry32(seed0); const ps = projects(); const perProject = []; for (const name of ps) { const dir = path.join(ossRoot, name); if (!fs.existsSync(dir)) continue; const files = [...walkFiles(dir)].filter((f) => !f.endsWith(".d.ts")); // Fisher-Yates shuffle with the seeded RNG so the sample is reproducible. for (let i = files.length - 1; i > 0; i--) { const j = Math.floor(rand() * (i + 1)); [files[i], files[j]] = [files[j], files[i]]; } perProject.push({ name, files }); } const budget = mb * 1e6; let used = 0; const out = []; // Round-robin across projects so the sample isn't dominated by whichever repo sorts first. let idx = 0, remaining = perProject.map((p) => p.files.length).reduce((a, b) => a + b, 0); while (used < budget && remaining > 0) { const p = perProject[idx % perProject.length]; idx++; if (p.files.length === 0) { remaining = perProject.reduce((a, b) => a + b.files.length, 0); if (remaining === 0) break; continue; } const f = p.files.pop(); remaining--; const c = parseClean(f); if (!c) continue; out.push(c.src); used += Buffer.byteLength(c.src); } const dest = path.join(root, "corpus/oss-throughput-sample.json"); fs.writeFileSync(dest, JSON.stringify(out)); console.log(`oss-sample: ${out.length} files, ${(used / 1e6).toFixed(1)}MB -> ${dest}`);} else { console.error("usage: node scripts/oss-sample.mjs <bytediff|throughput> [project] [--mb=20] [--seed=1]"); process.exit(1);}