replacing bundling with externals in playground

DOCS.md

···
 @external
 namespace com.atproto.label.defs {
   model SelfLabels { }
+  @token model SomeToken { }  // use @token for tokens
 }
 ```
 

packages/playground/samples/build.js

···
 // @ts-check
-import { writeFileSync, mkdirSync } from "fs";
+import { writeFileSync, mkdirSync, readFileSync } from "fs";
 import { dirname, resolve, join } from "path";
 import { fileURLToPath } from "url";
+import { deepStrictEqual } from "assert";
 import { lexicons, bundleLexicon } from "./index.js";
+import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const outputDir = resolve(__dirname, "dist");
+const pkgRoot = await findTestPackageRoot(import.meta.url);
+
+// TypeSpec library setup for testing
+const TypelexTestLibrary = {
+  name: "@typelex/emitter",
+  packageRoot: pkgRoot.replace("/playground", "/emitter"),
+  files: [
+    { realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" },
+    { realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" },
+    { realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" },
+  ],
+};
 
 // Create output directory
 mkdirSync(outputDir, { recursive: true });
 
-// Write each bundled lexicon to disk
+// Write each bundled lexicon to disk and verify it compiles correctly
 const samplesList = {};
 
 for (const [namespace, lexicon] of lexicons) {
···
 
   writeFileSync(filepath, bundled);
 
+  const host = await createTestHost({ libraries: [TypelexTestLibrary] });
+  host.addTypeSpecFile("main.tsp", bundled);
+
+  const baseOutputPath = resolveVirtualPath("test-output/");
+  const [, diagnostics] = await host.compileAndDiagnose("main.tsp", {
+    outputDir: baseOutputPath,
+    noEmit: false,
+    emit: ["@typelex/emitter"],
+  });
+
+  if (diagnostics.length > 0) {
+    console.error(`❌ ${namespace}: Compilation errors`);
+    diagnostics.forEach(d => console.error(`  ${d.message}`));
+    process.exit(1);
+  }
+
+  // Get emitted JSON
+  const outputFiles = [...host.fs.entries()]
+    .filter(([name]) => name.startsWith(baseOutputPath))
+    .map(([name, value]) => {
+      let relativePath = name.replace(baseOutputPath, "");
+      if (relativePath.startsWith("@typelex/emitter/")) {
+        relativePath = relativePath.replace("@typelex/emitter/", "");
+      }
+      return [relativePath, value];
+    });
+
+  const expectedJsonPath = namespace.replace(/\./g, "/") + ".json";
+  const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath);
+
+  if (!emittedJson) {
+    console.error(`❌ ${namespace}: No JSON output found (expected ${expectedJsonPath})`);
+    process.exit(1);
+  }
+
+  // Compare with expected JSON
+  const expectedJsonFile = join(
+    pkgRoot.replace("/playground", "/emitter"),
+    "test/integration",
+    lexicon.suite,
+    "output",
+    lexicon.file.replace(".tsp", ".json")
+  );
+
+  const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8"));
+  const actualJson = JSON.parse(emittedJson[1]);
+
+  deepStrictEqual(actualJson, expectedJson);
+
   samplesList[namespace] = {
     filename: `samples/dist/${filename}`,
     preferredEmitter: "@typelex/emitter",
···
 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`;
 writeFileSync(join(outputDir, "samples.js"), samplesIndex);
 
-console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`);
+console.log(`\n✅ ${lexicons.size} samples verified successfully`);

packages/playground/samples/index.js

···
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-// Get all tsp files
-function getAllTspFiles(dir, baseDir = dir) {
+// Get all tsp and json files
+function getAllFiles(dir, baseDir = dir) {
   const files = [];
   const entries = readdirSync(dir);
 
···
     const stat = statSync(fullPath);
 
     if (stat.isDirectory()) {
-      files.push(...getAllTspFiles(fullPath, baseDir));
-    } else if (entry.endsWith(".tsp")) {
+      files.push(...getAllFiles(fullPath, baseDir));
+    } else if (entry.endsWith(".tsp") || entry.endsWith(".json")) {
       files.push(relative(baseDir, fullPath));
     }
   }
···
   return files.sort();
 }
 
-// Extract dependencies from a file
-function extractDependencies(content) {
-  const deps = new Set();
-  // Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main"
-  // Pattern: word.word.word... followed by dot and identifier starting with capital letter
-  const pattern =
-    /\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g;
-  const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, "");
+// Extract all refs from JSON (recursively search for strings with #)
+function extractRefsFromJson(obj, refs = new Map()) {
+  if (typeof obj === "string") {
+    // Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref)
+    const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/);
+    if (match) {
+      const ns = match[1];
+      const def = match[2];
+      const modelName = def.charAt(0).toUpperCase() + def.slice(1);
+      if (!refs.has(ns)) {
+        refs.set(ns, new Set());
+      }
+      refs.get(ns).add(modelName);
+    } else {
+      // Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots)
+      const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/);
+      if (nsMatch) {
+        const ns = nsMatch[1];
+        if (!refs.has(ns)) {
+          refs.set(ns, new Set());
+        }
+        refs.get(ns).add("Main");
+      }
+    }
+  } else if (Array.isArray(obj)) {
+    for (const item of obj) {
+      extractRefsFromJson(item, refs);
+    }
+  } else if (obj && typeof obj === "object") {
+    for (const value of Object.values(obj)) {
+      extractRefsFromJson(value, refs);
+    }
+  }
+  return refs;
+}
 
-  const matches = withoutDeclaration.matchAll(pattern);
-  for (const match of matches) {
-    deps.add(match[1]);
-  }
+const integrationDir = join(__dirname, "../../emitter/test/integration");
+
+// Get all test suite directories
+const testSuites = readdirSync(integrationDir).filter((name) => {
+  const fullPath = join(integrationDir, name);
+  return statSync(fullPath).isDirectory() && !name.startsWith(".");
+});
 
-  return Array.from(deps);
-}
+// Build lexicons with refs extracted from JSON
+const lexicons = new Map(); // namespace -> { file, content, refs, suite }
 
-const atprotoInputDir = join(
-  __dirname,
-  "../../emitter/test/integration/atproto/input",
-);
-const lexiconExamplesDir = join(
-  __dirname,
-  "../../emitter/test/integration/lexicon-examples/input",
-);
+// Process all test suites
+for (const suite of testSuites) {
+  const inputDir = join(integrationDir, suite, "input");
+  const outputDir = join(integrationDir, suite, "output");
 
-const atprotoFiles = getAllTspFiles(atprotoInputDir);
-const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir);
+  const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp"));
 
-// Build dependency graph
-const lexicons = new Map(); // namespace -> { file, content, deps }
+  for (const file of inputFiles) {
+    const fullPath = join(inputDir, file);
+    const content = readFileSync(fullPath, "utf-8");
+    const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
 
-// Process atproto files
-for (const file of atprotoFiles) {
-  const fullPath = join(atprotoInputDir, file);
-  const content = readFileSync(fullPath, "utf-8");
-  const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
-  const deps = extractDependencies(content);
+    // Find corresponding JSON output
+    const jsonFile = file.replace(/\.tsp$/, ".json");
+    const jsonPath = join(outputDir, jsonFile);
+    const jsonContent = readFileSync(jsonPath, "utf-8");
+    const jsonData = JSON.parse(jsonContent);
+    const refs = extractRefsFromJson(jsonData);
 
-  lexicons.set(namespace, { file: `atproto/${file}`, content, deps });
+    lexicons.set(namespace, { file, content, refs, suite });
+  }
 }
 
-// Process lexicon-examples files
-for (const file of lexiconExampleFiles) {
-  const fullPath = join(lexiconExamplesDir, file);
-  const content = readFileSync(fullPath, "utf-8");
-  const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
-  const deps = extractDependencies(content);
+// TypeSpec reserved keywords that need escaping
+const TYPESPEC_KEYWORDS = new Set([
+  "record",
+  "pub",
+  "interface",
+  "model",
+  "namespace",
+  "op",
+  "import",
+  "export",
+  "using",
+  "alias",
+  "enum",
+  "union",
+  "scalar",
+  "extends",
+]);
 
-  lexicons.set(namespace, { file: `examples/${file}`, content, deps });
+// Escape a namespace part if it's a reserved keyword
+function escapeNamespacePart(part) {
+  return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part;
 }
 
-// Recursively collect all dependencies (topological sort)
-function collectDependencies(
-  namespace,
-  collected = new Set(),
-  visiting = new Set(),
-) {
-  if (collected.has(namespace)) return;
-  if (visiting.has(namespace)) return; // circular dependency
+// Escape a full namespace path
+function escapeNamespace(namespace) {
+  return namespace.split(".").map(escapeNamespacePart).join(".");
+}
 
+// Get the JSON for a lexicon to check its definitions
+function getLexiconJson(namespace) {
   const lexicon = lexicons.get(namespace);
-  if (!lexicon) return;
+  if (!lexicon) return null;
 
-  visiting.add(namespace);
+  const jsonPath = join(
+    integrationDir,
+    lexicon.suite,
+    "output",
+    lexicon.file.replace(".tsp", ".json"),
+  );
 
-  // First collect all dependencies
-  for (const dep of lexicon.deps) {
-    collectDependencies(dep, collected, visiting);
+  try {
+    return JSON.parse(readFileSync(jsonPath, "utf-8"));
+  } catch {
+    return null;
   }
+}
 
-  visiting.delete(namespace);
-  collected.add(namespace);
+// Check if a definition in JSON is a token
+function isToken(lexiconJson, defName) {
+  if (!lexiconJson || !lexiconJson.defs) return false;
+  const def = lexiconJson.defs[defName];
+  return def && def.type === "token";
 }
 
-// Bundle a lexicon with all its dependencies
+// Bundle a lexicon with stubs for referenced types (from JSON)
 function bundleLexicon(namespace) {
-  const collected = new Set();
-  collectDependencies(namespace, collected);
+  const mainLexicon = lexicons.get(namespace);
+  if (!mainLexicon) return "";
 
-  // Put the main lexicon FIRST, then its dependencies
-  const mainLexicon = lexicons.get(namespace);
-  const deps = Array.from(collected).filter((ns) => ns !== namespace);
+  let bundled = mainLexicon.content;
 
-  let bundled = 'import "@typelex/emitter";\n\n';
+  // Add stubs from refs extracted from JSON output (excluding self-references)
+  if (mainLexicon.refs.size > 0) {
+    let hasExternalRefs = false;
+    for (const [ns] of mainLexicon.refs) {
+      if (ns !== namespace) {
+        hasExternalRefs = true;
+        break;
+      }
+    }
 
-  // Main lexicon first (so it shows in the playground)
-  if (mainLexicon) {
-    const contentWithoutImport = mainLexicon.content.replace(
-      /^import "@typelex\/emitter";\s*\n/,
-      "",
-    );
-    bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`;
-  }
+    if (hasExternalRefs) {
+      bundled += "\n// --- Externals ---\n";
+    }
 
-  // Then dependencies
-  for (const ns of deps) {
-    const lexicon = lexicons.get(ns);
-    if (!lexicon) continue;
+    for (const [ns, models] of mainLexicon.refs) {
+      // Skip if this is the current namespace
+      if (ns === namespace) continue;
 
-    const contentWithoutImport = lexicon.content.replace(
-      /^import "@typelex\/emitter";\s*\n/,
-      "",
-    );
-    bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`;
+      // Get the JSON for this referenced namespace to check for tokens
+      const refJson = getLexiconJson(ns);
+
+      const escapedNs = escapeNamespace(ns);
+      bundled += `\n@external\nnamespace ${escapedNs} {\n`;
+      for (const model of models) {
+        // Check if this definition exists in the JSON and is a token
+        const defName = model.charAt(0).toLowerCase() + model.slice(1);
+        if (refJson && isToken(refJson, defName)) {
+          bundled += `  @token model ${model} { }\n`;
+        } else {
+          bundled += `  model ${model} { }\n`;
+        }
+      }
+      bundled += `}\n`;
+    }
   }
 
   return bundled;