[New feature] Externals #2

+15

packages/emitter/lib/decorators.tsp

··· 1 import "../dist/tsp-index.js"; 2 0 0 3 /** 4 * Specifies the maximum number of graphemes (user-perceived characters) allowed. 5 * Used alongside maxLength for proper Unicode text handling. ··· 159 * ``` 160 */ 161 extern dec errors(target: unknown, ...errors: unknown[]); 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 import "../dist/tsp-index.js"; 2 3 + using TypeSpec.Reflection; 4 + 5 /** 6 * Specifies the maximum number of graphemes (user-perceived characters) allowed. 7 * Used alongside maxLength for proper Unicode text handling. ··· 161 * ``` 162 */ 163 extern dec errors(target: unknown, ...errors: unknown[]); 164 + 165 + /** 166 + * Marks a namespace as external, preventing it from emitting JSON output. 167 + * This decorator can only be applied to namespaces. 168 + * Useful for importing definitions from other lexicons without re-emitting them. 169 + * 170 + * @example 171 + * ```typespec 172 + * @external 173 + * namespace com.atproto.repo.defs; 174 + * ``` 175 + */ 176 + extern dec external(target: Namespace);

+2

packages/emitter/src/tsp-index.ts

··· 14 $inline, 15 $maxBytes, 16 $minBytes, 0 17 } from "./decorators.js"; 18 19 /** @internal */ ··· 34 inline: $inline, 35 maxBytes: $maxBytes, 36 minBytes: $minBytes, 0 37 }, 38 };

··· 14 $inline, 15 $maxBytes, 16 $minBytes, 17 + $external, 18 } from "./decorators.js"; 19 20 /** @internal */ ··· 35 inline: $inline, 36 maxBytes: $maxBytes, 37 minBytes: $minBytes, 38 + external: $external, 39 }, 40 };

+23 -1

packages/emitter/test/spec.test.ts

··· 106 assert.deepStrictEqual(actual, expected); 107 }); 108 } else { 109 - it.skip(`TODO: ${expectedPath} (add ${inputPath})`, function () {}); 0 0 0 0 0 110 } 111 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 112 }); 113 } 114 });

··· 106 assert.deepStrictEqual(actual, expected); 107 }); 108 } else { 109 + it(`should emit ${expectedPath}`, function () { 110 + assert.fail( 111 + `Expected output file ${expectedPath} has no corresponding input file ${inputPath}. ` + 112 + `Either add the input file or remove the expected output.` 113 + ); 114 + }); 115 } 116 } 117 + 118 + // Check for unexpected emitted files 119 + it("should not emit unexpected files", function () { 120 + const emittedFiles = Object.keys(emitResult.files).filter(f => f.endsWith(".json")); 121 + const expectedPaths = Object.keys(expectedFiles) 122 + .filter(f => f.endsWith(".json")) 123 + .map(normalizePathToPosix); 124 + 125 + const unexpected = emittedFiles.filter(f => !expectedPaths.includes(f)); 126 + 127 + if (unexpected.length > 0) { 128 + assert.fail( 129 + `Unexpected files were emitted: ${unexpected.join(", ")}. ` + 130 + `Either add expected output files or ensure these should not be emitted.` 131 + ); 132 + } 133 + }); 134 }); 135 } 136 });

+21

packages/emitter/test/spec/basic/output/com/example/other.json

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + { 2 + "lexicon": 1, 3 + "id": "com.example.other", 4 + "defs": { 5 + "main": { 6 + "type": "object", 7 + "properties": {} 8 + }, 9 + "someDef": { 10 + "type": "object", 11 + "required": [ 12 + "value" 13 + ], 14 + "properties": { 15 + "value": { 16 + "type": "string" 17 + } 18 + } 19 + } 20 + } 21 + }

+7

packages/emitter/test/spec/external/input/test/normal.tsp

··· 0 0 0 0 0 0 0

··· 1 + import "@typelex/emitter"; 2 + 3 + namespace test.normal { 4 + model Main { 5 + name?: string; 6 + } 7 + }

+14

packages/emitter/test/spec/external/output/test/normal.json

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + { 2 + "lexicon": 1, 3 + "id": "test.normal", 4 + "defs": { 5 + "main": { 6 + "type": "object", 7 + "properties": { 8 + "name": { 9 + "type": "string" 10 + } 11 + } 12 + } 13 + } 14 + }

+66 -3

packages/playground/samples/build.js

··· 1 // @ts-check 2 - import { writeFileSync, mkdirSync } from "fs"; 3 import { dirname, resolve, join } from "path"; 4 import { fileURLToPath } from "url"; 0 5 import { lexicons, bundleLexicon } from "./index.js"; 0 6 7 const __dirname = dirname(fileURLToPath(import.meta.url)); 8 const outputDir = resolve(__dirname, "dist"); 0 0 0 0 0 0 0 0 0 0 0 0 9 10 // Create output directory 11 mkdirSync(outputDir, { recursive: true }); 12 13 - // Write each bundled lexicon to disk 14 const samplesList = {}; 15 16 for (const [namespace, lexicon] of lexicons) { ··· 20 21 writeFileSync(filepath, bundled); 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 samplesList[namespace] = { 24 filename: `samples/dist/${filename}`, 25 preferredEmitter: "@typelex/emitter", ··· 30 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`; 31 writeFileSync(join(outputDir, "samples.js"), samplesIndex); 32 33 - console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`);

··· 1 // @ts-check 2 + import { writeFileSync, mkdirSync, readFileSync } from "fs"; 3 import { dirname, resolve, join } from "path"; 4 import { fileURLToPath } from "url"; 5 + import { deepStrictEqual } from "assert"; 6 import { lexicons, bundleLexicon } from "./index.js"; 7 + import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing"; 8 9 const __dirname = dirname(fileURLToPath(import.meta.url)); 10 const outputDir = resolve(__dirname, "dist"); 11 + const pkgRoot = await findTestPackageRoot(import.meta.url); 12 + 13 + // TypeSpec library setup for testing 14 + const TypelexTestLibrary = { 15 + name: "@typelex/emitter", 16 + packageRoot: pkgRoot.replace("/playground", "/emitter"), 17 + files: [ 18 + { realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" }, 19 + { realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" }, 20 + { realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" }, 21 + ], 22 + }; 23 24 // Create output directory 25 mkdirSync(outputDir, { recursive: true }); 26 27 + // Write each bundled lexicon to disk and verify it compiles correctly 28 const samplesList = {}; 29 30 for (const [namespace, lexicon] of lexicons) { ··· 34 35 writeFileSync(filepath, bundled); 36 37 + const host = await createTestHost({ libraries: [TypelexTestLibrary] }); 38 + host.addTypeSpecFile("main.tsp", bundled); 39 + 40 + const baseOutputPath = resolveVirtualPath("test-output/"); 41 + const [, diagnostics] = await host.compileAndDiagnose("main.tsp", { 42 + outputDir: baseOutputPath, 43 + noEmit: false, 44 + emit: ["@typelex/emitter"], 45 + }); 46 + 47 + if (diagnostics.length > 0) { 48 + console.error(`❌ ${namespace}: Compilation errors`); 49 + diagnostics.forEach(d => console.error(` ${d.message}`)); 50 + process.exit(1); 51 + } 52 + 53 + // Get emitted JSON 54 + const outputFiles = [...host.fs.entries()] 55 + .filter(([name]) => name.startsWith(baseOutputPath)) 56 + .map(([name, value]) => { 57 + let relativePath = name.replace(baseOutputPath, ""); 58 + if (relativePath.startsWith("@typelex/emitter/")) { 59 + relativePath = relativePath.replace("@typelex/emitter/", ""); 60 + } 61 + return [relativePath, value]; 62 + }); 63 + 64 + const expectedJsonPath = namespace.replace(/\./g, "/") + ".json"; 65 + const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath); 66 + 67 + if (!emittedJson) { 68 + console.error(`❌ ${namespace}: No JSON output found (expected ${expectedJsonPath})`); 69 + process.exit(1); 70 + } 71 + 72 + // Compare with expected JSON 73 + const expectedJsonFile = join( 74 + pkgRoot.replace("/playground", "/emitter"), 75 + "test/integration", 76 + lexicon.suite, 77 + "output", 78 + lexicon.file.replace(".tsp", ".json") 79 + ); 80 + 81 + const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8")); 82 + const actualJson = JSON.parse(emittedJson[1]); 83 + 84 + deepStrictEqual(actualJson, expectedJson); 85 + 86 samplesList[namespace] = { 87 filename: `samples/dist/${filename}`, 88 preferredEmitter: "@typelex/emitter", ··· 93 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`; 94 writeFileSync(join(outputDir, "samples.js"), samplesIndex); 95 96 + console.log(`\n✅ ${lexicons.size} samples verified successfully`);

+151 -91

packages/playground/samples/index.js

··· 5 6 const __dirname = dirname(fileURLToPath(import.meta.url)); 7 8 - // Get all tsp files 9 - function getAllTspFiles(dir, baseDir = dir) { 10 const files = []; 11 const entries = readdirSync(dir); 12 ··· 15 const stat = statSync(fullPath); 16 17 if (stat.isDirectory()) { 18 - files.push(...getAllTspFiles(fullPath, baseDir)); 19 - } else if (entry.endsWith(".tsp")) { 20 files.push(relative(baseDir, fullPath)); 21 } 22 } ··· 24 return files.sort(); 25 } 26 27 - // Extract dependencies from a file 28 - function extractDependencies(content) { 29 - const deps = new Set(); 30 - // Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main" 31 - // Pattern: word.word.word... followed by dot and identifier starting with capital letter 32 - const pattern = 33 - /\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g; 34 - const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, ""); 35 - 36 - const matches = withoutDeclaration.matchAll(pattern); 37 - for (const match of matches) { 38 - deps.add(match[1]); 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 } 40 - 41 - return Array.from(deps); 42 } 43 44 - const atprotoInputDir = join( 45 - __dirname, 46 - "../../emitter/test/integration/atproto/input", 47 - ); 48 - const lexiconExamplesDir = join( 49 - __dirname, 50 - "../../emitter/test/integration/lexicon-examples/input", 51 - ); 52 - 53 - const atprotoFiles = getAllTspFiles(atprotoInputDir); 54 - const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir); 55 - 56 - // Build dependency graph 57 - const lexicons = new Map(); // namespace -> { file, content, deps } 58 - 59 - // Process atproto files 60 - for (const file of atprotoFiles) { 61 - const fullPath = join(atprotoInputDir, file); 62 - const content = readFileSync(fullPath, "utf-8"); 63 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 64 - const deps = extractDependencies(content); 65 - 66 - lexicons.set(namespace, { file: `atproto/${file}`, content, deps }); 67 - } 68 69 - // Process lexicon-examples files 70 - for (const file of lexiconExampleFiles) { 71 - const fullPath = join(lexiconExamplesDir, file); 72 - const content = readFileSync(fullPath, "utf-8"); 73 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 74 - const deps = extractDependencies(content); 75 76 - lexicons.set(namespace, { file: `examples/${file}`, content, deps }); 77 - } 78 79 - // Recursively collect all dependencies (topological sort) 80 - function collectDependencies( 81 - namespace, 82 - collected = new Set(), 83 - visiting = new Set(), 84 - ) { 85 - if (collected.has(namespace)) return; 86 - if (visiting.has(namespace)) return; // circular dependency 87 88 - const lexicon = lexicons.get(namespace); 89 - if (!lexicon) return; 0 0 0 0 90 91 - visiting.add(namespace); 0 0 0 0 0 92 93 - // First collect all dependencies 94 - for (const dep of lexicon.deps) { 95 - collectDependencies(dep, collected, visiting); 96 } 0 97 98 - visiting.delete(namespace); 99 - collected.add(namespace); 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 100 } 101 102 - // Bundle a lexicon with all its dependencies 103 - function bundleLexicon(namespace) { 104 - const collected = new Set(); 105 - collectDependencies(namespace, collected); 106 107 - // Put the main lexicon FIRST, then its dependencies 108 - const mainLexicon = lexicons.get(namespace); 109 - const deps = Array.from(collected).filter((ns) => ns !== namespace); 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 110 111 - let bundled = 'import "@typelex/emitter";\n\n'; 0 0 0 0 0 112 113 - // Main lexicon first (so it shows in the playground) 114 - if (mainLexicon) { 115 - const contentWithoutImport = mainLexicon.content.replace( 116 - /^import "@typelex\/emitter";\s*\n/, 117 - "", 118 - ); 119 - bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`; 120 - } 0 0 0 0 0 0 0 0 121 122 - // Then dependencies 123 - for (const ns of deps) { 124 - const lexicon = lexicons.get(ns); 125 - if (!lexicon) continue; 126 127 - const contentWithoutImport = lexicon.content.replace( 128 - /^import "@typelex\/emitter";\s*\n/, 129 - "", 130 - ); 131 - bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`; 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 132 } 133 134 return bundled;

··· 5 6 const __dirname = dirname(fileURLToPath(import.meta.url)); 7 8 + // Get all tsp and json files 9 + function getAllFiles(dir, baseDir = dir) { 10 const files = []; 11 const entries = readdirSync(dir); 12 ··· 15 const stat = statSync(fullPath); 16 17 if (stat.isDirectory()) { 18 + files.push(...getAllFiles(fullPath, baseDir)); 19 + } else if (entry.endsWith(".tsp") || entry.endsWith(".json")) { 20 files.push(relative(baseDir, fullPath)); 21 } 22 } ··· 24 return files.sort(); 25 } 26 27 + // Extract all refs from JSON (recursively search for strings with #) 28 + function extractRefsFromJson(obj, refs = new Map()) { 29 + if (typeof obj === "string") { 30 + // Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref) 31 + const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/); 32 + if (match) { 33 + const ns = match[1]; 34 + const def = match[2]; 35 + const modelName = def.charAt(0).toUpperCase() + def.slice(1); 36 + if (!refs.has(ns)) { 37 + refs.set(ns, new Set()); 38 + } 39 + refs.get(ns).add(modelName); 40 + } else { 41 + // Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots) 42 + const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/); 43 + if (nsMatch) { 44 + const ns = nsMatch[1]; 45 + if (!refs.has(ns)) { 46 + refs.set(ns, new Set()); 47 + } 48 + refs.get(ns).add("Main"); 49 + } 50 + } 51 + } else if (Array.isArray(obj)) { 52 + for (const item of obj) { 53 + extractRefsFromJson(item, refs); 54 + } 55 + } else if (obj && typeof obj === "object") { 56 + for (const value of Object.values(obj)) { 57 + extractRefsFromJson(value, refs); 58 + } 59 } 60 + return refs; 0 61 } 62 63 + const integrationDir = join(__dirname, "../../emitter/test/integration"); 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 64 65 + // Get all test suite directories 66 + const testSuites = readdirSync(integrationDir).filter((name) => { 67 + const fullPath = join(integrationDir, name); 68 + return statSync(fullPath).isDirectory() && !name.startsWith("."); 69 + }); 0 70 71 + // Build lexicons with refs extracted from JSON 72 + const lexicons = new Map(); // namespace -> { file, content, refs, suite } 73 74 + // Process all test suites 75 + for (const suite of testSuites) { 76 + const inputDir = join(integrationDir, suite, "input"); 77 + const outputDir = join(integrationDir, suite, "output"); 0 0 0 0 78 79 + const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp")); 80 + 81 + for (const file of inputFiles) { 82 + const fullPath = join(inputDir, file); 83 + const content = readFileSync(fullPath, "utf-8"); 84 + const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 85 86 + // Find corresponding JSON output 87 + const jsonFile = file.replace(/\.tsp$/, ".json"); 88 + const jsonPath = join(outputDir, jsonFile); 89 + const jsonContent = readFileSync(jsonPath, "utf-8"); 90 + const jsonData = JSON.parse(jsonContent); 91 + const refs = extractRefsFromJson(jsonData); 92 93 + lexicons.set(namespace, { file, content, refs, suite }); 0 0 94 } 95 + } 96 97 + // TypeSpec reserved keywords that need escaping 98 + const TYPESPEC_KEYWORDS = new Set([ 99 + "record", 100 + "pub", 101 + "interface", 102 + "model", 103 + "namespace", 104 + "op", 105 + "import", 106 + "export", 107 + "using", 108 + "alias", 109 + "enum", 110 + "union", 111 + "scalar", 112 + "extends", 113 + ]); 114 + 115 + // Escape a namespace part if it's a reserved keyword 116 + function escapeNamespacePart(part) { 117 + return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part; 118 } 119 120 + // Escape a full namespace path 121 + function escapeNamespace(namespace) { 122 + return namespace.split(".").map(escapeNamespacePart).join("."); 123 + } 124 125 + // Get the JSON for a lexicon to check its definitions 126 + function getLexiconJson(namespace) { 127 + const lexicon = lexicons.get(namespace); 128 + if (!lexicon) return null; 129 + 130 + const jsonPath = join( 131 + integrationDir, 132 + lexicon.suite, 133 + "output", 134 + lexicon.file.replace(".tsp", ".json"), 135 + ); 136 + 137 + try { 138 + return JSON.parse(readFileSync(jsonPath, "utf-8")); 139 + } catch { 140 + return null; 141 + } 142 + } 143 144 + // Check if a definition in JSON is a token 145 + function isToken(lexiconJson, defName) { 146 + if (!lexiconJson || !lexiconJson.defs) return false; 147 + const def = lexiconJson.defs[defName]; 148 + return def && def.type === "token"; 149 + } 150 151 + // Bundle a lexicon with stubs for referenced types (from JSON) 152 + function bundleLexicon(namespace) { 153 + const mainLexicon = lexicons.get(namespace); 154 + if (!mainLexicon) return ""; 155 + 156 + let bundled = mainLexicon.content; 157 + 158 + // Add stubs from refs extracted from JSON output (excluding self-references) 159 + if (mainLexicon.refs.size > 0) { 160 + let hasExternalRefs = false; 161 + for (const [ns] of mainLexicon.refs) { 162 + if (ns !== namespace) { 163 + hasExternalRefs = true; 164 + break; 165 + } 166 + } 167 168 + if (hasExternalRefs) { 169 + bundled += "\n// --- Externals ---\n"; 170 + } 0 171 172 + for (const [ns, models] of mainLexicon.refs) { 173 + // Skip if this is the current namespace 174 + if (ns === namespace) continue; 175 + 176 + // Get the JSON for this referenced namespace to check for tokens 177 + const refJson = getLexiconJson(ns); 178 + 179 + const escapedNs = escapeNamespace(ns); 180 + bundled += `\n@external\nnamespace ${escapedNs} {\n`; 181 + for (const model of models) { 182 + // Check if this definition exists in the JSON and is a token 183 + const defName = model.charAt(0).toLowerCase() + model.slice(1); 184 + if (refJson && isToken(refJson, defName)) { 185 + bundled += ` @token model ${model} { }\n`; 186 + } else { 187 + bundled += ` model ${model} { }\n`; 188 + } 189 + } 190 + bundled += `}\n`; 191 + } 192 } 193 194 return bundled;

+11

packages/emitter/src/emitter.ts

··· 124 125 // Skip external namespaces - they don't emit JSON files 126 if (isExternal(this.program, ns)) { 0 0 0 0 0 0 0 0 0 0 0 127 return; 128 } 129

··· 124 125 // Skip external namespaces - they don't emit JSON files 126 if (isExternal(this.program, ns)) { 127 + // Validate that all models in external namespaces are empty (stub-only) 128 + for (const [_, model] of ns.models) { 129 + if (model.properties && model.properties.size > 0) { 130 + this.program.reportDiagnostic({ 131 + code: "external-model-not-empty", 132 + severity: "error", 133 + message: `Models in @external namespaces must be empty stubs. Model '${model.name}' in namespace '${fullName}' has properties.`, 134 + target: model, 135 + }); 136 + } 137 + } 138 return; 139 } 140