[New feature] Externals #2

+15

packages/emitter/lib/decorators.tsp

··· 1 1 import "../dist/tsp-index.js"; 2 2 3 3 + using TypeSpec.Reflection; 4 4 + 3 5 /** 4 6 * Specifies the maximum number of graphemes (user-perceived characters) allowed. 5 7 * Used alongside maxLength for proper Unicode text handling. ··· 159 161 * ``` 160 162 */ 161 163 extern dec errors(target: unknown, ...errors: unknown[]); 164 164 + 165 165 + /** 166 166 + * Marks a namespace as external, preventing it from emitting JSON output. 167 167 + * This decorator can only be applied to namespaces. 168 168 + * Useful for importing definitions from other lexicons without re-emitting them. 169 169 + * 170 170 + * @example 171 171 + * ```typespec 172 172 + * @external 173 173 + * namespace com.atproto.repo.defs; 174 174 + * ``` 175 175 + */ 176 176 + extern dec external(target: Namespace);

+2

packages/emitter/src/tsp-index.ts

··· 14 14 $inline, 15 15 $maxBytes, 16 16 $minBytes, 17 17 + $external, 17 18 } from "./decorators.js"; 18 19 19 20 /** @internal */ ··· 34 35 inline: $inline, 35 36 maxBytes: $maxBytes, 36 37 minBytes: $minBytes, 38 38 + external: $external, 37 39 }, 38 40 };

+23 -1

packages/emitter/test/spec.test.ts

··· 106 106 assert.deepStrictEqual(actual, expected); 107 107 }); 108 108 } else { 109 109 - it.skip(`TODO: ${expectedPath} (add ${inputPath})`, function () {}); 109 109 + it(`should emit ${expectedPath}`, function () { 110 110 + assert.fail( 111 111 + `Expected output file ${expectedPath} has no corresponding input file ${inputPath}. ` + 112 112 + `Either add the input file or remove the expected output.` 113 113 + ); 114 114 + }); 110 115 } 111 116 } 117 117 + 118 118 + // Check for unexpected emitted files 119 119 + it("should not emit unexpected files", function () { 120 120 + const emittedFiles = Object.keys(emitResult.files).filter(f => f.endsWith(".json")); 121 121 + const expectedPaths = Object.keys(expectedFiles) 122 122 + .filter(f => f.endsWith(".json")) 123 123 + .map(normalizePathToPosix); 124 124 + 125 125 + const unexpected = emittedFiles.filter(f => !expectedPaths.includes(f)); 126 126 + 127 127 + if (unexpected.length > 0) { 128 128 + assert.fail( 129 129 + `Unexpected files were emitted: ${unexpected.join(", ")}. ` + 130 130 + `Either add expected output files or ensure these should not be emitted.` 131 131 + ); 132 132 + } 133 133 + }); 112 134 }); 113 135 } 114 136 });

+21

packages/emitter/test/spec/basic/output/com/example/other.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "com.example.other", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "object", 7 7 + "properties": {} 8 8 + }, 9 9 + "someDef": { 10 10 + "type": "object", 11 11 + "required": [ 12 12 + "value" 13 13 + ], 14 14 + "properties": { 15 15 + "value": { 16 16 + "type": "string" 17 17 + } 18 18 + } 19 19 + } 20 20 + } 21 21 + }

+7

packages/emitter/test/spec/external/input/test/normal.tsp

··· 1 1 + import "@typelex/emitter"; 2 2 + 3 3 + namespace test.normal { 4 4 + model Main { 5 5 + name?: string; 6 6 + } 7 7 + }

+14

packages/emitter/test/spec/external/output/test/normal.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "test.normal", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "object", 7 7 + "properties": { 8 8 + "name": { 9 9 + "type": "string" 10 10 + } 11 11 + } 12 12 + } 13 13 + } 14 14 + }

+66 -3

packages/playground/samples/build.js

··· 1 1 // @ts-check 2 2 - import { writeFileSync, mkdirSync } from "fs"; 2 2 + import { writeFileSync, mkdirSync, readFileSync } from "fs"; 3 3 import { dirname, resolve, join } from "path"; 4 4 import { fileURLToPath } from "url"; 5 5 + import { deepStrictEqual } from "assert"; 5 6 import { lexicons, bundleLexicon } from "./index.js"; 7 7 + import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing"; 6 8 7 9 const __dirname = dirname(fileURLToPath(import.meta.url)); 8 10 const outputDir = resolve(__dirname, "dist"); 11 11 + const pkgRoot = await findTestPackageRoot(import.meta.url); 12 12 + 13 13 + // TypeSpec library setup for testing 14 14 + const TypelexTestLibrary = { 15 15 + name: "@typelex/emitter", 16 16 + packageRoot: pkgRoot.replace("/playground", "/emitter"), 17 17 + files: [ 18 18 + { realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" }, 19 19 + { realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" }, 20 20 + { realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" }, 21 21 + ], 22 22 + }; 9 23 10 24 // Create output directory 11 25 mkdirSync(outputDir, { recursive: true }); 12 26 13 13 - // Write each bundled lexicon to disk 27 27 + // Write each bundled lexicon to disk and verify it compiles correctly 14 28 const samplesList = {}; 15 29 16 30 for (const [namespace, lexicon] of lexicons) { ··· 20 34 21 35 writeFileSync(filepath, bundled); 22 36 37 37 + const host = await createTestHost({ libraries: [TypelexTestLibrary] }); 38 38 + host.addTypeSpecFile("main.tsp", bundled); 39 39 + 40 40 + const baseOutputPath = resolveVirtualPath("test-output/"); 41 41 + const [, diagnostics] = await host.compileAndDiagnose("main.tsp", { 42 42 + outputDir: baseOutputPath, 43 43 + noEmit: false, 44 44 + emit: ["@typelex/emitter"], 45 45 + }); 46 46 + 47 47 + if (diagnostics.length > 0) { 48 48 + console.error(`❌ ${namespace}: Compilation errors`); 49 49 + diagnostics.forEach(d => console.error(` ${d.message}`)); 50 50 + process.exit(1); 51 51 + } 52 52 + 53 53 + // Get emitted JSON 54 54 + const outputFiles = [...host.fs.entries()] 55 55 + .filter(([name]) => name.startsWith(baseOutputPath)) 56 56 + .map(([name, value]) => { 57 57 + let relativePath = name.replace(baseOutputPath, ""); 58 58 + if (relativePath.startsWith("@typelex/emitter/")) { 59 59 + relativePath = relativePath.replace("@typelex/emitter/", ""); 60 60 + } 61 61 + return [relativePath, value]; 62 62 + }); 63 63 + 64 64 + const expectedJsonPath = namespace.replace(/\./g, "/") + ".json"; 65 65 + const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath); 66 66 + 67 67 + if (!emittedJson) { 68 68 + console.error(`❌ ${namespace}: No JSON output found (expected ${expectedJsonPath})`); 69 69 + process.exit(1); 70 70 + } 71 71 + 72 72 + // Compare with expected JSON 73 73 + const expectedJsonFile = join( 74 74 + pkgRoot.replace("/playground", "/emitter"), 75 75 + "test/integration", 76 76 + lexicon.suite, 77 77 + "output", 78 78 + lexicon.file.replace(".tsp", ".json") 79 79 + ); 80 80 + 81 81 + const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8")); 82 82 + const actualJson = JSON.parse(emittedJson[1]); 83 83 + 84 84 + deepStrictEqual(actualJson, expectedJson); 85 85 + 23 86 samplesList[namespace] = { 24 87 filename: `samples/dist/${filename}`, 25 88 preferredEmitter: "@typelex/emitter", ··· 30 93 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`; 31 94 writeFileSync(join(outputDir, "samples.js"), samplesIndex); 32 95 33 33 - console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`); 96 96 + console.log(`\n✅ ${lexicons.size} samples verified successfully`);

+151 -91

packages/playground/samples/index.js

··· 5 5 6 6 const __dirname = dirname(fileURLToPath(import.meta.url)); 7 7 8 8 - // Get all tsp files 9 9 - function getAllTspFiles(dir, baseDir = dir) { 8 8 + // Get all tsp and json files 9 9 + function getAllFiles(dir, baseDir = dir) { 10 10 const files = []; 11 11 const entries = readdirSync(dir); 12 12 ··· 15 15 const stat = statSync(fullPath); 16 16 17 17 if (stat.isDirectory()) { 18 18 - files.push(...getAllTspFiles(fullPath, baseDir)); 19 19 - } else if (entry.endsWith(".tsp")) { 18 18 + files.push(...getAllFiles(fullPath, baseDir)); 19 19 + } else if (entry.endsWith(".tsp") || entry.endsWith(".json")) { 20 20 files.push(relative(baseDir, fullPath)); 21 21 } 22 22 } ··· 24 24 return files.sort(); 25 25 } 26 26 27 27 - // Extract dependencies from a file 28 28 - function extractDependencies(content) { 29 29 - const deps = new Set(); 30 30 - // Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main" 31 31 - // Pattern: word.word.word... followed by dot and identifier starting with capital letter 32 32 - const pattern = 33 33 - /\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g; 34 34 - const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, ""); 35 35 - 36 36 - const matches = withoutDeclaration.matchAll(pattern); 37 37 - for (const match of matches) { 38 38 - deps.add(match[1]); 27 27 + // Extract all refs from JSON (recursively search for strings with #) 28 28 + function extractRefsFromJson(obj, refs = new Map()) { 29 29 + if (typeof obj === "string") { 30 30 + // Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref) 31 31 + const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/); 32 32 + if (match) { 33 33 + const ns = match[1]; 34 34 + const def = match[2]; 35 35 + const modelName = def.charAt(0).toUpperCase() + def.slice(1); 36 36 + if (!refs.has(ns)) { 37 37 + refs.set(ns, new Set()); 38 38 + } 39 39 + refs.get(ns).add(modelName); 40 40 + } else { 41 41 + // Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots) 42 42 + const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/); 43 43 + if (nsMatch) { 44 44 + const ns = nsMatch[1]; 45 45 + if (!refs.has(ns)) { 46 46 + refs.set(ns, new Set()); 47 47 + } 48 48 + refs.get(ns).add("Main"); 49 49 + } 50 50 + } 51 51 + } else if (Array.isArray(obj)) { 52 52 + for (const item of obj) { 53 53 + extractRefsFromJson(item, refs); 54 54 + } 55 55 + } else if (obj && typeof obj === "object") { 56 56 + for (const value of Object.values(obj)) { 57 57 + extractRefsFromJson(value, refs); 58 58 + } 39 59 } 40 40 - 41 41 - return Array.from(deps); 60 60 + return refs; 42 61 } 43 62 44 44 - const atprotoInputDir = join( 45 45 - __dirname, 46 46 - "../../emitter/test/integration/atproto/input", 47 47 - ); 48 48 - const lexiconExamplesDir = join( 49 49 - __dirname, 50 50 - "../../emitter/test/integration/lexicon-examples/input", 51 51 - ); 52 52 - 53 53 - const atprotoFiles = getAllTspFiles(atprotoInputDir); 54 54 - const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir); 55 55 - 56 56 - // Build dependency graph 57 57 - const lexicons = new Map(); // namespace -> { file, content, deps } 58 58 - 59 59 - // Process atproto files 60 60 - for (const file of atprotoFiles) { 61 61 - const fullPath = join(atprotoInputDir, file); 62 62 - const content = readFileSync(fullPath, "utf-8"); 63 63 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 64 64 - const deps = extractDependencies(content); 65 65 - 66 66 - lexicons.set(namespace, { file: `atproto/${file}`, content, deps }); 67 67 - } 63 63 + const integrationDir = join(__dirname, "../../emitter/test/integration"); 68 64 69 69 - // Process lexicon-examples files 70 70 - for (const file of lexiconExampleFiles) { 71 71 - const fullPath = join(lexiconExamplesDir, file); 72 72 - const content = readFileSync(fullPath, "utf-8"); 73 73 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 74 74 - const deps = extractDependencies(content); 65 65 + // Get all test suite directories 66 66 + const testSuites = readdirSync(integrationDir).filter((name) => { 67 67 + const fullPath = join(integrationDir, name); 68 68 + return statSync(fullPath).isDirectory() && !name.startsWith("."); 69 69 + }); 75 70 76 76 - lexicons.set(namespace, { file: `examples/${file}`, content, deps }); 77 77 - } 71 71 + // Build lexicons with refs extracted from JSON 72 72 + const lexicons = new Map(); // namespace -> { file, content, refs, suite } 78 73 79 79 - // Recursively collect all dependencies (topological sort) 80 80 - function collectDependencies( 81 81 - namespace, 82 82 - collected = new Set(), 83 83 - visiting = new Set(), 84 84 - ) { 85 85 - if (collected.has(namespace)) return; 86 86 - if (visiting.has(namespace)) return; // circular dependency 74 74 + // Process all test suites 75 75 + for (const suite of testSuites) { 76 76 + const inputDir = join(integrationDir, suite, "input"); 77 77 + const outputDir = join(integrationDir, suite, "output"); 87 78 88 88 - const lexicon = lexicons.get(namespace); 89 89 - if (!lexicon) return; 79 79 + const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp")); 80 80 + 81 81 + for (const file of inputFiles) { 82 82 + const fullPath = join(inputDir, file); 83 83 + const content = readFileSync(fullPath, "utf-8"); 84 84 + const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 90 85 91 91 - visiting.add(namespace); 86 86 + // Find corresponding JSON output 87 87 + const jsonFile = file.replace(/\.tsp$/, ".json"); 88 88 + const jsonPath = join(outputDir, jsonFile); 89 89 + const jsonContent = readFileSync(jsonPath, "utf-8"); 90 90 + const jsonData = JSON.parse(jsonContent); 91 91 + const refs = extractRefsFromJson(jsonData); 92 92 93 93 - // First collect all dependencies 94 94 - for (const dep of lexicon.deps) { 95 95 - collectDependencies(dep, collected, visiting); 93 93 + lexicons.set(namespace, { file, content, refs, suite }); 96 94 } 95 95 + } 97 96 98 98 - visiting.delete(namespace); 99 99 - collected.add(namespace); 97 97 + // TypeSpec reserved keywords that need escaping 98 98 + const TYPESPEC_KEYWORDS = new Set([ 99 99 + "record", 100 100 + "pub", 101 101 + "interface", 102 102 + "model", 103 103 + "namespace", 104 104 + "op", 105 105 + "import", 106 106 + "export", 107 107 + "using", 108 108 + "alias", 109 109 + "enum", 110 110 + "union", 111 111 + "scalar", 112 112 + "extends", 113 113 + ]); 114 114 + 115 115 + // Escape a namespace part if it's a reserved keyword 116 116 + function escapeNamespacePart(part) { 117 117 + return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part; 100 118 } 101 119 102 102 - // Bundle a lexicon with all its dependencies 103 103 - function bundleLexicon(namespace) { 104 104 - const collected = new Set(); 105 105 - collectDependencies(namespace, collected); 120 120 + // Escape a full namespace path 121 121 + function escapeNamespace(namespace) { 122 122 + return namespace.split(".").map(escapeNamespacePart).join("."); 123 123 + } 106 124 107 107 - // Put the main lexicon FIRST, then its dependencies 108 108 - const mainLexicon = lexicons.get(namespace); 109 109 - const deps = Array.from(collected).filter((ns) => ns !== namespace); 125 125 + // Get the JSON for a lexicon to check its definitions 126 126 + function getLexiconJson(namespace) { 127 127 + const lexicon = lexicons.get(namespace); 128 128 + if (!lexicon) return null; 129 129 + 130 130 + const jsonPath = join( 131 131 + integrationDir, 132 132 + lexicon.suite, 133 133 + "output", 134 134 + lexicon.file.replace(".tsp", ".json"), 135 135 + ); 136 136 + 137 137 + try { 138 138 + return JSON.parse(readFileSync(jsonPath, "utf-8")); 139 139 + } catch { 140 140 + return null; 141 141 + } 142 142 + } 110 143 111 111 - let bundled = 'import "@typelex/emitter";\n\n'; 144 144 + // Check if a definition in JSON is a token 145 145 + function isToken(lexiconJson, defName) { 146 146 + if (!lexiconJson || !lexiconJson.defs) return false; 147 147 + const def = lexiconJson.defs[defName]; 148 148 + return def && def.type === "token"; 149 149 + } 112 150 113 113 - // Main lexicon first (so it shows in the playground) 114 114 - if (mainLexicon) { 115 115 - const contentWithoutImport = mainLexicon.content.replace( 116 116 - /^import "@typelex\/emitter";\s*\n/, 117 117 - "", 118 118 - ); 119 119 - bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`; 120 120 - } 151 151 + // Bundle a lexicon with stubs for referenced types (from JSON) 152 152 + function bundleLexicon(namespace) { 153 153 + const mainLexicon = lexicons.get(namespace); 154 154 + if (!mainLexicon) return ""; 155 155 + 156 156 + let bundled = mainLexicon.content; 157 157 + 158 158 + // Add stubs from refs extracted from JSON output (excluding self-references) 159 159 + if (mainLexicon.refs.size > 0) { 160 160 + let hasExternalRefs = false; 161 161 + for (const [ns] of mainLexicon.refs) { 162 162 + if (ns !== namespace) { 163 163 + hasExternalRefs = true; 164 164 + break; 165 165 + } 166 166 + } 121 167 122 122 - // Then dependencies 123 123 - for (const ns of deps) { 124 124 - const lexicon = lexicons.get(ns); 125 125 - if (!lexicon) continue; 168 168 + if (hasExternalRefs) { 169 169 + bundled += "\n// --- Externals ---\n"; 170 170 + } 126 171 127 127 - const contentWithoutImport = lexicon.content.replace( 128 128 - /^import "@typelex\/emitter";\s*\n/, 129 129 - "", 130 130 - ); 131 131 - bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`; 172 172 + for (const [ns, models] of mainLexicon.refs) { 173 173 + // Skip if this is the current namespace 174 174 + if (ns === namespace) continue; 175 175 + 176 176 + // Get the JSON for this referenced namespace to check for tokens 177 177 + const refJson = getLexiconJson(ns); 178 178 + 179 179 + const escapedNs = escapeNamespace(ns); 180 180 + bundled += `\n@external\nnamespace ${escapedNs} {\n`; 181 181 + for (const model of models) { 182 182 + // Check if this definition exists in the JSON and is a token 183 183 + const defName = model.charAt(0).toLowerCase() + model.slice(1); 184 184 + if (refJson && isToken(refJson, defName)) { 185 185 + bundled += ` @token model ${model} { }\n`; 186 186 + } else { 187 187 + bundled += ` model ${model} { }\n`; 188 188 + } 189 189 + } 190 190 + bundled += `}\n`; 191 191 + } 132 192 } 133 193 134 194 return bundled;

+11

packages/emitter/src/emitter.ts

··· 124 124 125 125 // Skip external namespaces - they don't emit JSON files 126 126 if (isExternal(this.program, ns)) { 127 127 + // Validate that all models in external namespaces are empty (stub-only) 128 128 + for (const [_, model] of ns.models) { 129 129 + if (model.properties && model.properties.size > 0) { 130 130 + this.program.reportDiagnostic({ 131 131 + code: "external-model-not-empty", 132 132 + severity: "error", 133 133 + message: `Models in @external namespaces must be empty stubs. Model '${model.name}' in namespace '${fullName}' has properties.`, 134 134 + target: model, 135 135 + }); 136 136 + } 137 137 + } 127 138 return; 128 139 } 129 140