An experimental TypeSpec syntax for Lexicon

[New feature] Externals #2

closed opened by danabra.mov targeting main from external

A step towards https://tangled.org/@danabra.mov/typelex/issues/5.

This adds support for Externals, which are namespaces with @external decorator.

@external
namespace com.atproto.label.defs {
  model Label { }
}

@external
namespace app.bsky.graph.defs {
  model StarterPackViewBasic { }
  model ListViewBasic { }
}

They do NOT get emitted as JSON. They only exist to represent external Lexicons for which you have only JSON.

Models inside @external namespaces must be empty. However, they're allowed to be @tokens which we need to know to disambiguate unions and knownValues. (In the future, it's possible we won't rely on knowing that, but for now we do.)

I've converted the playground to use externals properly. I will convert tests in a follow-up.

Labels

None yet.

assignee

None yet.

Participants 1
AT URI
at://did:plc:fpruhuo22xkm5o7ttr2ktxdo/sh.tangled.repo.pull/3m2siinlkdr22
+1736 -100
Diff #1
+13
packages/emitter/lib/decorators.tsp
··· 159 * ``` 160 */ 161 extern dec errors(target: unknown, ...errors: unknown[]);
··· 159 * ``` 160 */ 161 extern dec errors(target: unknown, ...errors: unknown[]); 162 + 163 + /** 164 + * Marks a namespace as external, preventing it from emitting JSON output. 165 + * This decorator can only be applied to namespaces. 166 + * Useful for importing definitions from other lexicons without re-emitting them. 167 + * 168 + * @example 169 + * ```typespec 170 + * @external 171 + * namespace com.atproto.repo.defs; 172 + * ``` 173 + */ 174 + extern dec external(target: Namespace);
+23
packages/emitter/src/decorators.ts
··· 24 const inlineKey = Symbol("inline"); 25 const maxBytesKey = Symbol("maxBytes"); 26 const minBytesKey = Symbol("minBytes"); 27 28 /** 29 * @maxBytes decorator for maximum length of bytes type ··· 294 export function isReadOnly(program: Program, target: Type): boolean { 295 return program.stateSet(readOnlyKey).has(target); 296 }
··· 24 const inlineKey = Symbol("inline"); 25 const maxBytesKey = Symbol("maxBytes"); 26 const minBytesKey = Symbol("minBytes"); 27 + const externalKey = Symbol("external"); 28 29 /** 30 * @maxBytes decorator for maximum length of bytes type ··· 295 export function isReadOnly(program: Program, target: Type): boolean { 296 return program.stateSet(readOnlyKey).has(target); 297 } 298 + 299 + /** 300 + * @external decorator for marking a namespace as external 301 + * External namespaces are skipped during emission and don't produce JSON files 302 + */ 303 + export function $external(context: DecoratorContext, target: Type) { 304 + if (target.kind !== "Namespace") { 305 + context.program.reportDiagnostic({ 306 + code: "external-not-on-namespace", 307 + severity: "error", 308 + message: "@external decorator can only be applied to namespaces", 309 + target: target, 310 + }); 311 + return; 312 + } 313 + 314 + context.program.stateSet(externalKey).add(target); 315 + } 316 + 317 + export function isExternal(program: Program, target: Type): boolean { 318 + return program.stateSet(externalKey).has(target); 319 + }
+1382
packages/emitter/src/emitter.ts
··· 67 isInline, 68 getMaxBytes, 69 getMinBytes, 70 } from "./decorators.js"; 71 72 export interface EmitterOptions { ··· 121 return; 122 } 123 124 // Check for TypeSpec enum syntax and throw error 125 if (ns.enums && ns.enums.size > 0) { 126 for (const [_, enumType] of ns.enums) {
··· 67 isInline, 68 getMaxBytes, 69 getMinBytes, 70 + isExternal, 71 } from "./decorators.js"; 72 73 export interface EmitterOptions { ··· 122 return; 123 } 124 125 + // Skip external namespaces - they don't emit JSON files 126 + if (isExternal(this.program, ns)) { 127 + // Validate that all models in external namespaces are empty (stub-only) 128 + for (const [_, model] of ns.models) { 129 + if (model.properties && model.properties.size > 0) { 130 + this.program.reportDiagnostic({ 131 + code: "external-model-not-empty", 132 + severity: "error", 133 + message: `Models in @external namespaces must be empty stubs. Model '${model.name}' in namespace '${fullName}' has properties.`, 134 + target: model, 135 + }); 136 + } 137 + } 138 + return; 139 + } 140 + 141 // Check for TypeSpec enum syntax and throw error 142 if (ns.enums && ns.enums.size > 0) { 143 for (const [_, enumType] of ns.enums) { 144 + 145 + 146 + 147 + 148 + 149 + 150 + 151 + 152 + 153 + 154 + 155 + 156 + 157 + 158 + 159 + 160 + 161 + 162 + 163 + 164 + 165 + 166 + 167 + 168 + 169 + 170 + 171 + 172 + 173 + 174 + 175 + 176 + 177 + 178 + 179 + 180 + 181 + 182 + 183 + 184 + 185 + 186 + 187 + 188 + 189 + 190 + 191 + 192 + 193 + 194 + 195 + 196 + 197 + 198 + 199 + 200 + 201 + 202 + 203 + 204 + 205 + 206 + 207 + 208 + 209 + 210 + 211 + 212 + 213 + 214 + 215 + 216 + 217 + 218 + 219 + 220 + 221 + 222 + 223 + 224 + 225 + 226 + 227 + 228 + 229 + 230 + 231 + 232 + 233 + 234 + 235 + 236 + 237 + 238 + 239 + 240 + 241 + 242 + 243 + 244 + 245 + 246 + 247 + 248 + 249 + 250 + 251 + 252 + 253 + 254 + 255 + 256 + 257 + 258 + 259 + 260 + 261 + 262 + 263 + 264 + 265 + 266 + 267 + 268 + 269 + 270 + 271 + 272 + 273 + 274 + 275 + 276 + 277 + 278 + 279 + 280 + 281 + 282 + 283 + 284 + 285 + 286 + 287 + 288 + 289 + 290 + 291 + 292 + 293 + 294 + 295 + 296 + 297 + 298 + 299 + 300 + 301 + 302 + 303 + 304 + 305 + 306 + 307 + 308 + 309 + 310 + 311 + 312 + 313 + 314 + 315 + 316 + 317 + 318 + 319 + 320 + 321 + 322 + 323 + 324 + 325 + 326 + 327 + 328 + 329 + 330 + 331 + 332 + 333 + 334 + 335 + 336 + 337 + 338 + 339 + 340 + 341 + 342 + 343 + 344 + 345 + 346 + 347 + 348 + 349 + 350 + 351 + 352 + 353 + 354 + 355 + 356 + 357 + 358 + 359 + 360 + 361 + 362 + 363 + 364 + 365 + 366 + 367 + 368 + 369 + 370 + 371 + 372 + 373 + 374 + 375 + 376 + 377 + 378 + 379 + 380 + 381 + 382 + 383 + 384 + 385 + 386 + 387 + 388 + 389 + 390 + 391 + 392 + 393 + 394 + 395 + 396 + 397 + 398 + 399 + 400 + 401 + 402 + 403 + 404 + 405 + 406 + 407 + 408 + 409 + 410 + 411 + 412 + 413 + 414 + 415 + 416 + 417 + 418 + 419 + 420 + 421 + 422 + 423 + 424 + 425 + 426 + 427 + 428 + 429 + 430 + 431 + 432 + 433 + 434 + 435 + 436 + 437 + 438 + 439 + 440 + 441 + 442 + 443 + 444 + 445 + 446 + 447 + 448 + 449 + 450 + 451 + 452 + 453 + 454 + 455 + 456 + 457 + 458 + 459 + 460 + 461 + 462 + 463 + 464 + 465 + 466 + 467 + 468 + 469 + 470 + 471 + 472 + 473 + 474 + 475 + 476 + 477 + 478 + 479 + 480 + 481 + 482 + 483 + 484 + 485 + 486 + 487 + 488 + 489 + 490 + 491 + 492 + 493 + 494 + 495 + 496 + 497 + 498 + 499 + 500 + 501 + 502 + 503 + 504 + 505 + 506 + 507 + 508 + 509 + 510 + 511 + 512 + 513 + 514 + 515 + 516 + 517 + 518 + 519 + 520 + 521 + 522 + 523 + 524 + 525 + 526 + 527 + 528 + 529 + 530 + 531 + 532 + 533 + 534 + 535 + 536 + 537 + 538 + 539 + 540 + 541 + 542 + 543 + 544 + 545 + 546 + 547 + 548 + 549 + 550 + 551 + 552 + 553 + 554 + 555 + 556 + // Model reference union (including empty union with unknown) 557 + if (variants.unionRefs.length > 0 || variants.hasUnknown) { 558 + if ( 559 + variants.stringLiterals.length > 0 || 560 + variants.knownValueRefs.length > 0 561 + ) { 562 + this.program.reportDiagnostic({ 563 + code: "union-mixed-refs-literals", 564 + severity: "error", 565 + 566 + 567 + 568 + 569 + 570 + 571 + 572 + 573 + 574 + 575 + 576 + 577 + 578 + 579 + 580 + 581 + 582 + 583 + 584 + 585 + 586 + 587 + 588 + 589 + 590 + 591 + 592 + 593 + 594 + 595 + 596 + 597 + 598 + 599 + 600 + 601 + 602 + 603 + 604 + 605 + 606 + 607 + 608 + 609 + 610 + 611 + 612 + 613 + 614 + 615 + 616 + 617 + 618 + 619 + 620 + 621 + 622 + 623 + 624 + 625 + 626 + 627 + 628 + 629 + 630 + 631 + 632 + 633 + 634 + 635 + 636 + 637 + 638 + 639 + 640 + 641 + 642 + 643 + 644 + 645 + 646 + 647 + 648 + 649 + 650 + 651 + 652 + 653 + 654 + 655 + 656 + 657 + 658 + 659 + 660 + 661 + 662 + 663 + 664 + 665 + 666 + 667 + 668 + 669 + 670 + 671 + 672 + 673 + 674 + 675 + 676 + 677 + 678 + 679 + 680 + 681 + 682 + 683 + 684 + 685 + 686 + 687 + 688 + 689 + 690 + 691 + 692 + 693 + 694 + 695 + 696 + 697 + 698 + 699 + 700 + 701 + 702 + 703 + 704 + 705 + 706 + 707 + 708 + 709 + 710 + 711 + 712 + 713 + 714 + 715 + 716 + 717 + 718 + 719 + 720 + 721 + 722 + 723 + 724 + 725 + 726 + 727 + 728 + 729 + 730 + 731 + 732 + 733 + 734 + 735 + 736 + 737 + 738 + 739 + 740 + 741 + 742 + 743 + 744 + 745 + 746 + 747 + 748 + 749 + 750 + 751 + 752 + 753 + 754 + 755 + 756 + 757 + 758 + 759 + 760 + 761 + 762 + 763 + 764 + 765 + 766 + 767 + 768 + 769 + 770 + 771 + 772 + 773 + 774 + 775 + 776 + 777 + 778 + 779 + 780 + 781 + 782 + 783 + 784 + 785 + 786 + 787 + 788 + 789 + 790 + 791 + 792 + 793 + 794 + 795 + 796 + 797 + 798 + 799 + 800 + 801 + 802 + 803 + 804 + 805 + 806 + 807 + 808 + 809 + 810 + 811 + 812 + 813 + 814 + 815 + 816 + 817 + 818 + 819 + 820 + 821 + 822 + 823 + 824 + 825 + 826 + 827 + 828 + 829 + 830 + 831 + 832 + 833 + 834 + 835 + 836 + 837 + 838 + 839 + 840 + 841 + 842 + 843 + 844 + 845 + 846 + 847 + 848 + 849 + 850 + 851 + 852 + 853 + 854 + 855 + 856 + 857 + 858 + 859 + 860 + 861 + 862 + 863 + 864 + 865 + 866 + 867 + 868 + 869 + 870 + 871 + 872 + 873 + 874 + 875 + 876 + 877 + 878 + 879 + 880 + 881 + 882 + 883 + 884 + 885 + 886 + 887 + 888 + 889 + 890 + 891 + 892 + 893 + 894 + 895 + 896 + 897 + 898 + 899 + 900 + 901 + 902 + 903 + 904 + 905 + 906 + 907 + 908 + 909 + 910 + 911 + 912 + 913 + 914 + 915 + 916 + 917 + 918 + 919 + 920 + 921 + 922 + 923 + 924 + 925 + 926 + 927 + 928 + 929 + 930 + 931 + 932 + 933 + 934 + 935 + 936 + 937 + 938 + 939 + 940 + 941 + 942 + 943 + 944 + 945 + 946 + 947 + 948 + 949 + 950 + 951 + 952 + 953 + 954 + 955 + 956 + 957 + 958 + 959 + 960 + 961 + 962 + 963 + 964 + 965 + 966 + 967 + 968 + 969 + 970 + 971 + 972 + 973 + 974 + 975 + 976 + 977 + 978 + 979 + 980 + 981 + 982 + 983 + 984 + 985 + 986 + 987 + 988 + 989 + 990 + 991 + 992 + 993 + 994 + 995 + 996 + 997 + 998 + 999 + 1000 + 1001 + 1002 + 1003 + 1004 + 1005 + 1006 + 1007 + 1008 + 1009 + 1010 + 1011 + 1012 + 1013 + 1014 + 1015 + 1016 + 1017 + 1018 + 1019 + 1020 + 1021 + 1022 + 1023 + 1024 + 1025 + 1026 + 1027 + 1028 + 1029 + 1030 + 1031 + 1032 + 1033 + 1034 + 1035 + 1036 + 1037 + 1038 + 1039 + 1040 + 1041 + 1042 + 1043 + 1044 + 1045 + 1046 + 1047 + 1048 + 1049 + 1050 + 1051 + 1052 + 1053 + 1054 + 1055 + 1056 + 1057 + 1058 + 1059 + 1060 + 1061 + 1062 + 1063 + 1064 + 1065 + 1066 + 1067 + 1068 + 1069 + 1070 + 1071 + 1072 + 1073 + 1074 + 1075 + 1076 + 1077 + 1078 + 1079 + 1080 + 1081 + 1082 + 1083 + 1084 + 1085 + 1086 + 1087 + 1088 + 1089 + 1090 + 1091 + 1092 + 1093 + 1094 + 1095 + 1096 + 1097 + 1098 + 1099 + 1100 + 1101 + 1102 + 1103 + 1104 + 1105 + 1106 + 1107 + 1108 + 1109 + 1110 + 1111 + 1112 + 1113 + 1114 + 1115 + 1116 + 1117 + 1118 + 1119 + 1120 + 1121 + 1122 + 1123 + 1124 + 1125 + 1126 + 1127 + 1128 + 1129 + 1130 + 1131 + 1132 + 1133 + 1134 + 1135 + 1136 + 1137 + 1138 + 1139 + 1140 + 1141 + 1142 + 1143 + 1144 + 1145 + 1146 + 1147 + 1148 + 1149 + 1150 + 1151 + 1152 + 1153 + 1154 + 1155 + 1156 + 1157 + 1158 + 1159 + 1160 + 1161 + 1162 + 1163 + 1164 + 1165 + 1166 + 1167 + 1168 + 1169 + 1170 + 1171 + 1172 + 1173 + 1174 + 1175 + 1176 + 1177 + 1178 + 1179 + 1180 + 1181 + 1182 + 1183 + 1184 + 1185 + 1186 + 1187 + 1188 + 1189 + 1190 + 1191 + 1192 + 1193 + 1194 + 1195 + 1196 + 1197 + 1198 + 1199 + 1200 + 1201 + 1202 + 1203 + 1204 + 1205 + 1206 + 1207 + 1208 + 1209 + 1210 + 1211 + 1212 + 1213 + 1214 + 1215 + 1216 + 1217 + 1218 + 1219 + 1220 + 1221 + 1222 + 1223 + 1224 + 1225 + 1226 + 1227 + 1228 + 1229 + 1230 + 1231 + 1232 + 1233 + 1234 + 1235 + 1236 + 1237 + 1238 + 1239 + 1240 + 1241 + 1242 + 1243 + 1244 + 1245 + 1246 + 1247 + 1248 + 1249 + 1250 + 1251 + 1252 + 1253 + 1254 + 1255 + 1256 + 1257 + 1258 + 1259 + 1260 + 1261 + 1262 + 1263 + 1264 + 1265 + 1266 + 1267 + 1268 + 1269 + 1270 + 1271 + 1272 + 1273 + 1274 + 1275 + 1276 + 1277 + 1278 + 1279 + 1280 + 1281 + 1282 + 1283 + 1284 + 1285 + 1286 + 1287 + 1288 + 1289 + 1290 + 1291 + 1292 + 1293 + 1294 + 1295 + 1296 + 1297 + 1298 + 1299 + 1300 + 1301 + 1302 + 1303 + 1304 + 1305 + 1306 + 1307 + 1308 + 1309 + 1310 + 1311 + 1312 + 1313 + 1314 + 1315 + 1316 + 1317 + 1318 + 1319 + 1320 + 1321 + 1322 + 1323 + 1324 + 1325 + 1326 + 1327 + 1328 + 1329 + 1330 + 1331 + 1332 + 1333 + 1334 + 1335 + 1336 + 1337 + 1338 + 1339 + 1340 + 1341 + 1342 + 1343 + 1344 + 1345 + 1346 + 1347 + 1348 + 1349 + 1350 + 1351 + 1352 + 1353 + 1354 + 1355 + 1356 + 1357 + 1358 + 1359 + 1360 + 1361 + 1362 + 1363 + 1364 + 1365 + 1366 + 1367 + 1368 + 1369 + 1370 + 1371 + 1372 + 1373 + 1374 + 1375 + 1376 + 1377 + 1378 + 1379 + 1380 + 1381 + 1382 + 1383 + 1384 + 1385 + 1386 + 1387 + 1388 + 1389 + 1390 + 1391 + 1392 + 1393 + 1394 + 1395 + 1396 + 1397 + 1398 + 1399 + 1400 + 1401 + 1402 + 1403 + 1404 + 1405 + 1406 + 1407 + 1408 + 1409 + 1410 + 1411 + 1412 + 1413 + 1414 + 1415 + 1416 + 1417 + 1418 + 1419 + 1420 + 1421 + 1422 + 1423 + 1424 + 1425 + 1426 + 1427 + 1428 + 1429 + 1430 + 1431 + 1432 + 1433 + 1434 + 1435 + 1436 + 1437 + 1438 + 1439 + 1440 + 1441 + 1442 + 1443 + 1444 + 1445 + 1446 + 1447 + 1448 + 1449 + 1450 + 1451 + 1452 + 1453 + 1454 + 1455 + 1456 + 1457 + 1458 + 1459 + 1460 + 1461 + 1462 + 1463 + 1464 + 1465 + 1466 + 1467 + 1468 + 1469 + 1470 + 1471 + 1472 + 1473 + 1474 + 1475 + 1476 + 1477 + 1478 + 1479 + 1480 + 1481 + 1482 + 1483 + 1484 + 1485 + 1486 + 1487 + 1488 + 1489 + 1490 + 1491 + 1492 + 1493 + 1494 + 1495 + 1496 + 1497 + model: Model, 1498 + fullyQualified = false, 1499 + ): string | null { 1500 + return this.getReference( 1501 + model, 1502 + model.name, 1503 + model.namespace, 1504 + fullyQualified, 1505 + ); 1506 + } 1507 + 1508 + private getUnionReference(union: Union): string | null {
+2
packages/emitter/src/tsp-index.ts
··· 14 $inline, 15 $maxBytes, 16 $minBytes, 17 } from "./decorators.js"; 18 19 /** @internal */ ··· 34 inline: $inline, 35 maxBytes: $maxBytes, 36 minBytes: $minBytes, 37 }, 38 };
··· 14 $inline, 15 $maxBytes, 16 $minBytes, 17 + $external, 18 } from "./decorators.js"; 19 20 /** @internal */ ··· 35 inline: $inline, 36 maxBytes: $maxBytes, 37 minBytes: $minBytes, 38 + external: $external, 39 }, 40 };
+23 -1
packages/emitter/test/spec.test.ts
··· 106 assert.deepStrictEqual(actual, expected); 107 }); 108 } else { 109 - it.skip(`TODO: ${expectedPath} (add ${inputPath})`, function () {}); 110 } 111 } 112 }); 113 } 114 });
··· 106 assert.deepStrictEqual(actual, expected); 107 }); 108 } else { 109 + it(`should emit ${expectedPath}`, function () { 110 + assert.fail( 111 + `Expected output file ${expectedPath} has no corresponding input file ${inputPath}. ` + 112 + `Either add the input file or remove the expected output.` 113 + ); 114 + }); 115 } 116 } 117 + 118 + // Check for unexpected emitted files 119 + it("should not emit unexpected files", function () { 120 + const emittedFiles = Object.keys(emitResult.files).filter(f => f.endsWith(".json")); 121 + const expectedPaths = Object.keys(expectedFiles) 122 + .filter(f => f.endsWith(".json")) 123 + .map(normalizePathToPosix); 124 + 125 + const unexpected = emittedFiles.filter(f => !expectedPaths.includes(f)); 126 + 127 + if (unexpected.length > 0) { 128 + assert.fail( 129 + `Unexpected files were emitted: ${unexpected.join(", ")}. ` + 130 + `Either add expected output files or ensure these should not be emitted.` 131 + ); 132 + } 133 + }); 134 }); 135 } 136 });
+21
packages/emitter/test/spec/basic/output/com/example/other.json
···
··· 1 + { 2 + "lexicon": 1, 3 + "id": "com.example.other", 4 + "defs": { 5 + "main": { 6 + "type": "object", 7 + "properties": {} 8 + }, 9 + "someDef": { 10 + "type": "object", 11 + "required": [ 12 + "value" 13 + ], 14 + "properties": { 15 + "value": { 16 + "type": "string" 17 + } 18 + } 19 + } 20 + } 21 + }
+8
packages/emitter/test/spec/external/input/test/external.tsp
···
··· 1 + import "@typelex/emitter"; 2 + 3 + @external 4 + namespace test.external { 5 + model Main { } 6 + 7 + model AlsoNotEmitted { } 8 + }
+7
packages/emitter/test/spec/external/input/test/normal.tsp
···
··· 1 + import "@typelex/emitter"; 2 + 3 + namespace test.normal { 4 + model Main { 5 + name?: string; 6 + } 7 + }
+14
packages/emitter/test/spec/external/output/test/normal.json
···
··· 1 + { 2 + "lexicon": 1, 3 + "id": "test.normal", 4 + "defs": { 5 + "main": { 6 + "type": "object", 7 + "properties": { 8 + "name": { 9 + "type": "string" 10 + } 11 + } 12 + } 13 + } 14 + }
+26 -5
DOCS.md
··· 236 237 This works across files tooโ€”just remember to `import` the file with the definition. 238 239 - ### External Stubs 240 241 - If you don't have TypeSpec definitions for external Lexicons, you can stub them out: 242 243 ```typescript 244 import "@typelex/emitter"; ··· 249 } 250 } 251 252 - // Empty stub (like .d.ts in TypeScript) 253 namespace com.atproto.label.defs { 254 model SelfLabels { } 255 } 256 ``` 257 258 - You could collect stubs in one file and import them: 259 260 ```typescript 261 import "@typelex/emitter"; ··· 268 } 269 ``` 270 271 - You'll want to replace the stubbed lexicons in the output folder with their real JSON before running codegen. 272 273 ### Inline Models 274
··· 236 237 This works across files tooโ€”just remember to `import` the file with the definition. 238 239 + ### External References 240 241 + If you don't have TypeSpec definitions for external Lexicons, you can stub them out using the `@external` decorator: 242 243 ```typescript 244 import "@typelex/emitter"; ··· 249 } 250 } 251 252 + // External stub (like .d.ts in TypeScript) 253 + @external 254 namespace com.atproto.label.defs { 255 model SelfLabels { } 256 + @token model SomeToken { } // use @token for tokens 257 } 258 ``` 259 260 + The `@external` decorator tells the emitter to skip JSON output for that namespace. This is useful when referencing definitions from other Lexicons that you don't want to re-emit. 261 + 262 + You could collect external stubs in one file and import them: 263 264 ```typescript 265 import "@typelex/emitter"; ··· 272 } 273 ``` 274 275 + Then in `atproto-stubs.tsp`: 276 + 277 + ```typescript 278 + import "@typelex/emitter"; 279 + 280 + @external 281 + namespace com.atproto.label.defs { 282 + model SelfLabels { } 283 + } 284 + 285 + @external 286 + namespace com.atproto.repo.defs { 287 + model StrongRef { } 288 + } 289 + // ... more stubs 290 + ``` 291 + 292 + You'll want to ensure the real JSON for external Lexicons is available before running codegen. 293 294 ### Inline Models 295
+66 -3
packages/playground/samples/build.js
··· 1 // @ts-check 2 - import { writeFileSync, mkdirSync } from "fs"; 3 import { dirname, resolve, join } from "path"; 4 import { fileURLToPath } from "url"; 5 import { lexicons, bundleLexicon } from "./index.js"; 6 7 const __dirname = dirname(fileURLToPath(import.meta.url)); 8 const outputDir = resolve(__dirname, "dist"); 9 10 // Create output directory 11 mkdirSync(outputDir, { recursive: true }); 12 13 - // Write each bundled lexicon to disk 14 const samplesList = {}; 15 16 for (const [namespace, lexicon] of lexicons) { ··· 20 21 writeFileSync(filepath, bundled); 22 23 samplesList[namespace] = { 24 filename: `samples/dist/${filename}`, 25 preferredEmitter: "@typelex/emitter", ··· 30 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`; 31 writeFileSync(join(outputDir, "samples.js"), samplesIndex); 32 33 - console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`);
··· 1 // @ts-check 2 + import { writeFileSync, mkdirSync, readFileSync } from "fs"; 3 import { dirname, resolve, join } from "path"; 4 import { fileURLToPath } from "url"; 5 + import { deepStrictEqual } from "assert"; 6 import { lexicons, bundleLexicon } from "./index.js"; 7 + import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing"; 8 9 const __dirname = dirname(fileURLToPath(import.meta.url)); 10 const outputDir = resolve(__dirname, "dist"); 11 + const pkgRoot = await findTestPackageRoot(import.meta.url); 12 + 13 + // TypeSpec library setup for testing 14 + const TypelexTestLibrary = { 15 + name: "@typelex/emitter", 16 + packageRoot: pkgRoot.replace("/playground", "/emitter"), 17 + files: [ 18 + { realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" }, 19 + { realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" }, 20 + { realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" }, 21 + ], 22 + }; 23 24 // Create output directory 25 mkdirSync(outputDir, { recursive: true }); 26 27 + // Write each bundled lexicon to disk and verify it compiles correctly 28 const samplesList = {}; 29 30 for (const [namespace, lexicon] of lexicons) { ··· 34 35 writeFileSync(filepath, bundled); 36 37 + const host = await createTestHost({ libraries: [TypelexTestLibrary] }); 38 + host.addTypeSpecFile("main.tsp", bundled); 39 + 40 + const baseOutputPath = resolveVirtualPath("test-output/"); 41 + const [, diagnostics] = await host.compileAndDiagnose("main.tsp", { 42 + outputDir: baseOutputPath, 43 + noEmit: false, 44 + emit: ["@typelex/emitter"], 45 + }); 46 + 47 + if (diagnostics.length > 0) { 48 + console.error(`โŒ ${namespace}: Compilation errors`); 49 + diagnostics.forEach(d => console.error(` ${d.message}`)); 50 + process.exit(1); 51 + } 52 + 53 + // Get emitted JSON 54 + const outputFiles = [...host.fs.entries()] 55 + .filter(([name]) => name.startsWith(baseOutputPath)) 56 + .map(([name, value]) => { 57 + let relativePath = name.replace(baseOutputPath, ""); 58 + if (relativePath.startsWith("@typelex/emitter/")) { 59 + relativePath = relativePath.replace("@typelex/emitter/", ""); 60 + } 61 + return [relativePath, value]; 62 + }); 63 + 64 + const expectedJsonPath = namespace.replace(/\./g, "/") + ".json"; 65 + const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath); 66 + 67 + if (!emittedJson) { 68 + console.error(`โŒ ${namespace}: No JSON output found (expected ${expectedJsonPath})`); 69 + process.exit(1); 70 + } 71 + 72 + // Compare with expected JSON 73 + const expectedJsonFile = join( 74 + pkgRoot.replace("/playground", "/emitter"), 75 + "test/integration", 76 + lexicon.suite, 77 + "output", 78 + lexicon.file.replace(".tsp", ".json") 79 + ); 80 + 81 + const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8")); 82 + const actualJson = JSON.parse(emittedJson[1]); 83 + 84 + deepStrictEqual(actualJson, expectedJson); 85 + 86 samplesList[namespace] = { 87 filename: `samples/dist/${filename}`, 88 preferredEmitter: "@typelex/emitter", ··· 93 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`; 94 writeFileSync(join(outputDir, "samples.js"), samplesIndex); 95 96 + console.log(`\nโœ… ${lexicons.size} samples verified successfully`);
+151 -91
packages/playground/samples/index.js
··· 5 6 const __dirname = dirname(fileURLToPath(import.meta.url)); 7 8 - // Get all tsp files 9 - function getAllTspFiles(dir, baseDir = dir) { 10 const files = []; 11 const entries = readdirSync(dir); 12 ··· 15 const stat = statSync(fullPath); 16 17 if (stat.isDirectory()) { 18 - files.push(...getAllTspFiles(fullPath, baseDir)); 19 - } else if (entry.endsWith(".tsp")) { 20 files.push(relative(baseDir, fullPath)); 21 } 22 } ··· 24 return files.sort(); 25 } 26 27 - // Extract dependencies from a file 28 - function extractDependencies(content) { 29 - const deps = new Set(); 30 - // Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main" 31 - // Pattern: word.word.word... followed by dot and identifier starting with capital letter 32 - const pattern = 33 - /\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g; 34 - const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, ""); 35 - 36 - const matches = withoutDeclaration.matchAll(pattern); 37 - for (const match of matches) { 38 - deps.add(match[1]); 39 } 40 - 41 - return Array.from(deps); 42 } 43 44 - const atprotoInputDir = join( 45 - __dirname, 46 - "../../emitter/test/integration/atproto/input", 47 - ); 48 - const lexiconExamplesDir = join( 49 - __dirname, 50 - "../../emitter/test/integration/lexicon-examples/input", 51 - ); 52 - 53 - const atprotoFiles = getAllTspFiles(atprotoInputDir); 54 - const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir); 55 - 56 - // Build dependency graph 57 - const lexicons = new Map(); // namespace -> { file, content, deps } 58 - 59 - // Process atproto files 60 - for (const file of atprotoFiles) { 61 - const fullPath = join(atprotoInputDir, file); 62 - const content = readFileSync(fullPath, "utf-8"); 63 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 64 - const deps = extractDependencies(content); 65 - 66 - lexicons.set(namespace, { file: `atproto/${file}`, content, deps }); 67 - } 68 69 - // Process lexicon-examples files 70 - for (const file of lexiconExampleFiles) { 71 - const fullPath = join(lexiconExamplesDir, file); 72 - const content = readFileSync(fullPath, "utf-8"); 73 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 74 - const deps = extractDependencies(content); 75 76 - lexicons.set(namespace, { file: `examples/${file}`, content, deps }); 77 - } 78 79 - // Recursively collect all dependencies (topological sort) 80 - function collectDependencies( 81 - namespace, 82 - collected = new Set(), 83 - visiting = new Set(), 84 - ) { 85 - if (collected.has(namespace)) return; 86 - if (visiting.has(namespace)) return; // circular dependency 87 88 - const lexicon = lexicons.get(namespace); 89 - if (!lexicon) return; 90 91 - visiting.add(namespace); 92 93 - // First collect all dependencies 94 - for (const dep of lexicon.deps) { 95 - collectDependencies(dep, collected, visiting); 96 } 97 98 - visiting.delete(namespace); 99 - collected.add(namespace); 100 } 101 102 - // Bundle a lexicon with all its dependencies 103 - function bundleLexicon(namespace) { 104 - const collected = new Set(); 105 - collectDependencies(namespace, collected); 106 107 - // Put the main lexicon FIRST, then its dependencies 108 - const mainLexicon = lexicons.get(namespace); 109 - const deps = Array.from(collected).filter((ns) => ns !== namespace); 110 111 - let bundled = 'import "@typelex/emitter";\n\n'; 112 113 - // Main lexicon first (so it shows in the playground) 114 - if (mainLexicon) { 115 - const contentWithoutImport = mainLexicon.content.replace( 116 - /^import "@typelex\/emitter";\s*\n/, 117 - "", 118 - ); 119 - bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`; 120 - } 121 122 - // Then dependencies 123 - for (const ns of deps) { 124 - const lexicon = lexicons.get(ns); 125 - if (!lexicon) continue; 126 127 - const contentWithoutImport = lexicon.content.replace( 128 - /^import "@typelex\/emitter";\s*\n/, 129 - "", 130 - ); 131 - bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`; 132 } 133 134 return bundled;
··· 5 6 const __dirname = dirname(fileURLToPath(import.meta.url)); 7 8 + // Get all tsp and json files 9 + function getAllFiles(dir, baseDir = dir) { 10 const files = []; 11 const entries = readdirSync(dir); 12 ··· 15 const stat = statSync(fullPath); 16 17 if (stat.isDirectory()) { 18 + files.push(...getAllFiles(fullPath, baseDir)); 19 + } else if (entry.endsWith(".tsp") || entry.endsWith(".json")) { 20 files.push(relative(baseDir, fullPath)); 21 } 22 } ··· 24 return files.sort(); 25 } 26 27 + // Extract all refs from JSON (recursively search for strings with #) 28 + function extractRefsFromJson(obj, refs = new Map()) { 29 + if (typeof obj === "string") { 30 + // Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref) 31 + const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/); 32 + if (match) { 33 + const ns = match[1]; 34 + const def = match[2]; 35 + const modelName = def.charAt(0).toUpperCase() + def.slice(1); 36 + if (!refs.has(ns)) { 37 + refs.set(ns, new Set()); 38 + } 39 + refs.get(ns).add(modelName); 40 + } else { 41 + // Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots) 42 + const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/); 43 + if (nsMatch) { 44 + const ns = nsMatch[1]; 45 + if (!refs.has(ns)) { 46 + refs.set(ns, new Set()); 47 + } 48 + refs.get(ns).add("Main"); 49 + } 50 + } 51 + } else if (Array.isArray(obj)) { 52 + for (const item of obj) { 53 + extractRefsFromJson(item, refs); 54 + } 55 + } else if (obj && typeof obj === "object") { 56 + for (const value of Object.values(obj)) { 57 + extractRefsFromJson(value, refs); 58 + } 59 } 60 + return refs; 61 } 62 63 + const integrationDir = join(__dirname, "../../emitter/test/integration"); 64 65 + // Get all test suite directories 66 + const testSuites = readdirSync(integrationDir).filter((name) => { 67 + const fullPath = join(integrationDir, name); 68 + return statSync(fullPath).isDirectory() && !name.startsWith("."); 69 + }); 70 71 + // Build lexicons with refs extracted from JSON 72 + const lexicons = new Map(); // namespace -> { file, content, refs, suite } 73 74 + // Process all test suites 75 + for (const suite of testSuites) { 76 + const inputDir = join(integrationDir, suite, "input"); 77 + const outputDir = join(integrationDir, suite, "output"); 78 79 + const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp")); 80 + 81 + for (const file of inputFiles) { 82 + const fullPath = join(inputDir, file); 83 + const content = readFileSync(fullPath, "utf-8"); 84 + const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 85 86 + // Find corresponding JSON output 87 + const jsonFile = file.replace(/\.tsp$/, ".json"); 88 + const jsonPath = join(outputDir, jsonFile); 89 + const jsonContent = readFileSync(jsonPath, "utf-8"); 90 + const jsonData = JSON.parse(jsonContent); 91 + const refs = extractRefsFromJson(jsonData); 92 93 + lexicons.set(namespace, { file, content, refs, suite }); 94 } 95 + } 96 97 + // TypeSpec reserved keywords that need escaping 98 + const TYPESPEC_KEYWORDS = new Set([ 99 + "record", 100 + "pub", 101 + "interface", 102 + "model", 103 + "namespace", 104 + "op", 105 + "import", 106 + "export", 107 + "using", 108 + "alias", 109 + "enum", 110 + "union", 111 + "scalar", 112 + "extends", 113 + ]); 114 + 115 + // Escape a namespace part if it's a reserved keyword 116 + function escapeNamespacePart(part) { 117 + return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part; 118 } 119 120 + // Escape a full namespace path 121 + function escapeNamespace(namespace) { 122 + return namespace.split(".").map(escapeNamespacePart).join("."); 123 + } 124 125 + // Get the JSON for a lexicon to check its definitions 126 + function getLexiconJson(namespace) { 127 + const lexicon = lexicons.get(namespace); 128 + if (!lexicon) return null; 129 + 130 + const jsonPath = join( 131 + integrationDir, 132 + lexicon.suite, 133 + "output", 134 + lexicon.file.replace(".tsp", ".json"), 135 + ); 136 + 137 + try { 138 + return JSON.parse(readFileSync(jsonPath, "utf-8")); 139 + } catch { 140 + return null; 141 + } 142 + } 143 144 + // Check if a definition in JSON is a token 145 + function isToken(lexiconJson, defName) { 146 + if (!lexiconJson || !lexiconJson.defs) return false; 147 + const def = lexiconJson.defs[defName]; 148 + return def && def.type === "token"; 149 + } 150 151 + // Bundle a lexicon with stubs for referenced types (from JSON) 152 + function bundleLexicon(namespace) { 153 + const mainLexicon = lexicons.get(namespace); 154 + if (!mainLexicon) return ""; 155 + 156 + let bundled = mainLexicon.content; 157 + 158 + // Add stubs from refs extracted from JSON output (excluding self-references) 159 + if (mainLexicon.refs.size > 0) { 160 + let hasExternalRefs = false; 161 + for (const [ns] of mainLexicon.refs) { 162 + if (ns !== namespace) { 163 + hasExternalRefs = true; 164 + break; 165 + } 166 + } 167 168 + if (hasExternalRefs) { 169 + bundled += "\n// --- Externals ---\n"; 170 + } 171 172 + for (const [ns, models] of mainLexicon.refs) { 173 + // Skip if this is the current namespace 174 + if (ns === namespace) continue; 175 + 176 + // Get the JSON for this referenced namespace to check for tokens 177 + const refJson = getLexiconJson(ns); 178 + 179 + const escapedNs = escapeNamespace(ns); 180 + bundled += `\n@external\nnamespace ${escapedNs} {\n`; 181 + for (const model of models) { 182 + // Check if this definition exists in the JSON and is a token 183 + const defName = model.charAt(0).toLowerCase() + model.slice(1); 184 + if (refJson && isToken(refJson, defName)) { 185 + bundled += ` @token model ${model} { }\n`; 186 + } else { 187 + bundled += ` model ${model} { }\n`; 188 + } 189 + } 190 + bundled += `}\n`; 191 + } 192 } 193 194 return bundled;

Submissions

sign up or login to add to the discussion
danabra.mov submitted #1
5 commits
expand
add @external decorator
add test for @external
document @external
replacing bundling with externals in playground
enforce empty @externals
1/1 failed
expand
closed without merging
danabra.mov submitted #0
5 commits
expand
add @external decorator
add test for @external
document @external
replacing bundling with externals in playground
enforce empty @externals
1/1 success
expand