tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1{
2 writeShellScript,
3 writeText,
4 writers,
5 nix-prefetch-git,
6 formats,
7 lib,
8 coreutils,
9 curl,
10 xe,
11}:
12
13# Grammar list:
14# https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
15
16let
17 # Grammars we want to fetch from the tree-sitter github orga
18 knownTreeSitterOrgGrammarRepos = [
19 "tree-sitter-javascript"
20 "tree-sitter-c"
21 "tree-sitter-json"
22 "tree-sitter-cpp"
23 "tree-sitter-ruby"
24 "tree-sitter-go"
25 "tree-sitter-c-sharp"
26 "tree-sitter-python"
27 "tree-sitter-typescript"
28 "tree-sitter-rust"
29 "tree-sitter-bash"
30 "tree-sitter-php"
31 "tree-sitter-java"
32 "tree-sitter-scala"
33 "tree-sitter-ocaml"
34 "tree-sitter-julia"
35 "tree-sitter-html"
36 "tree-sitter-haskell"
37 "tree-sitter-regex"
38 "tree-sitter-css"
39 "tree-sitter-verilog"
40 "tree-sitter-jsdoc"
41 "tree-sitter-ql"
42 "tree-sitter-ql-dbscheme"
43 "tree-sitter-embedded-template"
44 "tree-sitter-tsq"
45 "tree-sitter-toml"
46 ];
47 knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
48
49 # repos of the tree-sitter github orga we want to ignore (not grammars)
50 ignoredTreeSitterOrgRepos = [
51 "tree-sitter"
52 "tree-sitter-cli"
53 # this is the haskell language bindings, tree-sitter-haskell is the grammar
54 "haskell-tree-sitter"
55 # this is the ruby language bindings, tree-sitter-ruby is the grammar
56 "ruby-tree-sitter.old"
57 # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
58 "rust-tree-sitter"
59 # this is the nodejs language bindings, tree-sitter-javascript is the grammar
60 "node-tree-sitter"
61 # this is the python language bindings, tree-sitter-python is the grammar
62 "py-tree-sitter"
63 # afl fuzzing for tree sitter
64 "afl-tree-sitter"
65 # this is the kotlin language bindings, tree-sitter-kotlin is the grammar
66 "kotlin-tree-sitter"
67 # this is the go language bindings, tree-sitter-go is the grammar
68 "go-tree-sitter"
69 # this is the java language bindings, tree-sitter-java is the grammar
70 "java-tree-sitter"
71 # archived
72 "highlight-schema"
73 # website
74 "tree-sitter.github.io"
75 # not maintained
76 "tree-sitter-razor"
77 # rust library for constructing arbitrary graph structures from source code
78 "tree-sitter-graph"
79 # abandoned
80 "tree-sitter-swift"
81 # this is the swift language bindings, tree-sitter-swift is the (abandoned) grammar
82 "swift-tree-sitter"
83 # abandoned
84 "tree-sitter-agda"
85 # abandoned
86 "tree-sitter-fluent"
87 # to unblock my update
88 "csharp-tree-sitter"
89 # (experimental) java bindings to the Tree-sitter parsing library
90 "java-tree-sitter"
91 # go bindings to the Tree-sitter parsing library
92 "go-tree-sitter"
93 # kotlin bindings to the Tree-sitter parsing library
94 "kotlin-tree-sitter"
95 # not ready to be used
96 "zig-tree-sitter"
97
98 # Non-grammar repositories
99 ".github"
100 "fuzz-action"
101 "parse-action"
102 "parser-setup-action"
103 "parser-test-action"
104 "parser-update-action"
105 "setup-action"
106 "workflows"
107 ];
108 ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
109
110 # Additional grammars that are not in the official github orga.
111 # If you need a grammar that already exists in the official orga,
112 # make sure to give it a different name.
113 otherGrammars = {
114 "tree-sitter-bitbake" = {
115 orga = "amaanq";
116 repo = "tree-sitter-bitbake";
117 };
118 "tree-sitter-beancount" = {
119 orga = "polarmutex";
120 repo = "tree-sitter-beancount";
121 };
122 "tree-sitter-bqn" = {
123 orga = "shnarazk";
124 repo = "tree-sitter-bqn";
125 };
126 "tree-sitter-clojure" = {
127 orga = "sogaiu";
128 repo = "tree-sitter-clojure";
129 };
130 "tree-sitter-comment" = {
131 orga = "stsewd";
132 repo = "tree-sitter-comment";
133 };
134 "tree-sitter-dart" = {
135 orga = "usernobody14";
136 repo = "tree-sitter-dart";
137 };
138 "tree-sitter-elisp" = {
139 orga = "wilfred";
140 repo = "tree-sitter-elisp";
141 };
142 "tree-sitter-just" = {
143 orga = "IndianBoy42";
144 repo = "tree-sitter-just";
145 };
146 "tree-sitter-nix" = {
147 orga = "nix-community";
148 repo = "tree-sitter-nix";
149 };
150 "tree-sitter-latex" = {
151 orga = "latex-lsp";
152 repo = "tree-sitter-latex";
153 };
154 "tree-sitter-lua" = {
155 orga = "MunifTanjim";
156 repo = "tree-sitter-lua";
157 };
158 "tree-sitter-factor" = {
159 orga = "erochest";
160 repo = "tree-sitter-factor";
161 };
162 "tree-sitter-fennel" = {
163 orga = "travonted";
164 repo = "tree-sitter-fennel";
165 };
166 "tree-sitter-make" = {
167 orga = "alemuller";
168 repo = "tree-sitter-make";
169 };
170 "tree-sitter-markdown" = {
171 orga = "MDeiml";
172 repo = "tree-sitter-markdown";
173 };
174 "tree-sitter-proto" = {
175 orga = "mitchellh";
176 repo = "tree-sitter-proto";
177 };
178 "tree-sitter-rego" = {
179 orga = "FallenAngel97";
180 repo = "tree-sitter-rego";
181 };
182 "tree-sitter-rst" = {
183 orga = "stsewd";
184 repo = "tree-sitter-rst";
185 };
186 "tree-sitter-svelte" = {
187 orga = "Himujjal";
188 repo = "tree-sitter-svelte";
189 };
190 "tree-sitter-sql" = {
191 orga = "derekstride";
192 repo = "tree-sitter-sql";
193 branch = "gh-pages";
194 };
195 "tree-sitter-talon" = {
196 orga = "wenkokke";
197 repo = "tree-sitter-talon";
198 };
199 "tree-sitter-typst" = {
200 orga = "uben0";
201 repo = "tree-sitter-typst";
202 };
203 "tree-sitter-vim" = {
204 orga = "vigoux";
205 repo = "tree-sitter-viml";
206 };
207 "tree-sitter-yaml" = {
208 orga = "ikatyang";
209 repo = "tree-sitter-yaml";
210 };
211 "tree-sitter-zig" = {
212 orga = "maxxnino";
213 repo = "tree-sitter-zig";
214 };
215 "tree-sitter-fish" = {
216 orga = "ram02z";
217 repo = "tree-sitter-fish";
218 };
219 "tree-sitter-dot" = {
220 orga = "rydesun";
221 repo = "tree-sitter-dot";
222 };
223 "tree-sitter-norg" = {
224 orga = "nvim-neorg";
225 repo = "tree-sitter-norg";
226 };
227 "tree-sitter-norg-meta" = {
228 orga = "nvim-neorg";
229 repo = "tree-sitter-norg-meta";
230 };
231 "tree-sitter-commonlisp" = {
232 orga = "thehamsta";
233 repo = "tree-sitter-commonlisp";
234 };
235 "tree-sitter-cuda" = {
236 orga = "thehamsta";
237 repo = "tree-sitter-cuda";
238 };
239 "tree-sitter-glsl" = {
240 orga = "thehamsta";
241 repo = "tree-sitter-glsl";
242 };
243 "tree-sitter-dockerfile" = {
244 orga = "camdencheek";
245 repo = "tree-sitter-dockerfile";
246 };
247 "tree-sitter-ledger" = {
248 orga = "cbarrete";
249 repo = "tree-sitter-ledger";
250 };
251 "tree-sitter-gomod" = {
252 orga = "camdencheek";
253 repo = "tree-sitter-go-mod";
254 };
255 "tree-sitter-gowork" = {
256 orga = "omertuc";
257 repo = "tree-sitter-go-work";
258 };
259 "tree-sitter-graphql" = {
260 orga = "bkegley";
261 repo = "tree-sitter-graphql";
262 };
263 "tree-sitter-pgn" = {
264 orga = "rolandwalker";
265 repo = "tree-sitter-pgn";
266 };
267 "tree-sitter-perl" = {
268 orga = "ganezdragon";
269 repo = "tree-sitter-perl";
270 };
271 "tree-sitter-kotlin" = {
272 orga = "fwcd";
273 repo = "tree-sitter-kotlin";
274 };
275 "tree-sitter-scss" = {
276 orga = "serenadeai";
277 repo = "tree-sitter-scss";
278 };
279 "tree-sitter-erlang" = {
280 orga = "abstractmachineslab";
281 repo = "tree-sitter-erlang";
282 };
283 "tree-sitter-elixir" = {
284 orga = "elixir-lang";
285 repo = "tree-sitter-elixir";
286 };
287 "tree-sitter-surface" = {
288 orga = "connorlay";
289 repo = "tree-sitter-surface";
290 };
291 "tree-sitter-eex" = {
292 orga = "connorlay";
293 repo = "tree-sitter-eex";
294 };
295 "tree-sitter-heex" = {
296 orga = "connorlay";
297 repo = "tree-sitter-heex";
298 };
299 "tree-sitter-supercollider" = {
300 orga = "madskjeldgaard";
301 repo = "tree-sitter-supercollider";
302 };
303 "tree-sitter-tlaplus" = {
304 orga = "tlaplus-community";
305 repo = "tree-sitter-tlaplus";
306 };
307 "tree-sitter-glimmer" = {
308 orga = "alexlafroscia";
309 repo = "tree-sitter-glimmer";
310 };
311 "tree-sitter-pug" = {
312 orga = "zealot128";
313 repo = "tree-sitter-pug";
314 };
315 "tree-sitter-vue" = {
316 orga = "ikatyang";
317 repo = "tree-sitter-vue";
318 };
319 "tree-sitter-elm" = {
320 orga = "elm-tooling";
321 repo = "tree-sitter-elm";
322 };
323 "tree-sitter-yang" = {
324 orga = "hubro";
325 repo = "tree-sitter-yang";
326 };
327 "tree-sitter-query" = {
328 orga = "nvim-treesitter";
329 repo = "tree-sitter-query";
330 };
331 "tree-sitter-sparql" = {
332 orga = "bonabeavis";
333 repo = "tree-sitter-sparql";
334 };
335 "tree-sitter-gdscript" = {
336 orga = "prestonknopp";
337 repo = "tree-sitter-gdscript";
338 };
339 "tree-sitter-gemini" = {
340 orga = "blessanabraham";
341 repo = "tree-sitter-gemini";
342 };
343 "tree-sitter-godot-resource" = {
344 orga = "prestonknopp";
345 repo = "tree-sitter-godot-resource";
346 };
347 "tree-sitter-turtle" = {
348 orga = "bonabeavis";
349 repo = "tree-sitter-turtle";
350 };
351 "tree-sitter-devicetree" = {
352 orga = "joelspadin";
353 repo = "tree-sitter-devicetree";
354 };
355 "tree-sitter-r" = {
356 orga = "r-lib";
357 repo = "tree-sitter-r";
358 };
359 "tree-sitter-bibtex" = {
360 orga = "latex-lsp";
361 repo = "tree-sitter-bibtex";
362 };
363 "tree-sitter-fortran" = {
364 orga = "stadelmanma";
365 repo = "tree-sitter-fortran";
366 };
367 "tree-sitter-cmake" = {
368 orga = "uyha";
369 repo = "tree-sitter-cmake";
370 };
371 "tree-sitter-janet-simple" = {
372 orga = "sogaiu";
373 repo = "tree-sitter-janet-simple";
374 };
375 "tree-sitter-json5" = {
376 orga = "joakker";
377 repo = "tree-sitter-json5";
378 };
379 "tree-sitter-pioasm" = {
380 orga = "leo60228";
381 repo = "tree-sitter-pioasm";
382 };
383 "tree-sitter-hjson" = {
384 orga = "winston0410";
385 repo = "tree-sitter-hjson";
386 };
387 "tree-sitter-llvm" = {
388 orga = "benwilliamgraham";
389 repo = "tree-sitter-llvm";
390 };
391 "tree-sitter-http" = {
392 orga = "ntbbloodbath";
393 repo = "tree-sitter-http";
394 };
395 "tree-sitter-prisma" = {
396 orga = "victorhqc";
397 repo = "tree-sitter-prisma";
398 };
399 "tree-sitter-org-nvim" = {
400 orga = "milisims";
401 repo = "tree-sitter-org";
402 };
403 "tree-sitter-hcl" = {
404 orga = "MichaHoffmann";
405 repo = "tree-sitter-hcl";
406 };
407 "tree-sitter-scheme" = {
408 orga = "6cdh";
409 repo = "tree-sitter-scheme";
410 };
411 "tree-sitter-tiger" = {
412 orga = "ambroisie";
413 repo = "tree-sitter-tiger";
414 };
415 "tree-sitter-nickel" = {
416 orga = "nickel-lang";
417 repo = "tree-sitter-nickel";
418 };
419 "tree-sitter-smithy" = {
420 orga = "indoorvivants";
421 repo = "tree-sitter-smithy";
422 };
423 "tree-sitter-jsonnet" = {
424 orga = "sourcegraph";
425 repo = "tree-sitter-jsonnet";
426 };
427 "tree-sitter-solidity" = {
428 orga = "JoranHonig";
429 repo = "tree-sitter-solidity";
430 };
431 "tree-sitter-nu" = {
432 orga = "nushell";
433 repo = "tree-sitter-nu";
434 };
435 "tree-sitter-cue" = {
436 orga = "eonpatapon";
437 repo = "tree-sitter-cue";
438 };
439 "tree-sitter-uiua" = {
440 orga = "shnarazk";
441 repo = "tree-sitter-uiua";
442 };
443 "tree-sitter-wing" = {
444 orga = "winglang";
445 repo = "tree-sitter-wing";
446 };
447 "tree-sitter-wgsl" = {
448 orga = "szebniok";
449 repo = "tree-sitter-wgsl";
450 };
451 "tree-sitter-templ" = {
452 orga = "vrischmann";
453 repo = "tree-sitter-templ";
454 };
455 "tree-sitter-gleam" = {
456 orga = "gleam-lang";
457 repo = "tree-sitter-gleam";
458 };
459 "tree-sitter-koka" = {
460 orga = "mtoohey31";
461 repo = "tree-sitter-koka";
462 };
463 "tree-sitter-earthfile" = {
464 orga = "glehmann";
465 repo = "tree-sitter-earthfile";
466 };
467 "tree-sitter-river" = {
468 orga = "grafana";
469 repo = "tree-sitter-river";
470 };
471 "tree-sitter-twig" = {
472 orga = "kaermorchen";
473 repo = "tree-sitter-twig";
474 };
475 "tree-sitter-hyprlang" = {
476 orga = "tree-sitter-grammars";
477 repo = "tree-sitter-hyprlang";
478 };
479 "tree-sitter-kdl" = {
480 orga = "tree-sitter-grammars";
481 repo = "tree-sitter-kdl";
482 };
483 "tree-sitter-tera" = {
484 orga = "uncenter";
485 repo = "tree-sitter-tera";
486 };
487 "tree-sitter-netlinx" = {
488 orga = "norgate-av";
489 repo = "tree-sitter-netlinx";
490 };
491 "tree-sitter-crystal" = {
492 orga = "crystal-lang-tools";
493 repo = "tree-sitter-crystal";
494 };
495 };
496
497 pinnedGrammars = [
498 "tree-sitter-bash"
499 "tree-sitter-bibtex"
500 "tree-sitter-c"
501 "tree-sitter-comment"
502 "tree-sitter-fortran"
503 "tree-sitter-hcl"
504 "tree-sitter-hyprlang"
505 "tree-sitter-llvm"
506 "tree-sitter-markdown"
507 "tree-sitter-query"
508 "tree-sitter-rust"
509 ];
510 pinnedGrammarsJson = jsonFile pinnedGrammars;
511
512 allGrammars =
513 let
514 treeSitterOrgaGrammars = lib.listToAttrs (
515 map (repo: {
516 name = repo;
517 value = {
518 orga = "tree-sitter";
519 inherit repo;
520 };
521 }) knownTreeSitterOrgGrammarRepos
522 );
523
524 in
525 lib.attrsets.unionOfDisjoint otherGrammars treeSitterOrgaGrammars;
526
527 jsonFile = name: val: (formats.json { }).generate name val;
528
529 # implementation of the updater
530 updateImpl =
531 passArgs "updateImpl-with-args"
532 {
533 binaries = {
534 curl = "${curl}/bin/curl";
535 nix-prefetch-git = "${nix-prefetch-git}/bin/nix-prefetch-git";
536 printf = "${coreutils}/bin/printf";
537 };
538 inherit
539 knownTreeSitterOrgGrammarRepos
540 ignoredTreeSitterOrgRepos
541 pinnedGrammars
542 ;
543 }
544 (
545 writers.writePython3 "updateImpl" {
546 flakeIgnore = [ "E501" ];
547 } ./update_impl.py
548 );
549
550 # Pass the given arguments to the command, in the ARGS environment variable.
551 # The arguments are just a json object that should be available in the script.
552 passArgs =
553 name: argAttrs: script:
554 writeShellScript name ''
555 env ARGS="$(< ${jsonFile "${name}-args" argAttrs})" \
556 ${script} "$@"
557 '';
558
559 foreachSh =
560 attrs: f: lib.concatMapStringsSep "\n" f (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
561
562 jsonNewlines = lib.concatMapStringsSep "\n" (lib.generators.toJSON { });
563
564 # Run the given script for each of the attr list.
565 # The attrs are passed to the script as a json value.
566 forEachParallel =
567 name: script: listOfAttrs:
568 writeShellScript "for-each-parallel.sh" ''
569 < ${writeText "${name}.json" (jsonNewlines listOfAttrs)} \
570 ${xe}/bin/xe -F -j5 ${script} {}
571 '';
572
573 # The output directory in the current source tree.
574 # This will depend on your local environment, but that is intentional.
575 outputDir = "${toString ./.}/grammars";
576
577 update-all-grammars = writeShellScript "update-all-grammars.sh" ''
578 set -euo pipefail
579 ${updateImpl} fetch-and-check-tree-sitter-repos '{}'
580 echo "writing files to ${outputDir}" 1>&2
581 mkdir -p "${outputDir}"
582 ${forEachParallel "repos-to-fetch"
583 (writeShellScript "fetch-repo" ''
584 ${updateImpl} fetch-repo "$1"
585 '')
586 (
587 lib.mapAttrsToList (
588 nixRepoAttrName: attrs:
589 attrs
590 // {
591 inherit
592 nixRepoAttrName
593 outputDir
594 ;
595 }
596 ) allGrammars
597 )
598 }
599 ${updateImpl} print-all-grammars-nix-file "$(< ${
600 jsonFile "all-grammars.json" {
601 allGrammars = lib.mapAttrsToList (
602 nixRepoAttrName: attrs:
603 attrs
604 // {
605 inherit nixRepoAttrName;
606 }
607 ) allGrammars;
608 inherit outputDir;
609 }
610 })"
611 '';
612
613in
614update-all-grammars