Commit
Author: Kevin Schoon [me@kevinschoon.com]
Hash: 63ec8ac10780b1838e6f769a5e15d03fa12827c4
Timestamp: Tue, 12 Sep 2023 16:08:24 +0000 (1 year ago)

+185 -408 +/-9 browse
split up build steps 
split up build steps

Split build steps into three seperate stages:

generate  # generate tree sitter parsers
package   # prepare the target dir for compiling and packaging
compile   # compile the shared libraries themselves
1diff --git a/.gitignore b/.gitignore
2index 3189293..c52dd7c 100644
3--- a/.gitignore
4+++ b/.gitignore
5 @@ -1,3 +1,4 @@
6 .ccls-cache
7 target
8 .ninja_log
9+ *.ninja
10 diff --git a/README.md b/README.md
11index ef558f0..2439034 100644
12--- a/README.md
13+++ b/README.md
14 @@ -29,7 +29,12 @@ as well as an ANSI C compiler.
15
16 ```sh
17 git submodule update --init --recursive
18- ninja
19+ # generate tree-sitter parsers
20+ ./generate.py
21+ # package all of the parsers into the target directory
22+ ./package.py
23+ # compile shared parser objects
24+ ./compile.py
25 ```
26
27 ## Adding New Languages
28 diff --git a/build.ninja b/build.ninja
29deleted file mode 100644
30index fe7ee53..0000000
31--- a/build.ninja
32+++ /dev/null
33 @@ -1,269 +0,0 @@
34- # automatically generated, do not edit.
35- cc = cc -shared -fno-exceptions -g -fPIC
36- rule copy
37- command = cp $in $out
38- rule mkdir
39- command = mkdir -p $out
40- rule ts-compile
41- command = cd $in && tree-sitter generate
42- rule fix-scanner-include
43- command = cat $in | sed "s/..\/..\/common\/scanner.h/common\/scanner.h/g" $
44- > $out
45- build target: mkdir
46- build target/bash: mkdir
47- build target/bash/parser.c: copy grammars/tree-sitter-bash/src/parser.c
48- build target/bash/scanner.c: copy grammars/tree-sitter-bash/src/scanner.c
49- build target/bash/queries/highlights.scm: copy $
50- grammars/tree-sitter-bash/src/../queries/highlights.scm
51- rule compile_bash
52- command = $cc $in -o $out
53- build target/bash/bash.so: compile_bash target/bash/parser.c $
54- target/bash/scanner.c
55- build target/c: mkdir
56- build target/c/parser.c: copy grammars/tree-sitter-c/src/parser.c
57- build target/c/queries/highlights.scm: copy $
58- grammars/tree-sitter-c/src/../queries/highlights.scm
59- rule compile_c
60- command = $cc $in -o $out
61- build target/c/c.so: compile_c target/c/parser.c
62- build target/c-sharp: mkdir
63- build target/c-sharp/parser.c: copy grammars/tree-sitter-c-sharp/src/parser.c
64- build target/c-sharp/scanner.c: copy $
65- grammars/tree-sitter-c-sharp/src/scanner.c
66- build target/c-sharp/queries/tags.scm: copy $
67- grammars/tree-sitter-c-sharp/src/../queries/tags.scm
68- build target/c-sharp/queries/highlights.scm: copy $
69- grammars/tree-sitter-c-sharp/src/../queries/highlights.scm
70- rule compile_c-sharp
71- command = $cc $in -o $out
72- build target/c-sharp/c-sharp.so: compile_c-sharp target/c-sharp/parser.c $
73- target/c-sharp/scanner.c
74- build target/diff: mkdir
75- build target/diff/parser.c: copy grammars/tree-sitter-diff/src/parser.c
76- build target/diff/queries/highlights.scm: copy $
77- grammars/tree-sitter-diff/src/../queries/highlights.scm
78- rule compile_diff
79- command = $cc $in -o $out
80- build target/diff/diff.so: compile_diff target/diff/parser.c
81- build target/go: mkdir
82- build target/go/parser.c: copy grammars/tree-sitter-go/src/parser.c
83- build target/go/queries/tags.scm: copy $
84- grammars/tree-sitter-go/src/../queries/tags.scm
85- build target/go/queries/highlights.scm: copy $
86- grammars/tree-sitter-go/src/../queries/highlights.scm
87- build target/go/queries/structure.scm: copy $
88- grammars/tree-sitter-go/src/../queries/structure.scm
89- rule compile_go
90- command = $cc $in -o $out
91- build target/go/go.so: compile_go target/go/parser.c
92- build target/haskell: mkdir
93- build target/haskell/parser.c: copy grammars/tree-sitter-haskell/src/parser.c
94- build target/haskell/scanner.c: copy $
95- grammars/tree-sitter-haskell/src/scanner.c
96- build target/haskell/unicode.h: copy $
97- grammars/tree-sitter-haskell/src/unicode.h
98- build target/haskell/queries/locals.scm: copy $
99- grammars/tree-sitter-haskell/src/../queries/locals.scm
100- build target/haskell/queries/highlights.scm: copy $
101- grammars/tree-sitter-haskell/src/../queries/highlights.scm
102- rule compile_haskell
103- command = $cc $in -o $out
104- build target/haskell/haskell.so: compile_haskell target/haskell/parser.c $
105- target/haskell/scanner.c
106- build target/html: mkdir
107- build target/html/parser.c: copy grammars/tree-sitter-html/src/parser.c
108- build target/html/scanner.c: copy grammars/tree-sitter-html/src/scanner.c
109- build target/html/tag.h: copy grammars/tree-sitter-html/src/tag.h
110- build target/html/queries/injections.scm: copy $
111- grammars/tree-sitter-html/src/../queries/injections.scm
112- build target/html/queries/highlights.scm: copy $
113- grammars/tree-sitter-html/src/../queries/highlights.scm
114- rule compile_html
115- command = $cc $in -o $out
116- build target/html/html.so: compile_html target/html/parser.c $
117- target/html/scanner.c
118- build target/java: mkdir
119- build target/java/parser.c: copy grammars/tree-sitter-java/src/parser.c
120- build target/java/queries/tags.scm: copy $
121- grammars/tree-sitter-java/src/../queries/tags.scm
122- build target/java/queries/highlights.scm: copy $
123- grammars/tree-sitter-java/src/../queries/highlights.scm
124- rule compile_java
125- command = $cc $in -o $out
126- build target/java/java.so: compile_java target/java/parser.c
127- build target/javascript: mkdir
128- build target/javascript/parser.c: copy $
129- grammars/tree-sitter-javascript/src/parser.c
130- build target/javascript/scanner.c: copy $
131- grammars/tree-sitter-javascript/src/scanner.c
132- build target/javascript/queries/tags.scm: copy $
133- grammars/tree-sitter-javascript/src/../queries/tags.scm
134- build target/javascript/queries/highlights-params.scm: copy $
135- grammars/tree-sitter-javascript/src/../queries/highlights-params.scm
136- build target/javascript/queries/injections.scm: copy $
137- grammars/tree-sitter-javascript/src/../queries/injections.scm
138- build target/javascript/queries/locals.scm: copy $
139- grammars/tree-sitter-javascript/src/../queries/locals.scm
140- build target/javascript/queries/highlights.scm: copy $
141- grammars/tree-sitter-javascript/src/../queries/highlights.scm
142- build target/javascript/queries/highlights-jsx.scm: copy $
143- grammars/tree-sitter-javascript/src/../queries/highlights-jsx.scm
144- rule compile_javascript
145- command = $cc $in -o $out
146- build target/javascript/javascript.so: compile_javascript $
147- target/javascript/parser.c target/javascript/scanner.c
148- build target/json: mkdir
149- build target/json/parser.c: copy grammars/tree-sitter-json/src/parser.c
150- build target/json/queries/highlights.scm: copy $
151- grammars/tree-sitter-json/src/../queries/highlights.scm
152- rule compile_json
153- command = $cc $in -o $out
154- build target/json/json.so: compile_json target/json/parser.c
155- build target/julia: mkdir
156- build target/julia/parser.c: copy grammars/tree-sitter-julia/src/parser.c
157- build target/julia/scanner.c: copy grammars/tree-sitter-julia/src/scanner.c
158- rule compile_julia
159- command = $cc $in -o $out
160- build target/julia/julia.so: compile_julia target/julia/parser.c $
161- target/julia/scanner.c
162- build target/markdown: mkdir
163- build target/markdown/parser.c: copy $
164- grammars/tree-sitter-markdown/tree-sitter-markdown/src/parser.c
165- build target/markdown/scanner.c: copy $
166- grammars/tree-sitter-markdown/tree-sitter-markdown/src/scanner.c
167- build target/markdown/queries/injections.scm: copy $
168- grammars/tree-sitter-markdown/tree-sitter-markdown/queries/injections.scm
169- build target/markdown/queries/highlights.scm: copy $
170- grammars/tree-sitter-markdown/tree-sitter-markdown/queries/highlights.scm
171- rule compile_markdown
172- command = $cc $in -o $out
173- build target/markdown/markdown.so: compile_markdown $
174- target/markdown/parser.c target/markdown/scanner.c
175- build target/ocaml: mkdir
176- build target/ocaml/parser.c: copy $
177- grammars/tree-sitter-ocaml/ocaml/src/parser.c
178- build target/ocaml/scanner.c: fix-scanner-include $
179- grammars/tree-sitter-ocaml/ocaml/src/scanner.c
180- build target/ocaml/common/scanner.h: copy $
181- grammars/tree-sitter-ocaml/common/scanner.h
182- build target/ocaml/queries/tags.scm: copy $
183- grammars/tree-sitter-ocaml/queries/tags.scm
184- build target/ocaml/queries/locals.scm: copy $
185- grammars/tree-sitter-ocaml/queries/locals.scm
186- build target/ocaml/queries/highlights.scm: copy $
187- grammars/tree-sitter-ocaml/queries/highlights.scm
188- rule compile_ocaml
189- command = $cc $in -o $out
190- build target/ocaml/ocaml.so: compile_ocaml target/ocaml/parser.c $
191- target/ocaml/scanner.c
192- build target/php: mkdir
193- build target/php/parser.c: copy grammars/tree-sitter-php/src/parser.c
194- build target/php/scanner.c: copy grammars/tree-sitter-php/src/scanner.c
195- build target/php/queries/tags.scm: copy $
196- grammars/tree-sitter-php/src/../queries/tags.scm
197- build target/php/queries/injections.scm: copy $
198- grammars/tree-sitter-php/src/../queries/injections.scm
199- build target/php/queries/highlights.scm: copy $
200- grammars/tree-sitter-php/src/../queries/highlights.scm
201- rule compile_php
202- command = $cc $in -o $out
203- build target/php/php.so: compile_php target/php/parser.c target/php/scanner.c
204- build target/python: mkdir
205- build target/python/parser.c: copy grammars/tree-sitter-python/src/parser.c
206- build target/python/scanner.c: copy grammars/tree-sitter-python/src/scanner.c
207- build target/python/queries/tags.scm: copy $
208- grammars/tree-sitter-python/src/../queries/tags.scm
209- build target/python/queries/highlights.scm: copy $
210- grammars/tree-sitter-python/src/../queries/highlights.scm
211- rule compile_python
212- command = $cc $in -o $out
213- build target/python/python.so: compile_python target/python/parser.c $
214- target/python/scanner.c
215- build target/regex: mkdir
216- build target/regex/parser.c: copy grammars/tree-sitter-regex/src/parser.c
217- build target/regex/queries/highlights.scm: copy $
218- grammars/tree-sitter-regex/src/../queries/highlights.scm
219- rule compile_regex
220- command = $cc $in -o $out
221- build target/regex/regex.so: compile_regex target/regex/parser.c
222- build target/rust: mkdir
223- build target/rust/parser.c: copy grammars/tree-sitter-rust/src/parser.c
224- build target/rust/scanner.c: copy grammars/tree-sitter-rust/src/scanner.c
225- build target/rust/queries/tags.scm: copy $
226- grammars/tree-sitter-rust/src/../queries/tags.scm
227- build target/rust/queries/injections.scm: copy $
228- grammars/tree-sitter-rust/src/../queries/injections.scm
229- build target/rust/queries/highlights.scm: copy $
230- grammars/tree-sitter-rust/src/../queries/highlights.scm
231- rule compile_rust
232- command = $cc $in -o $out
233- build target/rust/rust.so: compile_rust target/rust/parser.c $
234- target/rust/scanner.c
235- build target/scala: mkdir
236- build target/scala/parser.c: copy grammars/tree-sitter-scala/src/parser.c
237- build target/scala/scanner.c: copy grammars/tree-sitter-scala/src/scanner.c
238- build target/scala/stack.h: copy grammars/tree-sitter-scala/src/stack.h
239- build target/scala/queries/locals.scm: copy $
240- grammars/tree-sitter-scala/queries/scala/locals.scm
241- build target/scala/queries/highlights.scm: copy $
242- grammars/tree-sitter-scala/queries/scala/highlights.scm
243- rule compile_scala
244- command = $cc $in -o $out
245- build target/scala/scala.so: compile_scala target/scala/parser.c $
246- target/scala/scanner.c
247- build target/sql: mkdir
248- build target/sql/parser.c: copy grammars/tree-sitter-sql/src/parser.c
249- build target/sql/queries/indents.scm: copy $
250- grammars/tree-sitter-sql/src/../queries/indents.scm
251- build target/sql/queries/highlights.scm: copy $
252- grammars/tree-sitter-sql/src/../queries/highlights.scm
253- rule compile_sql
254- command = $cc $in -o $out
255- build target/sql/sql.so: compile_sql target/sql/parser.c
256- build target/toml: mkdir
257- build target/toml/parser.c: copy grammars/tree-sitter-toml/src/parser.c
258- build target/toml/scanner.c: copy grammars/tree-sitter-toml/src/scanner.c
259- build target/toml/queries/highlights.scm: copy $
260- grammars/tree-sitter-toml/src/../queries/highlights.scm
261- rule compile_toml
262- command = $cc $in -o $out
263- build target/toml/toml.so: compile_toml target/toml/parser.c $
264- target/toml/scanner.c
265- build target/typescript: mkdir
266- build target/typescript/parser.c: copy $
267- grammars/tree-sitter-typescript/typescript/src/parser.c
268- build target/typescript/scanner.c: fix-scanner-include $
269- grammars/tree-sitter-typescript/typescript/src/scanner.c
270- build target/typescript/common/scanner.h: copy $
271- grammars/tree-sitter-typescript/common/scanner.h
272- build target/typescript/queries/tags.scm: copy $
273- grammars/tree-sitter-typescript/queries/tags.scm
274- build target/typescript/queries/locals.scm: copy $
275- grammars/tree-sitter-typescript/queries/locals.scm
276- build target/typescript/queries/highlights.scm: copy $
277- grammars/tree-sitter-typescript/queries/highlights.scm
278- rule compile_typescript
279- command = $cc $in -o $out
280- build target/typescript/typescript.so: compile_typescript $
281- target/typescript/parser.c target/typescript/scanner.c
282- build target/tsx: mkdir
283- build target/tsx/parser.c: copy $
284- grammars/tree-sitter-typescript/tsx/src/parser.c
285- build target/tsx/scanner.c: fix-scanner-include $
286- grammars/tree-sitter-typescript/tsx/src/scanner.c
287- build target/tsx/common/scanner.h: copy $
288- grammars/tree-sitter-typescript/common/scanner.h
289- build target/tsx/queries/tags.scm: copy $
290- grammars/tree-sitter-typescript/queries/tags.scm
291- build target/tsx/queries/locals.scm: copy $
292- grammars/tree-sitter-typescript/queries/locals.scm
293- build target/tsx/queries/highlights.scm: copy $
294- grammars/tree-sitter-typescript/queries/highlights.scm
295- rule compile_tsx
296- command = $cc $in -o $out
297- build target/tsx/tsx.so: compile_tsx target/tsx/parser.c target/tsx/scanner.c
298- build target/verilog: mkdir
299- build target/verilog/parser.c: copy grammars/tree-sitter-verilog/src/parser.c
300- rule compile_verilog
301- command = $cc $in -o $out
302- build target/verilog/verilog.so: compile_verilog target/verilog/parser.c
303 diff --git a/build.py b/build.py
304deleted file mode 100755
305index 71e6c64..0000000
306--- a/build.py
307+++ /dev/null
308 @@ -1,138 +0,0 @@
309- #!/usr/bin/env python
310-
311- from glob import glob
312- from collections import namedtuple
313- from os import path
314-
315- import ninja_syntax
316-
317- # name of project and array of tuples where the first element is all of
318- # the files to include and the second element is the target directory to copy
319- # them into. If the second element is None it will go to the root path
320- Source = namedtuple("Source", ["name", "includes"])
321-
322-
323- # default source layout in use for most tree-sitter projects
324- def _default_config(name, basepath):
325- return Source(
326- name,
327- [
328- (glob(f"{basepath}/*.c"), None),
329- (glob(f"{basepath}/*.h"), None),
330- (glob(f"{basepath}/../queries/*.scm"), "queries"),
331- ],
332- )
333-
334-
335- sources = [
336- _default_config("bash", "grammars/tree-sitter-bash/src"),
337- _default_config("c", "grammars/tree-sitter-c/src"),
338- _default_config("c-sharp", "grammars/tree-sitter-c-sharp/src"),
339- _default_config("diff", "grammars/tree-sitter-diff/src"),
340- _default_config("go", "grammars/tree-sitter-go/src"),
341- _default_config("haskell", "grammars/tree-sitter-haskell/src"),
342- _default_config("html", "grammars/tree-sitter-html/src"),
343- _default_config("java", "grammars/tree-sitter-java/src"),
344- _default_config("javascript", "grammars/tree-sitter-javascript/src"),
345- _default_config("json", "grammars/tree-sitter-json/src"),
346- _default_config("julia", "grammars/tree-sitter-julia/src"),
347- Source(
348- "markdown",
349- [
350- (glob("grammars/tree-sitter-markdown/tree-sitter-markdown/src/*.c"), None),
351- (
352- glob(
353- "grammars/tree-sitter-markdown/tree-sitter-markdown/queries/*.scm"
354- ),
355- "queries",
356- ),
357- ],
358- ),
359- Source(
360- "ocaml",
361- [
362- (glob("grammars/tree-sitter-ocaml/ocaml/src/*.c"), None),
363- (glob("grammars/tree-sitter-ocaml/common/*.h"), "common"),
364- (glob("grammars/tree-sitter-ocaml/queries/*.scm"), "queries"),
365- ],
366- ),
367- _default_config("php", "grammars/tree-sitter-php/src"),
368- _default_config("python", "grammars/tree-sitter-python/src"),
369- _default_config("regex", "grammars/tree-sitter-regex/src"),
370- # TODO: ruby is using a cpp scanner and i'm not going to support it for now
371- # _default_config("ruby", "grammars/tree-sitter-ruby"),
372- _default_config("rust", "grammars/tree-sitter-rust/src"),
373- Source(
374- "scala",
375- [
376- (glob("grammars/tree-sitter-scala/src/*.c"), None),
377- (glob("grammars/tree-sitter-scala/src/*.h"), None),
378- (glob("grammars/tree-sitter-scala/queries/scala/*.scm"), "queries"),
379- ],
380- ),
381- _default_config("sql", "grammars/tree-sitter-sql/src"),
382- _default_config("toml", "grammars/tree-sitter-toml/src"),
383- Source(
384- "typescript",
385- [
386- (glob("grammars/tree-sitter-typescript/typescript/src/*.c"), None),
387- (glob("grammars/tree-sitter-typescript/typescript/src/*.h"), None),
388- (glob("grammars/tree-sitter-typescript/common/*.h"), "common"),
389- (glob("grammars/tree-sitter-typescript/queries/*.scm"), "queries"),
390- ],
391- ),
392- Source(
393- "tsx",
394- [
395- (glob("grammars/tree-sitter-typescript/tsx/src/*.c"), None),
396- (glob("grammars/tree-sitter-typescript/tsx/src/*.h"), None),
397- (glob("grammars/tree-sitter-typescript/common/*.h"), "common"),
398- (glob("grammars/tree-sitter-typescript/queries/*.scm"), "queries"),
399- ],
400- ),
401- _default_config("verilog", "grammars/tree-sitter-verilog/src"),
402- ]
403-
404- # these projects require an include fix
405- scanner_hacks = ["ocaml", "typescript", "tsx"]
406-
407-
408- if __name__ == "__main__":
409- with open("build.ninja", "w") as fp:
410- w = ninja_syntax.Writer(fp)
411- w.comment("automatically generated, do not edit.")
412- w.variable("cc", "cc -shared -fno-exceptions -g -fPIC")
413- w.rule("copy", command="cp $in $out")
414- w.rule("mkdir", command="mkdir -p $out")
415- w.rule("ts-compile", command="cd $in && tree-sitter generate")
416- sed_fix = "s/..\/..\/common\/scanner.h/common\/scanner.h/g"
417- w.rule("fix-scanner-include", command=f'cat $in | sed "{sed_fix}" > $out')
418- w.build("target", "mkdir")
419- for source in sources:
420- target_root = path.join("target", source.name)
421- w.build(target_root, "mkdir")
422- # NOTE: all parsers currently have committed their generated code
423- # so there isn't any reason to run this right now.
424- # w.build(parser_path, "ts-compile", [source.path])
425- compile_inputs = []
426- for include in source.includes:
427- if include[1]:
428- target_dir = path.join(target_root, include[1])
429- else:
430- target_dir = target_root
431- for include_path in include[0]:
432- filename = path.basename(include_path)
433- target_path = path.join(target_dir, filename)
434- # hacks for scanner.c in sub-language projects
435- if filename == "scanner.c" and source.name in scanner_hacks:
436- w.build(
437- target_path, "fix-scanner-include", inputs=[include_path]
438- )
439- else:
440- w.build(target_path, "copy", inputs=[include_path])
441- if filename in ["parser.c", "scanner.c"]:
442- compile_inputs.append(target_path)
443- lib_out = path.join(target_root, f"{source.name}.so")
444- w.rule(f"compile_{source.name}", command=f"$cc $in -o $out")
445- w.build(lib_out, f"compile_{source.name}", inputs=compile_inputs)
446- w.close()
447 diff --git a/compile.py b/compile.py
448new file mode 100755
449index 0000000..45a6e9b
450--- /dev/null
451+++ b/compile.py
452 @@ -0,0 +1,24 @@
453+ #!/usr/bin/env python
454+
455+ import subprocess
456+ import json
457+ from glob import glob
458+ from os import path
459+
460+ import ninja_syntax
461+
462+ if __name__ == "__main__":
463+ with open("grammars.json", "r") as fp:
464+ grammars = json.loads(fp.read())
465+
466+ with open("compile.ninja", "w") as fp:
467+ w = ninja_syntax.Writer(fp)
468+ w.comment("automatically generated, do not edit")
469+ w.rule("compile", "cc -shared -fno-exceptions -g -fPIC $in -o $out")
470+ for grammar in grammars:
471+ target_dir = path.join("target", grammar["name"])
472+ target_lib = path.join("target", grammar["name"], grammar["name"] + ".so")
473+ sources = glob(target_dir + "/*.c")
474+ w.build(target_lib, "compile", inputs=sources)
475+
476+ subprocess.call(["ninja", "-f", "compile.ninja"])
477 diff --git a/fixes.sed b/fixes.sed
478new file mode 100644
479index 0000000..f1e0e66
480--- /dev/null
481+++ b/fixes.sed
482 @@ -0,0 +1 @@
483+ s/..\/..\/common\/scanner.h/common\/scanner.h/g
484 diff --git a/generate.py b/generate.py
485new file mode 100755
486index 0000000..83d2ca8
487--- /dev/null
488+++ b/generate.py
489 @@ -0,0 +1,27 @@
490+ #!/usr/bin/env python
491+
492+ import subprocess
493+ import json
494+ from os import path
495+ from multiprocessing import cpu_count
496+
497+ import ninja_syntax
498+
499+ if __name__ == "__main__":
500+ with open("grammars.json", "r") as fp:
501+ grammars = json.loads(fp.read())
502+
503+ with open("generate.ninja", "w") as fp:
504+ w = ninja_syntax.Writer(fp)
505+ w.comment("automatically generated, do not edit.")
506+ w.rule("generate", "cd $in && tree-sitter generate")
507+ # NOTE: this will use a lot of memory on your system, you might decrase
508+ # the parallelism count here if you encounter issues.
509+ w.pool("generate", cpu_count())
510+ for grammar in grammars:
511+ parser_out = path.join(grammar["path"], "src/parser.c")
512+ w.build(parser_out, "generate", inputs=[grammar["path"]], pool="generate")
513+
514+ w.close()
515+
516+ subprocess.call(["ninja", "-f", "generate.ninja"])
517 diff --git a/grammars.json b/grammars.json
518new file mode 100644
519index 0000000..d1c1f3a
520--- /dev/null
521+++ b/grammars.json
522 @@ -0,0 +1,64 @@
523+ [
524+ {
525+ "name": "bash",
526+ "path": "grammars/tree-sitter-bash"
527+ },
528+ { "name": "c", "path": "grammars/tree-sitter-c" },
529+ {
530+ "name": "c-sharp",
531+ "path": "grammars/tree-sitter-c-sharp"
532+ },
533+ { "name": "diff", "path": "grammars/tree-sitter-diff" },
534+ { "name": "go", "path": "grammars/tree-sitter-go" },
535+ {
536+ "name": "haskell",
537+ "path": "grammars/tree-sitter-haskell"
538+ },
539+ { "name": "html", "path": "grammars/tree-sitter-html" },
540+ { "name": "java", "path": "grammars/tree-sitter-java" },
541+ {
542+ "name": "javascript",
543+ "path": "grammars/tree-sitter-javascript"
544+ },
545+ { "name": "json", "path": "grammars/tree-sitter-json" },
546+ { "name": "julia", "path": "grammars/tree-sitter-julia" },
547+ {
548+ "name": "markdown",
549+ "path": "grammars/tree-sitter-markdown/tree-sitter-markdown"
550+ },
551+ {
552+ "name": "markdown-inline",
553+ "path": "grammars/tree-sitter-markdown/tree-sitter-markdown-inline"
554+ },
555+ {
556+ "name": "ocaml",
557+ "path": "grammars/tree-sitter-ocaml/ocaml",
558+ "queries": "grammars/tree-sitter-ocaml/queries/*.scm",
559+ "includes": ["grammars/tree-sitter-ocaml/common/scanner.h"]
560+ },
561+ { "name": "php", "path": "grammars/tree-sitter-php" },
562+ { "name": "python", "path": "grammars/tree-sitter-python" },
563+ { "name": "regex", "path": "grammars/tree-sitter-regex" },
564+ { "name": "ruby", "path": "grammars/tree-sitter-ruby" },
565+ { "name": "rust", "path": "grammars/tree-sitter-rust" },
566+ {
567+ "name": "scala",
568+ "path": "grammars/tree-sitter-scala",
569+ "queries": "grammars/tree-sitter-scala/queries/scala/*.scm"
570+ },
571+ { "name": "sql", "path": "grammars/tree-sitter-sql" },
572+ { "name": "toml", "path": "grammars/tree-sitter-toml" },
573+ {
574+ "name": "tsx",
575+ "path": "grammars/tree-sitter-typescript/tsx",
576+ "queries": "grammars/tree-sitter-typescript/queries/*.scm",
577+ "includes": ["grammars/tree-sitter-typescript/common/scanner.h"]
578+ },
579+ {
580+ "name": "typescript",
581+ "path": "grammars/tree-sitter-typescript/typescript",
582+ "queries": "grammars/tree-sitter-typescript/queries/*.scm",
583+ "includes": ["grammars/tree-sitter-typescript/common/scanner.h"]
584+ },
585+ { "name": "verilog", "path": "grammars/tree-sitter-verilog" }
586+ ]
587 diff --git a/package.py b/package.py
588new file mode 100755
589index 0000000..c365d3a
590--- /dev/null
591+++ b/package.py
592 @@ -0,0 +1,62 @@
593+ #!/usr/bin/env python
594+
595+ import json
596+ import subprocess
597+ from glob import glob
598+ from os import path
599+
600+ import ninja_syntax
601+
602+
603+ if __name__ == "__main__":
604+ with open("grammars.json", "r") as fp:
605+ grammars = json.loads(fp.read())
606+
607+ with open("package.ninja", "w") as fp:
608+ w = ninja_syntax.Writer(fp)
609+ w.comment("automatically generated, do not edit.")
610+ w.rule("copy", command="cp $in $out")
611+ w.rule("mkdir", command="mkdir -p $out")
612+ w.rule("sed-fix", command=f"cat $in | sed -f fixes.sed > $out")
613+ w.build("target", "mkdir")
614+ for grammar in grammars:
615+ base_path = grammar["path"]
616+ target_dir = path.join("target", grammar["name"])
617+ w.build(target_dir, "mkdir")
618+ if "queries" in grammar and grammar["queries"]:
619+ queries = glob(grammar["queries"])
620+ else:
621+ queries = glob(base_path + "/queries/*.scm")
622+ for query in queries:
623+ filename = path.basename(query)
624+ query_target = path.join(target_dir, "queries", filename)
625+ w.build(query_target, "copy", inputs=[query])
626+ if "includes" in grammar and grammar["includes"]:
627+ includes = []
628+ for include in grammar["includes"]:
629+ includes += glob(include)
630+ unique_dirs = set(
631+ [
632+ path.join(target_dir, path.basename(path.dirname(include)))
633+ for include in includes
634+ ]
635+ )
636+ for unique_dir in unique_dirs:
637+ w.build(unique_dir, "mkdir", unique_dir)
638+ for include in includes:
639+ include_target_dir = path.basename(path.dirname(include))
640+ target_path = path.join(
641+ target_dir, include_target_dir, path.basename(include)
642+ )
643+ w.build(target_path, "sed-fix", inputs=[include])
644+ source_dir = path.join(base_path, "src")
645+ sources = []
646+ sources += glob(source_dir + "/*.c")
647+ sources += glob(source_dir + "/*.h")
648+ for source in sources:
649+ filename = path.basename(source)
650+ source_target = path.join(target_dir, filename)
651+ w.build(source_target, "sed-fix", inputs=[source])
652+ w.close()
653+
654+ subprocess.call(["ninja", "-f", "package.ninja"])