Commit
Author: Kevin Schoon [me@kevinschoon.com]
Hash: 3602b3b95a1ad4092b90da527c0b9a6fdd438f25
Timestamp: Sat, 15 Jul 2023 09:53:56 +0000 (1 year ago)

+199 -0 +/-4 browse
init
1diff --git a/.gitignore b/.gitignore
2new file mode 100644
3index 0000000..a173085
4--- /dev/null
5+++ b/.gitignore
6 @@ -0,0 +1 @@
7+ .ccls-cache
8 diff --git a/.gitmodules b/.gitmodules
9new file mode 100644
10index 0000000..ba3e6d3
11--- /dev/null
12+++ b/.gitmodules
13 @@ -0,0 +1,92 @@
14+ [submodule "grammars/tree-sitter-python"]
15+ path = grammars/tree-sitter-python
16+ url = https://github.com/tree-sitter/tree-sitter-python.git
17+ shallow = true
18+ [submodule "grammars/tree-sitter-sql"]
19+ path = grammars/tree-sitter-sql
20+ url = https://github.com/DerekStride/tree-sitter-sql.git
21+ shallow = true
22+ [submodule "grammars/tree-sitter-markdown"]
23+ path = grammars/tree-sitter-markdown
24+ url = https://github.com/MDeiml/tree-sitter-markdown.git
25+ shallow = true
26+ [submodule "grammars/tree-sitter-bash"]
27+ path = grammars/tree-sitter-bash
28+ url = https://github.com/tree-sitter/tree-sitter-bash.git
29+ shallow = true
30+ [submodule "grammars/tree-sitter-go"]
31+ path = grammars/tree-sitter-go
32+ url = https://github.com/tree-sitter/tree-sitter-go.git
33+ shallow = true
34+ [submodule "grammars/tree-sitter-rust"]
35+ path = grammars/tree-sitter-rust
36+ url = https://github.com/tree-sitter/tree-sitter-rust.git
37+ shallow = true
38+ [submodule "grammars/tree-sitter-ocaml"]
39+ path = grammars/tree-sitter-ocaml
40+ url = https://github.com/tree-sitter/tree-sitter-ocaml.git
41+ shallow = true
42+ [submodule "grammars/tree-sitter-regex"]
43+ path = grammars/tree-sitter-regex
44+ url = https://github.com/tree-sitter/tree-sitter-regex.git
45+ shallow = true
46+ [submodule "grammars/tree-sitter-java"]
47+ path = grammars/tree-sitter-java
48+ url = https://github.com/tree-sitter/tree-sitter-java.git
49+ shallow = true
50+ [submodule "grammars/tree-sitter-json"]
51+ path = grammars/tree-sitter-json
52+ url = https://github.com/tree-sitter/tree-sitter-json.git
53+ shallow = true
54+ [submodule "grammars/tree-sitter-html"]
55+ path = grammars/tree-sitter-html
56+ url = https://github.com/tree-sitter/tree-sitter-html.git
57+ shallow = true
58+ [submodule "grammars/tree-sitter-verilog"]
59+ path = grammars/tree-sitter-verilog
60+ url = https://github.com/tree-sitter/tree-sitter-verilog.git
61+ shallow = true
62+ [submodule "grammars/tree-sitter-julia"]
63+ path = grammars/tree-sitter-julia
64+ url = https://github.com/tree-sitter/tree-sitter-julia.git
65+ shallow = true
66+ [submodule "grammars/tree-sitter-scala"]
67+ path = grammars/tree-sitter-scala
68+ url = https://github.com/tree-sitter/tree-sitter-scala.git
69+ shallow = true
70+ [submodule "grammars/tree-sitter-typescript"]
71+ path = grammars/tree-sitter-typescript
72+ url = https://github.com/tree-sitter/tree-sitter-typescript.git
73+ shallow = true
74+ [submodule "grammars/tree-sitter-php"]
75+ path = grammars/tree-sitter-php
76+ url = https://github.com/tree-sitter/tree-sitter-php.git
77+ shallow = true
78+ [submodule "grammars/tree-sitter-c-sharp"]
79+ path = grammars/tree-sitter-c-sharp
80+ url = https://github.com/tree-sitter/tree-sitter-c-sharp.git
81+ shallow = true
82+ [submodule "grammars/tree-sitter-toml"]
83+ path = grammars/tree-sitter-toml
84+ url = https://github.com/tree-sitter/tree-sitter-toml.git
85+ shallow = true
86+ [submodule "grammars/tree-sitter-haskell"]
87+ path = grammars/tree-sitter-haskell
88+ url = https://github.com/tree-sitter/tree-sitter-haskell.git
89+ shallow = true
90+ [submodule "grammars/tree-sitter-ruby"]
91+ path = grammars/tree-sitter-ruby
92+ url = https://github.com/tree-sitter/tree-sitter-ruby.git
93+ shallow = true
94+ [submodule "grammars/tree-sitter-diff"]
95+ path = grammars/tree-sitter-diff
96+ url = https://github.com/the-mikedavis/tree-sitter-diff.git
97+ shallow = true
98+ [submodule "grammars/tree-sitter-javascript"]
99+ path = grammars/tree-sitter-javascript
100+ url = https://github.com/tree-sitter/tree-sitter-javascript.git
101+ shallow = true
102+ [submodule "grammars/tree-sitter-c"]
103+ path = grammars/tree-sitter-c
104+ url = https://github.com/tree-sitter/tree-sitter-c.git
105+ shallow = true
106 diff --git a/README.md b/README.md
107new file mode 100644
108index 0000000..1dc3c73
109--- /dev/null
110+++ b/README.md
111 @@ -0,0 +1,4 @@
112+ # tree-sitter-amalgamation
113+
114+ All tree-sitter parsers imported as submodules and compiled for easy use
115+ in downstream packages.
116 diff --git a/generate.py b/generate.py
117new file mode 100755
118index 0000000..f28b7ad
119--- /dev/null
120+++ b/generate.py
121 @@ -0,0 +1,102 @@
122+ #!/usr/bin/env python
123+
124+ import glob
125+ import os
126+ import shutil
127+ import subprocess
128+ from collections import namedtuple
129+
130+ recompile = False
131+
132+ Source = namedtuple("Source", ["name", "path", "includes", "highlights"])
133+
134+ sources = [
135+ Source("bash", "grammars/tree-sitter-bash", [], "queries"),
136+ Source("c", "grammars/tree-sitter-c", [], "queries"),
137+ Source("c-sharp","grammars/tree-sitter-c-sharp", [], "queries"),
138+ Source("diff","grammars/tree-sitter-diff", [], "queries"),
139+ Source("go","grammars/tree-sitter-go", [], "queries"),
140+ Source("haskell", "grammars/tree-sitter-haskell", [], "queries"),
141+ Source("html","grammars/tree-sitter-html", [], "queries"),
142+ Source("java","grammars/tree-sitter-java", [], "queries"),
143+ Source("javascript","grammars/tree-sitter-javascript", [], "queries"),
144+ Source("json","grammars/tree-sitter-json", [], "queries"),
145+ Source("julia","grammars/tree-sitter-julia", [], "queries"),
146+ Source("markdown","grammars/tree-sitter-markdown/tree-sitter-markdown", [], None),
147+ Source("markdown-inline","grammars/tree-sitter-markdown/tree-sitter-markdown-inline", [], "queries"),
148+ Source("ocaml","grammars/tree-sitter-ocaml/ocaml", [("common", "../../common")], "../queries"),
149+ Source("php","grammars/tree-sitter-php", [], "queries"),
150+ Source("python","grammars/tree-sitter-python", [], "queries"),
151+ Source("regex","grammars/tree-sitter-regex", [], "queries"),
152+ Source("ruby","grammars/tree-sitter-ruby", [], "queries"),
153+ Source("rust","grammars/tree-sitter-rust", [], "queries"),
154+ Source("scala","grammars/tree-sitter-scala", [], "queries/scala"),
155+ Source("sql","grammars/tree-sitter-sql", [], "queries") ,
156+ Source("toml","grammars/tree-sitter-toml", [], "queries"),
157+ Source("typescript","grammars/tree-sitter-typescript/typescript", [("common", "../../common")], "queries"),
158+ Source("tsx","grammars/tree-sitter-typescript/tsx", [("common", "../../common")], "queries"),
159+ Source("verilog","grammars/tree-sitter-verilog", [], "queries"),
160+ ]
161+
162+ def _fix_scanner_include():
163+ current_dir = os.path.abspath(".")
164+ scanner_path = f"{current_dir}/scanner.c"
165+ # yep this is what im doing
166+ fix = """s/..\/..\/common\/scanner.h/common\/scanner.h/g"""
167+ subprocess.call(["sed", "-i", fix, scanner_path])
168+
169+ def _hack_bash(src):
170+ pass
171+
172+ def _hack_ocaml(src):
173+ _fix_scanner_include()
174+
175+ def _hack_typescript(src):
176+ _fix_scanner_include()
177+
178+ def _hack_tsx(src):
179+ _fix_scanner_include()
180+
181+
182+ # some hand written scanners use relative paths that are picked up by the
183+ # tree-sitter binary but not by build systems
184+ hacks = {
185+ "bash": _hack_bash,
186+ "ocaml": _hack_ocaml,
187+ "typescript": _hack_typescript,
188+ "tsx": _hack_tsx,
189+ }
190+
191+ def compile_all():
192+ if not os.path.exists("target"):
193+ os.mkdir("target")
194+ for source in sources:
195+ print(source.name, source.path)
196+ target = f"target/{source.name}"
197+ source_dir = f"{source.path}/src"
198+ parser_path = f"{source_dir}/parser.c"
199+ if not os.path.exists(parser_path) or recompile:
200+ subprocess.check_call(["tree-sitter", "generate"], cwd=source.name)
201+ shutil.rmtree(target, ignore_errors=True)
202+ shutil.copytree(source_dir, target)
203+ for extra_dir in source.includes:
204+ extra = os.path.join(source_dir, extra_dir[1])
205+ shutil.copytree(extra, os.path.join(target, extra_dir[0]))
206+ # seemingly a convention used in parsers support "related" languages
207+ # like ocaml interfaces
208+ common_path = f"{source_dir}/common"
209+ if os.path.exists(common_path):
210+ shutil.copytree(common_path, target)
211+ if source.highlights:
212+ highlights_path = os.path.join(source.path, source.highlights)
213+ if os.path.exists(highlights_path):
214+ shutil.copytree(highlights_path, target + "/queries")
215+ current_dir = os.path.abspath(".")
216+ if source.name in hacks:
217+ os.chdir(target)
218+ hacks[source.name](source.path)
219+ os.chdir(current_dir)
220+
221+
222+ if __name__ == "__main__":
223+ compile_all()