Commit

Author:

Hash:

Timestamp:

+3183 -1261 +/-162 browse

Kevin Schoon [me@kevinschoon.com]

6628b7b36b989c9e15331b1ab75f7b26d3f5b9e8

Sat, 27 Sep 2025 12:23:25 +0000 (1 month ago)

revive ayllu-build with common database 
revive ayllu-build with common database

This revives the ayllu-build component providing a highly primitive
and insecure engine for executing synchronous DAGs. The SQLite database
has been restored now and is exposed to the ayllu web frontend when the
[builder] configuration is enabled.
1diff --git a/.ayllu-build.json b/.ayllu-build.json
2new file mode 100644
3index 0000000..9e90a84
4--- /dev/null
5+++ b/.ayllu-build.json
6 @@ -0,0 +1,53 @@
7+ {
8+ "workflows": [
9+ {
10+ "name": "lint",
11+ "steps": [
12+ {
13+ "name": "cargo-fmt",
14+ "input": "cargo fmt --check"
15+ }
16+ ]
17+ },
18+ {
19+ "name": "test",
20+ "depends_on": [
21+ "lint"
22+ ],
23+ "steps": [
24+ {
25+ "name": "cargo-test",
26+ "input": "cargo test"
27+ },
28+ {
29+ "name": "cargo-clippy",
30+ "input": "cargo clippy"
31+ }
32+ ]
33+ },
34+ {
35+ "name": "build",
36+ "depends_on": [
37+ "test"
38+ ],
39+ "steps": [
40+ {
41+ "name": "cargo-build-ayllu",
42+ "input": "cargo build --package ayllu"
43+ },
44+ {
45+ "name": "cargo-build-ayllu-shell",
46+ "input": "cargo build --package ayllu-shell"
47+ },
48+ {
49+ "name": "cargo-build-ayllu-keys",
50+ "input": "cargo build --package ayllu-keys"
51+ },
52+ {
53+ "name": "cargo-build-ayllu-build",
54+ "input": "cargo build --package ayllu-build"
55+ }
56+ ]
57+ }
58+ ]
59+ }
60\ No newline at end of file
61 diff --git a/.ayllu/build/main.ncl b/.ayllu/build/main.ncl
62deleted file mode 100644
63index ceb7bad..0000000
64--- a/.ayllu/build/main.ncl
65+++ /dev/null
66 @@ -1,24 +0,0 @@
67- {
68- workflows = [
69- {
70- name = "Simple",
71- steps = [
72- {
73- name = "Format",
74- shell = "/bin/sh",
75- input = "cargo fmt --all --check",
76- },
77- {
78- name = "Clippy",
79- shell = "/bin/sh",
80- input = "cargo clippy --all -- --deny warnings",
81- },
82- {
83- name = "Run Tests",
84- shell = "/bin/sh",
85- input = "cargo test --all",
86- }
87- ]
88- }
89- ]
90- }
91 diff --git a/Cargo.lock b/Cargo.lock
92index 9272d64..b3a3ca9 100644
93--- a/Cargo.lock
94+++ b/Cargo.lock
95 @@ -27,6 +27,12 @@ dependencies = [
96 ]
97
98 [[package]]
99+ name = "allocator-api2"
100+ version = "0.2.21"
101+ source = "registry+https://github.com/rust-lang/crates.io-index"
102+ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
103+
104+ [[package]]
105 name = "android-tzdata"
106 version = "0.1.1"
107 source = "registry+https://github.com/rust-lang/crates.io-index"
108 @@ -154,6 +160,15 @@ dependencies = [
109 ]
110
111 [[package]]
112+ name = "atoi"
113+ version = "2.0.0"
114+ source = "registry+https://github.com/rust-lang/crates.io-index"
115+ checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
116+ dependencies = [
117+ "num-traits",
118+ ]
119+
120+ [[package]]
121 name = "atom_syndication"
122 version = "0.12.7"
123 source = "registry+https://github.com/rust-lang/crates.io-index"
124 @@ -282,6 +297,7 @@ dependencies = [
125 "ayllu_api",
126 "ayllu_cmd",
127 "ayllu_config",
128+ "ayllu_database",
129 "ayllu_git",
130 "ayllu_identity",
131 "bytes",
132 @@ -291,14 +307,16 @@ dependencies = [
133 "git2",
134 "headers",
135 "httparse",
136+ "layout-rs",
137 "lazy_static",
138 "libloading",
139 "mime",
140 "mime_guess",
141 "nix",
142 "openssh-keys",
143+ "petgraph",
144 "quick-xml 0.38.0",
145- "rand",
146+ "rand 0.9.2",
147 "rss",
148 "rustc_version",
149 "serde",
150 @@ -322,6 +340,27 @@ dependencies = [
151 ]
152
153 [[package]]
154+ name = "ayllu-build"
155+ version = "0.2.1"
156+ dependencies = [
157+ "async-trait",
158+ "ayllu_api",
159+ "ayllu_config",
160+ "ayllu_database",
161+ "ayllu_git",
162+ "ayllu_logging",
163+ "clap",
164+ "clap_complete",
165+ "petgraph",
166+ "rand 0.9.2",
167+ "serde",
168+ "serde_json",
169+ "sysinfo",
170+ "tokio",
171+ "tracing",
172+ ]
173+
174+ [[package]]
175 name = "ayllu-keys"
176 version = "0.5.1"
177 dependencies = [
178 @@ -332,6 +371,19 @@ dependencies = [
179 ]
180
181 [[package]]
182+ name = "ayllu-migrate"
183+ version = "0.1.0"
184+ dependencies = [
185+ "ayllu_cmd",
186+ "ayllu_config",
187+ "ayllu_database",
188+ "ayllu_logging",
189+ "serde",
190+ "tokio",
191+ "tracing",
192+ ]
193+
194+ [[package]]
195 name = "ayllu-shell"
196 version = "0.5.1"
197 dependencies = [
198 @@ -379,11 +431,24 @@ dependencies = [
199 ]
200
201 [[package]]
202+ name = "ayllu_database"
203+ version = "0.2.1"
204+ dependencies = [
205+ "async-trait",
206+ "futures",
207+ "serde",
208+ "serde_json",
209+ "sqlx",
210+ "time",
211+ "tracing",
212+ ]
213+
214+ [[package]]
215 name = "ayllu_git"
216 version = "0.2.1"
217 dependencies = [
218 "git2",
219- "rand",
220+ "rand 0.9.2",
221 "serde",
222 "tempfile",
223 "tokio",
224 @@ -435,6 +500,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
225 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
226
227 [[package]]
228+ name = "base64ct"
229+ version = "1.8.0"
230+ source = "registry+https://github.com/rust-lang/crates.io-index"
231+ checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba"
232+
233+ [[package]]
234 name = "basic-toml"
235 version = "0.1.10"
236 source = "registry+https://github.com/rust-lang/crates.io-index"
237 @@ -448,6 +519,9 @@ name = "bitflags"
238 version = "2.9.1"
239 source = "registry+https://github.com/rust-lang/crates.io-index"
240 checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
241+ dependencies = [
242+ "serde",
243+ ]
244
245 [[package]]
246 name = "block-buffer"
247 @@ -601,6 +675,15 @@ dependencies = [
248 ]
249
250 [[package]]
251+ name = "concurrent-queue"
252+ version = "2.5.0"
253+ source = "registry+https://github.com/rust-lang/crates.io-index"
254+ checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
255+ dependencies = [
256+ "crossbeam-utils",
257+ ]
258+
259+ [[package]]
260 name = "console"
261 version = "0.15.11"
262 source = "registry+https://github.com/rust-lang/crates.io-index"
263 @@ -614,6 +697,12 @@ dependencies = [
264 ]
265
266 [[package]]
267+ name = "const-oid"
268+ version = "0.9.6"
269+ source = "registry+https://github.com/rust-lang/crates.io-index"
270+ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
271+
272+ [[package]]
273 name = "convert_case"
274 version = "0.6.0"
275 source = "registry+https://github.com/rust-lang/crates.io-index"
276 @@ -659,6 +748,55 @@ dependencies = [
277 ]
278
279 [[package]]
280+ name = "crc"
281+ version = "3.3.0"
282+ source = "registry+https://github.com/rust-lang/crates.io-index"
283+ checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
284+ dependencies = [
285+ "crc-catalog",
286+ ]
287+
288+ [[package]]
289+ name = "crc-catalog"
290+ version = "2.4.0"
291+ source = "registry+https://github.com/rust-lang/crates.io-index"
292+ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
293+
294+ [[package]]
295+ name = "crossbeam-deque"
296+ version = "0.8.6"
297+ source = "registry+https://github.com/rust-lang/crates.io-index"
298+ checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
299+ dependencies = [
300+ "crossbeam-epoch",
301+ "crossbeam-utils",
302+ ]
303+
304+ [[package]]
305+ name = "crossbeam-epoch"
306+ version = "0.9.18"
307+ source = "registry+https://github.com/rust-lang/crates.io-index"
308+ checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
309+ dependencies = [
310+ "crossbeam-utils",
311+ ]
312+
313+ [[package]]
314+ name = "crossbeam-queue"
315+ version = "0.3.12"
316+ source = "registry+https://github.com/rust-lang/crates.io-index"
317+ checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
318+ dependencies = [
319+ "crossbeam-utils",
320+ ]
321+
322+ [[package]]
323+ name = "crossbeam-utils"
324+ version = "0.8.21"
325+ source = "registry+https://github.com/rust-lang/crates.io-index"
326+ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
327+
328+ [[package]]
329 name = "crypto-common"
330 version = "0.1.6"
331 source = "registry+https://github.com/rust-lang/crates.io-index"
332 @@ -704,6 +842,17 @@ dependencies = [
333 ]
334
335 [[package]]
336+ name = "der"
337+ version = "0.7.10"
338+ source = "registry+https://github.com/rust-lang/crates.io-index"
339+ checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
340+ dependencies = [
341+ "const-oid",
342+ "pem-rfc7468",
343+ "zeroize",
344+ ]
345+
346+ [[package]]
347 name = "deranged"
348 version = "0.4.0"
349 source = "registry+https://github.com/rust-lang/crates.io-index"
350 @@ -768,7 +917,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
351 checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
352 dependencies = [
353 "block-buffer",
354+ "const-oid",
355 "crypto-common",
356+ "subtle",
357 ]
358
359 [[package]]
360 @@ -801,12 +952,27 @@ dependencies = [
361 ]
362
363 [[package]]
364+ name = "dotenvy"
365+ version = "0.15.7"
366+ source = "registry+https://github.com/rust-lang/crates.io-index"
367+ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
368+
369+ [[package]]
370 name = "dyn-clone"
371 version = "1.0.19"
372 source = "registry+https://github.com/rust-lang/crates.io-index"
373 checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
374
375 [[package]]
376+ name = "either"
377+ version = "1.15.0"
378+ source = "registry+https://github.com/rust-lang/crates.io-index"
379+ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
380+ dependencies = [
381+ "serde",
382+ ]
383+
384+ [[package]]
385 name = "encode_unicode"
386 version = "1.0.0"
387 source = "registry+https://github.com/rust-lang/crates.io-index"
388 @@ -844,6 +1010,28 @@ dependencies = [
389 ]
390
391 [[package]]
392+ name = "etcetera"
393+ version = "0.8.0"
394+ source = "registry+https://github.com/rust-lang/crates.io-index"
395+ checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"
396+ dependencies = [
397+ "cfg-if",
398+ "home",
399+ "windows-sys 0.48.0",
400+ ]
401+
402+ [[package]]
403+ name = "event-listener"
404+ version = "5.4.1"
405+ source = "registry+https://github.com/rust-lang/crates.io-index"
406+ checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
407+ dependencies = [
408+ "concurrent-queue",
409+ "parking",
410+ "pin-project-lite",
411+ ]
412+
413+ [[package]]
414 name = "fastrand"
415 version = "2.3.0"
416 source = "registry+https://github.com/rust-lang/crates.io-index"
417 @@ -859,12 +1047,35 @@ dependencies = [
418 ]
419
420 [[package]]
421+ name = "fixedbitset"
422+ version = "0.4.2"
423+ source = "registry+https://github.com/rust-lang/crates.io-index"
424+ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
425+
426+ [[package]]
427+ name = "flume"
428+ version = "0.11.1"
429+ source = "registry+https://github.com/rust-lang/crates.io-index"
430+ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
431+ dependencies = [
432+ "futures-core",
433+ "futures-sink",
434+ "spin",
435+ ]
436+
437+ [[package]]
438 name = "fnv"
439 version = "1.0.7"
440 source = "registry+https://github.com/rust-lang/crates.io-index"
441 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
442
443 [[package]]
444+ name = "foldhash"
445+ version = "0.1.5"
446+ source = "registry+https://github.com/rust-lang/crates.io-index"
447+ checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
448+
449+ [[package]]
450 name = "foreign-types"
451 version = "0.3.2"
452 source = "registry+https://github.com/rust-lang/crates.io-index"
453 @@ -931,6 +1142,17 @@ dependencies = [
454 ]
455
456 [[package]]
457+ name = "futures-intrusive"
458+ version = "0.5.0"
459+ source = "registry+https://github.com/rust-lang/crates.io-index"
460+ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
461+ dependencies = [
462+ "futures-core",
463+ "lock_api",
464+ "parking_lot",
465+ ]
466+
467+ [[package]]
468 name = "futures-io"
469 version = "0.3.31"
470 source = "registry+https://github.com/rust-lang/crates.io-index"
471 @@ -1059,6 +1281,20 @@ name = "hashbrown"
472 version = "0.15.4"
473 source = "registry+https://github.com/rust-lang/crates.io-index"
474 checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
475+ dependencies = [
476+ "allocator-api2",
477+ "equivalent",
478+ "foldhash",
479+ ]
480+
481+ [[package]]
482+ name = "hashlink"
483+ version = "0.10.0"
484+ source = "registry+https://github.com/rust-lang/crates.io-index"
485+ checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
486+ dependencies = [
487+ "hashbrown 0.15.4",
488+ ]
489
490 [[package]]
491 name = "headers"
492 @@ -1097,6 +1333,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
493 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
494
495 [[package]]
496+ name = "hkdf"
497+ version = "0.12.4"
498+ source = "registry+https://github.com/rust-lang/crates.io-index"
499+ checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7"
500+ dependencies = [
501+ "hmac",
502+ ]
503+
504+ [[package]]
505+ name = "hmac"
506+ version = "0.12.1"
507+ source = "registry+https://github.com/rust-lang/crates.io-index"
508+ checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
509+ dependencies = [
510+ "digest",
511+ ]
512+
513+ [[package]]
514+ name = "home"
515+ version = "0.5.11"
516+ source = "registry+https://github.com/rust-lang/crates.io-index"
517+ checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
518+ dependencies = [
519+ "windows-sys 0.59.0",
520+ ]
521+
522+ [[package]]
523 name = "http"
524 version = "1.3.1"
525 source = "registry+https://github.com/rust-lang/crates.io-index"
526 @@ -1461,10 +1724,19 @@ dependencies = [
527 ]
528
529 [[package]]
530+ name = "layout-rs"
531+ version = "0.1.3"
532+ source = "registry+https://github.com/rust-lang/crates.io-index"
533+ checksum = "5b8b38bc67665e362eb770c6b6ae88b48d040d94a0a10c4904c37bc79d263b95"
534+
535+ [[package]]
536 name = "lazy_static"
537 version = "1.5.0"
538 source = "registry+https://github.com/rust-lang/crates.io-index"
539 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
540+ dependencies = [
541+ "spin",
542+ ]
543
544 [[package]]
545 name = "libc"
546 @@ -1495,6 +1767,34 @@ dependencies = [
547 ]
548
549 [[package]]
550+ name = "libm"
551+ version = "0.2.15"
552+ source = "registry+https://github.com/rust-lang/crates.io-index"
553+ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
554+
555+ [[package]]
556+ name = "libredox"
557+ version = "0.1.10"
558+ source = "registry+https://github.com/rust-lang/crates.io-index"
559+ checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb"
560+ dependencies = [
561+ "bitflags",
562+ "libc",
563+ "redox_syscall",
564+ ]
565+
566+ [[package]]
567+ name = "libsqlite3-sys"
568+ version = "0.30.1"
569+ source = "registry+https://github.com/rust-lang/crates.io-index"
570+ checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
571+ dependencies = [
572+ "cc",
573+ "pkg-config",
574+ "vcpkg",
575+ ]
576+
577+ [[package]]
578 name = "libz-sys"
579 version = "1.1.22"
580 source = "registry+https://github.com/rust-lang/crates.io-index"
581 @@ -1643,6 +1943,15 @@ dependencies = [
582 ]
583
584 [[package]]
585+ name = "ntapi"
586+ version = "0.4.1"
587+ source = "registry+https://github.com/rust-lang/crates.io-index"
588+ checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
589+ dependencies = [
590+ "winapi",
591+ ]
592+
593+ [[package]]
594 name = "nu-ansi-term"
595 version = "0.46.0"
596 source = "registry+https://github.com/rust-lang/crates.io-index"
597 @@ -1653,18 +1962,56 @@ dependencies = [
598 ]
599
600 [[package]]
601+ name = "num-bigint-dig"
602+ version = "0.8.4"
603+ source = "registry+https://github.com/rust-lang/crates.io-index"
604+ checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151"
605+ dependencies = [
606+ "byteorder",
607+ "lazy_static",
608+ "libm",
609+ "num-integer",
610+ "num-iter",
611+ "num-traits",
612+ "rand 0.8.5",
613+ "smallvec",
614+ "zeroize",
615+ ]
616+
617+ [[package]]
618 name = "num-conv"
619 version = "0.1.0"
620 source = "registry+https://github.com/rust-lang/crates.io-index"
621 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
622
623 [[package]]
624+ name = "num-integer"
625+ version = "0.1.46"
626+ source = "registry+https://github.com/rust-lang/crates.io-index"
627+ checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
628+ dependencies = [
629+ "num-traits",
630+ ]
631+
632+ [[package]]
633+ name = "num-iter"
634+ version = "0.1.45"
635+ source = "registry+https://github.com/rust-lang/crates.io-index"
636+ checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
637+ dependencies = [
638+ "autocfg",
639+ "num-integer",
640+ "num-traits",
641+ ]
642+
643+ [[package]]
644 name = "num-traits"
645 version = "0.2.19"
646 source = "registry+https://github.com/rust-lang/crates.io-index"
647 checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
648 dependencies = [
649 "autocfg",
650+ "libm",
651 ]
652
653 [[package]]
654 @@ -1776,6 +2123,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
655 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
656
657 [[package]]
658+ name = "parking"
659+ version = "2.2.1"
660+ source = "registry+https://github.com/rust-lang/crates.io-index"
661+ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
662+
663+ [[package]]
664 name = "parking_lot"
665 version = "0.12.4"
666 source = "registry+https://github.com/rust-lang/crates.io-index"
667 @@ -1799,12 +2152,33 @@ dependencies = [
668 ]
669
670 [[package]]
671+ name = "pem-rfc7468"
672+ version = "0.7.0"
673+ source = "registry+https://github.com/rust-lang/crates.io-index"
674+ checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412"
675+ dependencies = [
676+ "base64ct",
677+ ]
678+
679+ [[package]]
680 name = "percent-encoding"
681 version = "2.3.1"
682 source = "registry+https://github.com/rust-lang/crates.io-index"
683 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
684
685 [[package]]
686+ name = "petgraph"
687+ version = "0.6.5"
688+ source = "registry+https://github.com/rust-lang/crates.io-index"
689+ checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
690+ dependencies = [
691+ "fixedbitset",
692+ "indexmap 2.10.0",
693+ "serde",
694+ "serde_derive",
695+ ]
696+
697+ [[package]]
698 name = "pin-project-lite"
699 version = "0.2.16"
700 source = "registry+https://github.com/rust-lang/crates.io-index"
701 @@ -1817,6 +2191,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
702 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
703
704 [[package]]
705+ name = "pkcs1"
706+ version = "0.7.5"
707+ source = "registry+https://github.com/rust-lang/crates.io-index"
708+ checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
709+ dependencies = [
710+ "der",
711+ "pkcs8",
712+ "spki",
713+ ]
714+
715+ [[package]]
716+ name = "pkcs8"
717+ version = "0.10.2"
718+ source = "registry+https://github.com/rust-lang/crates.io-index"
719+ checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
720+ dependencies = [
721+ "der",
722+ "spki",
723+ ]
724+
725+ [[package]]
726 name = "pkg-config"
727 version = "0.3.32"
728 source = "registry+https://github.com/rust-lang/crates.io-index"
729 @@ -1912,12 +2307,33 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
730
731 [[package]]
732 name = "rand"
733+ version = "0.8.5"
734+ source = "registry+https://github.com/rust-lang/crates.io-index"
735+ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
736+ dependencies = [
737+ "libc",
738+ "rand_chacha 0.3.1",
739+ "rand_core 0.6.4",
740+ ]
741+
742+ [[package]]
743+ name = "rand"
744 version = "0.9.2"
745 source = "registry+https://github.com/rust-lang/crates.io-index"
746 checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
747 dependencies = [
748- "rand_chacha",
749- "rand_core",
750+ "rand_chacha 0.9.0",
751+ "rand_core 0.9.3",
752+ ]
753+
754+ [[package]]
755+ name = "rand_chacha"
756+ version = "0.3.1"
757+ source = "registry+https://github.com/rust-lang/crates.io-index"
758+ checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
759+ dependencies = [
760+ "ppv-lite86",
761+ "rand_core 0.6.4",
762 ]
763
764 [[package]]
765 @@ -1927,7 +2343,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
766 checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
767 dependencies = [
768 "ppv-lite86",
769- "rand_core",
770+ "rand_core 0.9.3",
771+ ]
772+
773+ [[package]]
774+ name = "rand_core"
775+ version = "0.6.4"
776+ source = "registry+https://github.com/rust-lang/crates.io-index"
777+ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
778+ dependencies = [
779+ "getrandom 0.2.16",
780 ]
781
782 [[package]]
783 @@ -1940,6 +2365,26 @@ dependencies = [
784 ]
785
786 [[package]]
787+ name = "rayon"
788+ version = "1.11.0"
789+ source = "registry+https://github.com/rust-lang/crates.io-index"
790+ checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
791+ dependencies = [
792+ "either",
793+ "rayon-core",
794+ ]
795+
796+ [[package]]
797+ name = "rayon-core"
798+ version = "1.13.0"
799+ source = "registry+https://github.com/rust-lang/crates.io-index"
800+ checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
801+ dependencies = [
802+ "crossbeam-deque",
803+ "crossbeam-utils",
804+ ]
805+
806+ [[package]]
807 name = "redox_syscall"
808 version = "0.5.16"
809 source = "registry+https://github.com/rust-lang/crates.io-index"
810 @@ -2073,6 +2518,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
811 checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3"
812
813 [[package]]
814+ name = "rsa"
815+ version = "0.9.8"
816+ source = "registry+https://github.com/rust-lang/crates.io-index"
817+ checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b"
818+ dependencies = [
819+ "const-oid",
820+ "digest",
821+ "num-bigint-dig",
822+ "num-integer",
823+ "num-traits",
824+ "pkcs1",
825+ "pkcs8",
826+ "rand_core 0.6.4",
827+ "signature",
828+ "spki",
829+ "subtle",
830+ "zeroize",
831+ ]
832+
833+ [[package]]
834 name = "rss"
835 version = "2.0.12"
836 source = "registry+https://github.com/rust-lang/crates.io-index"
837 @@ -2393,6 +2858,16 @@ dependencies = [
838 ]
839
840 [[package]]
841+ name = "signature"
842+ version = "2.2.0"
843+ source = "registry+https://github.com/rust-lang/crates.io-index"
844+ checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
845+ dependencies = [
846+ "digest",
847+ "rand_core 0.6.4",
848+ ]
849+
850+ [[package]]
851 name = "slab"
852 version = "0.4.10"
853 source = "registry+https://github.com/rust-lang/crates.io-index"
854 @@ -2413,6 +2888,9 @@ name = "smallvec"
855 version = "1.15.1"
856 source = "registry+https://github.com/rust-lang/crates.io-index"
857 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
858+ dependencies = [
859+ "serde",
860+ ]
861
862 [[package]]
863 name = "socket2"
864 @@ -2425,6 +2903,213 @@ dependencies = [
865 ]
866
867 [[package]]
868+ name = "spin"
869+ version = "0.9.8"
870+ source = "registry+https://github.com/rust-lang/crates.io-index"
871+ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
872+ dependencies = [
873+ "lock_api",
874+ ]
875+
876+ [[package]]
877+ name = "spki"
878+ version = "0.7.3"
879+ source = "registry+https://github.com/rust-lang/crates.io-index"
880+ checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
881+ dependencies = [
882+ "base64ct",
883+ "der",
884+ ]
885+
886+ [[package]]
887+ name = "sqlx"
888+ version = "0.8.6"
889+ source = "registry+https://github.com/rust-lang/crates.io-index"
890+ checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc"
891+ dependencies = [
892+ "sqlx-core",
893+ "sqlx-macros",
894+ "sqlx-mysql",
895+ "sqlx-postgres",
896+ "sqlx-sqlite",
897+ ]
898+
899+ [[package]]
900+ name = "sqlx-core"
901+ version = "0.8.6"
902+ source = "registry+https://github.com/rust-lang/crates.io-index"
903+ checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6"
904+ dependencies = [
905+ "base64 0.22.1",
906+ "bytes",
907+ "crc",
908+ "crossbeam-queue",
909+ "either",
910+ "event-listener",
911+ "futures-core",
912+ "futures-intrusive",
913+ "futures-io",
914+ "futures-util",
915+ "hashbrown 0.15.4",
916+ "hashlink",
917+ "indexmap 2.10.0",
918+ "log",
919+ "memchr",
920+ "once_cell",
921+ "percent-encoding",
922+ "serde",
923+ "serde_json",
924+ "sha2",
925+ "smallvec",
926+ "thiserror 2.0.12",
927+ "tokio",
928+ "tokio-stream",
929+ "tracing",
930+ "url",
931+ ]
932+
933+ [[package]]
934+ name = "sqlx-macros"
935+ version = "0.8.6"
936+ source = "registry+https://github.com/rust-lang/crates.io-index"
937+ checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d"
938+ dependencies = [
939+ "proc-macro2",
940+ "quote",
941+ "sqlx-core",
942+ "sqlx-macros-core",
943+ "syn",
944+ ]
945+
946+ [[package]]
947+ name = "sqlx-macros-core"
948+ version = "0.8.6"
949+ source = "registry+https://github.com/rust-lang/crates.io-index"
950+ checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b"
951+ dependencies = [
952+ "dotenvy",
953+ "either",
954+ "heck",
955+ "hex",
956+ "once_cell",
957+ "proc-macro2",
958+ "quote",
959+ "serde",
960+ "serde_json",
961+ "sha2",
962+ "sqlx-core",
963+ "sqlx-mysql",
964+ "sqlx-postgres",
965+ "sqlx-sqlite",
966+ "syn",
967+ "tokio",
968+ "url",
969+ ]
970+
971+ [[package]]
972+ name = "sqlx-mysql"
973+ version = "0.8.6"
974+ source = "registry+https://github.com/rust-lang/crates.io-index"
975+ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526"
976+ dependencies = [
977+ "atoi",
978+ "base64 0.22.1",
979+ "bitflags",
980+ "byteorder",
981+ "bytes",
982+ "crc",
983+ "digest",
984+ "dotenvy",
985+ "either",
986+ "futures-channel",
987+ "futures-core",
988+ "futures-io",
989+ "futures-util",
990+ "generic-array",
991+ "hex",
992+ "hkdf",
993+ "hmac",
994+ "itoa",
995+ "log",
996+ "md-5",
997+ "memchr",
998+ "once_cell",
999+ "percent-encoding",
1000+ "rand 0.8.5",
1001+ "rsa",
1002+ "serde",
1003+ "sha1",
1004+ "sha2",
1005+ "smallvec",
1006+ "sqlx-core",
1007+ "stringprep",
1008+ "thiserror 2.0.12",
1009+ "tracing",
1010+ "whoami",
1011+ ]
1012+
1013+ [[package]]
1014+ name = "sqlx-postgres"
1015+ version = "0.8.6"
1016+ source = "registry+https://github.com/rust-lang/crates.io-index"
1017+ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46"
1018+ dependencies = [
1019+ "atoi",
1020+ "base64 0.22.1",
1021+ "bitflags",
1022+ "byteorder",
1023+ "crc",
1024+ "dotenvy",
1025+ "etcetera",
1026+ "futures-channel",
1027+ "futures-core",
1028+ "futures-util",
1029+ "hex",
1030+ "hkdf",
1031+ "hmac",
1032+ "home",
1033+ "itoa",
1034+ "log",
1035+ "md-5",
1036+ "memchr",
1037+ "once_cell",
1038+ "rand 0.8.5",
1039+ "serde",
1040+ "serde_json",
1041+ "sha2",
1042+ "smallvec",
1043+ "sqlx-core",
1044+ "stringprep",
1045+ "thiserror 2.0.12",
1046+ "tracing",
1047+ "whoami",
1048+ ]
1049+
1050+ [[package]]
1051+ name = "sqlx-sqlite"
1052+ version = "0.8.6"
1053+ source = "registry+https://github.com/rust-lang/crates.io-index"
1054+ checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea"
1055+ dependencies = [
1056+ "atoi",
1057+ "flume",
1058+ "futures-channel",
1059+ "futures-core",
1060+ "futures-executor",
1061+ "futures-intrusive",
1062+ "futures-util",
1063+ "libsqlite3-sys",
1064+ "log",
1065+ "percent-encoding",
1066+ "serde",
1067+ "serde_urlencoded",
1068+ "sqlx-core",
1069+ "thiserror 2.0.12",
1070+ "tracing",
1071+ "url",
1072+ ]
1073+
1074+ [[package]]
1075 name = "stable_deref_trait"
1076 version = "1.2.0"
1077 source = "registry+https://github.com/rust-lang/crates.io-index"
1078 @@ -2437,6 +3122,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1079 checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
1080
1081 [[package]]
1082+ name = "stringprep"
1083+ version = "0.1.5"
1084+ source = "registry+https://github.com/rust-lang/crates.io-index"
1085+ checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
1086+ dependencies = [
1087+ "unicode-bidi",
1088+ "unicode-normalization",
1089+ "unicode-properties",
1090+ ]
1091+
1092+ [[package]]
1093 name = "strsim"
1094 version = "0.11.1"
1095 source = "registry+https://github.com/rust-lang/crates.io-index"
1096 @@ -2480,6 +3176,21 @@ dependencies = [
1097 ]
1098
1099 [[package]]
1100+ name = "sysinfo"
1101+ version = "0.29.11"
1102+ source = "registry+https://github.com/rust-lang/crates.io-index"
1103+ checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666"
1104+ dependencies = [
1105+ "cfg-if",
1106+ "core-foundation-sys",
1107+ "libc",
1108+ "ntapi",
1109+ "once_cell",
1110+ "rayon",
1111+ "winapi",
1112+ ]
1113+
1114+ [[package]]
1115 name = "system-configuration"
1116 version = "0.6.1"
1117 source = "registry+https://github.com/rust-lang/crates.io-index"
1118 @@ -2908,6 +3619,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1119 checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
1120
1121 [[package]]
1122+ name = "unicode-bidi"
1123+ version = "0.3.18"
1124+ source = "registry+https://github.com/rust-lang/crates.io-index"
1125+ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
1126+
1127+ [[package]]
1128 name = "unicode-ident"
1129 version = "1.0.18"
1130 source = "registry+https://github.com/rust-lang/crates.io-index"
1131 @@ -2923,6 +3640,12 @@ dependencies = [
1132 ]
1133
1134 [[package]]
1135+ name = "unicode-properties"
1136+ version = "0.1.3"
1137+ source = "registry+https://github.com/rust-lang/crates.io-index"
1138+ checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
1139+
1140+ [[package]]
1141 name = "unicode-segmentation"
1142 version = "1.12.0"
1143 source = "registry+https://github.com/rust-lang/crates.io-index"
1144 @@ -3019,6 +3742,12 @@ dependencies = [
1145 ]
1146
1147 [[package]]
1148+ name = "wasite"
1149+ version = "0.1.0"
1150+ source = "registry+https://github.com/rust-lang/crates.io-index"
1151+ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
1152+
1153+ [[package]]
1154 name = "wasm-bindgen"
1155 version = "0.2.100"
1156 source = "registry+https://github.com/rust-lang/crates.io-index"
1157 @@ -3138,6 +3867,16 @@ dependencies = [
1158 ]
1159
1160 [[package]]
1161+ name = "whoami"
1162+ version = "1.6.1"
1163+ source = "registry+https://github.com/rust-lang/crates.io-index"
1164+ checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d"
1165+ dependencies = [
1166+ "libredox",
1167+ "wasite",
1168+ ]
1169+
1170+ [[package]]
1171 name = "winapi"
1172 version = "0.3.9"
1173 source = "registry+https://github.com/rust-lang/crates.io-index"
1174 @@ -3231,6 +3970,15 @@ dependencies = [
1175
1176 [[package]]
1177 name = "windows-sys"
1178+ version = "0.48.0"
1179+ source = "registry+https://github.com/rust-lang/crates.io-index"
1180+ checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
1181+ dependencies = [
1182+ "windows-targets 0.48.5",
1183+ ]
1184+
1185+ [[package]]
1186+ name = "windows-sys"
1187 version = "0.52.0"
1188 source = "registry+https://github.com/rust-lang/crates.io-index"
1189 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
1190 @@ -3258,6 +4006,21 @@ dependencies = [
1191
1192 [[package]]
1193 name = "windows-targets"
1194+ version = "0.48.5"
1195+ source = "registry+https://github.com/rust-lang/crates.io-index"
1196+ checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
1197+ dependencies = [
1198+ "windows_aarch64_gnullvm 0.48.5",
1199+ "windows_aarch64_msvc 0.48.5",
1200+ "windows_i686_gnu 0.48.5",
1201+ "windows_i686_msvc 0.48.5",
1202+ "windows_x86_64_gnu 0.48.5",
1203+ "windows_x86_64_gnullvm 0.48.5",
1204+ "windows_x86_64_msvc 0.48.5",
1205+ ]
1206+
1207+ [[package]]
1208+ name = "windows-targets"
1209 version = "0.52.6"
1210 source = "registry+https://github.com/rust-lang/crates.io-index"
1211 checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
1212 @@ -3290,6 +4053,12 @@ dependencies = [
1213
1214 [[package]]
1215 name = "windows_aarch64_gnullvm"
1216+ version = "0.48.5"
1217+ source = "registry+https://github.com/rust-lang/crates.io-index"
1218+ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
1219+
1220+ [[package]]
1221+ name = "windows_aarch64_gnullvm"
1222 version = "0.52.6"
1223 source = "registry+https://github.com/rust-lang/crates.io-index"
1224 checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
1225 @@ -3302,6 +4071,12 @@ checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
1226
1227 [[package]]
1228 name = "windows_aarch64_msvc"
1229+ version = "0.48.5"
1230+ source = "registry+https://github.com/rust-lang/crates.io-index"
1231+ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
1232+
1233+ [[package]]
1234+ name = "windows_aarch64_msvc"
1235 version = "0.52.6"
1236 source = "registry+https://github.com/rust-lang/crates.io-index"
1237 checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
1238 @@ -3314,6 +4089,12 @@ checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
1239
1240 [[package]]
1241 name = "windows_i686_gnu"
1242+ version = "0.48.5"
1243+ source = "registry+https://github.com/rust-lang/crates.io-index"
1244+ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
1245+
1246+ [[package]]
1247+ name = "windows_i686_gnu"
1248 version = "0.52.6"
1249 source = "registry+https://github.com/rust-lang/crates.io-index"
1250 checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
1251 @@ -3338,6 +4119,12 @@ checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
1252
1253 [[package]]
1254 name = "windows_i686_msvc"
1255+ version = "0.48.5"
1256+ source = "registry+https://github.com/rust-lang/crates.io-index"
1257+ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
1258+
1259+ [[package]]
1260+ name = "windows_i686_msvc"
1261 version = "0.52.6"
1262 source = "registry+https://github.com/rust-lang/crates.io-index"
1263 checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
1264 @@ -3350,6 +4137,12 @@ checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
1265
1266 [[package]]
1267 name = "windows_x86_64_gnu"
1268+ version = "0.48.5"
1269+ source = "registry+https://github.com/rust-lang/crates.io-index"
1270+ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
1271+
1272+ [[package]]
1273+ name = "windows_x86_64_gnu"
1274 version = "0.52.6"
1275 source = "registry+https://github.com/rust-lang/crates.io-index"
1276 checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
1277 @@ -3362,6 +4155,12 @@ checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
1278
1279 [[package]]
1280 name = "windows_x86_64_gnullvm"
1281+ version = "0.48.5"
1282+ source = "registry+https://github.com/rust-lang/crates.io-index"
1283+ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
1284+
1285+ [[package]]
1286+ name = "windows_x86_64_gnullvm"
1287 version = "0.52.6"
1288 source = "registry+https://github.com/rust-lang/crates.io-index"
1289 checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
1290 @@ -3374,6 +4173,12 @@ checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
1291
1292 [[package]]
1293 name = "windows_x86_64_msvc"
1294+ version = "0.48.5"
1295+ source = "registry+https://github.com/rust-lang/crates.io-index"
1296+ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
1297+
1298+ [[package]]
1299+ name = "windows_x86_64_msvc"
1300 version = "0.52.6"
1301 source = "registry+https://github.com/rust-lang/crates.io-index"
1302 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
1303 diff --git a/Cargo.toml b/Cargo.toml
1304index 5cf4932..bf76ee8 100644
1305--- a/Cargo.toml
1306+++ b/Cargo.toml
1307 @@ -3,20 +3,19 @@ resolver = "2"
1308 members = [
1309 "crates/api",
1310 "crates/config",
1311+ "crates/cmd",
1312 "crates/git",
1313 "crates/logging",
1314- "crates/timeutil",
1315 "crates/identity",
1316- # "crates/scheduler",
1317- # "crates/database",
1318+ "crates/timeutil",
1319+ "crates/database",
1320 "ayllu",
1321- # "ayllu-build",
1322+ "ayllu-build",
1323 # "ayllu-mail",
1324 "ayllu-shell",
1325 "ayllu-keys",
1326- # "ayllu-jobs",
1327- # "ayllu-xmpp",
1328- "quipu", "crates/identity", "crates/cmd", "xtask",
1329+ "quipu",
1330+ "xtask", "ayllu-migrate",
1331 ]
1332
1333 [workspace.dependencies]
1334 @@ -25,15 +24,19 @@ bytes = "1.10.1"
1335 clap = { version = "4.4.18", features = ["derive"] }
1336 clap_complete = { version = "4.4.10" }
1337 serde = { version = "1.0", features = ["derive"] }
1338+ serde_json = "1.0.108"
1339+ petgraph = { version = "0.6.4", features = ["serde-1"] }
1340 git2 = { version = "0.20.2", default-features = false, features = [] }
1341 rand = "0.9.1"
1342 thiserror = "2.0.12"
1343 tracing = { version = "0.1.41", features=["log"] }
1344 toml = "0.8.23"
1345+ time = "0.3.41"
1346 futures = "0.3.31"
1347 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
1348 openssh-keys = "0.6.4"
1349 url = { version = "2.5.4", features = ["serde"]}
1350+ sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] }
1351
1352 tokio = { version = "1.46.1", features = ["full"] }
1353 tokio-util = { version = "0.7.15", features = ["io", "compat"] }
1354 diff --git a/ayllu-build/Cargo.toml b/ayllu-build/Cargo.toml
1355index 0cdede2..83886cd 100644
1356--- a/ayllu-build/Cargo.toml
1357+++ b/ayllu-build/Cargo.toml
1358 @@ -10,20 +10,18 @@ name = "ayllu-build"
1359 ayllu_api = {path = "../crates/api"}
1360 ayllu_config = {path = "../crates/config"}
1361 ayllu_database = {path = "../crates/database"}
1362- ayllu_rpc = {path = "../crates/rpc"}
1363+ ayllu_logging = {path = "../crates/logging"}
1364+ # ayllu_rpc = {path = "../crates/rpc"}
1365 ayllu_git = {path = "../crates/git"}
1366
1367- anyhow = "1.0.75"
1368- clap = "4.4.8"
1369- nickel-lang-core = { version = "0.3.0", default-features = false }
1370- petgraph = "0.6.4"
1371- rand = "0.8.5"
1372- serde = { version = "1.0", features = ["derive"] }
1373- tracing = "0.1.40"
1374- tracing-subscriber = "0.3.18"
1375- async-trait = "0.1.74"
1376- sqlx = { version = "0.7.3", features = ["sqlite", "macros"] }
1377- tokio = { version = "1.34.0", features = ["full"] }
1378+ async-trait = { workspace = true }
1379+ clap = { workspace = true }
1380+ clap_complete = { workspace = true }
1381+ tracing = { workspace = true }
1382+ rand = { workspace = true }
1383+ serde = { workspace = true }
1384+ petgraph = { workspace = true }
1385+ tokio = { workspace = true }
1386+
1387 sysinfo = "0.29.11"
1388 serde_json = "1.0.108"
1389- clap_complete = "4.4.9"
1390 diff --git a/ayllu-build/migrations/20231204194038_init.sql b/ayllu-build/migrations/20231204194038_init.sql
1391deleted file mode 100644
1392index e072e06..0000000
1393--- a/ayllu-build/migrations/20231204194038_init.sql
1394+++ /dev/null
1395 @@ -1,54 +0,0 @@
1396- CREATE TABLE manifests (
1397- id INTEGER PRIMARY KEY,
1398- repository_url TEXT NOT NULL,
1399- git_hash TEXT NOT NULL,
1400- created_at INTEGER NOT NULL DEFAULT (UNIXEPOCH()),
1401- started_at INTEGER CHECK (started_at > 0),
1402- duration INTEGER CHECK (duration > 0)
1403- ) STRICT ;
1404-
1405- CREATE TABLE workflows (
1406- id INTEGER PRIMARY KEY,
1407- manifest_id INTEGER REFERENCES manifests(id) ON DELETE CASCADE NOT NULL,
1408- name TEXT NOT NULL,
1409- started_at INTEGER CHECK (started_at > 0),
1410- duration INTEGER CHECK (duration > 0)
1411- ) STRICT ;
1412-
1413- CREATE TABLE steps (
1414- id INTEGER PRIMARY KEY,
1415- name TEXT NOT NULL,
1416- workflow_id INTEGER REFERENCES workflows(id) ON DELETE CASCADE NOT NULL,
1417- shell TEXT NOT NULL DEFAULT "/bin/sh",
1418- environment_json TEXT,
1419- input TEXT NOT NULL,
1420- started_at INTEGER CHECK (started_at > 0),
1421- duration INTEGER CHECK (duration > 0),
1422- stdout TEXT,
1423- stderr TEXT,
1424- exit_code INTEGER
1425- ) STRICT ;
1426-
1427- CREATE TABLE dags (
1428- id INTEGER PRIMARY KEY,
1429- manifest_id INTEGER REFERENCES manifests(id) ON DELETE CASCADE NOT NULL,
1430- dot_content TEXT NOT NULL
1431- ) STRICT ;
1432-
1433- CREATE TABLE samples (
1434- id INTEGER PRIMARY KEY,
1435- manifest_id INTEGER REFERENCES manifests(id) ON DELETE CASCADE NOT NULL,
1436- timestamp INTEGER NOT NULL DEFAULT (UNIXEPOCH()),
1437- load_1m INTEGER,
1438- load_5m INTEGER,
1439- load_15m INTEGER,
1440- disk_total_bytes INTEGER,
1441- disk_free_bytes INTEGER,
1442- net_sent_bytes INTEGER,
1443- net_received_bytes INTEGER,
1444- net_sent_packets INTEGER,
1445- net_received_packets INTEGER,
1446- mem_total_bytes INTEGER,
1447- mem_free_bytes INTEGER,
1448- mem_available_bytes INTEGER
1449- ) STRICT ;
1450 diff --git a/ayllu-build/queries/dags_create.sql b/ayllu-build/queries/dags_create.sql
1451deleted file mode 100644
1452index 7214c60..0000000
1453--- a/ayllu-build/queries/dags_create.sql
1454+++ /dev/null
1455 @@ -1,2 +0,0 @@
1456- INSERT INTO dags (manifest_id, dot_content)
1457- VALUES (?, ?) RETURNING id
1458 diff --git a/ayllu-build/queries/manifests_create.sql b/ayllu-build/queries/manifests_create.sql
1459deleted file mode 100644
1460index a7ded3e..0000000
1461--- a/ayllu-build/queries/manifests_create.sql
1462+++ /dev/null
1463 @@ -1,2 +0,0 @@
1464- INSERT INTO manifests (repository_url, git_hash)
1465- VALUES (?, ?) RETURNING id
1466 diff --git a/ayllu-build/queries/manifests_list.sql b/ayllu-build/queries/manifests_list.sql
1467deleted file mode 100644
1468index af58a4e..0000000
1469--- a/ayllu-build/queries/manifests_list.sql
1470+++ /dev/null
1471 @@ -1,11 +0,0 @@
1472- SELECT id,
1473- repository_url,
1474- git_hash,
1475- created_at,
1476- started_at,
1477- finished_at,
1478- success
1479- FROM manifests
1480- WHERE id = ?
1481- LIMIT ?
1482- OFFSET ?
1483 diff --git a/ayllu-build/queries/manifests_read.sql b/ayllu-build/queries/manifests_read.sql
1484deleted file mode 100644
1485index 073fa75..0000000
1486--- a/ayllu-build/queries/manifests_read.sql
1487+++ /dev/null
1488 @@ -1,10 +0,0 @@
1489- SELECT manifests.id,
1490- repository_url,
1491- git_hash,
1492- created_at,
1493- started_at,
1494- duration,
1495- dags.dot_content AS "dot_content!"
1496- FROM manifests
1497- LEFT JOIN dags ON dags.manifest_id = manifests.id
1498- WHERE manifests.id = ?
1499 diff --git a/ayllu-build/queries/manifests_update_finish.sql b/ayllu-build/queries/manifests_update_finish.sql
1500deleted file mode 100644
1501index 52f4016..0000000
1502--- a/ayllu-build/queries/manifests_update_finish.sql
1503+++ /dev/null
1504 @@ -1,4 +0,0 @@
1505- UPDATE manifests
1506- SET
1507- duration = ?
1508- WHERE id = ?
1509 diff --git a/ayllu-build/queries/manifests_update_start.sql b/ayllu-build/queries/manifests_update_start.sql
1510deleted file mode 100644
1511index fc85a4d..0000000
1512--- a/ayllu-build/queries/manifests_update_start.sql
1513+++ /dev/null
1514 @@ -1,3 +0,0 @@
1515- UPDATE manifests
1516- SET started_at = UNIXEPOCH()
1517- WHERE id = ?
1518 diff --git a/ayllu-build/queries/samples_create.sql b/ayllu-build/queries/samples_create.sql
1519deleted file mode 100644
1520index 7fc297f..0000000
1521--- a/ayllu-build/queries/samples_create.sql
1522+++ /dev/null
1523 @@ -1,18 +0,0 @@
1524- INSERT INTO samples
1525- (
1526- manifest_id,
1527- load_1m,
1528- load_5m,
1529- load_15m,
1530- disk_total_bytes,
1531- disk_free_bytes,
1532- net_sent_bytes,
1533- net_received_bytes,
1534- net_sent_packets,
1535- net_received_packets,
1536- mem_total_bytes,
1537- mem_free_bytes,
1538- mem_available_bytes
1539- )
1540- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1541- RETURNING id
1542 diff --git a/ayllu-build/queries/samples_read.sql b/ayllu-build/queries/samples_read.sql
1543deleted file mode 100644
1544index 6842939..0000000
1545--- a/ayllu-build/queries/samples_read.sql
1546+++ /dev/null
1547 @@ -1,15 +0,0 @@
1548- SELECT id,
1549- load_1m,
1550- load_5m,
1551- load_15m,
1552- disk_total_bytes,
1553- disk_free_bytes,
1554- net_sent_bytes,
1555- net_received_bytes,
1556- net_sent_packets,
1557- net_received_packets,
1558- mem_total_bytes,
1559- mem_free_bytes,
1560- mem_available_bytes
1561- FROM samples
1562- WHERE manifest_id = ?
1563 diff --git a/ayllu-build/queries/steps_create.sql b/ayllu-build/queries/steps_create.sql
1564deleted file mode 100644
1565index ababf23..0000000
1566--- a/ayllu-build/queries/steps_create.sql
1567+++ /dev/null
1568 @@ -1,2 +0,0 @@
1569- INSERT INTO steps (workflow_id, name, input, shell, environment_json)
1570- VALUES (?, ?, ?, ?, json_array(?)) RETURNING id
1571 diff --git a/ayllu-build/queries/steps_list.sql b/ayllu-build/queries/steps_list.sql
1572deleted file mode 100644
1573index 7aa1ee7..0000000
1574--- a/ayllu-build/queries/steps_list.sql
1575+++ /dev/null
1576 @@ -1,10 +0,0 @@
1577- SELECT id,
1578- name,
1579- input,
1580- stdout,
1581- stderr,
1582- started_at,
1583- duration,
1584- exit_code
1585- FROM steps
1586- WHERE workflow_id = ?
1587 diff --git a/ayllu-build/queries/steps_read.sql b/ayllu-build/queries/steps_read.sql
1588deleted file mode 100644
1589index efd3761..0000000
1590--- a/ayllu-build/queries/steps_read.sql
1591+++ /dev/null
1592 @@ -1 +0,0 @@
1593- SELECT * FROM steps WHERE workflow_id = ?
1594 diff --git a/ayllu-build/queries/steps_update_finish.sql b/ayllu-build/queries/steps_update_finish.sql
1595deleted file mode 100644
1596index 60293fc..0000000
1597--- a/ayllu-build/queries/steps_update_finish.sql
1598+++ /dev/null
1599 @@ -1,8 +0,0 @@
1600- UPDATE steps
1601- SET
1602- duration = ?,
1603- stdout = ?,
1604- stderr = ?,
1605- exit_code = ?
1606- WHERE
1607- id = ?
1608 diff --git a/ayllu-build/queries/steps_update_start.sql b/ayllu-build/queries/steps_update_start.sql
1609deleted file mode 100644
1610index 387654b..0000000
1611--- a/ayllu-build/queries/steps_update_start.sql
1612+++ /dev/null
1613 @@ -1,5 +0,0 @@
1614- UPDATE steps
1615- SET
1616- started_at = UNIXEPOCH()
1617- WHERE
1618- id = ?
1619 diff --git a/ayllu-build/queries/workflows_create.sql b/ayllu-build/queries/workflows_create.sql
1620deleted file mode 100644
1621index 0063b5f..0000000
1622--- a/ayllu-build/queries/workflows_create.sql
1623+++ /dev/null
1624 @@ -1,2 +0,0 @@
1625- INSERT INTO workflows (manifest_id, name)
1626- VALUES (?, ?) RETURNING id
1627 diff --git a/ayllu-build/queries/workflows_list.sql b/ayllu-build/queries/workflows_list.sql
1628deleted file mode 100644
1629index b0cb1fc..0000000
1630--- a/ayllu-build/queries/workflows_list.sql
1631+++ /dev/null
1632 @@ -1,6 +0,0 @@
1633- SELECT id,
1634- name,
1635- started_at,
1636- duration
1637- FROM workflows
1638- WHERE manifest_id = ?
1639 diff --git a/ayllu-build/queries/workflows_read.sql b/ayllu-build/queries/workflows_read.sql
1640deleted file mode 100644
1641index 5752bfd..0000000
1642--- a/ayllu-build/queries/workflows_read.sql
1643+++ /dev/null
1644 @@ -1 +0,0 @@
1645- SELECT * FROM workflows WHERE manifest_id = ?
1646 diff --git a/ayllu-build/queries/workflows_update_finish.sql b/ayllu-build/queries/workflows_update_finish.sql
1647deleted file mode 100644
1648index d4d2476..0000000
1649--- a/ayllu-build/queries/workflows_update_finish.sql
1650+++ /dev/null
1651 @@ -1,5 +0,0 @@
1652- UPDATE workflows
1653- SET
1654- duration = ?
1655- WHERE
1656- id = ?
1657 diff --git a/ayllu-build/queries/workflows_update_start.sql b/ayllu-build/queries/workflows_update_start.sql
1658deleted file mode 100644
1659index 11be057..0000000
1660--- a/ayllu-build/queries/workflows_update_start.sql
1661+++ /dev/null
1662 @@ -1,5 +0,0 @@
1663- UPDATE workflows
1664- SET
1665- started_at = UNIXEPOCH()
1666- WHERE
1667- id = ?
1668 diff --git a/ayllu-build/src/config.rs b/ayllu-build/src/config.rs
1669index 7f34e57..eff7909 100644
1670--- a/ayllu-build/src/config.rs
1671+++ b/ayllu-build/src/config.rs
1672 @@ -1,56 +1,44 @@
1673- use std::path::Path;
1674+ use std::path::{Path, PathBuf};
1675
1676 use serde::{Deserialize, Serialize};
1677
1678- use ayllu_config::{data_dir, runtime_dir, Configurable, Error, Reader};
1679+ use ayllu_config::{data_dir, runtime_dir, Configurable, Database, Error, Reader};
1680
1681 #[derive(Deserialize, Serialize, Clone, Debug)]
1682- pub struct Database {
1683- #[serde(default = "Database::path_default")]
1684- pub path: String,
1685- #[serde(default = "Database::migrate_default")]
1686- pub migrate: bool,
1687+ pub struct Builder {
1688+ #[serde(default = "Builder::address_default")]
1689+ pub address: String,
1690+ #[serde(default = "Builder::log_path_default")]
1691+ pub log_path: PathBuf,
1692 }
1693
1694- impl Database {
1695- fn path_default() -> String {
1696- let mut data_path = data_dir();
1697- data_path.push("worker.db");
1698- String::from(data_path.to_str().unwrap())
1699+ impl Builder {
1700+ fn address_default() -> String {
1701+ runtime_dir().to_str().unwrap().to_string() + "/ayllu-build.sock"
1702 }
1703
1704- fn migrate_default() -> bool {
1705- true
1706+ fn log_path_default() -> PathBuf {
1707+ data_dir().join("worker_logs")
1708 }
1709 }
1710
1711- #[derive(Deserialize, Serialize, Clone, Debug)]
1712- pub struct Builder {
1713- #[serde(default = "Config::address_default")]
1714- pub address: String,
1715- pub database: Database,
1716- #[serde(default = "Config::log_path")]
1717- pub log_path: String,
1718+ impl Default for Builder {
1719+ fn default() -> Self {
1720+ Self {
1721+ address: Builder::address_default(),
1722+ log_path: Builder::log_path_default(),
1723+ }
1724+ }
1725 }
1726
1727 #[derive(Deserialize, Serialize, Clone, Debug)]
1728 pub struct Config {
1729- pub log_level: String,
1730+ #[serde(default = "Database::default")]
1731+ pub database: Database,
1732+ #[serde(default = "Builder::default")]
1733 pub builder: Builder,
1734 }
1735
1736- impl Config {
1737- fn address_default() -> String {
1738- runtime_dir().to_str().unwrap().to_string() + "/ayllu-build.sock"
1739- }
1740-
1741- fn log_path() -> String {
1742- let mut data_path = data_dir();
1743- data_path.push("worker_logs");
1744- data_path.to_str().unwrap().to_string()
1745- }
1746- }
1747-
1748 impl Configurable for Config {}
1749
1750 pub fn load(path: Option<&Path>) -> Result<Config, Error> {
1751 diff --git a/ayllu-build/src/error.rs b/ayllu-build/src/error.rs
1752new file mode 100644
1753index 0000000..c91a359
1754--- /dev/null
1755+++ b/ayllu-build/src/error.rs
1756 @@ -0,0 +1,44 @@
1757+ use std::path::PathBuf;
1758+
1759+ #[derive(Debug)]
1760+ pub enum Error {
1761+ CannotReadManifest {
1762+ path: PathBuf,
1763+ io_err: std::io::Error,
1764+ },
1765+ InvalidManifest {
1766+ path: PathBuf,
1767+ json_err: serde_json::Error,
1768+ },
1769+ EmptyWorkflow {
1770+ name: String,
1771+ },
1772+ CycleDetected,
1773+ DuplicateStepNames {
1774+ name: String,
1775+ },
1776+ DuplicateWorkflows,
1777+ // general io error during execution
1778+ Io(std::io::Error),
1779+ Db(ayllu_database::Error),
1780+ }
1781+
1782+ impl std::fmt::Display for Error {
1783+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1784+ todo!()
1785+ }
1786+ }
1787+
1788+ impl std::error::Error for Error {}
1789+
1790+ impl From<ayllu_database::Error> for Error {
1791+ fn from(value: ayllu_database::Error) -> Self {
1792+ Error::Db(value)
1793+ }
1794+ }
1795+
1796+ impl From<std::io::Error> for Error {
1797+ fn from(value: std::io::Error) -> Self {
1798+ Error::Io(value)
1799+ }
1800+ }
1801 diff --git a/ayllu-build/src/evaluate.rs b/ayllu-build/src/evaluate.rs
1802index 644b4d1..9c9e0eb 100644
1803--- a/ayllu-build/src/evaluate.rs
1804+++ b/ayllu-build/src/evaluate.rs
1805 @@ -1,114 +1,27 @@
1806 use std::collections::HashMap;
1807- use std::fmt::Display;
1808 use std::fs::metadata;
1809 use std::path::PathBuf;
1810- use std::time::Instant;
1811
1812- use anyhow::{format_err, Result};
1813 use petgraph::{
1814 algo::is_cyclic_directed,
1815 dot::Dot,
1816 graph::{Graph, NodeIndex},
1817 visit::Topo,
1818 };
1819- use serde::Deserialize;
1820- use tracing::log::{debug, info, warn};
1821
1822- use crate::database_ext::BuildExt;
1823- use crate::executor::{Context, Executor, Local};
1824- use crate::models::{Manifest, Step, Workflow};
1825- use ayllu_database::Wrapper as Database;
1826-
1827- type Ids<'a> = HashMap<&'a str, (i64, HashMap<&'a str, i64>)>;
1828-
1829- pub(crate) struct State<'a> {
1830- start_time: Instant,
1831- ids: Ids<'a>,
1832- // active workflow
1833- workflow: Option<(String, Instant)>,
1834- // active step
1835- step: Option<(String, Instant)>,
1836- }
1837-
1838- impl<'a> State<'a> {
1839- pub fn new(ids: Ids<'a>) -> Self {
1840- State {
1841- start_time: Instant::now(),
1842- ids,
1843- workflow: None,
1844- step: None,
1845- }
1846- }
1847-
1848- pub fn id(&self, unit: &Unit) -> i64 {
1849- match unit {
1850- Unit::Step(step) => match &self.workflow {
1851- Some(current) => self.ids[current.0.as_str()].1[step.name.as_str()],
1852- None => unreachable!(),
1853- },
1854- Unit::Workflow(workflow) => self.ids[workflow.name.as_str()].0,
1855- }
1856- }
1857-
1858- pub fn start(&mut self, unit: &Unit) {
1859- match unit {
1860- Unit::Step(step) => {
1861- self.step = Some((step.name.to_string(), Instant::now()));
1862- }
1863- Unit::Workflow(workflow) => {
1864- self.workflow = Some((workflow.name.to_string(), Instant::now()));
1865- }
1866- }
1867- }
1868-
1869- pub fn current_workflow(&self) -> Option<(i64, i64)> {
1870- match &self.workflow {
1871- Some(current) => {
1872- let id = self.ids[current.0.as_str()].0;
1873- let duration = Instant::now().duration_since(current.1).as_millis() as i64;
1874- Some((id, duration))
1875- }
1876- None => None,
1877- }
1878- }
1879-
1880- pub fn current_step(&self) -> Option<(i64, i64)> {
1881- match &self.step {
1882- Some(current_step) => match &self.workflow {
1883- Some(current_workflow) => {
1884- let id = self.ids[current_workflow.0.as_str()].1[current_step.0.as_str()];
1885- let duration = Instant::now().duration_since(current_step.1).as_millis() as i64;
1886- Some((id, duration))
1887- }
1888- None => unreachable!(),
1889- },
1890- None => todo!(),
1891- }
1892- }
1893-
1894- pub fn runtime(&self) -> i64 {
1895- Instant::now().duration_since(self.start_time).as_millis() as i64
1896- }
1897- }
1898+ use crate::models::{Manifest, Step};
1899+ use crate::{
1900+ error::Error,
1901+ executor::{Context, Executor, Local},
1902+ };
1903+ use ayllu_database::{
1904+ build::{BuildExt, BuildTx},
1905+ Tx, Wrapper as Database,
1906+ };
1907
1908- #[derive(Deserialize, Debug, Clone)]
1909- pub enum Unit {
1910- Step(Step),
1911- Workflow(Workflow),
1912- }
1913+ use ayllu_api::build::Unit;
1914
1915- impl Display for Unit {
1916- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1917- match self {
1918- Unit::Step(step) => {
1919- write!(f, "Step: {}", step.name)
1920- }
1921- Unit::Workflow(workflow) => {
1922- write!(f, "Workflow: {}", workflow.name)
1923- }
1924- }
1925- }
1926- }
1927+ pub type BuildGraph = Graph<Unit, u8>;
1928
1929 #[derive(Clone)]
1930 #[allow(dead_code)]
1931 @@ -121,63 +34,17 @@ pub enum Source {
1932 Clone((String, String)),
1933 }
1934
1935- #[derive(Default)]
1936- pub struct RuntimeBuilder {
1937- parallelism: bool,
1938- tee_output: bool,
1939- log_dir: Option<PathBuf>,
1940- source: Option<Source>,
1941- db: Option<Database>,
1942- }
1943-
1944- impl RuntimeBuilder {
1945- pub fn database(mut self, db: Database) -> RuntimeBuilder {
1946- self.db = Some(db);
1947- self
1948- }
1949-
1950- pub fn source(mut self, source: Source) -> RuntimeBuilder {
1951- self.source = Some(source);
1952- self
1953- }
1954-
1955- pub fn log_dir(mut self, log_dir: PathBuf) -> RuntimeBuilder {
1956- self.log_dir = Some(log_dir);
1957- self
1958- }
1959-
1960- pub fn tee_output(mut self, tee_output: bool) -> RuntimeBuilder {
1961- self.tee_output = tee_output;
1962- self
1963- }
1964-
1965- #[allow(dead_code)]
1966- pub fn enable_parallelism(self, _enabled: bool) -> RuntimeBuilder {
1967- todo!();
1968- }
1969-
1970- pub fn build(&self) -> Runtime {
1971- Runtime {
1972- parallelism: self.parallelism,
1973- tee_output: self.tee_output,
1974- db: self.db.clone(),
1975- source: self.source.clone().unwrap(),
1976- log_dir: self.log_dir.clone().unwrap_or(String::from("/tmp").into()),
1977- }
1978- }
1979- }
1980-
1981 #[allow(dead_code)]
1982 pub(crate) struct Runtime {
1983- parallelism: bool,
1984- tee_output: bool,
1985- db: Option<Database>,
1986- source: Source,
1987- log_dir: PathBuf,
1988+ // parallelism: bool,
1989+ pub tee_output: bool,
1990+ pub db: Database,
1991+ pub source: Source,
1992+ pub log_dir: PathBuf,
1993 }
1994
1995 impl Runtime {
1996- fn setup(&self) -> Result<Manifest> {
1997+ fn setup(&self) -> Result<Manifest, Error> {
1998 match &self.source {
1999 Source::Manifest(manifest) => Ok(manifest.clone()),
2000 Source::Path(path) => {
2001 @@ -189,7 +56,10 @@ impl Runtime {
2002 Manifest::from_dir(path)
2003 }
2004 }
2005- Err(err) => Err(err.into()),
2006+ Err(err) => Err(Error::CannotReadManifest {
2007+ path: path.to_path_buf(),
2008+ io_err: err,
2009+ }),
2010 }
2011 // let manifest = Manifest::from_path(path)?;
2012 }
2013 @@ -197,26 +67,56 @@ impl Runtime {
2014 }
2015 }
2016
2017- pub fn plan(&self) -> Result<Graph<Unit, i32>> {
2018+ // FIXME: Fundementally broken, assumes everything is a git repo with fragile paths
2019+ fn git_information(&self) -> Result<(String, String, String), ayllu_git::Error> {
2020+ let manifest_path = match &self.source {
2021+ Source::Manifest(_) => todo!(),
2022+ Source::Path(path_buf) => std::fs::canonicalize(path_buf).unwrap(),
2023+ Source::Clone(_) => todo!(),
2024+ };
2025+ let repo_path = manifest_path.parent().unwrap();
2026+ let (collection, name) = ayllu_git::collection_and_name(repo_path);
2027+ let repository = ayllu_git::Wrapper::new(repo_path)?;
2028+ let latest_hash = repository.latest_hash()?.unwrap();
2029+ Ok((collection, name, latest_hash))
2030+ }
2031+
2032+ /// Allocate the manifest in the database along with it's DAG
2033+ async fn allocate(&self, tx: &mut Tx) -> Result<(i64, BuildGraph), Error> {
2034 let manifest = self.setup()?;
2035+ manifest.validate()?;
2036+ let (collection, name, git_hash) = self.git_information().unwrap();
2037+ let manifest_id = tx.create_manifest(&collection, &name, &git_hash).await?;
2038 let mut workflows_by_name: HashMap<String, NodeIndex> = HashMap::new();
2039 let mut steps_by_name: HashMap<String, HashMap<String, NodeIndex>> = HashMap::new();
2040- let mut graph: Graph<Unit, i32> = Graph::new();
2041+ let mut graph: BuildGraph = Graph::new();
2042+ // Allocate components into the database
2043 for workflow in manifest.workflows.iter() {
2044 if workflow.steps.is_empty() {
2045- return Err(format_err!("workflow {} contains no steps", workflow.name));
2046+ return Err(Error::EmptyWorkflow {
2047+ name: workflow.name.clone(),
2048+ });
2049 }
2050+ let workflow_id = tx.create_workflow(manifest_id, &workflow.name).await?;
2051 workflows_by_name.insert(
2052 workflow.name.clone(),
2053- graph.add_node(Unit::Workflow(workflow.clone())),
2054+ graph.add_node(Unit::Workflow(workflow_id)),
2055 );
2056 let mut by_name: HashMap<String, NodeIndex> = HashMap::new();
2057 for step in workflow.steps.iter() {
2058- by_name.insert(step.name.clone(), graph.add_node(Unit::Step(step.clone())));
2059+ let step_id = tx
2060+ .create_step(
2061+ workflow_id,
2062+ &step.name,
2063+ &step.input,
2064+ &step.shell,
2065+ step.environment.clone(),
2066+ )
2067+ .await?;
2068+ by_name.insert(step.name.clone(), graph.add_node(Unit::Step(step_id)));
2069 }
2070 steps_by_name.insert(workflow.name.clone(), by_name);
2071 }
2072-
2073 // set edges for steps from each workflow
2074 for workflow in manifest.workflows.iter() {
2075 let workflow_steps = steps_by_name.get(&workflow.name).unwrap();
2076 @@ -228,6 +128,7 @@ impl Runtime {
2077 }
2078 }
2079 }
2080+
2081 for workflow in manifest.workflows.iter() {
2082 let workflow_index = workflows_by_name.get(&workflow.name).unwrap();
2083 let steps_with_no_dependencies: Vec<&Step> = workflow
2084 @@ -254,82 +155,64 @@ impl Runtime {
2085 }
2086
2087 if is_cyclic_directed(&graph) {
2088- return Err(format_err!("graph contains a cycle"));
2089+ return Err(Error::CycleDetected);
2090 }
2091
2092- Ok(graph)
2093- }
2094+ let dag_json = serde_json::ser::to_string(&graph).unwrap();
2095+
2096+ tx.create_dag(manifest_id, &dag_json).await?;
2097
2098- pub fn to_dot(&self, graph: &Graph<Unit, i32>) -> Result<String> {
2099- Ok(Dot::to_string(&Dot::with_config(graph, &[])))
2100+ Ok((manifest_id, graph))
2101 }
2102
2103- pub async fn eval(&self) -> Result<()> {
2104- // TODO: samples
2105- let manifest = self.setup()?;
2106- let job_graph = self.plan()?;
2107- let db = self.db.as_ref().unwrap();
2108- let manifest_id = db.create_manifest("todo", "todo").await?;
2109- let graph = self.plan()?;
2110- let dag_content = self.to_dot(&graph)?;
2111- let dag_id = db.create_dag(manifest_id, &dag_content).await?;
2112- info!(
2113+ /// Allocate a job graph and then execute it sequentially
2114+ pub async fn evaluate(&self) -> Result<i64, Error> {
2115+ let mut tx = self.db.begin().await?;
2116+ let (manifest_id, graph) = match self.allocate(&mut tx).await {
2117+ Ok(db_op) => {
2118+ tx.commit().await?;
2119+ db_op
2120+ }
2121+ Err(e) => {
2122+ tracing::error!("Failed to allocate graph: {e:?}");
2123+ tx.rollback().await?;
2124+ // FIXME rollback but how
2125+ return Err(e);
2126+ }
2127+ };
2128+ let dot_string = Dot::new(&graph).to_string();
2129+ tracing::info!(
2130 "evaluating DAG [{}] [n_nodes={}]:\n{}",
2131- dag_id,
2132- job_graph.node_count(),
2133- dag_content
2134+ manifest_id,
2135+ graph.node_count(),
2136+ dot_string
2137 );
2138- debug!("created manifest: {}", manifest_id);
2139- let mut ids: Ids = HashMap::new();
2140- for workflow in manifest.workflows.iter() {
2141- let workflow_id = db.create_workflow(manifest_id, &workflow.name).await?;
2142- debug!("created workflow {}", workflow_id);
2143- let mut step_ids: HashMap<&str, i64> = HashMap::new();
2144- for step in workflow.steps.iter() {
2145- let step_id = db
2146- .create_step(
2147- workflow_id,
2148- &step.name,
2149- &step.input,
2150- &step.shell,
2151- step.environment.clone(),
2152- )
2153- .await?;
2154- debug!("created step {}", step_id);
2155- step_ids.insert(step.name.as_str(), step_id);
2156- }
2157- ids.insert(workflow.name.as_str(), (workflow_id, step_ids));
2158- }
2159-
2160- db.update_manifest_start(manifest_id).await?;
2161- let mut state = State::new(ids);
2162-
2163+ self.db.update_manifest_start(manifest_id).await?;
2164 // TODO: Currently no parallelism is supported, need to implement the
2165 // options in the manifest and then break steps into asynchronous chunks
2166 // that can be run in parallel where appropriate.
2167- let mut topo = Topo::new(&job_graph);
2168- while let Some(nx) = topo.next(&job_graph) {
2169- let unit = &job_graph[nx];
2170+ let mut topo = Topo::new(&graph);
2171+ let mut current_workflow: Option<i64> = None;
2172+ while let Some(nx) = topo.next(&graph) {
2173+ let unit = &graph[nx];
2174 match unit {
2175- Unit::Workflow(workflow) => {
2176- info!("starting workflow {}", workflow.name);
2177- match state.current_workflow() {
2178- Some((current_workflow_id, duration)) => {
2179- db.update_workflow_finish(current_workflow_id, duration)
2180- .await?;
2181- state.start(unit);
2182- db.update_workflow_start(state.id(unit)).await?;
2183+ Unit::Workflow(next_workflow_id) => {
2184+ let next_workflow = self.db.read_workflow(*next_workflow_id).await?;
2185+ tracing::info!("starting workflow {}", next_workflow.name);
2186+ match current_workflow.as_ref() {
2187+ Some(current_workflow_id) => {
2188+ self.db.update_workflow_finish(*current_workflow_id).await?;
2189 }
2190 None => {
2191- state.start(unit);
2192- db.update_workflow_start(state.id(unit)).await?;
2193+ self.db.update_workflow_start(next_workflow.id).await?;
2194 }
2195- }
2196+ };
2197+ current_workflow = Some(*next_workflow_id);
2198 }
2199- Unit::Step(step) => {
2200- info!("starting step: {}", step.name);
2201- state.start(unit);
2202- db.update_step_start(state.id(unit)).await?;
2203+ Unit::Step(next_step_id) => {
2204+ let next_step = self.db.read_step(*next_step_id).await?;
2205+ tracing::info!("starting step: {}", next_step.name);
2206+ self.db.update_step_start(next_step.id).await?;
2207 let executor = Local {
2208 temp_dir: self.log_dir.clone(),
2209 tee_output: self.tee_output,
2210 @@ -338,102 +221,102 @@ impl Runtime {
2211 // TODO: more context
2212 let ctx = Context {
2213 manifest_id,
2214- workflow_id: state.current_workflow().unwrap().0,
2215+ workflow_id: current_workflow.unwrap(),
2216 ..Default::default()
2217 };
2218
2219- let (stdout, stderr, exit_code) = executor.execute(step, ctx)?;
2220+ let (stdout, stderr, exit_code) = executor.execute(&next_step, ctx)?;
2221 if !exit_code.success() {
2222- warn!("step {} has failed: {:?}", step.name, exit_code);
2223+ tracing::warn!("step {} has failed: {:?}", next_step.name, exit_code);
2224 }
2225- let (step_id, duration) = state.current_step().unwrap();
2226- db.update_step_finish(
2227- step_id,
2228- duration,
2229- &stdout,
2230- &stderr,
2231- exit_code.code().unwrap() as i8,
2232- )
2233- .await?;
2234+ // let (step_id, duration) = state.current_step().unwrap();
2235+ self.db
2236+ .update_step_finish(
2237+ *next_step_id,
2238+ &stdout,
2239+ &stderr,
2240+ exit_code.code().unwrap() as i8,
2241+ )
2242+ .await?;
2243 }
2244 }
2245 }
2246 // clean up last workflow
2247- if let Some((id, duration)) = state.current_workflow() {
2248- db.update_workflow_finish(id, duration).await?;
2249+ if let Some(id) = current_workflow {
2250+ self.db.update_workflow_finish(id).await?;
2251 }
2252
2253- db.update_manifest_finish(manifest_id, state.runtime())
2254- .await?;
2255- Ok(())
2256+ self.db.update_manifest_finish(manifest_id).await?;
2257+
2258+ Ok(manifest_id)
2259 }
2260 }
2261
2262 #[cfg(test)]
2263 mod tests {
2264
2265- use super::*;
2266- use std::path::Path;
2267-
2268- use ayllu_database::Builder;
2269-
2270- #[tokio::test]
2271- async fn test_manifest_simple() {
2272- let db = Builder::default()
2273- .migrations("./migrations")
2274- .build()
2275- .await
2276- .expect("failed to setup db");
2277- let rt = RuntimeBuilder::default()
2278- .source(Source::Path(PathBuf::from("tests/simple.ncl")))
2279- .log_dir(Path::new("logs").to_path_buf())
2280- .database(db.clone())
2281- .build();
2282- rt.eval().await.expect("failed to evaluate manifest");
2283- let normalized = db
2284- .read_manifest(1)
2285- .await
2286- .expect("failed to retreive manifest");
2287- assert!(normalized.manifest.id == 1);
2288- let workflow = normalized
2289- .workflows
2290- .first()
2291- .expect("first workflow is missing");
2292- assert!(workflow.name == "Simple");
2293- assert!(workflow.duration.is_some_and(|duration| duration > 0));
2294- let steps = normalized
2295- .steps
2296- .first()
2297- .expect("missing first set of steps");
2298- let step_1 = steps.first().expect("first step is missing");
2299- assert!(step_1.id == 1);
2300- assert!(step_1.name == "Hello");
2301- assert!(step_1
2302- .stdout
2303- .as_ref()
2304- .is_some_and(|output| output == "Hello"));
2305- let step_2 = steps.get(1).expect("second step is missing");
2306- assert!(step_2.id == 2);
2307- assert!(step_2.name == "World");
2308- assert!(step_2
2309- .stdout
2310- .as_ref()
2311- .is_some_and(|output| output == "World"));
2312- }
2313-
2314- #[tokio::test]
2315- async fn test_manifest_complex() {
2316- let db = Builder::default()
2317- .migrations("./migrations")
2318- .build()
2319- .await
2320- .expect("failed to setup db");
2321- let rt = RuntimeBuilder::default()
2322- .source(Source::Path(PathBuf::from("tests/complex.ncl")))
2323- .log_dir(Path::new("logs").to_path_buf())
2324- .database(db.clone())
2325- .build();
2326- rt.eval().await.expect("failed to evaluate manifest");
2327- // TODO order of evaluation needs to be preserved and tested
2328- }
2329+ // use super::*;
2330+ // use std::path::Path;
2331+
2332+ // use ayllu_database::Builder;
2333+
2334+ // #[tokio::test]
2335+ // async fn test_manifest_simple() {
2336+ // let db = Builder::default()
2337+ // .migrations(Path::new("./migrations"))
2338+ // .build()
2339+ // .await
2340+ // .expect("failed to setup db");
2341+ // let rt = RuntimeBuilder::default()
2342+ // .source(Source::Path(PathBuf::from("tests/simple.json")))
2343+ // .log_dir(Path::new("logs").to_path_buf())
2344+ // .database(db.clone())
2345+ // .build();
2346+ // rt.eval().await.expect("failed to evaluate manifest");
2347+ // let normalized = db
2348+ // .read_manifest(1)
2349+ // .await
2350+ // .expect("failed to retreive manifest");
2351+ // assert!(normalized.manifest.id == 1);
2352+ // let workflow = normalized
2353+ // .workflows
2354+ // .first()
2355+ // .expect("first workflow is missing");
2356+ // assert!(workflow.name == "Simple");
2357+ // assert!(workflow.duration.is_some_and(|duration| duration > 0));
2358+ // let steps = normalized
2359+ // .steps
2360+ // .first()
2361+ // .expect("missing first set of steps");
2362+ // let step_1 = steps.first().expect("first step is missing");
2363+ // assert!(step_1.id == 1);
2364+ // assert!(step_1.name == "Hello");
2365+ // assert!(step_1
2366+ // .stdout
2367+ // .as_ref()
2368+ // .is_some_and(|output| output == "Hello"));
2369+ // let step_2 = steps.get(1).expect("second step is missing");
2370+ // assert!(step_2.id == 2);
2371+ // assert!(step_2.name == "World");
2372+ // assert!(step_2
2373+ // .stdout
2374+ // .as_ref()
2375+ // .is_some_and(|output| output == "World"));
2376+ // }
2377+
2378+ // #[tokio::test]
2379+ // async fn test_manifest_complex() {
2380+ // let db = Builder::default()
2381+ // .migrations("./migrations")
2382+ // .build()
2383+ // .await
2384+ // .expect("failed to setup db");
2385+ // let rt = RuntimeBuilder::default()
2386+ // .source(Source::Path(PathBuf::from("tests/complex.ncl")))
2387+ // .log_dir(Path::new("logs").to_path_buf())
2388+ // .database(db.clone())
2389+ // .build();
2390+ // rt.eval().await.expect("failed to evaluate manifest");
2391+ // // TODO order of evaluation needs to be preserved and tested
2392+ // }
2393 }
2394 diff --git a/ayllu-build/src/executor.rs b/ayllu-build/src/executor.rs
2395index 56371aa..a97e2d7 100644
2396--- a/ayllu-build/src/executor.rs
2397+++ b/ayllu-build/src/executor.rs
2398 @@ -10,11 +10,11 @@ use std::{
2399 io::{Read, Write},
2400 };
2401
2402- use anyhow::Result;
2403+ use ayllu_database::build::Step;
2404 use serde::Deserialize;
2405 use tracing::log::{debug, info};
2406
2407- use crate::models::Step;
2408+ use crate::error::Error;
2409
2410 /// standard stream when logging output
2411 enum Output {
2412 @@ -42,12 +42,14 @@ pub struct Context {
2413 pub workflow_id: i64,
2414 pub git_hash: String,
2415 pub repo_url: String,
2416+ pub environment: HashMap<String, Option<String>>,
2417 }
2418
2419 // An executor runs the step in a process container of some kind. Currently
2420 // the only executor that exists is the Local exector.
2421 pub trait Executor {
2422- fn execute(&self, step: &Step, context: Context) -> Result<(String, String, ExitStatus)>;
2423+ fn execute(&self, step: &Step, context: Context)
2424+ -> Result<(String, String, ExitStatus), Error>;
2425 }
2426
2427 // build executor that runs with the same permissions as the build server.
2428 @@ -92,7 +94,11 @@ fn write_stream(
2429
2430 impl Executor for Local {
2431 // TODO: once parallelism is enabled this should be converted to tokio
2432- fn execute(&self, step: &Step, context: Context) -> Result<(String, String, ExitStatus)> {
2433+ fn execute(
2434+ &self,
2435+ step: &Step,
2436+ context: Context,
2437+ ) -> Result<(String, String, ExitStatus), Error> {
2438 let temp_dir = self
2439 .temp_dir
2440 .as_path()
2441 @@ -107,7 +113,11 @@ impl Executor for Local {
2442 let mut filtered_env: HashMap<String, String> = env::vars()
2443 .filter(|(k, _)| k == "TERM" || k == "TZ" || k == "LANG" || k == "PATH")
2444 .collect();
2445- filtered_env.extend(step.environment.clone());
2446+ // FIXME: If user specified None for an env it should override the defaults
2447+ filtered_env.extend(context.environment.iter().filter_map(|env| match env.1 {
2448+ Some(value) => Some((env.0.clone(), value.clone())),
2449+ None => None,
2450+ }));
2451 filtered_env.extend([
2452 (String::from("AYLLU_GIT_HASH"), context.git_hash.clone()),
2453 (String::from("AYLLU_REPO_URL"), context.repo_url.clone()),
2454 @@ -162,27 +172,27 @@ impl Executor for Local {
2455 #[cfg(test)]
2456 mod tests {
2457
2458- use super::*;
2459-
2460- #[tokio::test]
2461- async fn test_local_executor() {
2462- let executor = Local {
2463- temp_dir: Path::new("logs").to_path_buf(),
2464- tee_output: true,
2465- };
2466- let result = executor
2467- .execute(
2468- &Step {
2469- name: String::from("test"),
2470- shell: String::from("/bin/sh"),
2471- input: String::from("echo -n hello"),
2472- depends_on: Vec::new(),
2473- environment: HashMap::new(),
2474- },
2475- Context::default(),
2476- )
2477- .expect("failed to run command");
2478- assert!(result.2.success());
2479- assert!(result.0 == "hello");
2480- }
2481+ // use super::*;
2482+
2483+ // #[tokio::test]
2484+ // async fn test_local_executor() {
2485+ // let executor = Local {
2486+ // temp_dir: Path::new("logs").to_path_buf(),
2487+ // tee_output: true,
2488+ // };
2489+ // let result = executor
2490+ // .execute(
2491+ // &Step {
2492+ // name: String::from("test"),
2493+ // shell: String::from("/bin/sh"),
2494+ // input: String::from("echo -n hello"),
2495+ // depends_on: Vec::new(),
2496+ // environment: HashMap::new(),
2497+ // },
2498+ // Context::default(),
2499+ // )
2500+ // .expect("failed to run command");
2501+ // assert!(result.2.success());
2502+ // assert!(result.0 == "hello");
2503+ // }
2504 }
2505 diff --git a/ayllu-build/src/main.rs b/ayllu-build/src/main.rs
2506index 0333534..a8247ea 100644
2507--- a/ayllu-build/src/main.rs
2508+++ b/ayllu-build/src/main.rs
2509 @@ -1,20 +1,20 @@
2510 use std::io::stdout;
2511 use std::path::{Path, PathBuf};
2512- use std::str::FromStr;
2513
2514- use anyhow::Result;
2515 use clap::{arg, Command, CommandFactory, Parser, Subcommand};
2516 use clap_complete::{generate, Generator, Shell};
2517- use tokio::task::LocalSet;
2518 use tracing::Level;
2519
2520 use ayllu_database::Builder;
2521+
2522+ use crate::evaluate::{Runtime, Source};
2523 mod config;
2524- mod database_ext;
2525+ mod error;
2526 mod evaluate;
2527 mod executor;
2528 mod models;
2529- mod rpc_server;
2530+
2531+ const DEFAULT_BUILD_FILE: &str = ".ayllu-build.json";
2532
2533 #[derive(Parser)]
2534 #[command(author, version, about, long_about = None)]
2535 @@ -40,10 +40,6 @@ enum Commands {
2536 #[arg(long)]
2537 shell: Shell,
2538 },
2539- /// run migrations against the database
2540- Migrate {},
2541- /// run the build server and listen for commands
2542- Serve {},
2543 /// evaluate a local build script
2544 Evaluate {
2545 /// Path to your configuration file
2546 @@ -53,12 +49,6 @@ enum Commands {
2547 #[arg(short, long, action)]
2548 tee_output: bool,
2549 },
2550- /// generate an execution plan and output a DOT script
2551- Plan {
2552- /// Path to your configuration file
2553- #[arg(short, long, value_name = "FILE")]
2554- alternate_path: Option<PathBuf>,
2555- },
2556 }
2557
2558 fn print_completions<G: Generator>(gen: G, cmd: &mut Command) {
2559 @@ -66,69 +56,34 @@ fn print_completions<G: Generator>(gen: G, cmd: &mut Command) {
2560 }
2561
2562 #[tokio::main(flavor = "current_thread")]
2563- async fn main() -> Result<()> {
2564+ async fn main() -> Result<(), Box<dyn std::error::Error>> {
2565 let cli = Cli::parse();
2566 let cfg = config::load(cli.config.as_deref())?;
2567- let log_level = Level::from_str(&cfg.log_level)?;
2568- tracing_subscriber::fmt()
2569- .compact()
2570- .with_line_number(true)
2571- .with_level(true)
2572- .with_max_level(cli.level.unwrap_or(log_level))
2573- .init();
2574- tracing::info!("logger initialized");
2575- let default_build_dir = PathBuf::from_str(".ayllu/build").unwrap();
2576+ ayllu_logging::init(Level::INFO); // FIXME
2577 match cli.command {
2578 Commands::Complete { shell } => {
2579 let mut cmd = Cli::command();
2580 print_completions(shell, &mut cmd);
2581 Ok(())
2582 }
2583- Commands::Migrate {} => {
2584- let db = Builder::default()
2585- .url(&cfg.builder.database.path)
2586- .migrations("./migrations")
2587- .build()
2588- .await?;
2589- db.close().await?;
2590- Ok(())
2591- }
2592- Commands::Serve {} => {
2593- LocalSet::new()
2594- .run_until(rpc_server::serve(&cfg.builder))
2595- .await?;
2596- Ok(())
2597- }
2598 Commands::Evaluate {
2599 alternate_path,
2600 tee_output,
2601 } => {
2602- // TODO fix me
2603 let db = Builder::default()
2604- .url(&cfg.builder.database.path)
2605- .log_queries(log_level == Level::DEBUG)
2606+ .path(cfg.database.path.as_path())
2607+ .log_queries(false) // FIXME
2608 .build()
2609 .await?;
2610- let rt = evaluate::RuntimeBuilder::default()
2611- .source(evaluate::Source::Path(
2612- alternate_path.unwrap_or(default_build_dir),
2613- ))
2614- .database(db)
2615- .log_dir(Path::new(&cfg.builder.log_path).to_path_buf())
2616- .tee_output(tee_output)
2617- .build();
2618- rt.eval().await?;
2619- Ok(())
2620- }
2621- Commands::Plan { alternate_path } => {
2622- let rt = evaluate::RuntimeBuilder::default()
2623- .source(evaluate::Source::Path(
2624- alternate_path.unwrap_or(default_build_dir),
2625- ))
2626- .build();
2627- let graph = rt.plan()?;
2628- let dot = rt.to_dot(&graph)?;
2629- print!("{}", dot);
2630+ let rt = Runtime {
2631+ db,
2632+ source: Source::Path(
2633+ alternate_path.unwrap_or(Path::new(DEFAULT_BUILD_FILE).to_path_buf()),
2634+ ),
2635+ tee_output,
2636+ log_dir: cfg.builder.log_path,
2637+ };
2638+ rt.evaluate().await?;
2639 Ok(())
2640 }
2641 }
2642 diff --git a/ayllu-build/src/models.rs b/ayllu-build/src/models.rs
2643index 1126c50..ee4e282 100644
2644--- a/ayllu-build/src/models.rs
2645+++ b/ayllu-build/src/models.rs
2646 @@ -1,12 +1,10 @@
2647 use std::collections::{HashMap, HashSet};
2648 use std::fmt::Display;
2649- use std::fs::read_dir;
2650- use std::path::{Path, PathBuf};
2651+ use std::path::Path;
2652
2653- use anyhow::{format_err, Result};
2654- use nickel_lang_core::{eval::cache::lazy::CBNCache, program::Program};
2655 use serde::Deserialize;
2656- use tracing::debug;
2657+
2658+ use crate::error::Error;
2659
2660 // pub type TestGraph = DiGraph<(Job, DiGraph<Step, i32>), i32>;
2661
2662 @@ -25,7 +23,7 @@ pub struct Step {
2663 }
2664
2665 impl Step {
2666- pub fn validate(&self) -> Result<()> {
2667+ pub fn validate(&self) -> Result<(), Error> {
2668 Ok(())
2669 }
2670
2671 @@ -43,19 +41,18 @@ pub struct Workflow {
2672 }
2673
2674 impl Workflow {
2675- pub fn validate(&self) -> Result<()> {
2676+ pub fn validate(&self) -> Result<(), Error> {
2677 let mut names = HashSet::new();
2678 if !self
2679 .steps
2680 .iter()
2681 .all(move |step| names.insert(step.name.clone()))
2682 {
2683- return Err(format_err!(
2684- "workflow {} contains steps with duplicate names",
2685- self.name
2686- ));
2687+ return Err(Error::DuplicateStepNames {
2688+ name: self.name.clone(),
2689+ });
2690 }
2691- let result: Result<()> = self.steps.iter().try_for_each(|step| step.validate());
2692+ let result: Result<(), Error> = self.steps.iter().try_for_each(|step| step.validate());
2693 result?;
2694 Ok(())
2695 }
2696 @@ -73,80 +70,70 @@ pub struct Manifest {
2697 }
2698
2699 impl Manifest {
2700- pub fn validate(&self) -> Result<()> {
2701+ pub fn validate(&self) -> Result<(), Error> {
2702 let mut names = HashSet::new();
2703 if !self
2704 .workflows
2705 .iter()
2706 .all(move |workflow| names.insert(workflow.name.clone()))
2707 {
2708- return Err(format_err!(
2709- "manifest contains workflows with duplicate names"
2710- ));
2711+ return Err(Error::DuplicateWorkflows);
2712 }
2713- let result: Result<()> = self
2714+ let result: Result<(), Error> = self
2715 .workflows
2716- .iter().try_for_each(|workflow| workflow.validate());
2717+ .iter()
2718+ .try_for_each(|workflow| workflow.validate());
2719 result?;
2720 Ok(())
2721 }
2722 }
2723
2724 impl Manifest {
2725- pub fn from_file(path: &Path) -> Result<Self> {
2726- let mut result = Program::<CBNCache>::new_from_file(path, Vec::new())?;
2727- match result.eval_full() {
2728- Ok(term) => {
2729- let manifest = Manifest::deserialize(term)?;
2730- manifest.validate()?;
2731- Ok(manifest)
2732- }
2733- Err(err) => Err(format_err!("failed to load manifest: {:?}", err)),
2734- }
2735+ pub fn from_file(path: &Path) -> Result<Self, Error> {
2736+ let manifest_str =
2737+ std::fs::read_to_string(path).map_err(|e| Error::CannotReadManifest {
2738+ path: path.to_path_buf(),
2739+ io_err: e,
2740+ })?;
2741+ serde_json::de::from_str(&manifest_str).map_err(|e| Error::InvalidManifest {
2742+ path: path.to_path_buf(),
2743+ json_err: e,
2744+ })
2745 }
2746
2747- pub fn from_dir(path: &Path) -> Result<Self> {
2748- let mut files: Vec<PathBuf> = Vec::new();
2749- for dir_entry in read_dir(path)? {
2750- let entry = dir_entry?;
2751- if entry.file_type()?.is_file() {
2752- debug!("found file {:?}", entry);
2753- files.push(entry.path())
2754- }
2755- }
2756- let mut result = Program::<CBNCache>::new_from_files(files, Vec::new())?;
2757- match result.eval_full() {
2758- Ok(term) => {
2759- let manifest = Manifest::deserialize(term)?;
2760- manifest.validate()?;
2761- Ok(manifest)
2762- }
2763- Err(err) => Err(format_err!("failed to load manifest: {:?}", err)),
2764- }
2765+ pub fn from_dir(path: &Path) -> Result<Self, Error> {
2766+ todo!()
2767+ // let mut files: Vec<PathBuf> = Vec::new();
2768+ // for dir_entry in read_dir(path)? {
2769+ // let entry = dir_entry?;
2770+ // if entry.file_type()?.is_file() {
2771+ // debug!("found file {:?}", entry);
2772+ // files.push(entry.path())
2773+ // }
2774+ // }
2775+ // let mut result = Program::<CBNCache>::new_from_files(files, Vec::new())?;
2776+ // match result.eval_full() {
2777+ // Ok(term) => {
2778+ // let manifest = Manifest::deserialize(term)?;
2779+ // manifest.validate()?;
2780+ // Ok(manifest)
2781+ // }
2782+ // Err(err) => Err(format_err!("failed to load manifest: {:?}", err)),
2783+ // }
2784+ // }
2785 }
2786 }
2787
2788 #[cfg(test)]
2789 mod tests {
2790- use std::path::Path;
2791-
2792- use super::*;
2793+ // use std::path::Path;
2794
2795- #[test]
2796- fn test_manifest_simple() {
2797- let manifest =
2798- Manifest::from_file(Path::new("tests/simple.ncl")).expect("failed to load manifest");
2799- assert!(manifest.workflows.len() == 1);
2800- }
2801+ // use super::*;
2802
2803- #[test]
2804- fn test_read_manifest_test_files() {
2805- read_dir("tests")
2806- .expect("failed to read_dir")
2807- .for_each(|file| {
2808- let file_name = file.as_ref().unwrap().file_name();
2809- Manifest::from_file(&file.unwrap().path())
2810- .unwrap_or_else(|_| panic!("failed to load manifest: {:?}", &file_name));
2811- });
2812- }
2813+ // #[test]
2814+ // fn test_manifest_simple() {
2815+ // let manifest =
2816+ // Manifest::from_file(Path::new("tests/simple.ncl")).expect("failed to load manifest");
2817+ // assert!(manifest.workflows.len() == 1);
2818+ // }
2819 }
2820 diff --git a/ayllu-build/src/rpc_server.rs b/ayllu-build/src/rpc_server.rs
2821deleted file mode 100644
2822index b703541..0000000
2823--- a/ayllu-build/src/rpc_server.rs
2824+++ /dev/null
2825 @@ -1,80 +0,0 @@
2826- use std::path::Path;
2827-
2828- use anyhow::Result;
2829- use tracing::log::info;
2830-
2831- use ayllu_api::{
2832- build::{Event, Manifest, Server},
2833- error::ApiError,
2834- };
2835- use ayllu_database::{Builder, Wrapper as Database};
2836- use ayllu_rpc::{
2837- futures::prelude::*,
2838- init_socket, spawn,
2839- tarpc::{
2840- context::Context,
2841- serde_transport::unix,
2842- server::{BaseChannel, Channel},
2843- tokio_serde::formats::Bincode,
2844- },
2845- };
2846-
2847- use crate::config::Builder as Config;
2848-
2849- #[derive(Clone)]
2850- #[allow(dead_code)]
2851- struct ServerImpl {
2852- db: Database,
2853- }
2854-
2855- impl Server for ServerImpl {
2856- #[doc = r" submit a new event which may result in manifests being run"]
2857- async fn submit(self, _context: Context, _event: Event) -> Result<(), ApiError> {
2858- todo!()
2859- }
2860-
2861- #[doc = r" list manifests available on the worker"]
2862- async fn list_manifests(
2863- self,
2864- _context: Context,
2865- _offset: i64,
2866- _limit: i64,
2867- ) -> Result<Vec<Manifest>, ApiError> {
2868- todo!()
2869- }
2870-
2871- #[doc = r" read a manifest including it's status and all job output"]
2872- async fn read_manifest(
2873- self,
2874- _context: Context,
2875- _manifest_id: String,
2876- ) -> Result<Manifest, ApiError> {
2877- todo!()
2878- }
2879- }
2880-
2881- pub async fn serve(cfg: &Config) -> Result<()> {
2882- let db = Builder::default()
2883- .url(&cfg.database.path)
2884- .migrations("./migrations")
2885- .build()
2886- .await?;
2887- info!("build server listening @ {}", cfg.address);
2888- let socket_path = Path::new(&cfg.address);
2889- init_socket(socket_path)?;
2890- let mut listener = unix::listen(socket_path, Bincode::default).await?;
2891- listener.config_mut().max_frame_length(usize::MAX);
2892- listener
2893- // Ignore accept errors.
2894- .filter_map(|r| future::ready(r.ok()))
2895- .map(BaseChannel::with_defaults)
2896- .map(move |channel| {
2897- let server = ServerImpl { db: db.clone() };
2898- channel.execute(server.serve()).for_each(spawn)
2899- })
2900- // Max 10 channels.
2901- .buffer_unordered(10)
2902- .for_each(|_| async {})
2903- .await;
2904- Ok(())
2905- }
2906 diff --git a/ayllu-build/tests/complex.ncl b/ayllu-build/tests/complex.ncl
2907deleted file mode 100644
2908index 15e357d..0000000
2909--- a/ayllu-build/tests/complex.ncl
2910+++ /dev/null
2911 @@ -1,61 +0,0 @@
2912- {
2913- workflows = [
2914- {
2915- name = "workflow_3",
2916- steps = [
2917- {
2918- name = "W3S1",
2919- input = "true",
2920- depends_on = ["W3S3"]
2921- },
2922- {
2923- name = "W3S2",
2924- input = "true",
2925- depends_on = ["W3S3"]
2926- },
2927- {
2928- name = "W3S3",
2929- input = "true",
2930- }
2931- ]
2932- },
2933- {
2934- name = "workflow_1",
2935- depends_on = ["workflow_2"],
2936- steps = [
2937- {
2938- name = "W1S1",
2939- input = "true",
2940- depends_on = ["W1S2", "W1S4"]
2941- },
2942- {
2943- name = "W1S2",
2944- input = "true",
2945- },
2946- {
2947- name = "W1S3",
2948- input = "true",
2949- },
2950- {
2951- name = "W1S4",
2952-
2953- input = "true",
2954- depends_on = ["W1S3"]
2955- }
2956- ]
2957- },
2958- {
2959- name = "workflow_2",
2960- steps = [
2961- {
2962- name = "W2S2",
2963- input = "true"
2964- },
2965- {
2966- name = "W2S1",
2967- input = "true",
2968- }
2969- ]
2970- },
2971- ]
2972- }
2973 diff --git a/ayllu-build/tests/simple.json b/ayllu-build/tests/simple.json
2974new file mode 100644
2975index 0000000..7a7443d
2976--- /dev/null
2977+++ b/ayllu-build/tests/simple.json
2978 @@ -0,0 +1,35 @@
2979+ {
2980+ "workflows": [
2981+ {
2982+ "name": "lint",
2983+ "steps": [
2984+ {
2985+ "name": "cargo-lint",
2986+ "input": "echo -n cargo lint ; sleep 0.1"
2987+ }
2988+ ]
2989+ },
2990+ {
2991+ "name": "test",
2992+ "steps": [
2993+ {
2994+ "name": "cargo-test",
2995+ "input": "echo -n cargo test ; sleep 0.1"
2996+ },
2997+ {
2998+ "name": "cargo-clippy",
2999+ "input": "echo -n cargo clippy ; sleep 0.1"
3000+ }
3001+ ]
3002+ },
3003+ {
3004+ "name": "build",
3005+ "steps": [
3006+ {
3007+ "name": "cargo-build",
3008+ "input": "echo -n cargo build ; sleep 0.1"
3009+ }
3010+ ]
3011+ }
3012+ ]
3013+ }
3014\ No newline at end of file
3015 diff --git a/ayllu-build/tests/simple.ncl b/ayllu-build/tests/simple.ncl
3016deleted file mode 100644
3017index a6d9705..0000000
3018--- a/ayllu-build/tests/simple.ncl
3019+++ /dev/null
3020 @@ -1,17 +0,0 @@
3021- {
3022- workflows = [
3023- {
3024- name = "Simple",
3025- steps = [
3026- {
3027- name = "Hello",
3028- input = "echo -n Hello"
3029- },
3030- {
3031- name = "World",
3032- input = "echo -n World"
3033- }
3034- ]
3035- }
3036- ]
3037- }
3038 diff --git a/ayllu-migrate/Cargo.toml b/ayllu-migrate/Cargo.toml
3039new file mode 100644
3040index 0000000..b0bbed9
3041--- /dev/null
3042+++ b/ayllu-migrate/Cargo.toml
3043 @@ -0,0 +1,15 @@
3044+ [package]
3045+ name = "ayllu-migrate"
3046+ version = "0.1.0"
3047+ edition = "2024"
3048+
3049+ [dependencies]
3050+
3051+ ayllu_logging = { path = "../crates/logging" }
3052+ ayllu_config = { path = "../crates/config" }
3053+ ayllu_cmd = { path = "../crates/cmd" }
3054+ ayllu_database = { path = "../crates/database" }
3055+
3056+ serde = { workspace = true }
3057+ tokio = { workspace = true }
3058+ tracing = { workspace = true }
3059 diff --git a/ayllu-migrate/src/main.rs b/ayllu-migrate/src/main.rs
3060new file mode 100644
3061index 0000000..ba7e127
3062--- /dev/null
3063+++ b/ayllu-migrate/src/main.rs
3064 @@ -0,0 +1,39 @@
3065+ use ayllu_cmd::migrate::Command;
3066+ use ayllu_config::{Configurable, Database};
3067+ use serde::{Deserialize, Serialize};
3068+ use tracing::Level;
3069+
3070+ #[derive(Deserialize, Serialize)]
3071+ pub struct Config {
3072+ #[serde(default = "Database::default")]
3073+ pub database: Database,
3074+ }
3075+
3076+ impl Configurable for Config {}
3077+
3078+ #[tokio::main(flavor = "current_thread")]
3079+ async fn main() -> Result<(), Box<dyn std::error::Error>> {
3080+ let args = ayllu_cmd::parse::<Command>();
3081+ let config: Config = ayllu_config::Reader::load(args.config.as_deref())?;
3082+
3083+ ayllu_logging::init(Level::INFO);
3084+
3085+ let database_path = config.database.path.as_path();
3086+
3087+ let migrations_path = args.migrations.unwrap_or(config.database.migrations);
3088+
3089+ tracing::info!("Running migrations from {migrations_path:?} against {database_path:?}");
3090+
3091+ let wrapper = ayllu_database::Builder::default()
3092+ .path(database_path)
3093+ .migrations(migrations_path.as_path())
3094+ .log_queries(true)
3095+ .build()
3096+ .await?;
3097+
3098+ wrapper.close().await?;
3099+
3100+ tracing::info!("Migrations applied successfully");
3101+
3102+ Ok(())
3103+ }
3104 diff --git a/ayllu/Cargo.toml b/ayllu/Cargo.toml
3105index 033fa3b..952295b 100644
3106--- a/ayllu/Cargo.toml
3107+++ b/ayllu/Cargo.toml
3108 @@ -13,6 +13,7 @@ ayllu_cmd = { path = "../crates/cmd" }
3109 ayllu_git = { path = "../crates/git" }
3110 ayllu_identity = {path = "../crates/identity"}
3111 ayllu_config = { path = "../crates/config" }
3112+ ayllu_database = { path = "../crates/database" }
3113 timeutil = {path = "../crates/timeutil"}
3114
3115 async-trait = { workspace = true }
3116 @@ -21,17 +22,18 @@ futures = { workspace = true }
3117 thiserror = { workspace = true }
3118 tracing = { workspace = true }
3119 rand = { workspace = true }
3120+ petgraph = { workspace = true }
3121 url = {workspace = true}
3122+ time = { workspace = true }
3123+ serde = { workspace = true }
3124 tokio = { workspace = true }
3125 tokio-stream = { workspace = true }
3126 toml = { workspace = true }
3127 tokio-util = { workspace = true }
3128
3129-
3130- serde = { version = "1.0", features = ["derive"] }
3131 comrak = { version = "0.39.1", default-features = false }
3132 tree-sitter-highlight = "0.25.6"
3133- time = "0.3.41"
3134+
3135 file-mode = "0.1.2"
3136 lazy_static = "1.5.0"
3137 mime_guess = "2.0.5"
3138 @@ -52,6 +54,7 @@ quick-xml = { version = "0.38.0", features = ["encoding"] }
3139 askama = { version = "0.14.0" }
3140 openssh-keys = "0.6.4"
3141 nix = { version = "0.30.1", default-features = false, features = ["user"] }
3142+ layout-rs = "0.1.3"
3143
3144 [dev-dependencies]
3145 tempfile = {workspace = true}
3146 diff --git a/ayllu/src/config.rs b/ayllu/src/config.rs
3147index 99b21b3..202032b 100644
3148--- a/ayllu/src/config.rs
3149+++ b/ayllu/src/config.rs
3150 @@ -2,7 +2,7 @@ use std::fs::metadata;
3151 use std::{collections::HashMap, path::PathBuf};
3152 use url::Url;
3153
3154- use ayllu_config::{Configurable, Error};
3155+ use ayllu_config::{Configurable, Database, Error};
3156 use ayllu_identity::Identity;
3157
3158 use serde::{Deserialize, Serialize};
3159 @@ -249,6 +249,10 @@ pub struct Git {
3160 pub clone_url: Option<String>,
3161 }
3162
3163+ /// empty builder configuration to test if builds are configured
3164+ #[derive(Default, Deserialize, Serialize, Clone, Debug)]
3165+ pub struct Builder {}
3166+
3167 #[derive(Deserialize, Serialize, Clone, Debug)]
3168 pub struct Config {
3169 #[serde(default = "Config::default_site_name")]
3170 @@ -286,6 +290,8 @@ pub struct Config {
3171 pub lfs: Option<Lfs>,
3172 #[serde(default = "Vec::new")]
3173 pub identities: Vec<Identity>,
3174+ pub database: Option<Database>,
3175+ pub builder: Option<Builder>,
3176 }
3177
3178 impl Configurable for Config {
3179 @@ -336,6 +342,12 @@ impl Configurable for Config {
3180 });
3181 };
3182
3183+ if self.builder.is_some() && self.database.is_none() {
3184+ return Err(Error::Validation(
3185+ "Builder is enabled but the database in unconfigured".to_string(),
3186+ ));
3187+ }
3188+
3189 Ok(())
3190 }
3191 }
3192 diff --git a/ayllu/src/web2/dag.rs b/ayllu/src/web2/dag.rs
3193new file mode 100644
3194index 0000000..39b9c6d
3195--- /dev/null
3196+++ b/ayllu/src/web2/dag.rs
3197 @@ -0,0 +1,98 @@
3198+ use std::{collections::HashMap, pin::Pin};
3199+
3200+ use ayllu_api::build::Unit;
3201+ use ayllu_database::build::{ManifestView, Step, Workflow};
3202+ use petgraph::{dot, graph::NodeIndex, Graph};
3203+
3204+ #[derive(Debug)]
3205+ enum MappedUnit<'a> {
3206+ Step(&'a Step),
3207+ Workflow((&'a Workflow, bool)),
3208+ }
3209+
3210+ impl std::fmt::Display for MappedUnit<'_> {
3211+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3212+ match self {
3213+ MappedUnit::Step(step) => write!(f, "{}", step.name),
3214+ MappedUnit::Workflow(workflow) => write!(f, "{}", workflow.0.name),
3215+ }
3216+ }
3217+ }
3218+
3219+ /// Generate a DAG as an SVG chart
3220+ pub fn make_svg(view: &ManifestView) -> String {
3221+ let workflows_by_id: Vec<(i64, (&Workflow, bool))> = view
3222+ .workflows
3223+ .iter()
3224+ .map(|(workflow, steps)| {
3225+ (
3226+ workflow.id,
3227+ (
3228+ workflow,
3229+ steps
3230+ .iter()
3231+ .all(|step| step.exit_code.is_some_and(|code| code == 0)),
3232+ ),
3233+ )
3234+ })
3235+ .collect();
3236+ let workflows_by_id: HashMap<i64, (&Workflow, bool)> = HashMap::from_iter(workflows_by_id);
3237+ let steps_by_id: Vec<(i64, &Step)> = view
3238+ .workflows
3239+ .iter()
3240+ .flat_map(|workflow| {
3241+ workflow
3242+ .1
3243+ .iter()
3244+ .map(|step| (step.id, step))
3245+ .collect::<Vec<(i64, &Step)>>()
3246+ })
3247+ .collect();
3248+ let steps_by_id: HashMap<i64, &Step> = HashMap::from_iter(steps_by_id);
3249+ let graph: Graph<Unit, u8> = serde_json::de::from_str(&view.dag.dag_content).unwrap();
3250+ let mapped: Graph<MappedUnit, u8> = graph.map(
3251+ |_, unit| match unit {
3252+ Unit::Step(step_id) => MappedUnit::Step(steps_by_id[step_id]),
3253+ Unit::Workflow(workflow_id) => MappedUnit::Workflow(workflows_by_id[workflow_id]),
3254+ },
3255+ |_, weight| *weight,
3256+ );
3257+ let dot = dot::Dot::with_attr_getters(
3258+ &mapped,
3259+ &[dot::Config::EdgeNoLabel],
3260+ &|_, _| String::default(),
3261+ &|_, (_, unit)| match unit {
3262+ MappedUnit::Step(step) => match step.exit_code {
3263+ // FIXME: If we fork layout we can set classes / tags on the elements and
3264+ // style them via CSS.
3265+ Some(code) => {
3266+ if code == 0 {
3267+ "fillcolor = lightgreen, comment = helloworld".to_string()
3268+ } else {
3269+ "fillcolor = pink".to_string()
3270+ }
3271+ }
3272+ None => "fillcolor = grey".to_string(),
3273+ },
3274+ MappedUnit::Workflow((workflow, success)) => match workflow.finished_at {
3275+ Some(_) => {
3276+ if *success {
3277+ "fillcolor = lightgreen".to_string()
3278+ } else {
3279+ "fillcolor = pink".to_string()
3280+ }
3281+ }
3282+ None => "fillcolor = grey".to_string(),
3283+ },
3284+ },
3285+ );
3286+
3287+ let mut parser = layout::gv::DotParser::new(&dot.to_string());
3288+ let g = parser.process().unwrap();
3289+ let mut gb = layout::gv::GraphBuilder::new();
3290+ gb.visit_graph(&g);
3291+ let mut vg = gb.get();
3292+ let mut svg = layout::backends::svg::SVGWriter::new();
3293+ vg.do_it(false, false, false, &mut svg);
3294+ svg.finalize()
3295+ }
3296 diff --git a/ayllu/src/web2/error.rs b/ayllu/src/web2/error.rs
3297index 57c776d..99ac7e6 100644
3298--- a/ayllu/src/web2/error.rs
3299+++ b/ayllu/src/web2/error.rs
3300 @@ -19,6 +19,8 @@ pub enum Error {
3301 #[allow(dead_code)]
3302 #[error("Component not enabled: {0}")]
3303 ComponentNotEnabled(String),
3304+ #[error("Database error: {message}")]
3305+ Database { message: String },
3306 }
3307
3308 impl IntoResponse for Error {
3309 @@ -54,3 +56,11 @@ impl From<askama::Error> for Error {
3310 Error::Message(format!("Template Error: {value:#?}"))
3311 }
3312 }
3313+
3314+ impl From<ayllu_database::Error> for Error {
3315+ fn from(value: ayllu_database::Error) -> Self {
3316+ Error::Database {
3317+ message: value.to_string(),
3318+ }
3319+ }
3320+ }
3321 diff --git a/ayllu/src/web2/middleware/database.rs b/ayllu/src/web2/middleware/database.rs
3322new file mode 100644
3323index 0000000..8b5ccf2
3324--- /dev/null
3325+++ b/ayllu/src/web2/middleware/database.rs
3326 @@ -0,0 +1,35 @@
3327+ use axum::{
3328+ extract::{Request, State},
3329+ middleware::Next,
3330+ response::Response,
3331+ };
3332+
3333+ use crate::{config::Config, web2::error::Error};
3334+
3335+ #[derive(Clone)]
3336+ pub struct Builds(pub(crate) Option<ayllu_database::Wrapper>);
3337+
3338+ impl Builds {
3339+ pub fn enabled(&self) -> bool {
3340+ self.0.is_some()
3341+ }
3342+
3343+ pub fn db(&self) -> Result<&ayllu_database::Wrapper, Error> {
3344+ match self.0.as_ref() {
3345+ Some(wrapper) => Ok(wrapper),
3346+ None => Err(Error::ComponentNotEnabled("builds".to_string())),
3347+ }
3348+ }
3349+ }
3350+
3351+ pub async fn middleware(
3352+ State((config, db)): State<(Config, Option<ayllu_database::Wrapper>)>,
3353+ mut req: Request,
3354+ next: Next,
3355+ ) -> Response {
3356+ if config.builder.is_some() {
3357+ assert!(db.is_some()); // BUG if not enabled
3358+ req.extensions_mut().insert(Builds(db));
3359+ }
3360+ next.run(req).await
3361+ }
3362 diff --git a/ayllu/src/web2/middleware/error.rs b/ayllu/src/web2/middleware/error.rs
3363index e44af52..d20b482 100644
3364--- a/ayllu/src/web2/middleware/error.rs
3365+++ b/ayllu/src/web2/middleware/error.rs
3366 @@ -42,6 +42,7 @@ pub async fn middleware(
3367 Error::Message(_) => StatusCode::INTERNAL_SERVER_ERROR,
3368 Error::NotFound(_) => StatusCode::NOT_FOUND,
3369 Error::ComponentNotEnabled(_) => StatusCode::INTERNAL_SERVER_ERROR,
3370+ Error::Database { message: _ } => StatusCode::INTERNAL_SERVER_ERROR,
3371 };
3372 if status_code == StatusCode::NOT_FOUND {
3373 tracing::warn!("Not Found: {error}");
3374 diff --git a/ayllu/src/web2/middleware/mod.rs b/ayllu/src/web2/middleware/mod.rs
3375index 3f2a883..bd63d38 100644
3376--- a/ayllu/src/web2/middleware/mod.rs
3377+++ b/ayllu/src/web2/middleware/mod.rs
3378 @@ -1,3 +1,4 @@
3379+ pub mod database;
3380 pub mod error;
3381 pub mod repository;
3382 pub mod sites;
3383 diff --git a/ayllu/src/web2/mod.rs b/ayllu/src/web2/mod.rs
3384index fa88894..824ce95 100644
3385--- a/ayllu/src/web2/mod.rs
3386+++ b/ayllu/src/web2/mod.rs
3387 @@ -1,4 +1,5 @@
3388 mod config;
3389+ mod dag;
3390 mod error;
3391 mod extractors;
3392 mod middleware;
3393 diff --git a/ayllu/src/web2/navigation.rs b/ayllu/src/web2/navigation.rs
3394index 1b67ddc..4e41bd8 100644
3395--- a/ayllu/src/web2/navigation.rs
3396+++ b/ayllu/src/web2/navigation.rs
3397 @@ -36,9 +36,9 @@ pub fn primary(current_page: &str, collection: &str, name: &str) -> Items {
3398 // current_page == "charts",
3399 // ),
3400 (
3401- String::from("project"),
3402- format!("/{collection}/{name}"),
3403- current_page == "project",
3404+ String::from("builds"),
3405+ format!("/{collection}/{name}/builds"),
3406+ current_page == "builds",
3407 ),
3408 (
3409 String::from("log"),
3410 @@ -46,6 +46,11 @@ pub fn primary(current_page: &str, collection: &str, name: &str) -> Items {
3411 current_page == "log",
3412 ),
3413 (
3414+ String::from("project"),
3415+ format!("/{collection}/{name}"),
3416+ current_page == "project",
3417+ ),
3418+ (
3419 String::from("refs"),
3420 format!("/{collection}/{name}/refs"),
3421 current_page == "refs",
3422 diff --git a/ayllu/src/web2/routes/build.rs b/ayllu/src/web2/routes/build.rs
3423new file mode 100644
3424index 0000000..4c2cb14
3425--- /dev/null
3426+++ b/ayllu/src/web2/routes/build.rs
3427 @@ -0,0 +1,140 @@
3428+ use askama::Template;
3429+ use axum::extract::Path;
3430+ use axum::{extract::Extension, response::Html};
3431+ use serde::Deserialize;
3432+
3433+ use crate::config::Config;
3434+ use crate::highlight::Highlighter;
3435+ use crate::web2::dag::make_svg;
3436+ use crate::web2::error::Error;
3437+ use crate::web2::middleware::database::Builds;
3438+ use crate::web2::middleware::repository::Preamble;
3439+ use crate::web2::template::filters;
3440+ use crate::web2::template::Base;
3441+
3442+ use ayllu_database::build::{BuildExt, Manifest, ManifestItem, Step, Workflow};
3443+
3444+ #[derive(Deserialize)]
3445+ pub struct BuildParams {
3446+ pub manifest_id: i64,
3447+ pub workflow_id: Option<i64>,
3448+ pub step_id: Option<i64>,
3449+ }
3450+
3451+ #[derive(askama::Template)]
3452+ #[template(path = "build.html")]
3453+ struct BuildTemplate<'a> {
3454+ pub base: Base,
3455+ pub collection: &'a str,
3456+ pub name: &'a str,
3457+ pub manifest: Manifest,
3458+ pub workflows: Vec<(Workflow, Vec<Step>)>,
3459+ pub dag_svg: String,
3460+ pub current_workflow: i64,
3461+ pub current_step: Option<(i64, Step)>,
3462+ pub step_output: Option<String>,
3463+ }
3464+
3465+ pub async fn build(
3466+ Extension(cfg): Extension<Config>,
3467+ Extension(builds): Extension<Builds>,
3468+ Extension(preamble): Extension<Preamble>,
3469+ Extension(highlighter): Extension<Highlighter>,
3470+ Extension(mut base): Extension<Base>,
3471+ Path(BuildParams {
3472+ manifest_id,
3473+ workflow_id,
3474+ step_id,
3475+ }): Path<BuildParams>,
3476+ ) -> Result<Html<String>, Error> {
3477+ let db = builds.db()?;
3478+ let view = db
3479+ .read_manifest(&preamble.collection_name, &preamble.repo_name, manifest_id)
3480+ .await?;
3481+
3482+ let current_step = if let Some(step_id) = step_id {
3483+ view.workflows.iter().find_map(|wf| {
3484+ wf.1.iter()
3485+ .find(|step| step.id == step_id)
3486+ .map(|step| (wf.0.id, step.clone()))
3487+ })
3488+ } else {
3489+ None
3490+ };
3491+
3492+ let step_output = match &current_step {
3493+ Some((_, step)) => {
3494+ // FIXME: We need to change the db representaiton of log lines and merge
3495+ // them together here with stdout/stderr highlighting. For now I concat
3496+ // them together which is excessively stupid / annoying.
3497+ let stdout = step.stdout.as_ref().cloned().unwrap_or_default();
3498+ let stderr = step.stderr.as_ref().cloned().unwrap_or_default();
3499+ let combined = format!("{stdout}\n{stderr}");
3500+ let (_, highlighted) = highlighter.highlight(&combined, None, None, None, true);
3501+ Some(highlighted)
3502+ }
3503+ None => None,
3504+ };
3505+
3506+ base.nav_elements =
3507+ crate::web2::navigation::primary("builds", &preamble.collection_name, &preamble.repo_name);
3508+ base.current_time = timeutil::timestamp();
3509+
3510+ let dag_svg = make_svg(&view);
3511+
3512+ Ok(Html(
3513+ BuildTemplate {
3514+ base,
3515+ collection: &preamble.collection_name,
3516+ name: &preamble.repo_name,
3517+ manifest: view.manifest,
3518+ workflows: view.workflows,
3519+ dag_svg,
3520+ current_workflow: workflow_id.unwrap_or(-1),
3521+ current_step,
3522+ step_output,
3523+ }
3524+ .render()?,
3525+ ))
3526+ }
3527+
3528+ #[derive(askama::Template)]
3529+ #[template(path = "builds.html")]
3530+ struct BuildsTemplate<'a> {
3531+ pub base: Base,
3532+ pub collection: &'a str,
3533+ pub name: &'a str,
3534+ pub items: Vec<ManifestItem>,
3535+ }
3536+
3537+ pub async fn builds(
3538+ Extension(cfg): Extension<Config>,
3539+ Extension(builds): Extension<Builds>,
3540+ Extension(preamble): Extension<Preamble>,
3541+ Extension(mut base): Extension<Base>,
3542+ ) -> Result<Html<String>, Error> {
3543+ let db = builds.db()?;
3544+ let items = db
3545+ .list_manifest(
3546+ Some(&preamble.collection_name),
3547+ Some(&preamble.repo_name),
3548+ None,
3549+ 9999999,
3550+ 0,
3551+ )
3552+ .await?;
3553+ // FIXME
3554+ // let manifests = db.list_manifest(999999, 0).await?;
3555+ base.nav_elements =
3556+ crate::web2::navigation::primary("builds", &preamble.collection_name, &preamble.repo_name);
3557+ base.current_time = timeutil::timestamp();
3558+ Ok(Html(
3559+ BuildsTemplate {
3560+ base,
3561+ items,
3562+ collection: &preamble.collection_name,
3563+ name: &preamble.repo_name,
3564+ }
3565+ .render()?,
3566+ ))
3567+ }
3568 diff --git a/ayllu/src/web2/routes/mod.rs b/ayllu/src/web2/routes/mod.rs
3569index f2dca6f..7572af5 100644
3570--- a/ayllu/src/web2/routes/mod.rs
3571+++ b/ayllu/src/web2/routes/mod.rs
3572 @@ -1,6 +1,7 @@
3573 pub mod about;
3574 pub mod assets;
3575 pub mod blob;
3576+ pub mod build;
3577 pub mod commit;
3578 pub mod config;
3579 pub mod finger;
3580 diff --git a/ayllu/src/web2/routes/repo.rs b/ayllu/src/web2/routes/repo.rs
3581index b32c723..68e2f3f 100644
3582--- a/ayllu/src/web2/routes/repo.rs
3583+++ b/ayllu/src/web2/routes/repo.rs
3584 @@ -6,7 +6,10 @@ use serde::Serialize;
3585
3586 use crate::{
3587 config::Config,
3588- web2::template::{Base, DEFAULT_ANCHOR_SYMBOL},
3589+ web2::{
3590+ middleware::database::Builds,
3591+ template::{Base, DEFAULT_ANCHOR_SYMBOL},
3592+ },
3593 with_preamble,
3594 };
3595
3596 @@ -91,6 +94,7 @@ pub async fn serve(
3597 Extension(cfg): Extension<Config>,
3598 Extension(preamble): Extension<Preamble>,
3599 Extension(adapter): Extension<TreeSitterAdapter>,
3600+ Extension(builds): Extension<Builds>,
3601 Extension(mut base): Extension<Base>,
3602 ) -> Result<Html<String>, Error> {
3603 let repository = Wrapper::new(preamble.repo_path.as_path())?;
3604 diff --git a/ayllu/src/web2/server.rs b/ayllu/src/web2/server.rs
3605index 946e7ec..7f957cd 100644
3606--- a/ayllu/src/web2/server.rs
3607+++ b/ayllu/src/web2/server.rs
3608 @@ -17,12 +17,14 @@ use tracing::{Level, Span};
3609
3610 use crate::highlight::{Highlighter, Loader, TreeSitterAdapter};
3611 use crate::languages::{Hint, LANGUAGE_TABLE};
3612+ use crate::web2::middleware::database as db;
3613 use crate::web2::middleware::error;
3614 use crate::web2::middleware::repository;
3615 use crate::web2::middleware::sites;
3616 use crate::web2::routes::about;
3617 use crate::web2::routes::assets;
3618 use crate::web2::routes::blob;
3619+ use crate::web2::routes::build;
3620 use crate::web2::routes::commit;
3621 use crate::web2::routes::config;
3622 use crate::web2::routes::finger;
3623 @@ -82,6 +84,24 @@ pub async fn serve(cfg: &Config) -> Result<(), Box<dyn Error>> {
3624 Vec::new()
3625 };
3626
3627+ let mut database: Option<ayllu_database::Wrapper> = None;
3628+
3629+ if cfg.builder.is_some() {
3630+ tracing::info!("Builder is enabled thus the database is configured");
3631+ database = Some(
3632+ ayllu_database::Builder::default()
3633+ .path(cfg.database.as_ref().unwrap().path.as_path())
3634+ .read_only(true)
3635+ .log_queries(false) // FIXME
3636+ .build()
3637+ .await?,
3638+ );
3639+ }
3640+
3641+ if database.is_some() {
3642+ tracing::info!("SQLite Database is serving");
3643+ }
3644+
3645 let address: SocketAddrV4 = cfg.http.address.parse()?;
3646 let app = NormalizePathLayer::trim_trailing_slash().layer(
3647 Router::new()
3648 @@ -132,6 +152,16 @@ pub async fn serve(cfg: &Config) -> Result<(), Box<dyn Error>> {
3649 .route("/refs", routing::get(refs::refs))
3650 .route("/refs/tag/{tag_id}", routing::get(refs::tag))
3651 .route("/refs/archive/{ref_id}", routing::get(refs::archive))
3652+ .route("/builds", routing::get(build::builds))
3653+ .route("/builds/{manifest_id}", routing::get(build::build))
3654+ .route(
3655+ "/builds/{manifest_id}/{workflow_id}",
3656+ routing::get(build::build),
3657+ )
3658+ .route(
3659+ "/builds/{manifest_id}/{workflow_id}/{step_id}",
3660+ routing::get(build::build),
3661+ )
3662 // git smart http clone
3663 // /(HEAD|info/refs|objects/info/.*|git-upload-pack).*$
3664 .route("/HEAD", routing::get(git::handle))
3665 @@ -159,6 +189,7 @@ pub async fn serve(cfg: &Config) -> Result<(), Box<dyn Error>> {
3666 .layer(Extension(cfg.clone()))
3667 .layer(Extension(highlighter))
3668 .layer(Extension(adapter))
3669+ .layer(from_fn_with_state((cfg.clone(), database), db::middleware))
3670 .layer(from_fn_with_state(cfg.clone(), template::middleware))
3671 // error handling
3672 .layer(from_fn_with_state(Arc::new(cfg.clone()), error::middleware))
3673 diff --git a/ayllu/src/web2/template.rs b/ayllu/src/web2/template.rs
3674index eecfcaf..28a865c 100644
3675--- a/ayllu/src/web2/template.rs
3676+++ b/ayllu/src/web2/template.rs
3677 @@ -104,6 +104,16 @@ pub mod filters {
3678 Ok(timeutil::friendly(*epoch as u64))
3679 }
3680
3681+ pub fn friendly_time_maybe(
3682+ epoch: &Option<i64>,
3683+ values: &dyn askama::Values,
3684+ ) -> askama::Result<String> {
3685+ match epoch {
3686+ Some(value) => friendly_time(value, values),
3687+ None => Ok("?".to_string()),
3688+ }
3689+ }
3690+
3691 pub fn format_epoch(epoch: &i64, _: &dyn askama::Values) -> askama::Result<String> {
3692 let ts = OffsetDateTime::from_unix_timestamp(*epoch).unwrap();
3693 let formatted = ts.format(&well_known::Rfc2822).unwrap();
3694 @@ -123,14 +133,27 @@ pub mod filters {
3695 Ok(util::human_bytes(*bytes as f64))
3696 }
3697
3698+ pub fn friendly_duration_i64(
3699+ input: &Option<i64>,
3700+ _: &dyn askama::Values,
3701+ ) -> askama::Result<String> {
3702+ match input {
3703+ Some(value) => Ok(format!(
3704+ "{:?}",
3705+ std::time::Duration::from_millis(*value as u64)
3706+ )),
3707+ None => Ok("?".to_string()),
3708+ }
3709+ }
3710+
3711 pub fn verified_class_name(
3712 verified: &Option<bool>,
3713 _: &dyn askama::Values,
3714 ) -> askama::Result<String> {
3715- if verified.is_some_and(|verified| verified) {
3716- Ok(String::from("positive"))
3717- } else {
3718- Ok(String::from("negative"))
3719+ match verified {
3720+ Some(true) => Ok(String::from("positive")),
3721+ Some(false) => Ok(String::from("negative")),
3722+ None => Ok(String::from("unknown")),
3723 }
3724 }
3725
3726 diff --git a/ayllu/templates/build.html b/ayllu/templates/build.html
3727new file mode 100644
3728index 0000000..4c9ad9d
3729--- /dev/null
3730+++ b/ayllu/templates/build.html
3731 @@ -0,0 +1,92 @@
3732+ {% extends "base.html" %}
3733+ {% block content %}
3734+ <section id="builds" class="scrollable raised">
3735+ <section class="info-bar">
3736+ <section class="title"><a href="/{{collection}}/{{name}}/builds/{{manifest.id}}">Build for {{collection}}/{{name}}: {{manifest.id}}</a></section>
3737+ </section>
3738+ <section class="raised lower-half">
3739+ <section class="dag scrollable">
3740+ {{ dag_svg | safe }}
3741+ </section>
3742+ {% for workflow in workflows %}
3743+ <section class="scrollable">
3744+ <section class="info-bar flex-group">
3745+ <section class="title">
3746+ <a href="/{{collection}}/{{name}}/builds/{{manifest.id}}/{{workflow.0.id}}">{{ workflow.0.name }} {%- if workflow.0.id == current_workflow %} <{%- endif -%}</a>
3747+ </section>
3748+ <section> Workflow </section>
3749+ </section>
3750+ <table class="data-table lower-half">
3751+ <thead>
3752+ <tr>
3753+ <th>ID</th>
3754+ <th>Name</th>
3755+ <th>StartedAt</th>
3756+ <!-- <th>Duration</th> -->
3757+ <th>ExitCode</th>
3758+ </tr>
3759+ </thead>
3760+ <tbody>
3761+ {% for step in workflow.1 %}
3762+ {%- if let Some(current_step) = current_step -%}
3763+ {%- if current_step.1.id == step.id -%}
3764+ <tr class="selected">
3765+ {%- else -%}
3766+ <tr>
3767+ {%- endif -%}
3768+ {%- else -%}
3769+ <tr>
3770+ {%- endif -%}
3771+ <td><a href="/{{collection}}/{{name}}/builds/{{manifest.id}}/{{workflow.0.id}}/{{step.id}}">
3772+ {{step.id}}
3773+ </td>
3774+ <td>{{step.name}}</td>
3775+ <td>{{step.started_at | friendly_time_maybe }}</td>
3776+ <td>{%- if let Some(exit_code) = step.exit_code -%} {{ exit_code }} {%- else -%} ? {%- endif -%}</td>
3777+ </tr>
3778+ {% endfor %}
3779+ </tbody>
3780+ </table>
3781+ {%- if let Some(current_step) = current_step -%}
3782+ {%- if current_step.0 == workflow.0.id -%}
3783+ <section class="scrollable raised inner">
3784+ <section class="info-bar flex-group selected">
3785+ <section class="title">
3786+ {{ current_step.1.name }}
3787+ </section>
3788+ </section>
3789+ <section class="lower-half">
3790+ <table class="data-table">
3791+ <thead>
3792+ <tr>
3793+ <th>Name</th>
3794+ <th>Value</th>
3795+ </tr>
3796+ </thead>
3797+ <tbody>
3798+ <tr>
3799+ <td> Shell </td>
3800+ <td> {{ current_step.1.shell }} </td>
3801+ </tr>
3802+ <tr>
3803+ <td> Command </td>
3804+ <td><code>{{ current_step.1.input }}</code></td>
3805+ </tr>
3806+ </tbody>
3807+ </table>
3808+
3809+ {%- if let Some(step_output) = step_output -%}
3810+ <section id="code-viewer" class="scrollable">
3811+ {{ step_output | safe}}
3812+ </section>
3813+ {%- endif -%}
3814+
3815+ </section>
3816+ </section>
3817+ {%- endif -%}
3818+ {%- endif -%}
3819+ </section>
3820+ {% endfor %}
3821+ </section>
3822+ </section>
3823+ {% endblock %}
3824 diff --git a/ayllu/templates/builds.html b/ayllu/templates/builds.html
3825new file mode 100644
3826index 0000000..01ab8d8
3827--- /dev/null
3828+++ b/ayllu/templates/builds.html
3829 @@ -0,0 +1,42 @@
3830+ {% extends "base.html" %}
3831+ {% block content %}
3832+ <section id="builds" class="scrollable raised">
3833+ <section class="info-bar">
3834+ <section class="title">Builds</section>
3835+ </section>
3836+ <table class="data-table lower-half">
3837+ <thead>
3838+ <tr>
3839+ <th>ID</th>
3840+ <th>Status</th>
3841+ <th>CreatedAt</th>
3842+ <!-- <th>Duration</th> -->
3843+ </tr>
3844+ </thead>
3845+ <tbody>
3846+ {% for item in items %}
3847+ <tr class="build">
3848+ <td>
3849+ <div class="name">
3850+ <a href="/{{collection}}/{{name}}/builds/{{item.manifest.id}}">{{item.manifest.id}}</a>
3851+ </div>
3852+ </td>
3853+ <td>
3854+ {%- for workflow in item.workflows -%}
3855+ <span class="workflow">
3856+ <a href="/{{collection}}/{{name}}/builds/{{item.manifest.id}}/{{- workflow.0 -}}">{{- workflow.1 -}}:</a>
3857+ {%- for step in workflow.2 -%}
3858+ <span class="step {{step.2 | verified_class_name }}">
3859+ <a href="/{{collection}}/{{name}}/builds/{{item.manifest.id}}/{{- workflow.0 -}}/{{- step.0 -}}">{{- step.1 -}}</a>
3860+ </span>
3861+ {%- endfor -%}
3862+ </span>
3863+ {%- endfor -%}
3864+ </td>
3865+ <td>{{ item.manifest.created_at | friendly_time }}</td>
3866+ </tr>
3867+ {% endfor %}
3868+ </tbody>
3869+ </table>
3870+ </section>
3871+ {% endblock %}
3872 diff --git a/ayllu/templates/step.html b/ayllu/templates/step.html
3873new file mode 100644
3874index 0000000..2b93aa4
3875--- /dev/null
3876+++ b/ayllu/templates/step.html
3877 @@ -0,0 +1,10 @@
3878+ {% extends "base.html" %}
3879+ {% block content %}
3880+ <section id="builds" class="scrollable raised">
3881+ <section class="info-bar">
3882+ <section class="title">Step {{ step.id }}</section>
3883+ </section>
3884+ <section class="raised lower-half">
3885+ </section>
3886+ </section>
3887+ {% endblock %}
3888 diff --git a/ayllu/templates/workflow.html b/ayllu/templates/workflow.html
3889new file mode 100644
3890index 0000000..bf1a8f0
3891--- /dev/null
3892+++ b/ayllu/templates/workflow.html
3893 @@ -0,0 +1,34 @@
3894+ {% extends "base.html" %}
3895+ {% block content %}
3896+ <section id="builds" class="scrollable raised">
3897+ <section class="info-bar">
3898+ <section class="title">Workflow: {{manifest_id}}/{{workflow.id}}</section>
3899+ </section>
3900+ <section class="raised lower-half">
3901+ <section class="info-bar">
3902+ <section class="title">Workflow: <a href="/{{collection}}/{{name}}/builds/{{manifest_id}}/{{workflow.id}}">{{ workflow.name }}</a></section>
3903+ </section>
3904+ <table class="data-table lower-half">
3905+ <thead>
3906+ <tr>
3907+ <th>ID</th>
3908+ <th>Name</th>
3909+ <th>StartedAt</th>
3910+ <!-- <th>Duration</th> -->
3911+ <th>ExitCode</th>
3912+ </tr>
3913+ </thead>
3914+ <tbody>
3915+ {% for step in steps %}
3916+ <tr>
3917+ <td><a href="/{{collection}}/{{name}}/builds/{{manifest_id}}/{{workflow.id}}/{{step.id}}">{{step.id}}</td>
3918+ <td>{{step.name}}</td>
3919+ <td>{{step.started_at | friendly_time_maybe }}</td>
3920+ <td>{%- if let Some(exit_code) = step.exit_code -%} {{ exit_code }} {%- else -%} ? {%- endif -%}</td>
3921+ </tr>
3922+ {% endfor %}
3923+ </tbody>
3924+ </table>
3925+ </section>
3926+ </section>
3927+ {% endblock %}
3928 diff --git a/ayllu/themes/base.css b/ayllu/themes/base.css
3929index 7b2a607..cf04145 100644
3930--- a/ayllu/themes/base.css
3931+++ b/ayllu/themes/base.css
3932 @@ -24,10 +24,14 @@ main {
3933 white-space: nowrap;
3934 }
3935
3936- section#log > section > .data-table > tbody > tr > td:first-child {
3937+ section#log>section>.data-table>tbody>tr>td:first-child {
3938 text-align: center;
3939 }
3940
3941+ section.inner {
3942+ padding: 5px;
3943+ }
3944+
3945 a:visited {
3946 text-decoration: none;
3947 }
3948 @@ -91,7 +95,7 @@ section.info-bar {
3949 text-decoration: underline;
3950 }
3951
3952- section.clone > section.title {
3953+ section.clone>section.title {
3954 padding-bottom: 10px;
3955 }
3956
3957 @@ -99,14 +103,14 @@ section.info-bar.single {
3958 border-bottom: solid 2px;
3959 }
3960
3961- section.info-bar > section {
3962+ section.info-bar>section {
3963 align-content: center;
3964 justify-content: center;
3965 }
3966
3967- section.badge-group > section:first-child {
3968+ section.badge-group>section:first-child {
3969 margin-top: 1em;
3970- margin-bottom: 1em;
3971+ margin-bottom: 1em;
3972 }
3973
3974 section#code-viewer {
3975 @@ -127,7 +131,7 @@ section#commit-summary {
3976 border-radius: 5px;
3977 }
3978
3979- section#commit-summary > section > pre {
3980+ section#commit-summary>section>pre {
3981 border-radius: 5px;
3982 padding: 1em;
3983 }
3984 @@ -145,6 +149,14 @@ section#commit-message {
3985 margin-bottom: 0;
3986 }
3987
3988+ /* span.workflow { */
3989+ /* border: solid; */
3990+ /* } */
3991+
3992+ span.workflow>.step::before {
3993+ content: "•"
3994+ }
3995+
3996 span#ref-badge::before {
3997 content: "Ref:: ";
3998 }
3999 @@ -157,7 +169,7 @@ section#rss-links {
4000 display: flex;
4001 }
4002
4003- section#rss-links > section#rss-link-items {
4004+ section#rss-links>section#rss-link-items {
4005 display: flex;
4006 justify-content: flex-start;
4007 align-items: center;
4008 @@ -167,7 +179,7 @@ section#readme {
4009 padding: 1em;
4010 }
4011
4012- section#readme > header {
4013+ section#readme>header {
4014 margin: 5px;
4015 }
4016
4017 @@ -230,11 +242,11 @@ section#repo-details-panel {
4018 padding: 1em;
4019 }
4020
4021- section#config-panel > section {
4022+ section#config-panel>section {
4023 padding: 1em;
4024 }
4025
4026- section#config-panel > section > form > button {
4027+ section#config-panel>section>form>button {
4028 margin-top: 1em;
4029 display: block;
4030 }
4031 @@ -253,7 +265,7 @@ section.selectable-text {
4032 text-decoration: none;
4033 }
4034
4035- section.selectable-text > span.protocol-badge {
4036+ section.selectable-text>span.protocol-badge {
4037 font-variant: all-petite-caps;
4038 border-style: solid;
4039 border-width: 3px 3px 3px 3px;
4040 @@ -261,7 +273,7 @@ section.selectable-text > span.protocol-badge {
4041 user-select: none;
4042 }
4043
4044- section.selectable-text > span.selectable {
4045+ section.selectable-text>span.selectable {
4046 max-width: 300px;
4047 overflow: scroll;
4048 border-style: solid;
4049 @@ -271,6 +283,10 @@ section.selectable-text > span.selectable {
4050 cursor: pointer;
4051 }
4052
4053+ section.dag {
4054+ text-align: center;
4055+ }
4056+
4057 footer {
4058 padding-top: 2em;
4059 text-align: center;
4060 @@ -331,7 +347,7 @@ code.highlighted {
4061 overflow: scroll;
4062 }
4063
4064- .code > tbody > tr > td {
4065+ .code>tbody>tr>td {
4066 text-align: revert;
4067 padding: 0 0 0 0;
4068 padding-right: 0.5em;
4069 @@ -372,8 +388,8 @@ li.active {
4070 flex-direction: column;
4071 }
4072
4073- section.flex-group > section {
4074+ section.flex-group>section {
4075 width: 100%;
4076 height: auto;
4077 }
4078- }
4079+ }
4080\ No newline at end of file
4081 diff --git a/ayllu/themes/catppuccin.css b/ayllu/themes/catppuccin.css
4082index 3ded025..e6f3905 100644
4083--- a/ayllu/themes/catppuccin.css
4084+++ b/ayllu/themes/catppuccin.css
4085 @@ -62,6 +62,14 @@ a {
4086 color: var(--ctp-latte-blue);
4087 }
4088
4089+ .selected {
4090+ background-color: var(--ctp-latte-lavender);
4091+ }
4092+
4093+ section.dag {
4094+ background-color: var(--ctp-latte-base);
4095+ }
4096+
4097 body {
4098 color: var(--ctp-latte-text);
4099 background-color: var(--ctp-latte-base);
4100 @@ -73,6 +81,7 @@ nav {
4101 font-weight: bold;
4102 font-size: large;
4103 }
4104+
4105 color: var(--ctp-latte-text);
4106 background-color: var(--ctp-latte-mantle);
4107 }
4108 @@ -94,13 +103,13 @@ table.data-table tbody tr:nth-child(even) {
4109 background-color: var(--ctp-latte-surface1);
4110 }
4111
4112- section.selectable-text > span.selectable {
4113+ section.selectable-text>span.selectable {
4114 color: var(--ctp-latte-text);
4115 border-color: var(--ctp-latte-lavender);
4116 background-color: var(--ctp-latte-mantle);
4117 }
4118
4119- section.selectable-text > span.protocol-badge {
4120+ section.selectable-text>span.protocol-badge {
4121 color: var(--ctp-latte-mantle);
4122 border-color: var(--ctp-latte-lavender);
4123 background-color: var(--ctp-latte-text);
4124 @@ -139,7 +148,7 @@ section#commit-summary {
4125 background: var(--ctp-latte-surface0);
4126 }
4127
4128- section#commit-summary > section > pre {
4129+ section#commit-summary>section>pre {
4130 background-color: var(--ctp-latte-mantle);
4131 }
4132
4133 @@ -228,10 +237,17 @@ span.ts_addition {
4134 font-weight: bold;
4135 font-size: large;
4136 }
4137+
4138 color: var(--ctp-mocha-lavender);
4139 background-color: var(--ctp-mocha-surface0);
4140 }
4141
4142+
4143+
4144+ section.dag {
4145+ background-color: var(--ctp-mocha-overlay0);
4146+ }
4147+
4148 nav .subnav {
4149 background-color: var(--ctp-mocha-surface1);
4150 }
4151 @@ -241,13 +257,14 @@ span.ts_addition {
4152 background-color: var(--ctp-mocha-base);
4153 }
4154
4155- section.selectable-text > .selectable {
4156+ section.selectable-text>.selectable {
4157 color: var(--ctp-mocha-base);
4158 background-color: var(--ctp-mocha-lavender);
4159 }
4160
4161 .data-table {
4162 color: var(--ctp-mocha-text);
4163+
4164 a {
4165 color: var(--ctp-mocha-blue);
4166 }
4167 @@ -285,7 +302,7 @@ span.ts_addition {
4168 background-color: var(--ctp-mocha-surface1);
4169 }
4170
4171- section#commit-summary > section > pre {
4172+ section#commit-summary>section>pre {
4173 background-color: var(--ctp-mocha-mantle);
4174 }
4175- }
4176+ }
4177\ No newline at end of file
4178 diff --git a/ayllu/themes/default.css b/ayllu/themes/default.css
4179index 07d26ef..3d693fd 100644
4180--- a/ayllu/themes/default.css
4181+++ b/ayllu/themes/default.css
4182 @@ -11,6 +11,11 @@
4183 }
4184 }
4185
4186+ .selected {
4187+ color: var(--secondary-color);
4188+ background-color: var(--primary-color);
4189+ }
4190+
4191 body {
4192 color: var(--primary-color);
4193 background-color: var(--secondary-color);
4194 @@ -104,4 +109,4 @@ span.ts_removal {
4195
4196 span.ts_addition {
4197 color: var(--primary-color);
4198- }
4199+ }
4200\ No newline at end of file
4201 diff --git a/ayllu/themes/nord.css b/ayllu/themes/nord.css
4202index f77bf15..692dbc1 100644
4203--- a/ayllu/themes/nord.css
4204+++ b/ayllu/themes/nord.css
4205 @@ -37,6 +37,7 @@ nav {
4206 font-weight: bold;
4207 font-size: large;
4208 }
4209+
4210 color: var(--nord6);
4211 background-color: var(--nord9);
4212 }
4213 @@ -58,13 +59,13 @@ table.data-table thead {
4214 background-color: var(--nord10);
4215 }
4216
4217- section.selectable-text > span.text {
4218+ section.selectable-text>span.text {
4219 color: var(--nord0);
4220 border-color: var(--nord10);
4221 background-color: var(--nord5);
4222 }
4223
4224- section.selectable-text > span.protocol-badge {
4225+ section.selectable-text>span.protocol-badge {
4226 color: var(--nord6);
4227 border-color: var(--nord10);
4228 background-color: var(--nord0);
4229 @@ -74,7 +75,7 @@ section#readme-header {
4230 background-color: var(--nord7);
4231 }
4232
4233- section > a {
4234+ section>a {
4235 color: var(--nord5);
4236 }
4237
4238 @@ -88,7 +89,7 @@ section#commit-summary {
4239 background-color: var(--nord4);
4240 }
4241
4242- section#commit-summary > section > pre {
4243+ section#commit-summary>section>pre {
4244 background-color: var(--nord5);
4245 }
4246
4247 @@ -104,6 +105,14 @@ section.info-bar.single {
4248 text-shadow: 1px 1px 5px var(--nord11);
4249 }
4250
4251+ .selected {
4252+ background-color: var(--nord7) !important;
4253+ }
4254+
4255+ section.dag {
4256+ background-color: var(--nord4);
4257+ }
4258+
4259 /* Syntax Highlighting */
4260
4261 section#code-viewer {
4262 @@ -188,10 +197,12 @@ span.ts_addition {
4263 color: var(--nord4);
4264 background-color: var(--nord0);
4265 }
4266+
4267 nav {
4268 a {
4269 color: var(--nord6);
4270 }
4271+
4272 color: var(--nord6);
4273 background-color: var(--nord2);
4274 }
4275 @@ -200,11 +211,16 @@ span.ts_addition {
4276 background-color: var(--nord3);
4277 }
4278
4279- section.selectable-text > span.text {
4280+
4281+ section.dag {
4282+ background-color: var(--nord3);
4283+ }
4284+
4285+ section.selectable-text>span.text {
4286 border-color: var(--nord6);
4287 }
4288
4289- section.selectable-text > span.protocol-badge {
4290+ section.selectable-text>span.protocol-badge {
4291 background-color: var(--nord4);
4292 color: var(--nord0);
4293 border-color: var(--nord6);
4294 @@ -214,7 +230,7 @@ span.ts_addition {
4295 background-color: var(--nord3);
4296 }
4297
4298- section#tree > table > thead {
4299+ section#tree>table>thead {
4300 color: var(--nord0);
4301 background-color: var(--nord4);
4302 }
4303 @@ -251,7 +267,7 @@ span.ts_addition {
4304 border-bottom-color: var(--nord5);
4305 }
4306
4307- section > a {
4308+ section>a {
4309 color: var(--nord5);
4310 }
4311
4312 @@ -259,7 +275,7 @@ span.ts_addition {
4313 background-color: var(--nord2);
4314 }
4315
4316- section#commit-summary > section > pre {
4317+ section#commit-summary>section>pre {
4318 background-color: var(--nord3);
4319 }
4320
4321 @@ -267,4 +283,4 @@ span.ts_addition {
4322 color: var(--nord8);
4323 }
4324
4325- }
4326+ }
4327\ No newline at end of file
4328 diff --git a/config.example.toml b/config.example.toml
4329index 1df821d..75cfa82 100644
4330--- a/config.example.toml
4331+++ b/config.example.toml
4332 @@ -301,3 +301,15 @@ A Hyper Performant & Hackable Code Forge Built on Open Standards.
4333 # authorized_keys = [
4334 # ".. your key here .."
4335 # ]
4336+
4337+ # Global SQLite database used for all persistent storage outside of Git.
4338+ # You need to run ayllu-migrate to apply any migrations
4339+ # [database]
4340+ # Path to the SQLite database
4341+ # path = "/usr/share/ayllu/state.db"
4342+ # Directory of Migrations
4343+ # migrations = "/usr/lib/ayllu/migrations"
4344+
4345+ # Global Builder configuration
4346+ # Used to configure the runtime properties of ayllu-build
4347+ # [builder]
4348 diff --git a/crates/api/src/build.rs b/crates/api/src/build.rs
4349index 426f91d..de3c49c 100644
4350--- a/crates/api/src/build.rs
4351+++ b/crates/api/src/build.rs
4352 @@ -1,62 +1,23 @@
4353- use serde::{Deserialize, Serialize};
4354-
4355- #[derive(Debug, Deserialize, Serialize)]
4356- /// Defines some kind of event that occurs within a repository
4357- pub enum Kind {
4358- Commit,
4359- Tag,
4360- }
4361-
4362- #[derive(Debug, Deserialize, Serialize)]
4363- pub struct Event {
4364- pub repository_url: String,
4365- pub commit_hash: String,
4366- pub kind: Kind,
4367- }
4368+ use std::fmt::Display;
4369
4370- #[derive(Debug, Deserialize, Serialize)]
4371- /// array of samples taken to construct informative charts in the UI
4372- pub struct Samples {
4373- load_1m: Vec<f32>,
4374- load_5m: Vec<f32>,
4375- load_15m: Vec<f32>,
4376- disk_total_bytes: Vec<i64>,
4377- disk_free_bytes: Vec<i64>,
4378- net_sent_bytes: Vec<i64>,
4379- net_received_bytes: Vec<i64>,
4380- net_sent_packets: Vec<i64>,
4381- net_received_packets: Vec<i64>,
4382- mem_total_bytes: Vec<i64>,
4383- mem_free_bytes: Vec<i64>,
4384- mem_available_bytes: Vec<i64>,
4385- }
4386-
4387- #[derive(Debug, Deserialize, Serialize)]
4388- pub struct Step {
4389- pub id: i64,
4390- pub input: String,
4391- pub shell: String,
4392- }
4393+ use serde::{Deserialize, Serialize};
4394
4395- #[derive(Debug, Deserialize, Serialize)]
4396- pub struct Job {
4397- pub id: i64,
4398- pub name: String,
4399- pub steps: Vec<Step>,
4400+ /// Unit of execution which is either a workflow or a step
4401+ #[derive(Serialize, Deserialize, Debug, Clone)]
4402+ pub enum Unit {
4403+ Step(i64),
4404+ Workflow(i64),
4405 }
4406
4407- #[derive(Debug, Deserialize, Serialize)]
4408- pub struct Manifest {
4409- pub id: i64,
4410- repository_url: String,
4411- git_hash: String,
4412- /// DOT representation of DAG rendered via manifest
4413- dot_chart: String,
4414- created_at: i64,
4415- started_at: i64,
4416- finished_at: i64,
4417- success: bool,
4418- // not included on manfest list calls otherwise ordered in the way
4419- // that they are evaluated via dependency graph
4420- jobs: Vec<Job>,
4421+ impl Display for Unit {
4422+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4423+ match self {
4424+ Unit::Step(id) => {
4425+ write!(f, "Step: {id}")
4426+ }
4427+ Unit::Workflow(id) => {
4428+ write!(f, "Workflow: {id}")
4429+ }
4430+ }
4431+ }
4432 }
4433 diff --git a/crates/cmd/src/lib.rs b/crates/cmd/src/lib.rs
4434index 473f986..9ea472d 100644
4435--- a/crates/cmd/src/lib.rs
4436+++ b/crates/cmd/src/lib.rs
4437 @@ -1,5 +1,6 @@
4438 pub mod ayllu;
4439 pub mod keys;
4440+ pub mod migrate;
4441 pub mod quipu;
4442 pub mod shell;
4443
4444 diff --git a/crates/cmd/src/migrate.rs b/crates/cmd/src/migrate.rs
4445new file mode 100644
4446index 0000000..ac86806
4447--- /dev/null
4448+++ b/crates/cmd/src/migrate.rs
4449 @@ -0,0 +1,23 @@
4450+ use std::path::PathBuf;
4451+
4452+ use clap::Parser;
4453+
4454+ const LONG_ABOUT_DESCRIPTION: &str = r#"
4455+
4456+ ayllu-migrate is used to manage the shared global sqlite database used across
4457+ all ayllu binaries installed on the system. Anytime a schema change is introduced
4458+ ayllu-migrate must be invoked. Typically this should only be ran by your package
4459+ management system but may also be useful as a standalone component.
4460+ "#;
4461+
4462+ /// Manage the global Ayllu sqlite database
4463+ #[derive(Parser, Debug)]
4464+ #[clap(version, about, name = "ayllu-migrate", long_about = LONG_ABOUT_DESCRIPTION)]
4465+ pub struct Command {
4466+ /// Path to your configuration file
4467+ #[arg(short, long, value_name = "FILE")]
4468+ pub config: Option<PathBuf>,
4469+ #[arg(short, long, value_name = "FILE")]
4470+ /// Optional path to migration files
4471+ pub migrations: Option<PathBuf>,
4472+ }
4473 diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs
4474index 2327afc..322c1a8 100644
4475--- a/crates/config/src/lib.rs
4476+++ b/crates/config/src/lib.rs
4477 @@ -1,8 +1,46 @@
4478+ use std::path::{Path, PathBuf};
4479+
4480 pub use error::Error;
4481 pub use flags::Command;
4482 pub use reader::{data_dir, runtime_dir, Configurable, Reader};
4483+ use serde::{Deserialize, Serialize};
4484
4485 mod edit;
4486 mod error;
4487 mod flags;
4488 mod reader;
4489+
4490+ /// Global database configuration used in all ayllu components
4491+ #[derive(Deserialize, Serialize, Clone, Debug)]
4492+ pub struct Database {
4493+ #[serde(default = "Database::path_default")]
4494+ pub path: PathBuf,
4495+ #[serde(default = "Database::migrate_default")]
4496+ pub migrate: bool,
4497+ #[serde(default = "Database::migrations_default")]
4498+ pub migrations: PathBuf,
4499+ }
4500+
4501+ impl Database {
4502+ fn path_default() -> PathBuf {
4503+ data_dir().join("ayllu.sqlite").to_path_buf()
4504+ }
4505+
4506+ fn migrations_default() -> PathBuf {
4507+ Path::new("/usr/lib/ayllu/migrations").to_path_buf()
4508+ }
4509+
4510+ fn migrate_default() -> bool {
4511+ true
4512+ }
4513+ }
4514+
4515+ impl Default for Database {
4516+ fn default() -> Self {
4517+ Self {
4518+ path: Database::path_default(),
4519+ migrate: Database::migrate_default(),
4520+ migrations: Database::migrations_default(),
4521+ }
4522+ }
4523+ }
4524 diff --git a/crates/database/Cargo.toml b/crates/database/Cargo.toml
4525index 88f48cf..d7a1022 100644
4526--- a/crates/database/Cargo.toml
4527+++ b/crates/database/Cargo.toml
4528 @@ -8,7 +8,8 @@ edition = "2021"
4529 [dependencies]
4530 async-trait = { workspace = true }
4531 futures = { workspace = true }
4532- serde = { workspace = true }
4533+ serde = { workspace = true }
4534+ serde_json = { workspace = true }
4535 sqlx = { workspace = true }
4536- time = "0.3.41"
4537+ time = { workspace = true }
4538 tracing = { workspace = true }
4539 diff --git a/crates/database/queries/authors_list_project.sql b/crates/database/queries/authors_list_project.sql
4540new file mode 100644
4541index 0000000..832e3de
4542--- /dev/null
4543+++ b/crates/database/queries/authors_list_project.sql
4544 @@ -0,0 +1,30 @@
4545+ WITH RECURSIVE max_git_id AS
4546+ (SELECT contributions.id AS hash
4547+ FROM contributions
4548+ WHERE git_hash = ?
4549+ AND repo_path = ?
4550+ ORDER BY id DESC
4551+ LIMIT 1),
4552+ total_contributions AS
4553+ (SELECT COUNT(*) AS contributions
4554+ FROM contributions
4555+ WHERE repo_path = ?
4556+ AND id <=
4557+ (SELECT hash
4558+ FROM max_git_id) )
4559+ SELECT authors.username,
4560+ authors.email,
4561+ COUNT(contributions.id) AS "count: i64",
4562+ SUM(contributions.lines_added) AS "lines_added: i64",
4563+ SUM(contributions.lines_removed) AS "lines_removed: i64",
4564+ ROUND((COUNT(contributions.id)/CAST(
4565+ (SELECT contributions
4566+ FROM total_contributions) AS REAL))*100, 2) AS "percentage: f64"
4567+ FROM contributions
4568+ LEFT JOIN authors ON (contributions.author_id = authors.id)
4569+ WHERE repo_path = ?
4570+ AND contributions.id <=
4571+ (SELECT hash
4572+ FROM max_git_id)
4573+ GROUP BY authors.email
4574+ ORDER BY "count: i64" DESC
4575 diff --git a/crates/database/queries/authors_upsert.sql b/crates/database/queries/authors_upsert.sql
4576new file mode 100644
4577index 0000000..1c55636
4578--- /dev/null
4579+++ b/crates/database/queries/authors_upsert.sql
4580 @@ -0,0 +1,10 @@
4581+ INSERT INTO authors
4582+ (username, email)
4583+ VALUES
4584+ (?, ?)
4585+ ON CONFLICT
4586+ DO UPDATE
4587+ SET
4588+ username = username,
4589+ email = email
4590+ RETURNING id
4591 diff --git a/crates/database/queries/commits_count.sql b/crates/database/queries/commits_count.sql
4592new file mode 100644
4593index 0000000..0088bb8
4594--- /dev/null
4595+++ b/crates/database/queries/commits_count.sql
4596 @@ -0,0 +1,9 @@
4597+ SELECT
4598+ COUNT(id) AS count
4599+ FROM contributions
4600+ WHERE
4601+ repo_path = ? AND
4602+ id <= (
4603+ SELECT id FROM contributions
4604+ WHERE git_hash = ? AND repo_path = ?
4605+ )
4606 diff --git a/crates/database/queries/contribution_add.sql b/crates/database/queries/contribution_add.sql
4607new file mode 100644
4608index 0000000..e1085e3
4609--- /dev/null
4610+++ b/crates/database/queries/contribution_add.sql
4611 @@ -0,0 +1,11 @@
4612+ WITH previous(total) AS (
4613+ SELECT total FROM contribution_tally
4614+ WHERE
4615+ author_id = ? AND repo_path = ?
4616+ ORDER BY id DESC
4617+ LIMIT 1
4618+ )
4619+ INSERT INTO contribution_tally
4620+ (git_hash, repo_path, author_id, total)
4621+ VALUES
4622+ (?, ?, ?, COALESCE((SELECT(total) FROM previous), 0)+1)
4623 diff --git a/crates/database/queries/contribution_delete.sql b/crates/database/queries/contribution_delete.sql
4624new file mode 100644
4625index 0000000..6fdbe5c
4626--- /dev/null
4627+++ b/crates/database/queries/contribution_delete.sql
4628 @@ -0,0 +1,3 @@
4629+ DELETE FROM contribution_tally
4630+ WHERE
4631+ repo_path = ?
4632 diff --git a/crates/database/queries/contributions_add.sql b/crates/database/queries/contributions_add.sql
4633new file mode 100644
4634index 0000000..9883d94
4635--- /dev/null
4636+++ b/crates/database/queries/contributions_add.sql
4637 @@ -0,0 +1,4 @@
4638+ INSERT INTO contributions
4639+ (author_id, git_hash, repo_path, time, lines_added, lines_removed)
4640+ VALUES
4641+ (?, ?, ?, ?, ?, ?)
4642 diff --git a/crates/database/queries/contributions_bucket.sql b/crates/database/queries/contributions_bucket.sql
4643new file mode 100644
4644index 0000000..c4e2e55
4645--- /dev/null
4646+++ b/crates/database/queries/contributions_bucket.sql
4647 @@ -0,0 +1,10 @@
4648+ SELECT
4649+ COUNT(id) AS "count!: i64",
4650+ time,
4651+ SUM(lines_added) AS "added!: i64",
4652+ SUM(lines_removed) AS "removed!: i64"
4653+ FROM contributions WHERE
4654+ repo_path = ? AND
4655+ id <= (SELECT id FROM contributions WHERE git_hash = ?) AND
4656+ time <= ? AND time >= ?
4657+ GROUP BY strftime(?, time)
4658 diff --git a/crates/database/queries/contributions_delete.sql b/crates/database/queries/contributions_delete.sql
4659new file mode 100644
4660index 0000000..0ee5563
4661--- /dev/null
4662+++ b/crates/database/queries/contributions_delete.sql
4663 @@ -0,0 +1,2 @@
4664+ DELETE FROM contributions
4665+ WHERE repo_path = ?
4666 diff --git a/crates/database/queries/contributions_list.sql b/crates/database/queries/contributions_list.sql
4667new file mode 100644
4668index 0000000..fae5633
4669--- /dev/null
4670+++ b/crates/database/queries/contributions_list.sql
4671 @@ -0,0 +1,42 @@
4672+ WITH RECURSIVE
4673+ tallys AS (
4674+ SELECT
4675+ authors.username AS name,
4676+ authors.email AS email,
4677+ MAX(contribution_tally.total) AS total
4678+ FROM contribution_tally
4679+ LEFT JOIN authors ON
4680+ (authors.id = contribution_tally.author_id)
4681+ WHERE
4682+ contribution_tally.id < (
4683+ SELECT id FROM contributions
4684+ WHERE
4685+ repo_path = ? AND git_hash = ?
4686+ ORDER BY id DESC
4687+ LIMIT 1
4688+ ) AND
4689+ contribution_tally.repo_path = ?
4690+ GROUP BY authors.email
4691+ ORDER BY contribution_tally.total DESC
4692+ LIMIT 5),
4693+ raw_commits AS (
4694+ SELECT
4695+ COUNT(*) as total
4696+ FROM contributions
4697+ WHERE
4698+ contributions.id < (
4699+ SELECT id FROM contributions
4700+ WHERE
4701+ repo_path = ? AND git_hash = ?
4702+ ORDER BY id DESC
4703+ LIMIT 1
4704+ ) AND
4705+ contributions.repo_path = ?
4706+ )
4707+ SELECT
4708+ name,
4709+ email,
4710+ (
4711+ CAST(tallys.total AS REAL)/(SELECT total FROM raw_commits)
4712+ ) * 100 AS "percentage: i64"
4713+ FROM tallys
4714 diff --git a/crates/database/queries/dags_create.sql b/crates/database/queries/dags_create.sql
4715new file mode 100644
4716index 0000000..07bd1bb
4717--- /dev/null
4718+++ b/crates/database/queries/dags_create.sql
4719 @@ -0,0 +1,2 @@
4720+ INSERT INTO dags (manifest_id, dag_content)
4721+ VALUES (?, ?) RETURNING id
4722 diff --git a/crates/database/queries/dags_read.sql b/crates/database/queries/dags_read.sql
4723new file mode 100644
4724index 0000000..d2bb6ca
4725--- /dev/null
4726+++ b/crates/database/queries/dags_read.sql
4727 @@ -0,0 +1,3 @@
4728+ SELECT
4729+ id, manifest_id, dag_content
4730+ FROM dags WHERE manifest_id = ?
4731 diff --git a/crates/database/queries/job_tracking_add.sql b/crates/database/queries/job_tracking_add.sql
4732new file mode 100644
4733index 0000000..7e70756
4734--- /dev/null
4735+++ b/crates/database/queries/job_tracking_add.sql
4736 @@ -0,0 +1,4 @@
4737+ INSERT INTO job_tracking
4738+ (repo_path, git_hash, kind, job_id)
4739+ VALUES
4740+ (?, ?, ?, ?)
4741 diff --git a/crates/database/queries/job_tracking_delete.sql b/crates/database/queries/job_tracking_delete.sql
4742new file mode 100644
4743index 0000000..3aef33d
4744--- /dev/null
4745+++ b/crates/database/queries/job_tracking_delete.sql
4746 @@ -0,0 +1,3 @@
4747+ DELETE FROM job_tracking
4748+ WHERE
4749+ repo_path = ?
4750 diff --git a/crates/database/queries/job_tracking_delete_by_repo.sql b/crates/database/queries/job_tracking_delete_by_repo.sql
4751new file mode 100644
4752index 0000000..ebd0363
4753--- /dev/null
4754+++ b/crates/database/queries/job_tracking_delete_by_repo.sql
4755 @@ -0,0 +1,3 @@
4756+ DELETE FROM jobs
4757+ WHERE
4758+ repo_path = ?
4759 diff --git a/crates/database/queries/job_tracking_read.sql b/crates/database/queries/job_tracking_read.sql
4760new file mode 100644
4761index 0000000..823dfc5
4762--- /dev/null
4763+++ b/crates/database/queries/job_tracking_read.sql
4764 @@ -0,0 +1,8 @@
4765+ SELECT
4766+ git_hash AS "git_hash!"
4767+ FROM job_tracking
4768+ WHERE
4769+ repo_path = ? AND
4770+ kind = ?
4771+ ORDER BY id DESC
4772+ LIMIT 1
4773 diff --git a/crates/database/queries/jobs_create.sql b/crates/database/queries/jobs_create.sql
4774new file mode 100644
4775index 0000000..4a1bea5
4776--- /dev/null
4777+++ b/crates/database/queries/jobs_create.sql
4778 @@ -0,0 +1,5 @@
4779+ INSERT INTO jobs
4780+ (created_at, repo_path, kind)
4781+ VALUES
4782+ (?, ?, ?)
4783+ RETURNING id
4784 diff --git a/crates/database/queries/jobs_delete.sql b/crates/database/queries/jobs_delete.sql
4785new file mode 100644
4786index 0000000..c70ba25
4787--- /dev/null
4788+++ b/crates/database/queries/jobs_delete.sql
4789 @@ -0,0 +1,2 @@
4790+ DELETE FROM jobs
4791+ WHERE jobs.id = ?
4792 diff --git a/crates/database/queries/jobs_list.sql b/crates/database/queries/jobs_list.sql
4793new file mode 100644
4794index 0000000..b1e7f72
4795--- /dev/null
4796+++ b/crates/database/queries/jobs_list.sql
4797 @@ -0,0 +1,15 @@
4798+ SELECT
4799+ jobs.id AS "id!",
4800+ created_at AS "created_at: OffsetDateTime",
4801+ jobs.repo_path AS "repo_path!: String",
4802+ jobs.kind AS "kind!: String",
4803+ runtime AS "runtime: u32",
4804+ success AS "success: bool",
4805+ (
4806+ SELECT COUNT(id) FROM job_tracking
4807+ WHERE job_tracking.job_id = jobs.id
4808+ ) AS "commits!: i32"
4809+ FROM jobs
4810+ WHERE
4811+ jobs.repo_path = ? OR ? IS NULL
4812+ ORDER BY DATETIME(created_at) ASC
4813 diff --git a/crates/database/queries/jobs_update.sql b/crates/database/queries/jobs_update.sql
4814new file mode 100644
4815index 0000000..ea7e521
4816--- /dev/null
4817+++ b/crates/database/queries/jobs_update.sql
4818 @@ -0,0 +1,6 @@
4819+ UPDATE jobs
4820+ SET
4821+ runtime = ?,
4822+ success = ?
4823+ WHERE
4824+ jobs.id = ?
4825 diff --git a/crates/database/queries/languages_add.sql b/crates/database/queries/languages_add.sql
4826new file mode 100644
4827index 0000000..54551b3
4828--- /dev/null
4829+++ b/crates/database/queries/languages_add.sql
4830 @@ -0,0 +1,4 @@
4831+ INSERT INTO languages
4832+ (git_hash, repo_path, language, loc)
4833+ VALUES
4834+ (?,?,?,?)
4835 diff --git a/crates/database/queries/languages_delete.sql b/crates/database/queries/languages_delete.sql
4836new file mode 100644
4837index 0000000..06c4349
4838--- /dev/null
4839+++ b/crates/database/queries/languages_delete.sql
4840 @@ -0,0 +1,3 @@
4841+ DELETE FROM languages
4842+ WHERE
4843+ repo_path = ?
4844 diff --git a/crates/database/queries/languages_list.sql b/crates/database/queries/languages_list.sql
4845new file mode 100644
4846index 0000000..608a719
4847--- /dev/null
4848+++ b/crates/database/queries/languages_list.sql
4849 @@ -0,0 +1,13 @@
4850+ SELECT
4851+ language,
4852+ loc,
4853+ ROUND(
4854+ CAST(loc AS REAL)/CAST(
4855+ (SELECT SUM(loc) FROM languages
4856+ WHERE repo_path = ? AND git_hash = ?) AS REAL)*100)
4857+ AS "percentage: f64"
4858+ FROM languages
4859+ WHERE
4860+ repo_path = ? AND git_hash = ?
4861+ ORDER BY "percentage: f64" DESC
4862+ LIMIT 5
4863 diff --git a/crates/database/queries/latest_commit.sql b/crates/database/queries/latest_commit.sql
4864new file mode 100644
4865index 0000000..bd26eac
4866--- /dev/null
4867+++ b/crates/database/queries/latest_commit.sql
4868 @@ -0,0 +1,7 @@
4869+ SELECT
4870+ git_hash
4871+ FROM contributions
4872+ WHERE
4873+ repo_path = ?
4874+ ORDER BY id DESC
4875+ LIMIT 1
4876 diff --git a/crates/database/queries/mail_deactivate_unused.sql b/crates/database/queries/mail_deactivate_unused.sql
4877new file mode 100644
4878index 0000000..730dc92
4879--- /dev/null
4880+++ b/crates/database/queries/mail_deactivate_unused.sql
4881 @@ -0,0 +1 @@
4882+ UPDATE lists SET enabled = 0 WHERE address NOT IN (?)
4883 diff --git a/crates/database/queries/mail_deliver_message.sql b/crates/database/queries/mail_deliver_message.sql
4884new file mode 100644
4885index 0000000..6763a82
4886--- /dev/null
4887+++ b/crates/database/queries/mail_deliver_message.sql
4888 @@ -0,0 +1,10 @@
4889+ INSERT INTO messages
4890+ (message_id, list_id, in_reply_to, subject, mail_from, content_body, raw_message)
4891+ VALUES (
4892+ ?,
4893+ ( SELECT id FROM lists WHERE address = ? LIMIT 1 ),
4894+ ?,
4895+ ?,
4896+ ?,
4897+ ?, ?
4898+ ) RETURNING messages.id
4899 diff --git a/crates/database/queries/mail_has_message.sql b/crates/database/queries/mail_has_message.sql
4900new file mode 100644
4901index 0000000..e32ce95
4902--- /dev/null
4903+++ b/crates/database/queries/mail_has_message.sql
4904 @@ -0,0 +1,5 @@
4905+ SELECT
4906+ CASE
4907+ WHEN (SELECT COUNT(*) FROM messages WHERE message_id = ?) >= 1 THEN TRUE
4908+ ELSE FALSE
4909+ END AS "has_message!: bool"
4910 diff --git a/crates/database/queries/mail_list_create.sql b/crates/database/queries/mail_list_create.sql
4911new file mode 100644
4912index 0000000..770b76a
4913--- /dev/null
4914+++ b/crates/database/queries/mail_list_create.sql
4915 @@ -0,0 +1,9 @@
4916+ INSERT INTO lists
4917+ (name, address, description, enabled)
4918+ VALUES
4919+ (?, ?, ?, ?)
4920+ ON CONFLICT (address) DO
4921+ UPDATE SET
4922+ name = ?,
4923+ description = ?,
4924+ enabled = ?
4925 diff --git a/crates/database/queries/mail_list_outbox.sql b/crates/database/queries/mail_list_outbox.sql
4926new file mode 100644
4927index 0000000..8eaa79e
4928--- /dev/null
4929+++ b/crates/database/queries/mail_list_outbox.sql
4930 @@ -0,0 +1,13 @@
4931+ SELECT
4932+ outbox.id,
4933+ messages.message_id AS "message_id!: String",
4934+ lists.name AS "list_name!: String",
4935+ participants.address AS "address!: String",
4936+ messages.raw_message AS raw_message
4937+ FROM outbox
4938+ LEFT JOIN participants ON outbox.recipient = participants.address
4939+ LEFT JOIN messages ON messages.id = outbox.message_id
4940+ LEFT JOIN lists ON messages.list_id = lists.id
4941+ WHERE
4942+ (SELECT COUNT(*) FROM delivery_failures
4943+ WHERE delivery_failures.outbox_id = outbox.id) < ?
4944 diff --git a/crates/database/queries/mail_outbox_failed.sql b/crates/database/queries/mail_outbox_failed.sql
4945new file mode 100644
4946index 0000000..c7dafcb
4947--- /dev/null
4948+++ b/crates/database/queries/mail_outbox_failed.sql
4949 @@ -0,0 +1,3 @@
4950+ INSERT INTO delivery_failures
4951+ (outbox_id, error_kind, message)
4952+ VALUES (?, ?, ?)
4953 diff --git a/crates/database/queries/mail_outbox_successful.sql b/crates/database/queries/mail_outbox_successful.sql
4954new file mode 100644
4955index 0000000..0b0269c
4956--- /dev/null
4957+++ b/crates/database/queries/mail_outbox_successful.sql
4958 @@ -0,0 +1 @@
4959+ DELETE FROM outbox WHERE id = ?
4960 diff --git a/crates/database/queries/mail_read_message.sql b/crates/database/queries/mail_read_message.sql
4961new file mode 100644
4962index 0000000..1696cae
4963--- /dev/null
4964+++ b/crates/database/queries/mail_read_message.sql
4965 @@ -0,0 +1 @@
4966+ SELECT * FROM messages WHERE id = ?
4967 diff --git a/crates/database/queries/mail_read_thread.sql b/crates/database/queries/mail_read_thread.sql
4968new file mode 100644
4969index 0000000..09342d0
4970--- /dev/null
4971+++ b/crates/database/queries/mail_read_thread.sql
4972 @@ -0,0 +1,18 @@
4973+ WITH RECURSIVE tree AS (
4974+ SELECT messages.*, 0 AS depth FROM messages
4975+ WHERE messages.message_id = ?
4976+ UNION ALL
4977+ SELECT other.*, depth + 1 FROM messages other
4978+ JOIN tree ON other.in_reply_to = tree.message_id
4979+ )
4980+ SELECT
4981+ message_id,
4982+ in_reply_to,
4983+ subject,
4984+ (SELECT address FROM participants WHERE participants.id = tree.mail_from LIMIT 1)
4985+ mail_from,
4986+ raw_message,
4987+ content_body,
4988+ reply_count,
4989+ depth AS "depth!: i64"
4990+ FROM tree ORDER BY depth
4991 diff --git a/crates/database/queries/mail_read_threads.sql b/crates/database/queries/mail_read_threads.sql
4992new file mode 100644
4993index 0000000..d2e3b4a
4994--- /dev/null
4995+++ b/crates/database/queries/mail_read_threads.sql
4996 @@ -0,0 +1,9 @@
4997+ SELECT
4998+ messages.message_id,
4999+ messages.subject,
5000+ messages.reply_count,
5001+ participants.address AS "mail_from!: String"
5002+ FROM messages
5003+ LEFT JOIN lists ON lists.id = messages.id
5004+ LEFT JOIN participants ON participants.id = messages.mail_from
5005+ WHERE lists.name = ? AND messages.in_reply_to IS NULL
5006 diff --git a/crates/database/queries/mail_thread_count.sql b/crates/database/queries/mail_thread_count.sql
5007new file mode 100644
5008index 0000000..5dc619a
5009--- /dev/null
5010+++ b/crates/database/queries/mail_thread_count.sql
5011 @@ -0,0 +1,4 @@
5012+ SELECT COUNT(*) AS "count: i64" FROM messages
5013+ WHERE
5014+ list_id = (SELECT list_id FROM lists WHERE list_id = ?) AND
5015+ in_reply_to IS NULL
5016 diff --git a/crates/database/queries/mail_thread_summary.sql b/crates/database/queries/mail_thread_summary.sql
5017new file mode 100644
5018index 0000000..c76aa55
5019--- /dev/null
5020+++ b/crates/database/queries/mail_thread_summary.sql
5021 @@ -0,0 +1,3 @@
5022+ SELECT messages.* FROM messages
5023+ LEFT JOIN lists ON lists.id = messages.id
5024+ WHERE lists.name = ? AND messages.reply_to IS NULL
5025 diff --git a/crates/database/queries/mail_upsert_participant.sql b/crates/database/queries/mail_upsert_participant.sql
5026new file mode 100644
5027index 0000000..67b0973
5028--- /dev/null
5029+++ b/crates/database/queries/mail_upsert_participant.sql
5030 @@ -0,0 +1,6 @@
5031+ INSERT INTO participants
5032+ (address, authorized_sender)
5033+ VALUES (?, ?)
5034+ ON CONFLICT DO NOTHING;
5035+
5036+ SELECT id FROM participants WHERE address = ?
5037 diff --git a/crates/database/queries/mail_upsert_subscription.sql b/crates/database/queries/mail_upsert_subscription.sql
5038new file mode 100644
5039index 0000000..78cd886
5040--- /dev/null
5041+++ b/crates/database/queries/mail_upsert_subscription.sql
5042 @@ -0,0 +1,6 @@
5043+ INSERT INTO subscriptions
5044+ (participant_id, list_id)
5045+ VALUES (
5046+ (SELECT id FROM participants WHERE address = ? LIMIT 1),
5047+ (SELECT id FROM lists WHERE address = ? LIMIT 1)
5048+ ) ON CONFLICT DO NOTHING;
5049 diff --git a/crates/database/queries/manifests_concise.sql b/crates/database/queries/manifests_concise.sql
5050new file mode 100644
5051index 0000000..ff04dcf
5052--- /dev/null
5053+++ b/crates/database/queries/manifests_concise.sql
5054 @@ -0,0 +1,25 @@
5055+ WITH params AS (
5056+ SELECT
5057+ ? AS collection,
5058+ ? AS name,
5059+ ? AS manifest_id,
5060+ ? AS git_hash
5061+ )
5062+ SELECT
5063+ steps.id AS step_id,
5064+ steps.name AS step_name,
5065+ workflows.id AS workflow_id,
5066+ workflows.name AS workflow_name,
5067+ manifests.id AS manifest_id,
5068+ steps.exit_code,
5069+ steps.finished_at
5070+ FROM steps
5071+ LEFT JOIN workflows ON workflows.id = steps.workflow_id
5072+ LEFT JOIN manifests ON manifests.id = workflows.manifest_id
5073+ WHERE
5074+ (manifests.collection IS (SELECT collection FROM params) OR (SELECT collection FROM params) IS NULL)
5075+ AND (manifests.name IS (SELECT name FROM params) OR (SELECT name FROM params) IS NULL)
5076+ AND (manifests.id IS (SELECT manifest_id FROM params) OR (SELECT manifest_id FROM params) IS NULL)
5077+ AND (manifests.git_hash IS (SELECT git_hash FROM params) OR (SELECT git_hash FROM params) IS NULL)
5078+ LIMIT ?
5079+ OFFSET ?
5080 diff --git a/crates/database/queries/manifests_create.sql b/crates/database/queries/manifests_create.sql
5081new file mode 100644
5082index 0000000..e826336
5083--- /dev/null
5084+++ b/crates/database/queries/manifests_create.sql
5085 @@ -0,0 +1,2 @@
5086+ INSERT INTO manifests (collection, name, git_hash)
5087+ VALUES (?, ?, ?) RETURNING id
5088 diff --git a/crates/database/queries/manifests_list.sql b/crates/database/queries/manifests_list.sql
5089new file mode 100644
5090index 0000000..55ea5b1
5091--- /dev/null
5092+++ b/crates/database/queries/manifests_list.sql
5093 @@ -0,0 +1,14 @@
5094+ SELECT id,
5095+ collection,
5096+ name,
5097+ git_hash,
5098+ created_at,
5099+ started_at,
5100+ duration
5101+ FROM manifests
5102+ WHERE
5103+ collection = ? OR collection IS NULL
5104+ AND name = ? OR name IS NULL
5105+ AND git_hash = ? OR git_hash IS NULL
5106+ LIMIT ?
5107+ OFFSET ?
5108 diff --git a/crates/database/queries/manifests_read.sql b/crates/database/queries/manifests_read.sql
5109new file mode 100644
5110index 0000000..e74b333
5111--- /dev/null
5112+++ b/crates/database/queries/manifests_read.sql
5113 @@ -0,0 +1,18 @@
5114+ WITH params AS (
5115+ SELECT
5116+ ? AS _manifest_id,
5117+ ? AS _collection,
5118+ ? AS _name
5119+ )
5120+ SELECT manifests.id,
5121+ name,
5122+ collection,
5123+ git_hash,
5124+ created_at,
5125+ started_at,
5126+ finished_at
5127+ FROM manifests
5128+ WHERE
5129+ manifests.id = (SELECT _manifest_id FROM params)
5130+ AND (manifests.collection IS (SELECT _collection FROM params) OR (SELECT _collection FROM params) IS NULL)
5131+ AND (manifests.name IS (SELECT _name FROM params) OR (SELECT _name FROM params) IS NULL)
5132 diff --git a/crates/database/queries/manifests_update_finish.sql b/crates/database/queries/manifests_update_finish.sql
5133new file mode 100644
5134index 0000000..d0a6513
5135--- /dev/null
5136+++ b/crates/database/queries/manifests_update_finish.sql
5137 @@ -0,0 +1,4 @@
5138+ UPDATE manifests
5139+ SET
5140+ finished_at = UNIXEPOCH()
5141+ WHERE id = ?
5142 diff --git a/crates/database/queries/manifests_update_start.sql b/crates/database/queries/manifests_update_start.sql
5143new file mode 100644
5144index 0000000..fc85a4d
5145--- /dev/null
5146+++ b/crates/database/queries/manifests_update_start.sql
5147 @@ -0,0 +1,3 @@
5148+ UPDATE manifests
5149+ SET started_at = UNIXEPOCH()
5150+ WHERE id = ?
5151 diff --git a/crates/database/queries/samples_create.sql b/crates/database/queries/samples_create.sql
5152new file mode 100644
5153index 0000000..7fc297f
5154--- /dev/null
5155+++ b/crates/database/queries/samples_create.sql
5156 @@ -0,0 +1,18 @@
5157+ INSERT INTO samples
5158+ (
5159+ manifest_id,
5160+ load_1m,
5161+ load_5m,
5162+ load_15m,
5163+ disk_total_bytes,
5164+ disk_free_bytes,
5165+ net_sent_bytes,
5166+ net_received_bytes,
5167+ net_sent_packets,
5168+ net_received_packets,
5169+ mem_total_bytes,
5170+ mem_free_bytes,
5171+ mem_available_bytes
5172+ )
5173+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5174+ RETURNING id
5175 diff --git a/crates/database/queries/samples_read.sql b/crates/database/queries/samples_read.sql
5176new file mode 100644
5177index 0000000..6842939
5178--- /dev/null
5179+++ b/crates/database/queries/samples_read.sql
5180 @@ -0,0 +1,15 @@
5181+ SELECT id,
5182+ load_1m,
5183+ load_5m,
5184+ load_15m,
5185+ disk_total_bytes,
5186+ disk_free_bytes,
5187+ net_sent_bytes,
5188+ net_received_bytes,
5189+ net_sent_packets,
5190+ net_received_packets,
5191+ mem_total_bytes,
5192+ mem_free_bytes,
5193+ mem_available_bytes
5194+ FROM samples
5195+ WHERE manifest_id = ?
5196 diff --git a/crates/database/queries/steps_create.sql b/crates/database/queries/steps_create.sql
5197new file mode 100644
5198index 0000000..ababf23
5199--- /dev/null
5200+++ b/crates/database/queries/steps_create.sql
5201 @@ -0,0 +1,2 @@
5202+ INSERT INTO steps (workflow_id, name, input, shell, environment_json)
5203+ VALUES (?, ?, ?, ?, json_array(?)) RETURNING id
5204 diff --git a/crates/database/queries/steps_list.sql b/crates/database/queries/steps_list.sql
5205new file mode 100644
5206index 0000000..4ab824b
5207--- /dev/null
5208+++ b/crates/database/queries/steps_list.sql
5209 @@ -0,0 +1,11 @@
5210+ SELECT id,
5211+ name,
5212+ shell,
5213+ input,
5214+ stdout,
5215+ stderr,
5216+ started_at,
5217+ finished_at,
5218+ exit_code
5219+ FROM steps
5220+ WHERE workflow_id = ?
5221 diff --git a/crates/database/queries/steps_read.sql b/crates/database/queries/steps_read.sql
5222new file mode 100644
5223index 0000000..2b5c4e3
5224--- /dev/null
5225+++ b/crates/database/queries/steps_read.sql
5226 @@ -0,0 +1,11 @@
5227+ SELECT
5228+ id,
5229+ name,
5230+ shell,
5231+ input,
5232+ stdout,
5233+ stderr,
5234+ started_at,
5235+ finished_at,
5236+ exit_code
5237+ FROM steps WHERE id = ?
5238 diff --git a/crates/database/queries/steps_read_env.sql b/crates/database/queries/steps_read_env.sql
5239new file mode 100644
5240index 0000000..2312f2b
5241--- /dev/null
5242+++ b/crates/database/queries/steps_read_env.sql
5243 @@ -0,0 +1 @@
5244+ SELECT name, value FROM steps_env_vars WHERE step_id = ?
5245 diff --git a/crates/database/queries/steps_update_finish.sql b/crates/database/queries/steps_update_finish.sql
5246new file mode 100644
5247index 0000000..d80c0e3
5248--- /dev/null
5249+++ b/crates/database/queries/steps_update_finish.sql
5250 @@ -0,0 +1,8 @@
5251+ UPDATE steps
5252+ SET
5253+ finished_at = UNIXEPOCH(),
5254+ stdout = ?,
5255+ stderr = ?,
5256+ exit_code = ?
5257+ WHERE
5258+ id = ?
5259 diff --git a/crates/database/queries/steps_update_start.sql b/crates/database/queries/steps_update_start.sql
5260new file mode 100644
5261index 0000000..387654b
5262--- /dev/null
5263+++ b/crates/database/queries/steps_update_start.sql
5264 @@ -0,0 +1,5 @@
5265+ UPDATE steps
5266+ SET
5267+ started_at = UNIXEPOCH()
5268+ WHERE
5269+ id = ?
5270 diff --git a/crates/database/queries/workflows_create.sql b/crates/database/queries/workflows_create.sql
5271new file mode 100644
5272index 0000000..0063b5f
5273--- /dev/null
5274+++ b/crates/database/queries/workflows_create.sql
5275 @@ -0,0 +1,2 @@
5276+ INSERT INTO workflows (manifest_id, name)
5277+ VALUES (?, ?) RETURNING id
5278 diff --git a/crates/database/queries/workflows_list.sql b/crates/database/queries/workflows_list.sql
5279new file mode 100644
5280index 0000000..7ba11c6
5281--- /dev/null
5282+++ b/crates/database/queries/workflows_list.sql
5283 @@ -0,0 +1,6 @@
5284+ SELECT id,
5285+ name,
5286+ started_at,
5287+ finished_at
5288+ FROM workflows
5289+ WHERE manifest_id = ?
5290 diff --git a/crates/database/queries/workflows_read.sql b/crates/database/queries/workflows_read.sql
5291new file mode 100644
5292index 0000000..a772b4b
5293--- /dev/null
5294+++ b/crates/database/queries/workflows_read.sql
5295 @@ -0,0 +1 @@
5296+ SELECT id, name, started_at, finished_at FROM workflows WHERE id = ?
5297 diff --git a/crates/database/queries/workflows_update_finish.sql b/crates/database/queries/workflows_update_finish.sql
5298new file mode 100644
5299index 0000000..769a8be
5300--- /dev/null
5301+++ b/crates/database/queries/workflows_update_finish.sql
5302 @@ -0,0 +1,5 @@
5303+ UPDATE workflows
5304+ SET
5305+ finished_at = UNIXEPOCH()
5306+ WHERE
5307+ id = ?
5308 diff --git a/crates/database/queries/workflows_update_start.sql b/crates/database/queries/workflows_update_start.sql
5309new file mode 100644
5310index 0000000..11be057
5311--- /dev/null
5312+++ b/crates/database/queries/workflows_update_start.sql
5313 @@ -0,0 +1,5 @@
5314+ UPDATE workflows
5315+ SET
5316+ started_at = UNIXEPOCH()
5317+ WHERE
5318+ id = ?
5319 diff --git a/crates/database/src/build.rs b/crates/database/src/build.rs
5320new file mode 100644
5321index 0000000..8ac8030
5322--- /dev/null
5323+++ b/crates/database/src/build.rs
5324 @@ -0,0 +1,562 @@
5325+ use std::collections::HashMap;
5326+
5327+ use async_trait::async_trait;
5328+ use serde::{Deserialize, Serialize};
5329+
5330+ use crate::{Error, Tx, Wrapper as Database};
5331+
5332+ pub type ConciseStep = (i64, String, Option<bool>);
5333+ pub type ConciseWorkflow = (i64, String, Vec<ConciseStep>);
5334+
5335+ /// An environment variable
5336+ pub struct Env((String, Option<String>));
5337+ /// Environment Variables
5338+ pub struct Environment(Vec<Env>);
5339+
5340+ impl From<Environment> for HashMap<String, Option<String>> {
5341+ fn from(val: Environment) -> Self {
5342+ let values: Vec<(String, Option<String>)> = val
5343+ .0
5344+ .iter()
5345+ .map(|env| (env.0 .0.clone(), env.0 .1.clone()))
5346+ .collect();
5347+ HashMap::from_iter(values)
5348+ }
5349+ }
5350+
5351+ // Manifest item with all of it's inner state for quick display
5352+ #[derive(Clone, Serialize, Deserialize)]
5353+ pub struct ManifestItem {
5354+ pub manifest: Manifest,
5355+ // Present if the manifest job is completed, true if all items within were successful
5356+ pub success: Option<bool>,
5357+ // Concise representation of workflows and steps for display in the UI
5358+ pub workflows: Vec<ConciseWorkflow>,
5359+ }
5360+
5361+ #[derive(Clone, Serialize, Deserialize)]
5362+ // everything needed to populate the build status page
5363+ pub struct ManifestView {
5364+ pub manifest: Manifest,
5365+ // pub samples: Vec<Sample>,
5366+ pub workflows: Vec<(Workflow, Vec<Step>)>,
5367+ pub dag: Dag,
5368+ }
5369+
5370+ #[derive(Clone, Default, Serialize, Deserialize)]
5371+ pub struct Sample {
5372+ pub id: i64,
5373+ pub load_1m: Option<i64>,
5374+ pub load_5m: Option<i64>,
5375+ pub load_15m: Option<i64>,
5376+ pub disk_total_bytes: Option<i64>,
5377+ pub disk_free_bytes: Option<i64>,
5378+ pub net_sent_bytes: Option<i64>,
5379+ pub net_received_bytes: Option<i64>,
5380+ pub net_sent_packets: Option<i64>,
5381+ pub net_received_packets: Option<i64>,
5382+ pub mem_total_bytes: Option<i64>,
5383+ pub mem_free_bytes: Option<i64>,
5384+ pub mem_available_bytes: Option<i64>,
5385+ }
5386+
5387+ #[derive(Clone, Serialize, Deserialize)]
5388+ pub struct Manifest {
5389+ pub id: i64,
5390+ pub collection: String,
5391+ pub name: String,
5392+ pub git_hash: String,
5393+ pub created_at: i64,
5394+ pub started_at: Option<i64>,
5395+ pub finished_at: Option<i64>,
5396+ // pub dot_content: String,
5397+ }
5398+
5399+ // TODO
5400+ //
5401+ #[derive(Debug, Clone, Serialize, Deserialize)]
5402+ pub struct Workflow {
5403+ pub id: i64,
5404+ pub name: String,
5405+ pub started_at: Option<i64>,
5406+ pub finished_at: Option<i64>,
5407+ }
5408+
5409+ pub struct WorkflowView {
5410+ pub workflow: Workflow,
5411+ pub steps: Vec<Step>,
5412+ }
5413+
5414+ #[derive(Debug, Clone, Serialize, Deserialize)]
5415+ pub struct Step {
5416+ pub id: i64,
5417+ pub name: String,
5418+ pub shell: String,
5419+ pub input: String,
5420+ pub stdout: Option<String>,
5421+ pub stderr: Option<String>,
5422+ pub started_at: Option<i64>,
5423+ pub finished_at: Option<i64>,
5424+ pub exit_code: Option<i64>,
5425+ }
5426+
5427+ #[derive(Clone, Serialize, Deserialize)]
5428+ pub struct Dag {
5429+ pub id: i64,
5430+ pub manifest_id: i64,
5431+ pub dag_content: String,
5432+ }
5433+
5434+ /// Used to list manifests from the UI
5435+ #[derive(Clone)]
5436+ pub enum Identifier {
5437+ GitSHA(String),
5438+ Id(i64),
5439+ }
5440+
5441+ #[async_trait]
5442+ pub trait BuildTx {
5443+ async fn create_manifest(
5444+ &mut self,
5445+ collection: &str,
5446+ name: &str,
5447+ git_hash: &str,
5448+ ) -> Result<i64, Error>;
5449+ async fn create_workflow(&mut self, manifest_id: i64, name: &str) -> Result<i64, Error>;
5450+ async fn create_step(
5451+ &mut self,
5452+ job_id: i64,
5453+ name: &str,
5454+ input: &str,
5455+ shell: &str,
5456+ environment: HashMap<String, String>,
5457+ ) -> Result<i64, Error>;
5458+ async fn create_dag(&mut self, manifest_id: i64, dag_content: &str) -> Result<i64, Error>;
5459+ }
5460+
5461+ #[async_trait]
5462+ impl BuildTx for Tx {
5463+ // manifests
5464+ async fn create_manifest(
5465+ &mut self,
5466+ collection: &str,
5467+ name: &str,
5468+ git_hash: &str,
5469+ ) -> Result<i64, Error> {
5470+ let ret = sqlx::query_file!("queries/manifests_create.sql", collection, name, git_hash)
5471+ .fetch_one(&mut *self.inner)
5472+ .await?;
5473+ Ok(ret.id)
5474+ }
5475+
5476+ async fn create_workflow(&mut self, manifest_id: i64, name: &str) -> Result<i64, Error> {
5477+ let ret = sqlx::query_file!("queries/workflows_create.sql", manifest_id, name)
5478+ .fetch_one(&mut *self.inner)
5479+ .await?;
5480+ Ok(ret.id)
5481+ }
5482+
5483+ async fn create_step(
5484+ &mut self,
5485+ workflow_id: i64,
5486+ name: &str,
5487+ input: &str,
5488+ shell: &str,
5489+ environment: HashMap<String, String>,
5490+ ) -> Result<i64, Error> {
5491+ let environment_json = serde_json::ser::to_string(&environment).unwrap();
5492+ let ret = sqlx::query_file!(
5493+ "queries/steps_create.sql",
5494+ workflow_id,
5495+ name,
5496+ input,
5497+ shell,
5498+ environment_json
5499+ )
5500+ .fetch_one(&mut *self.inner)
5501+ .await?;
5502+ Ok(ret.id)
5503+ }
5504+
5505+ async fn create_dag(&mut self, manifest_id: i64, dag_content: &str) -> Result<i64, Error> {
5506+ let ret = sqlx::query_file!("queries/dags_create.sql", manifest_id, dag_content)
5507+ .fetch_one(&mut *self.inner)
5508+ .await?;
5509+ Ok(ret.id)
5510+ }
5511+ }
5512+
5513+ #[async_trait]
5514+ pub trait BuildExt {
5515+ // manifests
5516+ async fn create_manifest(
5517+ &self,
5518+ collection: &str,
5519+ name: &str,
5520+ git_hash: &str,
5521+ ) -> Result<i64, Error>;
5522+ async fn update_manifest_start(&self, manifest_id: i64) -> Result<(), Error>;
5523+ async fn update_manifest_finish(&self, manifest_id: i64) -> Result<(), Error>;
5524+ async fn list_manifest(
5525+ &self,
5526+ collection: Option<&str>,
5527+ name: Option<&str>,
5528+ filter: Option<&Identifier>,
5529+ limit: i64,
5530+ offset: i64,
5531+ ) -> Result<Vec<ManifestItem>, Error>;
5532+ async fn read_manifest(
5533+ &self,
5534+ collection: &str,
5535+ name: &str,
5536+ manifest_id: i64,
5537+ ) -> Result<ManifestView, Error>;
5538+
5539+ // workflows
5540+ async fn create_workflow(&self, manifest_id: i64, name: &str) -> Result<i64, Error>;
5541+ async fn update_workflow_start(&self, workflow_id: i64) -> Result<(), Error>;
5542+ async fn update_workflow_finish(&self, workflow_id: i64) -> Result<(), Error>;
5543+ async fn read_workflow(&self, workflow_id: i64) -> Result<Workflow, Error>;
5544+ async fn read_workflow_view(&self, workflow_id: i64) -> Result<WorkflowView, Error>;
5545+ // steps
5546+ async fn create_step(
5547+ &self,
5548+ job_id: i64,
5549+ name: &str,
5550+ input: &str,
5551+ shell: &str,
5552+ environment: HashMap<String, String>,
5553+ ) -> Result<i64, Error>;
5554+ async fn read_step(&self, step_id: i64) -> Result<Step, Error>;
5555+ async fn read_step_env(&self, step_id: i64) -> Result<Environment, Error>;
5556+ async fn update_step_start(&self, step_id: i64) -> Result<(), Error>;
5557+ async fn update_step_finish(
5558+ &self,
5559+ step_id: i64,
5560+ stdout: &str,
5561+ stderr: &str,
5562+ exit_code: i8,
5563+ ) -> Result<(), Error>;
5564+
5565+ // misc
5566+ async fn create_sample(&self, manifest_id: i64, sample: Sample) -> Result<i64, Error>;
5567+
5568+ async fn read_dag(&self, manifest_id: i64) -> Result<Dag, Error>;
5569+ }
5570+
5571+ #[async_trait]
5572+ impl BuildExt for Database {
5573+ // manifests
5574+ async fn create_manifest(
5575+ &self,
5576+ collection: &str,
5577+ name: &str,
5578+ git_hash: &str,
5579+ ) -> Result<i64, Error> {
5580+ let ret = sqlx::query_file!("queries/manifests_create.sql", collection, name, git_hash)
5581+ .fetch_one(&self.pool)
5582+ .await?;
5583+ Ok(ret.id)
5584+ }
5585+
5586+ async fn update_manifest_start(&self, manifest_id: i64) -> Result<(), Error> {
5587+ sqlx::query_file!("queries/manifests_update_start.sql", manifest_id,)
5588+ .execute(&self.pool)
5589+ .await?;
5590+ Ok(())
5591+ }
5592+
5593+ async fn update_manifest_finish(&self, manifest_id: i64) -> Result<(), Error> {
5594+ sqlx::query_file!("queries/manifests_update_finish.sql", manifest_id)
5595+ .execute(&self.pool)
5596+ .await?;
5597+ Ok(())
5598+ }
5599+
5600+ async fn list_manifest(
5601+ &self,
5602+ collection: Option<&str>,
5603+ name: Option<&str>,
5604+ filter: Option<&Identifier>,
5605+ limit: i64,
5606+ offset: i64,
5607+ ) -> Result<Vec<ManifestItem>, Error> {
5608+ let (manifest_id, git_hash) = match filter {
5609+ Some(Identifier::GitSHA(gitsha)) => (None, Some(gitsha.clone())),
5610+ Some(Identifier::Id(manifest_id)) => (Some(manifest_id.to_string()), None),
5611+ None => (None, None),
5612+ };
5613+ let records = sqlx::query_file!(
5614+ "queries/manifests_concise.sql",
5615+ collection,
5616+ name,
5617+ manifest_id,
5618+ git_hash,
5619+ limit,
5620+ offset
5621+ )
5622+ .fetch_all(&self.pool)
5623+ .await?;
5624+ let invocations: HashMap<i64, HashMap<i64, (String, Vec<ConciseStep>)>> =
5625+ records.iter().fold(HashMap::new(), |mut mapped, record| {
5626+ match mapped.get_mut(&record.manifest_id) {
5627+ Some(workflow) => match workflow.get_mut(&record.workflow_id) {
5628+ Some(steps) => steps.1.push((
5629+ record.step_id,
5630+ record.step_name.clone(),
5631+ record.exit_code.map(|code| code == 0),
5632+ )),
5633+ None => {
5634+ workflow.insert(
5635+ record.workflow_id,
5636+ (
5637+ record.workflow_name.clone().unwrap_or_default(),
5638+ vec![(
5639+ record.step_id,
5640+ record.step_name.clone(),
5641+ record.exit_code.map(|code| code == 0),
5642+ )],
5643+ ),
5644+ );
5645+ }
5646+ },
5647+ None => {
5648+ let workflow: HashMap<i64, (String, Vec<ConciseStep>)> = HashMap::from([(
5649+ record.workflow_id,
5650+ (
5651+ record.workflow_name.clone().unwrap_or_default(),
5652+ vec![(
5653+ record.step_id,
5654+ record.step_name.clone(),
5655+ record.exit_code.map(|code| code == 0),
5656+ )],
5657+ ),
5658+ )]);
5659+ mapped.insert(record.manifest_id, workflow);
5660+ }
5661+ };
5662+ mapped
5663+ });
5664+
5665+ let mut as_ordered_vec: Vec<(i64, Vec<ConciseWorkflow>)> =
5666+ invocations
5667+ .iter()
5668+ .fold(Vec::new(), |mut accm, (manifest_id, workflows)| {
5669+ let mut workflows: Vec<ConciseWorkflow> = workflows.iter().fold(
5670+ Vec::new(),
5671+ |mut wf_accm, (workflow_id, (workflow_name, steps))| {
5672+ wf_accm.push((*workflow_id, workflow_name.clone(), steps.clone()));
5673+ wf_accm
5674+ },
5675+ );
5676+ workflows.sort_by(|first, second| first.0.cmp(&second.0));
5677+ accm.push((*manifest_id, workflows));
5678+ accm
5679+ });
5680+
5681+ as_ordered_vec.sort_by(|first, second| first.0.cmp(&second.0));
5682+
5683+ let mut items: Vec<ManifestItem> = Vec::new();
5684+ for (manifest_id, workflows) in as_ordered_vec {
5685+ let manifest = sqlx::query_file_as!(
5686+ Manifest,
5687+ "queries/manifests_read.sql",
5688+ manifest_id,
5689+ None::<String>,
5690+ None::<String>
5691+ )
5692+ .fetch_one(&self.pool)
5693+ .await?;
5694+ let success = workflows
5695+ .iter()
5696+ .all(|(_, _, steps)| steps.iter().all(|(_, _, success)| success.is_some()))
5697+ .then(|| {
5698+ workflows.iter().all(|(_, _, steps)| {
5699+ steps
5700+ .iter()
5701+ .all(|(_, _, success)| success.is_some_and(|success| success))
5702+ })
5703+ });
5704+
5705+ items.push(ManifestItem {
5706+ manifest,
5707+ success,
5708+ workflows,
5709+ });
5710+ }
5711+ items.sort_by(|first, second| second.manifest.id.cmp(&first.manifest.id));
5712+ Ok(items)
5713+ }
5714+
5715+ async fn read_manifest(
5716+ &self,
5717+ collection: &str,
5718+ name: &str,
5719+ manifest_id: i64,
5720+ ) -> Result<ManifestView, Error> {
5721+ let manifest = sqlx::query_file_as!(
5722+ Manifest,
5723+ "queries/manifests_read.sql",
5724+ manifest_id,
5725+ collection,
5726+ name
5727+ )
5728+ .fetch_one(&self.pool)
5729+ .await?;
5730+ let dag = sqlx::query_file_as!(Dag, "queries/dags_read.sql", manifest_id)
5731+ .fetch_one(&self.pool)
5732+ .await?;
5733+ let mut normalized = ManifestView {
5734+ manifest,
5735+ workflows: Vec::default(),
5736+ dag,
5737+ };
5738+ let workflows = sqlx::query_file_as!(Workflow, "queries/workflows_list.sql", manifest_id)
5739+ .fetch_all(&self.pool)
5740+ .await?;
5741+ for workflow in workflows {
5742+ let steps = sqlx::query_file_as!(Step, "queries/steps_list.sql", workflow.id)
5743+ .fetch_all(&self.pool)
5744+ .await?;
5745+ normalized.workflows.push((workflow, steps));
5746+ }
5747+ Ok(normalized)
5748+ }
5749+
5750+ // workflows
5751+
5752+ async fn read_workflow(&self, workflow_id: i64) -> Result<Workflow, Error> {
5753+ let workflow = sqlx::query_file_as!(Workflow, "queries/workflows_read.sql", workflow_id)
5754+ .fetch_one(&self.pool)
5755+ .await?;
5756+ Ok(workflow)
5757+ }
5758+
5759+ async fn read_workflow_view(&self, workflow_id: i64) -> Result<WorkflowView, Error> {
5760+ let workflow = sqlx::query_file_as!(Workflow, "queries/workflows_read.sql", workflow_id)
5761+ .fetch_one(&self.pool)
5762+ .await?;
5763+ let steps = sqlx::query_file_as!(Step, "queries/steps_list.sql", workflow_id)
5764+ .fetch_all(&self.pool)
5765+ .await?;
5766+ Ok(WorkflowView { workflow, steps })
5767+ }
5768+
5769+ async fn create_workflow(&self, manifest_id: i64, name: &str) -> Result<i64, Error> {
5770+ let ret = sqlx::query_file!("queries/workflows_create.sql", manifest_id, name)
5771+ .fetch_one(&self.pool)
5772+ .await?;
5773+ Ok(ret.id)
5774+ }
5775+
5776+ async fn update_workflow_start(&self, workflow_id: i64) -> Result<(), Error> {
5777+ sqlx::query_file!("queries/workflows_update_start.sql", workflow_id,)
5778+ .execute(&self.pool)
5779+ .await?;
5780+ Ok(())
5781+ }
5782+
5783+ async fn update_workflow_finish(&self, workflow_id: i64) -> Result<(), Error> {
5784+ sqlx::query_file!("queries/workflows_update_finish.sql", workflow_id)
5785+ .execute(&self.pool)
5786+ .await?;
5787+ Ok(())
5788+ }
5789+
5790+ // steps
5791+
5792+ async fn create_step(
5793+ &self,
5794+ workflow_id: i64,
5795+ name: &str,
5796+ input: &str,
5797+ shell: &str,
5798+ environment: HashMap<String, String>,
5799+ ) -> Result<i64, Error> {
5800+ let environment_json = serde_json::ser::to_string(&environment).unwrap();
5801+ let ret = sqlx::query_file!(
5802+ "queries/steps_create.sql",
5803+ workflow_id,
5804+ name,
5805+ input,
5806+ shell,
5807+ environment_json
5808+ )
5809+ .fetch_one(&self.pool)
5810+ .await?;
5811+ Ok(ret.id)
5812+ }
5813+
5814+ async fn read_step(&self, step_id: i64) -> Result<Step, Error> {
5815+ sqlx::query_file_as!(Step, "queries/steps_read.sql", step_id)
5816+ .fetch_one(&self.pool)
5817+ .await
5818+ }
5819+
5820+ async fn read_step_env(&self, step_id: i64) -> Result<Environment, Error> {
5821+ let rows = sqlx::query_file!("queries/steps_read_env.sql", step_id)
5822+ .fetch_all(&self.pool)
5823+ .await?;
5824+
5825+ let envs: Vec<Env> = rows
5826+ .iter()
5827+ .map(|record| Env((record.name.clone(), record.value.clone())))
5828+ .collect();
5829+
5830+ Ok(Environment(envs))
5831+ }
5832+
5833+ async fn update_step_start(&self, step_id: i64) -> Result<(), Error> {
5834+ sqlx::query_file!("queries/steps_update_start.sql", step_id)
5835+ .execute(&self.pool)
5836+ .await?;
5837+ Ok(())
5838+ }
5839+
5840+ async fn update_step_finish(
5841+ &self,
5842+ step_id: i64,
5843+ stdout: &str,
5844+ stderr: &str,
5845+ exit_code: i8,
5846+ ) -> Result<(), Error> {
5847+ sqlx::query_file!(
5848+ "queries/steps_update_finish.sql",
5849+ stdout,
5850+ stderr,
5851+ exit_code,
5852+ step_id,
5853+ )
5854+ .execute(&self.pool)
5855+ .await?;
5856+ Ok(())
5857+ }
5858+
5859+ async fn create_sample(&self, manifest_id: i64, sample: Sample) -> Result<i64, Error> {
5860+ let ret = sqlx::query_file!(
5861+ "queries/samples_create.sql",
5862+ manifest_id,
5863+ sample.load_1m,
5864+ sample.load_5m,
5865+ sample.load_15m,
5866+ sample.disk_total_bytes,
5867+ sample.disk_free_bytes,
5868+ sample.net_sent_bytes,
5869+ sample.net_received_bytes,
5870+ sample.net_sent_packets,
5871+ sample.net_received_packets,
5872+ sample.mem_total_bytes,
5873+ sample.mem_free_bytes,
5874+ sample.mem_available_bytes,
5875+ )
5876+ .fetch_one(&self.pool)
5877+ .await?;
5878+ Ok(ret.id)
5879+ }
5880+
5881+ async fn read_dag(&self, manifest_id: i64) -> Result<Dag, Error> {
5882+ sqlx::query_file_as!(Dag, "queries/dags_read.sql", manifest_id)
5883+ .fetch_one(&self.pool)
5884+ .await
5885+ }
5886+ }
5887 diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs
5888index 0ceda9f..cee98bc 100644
5889--- a/crates/database/src/lib.rs
5890+++ b/crates/database/src/lib.rs
5891 @@ -1,17 +1,19 @@
5892 use std::fs::create_dir_all;
5893- use std::path::Path;
5894+ use std::path::{Path, PathBuf};
5895
5896 use sqlx::migrate::Migrator;
5897 use sqlx::sqlite::{SqliteConnectOptions, SqliteLockingMode, SqlitePool, SqlitePoolOptions};
5898 use sqlx::ConnectOptions;
5899- use sqlx::Error as SqlxError;
5900+ use sqlx::{Error as SqlxError, Sqlite, Transaction};
5901 use tracing::log;
5902
5903- pub mod contributors;
5904- pub mod jobs;
5905- pub mod languages;
5906- pub mod mail;
5907- pub mod stats;
5908+ // pub mod contributors;
5909+ // pub mod jobs;
5910+ // pub mod languages;
5911+ // pub mod mail;
5912+ // pub mod stats;
5913+
5914+ pub mod build;
5915
5916 pub type Error = SqlxError;
5917
5918 @@ -36,12 +38,12 @@ pub struct Builder {
5919 pub url: Option<String>,
5920 pub read_only: bool,
5921 pub log_queries: bool,
5922- pub migrations: Option<String>,
5923+ pub migrations: Option<PathBuf>,
5924 }
5925
5926 impl Builder {
5927- pub fn url(mut self, url: &str) -> Self {
5928- self.url = Some(url.to_string());
5929+ pub fn path(mut self, path: &Path) -> Self {
5930+ self.url = Some(path.to_str().unwrap().to_string());
5931 self
5932 }
5933
5934 @@ -55,8 +57,8 @@ impl Builder {
5935 self
5936 }
5937
5938- pub fn migrations(mut self, path: &str) -> Self {
5939- self.migrations = Some(path.to_string());
5940+ pub fn migrations(mut self, path: &Path) -> Self {
5941+ self.migrations = Some(path.to_path_buf());
5942 self
5943 }
5944
5945 @@ -109,12 +111,31 @@ impl Builder {
5946 }
5947 }
5948
5949+ pub struct Tx {
5950+ inner: Transaction<'static, Sqlite>,
5951+ }
5952+
5953+ impl Tx {
5954+ pub async fn commit(self) -> Result<(), Error> {
5955+ self.inner.commit().await
5956+ }
5957+
5958+ pub async fn rollback(self) -> Result<(), Error> {
5959+ self.inner.rollback().await
5960+ }
5961+ }
5962+
5963 #[derive(Clone, Debug)]
5964 pub struct Wrapper {
5965 pub pool: SqlitePool,
5966 }
5967
5968 impl Wrapper {
5969+ pub async fn begin(&self) -> Result<Tx, Error> {
5970+ let inner = self.pool.begin().await?;
5971+ Ok(Tx { inner })
5972+ }
5973+
5974 pub async fn close(&self) -> Result<(), Error> {
5975 self.pool.close().await;
5976 Ok(())
5977 diff --git a/migrations/20231204194038_init.sql b/migrations/20231204194038_init.sql
5978new file mode 100644
5979index 0000000..2882f7e
5980--- /dev/null
5981+++ b/migrations/20231204194038_init.sql
5982 @@ -0,0 +1,62 @@
5983+ CREATE TABLE manifests (
5984+ id INTEGER PRIMARY KEY NOT NULL,
5985+ collection TEXT NOT NULL,
5986+ name TEXT NOT NULL,
5987+ git_hash TEXT NOT NULL,
5988+ created_at INTEGER NOT NULL DEFAULT (UNIXEPOCH()),
5989+ started_at INTEGER CHECK (started_at > 0),
5990+ finished_at INTEGER CHECK (finished_at > 0)
5991+ ) STRICT ;
5992+
5993+ CREATE TABLE workflows (
5994+ id INTEGER PRIMARY KEY NOT NULL,
5995+ manifest_id INTEGER REFERENCES manifests(id) ON DELETE CASCADE NOT NULL,
5996+ name TEXT NOT NULL,
5997+ started_at INTEGER CHECK (started_at > 0),
5998+ finished_at INTEGER CHECK (finished_at > 0)
5999+ ) STRICT ;
6000+
6001+ CREATE TABLE steps (
6002+ id INTEGER PRIMARY KEY NOT NULL,
6003+ name TEXT NOT NULL,
6004+ workflow_id INTEGER REFERENCES workflows(id) ON DELETE CASCADE NOT NULL,
6005+ shell TEXT NOT NULL DEFAULT "/bin/sh",
6006+ environment_json TEXT,
6007+ input TEXT NOT NULL,
6008+ started_at INTEGER CHECK (started_at > 0),
6009+ finished_at INTEGER CHECK (finished_at > 0),
6010+ stdout TEXT,
6011+ stderr TEXT,
6012+ exit_code INTEGER CHECK (exit_code <= 255)
6013+ ) STRICT ;
6014+
6015+ CREATE TABLE steps_env_vars (
6016+ id INTEGER PRIMARY KEY NOT NULL,
6017+ step_id INTEGER REFERENCES steps(id) ON DELETE CASCADE NOT NULL,
6018+ name TEXT NOT NULL,
6019+ value TEXT
6020+ ) STRICT ;
6021+
6022+ CREATE TABLE dags (
6023+ id INTEGER PRIMARY KEY NOT NULL,
6024+ manifest_id INTEGER REFERENCES manifests(id) ON DELETE CASCADE NOT NULL,
6025+ dag_content TEXT NOT NULL
6026+ ) STRICT ;
6027+
6028+ CREATE TABLE samples (
6029+ id INTEGER PRIMARY KEY NOT NULL,
6030+ manifest_id INTEGER REFERENCES manifests(id) ON DELETE CASCADE NOT NULL,
6031+ timestamp INTEGER NOT NULL DEFAULT (UNIXEPOCH()),
6032+ load_1m INTEGER,
6033+ load_5m INTEGER,
6034+ load_15m INTEGER,
6035+ disk_total_bytes INTEGER,
6036+ disk_free_bytes INTEGER,
6037+ net_sent_bytes INTEGER,
6038+ net_received_bytes INTEGER,
6039+ net_sent_packets INTEGER,
6040+ net_received_packets INTEGER,
6041+ mem_total_bytes INTEGER,
6042+ mem_free_bytes INTEGER,
6043+ mem_available_bytes INTEGER
6044+ ) STRICT ;
6045 diff --git a/packaging/archlinux/ayllu-git/PKGBUILD b/packaging/archlinux/ayllu-git/PKGBUILD
6046index f2756ea..6e1751f 100644
6047--- a/packaging/archlinux/ayllu-git/PKGBUILD
6048+++ b/packaging/archlinux/ayllu-git/PKGBUILD
6049 @@ -20,7 +20,7 @@ makedepends=(
6050 provides=("ayllu-git")
6051 optdepends=()
6052 source=(
6053- "$_pkgname::git+https://ayllu-forge.org/ayllu/ayllu#branch=main"
6054+ "$_pkgname::git+https://ayllu-forge.org/ayllu/ayllu#branch=ayllu-build-round-2"
6055 )
6056 # See: https://gitlab.archlinux.org/archlinux/packaging/packages/pacman/-/issues/20
6057 options=(!lto)
6058 @@ -40,6 +40,9 @@ build() {
6059 }
6060
6061 package() {
6062+ mkdir -p "${pkgdir}/usr/lib/ayllu/migrations"
6063+ find "${srcdir}/ayllu/migrations" -name "*.sql" \
6064+ | xargs install -Dm644 -t "${pkgdir}/usr/lib/ayllu/migrations"
6065 install -Dm755 \
6066 "${srcdir}/target/release/ayllu" "${pkgdir}/usr/bin/ayllu"
6067 install -Dm755 \
6068 @@ -48,6 +51,10 @@ package() {
6069 "${srcdir}/target/release/ayllu-shell" "${pkgdir}/usr/bin/ayllu-shell"
6070 install -Dm755 \
6071 "${srcdir}/target/release/ayllu-keys" "${pkgdir}/usr/bin/ayllu-keys"
6072+ install -Dm755 \
6073+ "${srcdir}/target/release/ayllu-migrate" "${pkgdir}/usr/bin/ayllu-migrate"
6074+ install -Dm755 \
6075+ "${srcdir}/target/release/ayllu-build" "${pkgdir}/usr/bin/ayllu-build"
6076 install -Dm644 \
6077 "${srcdir}/ayllu/LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}"
6078 install -Dm644 \
6079 diff --git a/packaging/debian/postinst b/packaging/debian/postinst
6080index 7a769f3..6a48aa1 100755
6081--- a/packaging/debian/postinst
6082+++ b/packaging/debian/postinst
6083 @@ -3,6 +3,7 @@
6084 AYLLU_SHELL_PATH="/usr/bin/ayllu-shell"
6085 AYLLU_HOME_PATH="/var/lib/ayllu"
6086 AYLLU_SHARE_PATH="/usr/share/ayllu"
6087+ AYLLU_LIB_PATH="/usr/lib/ayllu"
6088
6089 if [ ! "$(id ayllu)" ] ; then
6090 adduser --system --group --shell="$AYLLU_SHELL_PATH" --home="$AYLLU_HOME_PATH" ayllu
6091 @@ -12,3 +13,7 @@ if [ -d "$AYLLU_SHARE_PATH" ] ; then
6092 mkdir -p "$AYLLU_SHARE_PATH"
6093 chown ayllu:ayllu "$AYLLU_SHARE_PATH"
6094 fi
6095+
6096+ if [ -d "$ALLYU_LIB_PATH" ] ; then
6097+ mkdir -p "$AYLLU_LIB_PATH"
6098+ fi
6099 diff --git a/queries/authors_list_project.sql b/queries/authors_list_project.sql
6100deleted file mode 100644
6101index 832e3de..0000000
6102--- a/queries/authors_list_project.sql
6103+++ /dev/null
6104 @@ -1,30 +0,0 @@
6105- WITH RECURSIVE max_git_id AS
6106- (SELECT contributions.id AS hash
6107- FROM contributions
6108- WHERE git_hash = ?
6109- AND repo_path = ?
6110- ORDER BY id DESC
6111- LIMIT 1),
6112- total_contributions AS
6113- (SELECT COUNT(*) AS contributions
6114- FROM contributions
6115- WHERE repo_path = ?
6116- AND id <=
6117- (SELECT hash
6118- FROM max_git_id) )
6119- SELECT authors.username,
6120- authors.email,
6121- COUNT(contributions.id) AS "count: i64",
6122- SUM(contributions.lines_added) AS "lines_added: i64",
6123- SUM(contributions.lines_removed) AS "lines_removed: i64",
6124- ROUND((COUNT(contributions.id)/CAST(
6125- (SELECT contributions
6126- FROM total_contributions) AS REAL))*100, 2) AS "percentage: f64"
6127- FROM contributions
6128- LEFT JOIN authors ON (contributions.author_id = authors.id)
6129- WHERE repo_path = ?
6130- AND contributions.id <=
6131- (SELECT hash
6132- FROM max_git_id)
6133- GROUP BY authors.email
6134- ORDER BY "count: i64" DESC
6135 diff --git a/queries/authors_upsert.sql b/queries/authors_upsert.sql
6136deleted file mode 100644
6137index 1c55636..0000000
6138--- a/queries/authors_upsert.sql
6139+++ /dev/null
6140 @@ -1,10 +0,0 @@
6141- INSERT INTO authors
6142- (username, email)
6143- VALUES
6144- (?, ?)
6145- ON CONFLICT
6146- DO UPDATE
6147- SET
6148- username = username,
6149- email = email
6150- RETURNING id
6151 diff --git a/queries/commits_count.sql b/queries/commits_count.sql
6152deleted file mode 100644
6153index 0088bb8..0000000
6154--- a/queries/commits_count.sql
6155+++ /dev/null
6156 @@ -1,9 +0,0 @@
6157- SELECT
6158- COUNT(id) AS count
6159- FROM contributions
6160- WHERE
6161- repo_path = ? AND
6162- id <= (
6163- SELECT id FROM contributions
6164- WHERE git_hash = ? AND repo_path = ?
6165- )
6166 diff --git a/queries/contribution_add.sql b/queries/contribution_add.sql
6167deleted file mode 100644
6168index e1085e3..0000000
6169--- a/queries/contribution_add.sql
6170+++ /dev/null
6171 @@ -1,11 +0,0 @@
6172- WITH previous(total) AS (
6173- SELECT total FROM contribution_tally
6174- WHERE
6175- author_id = ? AND repo_path = ?
6176- ORDER BY id DESC
6177- LIMIT 1
6178- )
6179- INSERT INTO contribution_tally
6180- (git_hash, repo_path, author_id, total)
6181- VALUES
6182- (?, ?, ?, COALESCE((SELECT(total) FROM previous), 0)+1)
6183 diff --git a/queries/contribution_delete.sql b/queries/contribution_delete.sql
6184deleted file mode 100644
6185index 6fdbe5c..0000000
6186--- a/queries/contribution_delete.sql
6187+++ /dev/null
6188 @@ -1,3 +0,0 @@
6189- DELETE FROM contribution_tally
6190- WHERE
6191- repo_path = ?
6192 diff --git a/queries/contributions_add.sql b/queries/contributions_add.sql
6193deleted file mode 100644
6194index 9883d94..0000000
6195--- a/queries/contributions_add.sql
6196+++ /dev/null
6197 @@ -1,4 +0,0 @@
6198- INSERT INTO contributions
6199- (author_id, git_hash, repo_path, time, lines_added, lines_removed)
6200- VALUES
6201- (?, ?, ?, ?, ?, ?)
6202 diff --git a/queries/contributions_bucket.sql b/queries/contributions_bucket.sql
6203deleted file mode 100644
6204index c4e2e55..0000000
6205--- a/queries/contributions_bucket.sql
6206+++ /dev/null
6207 @@ -1,10 +0,0 @@
6208- SELECT
6209- COUNT(id) AS "count!: i64",
6210- time,
6211- SUM(lines_added) AS "added!: i64",
6212- SUM(lines_removed) AS "removed!: i64"
6213- FROM contributions WHERE
6214- repo_path = ? AND
6215- id <= (SELECT id FROM contributions WHERE git_hash = ?) AND
6216- time <= ? AND time >= ?
6217- GROUP BY strftime(?, time)
6218 diff --git a/queries/contributions_delete.sql b/queries/contributions_delete.sql
6219deleted file mode 100644
6220index 0ee5563..0000000
6221--- a/queries/contributions_delete.sql
6222+++ /dev/null
6223 @@ -1,2 +0,0 @@
6224- DELETE FROM contributions
6225- WHERE repo_path = ?
6226 diff --git a/queries/contributions_list.sql b/queries/contributions_list.sql
6227deleted file mode 100644
6228index fae5633..0000000
6229--- a/queries/contributions_list.sql
6230+++ /dev/null
6231 @@ -1,42 +0,0 @@
6232- WITH RECURSIVE
6233- tallys AS (
6234- SELECT
6235- authors.username AS name,
6236- authors.email AS email,
6237- MAX(contribution_tally.total) AS total
6238- FROM contribution_tally
6239- LEFT JOIN authors ON
6240- (authors.id = contribution_tally.author_id)
6241- WHERE
6242- contribution_tally.id < (
6243- SELECT id FROM contributions
6244- WHERE
6245- repo_path = ? AND git_hash = ?
6246- ORDER BY id DESC
6247- LIMIT 1
6248- ) AND
6249- contribution_tally.repo_path = ?
6250- GROUP BY authors.email
6251- ORDER BY contribution_tally.total DESC
6252- LIMIT 5),
6253- raw_commits AS (
6254- SELECT
6255- COUNT(*) as total
6256- FROM contributions
6257- WHERE
6258- contributions.id < (
6259- SELECT id FROM contributions
6260- WHERE
6261- repo_path = ? AND git_hash = ?
6262- ORDER BY id DESC
6263- LIMIT 1
6264- ) AND
6265- contributions.repo_path = ?
6266- )
6267- SELECT
6268- name,
6269- email,
6270- (
6271- CAST(tallys.total AS REAL)/(SELECT total FROM raw_commits)
6272- ) * 100 AS "percentage: i64"
6273- FROM tallys
6274 diff --git a/queries/job_tracking_add.sql b/queries/job_tracking_add.sql
6275deleted file mode 100644
6276index 7e70756..0000000
6277--- a/queries/job_tracking_add.sql
6278+++ /dev/null
6279 @@ -1,4 +0,0 @@
6280- INSERT INTO job_tracking
6281- (repo_path, git_hash, kind, job_id)
6282- VALUES
6283- (?, ?, ?, ?)
6284 diff --git a/queries/job_tracking_delete.sql b/queries/job_tracking_delete.sql
6285deleted file mode 100644
6286index 3aef33d..0000000
6287--- a/queries/job_tracking_delete.sql
6288+++ /dev/null
6289 @@ -1,3 +0,0 @@
6290- DELETE FROM job_tracking
6291- WHERE
6292- repo_path = ?
6293 diff --git a/queries/job_tracking_delete_by_repo.sql b/queries/job_tracking_delete_by_repo.sql
6294deleted file mode 100644
6295index ebd0363..0000000
6296--- a/queries/job_tracking_delete_by_repo.sql
6297+++ /dev/null
6298 @@ -1,3 +0,0 @@
6299- DELETE FROM jobs
6300- WHERE
6301- repo_path = ?
6302 diff --git a/queries/job_tracking_read.sql b/queries/job_tracking_read.sql
6303deleted file mode 100644
6304index 823dfc5..0000000
6305--- a/queries/job_tracking_read.sql
6306+++ /dev/null
6307 @@ -1,8 +0,0 @@
6308- SELECT
6309- git_hash AS "git_hash!"
6310- FROM job_tracking
6311- WHERE
6312- repo_path = ? AND
6313- kind = ?
6314- ORDER BY id DESC
6315- LIMIT 1
6316 diff --git a/queries/jobs_create.sql b/queries/jobs_create.sql
6317deleted file mode 100644
6318index 4a1bea5..0000000
6319--- a/queries/jobs_create.sql
6320+++ /dev/null
6321 @@ -1,5 +0,0 @@
6322- INSERT INTO jobs
6323- (created_at, repo_path, kind)
6324- VALUES
6325- (?, ?, ?)
6326- RETURNING id
6327 diff --git a/queries/jobs_delete.sql b/queries/jobs_delete.sql
6328deleted file mode 100644
6329index c70ba25..0000000
6330--- a/queries/jobs_delete.sql
6331+++ /dev/null
6332 @@ -1,2 +0,0 @@
6333- DELETE FROM jobs
6334- WHERE jobs.id = ?
6335 diff --git a/queries/jobs_list.sql b/queries/jobs_list.sql
6336deleted file mode 100644
6337index b1e7f72..0000000
6338--- a/queries/jobs_list.sql
6339+++ /dev/null
6340 @@ -1,15 +0,0 @@
6341- SELECT
6342- jobs.id AS "id!",
6343- created_at AS "created_at: OffsetDateTime",
6344- jobs.repo_path AS "repo_path!: String",
6345- jobs.kind AS "kind!: String",
6346- runtime AS "runtime: u32",
6347- success AS "success: bool",
6348- (
6349- SELECT COUNT(id) FROM job_tracking
6350- WHERE job_tracking.job_id = jobs.id
6351- ) AS "commits!: i32"
6352- FROM jobs
6353- WHERE
6354- jobs.repo_path = ? OR ? IS NULL
6355- ORDER BY DATETIME(created_at) ASC
6356 diff --git a/queries/jobs_update.sql b/queries/jobs_update.sql
6357deleted file mode 100644
6358index ea7e521..0000000
6359--- a/queries/jobs_update.sql
6360+++ /dev/null
6361 @@ -1,6 +0,0 @@
6362- UPDATE jobs
6363- SET
6364- runtime = ?,
6365- success = ?
6366- WHERE
6367- jobs.id = ?
6368 diff --git a/queries/languages_add.sql b/queries/languages_add.sql
6369deleted file mode 100644
6370index 54551b3..0000000
6371--- a/queries/languages_add.sql
6372+++ /dev/null
6373 @@ -1,4 +0,0 @@
6374- INSERT INTO languages
6375- (git_hash, repo_path, language, loc)
6376- VALUES
6377- (?,?,?,?)
6378 diff --git a/queries/languages_delete.sql b/queries/languages_delete.sql
6379deleted file mode 100644
6380index 06c4349..0000000
6381--- a/queries/languages_delete.sql
6382+++ /dev/null
6383 @@ -1,3 +0,0 @@
6384- DELETE FROM languages
6385- WHERE
6386- repo_path = ?
6387 diff --git a/queries/languages_list.sql b/queries/languages_list.sql
6388deleted file mode 100644
6389index 608a719..0000000
6390--- a/queries/languages_list.sql
6391+++ /dev/null
6392 @@ -1,13 +0,0 @@
6393- SELECT
6394- language,
6395- loc,
6396- ROUND(
6397- CAST(loc AS REAL)/CAST(
6398- (SELECT SUM(loc) FROM languages
6399- WHERE repo_path = ? AND git_hash = ?) AS REAL)*100)
6400- AS "percentage: f64"
6401- FROM languages
6402- WHERE
6403- repo_path = ? AND git_hash = ?
6404- ORDER BY "percentage: f64" DESC
6405- LIMIT 5
6406 diff --git a/queries/latest_commit.sql b/queries/latest_commit.sql
6407deleted file mode 100644
6408index bd26eac..0000000
6409--- a/queries/latest_commit.sql
6410+++ /dev/null
6411 @@ -1,7 +0,0 @@
6412- SELECT
6413- git_hash
6414- FROM contributions
6415- WHERE
6416- repo_path = ?
6417- ORDER BY id DESC
6418- LIMIT 1
6419 diff --git a/queries/mail_deactivate_unused.sql b/queries/mail_deactivate_unused.sql
6420deleted file mode 100644
6421index 730dc92..0000000
6422--- a/queries/mail_deactivate_unused.sql
6423+++ /dev/null
6424 @@ -1 +0,0 @@
6425- UPDATE lists SET enabled = 0 WHERE address NOT IN (?)
6426 diff --git a/queries/mail_deliver_message.sql b/queries/mail_deliver_message.sql
6427deleted file mode 100644
6428index 6763a82..0000000
6429--- a/queries/mail_deliver_message.sql
6430+++ /dev/null
6431 @@ -1,10 +0,0 @@
6432- INSERT INTO messages
6433- (message_id, list_id, in_reply_to, subject, mail_from, content_body, raw_message)
6434- VALUES (
6435- ?,
6436- ( SELECT id FROM lists WHERE address = ? LIMIT 1 ),
6437- ?,
6438- ?,
6439- ?,
6440- ?, ?
6441- ) RETURNING messages.id
6442 diff --git a/queries/mail_has_message.sql b/queries/mail_has_message.sql
6443deleted file mode 100644
6444index e32ce95..0000000
6445--- a/queries/mail_has_message.sql
6446+++ /dev/null
6447 @@ -1,5 +0,0 @@
6448- SELECT
6449- CASE
6450- WHEN (SELECT COUNT(*) FROM messages WHERE message_id = ?) >= 1 THEN TRUE
6451- ELSE FALSE
6452- END AS "has_message!: bool"
6453 diff --git a/queries/mail_list_create.sql b/queries/mail_list_create.sql
6454deleted file mode 100644
6455index 770b76a..0000000
6456--- a/queries/mail_list_create.sql
6457+++ /dev/null
6458 @@ -1,9 +0,0 @@
6459- INSERT INTO lists
6460- (name, address, description, enabled)
6461- VALUES
6462- (?, ?, ?, ?)
6463- ON CONFLICT (address) DO
6464- UPDATE SET
6465- name = ?,
6466- description = ?,
6467- enabled = ?
6468 diff --git a/queries/mail_list_outbox.sql b/queries/mail_list_outbox.sql
6469deleted file mode 100644
6470index 8eaa79e..0000000
6471--- a/queries/mail_list_outbox.sql
6472+++ /dev/null
6473 @@ -1,13 +0,0 @@
6474- SELECT
6475- outbox.id,
6476- messages.message_id AS "message_id!: String",
6477- lists.name AS "list_name!: String",
6478- participants.address AS "address!: String",
6479- messages.raw_message AS raw_message
6480- FROM outbox
6481- LEFT JOIN participants ON outbox.recipient = participants.address
6482- LEFT JOIN messages ON messages.id = outbox.message_id
6483- LEFT JOIN lists ON messages.list_id = lists.id
6484- WHERE
6485- (SELECT COUNT(*) FROM delivery_failures
6486- WHERE delivery_failures.outbox_id = outbox.id) < ?
6487 diff --git a/queries/mail_outbox_failed.sql b/queries/mail_outbox_failed.sql
6488deleted file mode 100644
6489index c7dafcb..0000000
6490--- a/queries/mail_outbox_failed.sql
6491+++ /dev/null
6492 @@ -1,3 +0,0 @@
6493- INSERT INTO delivery_failures
6494- (outbox_id, error_kind, message)
6495- VALUES (?, ?, ?)
6496 diff --git a/queries/mail_outbox_successful.sql b/queries/mail_outbox_successful.sql
6497deleted file mode 100644
6498index 0b0269c..0000000
6499--- a/queries/mail_outbox_successful.sql
6500+++ /dev/null
6501 @@ -1 +0,0 @@
6502- DELETE FROM outbox WHERE id = ?
6503 diff --git a/queries/mail_read_message.sql b/queries/mail_read_message.sql
6504deleted file mode 100644
6505index 1696cae..0000000
6506--- a/queries/mail_read_message.sql
6507+++ /dev/null
6508 @@ -1 +0,0 @@
6509- SELECT * FROM messages WHERE id = ?
6510 diff --git a/queries/mail_read_thread.sql b/queries/mail_read_thread.sql
6511deleted file mode 100644
6512index 09342d0..0000000
6513--- a/queries/mail_read_thread.sql
6514+++ /dev/null
6515 @@ -1,18 +0,0 @@
6516- WITH RECURSIVE tree AS (
6517- SELECT messages.*, 0 AS depth FROM messages
6518- WHERE messages.message_id = ?
6519- UNION ALL
6520- SELECT other.*, depth + 1 FROM messages other
6521- JOIN tree ON other.in_reply_to = tree.message_id
6522- )
6523- SELECT
6524- message_id,
6525- in_reply_to,
6526- subject,
6527- (SELECT address FROM participants WHERE participants.id = tree.mail_from LIMIT 1)
6528- mail_from,
6529- raw_message,
6530- content_body,
6531- reply_count,
6532- depth AS "depth!: i64"
6533- FROM tree ORDER BY depth
6534 diff --git a/queries/mail_read_threads.sql b/queries/mail_read_threads.sql
6535deleted file mode 100644
6536index d2e3b4a..0000000
6537--- a/queries/mail_read_threads.sql
6538+++ /dev/null
6539 @@ -1,9 +0,0 @@
6540- SELECT
6541- messages.message_id,
6542- messages.subject,
6543- messages.reply_count,
6544- participants.address AS "mail_from!: String"
6545- FROM messages
6546- LEFT JOIN lists ON lists.id = messages.id
6547- LEFT JOIN participants ON participants.id = messages.mail_from
6548- WHERE lists.name = ? AND messages.in_reply_to IS NULL
6549 diff --git a/queries/mail_thread_count.sql b/queries/mail_thread_count.sql
6550deleted file mode 100644
6551index 5dc619a..0000000
6552--- a/queries/mail_thread_count.sql
6553+++ /dev/null
6554 @@ -1,4 +0,0 @@
6555- SELECT COUNT(*) AS "count: i64" FROM messages
6556- WHERE
6557- list_id = (SELECT list_id FROM lists WHERE list_id = ?) AND
6558- in_reply_to IS NULL
6559 diff --git a/queries/mail_thread_summary.sql b/queries/mail_thread_summary.sql
6560deleted file mode 100644
6561index c76aa55..0000000
6562--- a/queries/mail_thread_summary.sql
6563+++ /dev/null
6564 @@ -1,3 +0,0 @@
6565- SELECT messages.* FROM messages
6566- LEFT JOIN lists ON lists.id = messages.id
6567- WHERE lists.name = ? AND messages.reply_to IS NULL
6568 diff --git a/queries/mail_upsert_participant.sql b/queries/mail_upsert_participant.sql
6569deleted file mode 100644
6570index 67b0973..0000000
6571--- a/queries/mail_upsert_participant.sql
6572+++ /dev/null
6573 @@ -1,6 +0,0 @@
6574- INSERT INTO participants
6575- (address, authorized_sender)
6576- VALUES (?, ?)
6577- ON CONFLICT DO NOTHING;
6578-
6579- SELECT id FROM participants WHERE address = ?
6580 diff --git a/queries/mail_upsert_subscription.sql b/queries/mail_upsert_subscription.sql
6581deleted file mode 100644
6582index 78cd886..0000000
6583--- a/queries/mail_upsert_subscription.sql
6584+++ /dev/null
6585 @@ -1,6 +0,0 @@
6586- INSERT INTO subscriptions
6587- (participant_id, list_id)
6588- VALUES (
6589- (SELECT id FROM participants WHERE address = ? LIMIT 1),
6590- (SELECT id FROM lists WHERE address = ? LIMIT 1)
6591- ) ON CONFLICT DO NOTHING;
6592 diff --git a/scripts/build_deb.sh b/scripts/build_deb.sh
6593index 26709db..8ecdf77 100755
6594--- a/scripts/build_deb.sh
6595+++ b/scripts/build_deb.sh
6596 @@ -12,10 +12,13 @@ cp -rvp packaging/debian/ "${BUILD_DIR}/DEBIAN/"
6597
6598 sed -i "s/Version: X.X.X/Version: ${AYLLU_VERSION}/g" "${BUILD_DIR}/DEBIAN/control"
6599
6600+ find migrations -name "*.sql" | xargs install -Dm644 -t "${BUILD_DIR}/usr/lib/ayllu/migrations"
6601 install -Dm755 "target/release/ayllu" "${BUILD_DIR}/usr/bin/ayllu"
6602 install -Dm755 "target/release/quipu" "${BUILD_DIR}/usr/bin/quipu"
6603 install -Dm755 "target/release/ayllu-shell" "${BUILD_DIR}/usr/bin/ayllu-shell"
6604 install -Dm755 "target/release/ayllu-keys" "${BUILD_DIR}/usr/bin/ayllu-keys"
6605+ install -Dm755 "target/release/ayllu-migrate" "${BUILD_DIR}/usr/bin/ayllu-migrate"
6606+ install -Dm755 "target/release/ayllu-build" "${BUILD_DIR}/usr/bin/ayllu-build"
6607 install -Dm644 "LICENSE" "${BUILD_DIR}/usr/share/licenses/ayllu"
6608 install -Dm644 "config.example.toml" "${BUILD_DIR}/etc/ayllu/config.example.toml"
6609 install -Dm644 \
6610 diff --git a/scripts/ensure_database.sh b/scripts/ensure_database.sh
6611index f954193..722ad25 100755
6612--- a/scripts/ensure_database.sh
6613+++ b/scripts/ensure_database.sh
6614 @@ -2,12 +2,11 @@
6615 set -ex
6616 # initialize a local database for builds and testing assuming the package
6617 # has database support.
6618-
6619+
6620 DB_PATH="db/ayllu.db"
6621 DB_URL="sqlite://${DB_PATH}"
6622
6623 mkdir -p "$(dirname "${DB_PATH}")"
6624-
6625 echo "$DB_PATH"
6626
6627 if [ ! -e "${DB_PATH}" ]; then