Commit: 7bb4264bd829b82de5f8f3b1a44ff7d958eb316c

Commit

Author:

Hash:

Timestamp:

+306 -377 +/-11 browse

Kevin Schoon [me@kevinschoon.com]

7bb4264bd829b82de5f8f3b1a44ff7d958eb316c

Thu, 14 May 2026 15:35:29 +0000 (3 weeks ago)

refactor ayllu-build to improve source loading

refactor ayllu-build to improve source loading

This also adds a new optional PreProcessor configuration to the builder
runtime which allows pipeing some source file through an external
command. After evaluating several configuration/scripting/dsl systems
this approach feels the most flexible and avoids having to tie Ayllu
to one particular config format.

Full fledged programming languages like Nickel feel wrong because
builds really don't need a static type system for their configuraiton
as serde handles this already. Embeddable languages like Lua or similar
are TOO flexible and powerful while everything but YAML lacks good
support for reducing duplication.

1	diff --git a/ayllu-build/src/config.rs b/ayllu-build/src/config.rs
2	index 0883c1a..fb4ab3a 100644
3	--- a/ayllu-build/src/config.rs
4	+++ b/ayllu-build/src/config.rs
5	@@ -5,6 +5,24 @@ use serde::{Deserialize, Serialize};
6
7	use ayllu_config::{Configurable, Database, Error, Reader, data_dir};
8
9	+ /// Optional pre-processor command which will pipe a source file through some
10	+ /// external program. The external program must return a valid manifest JSON
11	+ /// object.
12	+ ///
13	+ /// NOTE: This is designed with jsonnet in-mind which I settled on using after
14	+ /// evaluating many other configuration languages, DSLs, and embeddable
15	+ /// scripting engines.
16	+ ///
17	+ /// TODO: Probably we will settle on embedding a jsonnet runtime like rsjsonnet
18	+ /// or jrsonnet but I have not evaluated those yet.
19	+ #[derive(Deserialize, Serialize, Clone, Debug)]
20	+ pub struct Preprocessor {
21	+ pub source: PathBuf,
22	+ pub program: String,
23	+ #[serde(default = "Vec::new")]
24	+ pub args: Vec<String>,
25	+ }
26	+
27	#[derive(Deserialize, Serialize, Clone, Debug)]
28	pub struct Builder {
29	#[serde(default = "Builder::work_dir_default")]
30	@@ -13,6 +31,7 @@ pub struct Builder {
31	pub init_binary: PathBuf,
32	#[serde(default = "Builder::podman_socket_default")]
33	pub podman_socket: PathBuf,
34	+ pub pre_processor: Option<Preprocessor>,
35	}
36
37	impl Builder {
38	@@ -35,6 +54,7 @@ impl Default for Builder {
39	work_dir: Builder::work_dir_default(),
40	init_binary: Builder::init_binary_default(),
41	podman_socket: Builder::podman_socket_default(),
42	+ pre_processor: Default::default(),
43	}
44	}
45	}
46	diff --git a/ayllu-build/src/error.rs b/ayllu-build/src/error.rs
47	index 873ef1e..128558b 100644
48	--- a/ayllu-build/src/error.rs
49	+++ b/ayllu-build/src/error.rs
50	@@ -1,89 +1,40 @@
51	use std::{path::PathBuf, time::Duration};
52
53	- #[derive(Debug)]
54	+ #[derive(thiserror::Error, Debug)]
55	pub enum Error {
56	- CannotReadManifest {
57	- path: PathBuf,
58	- io_err: std::io::Error,
59	- },
60	- CannotReadManifestFromRepository {
61	- path: PathBuf,
62	- repo_err: ayllu_git::Error,
63	- },
64	- RepositoryDoesNotContainManifest {
65	- path: PathBuf,
66	- },
67	- InvalidManifest {
68	- path: PathBuf,
69	- json_err: serde_json::Error,
70	- },
71	- EmptyWorkflow {
72	- name: String,
73	+ #[error("Problem reading manifest from repository: {0}")]
74	+ CannotReadManifest(ayllu_git::Error),
75	+ #[error("Repository does not contain a manifest: {path}")]
76	+ RepositoryDoesNotContainManifest { path: PathBuf },
77	+ #[error("Cannot open repository: {0}")]
78	+ CannotOpenRepository(ayllu_git::Error),
79	+ #[error("Cannot find managed repository: {collection} {name}")]
80	+ CannotFindManagedRepository { collection: String, name: String },
81	+ #[error("Preprocessor command: {program} {args:?} failed: {io_error}")]
82	+ PreProcessorFailed {
83	+ program: String,
84	+ args: Vec<String>,
85	+ io_error: std::io::Error,
86	},
87	+ #[error("Cannot read manifest: {0}")]
88	+ InvalidManifest(#[from] serde_json::Error),
89	+ #[error("Workflow {name} contains no steps")]
90	+ EmptyWorkflow { name: String },
91	+ #[error("Manifest contains a cycle")]
92	CycleDetected,
93	- DuplicateStepNames {
94	- name: String,
95	- },
96	+ #[error("Manifest contains duplicate steps: {name}")]
97	+ DuplicateStepNames { name: String },
98	+ #[error("Mainfest contains duplicate workflows")]
99	DuplicateWorkflows,
100	- Database(ayllu_database::Error),
101	+ #[error("Problem connecting to SQLite Database: {0}")]
102	+ Database(#[from] ayllu_database::Error),
103	// general io error during execution
104	- Io(std::io::Error),
105	- Reqwest(reqwest::Error),
106	+ #[error("Some other IO error: {0}")]
107	+ Io(#[from] std::io::Error),
108	+ #[error("Libpod network error: {0}")]
109	+ Reqwest(#[from] reqwest::Error),
110	+ #[error("Libpod API error: {0:?}")]
111	Libpod(crate::libpod::ErrorMessage),
112	+ #[error("Container timed out after {0:?}")]
113	ContainerTimeout(Duration),
114	}
115	-
116	- impl std::fmt::Display for Error {
117	- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
118	- match self {
119	- Error::CannotReadManifest { path, io_err } => {
120	- write!(f, "Cannot read manifest: {path:?}, {io_err}")
121	- }
122	- Error::InvalidManifest { path, json_err } => {
123	- write!(f, "Manifest is invalid: {path:?}, {json_err}")
124	- }
125	- Error::EmptyWorkflow { name } => write!(f, "Workflow {name} is empty"),
126	- Error::CycleDetected => write!(f, "Cycle Detected!"),
127	- Error::DuplicateStepNames { name } => write!(f, "Duplicate step detected: {name}"),
128	- Error::DuplicateWorkflows => write!(f, "Duplicate Workflows!"),
129	- Error::Io(error) => write!(f, "Io Error: {error}"),
130	- Error::CannotReadManifestFromRepository { path, repo_err } => write!(
131	- f,
132	- "Cannot read manifest from repository {path:?}: {repo_err}"
133	- ),
134	- Error::RepositoryDoesNotContainManifest { path } => {
135	- write!(f, "Repository {path:?} does not contain a manifest")
136	- }
137	- Error::Database(error) => write!(f, "Database error: {error}"),
138	- Error::Reqwest(error) => write!(f, "Error making libpod request: {error:?}"),
139	- Error::Libpod(error_message) => write!(
140	- f,
141	- "Libpod API Error: {}: {} {}",
142	- error_message.cause, error_message.message, error_message.response
143	- ),
144	- Error::ContainerTimeout(duration) => {
145	- write!(f, "Container took too long to start up: {duration:?}")
146	- }
147	- }
148	- }
149	- }
150	-
151	- impl std::error::Error for Error {}
152	-
153	- impl From<ayllu_database::Error> for Error {
154	- fn from(value: ayllu_database::Error) -> Self {
155	- Self::Database(value)
156	- }
157	- }
158	-
159	- impl From<std::io::Error> for Error {
160	- fn from(value: std::io::Error) -> Self {
161	- Error::Io(value)
162	- }
163	- }
164	-
165	- impl From<reqwest::Error> for Error {
166	- fn from(value: reqwest::Error) -> Self {
167	- Self::Reqwest(value)
168	- }
169	- }
170	diff --git a/ayllu-build/src/evaluate.rs b/ayllu-build/src/evaluate.rs
171	index e4f7759..71893f1 100644
172	--- a/ayllu-build/src/evaluate.rs
173	+++ b/ayllu-build/src/evaluate.rs
174	@@ -1,6 +1,5 @@
175	- use std::fs::metadata;
176	+ use std::collections::HashMap;
177	use std::path::PathBuf;
178	- use std::{collections::HashMap, path::Path};
179
180	use ayllu_database::build::State;
181	use ayllu_database::build::workflows::CreateWorkflowArgs;
182	@@ -11,12 +10,10 @@ use petgraph::{
183	visit::Topo,
184	};
185
186	+ use crate::config::Config;
187	use crate::libpod_executor::InitializeArgs;
188	- use crate::package::Package;
189	- use crate::{
190	- DEFAULT_BUILD_FILE,
191	- models::{Manifest, Step},
192	- };
193	+ use crate::manifest::Step;
194	+ use crate::source::Source;
195	use crate::{error::Error, libpod_executor::Context};
196
197	use ayllu_api::build::Unit;
198	@@ -24,34 +21,6 @@ use ayllu_database::Wrapper as Database;
199
200	pub type BuildGraph = Graph<Unit, u8>;
201
202	- #[derive(Clone, Debug)]
203	- #[allow(dead_code)]
204	- pub enum Source {
205	- // already loaded manifest for testing
206	- Manifest(Manifest),
207	- // load a manifest from a file or a directory, typically useful for testing
208	- Path(PathBuf),
209	- // clone a repository and run the manifest from that is contained within
210	- Clone((String, String)),
211	- }
212	-
213	- impl Source {
214	- pub fn url(&self) -> String {
215	- match self {
216	- Source::Manifest(_) => todo!(),
217	- Source::Path(path_buf) => {
218	- if path_buf.ends_with(DEFAULT_BUILD_FILE) {
219	- let path = path_buf.parent().unwrap();
220	- path.to_string_lossy().to_string()
221	- } else {
222	- path_buf.to_string_lossy().to_string()
223	- }
224	- }
225	- Source::Clone(_) => todo!(),
226	- }
227	- }
228	- }
229	-
230	#[allow(dead_code)]
231	pub(crate) struct Runtime {
232	// parallelism: bool,
233	@@ -62,84 +31,11 @@ pub(crate) struct Runtime {
234	}
235
236	impl Runtime {
237	- fn setup(&self) -> Result<Manifest, Error> {
238	- match &self.source {
239	- Source::Manifest(manifest) => Ok(manifest.clone()),
240	- Source::Path(path) => {
241	- match metadata(path) {
242	- Ok(info) => {
243	- if info.is_file() {
244	- Manifest::from_file(path)
245	- } else {
246	- // detect if the path is a git dir and read from it's HEAD instead.
247	- // TODO: We can't just consider it's HEAD but need to accept flags
248	- // for a branch and or commit hash.
249	- if ayllu_git::git_dir(path.as_path())? {
250	- let repository =
251	- ayllu_git::Wrapper::new(path.as_path()).map_err(\|e\| {
252	- Error::CannotReadManifestFromRepository {
253	- path: path.to_path_buf(),
254	- repo_err: e,
255	- }
256	- })?;
257	- let manifest_str = repository
258	- .read_string(Path::new(DEFAULT_BUILD_FILE), None)
259	- .map_err(\|e\| Error::CannotReadManifestFromRepository {
260	- path: path.to_path_buf(),
261	- repo_err: e,
262	- })?;
263	- if let Some(manifest_str) = manifest_str {
264	- let manifest: Manifest =
265	- serde_json::de::from_str(&manifest_str).map_err(\|e\| {
266	- Error::InvalidManifest {
267	- path: path.to_path_buf(),
268	- json_err: e,
269	- }
270	- })?;
271	- Ok(manifest)
272	- } else {
273	- Err(Error::RepositoryDoesNotContainManifest {
274	- path: path.to_path_buf(),
275	- })
276	- }
277	- } else {
278	- Manifest::from_dir(path)
279	- }
280	- }
281	- }
282	- Err(err) => Err(Error::CannotReadManifest {
283	- path: path.to_path_buf(),
284	- io_err: err,
285	- }),
286	- }
287	- }
288	- Source::Clone((_url, _git_hash)) => todo!(),
289	- }
290	- }
291	-
292	- // FIXME: Fundementally broken, assumes everything is a git repo with fragile paths
293	- fn git_information(&self) -> Result<(String, String, String), ayllu_git::Error> {
294	- let manifest_path = match &self.source {
295	- Source::Manifest(_) => todo!(),
296	- Source::Path(path_buf) => std::fs::canonicalize(path_buf).unwrap(),
297	- Source::Clone(_) => todo!(),
298	- };
299	- let repo_path = if manifest_path.is_dir() {
300	- manifest_path.as_path()
301	- } else {
302	- manifest_path.parent().unwrap()
303	- };
304	- let (collection, name) = ayllu_git::collection_and_name(repo_path);
305	- let repository = ayllu_git::Wrapper::new(repo_path)?;
306	- let latest_hash = repository.latest_hash()?.unwrap();
307	- Ok((collection, name, latest_hash))
308	- }
309	-
310	/// Allocate the manifest in the database along with it's DAG
311	- fn allocate(&mut self) -> Result<(i32, BuildGraph), Error> {
312	- let manifest = self.setup()?;
313	+ fn allocate(&mut self, config: &Config) -> Result<(i32, BuildGraph), Error> {
314	+ let manifest = self.source.read(config)?;
315	+ let (collection, name, git_hash) = self.source.git_information(config)?;
316	manifest.validate()?;
317	- let (collection, name, git_hash) = self.git_information().unwrap();
318	self.db.with(move \|mut tx\| {
319	let manifest_id = tx.manifest_create(&collection, &name, &git_hash)?;
320
321	@@ -226,18 +122,10 @@ impl Runtime {
322	/// Allocate a job graph and then execute it sequentially
323	pub async fn evaluate(
324	&mut self,
325	+ config: &Config,
326	executor: &crate::libpod_executor::Libpod,
327	) -> Result<i32, Error> {
328	- let (manifest_id, graph) = self.allocate()?;
329	- let source_dir = match &self.source {
330	- Source::Path(path_buf) => path_buf.as_path(),
331	- _ => todo!(),
332	- };
333	- let package = Package {
334	- temp_dir: &self.work_dir.join(format!("ayllu-{manifest_id}")),
335	- source_dir,
336	- };
337	- let payload = package.build()?;
338	+ let (manifest_id, graph) = self.allocate(config)?;
339	let dot_string = Dot::new(&graph).to_string();
340	tracing::info!(
341	"evaluating DAG [{}] [n_nodes={}]:\n{}",
342	@@ -245,9 +133,9 @@ impl Runtime {
343	graph.node_count(),
344	dot_string
345	);
346	+ let bundle = self.source.bundle(config, manifest_id)?;
347	let mut conn = self.db.call();
348	conn.manifest_start(manifest_id)?;
349	-
350	// TODO: Currently no parallelism is supported, need to implement the
351	// options in the manifest and then break steps into asynchronous chunks
352	// that can be run in parallel where appropriate.
353	@@ -277,9 +165,9 @@ impl Runtime {
354	build_dir: self.work_dir.as_path(),
355	image: &next_workflow.image,
356	init_args: &[init_path.to_string().as_str()],
357	- source_dir,
358	+ // source_dir,
359	init_binary: &self.init_path,
360	- src_package: &payload,
361	+ src_package: &bundle,
362	})
363	.await?;
364	}
365	@@ -292,7 +180,7 @@ impl Runtime {
366	manifest_id,
367	workflow_id: current_workflow.unwrap(),
368	step_id: *next_step_id,
369	- repo_url: self.source.url(),
370	+ // repo_url: self.source.url(),
371	..Default::default()
372	};
373	let exit_code = executor.execute(&next_step, &ctx, &mut conn).await?;
374	diff --git a/ayllu-build/src/libpod_executor.rs b/ayllu-build/src/libpod_executor.rs
375	index f432fd0..4bb6d46 100644
376	--- a/ayllu-build/src/libpod_executor.rs
377	+++ b/ayllu-build/src/libpod_executor.rs
378	@@ -21,7 +21,7 @@ pub struct Context {
379	pub workflow_id: i32,
380	pub step_id: i32,
381	pub git_hash: String,
382	- pub repo_url: String,
383	+ // pub repo_url: String,
384	pub environment: HashMap<String, Option<String>>,
385	}
386
387	@@ -31,7 +31,7 @@ pub struct InitializeArgs<'a> {
388	pub build_dir: &'a Path,
389	pub image: &'a str,
390	pub init_args: &'a [&'a str],
391	- pub source_dir: &'a Path,
392	+ // pub source_dir: &'a Path,
393	pub init_binary: &'a Path, // Must be an absolute path
394	// path to source tree and other things needed in the ct runtime
395	pub src_package: &'a Path,
396	diff --git a/ayllu-build/src/main.rs b/ayllu-build/src/main.rs
397	index 5570f66..33ec53d 100644
398	--- a/ayllu-build/src/main.rs
399	+++ b/ayllu-build/src/main.rs
400	@@ -5,34 +5,15 @@ use tracing::Level;
401	use ayllu_cmd::build::{Command, Commands};
402	use ayllu_database::Wrapper as Database;
403
404	- use crate::{
405	- config::Config,
406	- evaluate::{Runtime, Source},
407	- };
408	+ use crate::{config::Config, evaluate::Runtime, source::Source};
409
410	mod config;
411	mod error;
412	mod evaluate;
413	mod libpod;
414	mod libpod_executor;
415	- mod models;
416	- mod package;
417	-
418	- const DEFAULT_BUILD_FILE: &str = ".ayllu-build.json";
419	-
420	- type Error = Box<dyn std::error::Error>;
421	-
422	- fn read_source(cfg: &Config, input: &str) -> Result<Source, Error> {
423	- let (collection, name) = input.split_once("/").unwrap();
424	- match ayllu_git::find(
425	- cfg.collections.as_slice(),
426	- collection,
427	- name.trim_end_matches("\n"),
428	- )? {
429	- Some(repo) => Ok(Source::Path(repo.path.to_path_buf())),
430	- None => Err("Cannot find repository".into()),
431	- }
432	- }
433	+ mod manifest;
434	+ mod source;
435
436	#[tokio::main(flavor = "current_thread")]
437	async fn main() -> Result<(), Box<dyn std::error::Error>> {
438	@@ -49,7 +30,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
439	tracing::info!("Reading source from stdin for build {name:?}");
440	let mut line = String::new();
441	std::io::stdin().read_line(&mut line).unwrap();
442	- read_source(&cfg, &line)?
443	+ let (collection, name) = line.split_once("/").unwrap();
444	+ Source::Named(
445	+ collection.to_string(),
446	+ name.trim_end_matches("\n").to_string(),
447	+ )
448	} else {
449	match source {
450	Some(path) => Source::Path(Path::new(&path).to_path_buf()),
451	@@ -66,7 +51,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
452	init_path: cfg.builder.init_binary.clone(),
453	};
454	let executor = libpod_executor::Libpod::new(&cfg.builder.podman_socket);
455	- rt.evaluate(&executor).await?;
456	+ rt.evaluate(&cfg, &executor).await?;
457	Ok(())
458	}
459	}
460	diff --git a/ayllu-build/src/manifest.rs b/ayllu-build/src/manifest.rs
461	new file mode 100644
462	index 0000000..98ef061
463	--- /dev/null
464	+++ b/ayllu-build/src/manifest.rs
465	@@ -0,0 +1,89 @@
466	+ use std::collections::{HashMap, HashSet};
467	+ use std::fmt::Display;
468	+
469	+ use serde::Deserialize;
470	+
471	+ use crate::error::Error;
472	+
473	+ // pub type TestGraph = DiGraph<(Job, DiGraph<Step, i32>), i32>;
474	+
475	+ #[derive(Deserialize, Debug, Clone, Default)]
476	+ pub struct Step {
477	+ pub name: String,
478	+ #[serde(default = "Step::default_shell")]
479	+ pub shell: String,
480	+ pub input: String,
481	+ #[serde(default = "Vec::new")]
482	+ pub depends_on: Vec<String>,
483	+ #[serde(default = "HashMap::new")]
484	+ pub environment: HashMap<String, String>,
485	+ // TODO
486	+ // pub secrets: HashMap<String, String>,
487	+ }
488	+
489	+ impl Step {
490	+ pub fn validate(&self) -> Result<(), Error> {
491	+ Ok(())
492	+ }
493	+
494	+ fn default_shell() -> String {
495	+ String::from("/bin/sh")
496	+ }
497	+ }
498	+
499	+ #[derive(Deserialize, Debug, Clone)]
500	+ pub struct Workflow {
501	+ pub name: String,
502	+ pub image: String,
503	+ pub steps: Vec<Step>,
504	+ #[serde(default = "Vec::new")]
505	+ pub depends_on: Vec<String>,
506	+ }
507	+
508	+ impl Workflow {
509	+ pub fn validate(&self) -> Result<(), Error> {
510	+ let mut names = HashSet::new();
511	+ if !self
512	+ .steps
513	+ .iter()
514	+ .all(move \|step\| names.insert(step.name.clone()))
515	+ {
516	+ return Err(Error::DuplicateStepNames {
517	+ name: self.name.clone(),
518	+ });
519	+ }
520	+ let result: Result<(), Error> = self.steps.iter().try_for_each(\|step\| step.validate());
521	+ result?;
522	+ Ok(())
523	+ }
524	+ }
525	+
526	+ impl Display for Workflow {
527	+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
528	+ writeln!(f, "Job: {}", self.name)
529	+ }
530	+ }
531	+
532	+ #[derive(Deserialize, Debug, Clone)]
533	+ pub struct Manifest {
534	+ pub workflows: Vec<Workflow>,
535	+ }
536	+
537	+ impl Manifest {
538	+ pub fn validate(&self) -> Result<(), Error> {
539	+ let mut names = HashSet::new();
540	+ if !self
541	+ .workflows
542	+ .iter()
543	+ .all(move \|workflow\| names.insert(workflow.name.clone()))
544	+ {
545	+ return Err(Error::DuplicateWorkflows);
546	+ }
547	+ let result: Result<(), Error> = self
548	+ .workflows
549	+ .iter()
550	+ .try_for_each(\|workflow\| workflow.validate());
551	+ result?;
552	+ Ok(())
553	+ }
554	+ }
555	diff --git a/ayllu-build/src/models.rs b/ayllu-build/src/models.rs
556	deleted file mode 100644
557	index ad97ea1..0000000
558	--- a/ayllu-build/src/models.rs
559	+++ /dev/null
560	@@ -1,126 +0,0 @@
561	- use std::collections::{HashMap, HashSet};
562	- use std::fmt::Display;
563	- use std::path::Path;
564	-
565	- use serde::Deserialize;
566	-
567	- use crate::error::Error;
568	-
569	- // pub type TestGraph = DiGraph<(Job, DiGraph<Step, i32>), i32>;
570	-
571	- #[derive(Deserialize, Debug, Clone, Default)]
572	- pub struct Step {
573	- pub name: String,
574	- #[serde(default = "Step::default_shell")]
575	- pub shell: String,
576	- pub input: String,
577	- #[serde(default = "Vec::new")]
578	- pub depends_on: Vec<String>,
579	- #[serde(default = "HashMap::new")]
580	- pub environment: HashMap<String, String>,
581	- // TODO
582	- // pub secrets: HashMap<String, String>,
583	- }
584	-
585	- impl Step {
586	- pub fn validate(&self) -> Result<(), Error> {
587	- Ok(())
588	- }
589	-
590	- fn default_shell() -> String {
591	- String::from("/bin/sh")
592	- }
593	- }
594	-
595	- #[derive(Deserialize, Debug, Clone)]
596	- pub struct Workflow {
597	- pub name: String,
598	- pub image: String,
599	- pub steps: Vec<Step>,
600	- #[serde(default = "Vec::new")]
601	- pub depends_on: Vec<String>,
602	- }
603	-
604	- impl Workflow {
605	- pub fn validate(&self) -> Result<(), Error> {
606	- let mut names = HashSet::new();
607	- if !self
608	- .steps
609	- .iter()
610	- .all(move \|step\| names.insert(step.name.clone()))
611	- {
612	- return Err(Error::DuplicateStepNames {
613	- name: self.name.clone(),
614	- });
615	- }
616	- let result: Result<(), Error> = self.steps.iter().try_for_each(\|step\| step.validate());
617	- result?;
618	- Ok(())
619	- }
620	- }
621	-
622	- impl Display for Workflow {
623	- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
624	- writeln!(f, "Job: {}", self.name)
625	- }
626	- }
627	-
628	- #[derive(Deserialize, Debug, Clone)]
629	- pub struct Manifest {
630	- pub workflows: Vec<Workflow>,
631	- }
632	-
633	- impl Manifest {
634	- pub fn validate(&self) -> Result<(), Error> {
635	- let mut names = HashSet::new();
636	- if !self
637	- .workflows
638	- .iter()
639	- .all(move \|workflow\| names.insert(workflow.name.clone()))
640	- {
641	- return Err(Error::DuplicateWorkflows);
642	- }
643	- let result: Result<(), Error> = self
644	- .workflows
645	- .iter()
646	- .try_for_each(\|workflow\| workflow.validate());
647	- result?;
648	- Ok(())
649	- }
650	- }
651	-
652	- impl Manifest {
653	- pub fn from_file(path: &Path) -> Result<Self, Error> {
654	- let manifest_str =
655	- std::fs::read_to_string(path).map_err(\|e\| Error::CannotReadManifest {
656	- path: path.to_path_buf(),
657	- io_err: e,
658	- })?;
659	- serde_json::de::from_str(&manifest_str).map_err(\|e\| Error::InvalidManifest {
660	- path: path.to_path_buf(),
661	- json_err: e,
662	- })
663	- }
664	-
665	- pub fn from_dir(_path: &Path) -> Result<Self, Error> {
666	- todo!()
667	- // let mut files: Vec<PathBuf> = Vec::new();
668	- // for dir_entry in read_dir(path)? {
669	- // let entry = dir_entry?;
670	- // if entry.file_type()?.is_file() {
671	- // debug!("found file {:?}", entry);
672	- // files.push(entry.path())
673	- // }
674	- // }
675	- // let mut result = Program::<CBNCache>::new_from_files(files, Vec::new())?;
676	- // match result.eval_full() {
677	- // Ok(term) => {
678	- // let manifest = Manifest::deserialize(term)?;
679	- // manifest.validate()?;
680	- // Ok(manifest)
681	- // }
682	- // Err(err) => Err(format_err!("failed to load manifest: {:?}", err)),
683	- // }
684	- // }
685	- }
686	- }
687	diff --git a/ayllu-build/src/package.rs b/ayllu-build/src/package.rs
688	index 949323e..8b13789 100644
689	--- a/ayllu-build/src/package.rs
690	+++ b/ayllu-build/src/package.rs
691	@@ -1,22 +1 @@
692	- use std::path::{Path, PathBuf};
693	- use tar::Builder;
694
695	- pub struct Package<'a> {
696	- pub temp_dir: &'a Path,
697	- pub source_dir: &'a Path,
698	- }
699	-
700	- impl Package<'_> {
701	- pub fn build(&self) -> Result<PathBuf, std::io::Error> {
702	- std::fs::create_dir_all(self.temp_dir)?;
703	- let tmp_bundle = self.temp_dir.join("src.bundle");
704	- ayllu_git::bundle(self.source_dir, tmp_bundle.as_path(), None).unwrap();
705	- let fp = std::fs::File::create(self.temp_dir.join("package.tar"))?;
706	- let mut a = Builder::new(fp);
707	- a.append_file(
708	- "src.bundle",
709	- &mut std::fs::File::open(tmp_bundle.as_path())?,
710	- )?;
711	- Ok(self.temp_dir.join("package.tar").to_path_buf())
712	- }
713	- }
714	diff --git a/ayllu-build/src/source.rs b/ayllu-build/src/source.rs
715	new file mode 100644
716	index 0000000..c097060
717	--- /dev/null
718	+++ b/ayllu-build/src/source.rs
719	@@ -0,0 +1,131 @@
720	+ use std::{
721	+ io::Write,
722	+ path::{Path, PathBuf},
723	+ process::{Command, Stdio},
724	+ };
725	+
726	+ use ayllu_git::Repository;
727	+ use tar::Builder;
728	+
729	+ use crate::{config::Config, error::Error, manifest::Manifest};
730	+
731	+ const DEFAULT_BUILD_FILE: &str = ".ayllu-build.json";
732	+
733	+ /// Source git repository which shall be built
734	+ #[derive(Clone, Debug)]
735	+ pub enum Source {
736	+ Path(PathBuf),
737	+ // Pair of Collection / Name
738	+ Named(String, String),
739	+ }
740	+
741	+ impl Source {
742	+ fn repository(&self, config: &Config) -> Result<Repository, Error> {
743	+ match self {
744	+ Source::Path(path_buf) => {
745	+ let abs_path = std::fs::canonicalize(path_buf.as_path()).unwrap();
746	+ assert!(ayllu_git::git_dir(path_buf).unwrap());
747	+ Ok(Repository {
748	+ name: ayllu_git::name(&abs_path),
749	+ path: abs_path,
750	+ })
751	+ }
752	+ Source::Named(collection, name) => {
753	+ ayllu_git::find(&config.collections, collection, name)?.map_or_else(
754	+ \|\| {
755	+ Err(Error::CannotFindManagedRepository {
756	+ collection: collection.clone(),
757	+ name: name.clone(),
758	+ })
759	+ },
760	+ Ok::<_, Error>,
761	+ )
762	+ }
763	+ }
764	+ }
765	+
766	+ pub fn git_information(&self, config: &Config) -> Result<(String, String, String), Error> {
767	+ let repository = self.repository(config)?;
768	+ let latest_hash = repository
769	+ .open()
770	+ .map_err(Error::CannotOpenRepository)?
771	+ .latest_hash()
772	+ .map_err(Error::CannotOpenRepository)?
773	+ .unwrap();
774	+ let (collection, name) = ayllu_git::collection_and_name(&repository.path);
775	+ Ok((collection, name, latest_hash))
776	+ }
777	+
778	+ /// Read the manifest from the source repository
779	+ pub fn read(&self, config: &Config) -> Result<Manifest, Error> {
780	+ let repo = self
781	+ .repository(config)?
782	+ .open()
783	+ .map_err(Error::CannotOpenRepository)?;
784	+ let manifest_str = if let Some(pre_processor) = config.builder.pre_processor.as_ref() {
785	+ tracing::info!(
786	+ "Passing source {:?} through pre-processor: {}",
787	+ pre_processor.source,
788	+ pre_processor.program
789	+ );
790	+ let source = repo
791	+ .read_string(&pre_processor.source, None)
792	+ .map_err(Error::CannotReadManifest)?
793	+ .map_or_else(
794	+ \|\| {
795	+ Err(Error::RepositoryDoesNotContainManifest {
796	+ path: pre_processor.source.to_path_buf(),
797	+ })
798	+ },
799	+ Ok::<_, Error>,
800	+ )?;
801	+ let program = pre_processor.program.clone();
802	+ let args = pre_processor.args.clone();
803	+ let fp = Command::new(&program)
804	+ .stdin(Stdio::piped())
805	+ .stdout(Stdio::piped())
806	+ .args(args.as_slice())
807	+ .spawn()
808	+ .map_err(\|e\| Error::PreProcessorFailed {
809	+ program: program.clone(),
810	+ args: args.clone(),
811	+ io_error: e,
812	+ })?;
813	+ fp.stdin
814	+ .as_ref()
815	+ .map(\|mut stdin\| stdin.write_all(source.as_bytes()))
816	+ .unwrap()?;
817	+ let output = fp.wait_with_output()?;
818	+ String::from_utf8_lossy(output.stdout.as_slice()).to_string()
819	+ } else {
820	+ repo.read_string(Path::new(DEFAULT_BUILD_FILE), None)
821	+ .map_err(Error::CannotReadManifest)?
822	+ .map_or_else(
823	+ \|\| {
824	+ Err(Error::RepositoryDoesNotContainManifest {
825	+ path: Path::new(DEFAULT_BUILD_FILE).to_path_buf(),
826	+ })
827	+ },
828	+ Ok::<_, Error>,
829	+ )?
830	+ };
831	+ tracing::debug!("Processing configuration:\n{manifest_str}");
832	+ Ok(serde_json::de::from_str::<Manifest>(&manifest_str)?)
833	+ }
834	+
835	+ /// Bundle the source repository for mounting in the build container
836	+ pub fn bundle(&self, config: &Config, manifest_id: i32) -> Result<PathBuf, Error> {
837	+ let temp_dir = config.builder.work_dir.join(format!("ayllu-{manifest_id}"));
838	+ std::fs::create_dir_all(&temp_dir)?;
839	+ let tmp_bundle = temp_dir.join("src.bundle");
840	+ let repository = self.repository(config)?;
841	+ ayllu_git::bundle(&repository.path, tmp_bundle.as_path(), None).unwrap();
842	+ let fp = std::fs::File::create(temp_dir.join("package.tar"))?;
843	+ let mut a = Builder::new(fp);
844	+ a.append_file(
845	+ "src.bundle",
846	+ &mut std::fs::File::open(tmp_bundle.as_path())?,
847	+ )?;
848	+ Ok(temp_dir.join("package.tar").to_path_buf())
849	+ }
850	+ }
851	diff --git a/config.example.toml b/config.example.toml
852	index 9af4a8f..5cd1187 100644
853	--- a/config.example.toml
854	+++ b/config.example.toml
855	@@ -345,3 +345,9 @@ A Hyper Performant & Hackable Code Forge Built on Open Standards.
856	[builder]
857	# Path to the init program for build containers
858	init_binary = "/usr/bin/ayllu-init"
859	+ [builder.pre_processor]
860	+ # Path to a configuration file in your repositories
861	+ source = ".ayllu-build.jsonnet"
862	+ # Program and arguments to pass to the pre-processor
863	+ program = "jsonnet"
864	+ args = ["-"]
865	diff --git a/crates/git/src/lib.rs b/crates/git/src/lib.rs
866	index 42d1da5..91e2e4b 100644
867	--- a/crates/git/src/lib.rs
868	+++ b/crates/git/src/lib.rs
869	@@ -39,3 +39,9 @@ pub struct Repository {
870	pub name: String,
871	pub path: PathBuf,
872	}
873	+
874	+ impl Repository {
875	+ pub fn open(&self) -> Result<crate::Wrapper, Error> {
876	+ Wrapper::new(&self.path)
877	+ }
878	+ }