Commit

Author:

Hash:

Timestamp:

+45 -24 +/-4 browse

Kevin Schoon [me@kevinschoon.com]

aaadeeb34e492ee3ad958ba8e4bc3b900647042e

Mon, 27 May 2024 07:37:34 +0000 (1.6 years ago)

make rpc timeout configurable for job operations
1diff --git a/ayllu/src/config.rs b/ayllu/src/config.rs
2index e602201..774288e 100644
3--- a/ayllu/src/config.rs
4+++ b/ayllu/src/config.rs
5 @@ -4,8 +4,6 @@ use std::fs::metadata;
6 use std::num::NonZeroUsize;
7 use std::path::PathBuf;
8 use std::thread::available_parallelism;
9-
10- use comrak::ComrakOptions;
11 use url::Url;
12
13 use ayllu_config::{data_dir, runtime_dir, Configurable};
14 @@ -227,6 +225,31 @@ pub struct Git {
15 }
16
17 #[derive(Deserialize, Serialize, Clone, Debug)]
18+ pub struct Jobs {
19+ #[serde(default = "Jobs::default_socket_path")]
20+ pub socket_path: String,
21+ #[serde(default = "Jobs::default_n_workers")]
22+ pub n_workers: NonZeroUsize,
23+ #[serde(default = "Jobs::default_timeout")]
24+ pub timeout: u64,
25+ }
26+
27+ impl Jobs {
28+ fn default_socket_path() -> String {
29+ runtime_dir().to_str().unwrap().to_string() + "/ayllu.sock"
30+ }
31+
32+ fn default_n_workers() -> NonZeroUsize {
33+ available_parallelism().unwrap()
34+ }
35+
36+ fn default_timeout() -> u64 {
37+ // 30m
38+ 1800
39+ }
40+ }
41+
42+ #[derive(Deserialize, Serialize, Clone, Debug)]
43 pub struct Config {
44 #[serde(default = "Config::default_site_name")]
45 pub site_name: String,
46 @@ -249,10 +272,7 @@ pub struct Config {
47 pub rss_time_to_live: Option<i64>,
48 pub web: Web,
49 pub http: Http,
50- #[serde(default = "Config::default_jobs_socket_path")]
51- pub jobs_socket_path: String,
52- #[serde(default = "Config::default_jobs_n_workers")]
53- pub jobs_n_workers: NonZeroUsize,
54+ pub jobs: Jobs,
55 pub database: Database,
56 #[serde(default = "Vec::new")]
57 pub collections: Vec<Collection>,
58 @@ -329,14 +349,6 @@ Disallow: /*/*/chart/*
59 NonZeroUsize::new(512).unwrap()
60 }
61
62- fn default_jobs_socket_path() -> String {
63- runtime_dir().to_str().unwrap().to_string() + "/ayllu.sock"
64- }
65-
66- fn default_jobs_n_workers() -> NonZeroUsize {
67- available_parallelism().unwrap()
68- }
69-
70 pub fn to_json(&self) -> String {
71 serde_json::to_string(self).unwrap()
72 }
73 diff --git a/ayllu/src/job_server/commands.rs b/ayllu/src/job_server/commands.rs
74index b3b9e3f..e08aaf6 100644
75--- a/ayllu/src/job_server/commands.rs
76+++ b/ayllu/src/job_server/commands.rs
77 @@ -59,11 +59,11 @@ pub async fn run_one(
78 vec![kind.to_string().into()]
79 });
80 let repo_path = resolve_path(repo_path.as_ref())?;
81- let client = jobs_client(&config.jobs_socket_path).await?;
82+ let client = jobs_client(&config.jobs.socket_path).await?;
83 for kind in kinds {
84 let mut ctx = context::current();
85 ctx.deadline = SystemTime::now()
86- .checked_add(StdDuration::from_secs(1200))
87+ .checked_add(StdDuration::from_secs(config.jobs.timeout))
88 .unwrap();
89 client
90 .submit(ctx, kind, repo_path.display().to_string(), max_depth)
91 @@ -76,7 +76,7 @@ pub async fn list(config: Config, repo_path: Option<PathBuf>) -> Result<()> {
92 let repo_path = repo_path
93 .map(|repo_path| resolve_path(Some(repo_path).as_ref()))
94 .transpose()?;
95- let client = jobs_client(&config.jobs_socket_path).await?;
96+ let client = jobs_client(&config.jobs.socket_path).await?;
97 let jobs = client
98 .list(
99 context::current(),
100 @@ -103,7 +103,7 @@ pub async fn list(config: Config, repo_path: Option<PathBuf>) -> Result<()> {
101
102 pub async fn purge(config: Config, repo_path: Option<PathBuf>) -> Result<()> {
103 let repo_path = resolve_path(repo_path.as_ref())?;
104- let client = jobs_client(&config.jobs_socket_path).await?;
105+ let client = jobs_client(&config.jobs.socket_path).await?;
106 client
107 .purge(context::current(), repo_path.display().to_string())
108 .await?;
109 diff --git a/ayllu/src/job_server/mod.rs b/ayllu/src/job_server/mod.rs
110index d81e0d5..b4f40d1 100644
111--- a/ayllu/src/job_server/mod.rs
112+++ b/ayllu/src/job_server/mod.rs
113 @@ -27,7 +27,7 @@ mod server;
114 pub async fn serve(cfg: &Config) -> Result<(), Box<dyn Error>> {
115 let db = Builder::default().url(&cfg.database.path).build().await?;
116 // run server
117- let socket_path = Path::new(&cfg.jobs_socket_path);
118+ let socket_path = Path::new(&cfg.jobs.socket_path);
119 init_socket(socket_path)?;
120 info!("job server listening @ {:?}", socket_path);
121 let mut listener = unix::listen(socket_path, Bincode::default).await?;
122 diff --git a/config.example.toml b/config.example.toml
123index 4afc1cd..7d04a35 100644
124--- a/config.example.toml
125+++ b/config.example.toml
126 @@ -22,11 +22,6 @@ sysadmin = "admin@ayllu-forge.org"
127 # runtime. See https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.max_blocking_threads
128 # max_blocking_threads = 512
129
130- # socket for the job server to listen to new requests on. This is typically
131- # used to communicate via git hooks to perform new computation on a repository
132- # after it has been updated.
133- # jobs_socket_path = "/var/run/user/1000/ayllu-jobs.sock"
134-
135
136
137 # logging level
138 @@ -78,6 +73,20 @@ clone_url = "git@localhost"
139 # for cloing to be permitted.
140 export_all = false
141
142+ # Job server configuration
143+
144+ [jobs]
145+
146+ # socket for the job server to listen to new requests on. This is typically
147+ # used to communicate via git hooks to perform new computation on a repository
148+ # after it has been updated.
149+ # jobs_socket_path = "/var/run/user/1000/ayllu-jobs.sock"
150+
151+ # Maximum time to allow a job to run before killing it in seconds.
152+ # This value needs to be high for large repositories since we have to run a
153+ # job for every commit in the repository.
154+ timeout = 1800
155+
156
157 # List of authors associated with this site as returned via webfinger queries
158 # see https://datatracker.ietf.org/doc/html/rfc7033 and https://webfinger.net/