commit 9a6c34ed61fc946fa258459e27f2872797626f68

Commit

Author: Kevin Schoon [me@kevinschoon.com]

Hash: 9a6c34ed61fc946fa258459e27f2872797626f68

Timestamp: Sun, 21 Jan 2024 13:19:05 +0000 (1 year ago)

+140 -57 +/-4 browse

make rss ttl fixed and configurable

make rss ttl fixed and configurable

This lets the user specify the TTL in the RSS response by the server.
Additionally the tests have been refactored to use a fixed timestamp instead
of being based on the current time. This was needed because two sequential
commits at the same second will cause the tests to periodically fail since
epoch times will overlap.

1	diff --git a/config.example.toml b/config.example.toml
2	index 7f94ccb..92bbbfa 100644
3	--- a/config.example.toml
4	+++ b/config.example.toml
5	@@ -57,6 +57,10 @@ blurb = """
6	# blocking operations.
7	# max_blocking_threads = 512
8
9	+ # The number of suggested seconds to wait before polling the server again
10	+ # across RSS feeds. The default of 1 hour is a reasonable default in most
11	+ # cases.
12	+ rss_time_to_live = 3600
13
14	# List of authors associated with this site as returned via webfinger queries
15	# see https://datatracker.ietf.org/doc/html/rfc7033 and https://webfinger.net/
16	diff --git a/crates/git/src/testing.rs b/crates/git/src/testing.rs
17	index 54a8333..587fa35 100644
18	--- a/crates/git/src/testing.rs
19	+++ b/crates/git/src/testing.rs
20	@@ -5,6 +5,14 @@ use std::process::Command;
21
22	use rand::{distributions::Alphanumeric, Rng};
23
24	+ /// return escaped shell variables with timestamps for use in testing
25	+ pub fn timestamp_envs(timestamp: &str) -> String {
26	+ format!(
27	+ "GIT_COMMITTER_DATE='{}' GIT_AUTHOR_DATE='{}'",
28	+ timestamp, timestamp
29	+ )
30	+ }
31	+
32	/// Helper struct that will setup a Git repository and then run a series of
33	/// commands in that repository. This is only useful for testing.
34	#[derive(Default, Debug)]
35	diff --git a/src/config.rs b/src/config.rs
36	index 66a91ed..1fd0dd0 100644
37	--- a/src/config.rs
38	+++ b/src/config.rs
39	@@ -234,6 +234,7 @@ pub struct Config {
40	pub log_level: String,
41	pub git_clone_url: Option<String>,
42	pub default_branch: Option<String>,
43	+ pub rss_time_to_live: Option<i64>,
44	pub web: Web,
45	pub http: Http,
46	#[serde(default = "Config::default_jobs_socket_path")]
47	diff --git a/src/web2/routes/rss.rs b/src/web2/routes/rss.rs
48	index d2e4dab..47ddf09 100644
49	--- a/src/web2/routes/rss.rs
50	+++ b/src/web2/routes/rss.rs
51	@@ -93,13 +93,13 @@ struct Data {
52	}
53
54	/// Channel builder for RSS feeds for on all repositories hosted on the server.
55	- /// NOTE: Most RSS readers don't support TTLs and subscribing to a summary
56	- /// channel will result in errorous updates.
57	struct Builder {
58	templates: Tera,
59	context: Context,
60	origin: String,
61	title: String,
62	+ time_to_live: Option<Duration>,
63	+ current_time: OffsetDateTime,
64	}
65
66	impl Builder {
67	@@ -125,7 +125,7 @@ impl Builder {
68	Ok(entries)
69	}
70
71	- fn channel(&self, items: Vec<Item>, time_to_live: Option<Duration>) -> Channel {
72	+ fn channel(&self, items: Vec<Item>) -> Channel {
73	let mut channel = Channel::default();
74	channel.set_title(self.title.to_string());
75	channel.set_last_build_date(items.last().map(\|item\| {
76	@@ -133,7 +133,7 @@ impl Builder {
77	.map_or(String::new(), \|date\| date.to_string())
78	}));
79	channel.set_items(items);
80	- match time_to_live {
81	+ match self.time_to_live {
82	Some(ttl) => {
83	channel.set_ttl(ttl.whole_minutes().to_string());
84	}
85	@@ -143,8 +143,7 @@ impl Builder {
86	}
87
88	fn firehose(&self, entries: Vec<(PathBuf, String)>, since: Duration) -> Result<Channel, Error> {
89	- let end = OffsetDateTime::now_utc();
90	- let start = end.saturating_sub(since);
91	+ let start = self.current_time.saturating_sub(since);
92	let mut items: Vec<(Item, i64)> = Vec::new();
93	for entry in entries {
94	let repository = Wrapper::new(entry.0.as_path())?;
95	@@ -152,8 +151,10 @@ impl Builder {
96	if config.hidden.is_some_and(\|hidden\| hidden) {
97	continue;
98	};
99	- let bugs = gitbug::Wrapper::new(entry.0.as_path())
100	- .list_range(Some((start.unix_timestamp(), end.unix_timestamp())))?;
101	+ let bugs = gitbug::Wrapper::new(entry.0.as_path()).list_range(Some((
102	+ start.unix_timestamp(),
103	+ self.current_time.unix_timestamp(),
104	+ )))?;
105	for bug in bugs {
106	let mut item = rss::Item::default();
107	item.set_title(format!("Bug: {}", bug.title));
108	@@ -172,18 +173,20 @@ impl Builder {
109	);
110	items.push((item, bug.create_time.timestamp as i64))
111	}
112	- for tag in
113	- repository.tags_range(Some((start.unix_timestamp(), end.unix_timestamp())))?
114	- {
115	+ for tag in repository.tags_range(Some((
116	+ start.unix_timestamp(),
117	+ self.current_time.unix_timestamp(),
118	+ )))? {
119	let mut item = rss::Item::default();
120	item.set_title(format!("Tag {}", tag.name));
121	item.set_description(tag.summary);
122	item.set_author(tag.author_name);
123	items.push((item, tag.commit.epoch));
124	}
125	- for commit in repository
126	- .commits_range(None, Some((start.unix_timestamp(), end.unix_timestamp())))?
127	- {
128	+ for commit in repository.commits_range(
129	+ None,
130	+ Some((start.unix_timestamp(), self.current_time.unix_timestamp())),
131	+ )? {
132	let mut item = rss::Item::default();
133	let link = format!("{}/commit/{}", entry.1, commit.id);
134	item.set_title(format!("Commit: {}", commit.summary));
135	@@ -201,8 +204,8 @@ impl Builder {
136	}
137	// reorder everything by time
138	items.sort_by(\|a, b\| a.1.cmp(&b.1));
139	- items.reverse();
140	- Ok(self.channel(items.iter().map(\|item\| item.0.clone()).collect(), None))
141	+ // items.reverse();
142	+ Ok(self.channel(items.iter().map(\|item\| item.0.clone()).collect()))
143	}
144
145	fn summary(
146	@@ -210,7 +213,7 @@ impl Builder {
147	entries: Vec<(PathBuf, String)>,
148	timeframe: Timeframe,
149	) -> Result<Channel, Error> {
150	- let (start, end) = timeframe.clamp(OffsetDateTime::now_utc());
151	+ let (start, end) = timeframe.clamp(self.current_time);
152	let mut items: Vec<Item> = Vec::new();
153	let mut all_data: Vec<Data> = Vec::new();
154	for entry in entries {
155	@@ -269,14 +272,7 @@ impl Builder {
156	permalink: false,
157	}));
158	items.push(item);
159	- let time_to_live = match timeframe {
160	- Timeframe::DAILY => None,
161	- // check twice a week
162	- Timeframe::WEEKLY => Some(Duration::days(3)),
163	- // check twice a month
164	- Timeframe::MONTHLY => Some(Duration::days(15)),
165	- };
166	- Ok(self.channel(items, time_to_live))
167	+ Ok(self.channel(items))
168	}
169	}
170
171	@@ -289,6 +285,10 @@ pub async fn feed_firehose(
172	context,
173	origin: cfg.origin,
174	title: String::from("Firehose"),
175	+ time_to_live: cfg
176	+ .rss_time_to_live
177	+ .map(\|seconds\| Duration::seconds(seconds)),
178	+ current_time: OffsetDateTime::now_utc(),
179	};
180	let channel = builder.firehose(
181	builder.scan_repositories(cfg.collections)?,
182	@@ -310,6 +310,10 @@ pub async fn feed_1d(
183	context,
184	origin: cfg.origin,
185	title: String::from("Daily Update Summary"),
186	+ time_to_live: cfg
187	+ .rss_time_to_live
188	+ .map(\|seconds\| Duration::seconds(seconds)),
189	+ current_time: OffsetDateTime::now_utc(),
190	};
191	let channel = builder.summary(
192	builder.scan_repositories(cfg.collections)?,
193	@@ -331,6 +335,10 @@ pub async fn feed_1w(
194	context,
195	origin: cfg.origin,
196	title: String::from("Weekly Update Summary"),
197	+ time_to_live: cfg
198	+ .rss_time_to_live
199	+ .map(\|seconds\| Duration::seconds(seconds)),
200	+ current_time: OffsetDateTime::now_utc(),
201	};
202	let channel = builder.summary(
203	builder.scan_repositories(cfg.collections)?,
204	@@ -352,6 +360,10 @@ pub async fn feed_1m(
205	context,
206	origin: cfg.origin,
207	title: String::from("Monthly Update Summary"),
208	+ time_to_live: cfg
209	+ .rss_time_to_live
210	+ .map(\|seconds\| Duration::seconds(seconds)),
211	+ current_time: OffsetDateTime::now_utc(),
212	};
213	let channel = builder.summary(
214	builder.scan_repositories(cfg.collections)?,
215	@@ -379,6 +391,10 @@ pub async fn feed_repository_firehose(
216	"Firehose for {}/{}",
217	preamble.collection_name, preamble.repo_name
218	),
219	+ time_to_live: cfg
220	+ .rss_time_to_live
221	+ .map(\|seconds\| Duration::seconds(seconds)),
222	+ current_time: OffsetDateTime::now_utc(),
223	};
224	let channel = builder.firehose(vec![(preamble.repo_path, project_url)], Duration::days(7))?;
225	let response = Response::builder()
226	@@ -403,6 +419,10 @@ pub async fn feed_repository_1d(
227	"Daily Update Summary for {}/{}",
228	preamble.collection_name, preamble.repo_name
229	),
230	+ time_to_live: cfg
231	+ .rss_time_to_live
232	+ .map(\|seconds\| Duration::seconds(seconds)),
233	+ current_time: OffsetDateTime::now_utc(),
234	};
235	let channel = builder.summary(vec![(preamble.repo_path, project_url)], Timeframe::DAILY)?;
236	let response = Response::builder()
237	@@ -427,6 +447,10 @@ pub async fn feed_repository_1w(
238	"Weekly Update Summary for {}/{}",
239	preamble.collection_name, preamble.repo_name
240	),
241	+ time_to_live: cfg
242	+ .rss_time_to_live
243	+ .map(\|seconds\| Duration::seconds(seconds)),
244	+ current_time: OffsetDateTime::now_utc(),
245	};
246	let channel = builder.summary(vec![(preamble.repo_path, project_url)], Timeframe::WEEKLY)?;
247	let response = Response::builder()
248	@@ -451,6 +475,10 @@ pub async fn feed_repository_1m(
249	"Monthly Update Summary for {}/{}",
250	preamble.collection_name, preamble.repo_name
251	),
252	+ time_to_live: cfg
253	+ .rss_time_to_live
254	+ .map(\|seconds\| Duration::seconds(seconds)),
255	+ current_time: OffsetDateTime::now_utc(),
256	};
257	let channel = builder.summary(vec![(preamble.repo_path, project_url)], Timeframe::MONTHLY)?;
258	let response = Response::builder()
259	@@ -465,19 +493,34 @@ mod tests {
260
261	use super::*;
262	use tera::Tera;
263	- use time::{format_description::well_known::Rfc2822, macros::datetime};
264	+ use time::{format_description::well_known::Rfc2822, macros::datetime, OffsetDateTime};
265
266	use ayllu_git::testing;
267
268	+ // Thu, 07 Apr 2005 22:13:13 +0200
269	+
270	+ /// all tests assume the current time is as below
271	+ const CURRENT_TIME: &'static str = "Tue, 19 Dec 2023 20:55:10 +0000";
272	+
273	#[test]
274	fn test_firehose_commits() {
275	- let timestamp = "Tue Dec 19 20:55:10 2023 +0000";
276	let mut test_repo = testing::Builder::default().with_commands(vec![
277	// an old commit to be filtered out
278	- format!("echo 'content' > file_1.txt && git add file_1.txt && GIT_COMMITTER_DATE='{}' GIT_AUTHOR_DATE='{}' git commit -m 'commit 1'",
279	- timestamp, timestamp).as_str(),
280	- "echo 'content' > file_2.txt && git add file_2.txt && git commit -m 'commit 2'",
281	- "echo 'content' > file_3.txt && git add file_3.txt && git commit -m 'commit 3'",
282	+ format!(
283	+ "echo 'content' > file_1.txt && git add file_1.txt && {} git commit -m 'commit 1'",
284	+ testing::timestamp_envs("Tue, 14 Dec 2023 20:55:10 +0000")
285	+ )
286	+ .as_str(),
287	+ format!(
288	+ "echo 'content' > file_2.txt && git add file_2.txt && {} git commit -m 'commit 2'",
289	+ testing::timestamp_envs("Tue, 19 Dec 2023 20:00:00 +0000")
290	+ )
291	+ .as_str(),
292	+ format!(
293	+ "echo 'content' > file_3.txt && git add file_3.txt && {} git commit -m 'commit 3'",
294	+ testing::timestamp_envs("Tue, 19 Dec 2023 20:01:00 +0000")
295	+ )
296	+ .as_str(),
297	]);
298	let (name, path) = test_repo.build().expect("failed to init repo");
299	let templates =
300	@@ -488,11 +531,12 @@ mod tests {
301	context,
302	origin: String::from("localhost:8080"),
303	title: String::from("test"),
304	+ time_to_live: Some(Duration::HOUR),
305	+ current_time: OffsetDateTime::parse(CURRENT_TIME, &Rfc2822).unwrap(),
306	};
307	let channel = builder
308	.firehose(vec![(path, name)], Duration::days(1))
309	.expect("failed to build items");
310	- println!("LEN: {}", channel.items.len());
311	assert!(channel.items.len() == 2);
312	assert!(channel.items[0]
313	.title
314	@@ -508,10 +552,22 @@ mod tests {
315	#[test]
316	fn test_firehose_releases() {
317	let mut test_repo = testing::Builder::default().with_commands(vec![
318	- "echo 'content' > file_1.txt && git add file_1.txt && git commit -m 'commit 1'",
319	- "echo 'content' > file_2.txt && git add file_2.txt && git commit -m 'commit 2'",
320	+ format!(
321	+ "echo 'content' > file_1.txt && git add file_1.txt && {} git commit -m 'commit 1'",
322	+ testing::timestamp_envs("Tue Dec 19 20:00:00 2023 +0000")
323	+ )
324	+ .as_str(),
325	+ format!(
326	+ "echo 'content' > file_2.txt && git add file_2.txt && {} git commit -m 'commit 2'",
327	+ testing::timestamp_envs("Tue Dec 19 20:01:00 2023 +0000")
328	+ )
329	+ .as_str(),
330	// release a new version
331	- "git tag -m 'release version 0.0.1!' v0.0.1",
332	+ format!(
333	+ "{} git tag -m 'release version 0.0.1!' v0.0.1",
334	+ testing::timestamp_envs("Tue Dec 19 20:02:00 2023 +0000")
335	+ )
336	+ .as_str(),
337	]);
338	let (name, path) = test_repo.build().expect("failed to init repo");
339	let templates =
340	@@ -522,6 +578,8 @@ mod tests {
341	context,
342	origin: String::from("localhost:8080"),
343	title: String::from("test"),
344	+ time_to_live: Some(Duration::HOUR),
345	+ current_time: OffsetDateTime::parse(CURRENT_TIME, &Rfc2822).unwrap(),
346	};
347	let channel = builder
348	.firehose(vec![(path, name)], Duration::days(1))
349	@@ -532,14 +590,24 @@ mod tests {
350	.as_ref()
351	.is_some_and(\|title\| title == "Commit: commit 1"));
352	assert!(channel.items[0].guid.is_some());
353	+ // TODO: validate tag
354	test_repo.cleanup().expect("failed to cleanup repo");
355	}
356
357	#[test]
358	fn test_feed_1w() {
359	+ // TODO: I think this is broken
360	let mut test_repo = testing::Builder::default().with_commands(vec![
361	- "echo 'content' > file_1.txt && git add file_1.txt && git commit -m 'commit 1'",
362	- "echo 'content' > file_2.txt && git add file_2.txt && git commit -m 'commit 2'",
363	+ format!(
364	+ "echo 'content' > file_1.txt && git add file_1.txt && {} git commit -m 'commit 1'",
365	+ testing::timestamp_envs("Tue Dec 19 20:00:00 2023 +0000")
366	+ )
367	+ .as_str(),
368	+ format!(
369	+ "echo 'content' > file_2.txt && git add file_2.txt && {} git commit -m 'commit 2'",
370	+ testing::timestamp_envs("Tue Dec 19 20:01:00 2023 +0000")
371	+ )
372	+ .as_str(),
373	]);
374	let (name, path) = test_repo.build().expect("failed to init repo");
375	let templates =
376	@@ -550,35 +618,36 @@ mod tests {
377	context,
378	origin: String::from("localhost:8080"),
379	title: String::from("test"),
380	+ time_to_live: Some(Duration::HOUR),
381	+ current_time: OffsetDateTime::parse(CURRENT_TIME, &Rfc2822).unwrap(),
382	};
383	let channel = builder
384	.summary(vec![(path, name)], Timeframe::WEEKLY)
385	.expect("failed to build items");
386	assert!(channel.items.len() == 1);
387	assert!(channel.items[0].guid.is_some());
388	- // check TTL for 3 days
389	- assert!(channel.ttl.as_ref().is_some_and(\|ttl\| ttl == "4320"))
390	+ assert!(channel.ttl.as_ref().is_some_and(\|ttl\| ttl == "60"))
391	}
392
393	#[test]
394	fn test_feed_1m() {
395	- let (start, _) = Timeframe::MONTHLY.clamp(OffsetDateTime::now_utc());
396	- let commit_1 = start
397	- .saturating_add(2 * Duration::DAY)
398	- .format(&Rfc2822)
399	- .unwrap();
400	- let commit_2 = start
401	- .saturating_add(3 * Duration::DAY)
402	- .format(&Rfc2822)
403	- .unwrap();
404	- println!("COMMITS: {} {}", commit_1, commit_2);
405	let mut test_repo = testing::Builder::default().with_commands(vec![
406	- format!("echo 'content' > file_1.txt && git add file_1.txt && GIT_COMMITTER_DATE='{}' GIT_AUTHOR_DATE='{}' git commit -m 'commit 1'",
407	- commit_1, commit_1).as_str(),
408	- format!("echo 'content' > file_2.txt && git add file_2.txt && GIT_COMMITTER_DATE='{}' GIT_AUTHOR_DATE='{}' git commit -m 'commit 2'",
409	- commit_2, commit_2).as_str(),
410	- // a new commit to be filtered out
411	- "echo 'content' > file_3.txt && git add file_3.txt && git commit -m 'commit 3'",
412	+ format!(
413	+ "echo 'content' > file_1.txt && git add file_1.txt && {} git commit -m 'commit 1'",
414	+ testing::timestamp_envs("Tue Nov 23 00:01:00 2023 +0000")
415	+ )
416	+ .as_str(),
417	+ format!(
418	+ "echo 'content' > file_2.txt && git add file_2.txt && {} git commit -m 'commit 2'",
419	+ testing::timestamp_envs("Tue Nov 23 00:01:00 2023 +0000")
420	+ )
421	+ .as_str(),
422	+ // a recent commit to be filtered out
423	+ format!(
424	+ "echo 'content' > file_3.txt && git add file_3.txt && {} git commit -m 'commit 3'",
425	+ testing::timestamp_envs("Tue Dec 18 00:01:00 2023 +0000")
426	+ )
427	+ .as_str(),
428	]);
429	let (name, path) = test_repo.build().expect("failed to init repo");
430	let templates =
431	@@ -589,6 +658,8 @@ mod tests {
432	context,
433	origin: String::from("localhost:8080"),
434	title: String::from("test"),
435	+ time_to_live: Some(Duration::HOUR),
436	+ current_time: OffsetDateTime::parse(CURRENT_TIME, &Rfc2822).unwrap(),
437	};
438	let channel = builder
439	.summary(vec![(path, name)], Timeframe::MONTHLY)
440	@@ -599,8 +670,7 @@ mod tests {
441	.as_ref()
442	.is_some_and(\|content\| content.contains("commit 1") && content.contains("commit 2")));
443	assert!(channel.items[0].guid.is_some());
444	- // check TTL for 30 days
445	- assert!(channel.ttl.as_ref().is_some_and(\|ttl\| ttl == "21600"))
446	+ assert!(channel.ttl.as_ref().is_some_and(\|ttl\| ttl == "60"))
447	}
448
449	#[test]