Author: Manos Pitsidianakis [manos@pitsidianak.is]
Hash: 89059c1411c550af635b628f1cfff9940e3633c2
Timestamp: Sat, 29 Jul 2023 07:59:29 +0000 (1 year ago)

+299 -2 +/-9 browse
core: Add MimeReject filter stub
1diff --git a/Makefile b/Makefile
2index fdc5415..4c2dc56 100644
3--- a/Makefile
4+++ b/Makefile
5 @@ -27,7 +27,7 @@ lint:
6
7 .PHONY: test
8 test: check lint
9- @$(CARGOBIN) test --all --no-fail-fast --all-features
10+ @$(CARGOBIN) nextest run --all --no-fail-fast --all-features
11
12 .PHONY: rustdoc
13 rustdoc:
14 diff --git a/core/create_migration.py b/core/create_migration.py
15new file mode 100644
16index 0000000..a4b3318
17--- /dev/null
18+++ b/core/create_migration.py
19 @@ -0,0 +1,87 @@
20+ import json
21+ from pathlib import Path
22+ import re
23+ import sys
24+ import pprint
25+ import argparse
26+
27+
28+ def make_undo(id: str) -> str:
29+ return f"DELETE FROM settings_json_schema WHERE id = '{id}';"
30+
31+
32+ def make_redo(id: str, value: str) -> str:
33+ return f"""INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('{id}', '{value}');"""
34+
35+
36+ class Migration:
37+ patt = re.compile(r"(\d+)[.].*sql")
38+
39+ def __init__(self, path: Path):
40+ name = path.name
41+ self.path = path
42+ self.is_data = "data" in name
43+ self.is_undo = "undo" in name
44+ m = self.patt.match(name)
45+ self.seq = int(m.group(1))
46+ self.name = name
47+
48+ def __str__(self) -> str:
49+ return str(self.seq)
50+
51+ def __repr__(self) -> str:
52+ return f"Migration(seq={self.seq},name={self.name},path={self.path},is_data={self.is_data},is_undo={self.is_undo})"
53+
54+
55+ if __name__ == "__main__":
56+ parser = argparse.ArgumentParser(
57+ prog="Create migrations", description="", epilog=""
58+ )
59+ parser.add_argument("--data", action="store_true")
60+ parser.add_argument("--settings", action="store_true")
61+ parser.add_argument("--name", type=str, default=None)
62+ parser.add_argument("--dry-run", action="store_true")
63+ args = parser.parse_args()
64+ migrations = {}
65+ last = -1
66+ for f in Path(".").glob("migrations/*.sql"):
67+ m = Migration(f)
68+ last = max(last, m.seq)
69+ seq = str(m)
70+ if seq not in migrations:
71+ if m.is_undo:
72+ migrations[seq] = (None, m)
73+ else:
74+ migrations[seq] = (m, None)
75+ else:
76+ if m.is_undo:
77+ redo, _ = migrations[seq]
78+ migrations[seq] = (redo, m)
79+ else:
80+ _, undo = migrations[seq]
81+ migrations[seq] = (m, undo)
82+ # pprint.pprint(migrations)
83+ if args.data:
84+ data = ".data"
85+ else:
86+ data = ""
87+ new_name = f"{last+1:0>3}{data}.sql"
88+ new_undo_name = f"{last+1:0>3}{data}.undo.sql"
89+ if not args.dry_run:
90+ redo = ""
91+ undo = ""
92+ if args.settings:
93+ if not args.name:
94+ print("Please define a --name.")
95+ sys.exit(1)
96+ redo = make_redo(args.name, "{}")
97+ undo = make_undo(args.name)
98+ name = args.name.lower() + ".json"
99+ with open(Path("settings_json_schemas") / name, "x") as file:
100+ file.write("{}")
101+ with open(Path("migrations") / new_name, "x") as file, open(
102+ Path("migrations") / new_undo_name, "x"
103+ ) as undo_file:
104+ file.write(redo)
105+ undo_file.write(undo)
106+ print(f"Created to {new_name} and {new_undo_name}.")
107 diff --git a/core/migrations/007.data.sql b/core/migrations/007.data.sql
108new file mode 100644
109index 0000000..c1bbfc2
110--- /dev/null
111+++ b/core/migrations/007.data.sql
112 @@ -0,0 +1,33 @@
113+ INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('MimeRejectSettings', '{
114+ "$schema": "http://json-schema.org/draft-07/schema",
115+ "$ref": "#/$defs/MimeRejectSettings",
116+ "$defs": {
117+ "MimeRejectSettings": {
118+ "title": "MimeRejectSettings",
119+ "description": "Settings for MimeReject message filter",
120+ "type": "object",
121+ "properties": {
122+ "enabled": {
123+ "title": "If true, list posts that contain mime types in the reject array are rejected.",
124+ "type": "boolean"
125+ },
126+ "reject": {
127+ "title": "Mime types to reject.",
128+ "type": "array",
129+ "minLength": 0,
130+ "items": { "$ref": "#/$defs/MimeType" }
131+ },
132+ "required": [
133+ "enabled"
134+ ]
135+ }
136+ },
137+ "MimeType": {
138+ "type": "string",
139+ "maxLength": 127,
140+ "minLength": 3,
141+ "uniqueItems": true,
142+ "pattern": "^[a-zA-Z!#$&-^_]+[/][a-zA-Z!#$&-^_]+$"
143+ }
144+ }
145+ }');
146 diff --git a/core/migrations/007.data.undo.sql b/core/migrations/007.data.undo.sql
147new file mode 100644
148index 0000000..cfd0945
149--- /dev/null
150+++ b/core/migrations/007.data.undo.sql
151 @@ -0,0 +1 @@
152+ DELETE FROM settings_json_schema WHERE id = 'MimeRejectSettings';
153\ No newline at end of file
154 diff --git a/core/settings_json_schemas/mimerejectsettings.json b/core/settings_json_schemas/mimerejectsettings.json
155new file mode 100644
156index 0000000..5bd0511
157--- /dev/null
158+++ b/core/settings_json_schemas/mimerejectsettings.json
159 @@ -0,0 +1,33 @@
160+ {
161+ "$schema": "http://json-schema.org/draft-07/schema",
162+ "$ref": "#/$defs/MimeRejectSettings",
163+ "$defs": {
164+ "MimeRejectSettings": {
165+ "title": "MimeRejectSettings",
166+ "description": "Settings for MimeReject message filter",
167+ "type": "object",
168+ "properties": {
169+ "enabled": {
170+ "title": "If true, list posts that contain mime types in the reject array are rejected.",
171+ "type": "boolean"
172+ },
173+ "reject": {
174+ "title": "Mime types to reject.",
175+ "type": "array",
176+ "minLength": 0,
177+ "items": { "$ref": "#/$defs/MimeType" }
178+ },
179+ "required": [
180+ "enabled"
181+ ]
182+ }
183+ },
184+ "MimeType": {
185+ "type": "string",
186+ "maxLength": 127,
187+ "minLength": 3,
188+ "uniqueItems": true,
189+ "pattern": "^[a-zA-Z!#$&-^_]+[/][a-zA-Z!#$&-^_]+$"
190+ }
191+ }
192+ }
193 diff --git a/core/src/message_filters.rs b/core/src/message_filters.rs
194index f30ce42..dd51d2a 100644
195--- a/core/src/message_filters.rs
196+++ b/core/src/message_filters.rs
197 @@ -55,6 +55,7 @@ impl Connection {
198 pub fn list_filters(&self, _list: &DbVal<MailingList>) -> Vec<Box<dyn PostFilter>> {
199 vec![
200 Box::new(PostRightsCheck),
201+ Box::new(MimeReject),
202 Box::new(FixCRLF),
203 Box::new(AddListHeaders),
204 Box::new(ArchivedAtLink),
205 @@ -376,3 +377,32 @@ impl PostFilter for FinalizeRecipients {
206 Ok((post, ctx))
207 }
208 }
209+
210+ /// Allow specific MIMEs only.
211+ pub struct MimeReject;
212+
213+ impl PostFilter for MimeReject {
214+ fn feed<'p, 'list>(
215+ self: Box<Self>,
216+ post: &'p mut PostEntry,
217+ ctx: &'p mut ListContext<'list>,
218+ ) -> std::result::Result<(&'p mut PostEntry, &'p mut ListContext<'list>), ()> {
219+ let reject = if let Some(mut settings) = ctx.filter_settings.remove("MimeRejectSettings") {
220+ let map = settings.as_object_mut().unwrap();
221+ let enabled = serde_json::from_value::<bool>(map.remove("enabled").unwrap()).unwrap();
222+ if !enabled {
223+ trace!(
224+ "MimeReject is disabled from settings found for list.pk = {} \
225+ skipping filter",
226+ ctx.list.pk
227+ );
228+ return Ok((post, ctx));
229+ }
230+ serde_json::from_value::<Vec<String>>(map.remove("reject").unwrap())
231+ } else {
232+ return Ok((post, ctx));
233+ };
234+ trace!("Running MimeReject filter with reject = {:?}", reject);
235+ Ok((post, ctx))
236+ }
237+ }
238 diff --git a/core/src/migrations.rs.inc b/core/src/migrations.rs.inc
239index d7e5d8a..7f9f7ab 100644
240--- a/core/src/migrations.rs.inc
241+++ b/core/src/migrations.rs.inc
242 @@ -242,4 +242,36 @@ DROP TABLE list_settings_json;"###),(5,r###"INSERT OR REPLACE INTO settings_json
243 ]
244 }
245 }
246- }');"###,r###"DELETE FROM settings_json_schema WHERE id = 'AddSubjectTagPrefixSettings';"###),]
247\ No newline at end of file
248+ }');"###,r###"DELETE FROM settings_json_schema WHERE id = 'AddSubjectTagPrefixSettings';"###),(7,r###"INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('MimeRejectSettings', '{
249+ "$schema": "http://json-schema.org/draft-07/schema",
250+ "$ref": "#/$defs/MimeRejectSettings",
251+ "$defs": {
252+ "MimeRejectSettings": {
253+ "title": "MimeRejectSettings",
254+ "description": "Settings for MimeReject message filter",
255+ "type": "object",
256+ "properties": {
257+ "enabled": {
258+ "title": "If true, list posts that contain mime types in the reject array are rejected.",
259+ "type": "boolean"
260+ },
261+ "reject": {
262+ "title": "Mime types to reject.",
263+ "type": "array",
264+ "minLength": 0,
265+ "items": { "$ref": "#/$defs/MimeType" }
266+ },
267+ "required": [
268+ "enabled"
269+ ]
270+ }
271+ },
272+ "MimeType": {
273+ "type": "string",
274+ "maxLength": 127,
275+ "minLength": 3,
276+ "uniqueItems": true,
277+ "pattern": "^[a-zA-Z!#$&-^_]+[/][a-zA-Z!#$&-^_]+$"
278+ }
279+ }
280+ }');"###,r###"DELETE FROM settings_json_schema WHERE id = 'MimeRejectSettings';"###),]
281\ No newline at end of file
282 diff --git a/core/src/schema.sql b/core/src/schema.sql
283index 713d615..ba839cb 100644
284--- a/core/src/schema.sql
285+++ b/core/src/schema.sql
286 @@ -613,3 +613,40 @@ INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('AddSubjectTagPref
287 }
288 }
289 }');
290+
291+
292+ -- 007.data.sql
293+
294+ INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('MimeRejectSettings', '{
295+ "$schema": "http://json-schema.org/draft-07/schema",
296+ "$ref": "#/$defs/MimeRejectSettings",
297+ "$defs": {
298+ "MimeRejectSettings": {
299+ "title": "MimeRejectSettings",
300+ "description": "Settings for MimeReject message filter",
301+ "type": "object",
302+ "properties": {
303+ "enabled": {
304+ "title": "If true, list posts that contain mime types in the reject array are rejected.",
305+ "type": "boolean"
306+ },
307+ "reject": {
308+ "title": "Mime types to reject.",
309+ "type": "array",
310+ "minLength": 0,
311+ "items": { "$ref": "#/$defs/MimeType" }
312+ },
313+ "required": [
314+ "enabled"
315+ ]
316+ }
317+ },
318+ "MimeType": {
319+ "type": "string",
320+ "maxLength": 127,
321+ "minLength": 3,
322+ "uniqueItems": true,
323+ "pattern": "^[a-zA-Z!#$&-^_]+[/][a-zA-Z!#$&-^_]+$"
324+ }
325+ }
326+ }');
327 diff --git a/core/tests/settings_json.rs b/core/tests/settings_json.rs
328index ee79842..82d459d 100644
329--- a/core/tests/settings_json.rs
330+++ b/core/tests/settings_json.rs
331 @@ -177,3 +177,47 @@ fn test_settings_json() {
332 assert!(is_valid);
333 assert!(new_last_modified != last_modified);
334 }
335+
336+ #[test]
337+ fn test_settings_json_schemas() {
338+ init_stderr_logging();
339+ let tmp_dir = TempDir::new().unwrap();
340+
341+ let db_path = tmp_dir.path().join("mpot.db");
342+ std::fs::copy("../mailpot-tests/for_testing.db", &db_path).unwrap();
343+ let mut perms = std::fs::metadata(&db_path).unwrap().permissions();
344+ #[allow(clippy::permissions_set_readonly_false)]
345+ perms.set_readonly(false);
346+ std::fs::set_permissions(&db_path, perms).unwrap();
347+
348+ let config = Configuration {
349+ send_mail: SendMail::ShellCommand("/usr/bin/false".to_string()),
350+ db_path,
351+ data_path: tmp_dir.path().to_path_buf(),
352+ administrators: vec![],
353+ };
354+ let db = Connection::open_or_create_db(config).unwrap().trusted();
355+
356+ let schemas: Vec<String> = {
357+ let mut stmt = db
358+ .connection
359+ .prepare("SELECT value FROM list_settings_json;")
360+ .unwrap();
361+ let iter = stmt
362+ .query_map([], |row| {
363+ let value: String = row.get("value")?;
364+ Ok(value)
365+ })
366+ .unwrap();
367+ let mut ret = vec![];
368+ for item in iter {
369+ ret.push(item.unwrap());
370+ }
371+ ret
372+ };
373+ println!("Testing that schemas are valid…");
374+ for schema in schemas {
375+ let schema: Value = serde_json::from_str(&schema).unwrap();
376+ let _compiled = JSONSchema::compile(&schema).expect("A valid schema");
377+ }
378+ }