Author: Manos Pitsidianakis [manos@pitsidianak.is]
Hash: f425cf0198a098cdf1f2c6672a8e34c5b9bb3302
Timestamp: Sat, 08 Jun 2024 20:10:44 +0000 (3 months ago)

+305 -115 +/-6 browse
mailpot: make sure inserted headers are properly encoded 
mailpot: make sure inserted headers are properly encoded

Closes #14

Link: <https://git.meli-email.org/meli/mailpot/issues/14>
Signed-off-by: Manos Pitsidianakis <manos@pitsidianak.is>
1diff --git a/Cargo.lock b/Cargo.lock
2index bf39166..dec3be9 100644
3--- a/Cargo.lock
4+++ b/Cargo.lock
5 @@ -2191,6 +2191,7 @@ version = "0.1.1"
6 dependencies = [
7 "anyhow",
8 "chrono",
9+ "data-encoding",
10 "jsonschema",
11 "log",
12 "mailpot-tests",
13 diff --git a/mailpot/Cargo.toml b/mailpot/Cargo.toml
14index 2b82de9..e0bedfd 100644
15--- a/mailpot/Cargo.toml
16+++ b/mailpot/Cargo.toml
17 @@ -16,6 +16,7 @@ doc-scrape-examples = true
18 [dependencies]
19 anyhow = "1.0.58"
20 chrono = { version = "^0.4", features = ["serde", ] }
21+ data-encoding = { version = "2.1.1" }
22 jsonschema = { version = "0.17", default-features = false }
23 log = "0.4"
24 melib = { version = "0.8.6", default-features = false, features = ["mbox", "smtp", "maildir"] }
25 diff --git a/mailpot/src/lib.rs b/mailpot/src/lib.rs
26index e56a80a..8576c57 100644
27--- a/mailpot/src/lib.rs
28+++ b/mailpot/src/lib.rs
29 @@ -257,3 +257,117 @@ const PATH: &AsciiSet = &FRAGMENT.add(b'#').add(b'?').add(b'{').add(b'}');
30
31 /// Set for percent encoding URL components.
32 pub const PATH_SEGMENT: &AsciiSet = &PATH.add(b'/').add(b'%');
33+
34+ mod helpers {
35+ use std::borrow::Cow;
36+
37+ use data_encoding::Encoding;
38+
39+ fn base64_encoding() -> Encoding {
40+ let mut spec = data_encoding::BASE64_MIME.specification();
41+ spec.ignore.clear();
42+ spec.wrap.width = 0;
43+ spec.wrap.separator.clear();
44+ spec.encoding().unwrap()
45+ }
46+
47+ /// Ensure `value` is in appropriate representation to be a header value.
48+ pub fn encode_header(value: &'_ [u8]) -> Cow<'_, [u8]> {
49+ if value.iter().all(|&b| b.is_ascii_graphic() || b == b' ') {
50+ return Cow::Borrowed(value);
51+ }
52+ Cow::Owned(_encode_header(value))
53+ }
54+
55+ /// Same as [`encode_header`] but for owned bytes.
56+ pub fn encode_header_owned(value: Vec<u8>) -> Vec<u8> {
57+ if value.iter().all(|&b| b.is_ascii_graphic() || b == b' ') {
58+ return value;
59+ }
60+ _encode_header(&value)
61+ }
62+
63+ fn _encode_header(value: &[u8]) -> Vec<u8> {
64+ let mut ret = Vec::with_capacity(value.len());
65+ let base64_mime = base64_encoding();
66+ let mut is_current_window_ascii = true;
67+ let mut current_window_start = 0;
68+ {
69+ for (idx, g) in value.iter().copied().enumerate() {
70+ match (g.is_ascii(), is_current_window_ascii) {
71+ (true, true) => {
72+ if g.is_ascii_graphic() || g == b' ' {
73+ ret.push(g);
74+ } else {
75+ current_window_start = idx;
76+ is_current_window_ascii = false;
77+ }
78+ }
79+ (true, false) => {
80+ /* If !g.is_whitespace()
81+ *
82+ * Whitespaces inside encoded tokens must be greedily taken,
83+ * instead of splitting each non-ascii word into separate encoded tokens. */
84+ if g != b' ' && !g.is_ascii_control() {
85+ ret.extend_from_slice(
86+ format!(
87+ "=?UTF-8?B?{}?=",
88+ base64_mime.encode(&value[current_window_start..idx]).trim()
89+ )
90+ .as_bytes(),
91+ );
92+ if idx != value.len() - 1
93+ && ((idx == 0)
94+ ^ (!value[idx - 1].is_ascii_control()
95+ && !value[idx - 1] != b' '))
96+ {
97+ ret.push(b' ');
98+ }
99+ is_current_window_ascii = true;
100+ current_window_start = idx;
101+ ret.push(g);
102+ }
103+ }
104+ (false, true) => {
105+ current_window_start = idx;
106+ is_current_window_ascii = false;
107+ }
108+ /* RFC2047 recommends:
109+ * 'While there is no limit to the length of a multiple-line header field,
110+ * each line of a header field that contains one or more
111+ * 'encoded-word's is limited to 76 characters.'
112+ * This is a rough compliance.
113+ */
114+ (false, false) if (((4 * (idx - current_window_start) / 3) + 3) & !3) > 33 => {
115+ ret.extend_from_slice(
116+ format!(
117+ "=?UTF-8?B?{}?=",
118+ base64_mime.encode(&value[current_window_start..idx]).trim()
119+ )
120+ .as_bytes(),
121+ );
122+ if idx != value.len() - 1 {
123+ ret.push(b' ');
124+ }
125+ current_window_start = idx;
126+ }
127+ (false, false) => {}
128+ }
129+ }
130+ }
131+ /* If the last part of the header value is encoded, it won't be pushed inside
132+ * the previous for block */
133+ if !is_current_window_ascii {
134+ ret.extend_from_slice(
135+ format!(
136+ "=?UTF-8?B?{}?=",
137+ base64_mime.encode(&value[current_window_start..]).trim()
138+ )
139+ .as_bytes(),
140+ );
141+ }
142+ ret
143+ }
144+ }
145+
146+ pub use helpers::*;
147 diff --git a/mailpot/src/message_filters.rs b/mailpot/src/message_filters.rs
148index 553a471..547cb64 100644
149--- a/mailpot/src/message_filters.rs
150+++ b/mailpot/src/message_filters.rs
151 @@ -166,21 +166,26 @@ impl PostFilter for AddListHeaders {
152 ) -> std::result::Result<(&'p mut PostEntry, &'p mut ListContext<'list>), ()> {
153 trace!("Running AddListHeaders filter");
154 let (mut headers, body) = melib::email::parser::mail(&post.bytes).unwrap();
155- let sender = format!("<{}>", ctx.list.address);
156- headers.push((HeaderName::SENDER, sender.as_bytes()));
157
158- let list_id = Some(ctx.list.id_header());
159- let list_help = ctx.list.help_header();
160- let list_post = ctx.list.post_header(ctx.post_policy.as_deref());
161+ let map_fn = |x| crate::encode_header_owned(String::into_bytes(x));
162+
163+ let sender = Some(format!("<{}>", ctx.list.address)).map(map_fn);
164+
165+ let list_id = Some(map_fn(ctx.list.id_header()));
166+ let list_help = ctx.list.help_header().map(map_fn);
167+ let list_post = ctx.list.post_header(ctx.post_policy.as_deref()).map(map_fn);
168 let list_unsubscribe = ctx
169 .list
170- .unsubscribe_header(ctx.subscription_policy.as_deref());
171+ .unsubscribe_header(ctx.subscription_policy.as_deref())
172+ .map(map_fn);
173 let list_subscribe = ctx
174 .list
175- .subscribe_header(ctx.subscription_policy.as_deref());
176- let list_archive = ctx.list.archive_header();
177+ .subscribe_header(ctx.subscription_policy.as_deref())
178+ .map(map_fn);
179+ let list_archive = ctx.list.archive_header().map(map_fn);
180
181 for (hdr, val) in [
182+ (HeaderName::SENDER, &sender),
183 (HeaderName::LIST_ID, &list_id),
184 (HeaderName::LIST_HELP, &list_help),
185 (HeaderName::LIST_POST, &list_post),
186 @@ -189,7 +194,7 @@ impl PostFilter for AddListHeaders {
187 (HeaderName::LIST_ARCHIVE, &list_archive),
188 ] {
189 if let Some(val) = val {
190- headers.push((hdr, val.as_bytes()));
191+ headers.push((hdr, val.as_slice()));
192 }
193 }
194
195 @@ -239,11 +244,12 @@ impl PostFilter for AddSubjectTagPrefix {
196 let (mut headers, body) = melib::email::parser::mail(&post.bytes).unwrap();
197 let mut subject;
198 if let Some((_, subj_val)) = headers.iter_mut().find(|(k, _)| k == HeaderName::SUBJECT) {
199- subject = format!("[{}] ", ctx.list.id).into_bytes();
200+ subject = crate::encode_header_owned(format!("[{}] ", ctx.list.id).into_bytes());
201 subject.extend(subj_val.iter().cloned());
202 *subj_val = subject.as_slice();
203 } else {
204- subject = format!("[{}] (no subject)", ctx.list.id).into_bytes();
205+ subject =
206+ crate::encode_header_owned(format!("[{}] (no subject)", ctx.list.id).into_bytes());
207 headers.push((HeaderName::SUBJECT, subject.as_slice()));
208 }
209
210 @@ -293,7 +299,7 @@ impl PostFilter for ArchivedAtLink {
211
212 let env = minijinja::Environment::new();
213 let message_id = post.message_id.to_string();
214- let header_val = env
215+ let header_val = crate::encode_header_owned(env
216 .render_named_str(
217 "ArchivedAtLinkSettings.template",
218 &template,
219 @@ -309,9 +315,9 @@ impl PostFilter for ArchivedAtLink {
220 )
221 .map_err(|err| {
222 log::error!("ArchivedAtLink: {}", err);
223- })?;
224+ })?.into_bytes());
225 let (mut headers, body) = melib::email::parser::mail(&post.bytes).unwrap();
226- headers.push((HeaderName::ARCHIVED_AT, header_val.as_bytes()));
227+ headers.push((HeaderName::ARCHIVED_AT, header_val.as_slice()));
228
229 let mut new_vec = Vec::with_capacity(
230 headers
231 diff --git a/mailpot/tests/message_filters.rs b/mailpot/tests/message_filters.rs
232new file mode 100644
233index 0000000..e5d3b0a
234--- /dev/null
235+++ b/mailpot/tests/message_filters.rs
236 @@ -0,0 +1,169 @@
237+ /*
238+ * This file is part of mailpot
239+ *
240+ * Copyright 2020 - Manos Pitsidianakis
241+ *
242+ * This program is free software: you can redistribute it and/or modify
243+ * it under the terms of the GNU Affero General Public License as
244+ * published by the Free Software Foundation, either version 3 of the
245+ * License, or (at your option) any later version.
246+ *
247+ * This program is distributed in the hope that it will be useful,
248+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
249+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
250+ * GNU Affero General Public License for more details.
251+ *
252+ * You should have received a copy of the GNU Affero General Public License
253+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
254+ */
255+
256+ use mailpot::{models::*, queue::Queue, Configuration, Connection, SendMail};
257+ use mailpot_tests::init_stderr_logging;
258+ use serde_json::json;
259+ use tempfile::TempDir;
260+
261+ #[test]
262+ fn test_post_filters() {
263+ init_stderr_logging();
264+ let tmp_dir = TempDir::new().unwrap();
265+
266+ let mut post_policy = PostPolicy {
267+ pk: -1,
268+ list: -1,
269+ announce_only: false,
270+ subscription_only: false,
271+ approval_needed: false,
272+ open: true,
273+ custom: false,
274+ };
275+ let db_path = tmp_dir.path().join("mpot.db");
276+ let config = Configuration {
277+ send_mail: SendMail::ShellCommand("/usr/bin/false".to_string()),
278+ db_path,
279+ data_path: tmp_dir.path().to_path_buf(),
280+ administrators: vec![],
281+ };
282+
283+ let db = Connection::open_or_create_db(config).unwrap().trusted();
284+ let foo_chat = db
285+ .create_list(MailingList {
286+ pk: 0,
287+ name: "foobar chat".into(),
288+ id: "foo-chat".into(),
289+ address: "foo-chat@example.com".into(),
290+ description: None,
291+ topics: vec![],
292+ archive_url: None,
293+ })
294+ .unwrap();
295+ post_policy.list = foo_chat.pk();
296+ db.add_subscription(
297+ foo_chat.pk(),
298+ ListSubscription {
299+ pk: -1,
300+ list: foo_chat.pk(),
301+ address: "user@example.com".into(),
302+ name: None,
303+ account: None,
304+ digest: false,
305+ enabled: true,
306+ verified: true,
307+ hide_address: false,
308+ receive_duplicates: true,
309+ receive_own_posts: true,
310+ receive_confirmation: false,
311+ },
312+ )
313+ .unwrap();
314+ db.set_list_post_policy(post_policy).unwrap();
315+
316+ println!("Check that List subject prefix is inserted and can be optionally disabled…");
317+ let post_bytes = b"From: Name <user@example.com>
318+ To: <foo-chat@example.com>
319+ Subject: This is a post
320+ Date: Thu, 29 Oct 2020 13:58:16 +0000
321+ Message-ID: <abcdefgh@sator.example.com>
322+ Content-Language: en-US
323+ Content-Type: text/html
324+ Content-Transfer-Encoding: base64
325+ MIME-Version: 1.0
326+
327+ PCFET0NUWVBFPjxodG1sPjxoZWFkPjx0aXRsZT5mb288L3RpdGxlPjwvaGVhZD48Ym9k
328+ eT48dGFibGUgY2xhc3M9ImZvbyI+PHRoZWFkPjx0cj48dGQ+Zm9vPC90ZD48L3RoZWFk
329+ Pjx0Ym9keT48dHI+PHRkPmZvbzE8L3RkPjwvdHI+PC90Ym9keT48L3RhYmxlPjwvYm9k
330+ eT48L2h0bWw+
331+ ";
332+ let envelope = melib::Envelope::from_bytes(post_bytes, None).expect("Could not parse message");
333+ db.post(&envelope, post_bytes, /* dry_run */ false).unwrap();
334+ let q = db.queue(Queue::Out).unwrap();
335+ assert_eq!(&q[0].subject, "[foo-chat] This is a post");
336+
337+ db.delete_from_queue(Queue::Out, vec![]).unwrap();
338+ {
339+ let mut stmt = db
340+ .connection
341+ .prepare(
342+ "INSERT INTO list_settings_json(name, list, value) \
343+ VALUES('AddSubjectTagPrefixSettings', ?, ?) RETURNING *;",
344+ )
345+ .unwrap();
346+ stmt.query_row(
347+ rusqlite::params![
348+ &foo_chat.pk(),
349+ &json!({
350+ "enabled": false
351+ }),
352+ ],
353+ |_| Ok(()),
354+ )
355+ .unwrap();
356+ }
357+ db.post(&envelope, post_bytes, /* dry_run */ false).unwrap();
358+ let q = db.queue(Queue::Out).unwrap();
359+ assert_eq!(&q[0].subject, "This is a post");
360+ db.delete_from_queue(Queue::Out, vec![]).unwrap();
361+
362+ println!("Check that List headers are encoded with MIME when necessary…");
363+ db.update_list(changesets::MailingListChangeset {
364+ pk: foo_chat.pk,
365+ description: Some(Some(
366+ "Why, I, in this weak piping time of peace,\nHave no delight to pass away the \
367+ time,\nUnless to spy my shadow in the sun."
368+ .to_string(),
369+ )),
370+ ..Default::default()
371+ })
372+ .unwrap();
373+ db.post(&envelope, post_bytes, /* dry_run */ false).unwrap();
374+ let q = db.queue(Queue::Out).unwrap();
375+ let q_env = melib::Envelope::from_bytes(&q[0].message, None).expect("Could not parse message");
376+ assert_eq!(
377+ &q_env.other_headers[melib::HeaderName::LIST_ID],
378+ "Why, I, in this weak piping time of peace,\nHave no delight to pass away the \
379+ time,\nUnless to spy my shadow in the sun. <foo-chat.example.com>"
380+ );
381+ db.delete_from_queue(Queue::Out, vec![]).unwrap();
382+
383+ db.update_list(changesets::MailingListChangeset {
384+ pk: foo_chat.pk,
385+ description: Some(Some(
386+ r#"<p>Discussion about mailpot, a mailing list manager software.</p>
387+
388+
389+ <ul>
390+ <li>Main git repository: <a href="https://git.meli-email.org/meli/mailpot">https://git.meli-email.org/meli/mailpot</a></li>
391+ <li>Mirror: <a href="https://github.com/meli/mailpot/">https://github.com/meli/mailpot/</a></li>
392+ </ul>"#
393+ .to_string(),
394+ )),
395+ ..Default::default()
396+ })
397+ .unwrap();
398+ db.post(&envelope, post_bytes, /* dry_run */ false).unwrap();
399+ let q = db.queue(Queue::Out).unwrap();
400+ let q_env = melib::Envelope::from_bytes(&q[0].message, None).expect("Could not parse message");
401+ assert_eq!(
402+ &q_env.other_headers[melib::HeaderName::LIST_ID],
403+ "<p>Discussion about mailpot, a mailing list manager software.</p>\n\n\n<ul>\n<li>Main git repository: <a href=\"https://git.meli-email.org/meli/mailpot\">https://git.meli-email.org/meli/mailpot</a></li>\n<li>Mirror: <a href=\"https://github.com/meli/mailpot/\">https://github.com/meli/mailpot/</a></li>\n</ul> <foo-chat.example.com>"
404+ );
405+ }
406 diff --git a/mailpot/tests/subscription.rs b/mailpot/tests/subscription.rs
407index 1f5468c..c92081a 100644
408--- a/mailpot/tests/subscription.rs
409+++ b/mailpot/tests/subscription.rs
410 @@ -19,7 +19,6 @@
411
412 use mailpot::{models::*, queue::Queue, Configuration, Connection, SendMail};
413 use mailpot_tests::init_stderr_logging;
414- use serde_json::json;
415 use tempfile::TempDir;
416
417 #[test]
418 @@ -228,103 +227,3 @@ eT48L2h0bWw+
419 }
420 }
421 }
422-
423- #[test]
424- fn test_post_filters() {
425- init_stderr_logging();
426- let tmp_dir = TempDir::new().unwrap();
427-
428- let mut post_policy = PostPolicy {
429- pk: -1,
430- list: -1,
431- announce_only: false,
432- subscription_only: false,
433- approval_needed: false,
434- open: true,
435- custom: false,
436- };
437- let db_path = tmp_dir.path().join("mpot.db");
438- let config = Configuration {
439- send_mail: SendMail::ShellCommand("/usr/bin/false".to_string()),
440- db_path,
441- data_path: tmp_dir.path().to_path_buf(),
442- administrators: vec![],
443- };
444-
445- let db = Connection::open_or_create_db(config).unwrap().trusted();
446- let foo_chat = db
447- .create_list(MailingList {
448- pk: 0,
449- name: "foobar chat".into(),
450- id: "foo-chat".into(),
451- address: "foo-chat@example.com".into(),
452- description: None,
453- topics: vec![],
454- archive_url: None,
455- })
456- .unwrap();
457- post_policy.list = foo_chat.pk();
458- db.add_subscription(
459- foo_chat.pk(),
460- ListSubscription {
461- pk: -1,
462- list: foo_chat.pk(),
463- address: "user@example.com".into(),
464- name: None,
465- account: None,
466- digest: false,
467- enabled: true,
468- verified: true,
469- hide_address: false,
470- receive_duplicates: true,
471- receive_own_posts: true,
472- receive_confirmation: false,
473- },
474- )
475- .unwrap();
476- db.set_list_post_policy(post_policy).unwrap();
477-
478- let post_bytes = b"From: Name <user@example.com>
479- To: <foo-chat@example.com>
480- Subject: This is a post
481- Date: Thu, 29 Oct 2020 13:58:16 +0000
482- Message-ID: <abcdefgh@sator.example.com>
483- Content-Language: en-US
484- Content-Type: text/html
485- Content-Transfer-Encoding: base64
486- MIME-Version: 1.0
487-
488- PCFET0NUWVBFPjxodG1sPjxoZWFkPjx0aXRsZT5mb288L3RpdGxlPjwvaGVhZD48Ym9k
489- eT48dGFibGUgY2xhc3M9ImZvbyI+PHRoZWFkPjx0cj48dGQ+Zm9vPC90ZD48L3RoZWFk
490- Pjx0Ym9keT48dHI+PHRkPmZvbzE8L3RkPjwvdHI+PC90Ym9keT48L3RhYmxlPjwvYm9k
491- eT48L2h0bWw+
492- ";
493- let envelope = melib::Envelope::from_bytes(post_bytes, None).expect("Could not parse message");
494- db.post(&envelope, post_bytes, /* dry_run */ false).unwrap();
495- let q = db.queue(Queue::Out).unwrap();
496- assert_eq!(&q[0].subject, "[foo-chat] This is a post");
497-
498- db.delete_from_queue(Queue::Out, vec![]).unwrap();
499- {
500- let mut stmt = db
501- .connection
502- .prepare(
503- "INSERT INTO list_settings_json(name, list, value) \
504- VALUES('AddSubjectTagPrefixSettings', ?, ?) RETURNING *;",
505- )
506- .unwrap();
507- stmt.query_row(
508- rusqlite::params![
509- &foo_chat.pk(),
510- &json!({
511- "enabled": false
512- }),
513- ],
514- |_| Ok(()),
515- )
516- .unwrap();
517- }
518- db.post(&envelope, post_bytes, /* dry_run */ false).unwrap();
519- let q = db.queue(Queue::Out).unwrap();
520- assert_eq!(&q[0].subject, "This is a post");
521- }