Commit

Author:

Hash:

Timestamp:

+138 -42 +/-10 browse

Kevin Schoon [me@kevinschoon.com]

3f73d8124dce097344f73f693fbd91ad6f33e262

Tue, 09 Jan 2024 14:42:33 +0000 (1.5 years ago)

add support for exporting via mbox
1diff --git a/Cargo.lock b/Cargo.lock
2index 4aeb8e0..105786c 100644
3--- a/Cargo.lock
4+++ b/Cargo.lock
5 @@ -2896,13 +2896,13 @@ dependencies = [
6 [[package]]
7 name = "mailpot"
8 version = "0.1.1"
9- source = "git+https://ayllu-forge.org/forks/mailpot?branch=ayllu-dev#30a34815d96cc642963ea9aaa5b9c4f36db459d2"
10+ source = "git+https://ayllu-forge.org/forks/mailpot?branch=ayllu-dev#974437e763b0e128effe6d317979a93058dcefe3"
11 dependencies = [
12 "anyhow",
13 "chrono",
14 "jsonschema",
15 "log",
16- "melib 0.7.2",
17+ "melib 0.8.5-rc.3",
18 "minijinja",
19 "percent-encoding",
20 "rusqlite",
21 @@ -2997,38 +2997,44 @@ dependencies = [
22
23 [[package]]
24 name = "melib"
25- version = "0.7.2"
26- source = "git+https://github.com/meli/meli?rev=2447a2c#2447a2cbfeaa8d6f7ec11a2a8a6f3be1ff2fea58"
27+ version = "0.8.4"
28+ source = "registry+https://github.com/rust-lang/crates.io-index"
29+ checksum = "19fcadbfc88bf288df9ae60c85ea37d499e0011205f9231c61fb9917732255ee"
30 dependencies = [
31 "async-stream",
32 "base64 0.13.1",
33- "bincode",
34- "bitflags 1.3.2",
35+ "bitflags 2.4.0",
36 "data-encoding",
37 "encoding",
38+ "encoding_rs",
39+ "flate2",
40 "futures",
41+ "imap-codec",
42 "indexmap 1.9.3",
43 "libc",
44 "libloading 0.7.4",
45+ "log",
46 "native-tls",
47 "nix 0.24.3",
48 "nom",
49 "notify",
50+ "polling 2.8.0",
51+ "regex",
52 "serde",
53 "serde_derive",
54+ "serde_json",
55+ "serde_path_to_error",
56 "smallvec",
57 "smol",
58- "unicode-segmentation",
59+ "socket2 0.4.9",
60 "uuid",
61 "xdg",
62- "xdg-utils",
63 ]
64
65 [[package]]
66 name = "melib"
67- version = "0.8.4"
68- source = "registry+https://github.com/rust-lang/crates.io-index"
69- checksum = "19fcadbfc88bf288df9ae60c85ea37d499e0011205f9231c61fb9917732255ee"
70+ version = "0.8.5-rc.3"
71+ source = "git+https://git.meli-email.org/meli/meli.git?rev=64e60cb#64e60cb0ee79841ab40e3dba94ac27150a264c5c"
72 dependencies = [
73 "async-stream",
74 "base64 0.13.1",
75 @@ -3038,7 +3044,6 @@ dependencies = [
76 "encoding_rs",
77 "flate2",
78 "futures",
79- "imap-codec",
80 "indexmap 1.9.3",
81 "libc",
82 "libloading 0.7.4",
83 @@ -3056,6 +3061,7 @@ dependencies = [
84 "smallvec",
85 "smol",
86 "socket2 0.4.9",
87+ "unicode-segmentation",
88 "uuid",
89 "xdg",
90 ]
91 @@ -6282,12 +6288,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
92 checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546"
93
94 [[package]]
95- name = "xdg-utils"
96- version = "0.4.0"
97- source = "registry+https://github.com/rust-lang/crates.io-index"
98- checksum = "db9fefe62d5969721e2cfc529e6a760901cc0da422b6d67e7bfd18e69490dba6"
99-
100- [[package]]
101 name = "xmpp"
102 version = "0.5.0"
103 source = "git+https://ayllu-forge.org/forks/xmpp-rs?branch=ayllu#3df636d178234f7b15bcc9a7c92ad33221fcf324"
104 diff --git a/ayllu-mail/src/declarative.rs b/ayllu-mail/src/declarative.rs
105index ad1dfd7..545eb0a 100644
106--- a/ayllu-mail/src/declarative.rs
107+++ b/ayllu-mail/src/declarative.rs
108 @@ -12,18 +12,18 @@ use crate::{config, config::Config};
109 pub fn initialize(cfg: &Config) -> MpResult<()> {
110 let db = Connection::open_or_create_db(cfg.mailpot_config())?.trusted();
111
112- let mut lists_by_name: HashMap<String, i64> = HashMap::new();
113+ let mut lists_by_id: HashMap<String, i64> = HashMap::new();
114 let mut managed_lists: Vec<String> = Vec::new();
115
116 for list in db.lists()? {
117- lists_by_name.insert(list.name.clone(), list.pk);
118+ lists_by_id.insert(list.id.clone(), list.pk);
119 }
120
121 // create missing lists
122
123 for list in cfg.mail.lists.iter() {
124 log::info!("configuring mailing list: {} [{}]", list.id, list.address);
125- if !lists_by_name.contains_key(&list.id) {
126+ if !lists_by_id.contains_key(&list.id) {
127 log::info!("creating mailing list: {}", list.address);
128 let db_list = db.create_list(MailingList {
129 pk: -1,
130 @@ -34,9 +34,9 @@ pub fn initialize(cfg: &Config) -> MpResult<()> {
131 description: list.description.clone(),
132 archive_url: None,
133 })?;
134- lists_by_name.insert(db_list.id.clone(), db_list.pk);
135+ lists_by_id.insert(db_list.id.clone(), db_list.pk);
136 } else {
137- let pk = *lists_by_name.get(&list.id).unwrap();
138+ let pk = *lists_by_id.get(&list.id).unwrap();
139 // ensure the configuration is consistent
140 db.update_list(MailingListChangeset {
141 pk,
142 @@ -54,7 +54,7 @@ pub fn initialize(cfg: &Config) -> MpResult<()> {
143 // set policies
144
145 for list in cfg.mail.lists.iter() {
146- let list_pk = *lists_by_name.get(&list.id).unwrap();
147+ let list_pk = *lists_by_id.get(&list.id).unwrap();
148 let post_policy = PostPolicy {
149 pk: 0,
150 list: list_pk,
151 @@ -97,7 +97,7 @@ pub fn initialize(cfg: &Config) -> MpResult<()> {
152 db.set_list_subscription_policy(sub_policy)?;
153 }
154
155- let extras: Vec<&String> = lists_by_name
156+ let extras: Vec<&String> = lists_by_id
157 .keys()
158 .into_iter()
159 .filter(|name| managed_lists.contains(name) == false)
160 @@ -110,7 +110,7 @@ pub fn initialize(cfg: &Config) -> MpResult<()> {
161 });
162 for extra in extras.iter() {
163 log::info!("disabling mailing list: {}", extra);
164- let pk = *lists_by_name.get(&extra.to_string()).unwrap();
165+ let pk = *lists_by_id.get(&extra.to_string()).unwrap();
166 db.update_list(MailingListChangeset {
167 pk,
168 enabled: Some(false),
169 diff --git a/ayllu-mail/src/server.rs b/ayllu-mail/src/server.rs
170index 6f3664d..6f26d66 100644
171--- a/ayllu-mail/src/server.rs
172+++ b/ayllu-mail/src/server.rs
173 @@ -1,5 +1,5 @@
174- use std::borrow::Cow;
175 use std::error::Error as StdError;
176+ use std::io::{Read, Write};
177
178 use anyhow::{format_err, Result};
179 use capnp::{capability::Promise, Error as CapnpError};
180 @@ -8,12 +8,16 @@ use mailpot::{
181 models::{DbVal, Post},
182 Connection, Error as MailpotError,
183 };
184- use melib::Envelope;
185+ use melib::{
186+ mbox::{MboxFormat, MboxMetadata},
187+ Envelope,
188+ };
189+ use tracing::log;
190
191 use crate::config::Config;
192 use ayllu_api::mail_capnp::server::{
193- Client, ListThreadsParams, ListThreadsResults, ReadPostParams, ReadPostResults,
194- ReadThreadParams, ReadThreadResults, Server,
195+ Client, ExportParams, ExportResults, ListThreadsParams, ListThreadsResults, ReadPostParams,
196+ ReadPostResults, ReadThreadParams, ReadThreadResults, Server,
197 };
198 use ayllu_rpc::Server as RpcHelper;
199
200 @@ -109,6 +113,7 @@ impl Server for ServerImpl {
201 first.set_subject(subject.as_str().into());
202 first.set_is_patch(is_patch(subject.as_str()));
203 }
204+ log::info!("read thread with {} replies", replies.len());
205 for (i, reply) in replies.iter().enumerate() {
206 let mut next = threads.reborrow().get(i as u32 + 1);
207 let envelope = Envelope::from_bytes(&reply.1.message, None).unwrap();
208 @@ -159,6 +164,7 @@ impl Server for ServerImpl {
209 }
210 let message_text = envelope.body_bytes(&post.message).text();
211 let message_body = String::from_utf8(post.message.to_vec()).unwrap();
212+ message.set_message_id(envelope.message_id.to_string().as_str().into());
213 message.set_body(message_body.as_str().into());
214 message.set_text(message_text.as_str().into());
215 message.set_timestamp(envelope.timestamp as i64);
216 @@ -174,6 +180,34 @@ impl Server for ServerImpl {
217 Err(e) => Promise::err(CapnpError::failed(e.to_string())),
218 }
219 }
220+
221+ fn export(&mut self, params: ExportParams, results: ExportResults) -> Promise<(), CapnpError> {
222+ let params = pry!(params.get());
223+ let list_id = pry!(params.get_id()).to_string().unwrap();
224+ let message_id = if params.has_message_id() {
225+ let message_id = pry!(params.get_message_id()).to_string().unwrap();
226+ Some(message_id)
227+ } else {
228+ None
229+ };
230+ let read_mbox = |mut results: ExportResults| {
231+ let list = self
232+ .db
233+ .list_by_id(&list_id)?
234+ .ok_or(format_err!("no mailing list with id: {}", list_id))?;
235+ let buf = self.db.export_mbox(
236+ list.pk,
237+ message_id.as_ref().map(|message| message.as_str()),
238+ params.get_as_thread(),
239+ )?;
240+ results.get().set_thread(&buf);
241+ Ok::<(), MailpotError>(())
242+ };
243+ match read_mbox(results) {
244+ Ok(_) => Promise::ok(()),
245+ Err(e) => Promise::err(CapnpError::failed(e.to_string())),
246+ }
247+ }
248 }
249
250 pub async fn serve(cfg: &Config) -> Result<(), Box<dyn StdError>> {
251 diff --git a/crates/api/v1/mail.capnp b/crates/api/v1/mail.capnp
252index b15c8df..b66637a 100644
253--- a/crates/api/v1/mail.capnp
254+++ b/crates/api/v1/mail.capnp
255 @@ -38,4 +38,6 @@ interface Server {
256 readThread @1 (id: Text, messageId: Text, offset: Int64, limit: Int64) -> (thread: List(Message));
257 # read a single post
258 readPost @2 (id: Text, messageId: Text) -> Message;
259+ # export one or more messages in mbox format
260+ export @3 (id: Text, messageId: Text, asThread: Bool) -> (thread :Data);
261 }
262 diff --git a/src/web2/routes/mail.rs b/src/web2/routes/mail.rs
263index c4b3c69..c67a43a 100644
264--- a/src/web2/routes/mail.rs
265+++ b/src/web2/routes/mail.rs
266 @@ -1,7 +1,9 @@
267 use axum::{
268+ body::Bytes,
269 debug_handler,
270 extract::{Extension, Path},
271- response::Html,
272+ http::header::CONTENT_TYPE,
273+ response::{Html, IntoResponse, Response},
274 };
275 use serde::{Deserialize, Serialize};
276
277 @@ -40,6 +42,7 @@ struct Message {
278 pub from_address: String,
279 pub body: String,
280 pub text: String,
281+ pub is_patch: bool,
282 }
283
284 pub async fn lists(
285 @@ -120,14 +123,14 @@ pub async fn thread(
286 params.list_id
287 ))),
288 }?;
289+ let thread_id = params.thread_id.clone();
290 let mail_client = initiator.client(InitiatorKind::Mail).unwrap();
291 let messages = mail_client
292 .invoke(move |c: MailClient| async move {
293 let mut messages: Vec<Message> = Vec::new();
294 let mut req = c.read_thread_request();
295 req.get().set_id(params.list_id.as_str().into());
296- req.get()
297- .set_message_id(params.thread_id.unwrap().as_str().into());
298+ req.get().set_message_id(thread_id.unwrap().as_str().into());
299 let result = req.send().promise.await?;
300 for message in result.get()?.get_thread()? {
301 let text = message.get_text().unwrap().to_string().unwrap();
302 @@ -146,6 +149,7 @@ pub async fn thread(
303 from_address: message.get_address()?.to_string().unwrap(),
304 body: message.get_body()?.to_string().unwrap(),
305 text,
306+ is_patch: message.get_is_patch(),
307 })
308 }
309 Ok(messages)
310 @@ -156,6 +160,7 @@ pub async fn thread(
311 ctx.insert("nav_elements", &navigation::global("mail", true));
312 ctx.insert("list", list);
313 ctx.insert("list_id", &list.id);
314+ ctx.insert("thread_id", &params.thread_id.clone());
315 ctx.insert("messages", &messages);
316 ctx.insert("subject", &subject);
317 let body = templates.render("thread.html", &ctx)?;
318 @@ -202,6 +207,7 @@ pub async fn message(
319 body: message.get_body()?.to_string().unwrap(),
320 text,
321 from_address: message.get_address()?.to_string().unwrap(),
322+ is_patch: message.get_is_patch(),
323 })
324 })
325 .await?;
326 @@ -209,7 +215,43 @@ pub async fn message(
327 ctx.insert("nav_elements", &navigation::global("mail", true));
328 ctx.insert("list", list);
329 ctx.insert("list_id", &list.id);
330+ ctx.insert("thread_id", &params.thread_id);
331 ctx.insert("message", &message);
332 let body = templates.render("post.html", &ctx)?;
333 Ok(Html(body))
334 }
335+
336+ pub async fn export(
337+ Path(params): Path<Params>,
338+ Extension(initiator): Extension<Initiator>,
339+ Extension(cfg): Extension<Config>,
340+ ) -> Result<Response, Error> {
341+ let lists = cfg.mail.unwrap().lists;
342+ let list = match lists.iter().find(|list| list.id == params.list_id) {
343+ Some(list) => Ok(list),
344+ None => Err(Error::Message(format!(
345+ "no list associated with: {}",
346+ params.list_id
347+ ))),
348+ }?;
349+ let list_id = list.id.clone();
350+ let mail_client = initiator.client(InitiatorKind::Mail).unwrap();
351+ let thread = mail_client
352+ .invoke(move |c: MailClient| async move {
353+ let mut req = c.export_request();
354+ req.get().set_id(list_id.as_str().into());
355+ if let Some(message_id) = params.message_id {
356+ req.get().set_message_id(message_id.as_str().into());
357+ req.get().set_as_thread(true);
358+ }
359+ let result = req.send().promise.await?;
360+ let post = result.get()?.get_thread()?;
361+ Ok(post.to_vec())
362+ })
363+ .await?;
364+ let mut response = Bytes::from(thread).into_response();
365+ response
366+ .headers_mut()
367+ .insert(CONTENT_TYPE, "application/mbox".parse().unwrap());
368+ Ok(response)
369+ }
370 diff --git a/src/web2/server.rs b/src/web2/server.rs
371index 1ede61a..024e2d8 100644
372--- a/src/web2/server.rs
373+++ b/src/web2/server.rs
374 @@ -174,8 +174,14 @@ pub async fn serve(cfg: &Config) -> Result<(), Box<dyn Error>> {
375 Router::new()
376 .route("/", routing::get(mail::lists))
377 .route("/:list_id", routing::get(mail::threads))
378- .route("/:list_id/thread/:thread_id", routing::get(mail::thread))
379- .route("/:list_id/message/:message_id", routing::get(mail::message))
380+ .route("/export/:list_id", routing::get(mail::export))
381+ .route("/export/:list_id/:thread_id", routing::get(mail::export))
382+ .route(
383+ "/export/:list_id/:thread_id/:message_id",
384+ routing::get(mail::export),
385+ )
386+ .route("/thread/:list_id/:thread_id", routing::get(mail::thread))
387+ .route("/message/:list_id/:message_id", routing::get(mail::message))
388 .layer(from_fn_with_state(
389 Arc::new((cfg.clone(), templates.clone(), mail_required_plugins)),
390 rpc_initiator::required,
391 diff --git a/themes/default/base.scss b/themes/default/base.scss
392index a0f4cb4..f66f084 100644
393--- a/themes/default/base.scss
394+++ b/themes/default/base.scss
395 @@ -76,6 +76,11 @@ pre {
396 margin-bottom: 2px;
397 }
398
399+ button.small {
400+ display: initial;
401+ width: initial;
402+ }
403+
404 nav {
405 border-radius: var(--border-radius);
406 margin-left: 10px;
407 diff --git a/themes/default/templates/post.html b/themes/default/templates/post.html
408index e422cb0..10a0e41 100644
409--- a/themes/default/templates/post.html
410+++ b/themes/default/templates/post.html
411 @@ -3,9 +3,11 @@
412 <section class="thread-view">
413 <article>
414 <header>
415- <b>From: {{ message.from_address }}</b></br>
416- <b><a href="/mail/{{list_id}}/message/{{message.message_id}}">{{ message.message_id }}</a></b>
417- <b><a href="/discuss/mail/post/{{list_id}}/{{message.message_id}}">{{ message.message_id }}</a></b>
418+ <h1>{{message.message_id}}</h1></br>
419+ <h4> Export Message </h4>
420+ <p> <a href="/mail/export/{{list_id}}/{{thread_id}}/{{message.message_id}}">mbox</a><p>
421+ <b>From: {{ message.from_address }}</b></br>
422+ <b>Subject: {{ message.subject }}</b></br>
423 <span class="right">{{ message.created_at | format_epoch }}</span>
424 </header>
425 <pre>{{ message.text | safe }}</pre>
426 diff --git a/themes/default/templates/thread.html b/themes/default/templates/thread.html
427index 33402dc..9d75245 100644
428--- a/themes/default/templates/thread.html
429+++ b/themes/default/templates/thread.html
430 @@ -1,16 +1,18 @@
431 {% extends "base.html" %}
432 {% block content %}
433 <section class="thread-view">
434-
435 <article>
436 <header>
437- <h1>{{ subject }}</h1>
438+ <h1>{{ subject }}</h1></br>
439+ <h4> Export Thread </h4>
440+ <p><a href="/mail/export/{{list_id}}/{{thread_id}}">mbox</a></p>
441 </header>
442 {% for reply in messages %}
443 <article>
444 <header>
445 <b>From: {{ reply.from_address }}</b></br>
446- <b><a href="/mail/{{list_id}}/message/{{reply.message_id}}">{{ reply.message_id }}</a></b>
447+ <b>Subject: {{ reply.subject }} </b></br>
448+ <b><a href="/mail/message/{{list_id}}/{{reply.message_id}}">{{ reply.message_id }}</a></b>
449 <span class="right">{{ reply.created_at | format_epoch }}</span>
450 </header>
451 <pre>{{ reply.text | safe }}</pre>
452 diff --git a/themes/default/templates/threads.html b/themes/default/templates/threads.html
453index c7f8cfb..83470aa 100644
454--- a/themes/default/templates/threads.html
455+++ b/themes/default/templates/threads.html
456 @@ -22,7 +22,10 @@
457 <p> Send an e-mail to <a href="mailto:{{ request_email }}?subject=subscribe">{{request_email}}</a> with the following subject: <code>subscribe</code> </p>
458 <h4> Unsubscribe </h4>
459 <p> Send an e-mail to <a href="mailto:{{ request_email }}?subject=unsubscribe">{{request_email}}</a> with the following subject: <code>unsubscribe</code> </p>
460+ <h4> Export List </h4>
461+ <p><a href="/mail/export/{{list.id}}">mbox</a></p>
462 </div>
463+ <h4> Messages </h4>
464 <table>
465 <thead>
466 <th> from </th>
467 @@ -35,7 +38,7 @@
468 <tr>
469 <td>{{ thread.from }}</a></td>
470 <td>{{thread.timestamp | format_epoch }}</td>
471- <td><a href="/mail/{{list.id}}/thread/{{thread.message_id}}">{{thread.subject}}</a></td>
472+ <td><a href="/mail/thread/{{list.id}}/{{thread.message_id}}">{{thread.subject}}</a></td>
473 <td>{{thread.n_replies}}</td>
474 </tr>
475 {% endfor %}