+380 -12 +/-12 browse
1 | diff --git a/Cargo.lock b/Cargo.lock |
2 | index 03fb72b..8a8d7b4 100644 |
3 | --- a/Cargo.lock |
4 | +++ b/Cargo.lock |
5 | @@ -2530,6 +2530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" |
6 | checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" |
7 | dependencies = [ |
8 | "bitflags", |
9 | + "chrono", |
10 | "fallible-iterator", |
11 | "fallible-streaming-iterator", |
12 | "hashlink", |
13 | diff --git a/Cargo.toml b/Cargo.toml |
14 | index ffa3357..5632c20 100644 |
15 | --- a/Cargo.toml |
16 | +++ b/Cargo.toml |
17 | @@ -7,3 +7,9 @@ members = [ |
18 | "rest-http", |
19 | "web", |
20 | ] |
21 | + |
22 | + [profile.release] |
23 | + lto = "fat" |
24 | + opt-level = "z" |
25 | + codegen-units = 1 |
26 | + split-debuginfo = "unpacked" |
27 | diff --git a/cli/src/lib.rs b/cli/src/lib.rs |
28 | index 67c6de3..3d4dc9a 100644 |
29 | --- a/cli/src/lib.rs |
30 | +++ b/cli/src/lib.rs |
31 | @@ -185,6 +185,28 @@ pub enum Command { |
32 | /// Is account enabled. |
33 | enabled: Option<Option<bool>>, |
34 | }, |
35 | + /// Show and fix possible data mistakes or inconsistencies. |
36 | + Repair { |
37 | + /// Fix errors (default: false) |
38 | + #[arg(long, default_value = "false")] |
39 | + fix: bool, |
40 | + /// Select all tests (default: false) |
41 | + #[arg(long, default_value = "false")] |
42 | + all: bool, |
43 | + /// Post `datetime` column must have the Date: header value, in RFC2822 |
44 | + /// format. |
45 | + #[arg(long, default_value = "false")] |
46 | + datetime_header_value: bool, |
47 | + /// Remove accounts that have no matching subscriptions. |
48 | + #[arg(long, default_value = "false")] |
49 | + remove_empty_accounts: bool, |
50 | + /// Remove subscription requests that have been accepted. |
51 | + #[arg(long, default_value = "false")] |
52 | + remove_accepted_subscription_requests: bool, |
53 | + /// Warn if a list has no owners. |
54 | + #[arg(long, default_value = "false")] |
55 | + warn_list_no_owner: bool, |
56 | + }, |
57 | } |
58 | |
59 | /// Postfix config values. |
60 | diff --git a/cli/src/lints.rs b/cli/src/lints.rs |
61 | new file mode 100644 |
62 | index 0000000..68b118f |
63 | --- /dev/null |
64 | +++ b/cli/src/lints.rs |
65 | @@ -0,0 +1,252 @@ |
66 | + /* |
67 | + * This file is part of mailpot |
68 | + * |
69 | + * Copyright 2020 - Manos Pitsidianakis |
70 | + * |
71 | + * This program is free software: you can redistribute it and/or modify |
72 | + * it under the terms of the GNU Affero General Public License as |
73 | + * published by the Free Software Foundation, either version 3 of the |
74 | + * License, or (at your option) any later version. |
75 | + * |
76 | + * This program is distributed in the hope that it will be useful, |
77 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
78 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
79 | + * GNU Affero General Public License for more details. |
80 | + * |
81 | + * You should have received a copy of the GNU Affero General Public License |
82 | + * along with this program. If not, see <https://www.gnu.org/licenses/>. |
83 | + */ |
84 | + |
85 | + use super::*; |
86 | + |
87 | + pub fn datetime_header_value_lint(db: &mut Connection, dry_run: bool) -> Result<()> { |
88 | + let mut col = vec![]; |
89 | + { |
90 | + let mut stmt = db.connection.prepare("SELECT * FROM post ORDER BY pk")?; |
91 | + let iter = stmt.query_map([], |row| { |
92 | + let pk: i64 = row.get("pk")?; |
93 | + let date_s: String = row.get("datetime")?; |
94 | + match melib::datetime::rfc822_to_timestamp(date_s.trim()) { |
95 | + Err(_) | Ok(0) => { |
96 | + let mut timestamp: i64 = row.get("timestamp")?; |
97 | + let created: i64 = row.get("created")?; |
98 | + if timestamp == 0 { |
99 | + timestamp = created; |
100 | + } |
101 | + timestamp = std::cmp::min(timestamp, created); |
102 | + let timestamp = if timestamp <= 0 { |
103 | + None |
104 | + } else { |
105 | + // safe because we checked it's not negative or zero above. |
106 | + Some(timestamp as u64) |
107 | + }; |
108 | + let message: Vec<u8> = row.get("message")?; |
109 | + Ok(Some((pk, date_s, message, timestamp))) |
110 | + } |
111 | + Ok(_) => Ok(None), |
112 | + } |
113 | + })?; |
114 | + |
115 | + for entry in iter { |
116 | + if let Some(s) = entry? { |
117 | + col.push(s); |
118 | + } |
119 | + } |
120 | + } |
121 | + let mut failures = 0; |
122 | + let tx = if dry_run { |
123 | + None |
124 | + } else { |
125 | + Some(db.connection.transaction()?) |
126 | + }; |
127 | + if col.is_empty() { |
128 | + println!("datetime_header_value: ok"); |
129 | + } else { |
130 | + println!("datetime_header_value: found {} entries", col.len()); |
131 | + println!("pk\tDate value\tshould be"); |
132 | + for (pk, val, message, timestamp) in col { |
133 | + let correct = if let Ok(v) = |
134 | + chrono::DateTime::<chrono::FixedOffset>::parse_from_rfc3339(&val) |
135 | + { |
136 | + v.to_rfc2822() |
137 | + } else if let Some(v) = timestamp.map(|t| { |
138 | + melib::datetime::timestamp_to_string(t, Some(melib::datetime::RFC822_DATE), true) |
139 | + }) { |
140 | + v |
141 | + } else if let Ok(v) = |
142 | + Envelope::from_bytes(&message, None).map(|env| env.date_as_str().to_string()) |
143 | + { |
144 | + v |
145 | + } else { |
146 | + failures += 1; |
147 | + println!("{pk}\t{val}\tCould not find any valid date value in the post metadata!"); |
148 | + continue; |
149 | + }; |
150 | + println!("{pk}\t{val}\t{correct}"); |
151 | + if let Some(tx) = tx.as_ref() { |
152 | + tx.execute( |
153 | + "UPDATE post SET datetime = ? WHERE pk = ?", |
154 | + rusqlite::params![&correct, pk], |
155 | + )?; |
156 | + } |
157 | + } |
158 | + } |
159 | + if let Some(tx) = tx { |
160 | + tx.commit()?; |
161 | + } |
162 | + if failures > 0 { |
163 | + println!( |
164 | + "datetime_header_value: {failures} failure{}", |
165 | + if failures == 1 { "" } else { "s" } |
166 | + ); |
167 | + } |
168 | + Ok(()) |
169 | + } |
170 | + |
171 | + pub fn remove_empty_accounts_lint(db: &mut Connection, dry_run: bool) -> Result<()> { |
172 | + let mut col = vec![]; |
173 | + { |
174 | + let mut stmt = db.connection.prepare( |
175 | + "SELECT * FROM account WHERE NOT EXISTS (SELECT 1 FROM subscription AS s WHERE \ |
176 | + s.address = address) ORDER BY pk", |
177 | + )?; |
178 | + let iter = stmt.query_map([], |row| { |
179 | + let pk = row.get("pk")?; |
180 | + Ok(DbVal( |
181 | + Account { |
182 | + pk, |
183 | + name: row.get("name")?, |
184 | + address: row.get("address")?, |
185 | + public_key: row.get("public_key")?, |
186 | + password: row.get("password")?, |
187 | + enabled: row.get("enabled")?, |
188 | + }, |
189 | + pk, |
190 | + )) |
191 | + })?; |
192 | + |
193 | + for entry in iter { |
194 | + let entry = entry?; |
195 | + col.push(entry); |
196 | + } |
197 | + } |
198 | + if col.is_empty() { |
199 | + println!("remove_empty_accounts: ok"); |
200 | + } else { |
201 | + let tx = if dry_run { |
202 | + None |
203 | + } else { |
204 | + Some(db.connection.transaction()?) |
205 | + }; |
206 | + println!("remove_empty_accounts: found {} entries", col.len()); |
207 | + println!("pk\tAddress"); |
208 | + for DbVal(Account { pk, address, .. }, _) in &col { |
209 | + println!("{pk}\t{address}"); |
210 | + } |
211 | + if let Some(tx) = tx { |
212 | + for DbVal(_, pk) in col { |
213 | + tx.execute("DELETE FROM account WHERE pk = ?", [pk])?; |
214 | + } |
215 | + tx.commit()?; |
216 | + } |
217 | + } |
218 | + Ok(()) |
219 | + } |
220 | + |
221 | + pub fn remove_accepted_subscription_requests_lint( |
222 | + db: &mut Connection, |
223 | + dry_run: bool, |
224 | + ) -> Result<()> { |
225 | + let mut col = vec![]; |
226 | + { |
227 | + let mut stmt = db.connection.prepare( |
228 | + "SELECT * FROM candidate_subscription WHERE accepted IS NOT NULL ORDER BY pk", |
229 | + )?; |
230 | + let iter = stmt.query_map([], |row| { |
231 | + let pk = row.get("pk")?; |
232 | + Ok(DbVal( |
233 | + ListSubscription { |
234 | + pk, |
235 | + list: row.get("list")?, |
236 | + address: row.get("address")?, |
237 | + account: row.get("account")?, |
238 | + name: row.get("name")?, |
239 | + digest: row.get("digest")?, |
240 | + enabled: row.get("enabled")?, |
241 | + verified: row.get("verified")?, |
242 | + hide_address: row.get("hide_address")?, |
243 | + receive_duplicates: row.get("receive_duplicates")?, |
244 | + receive_own_posts: row.get("receive_own_posts")?, |
245 | + receive_confirmation: row.get("receive_confirmation")?, |
246 | + }, |
247 | + pk, |
248 | + )) |
249 | + })?; |
250 | + |
251 | + for entry in iter { |
252 | + let entry = entry?; |
253 | + col.push(entry); |
254 | + } |
255 | + } |
256 | + if col.is_empty() { |
257 | + println!("remove_accepted_subscription_requests: ok"); |
258 | + } else { |
259 | + let tx = if dry_run { |
260 | + None |
261 | + } else { |
262 | + Some(db.connection.transaction()?) |
263 | + }; |
264 | + println!( |
265 | + "remove_accepted_subscription_requests: found {} entries", |
266 | + col.len() |
267 | + ); |
268 | + println!("pk\tAddress"); |
269 | + for DbVal(ListSubscription { pk, address, .. }, _) in &col { |
270 | + println!("{pk}\t{address}"); |
271 | + } |
272 | + if let Some(tx) = tx { |
273 | + for DbVal(_, pk) in col { |
274 | + tx.execute("DELETE FROM candidate_subscription WHERE pk = ?", [pk])?; |
275 | + } |
276 | + tx.commit()?; |
277 | + } |
278 | + } |
279 | + Ok(()) |
280 | + } |
281 | + |
282 | + pub fn warn_list_no_owner_lint(db: &mut Connection, _: bool) -> Result<()> { |
283 | + let mut stmt = db.connection.prepare( |
284 | + "SELECT * FROM list WHERE NOT EXISTS (SELECT 1 FROM owner AS o WHERE o.list = pk) ORDER \ |
285 | + BY pk", |
286 | + )?; |
287 | + let iter = stmt.query_map([], |row| { |
288 | + let pk = row.get("pk")?; |
289 | + Ok(DbVal( |
290 | + MailingList { |
291 | + pk, |
292 | + name: row.get("name")?, |
293 | + id: row.get("id")?, |
294 | + address: row.get("address")?, |
295 | + description: row.get("description")?, |
296 | + archive_url: row.get("archive_url")?, |
297 | + }, |
298 | + pk, |
299 | + )) |
300 | + })?; |
301 | + |
302 | + let mut col = vec![]; |
303 | + for entry in iter { |
304 | + let entry = entry?; |
305 | + col.push(entry); |
306 | + } |
307 | + if col.is_empty() { |
308 | + println!("warn_list_no_owner: ok"); |
309 | + } else { |
310 | + println!("warn_list_no_owner: found {} entries", col.len()); |
311 | + println!("pk\tName"); |
312 | + for DbVal(MailingList { pk, name, .. }, _) in col { |
313 | + println!("{pk}\t{name}"); |
314 | + } |
315 | + } |
316 | + Ok(()) |
317 | + } |
318 | diff --git a/cli/src/main.rs b/cli/src/main.rs |
319 | index b46049c..35fbca5 100644 |
320 | --- a/cli/src/main.rs |
321 | +++ b/cli/src/main.rs |
322 | @@ -24,6 +24,8 @@ use std::{ |
323 | process::Stdio, |
324 | }; |
325 | |
326 | + mod lints; |
327 | + use lints::*; |
328 | use mailpot::{ |
329 | melib::{backends::maildir::MaildirPathTrait, smol, Envelope, EnvelopeHash}, |
330 | models::{changesets::*, *}, |
331 | @@ -757,6 +759,52 @@ fn run_app(opt: Opt) -> Result<()> { |
332 | }; |
333 | db.update_account(changeset)?; |
334 | } |
335 | + Repair { |
336 | + fix, |
337 | + all, |
338 | + mut datetime_header_value, |
339 | + mut remove_empty_accounts, |
340 | + mut remove_accepted_subscription_requests, |
341 | + mut warn_list_no_owner, |
342 | + } => { |
343 | + type LintFn = |
344 | + fn(&'_ mut mailpot::Connection, bool) -> std::result::Result<(), mailpot::Error>; |
345 | + let dry_run = !fix; |
346 | + if all { |
347 | + datetime_header_value = true; |
348 | + remove_empty_accounts = true; |
349 | + remove_accepted_subscription_requests = true; |
350 | + warn_list_no_owner = true; |
351 | + } |
352 | + |
353 | + if !(datetime_header_value |
354 | + | remove_empty_accounts |
355 | + | remove_accepted_subscription_requests |
356 | + | warn_list_no_owner) |
357 | + { |
358 | + return Err( |
359 | + "No lints selected: specify them with flag arguments. See --help".into(), |
360 | + ); |
361 | + } |
362 | + |
363 | + if dry_run { |
364 | + println!("running without making modifications (dry run)"); |
365 | + } |
366 | + |
367 | + for (flag, lint_fn) in [ |
368 | + (datetime_header_value, datetime_header_value_lint as LintFn), |
369 | + (remove_empty_accounts, remove_empty_accounts_lint as _), |
370 | + ( |
371 | + remove_accepted_subscription_requests, |
372 | + remove_accepted_subscription_requests_lint as _, |
373 | + ), |
374 | + (warn_list_no_owner, warn_list_no_owner_lint as _), |
375 | + ] { |
376 | + if flag { |
377 | + lint_fn(&mut db, dry_run)?; |
378 | + } |
379 | + } |
380 | + } |
381 | } |
382 | |
383 | Ok(()) |
384 | @@ -773,7 +821,7 @@ fn main() -> std::result::Result<(), i32> { |
385 | .init() |
386 | .unwrap(); |
387 | if let Err(err) = run_app(opt) { |
388 | - println!("{}", err.display_chain()); |
389 | + print!("{}", err.display_chain()); |
390 | std::process::exit(-1); |
391 | } |
392 | Ok(()) |
393 | diff --git a/core/Cargo.toml b/core/Cargo.toml |
394 | index 127759f..6eb0d07 100644 |
395 | --- a/core/Cargo.toml |
396 | +++ b/core/Cargo.toml |
397 | @@ -17,7 +17,7 @@ error-chain = { version = "0.12.4", default-features = false } |
398 | log = "0.4" |
399 | melib = { version = "*", default-features = false, features = ["smtp", "unicode_algorithms", "maildir_backend"], git = "https://github.com/meli/meli", rev = "2447a2c" } |
400 | minijinja = { version = "0.31.0", features = ["source", ] } |
401 | - rusqlite = { version = "^0.28", features = ["bundled", "trace", "hooks", "serde_json", "array"] } |
402 | + rusqlite = { version = "^0.28", features = ["bundled", "trace", "hooks", "serde_json", "array", "chrono"] } |
403 | serde = { version = "^1", features = ["derive", ] } |
404 | serde_json = "^1" |
405 | toml = "^0.5" |
406 | diff --git a/core/src/db/posts.rs b/core/src/db/posts.rs |
407 | index 7b8cb59..ee733ed 100644 |
408 | --- a/core/src/db/posts.rs |
409 | +++ b/core/src/db/posts.rs |
410 | @@ -31,15 +31,15 @@ impl Connection { |
411 | } else { |
412 | from_[0].get_email() |
413 | }; |
414 | - let datetime: std::borrow::Cow<'_, str> = if env.timestamp != 0 { |
415 | + let datetime: std::borrow::Cow<'_, str> = if !env.date.as_str().is_empty() { |
416 | + env.date.as_str().into() |
417 | + } else { |
418 | melib::datetime::timestamp_to_string( |
419 | env.timestamp, |
420 | - Some(melib::datetime::RFC3339_FMT_WITH_TIME), |
421 | + Some(melib::datetime::RFC822_DATE), |
422 | true, |
423 | ) |
424 | .into() |
425 | - } else { |
426 | - env.date.as_str().into() |
427 | }; |
428 | let message_id = env.message_id_display(); |
429 | let mut stmt = self.connection.prepare( |
430 | diff --git a/core/src/db/queue.rs b/core/src/db/queue.rs |
431 | index 392bf45..97faafb 100644 |
432 | --- a/core/src/db/queue.rs |
433 | +++ b/core/src/db/queue.rs |
434 | @@ -87,7 +87,7 @@ pub struct QueueEntry { |
435 | /// Unix timestamp of date. |
436 | pub timestamp: u64, |
437 | /// Datetime as string. |
438 | - pub datetime: String, |
439 | + pub datetime: DateTime, |
440 | } |
441 | |
442 | impl std::fmt::Display for QueueEntry { |
443 | @@ -142,7 +142,7 @@ impl QueueEntry { |
444 | message_id: env.message_id().to_string(), |
445 | message: raw.to_vec(), |
446 | timestamp: now.timestamp() as u64, |
447 | - datetime: now.to_string(), |
448 | + datetime: now, |
449 | }) |
450 | } |
451 | } |
452 | @@ -197,7 +197,7 @@ impl Connection { |
453 | message_id: row.get::<_, String>("message_id")?, |
454 | message: row.get::<_, Vec<u8>>("message")?, |
455 | timestamp: row.get::<_, u64>("timestamp")?, |
456 | - datetime: row.get::<_, String>("datetime")?, |
457 | + datetime: row.get::<_, DateTime>("datetime")?, |
458 | }, |
459 | pk, |
460 | )) |
461 | @@ -227,7 +227,7 @@ impl Connection { |
462 | message_id: row.get::<_, String>("message_id")?, |
463 | message: row.get::<_, Vec<u8>>("message")?, |
464 | timestamp: row.get::<_, u64>("timestamp")?, |
465 | - datetime: row.get::<_, String>("datetime")?, |
466 | + datetime: row.get::<_, DateTime>("datetime")?, |
467 | }) |
468 | }; |
469 | let mut stmt = if index.is_empty() { |
470 | diff --git a/core/src/lib.rs b/core/src/lib.rs |
471 | index 0139c2a..d0caca0 100644 |
472 | --- a/core/src/lib.rs |
473 | +++ b/core/src/lib.rs |
474 | @@ -153,8 +153,12 @@ |
475 | #[macro_use] |
476 | extern crate error_chain; |
477 | pub extern crate anyhow; |
478 | + pub extern crate chrono; |
479 | pub extern crate rusqlite; |
480 | |
481 | + /// Alias for [`chrono::DateTime<chrono::Utc>`]. |
482 | + pub type DateTime = chrono::DateTime<chrono::Utc>; |
483 | + |
484 | #[macro_use] |
485 | pub extern crate serde; |
486 | pub extern crate log; |
487 | diff --git a/core/src/models.rs b/core/src/models.rs |
488 | index 9cdcfc7..d743829 100644 |
489 | --- a/core/src/models.rs |
490 | +++ b/core/src/models.rs |
491 | @@ -473,7 +473,7 @@ pub struct Post { |
492 | pub message: Vec<u8>, |
493 | /// Unix timestamp of date. |
494 | pub timestamp: u64, |
495 | - /// Datetime as string. |
496 | + /// Date header as string. |
497 | pub datetime: String, |
498 | /// Month-year as a `YYYY-mm` formatted string, for use in archives. |
499 | pub month_year: String, |
500 | diff --git a/docs/mpot.1 b/docs/mpot.1 |
501 | index 18f4b91..834545d 100644 |
502 | --- a/docs/mpot.1 |
503 | +++ b/docs/mpot.1 |
504 | @@ -891,5 +891,36 @@ Is account enabled. |
505 | [\fIpossible values: \fRtrue, false] |
506 | .ie \n(.g .ds Aq \(aq |
507 | .el .ds Aq ' |
508 | + .\fB |
509 | + .SS mpot repair |
510 | + .\fR |
511 | + .br |
512 | + |
513 | + .br |
514 | + |
515 | + mpot repair [\-\-fix \fIFIX\fR] [\-\-all \fIALL\fR] [\-\-datetime\-header\-value \fIDATETIME_HEADER_VALUE\fR] [\-\-remove\-empty\-accounts \fIREMOVE_EMPTY_ACCOUNTS\fR] [\-\-remove\-accepted\-subscription\-requests \fIREMOVE_ACCEPTED_SUBSCRIPTION_REQUESTS\fR] [\-\-warn\-list\-no\-owner \fIWARN_LIST_NO_OWNER\fR] |
516 | + .br |
517 | + |
518 | + Show and fix possible data mistakes or inconsistencies. |
519 | + .TP |
520 | + \-\-fix |
521 | + Fix errors (default: false). |
522 | + .TP |
523 | + \-\-all |
524 | + Select all tests (default: false). |
525 | + .TP |
526 | + \-\-datetime\-header\-value |
527 | + Post `datetime` column must have the Date: header value, in RFC2822 format. |
528 | + .TP |
529 | + \-\-remove\-empty\-accounts |
530 | + Remove accounts that have no matching subscriptions. |
531 | + .TP |
532 | + \-\-remove\-accepted\-subscription\-requests |
533 | + Remove subscription requests that have been accepted. |
534 | + .TP |
535 | + \-\-warn\-list\-no\-owner |
536 | + Warn if a list has no owners. |
537 | + .ie \n(.g .ds Aq \(aq |
538 | + .el .ds Aq ' |
539 | .SH AUTHORS |
540 | Manos Pitsidianakis <el13635@mail.ntua.gr> |
541 | diff --git a/web/src/lists.rs b/web/src/lists.rs |
542 | index 9e38d16..f148ab4 100644 |
543 | --- a/web/src/lists.rs |
544 | +++ b/web/src/lists.rs |
545 | @@ -73,7 +73,11 @@ pub async fn list( |
546 | .map(|(thread, length, _timestamp)| { |
547 | let post = &post_map[&thread.message_id.as_str()]; |
548 | //2019-07-14T14:21:02 |
549 | - if let Some(day) = post.datetime.get(8..10).and_then(|d| d.parse::<u64>().ok()) { |
550 | + if let Some(day) = |
551 | + chrono::DateTime::<chrono::FixedOffset>::parse_from_rfc2822(post.datetime.trim()) |
552 | + .ok() |
553 | + .map(|d| d.day()) |
554 | + { |
555 | hist.get_mut(&post.month_year).unwrap()[day.saturating_sub(1) as usize] += 1; |
556 | } |
557 | let envelope = melib::Envelope::from_bytes(post.message.as_slice(), None) |