Author: Manos Pitsidianakis [manos@pitsidianak.is]
Hash: a37851b1082b88fc57611f79fe09064b46dbb17e
Timestamp: Sat, 29 Apr 2023 14:14:31 +0000 (1 year ago)

+380 -12 +/-12 browse
cli: add repair command with some lints
1diff --git a/Cargo.lock b/Cargo.lock
2index 03fb72b..8a8d7b4 100644
3--- a/Cargo.lock
4+++ b/Cargo.lock
5 @@ -2530,6 +2530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
6 checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
7 dependencies = [
8 "bitflags",
9+ "chrono",
10 "fallible-iterator",
11 "fallible-streaming-iterator",
12 "hashlink",
13 diff --git a/Cargo.toml b/Cargo.toml
14index ffa3357..5632c20 100644
15--- a/Cargo.toml
16+++ b/Cargo.toml
17 @@ -7,3 +7,9 @@ members = [
18 "rest-http",
19 "web",
20 ]
21+
22+ [profile.release]
23+ lto = "fat"
24+ opt-level = "z"
25+ codegen-units = 1
26+ split-debuginfo = "unpacked"
27 diff --git a/cli/src/lib.rs b/cli/src/lib.rs
28index 67c6de3..3d4dc9a 100644
29--- a/cli/src/lib.rs
30+++ b/cli/src/lib.rs
31 @@ -185,6 +185,28 @@ pub enum Command {
32 /// Is account enabled.
33 enabled: Option<Option<bool>>,
34 },
35+ /// Show and fix possible data mistakes or inconsistencies.
36+ Repair {
37+ /// Fix errors (default: false)
38+ #[arg(long, default_value = "false")]
39+ fix: bool,
40+ /// Select all tests (default: false)
41+ #[arg(long, default_value = "false")]
42+ all: bool,
43+ /// Post `datetime` column must have the Date: header value, in RFC2822
44+ /// format.
45+ #[arg(long, default_value = "false")]
46+ datetime_header_value: bool,
47+ /// Remove accounts that have no matching subscriptions.
48+ #[arg(long, default_value = "false")]
49+ remove_empty_accounts: bool,
50+ /// Remove subscription requests that have been accepted.
51+ #[arg(long, default_value = "false")]
52+ remove_accepted_subscription_requests: bool,
53+ /// Warn if a list has no owners.
54+ #[arg(long, default_value = "false")]
55+ warn_list_no_owner: bool,
56+ },
57 }
58
59 /// Postfix config values.
60 diff --git a/cli/src/lints.rs b/cli/src/lints.rs
61new file mode 100644
62index 0000000..68b118f
63--- /dev/null
64+++ b/cli/src/lints.rs
65 @@ -0,0 +1,252 @@
66+ /*
67+ * This file is part of mailpot
68+ *
69+ * Copyright 2020 - Manos Pitsidianakis
70+ *
71+ * This program is free software: you can redistribute it and/or modify
72+ * it under the terms of the GNU Affero General Public License as
73+ * published by the Free Software Foundation, either version 3 of the
74+ * License, or (at your option) any later version.
75+ *
76+ * This program is distributed in the hope that it will be useful,
77+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
78+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
79+ * GNU Affero General Public License for more details.
80+ *
81+ * You should have received a copy of the GNU Affero General Public License
82+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
83+ */
84+
85+ use super::*;
86+
87+ pub fn datetime_header_value_lint(db: &mut Connection, dry_run: bool) -> Result<()> {
88+ let mut col = vec![];
89+ {
90+ let mut stmt = db.connection.prepare("SELECT * FROM post ORDER BY pk")?;
91+ let iter = stmt.query_map([], |row| {
92+ let pk: i64 = row.get("pk")?;
93+ let date_s: String = row.get("datetime")?;
94+ match melib::datetime::rfc822_to_timestamp(date_s.trim()) {
95+ Err(_) | Ok(0) => {
96+ let mut timestamp: i64 = row.get("timestamp")?;
97+ let created: i64 = row.get("created")?;
98+ if timestamp == 0 {
99+ timestamp = created;
100+ }
101+ timestamp = std::cmp::min(timestamp, created);
102+ let timestamp = if timestamp <= 0 {
103+ None
104+ } else {
105+ // safe because we checked it's not negative or zero above.
106+ Some(timestamp as u64)
107+ };
108+ let message: Vec<u8> = row.get("message")?;
109+ Ok(Some((pk, date_s, message, timestamp)))
110+ }
111+ Ok(_) => Ok(None),
112+ }
113+ })?;
114+
115+ for entry in iter {
116+ if let Some(s) = entry? {
117+ col.push(s);
118+ }
119+ }
120+ }
121+ let mut failures = 0;
122+ let tx = if dry_run {
123+ None
124+ } else {
125+ Some(db.connection.transaction()?)
126+ };
127+ if col.is_empty() {
128+ println!("datetime_header_value: ok");
129+ } else {
130+ println!("datetime_header_value: found {} entries", col.len());
131+ println!("pk\tDate value\tshould be");
132+ for (pk, val, message, timestamp) in col {
133+ let correct = if let Ok(v) =
134+ chrono::DateTime::<chrono::FixedOffset>::parse_from_rfc3339(&val)
135+ {
136+ v.to_rfc2822()
137+ } else if let Some(v) = timestamp.map(|t| {
138+ melib::datetime::timestamp_to_string(t, Some(melib::datetime::RFC822_DATE), true)
139+ }) {
140+ v
141+ } else if let Ok(v) =
142+ Envelope::from_bytes(&message, None).map(|env| env.date_as_str().to_string())
143+ {
144+ v
145+ } else {
146+ failures += 1;
147+ println!("{pk}\t{val}\tCould not find any valid date value in the post metadata!");
148+ continue;
149+ };
150+ println!("{pk}\t{val}\t{correct}");
151+ if let Some(tx) = tx.as_ref() {
152+ tx.execute(
153+ "UPDATE post SET datetime = ? WHERE pk = ?",
154+ rusqlite::params![&correct, pk],
155+ )?;
156+ }
157+ }
158+ }
159+ if let Some(tx) = tx {
160+ tx.commit()?;
161+ }
162+ if failures > 0 {
163+ println!(
164+ "datetime_header_value: {failures} failure{}",
165+ if failures == 1 { "" } else { "s" }
166+ );
167+ }
168+ Ok(())
169+ }
170+
171+ pub fn remove_empty_accounts_lint(db: &mut Connection, dry_run: bool) -> Result<()> {
172+ let mut col = vec![];
173+ {
174+ let mut stmt = db.connection.prepare(
175+ "SELECT * FROM account WHERE NOT EXISTS (SELECT 1 FROM subscription AS s WHERE \
176+ s.address = address) ORDER BY pk",
177+ )?;
178+ let iter = stmt.query_map([], |row| {
179+ let pk = row.get("pk")?;
180+ Ok(DbVal(
181+ Account {
182+ pk,
183+ name: row.get("name")?,
184+ address: row.get("address")?,
185+ public_key: row.get("public_key")?,
186+ password: row.get("password")?,
187+ enabled: row.get("enabled")?,
188+ },
189+ pk,
190+ ))
191+ })?;
192+
193+ for entry in iter {
194+ let entry = entry?;
195+ col.push(entry);
196+ }
197+ }
198+ if col.is_empty() {
199+ println!("remove_empty_accounts: ok");
200+ } else {
201+ let tx = if dry_run {
202+ None
203+ } else {
204+ Some(db.connection.transaction()?)
205+ };
206+ println!("remove_empty_accounts: found {} entries", col.len());
207+ println!("pk\tAddress");
208+ for DbVal(Account { pk, address, .. }, _) in &col {
209+ println!("{pk}\t{address}");
210+ }
211+ if let Some(tx) = tx {
212+ for DbVal(_, pk) in col {
213+ tx.execute("DELETE FROM account WHERE pk = ?", [pk])?;
214+ }
215+ tx.commit()?;
216+ }
217+ }
218+ Ok(())
219+ }
220+
221+ pub fn remove_accepted_subscription_requests_lint(
222+ db: &mut Connection,
223+ dry_run: bool,
224+ ) -> Result<()> {
225+ let mut col = vec![];
226+ {
227+ let mut stmt = db.connection.prepare(
228+ "SELECT * FROM candidate_subscription WHERE accepted IS NOT NULL ORDER BY pk",
229+ )?;
230+ let iter = stmt.query_map([], |row| {
231+ let pk = row.get("pk")?;
232+ Ok(DbVal(
233+ ListSubscription {
234+ pk,
235+ list: row.get("list")?,
236+ address: row.get("address")?,
237+ account: row.get("account")?,
238+ name: row.get("name")?,
239+ digest: row.get("digest")?,
240+ enabled: row.get("enabled")?,
241+ verified: row.get("verified")?,
242+ hide_address: row.get("hide_address")?,
243+ receive_duplicates: row.get("receive_duplicates")?,
244+ receive_own_posts: row.get("receive_own_posts")?,
245+ receive_confirmation: row.get("receive_confirmation")?,
246+ },
247+ pk,
248+ ))
249+ })?;
250+
251+ for entry in iter {
252+ let entry = entry?;
253+ col.push(entry);
254+ }
255+ }
256+ if col.is_empty() {
257+ println!("remove_accepted_subscription_requests: ok");
258+ } else {
259+ let tx = if dry_run {
260+ None
261+ } else {
262+ Some(db.connection.transaction()?)
263+ };
264+ println!(
265+ "remove_accepted_subscription_requests: found {} entries",
266+ col.len()
267+ );
268+ println!("pk\tAddress");
269+ for DbVal(ListSubscription { pk, address, .. }, _) in &col {
270+ println!("{pk}\t{address}");
271+ }
272+ if let Some(tx) = tx {
273+ for DbVal(_, pk) in col {
274+ tx.execute("DELETE FROM candidate_subscription WHERE pk = ?", [pk])?;
275+ }
276+ tx.commit()?;
277+ }
278+ }
279+ Ok(())
280+ }
281+
282+ pub fn warn_list_no_owner_lint(db: &mut Connection, _: bool) -> Result<()> {
283+ let mut stmt = db.connection.prepare(
284+ "SELECT * FROM list WHERE NOT EXISTS (SELECT 1 FROM owner AS o WHERE o.list = pk) ORDER \
285+ BY pk",
286+ )?;
287+ let iter = stmt.query_map([], |row| {
288+ let pk = row.get("pk")?;
289+ Ok(DbVal(
290+ MailingList {
291+ pk,
292+ name: row.get("name")?,
293+ id: row.get("id")?,
294+ address: row.get("address")?,
295+ description: row.get("description")?,
296+ archive_url: row.get("archive_url")?,
297+ },
298+ pk,
299+ ))
300+ })?;
301+
302+ let mut col = vec![];
303+ for entry in iter {
304+ let entry = entry?;
305+ col.push(entry);
306+ }
307+ if col.is_empty() {
308+ println!("warn_list_no_owner: ok");
309+ } else {
310+ println!("warn_list_no_owner: found {} entries", col.len());
311+ println!("pk\tName");
312+ for DbVal(MailingList { pk, name, .. }, _) in col {
313+ println!("{pk}\t{name}");
314+ }
315+ }
316+ Ok(())
317+ }
318 diff --git a/cli/src/main.rs b/cli/src/main.rs
319index b46049c..35fbca5 100644
320--- a/cli/src/main.rs
321+++ b/cli/src/main.rs
322 @@ -24,6 +24,8 @@ use std::{
323 process::Stdio,
324 };
325
326+ mod lints;
327+ use lints::*;
328 use mailpot::{
329 melib::{backends::maildir::MaildirPathTrait, smol, Envelope, EnvelopeHash},
330 models::{changesets::*, *},
331 @@ -757,6 +759,52 @@ fn run_app(opt: Opt) -> Result<()> {
332 };
333 db.update_account(changeset)?;
334 }
335+ Repair {
336+ fix,
337+ all,
338+ mut datetime_header_value,
339+ mut remove_empty_accounts,
340+ mut remove_accepted_subscription_requests,
341+ mut warn_list_no_owner,
342+ } => {
343+ type LintFn =
344+ fn(&'_ mut mailpot::Connection, bool) -> std::result::Result<(), mailpot::Error>;
345+ let dry_run = !fix;
346+ if all {
347+ datetime_header_value = true;
348+ remove_empty_accounts = true;
349+ remove_accepted_subscription_requests = true;
350+ warn_list_no_owner = true;
351+ }
352+
353+ if !(datetime_header_value
354+ | remove_empty_accounts
355+ | remove_accepted_subscription_requests
356+ | warn_list_no_owner)
357+ {
358+ return Err(
359+ "No lints selected: specify them with flag arguments. See --help".into(),
360+ );
361+ }
362+
363+ if dry_run {
364+ println!("running without making modifications (dry run)");
365+ }
366+
367+ for (flag, lint_fn) in [
368+ (datetime_header_value, datetime_header_value_lint as LintFn),
369+ (remove_empty_accounts, remove_empty_accounts_lint as _),
370+ (
371+ remove_accepted_subscription_requests,
372+ remove_accepted_subscription_requests_lint as _,
373+ ),
374+ (warn_list_no_owner, warn_list_no_owner_lint as _),
375+ ] {
376+ if flag {
377+ lint_fn(&mut db, dry_run)?;
378+ }
379+ }
380+ }
381 }
382
383 Ok(())
384 @@ -773,7 +821,7 @@ fn main() -> std::result::Result<(), i32> {
385 .init()
386 .unwrap();
387 if let Err(err) = run_app(opt) {
388- println!("{}", err.display_chain());
389+ print!("{}", err.display_chain());
390 std::process::exit(-1);
391 }
392 Ok(())
393 diff --git a/core/Cargo.toml b/core/Cargo.toml
394index 127759f..6eb0d07 100644
395--- a/core/Cargo.toml
396+++ b/core/Cargo.toml
397 @@ -17,7 +17,7 @@ error-chain = { version = "0.12.4", default-features = false }
398 log = "0.4"
399 melib = { version = "*", default-features = false, features = ["smtp", "unicode_algorithms", "maildir_backend"], git = "https://github.com/meli/meli", rev = "2447a2c" }
400 minijinja = { version = "0.31.0", features = ["source", ] }
401- rusqlite = { version = "^0.28", features = ["bundled", "trace", "hooks", "serde_json", "array"] }
402+ rusqlite = { version = "^0.28", features = ["bundled", "trace", "hooks", "serde_json", "array", "chrono"] }
403 serde = { version = "^1", features = ["derive", ] }
404 serde_json = "^1"
405 toml = "^0.5"
406 diff --git a/core/src/db/posts.rs b/core/src/db/posts.rs
407index 7b8cb59..ee733ed 100644
408--- a/core/src/db/posts.rs
409+++ b/core/src/db/posts.rs
410 @@ -31,15 +31,15 @@ impl Connection {
411 } else {
412 from_[0].get_email()
413 };
414- let datetime: std::borrow::Cow<'_, str> = if env.timestamp != 0 {
415+ let datetime: std::borrow::Cow<'_, str> = if !env.date.as_str().is_empty() {
416+ env.date.as_str().into()
417+ } else {
418 melib::datetime::timestamp_to_string(
419 env.timestamp,
420- Some(melib::datetime::RFC3339_FMT_WITH_TIME),
421+ Some(melib::datetime::RFC822_DATE),
422 true,
423 )
424 .into()
425- } else {
426- env.date.as_str().into()
427 };
428 let message_id = env.message_id_display();
429 let mut stmt = self.connection.prepare(
430 diff --git a/core/src/db/queue.rs b/core/src/db/queue.rs
431index 392bf45..97faafb 100644
432--- a/core/src/db/queue.rs
433+++ b/core/src/db/queue.rs
434 @@ -87,7 +87,7 @@ pub struct QueueEntry {
435 /// Unix timestamp of date.
436 pub timestamp: u64,
437 /// Datetime as string.
438- pub datetime: String,
439+ pub datetime: DateTime,
440 }
441
442 impl std::fmt::Display for QueueEntry {
443 @@ -142,7 +142,7 @@ impl QueueEntry {
444 message_id: env.message_id().to_string(),
445 message: raw.to_vec(),
446 timestamp: now.timestamp() as u64,
447- datetime: now.to_string(),
448+ datetime: now,
449 })
450 }
451 }
452 @@ -197,7 +197,7 @@ impl Connection {
453 message_id: row.get::<_, String>("message_id")?,
454 message: row.get::<_, Vec<u8>>("message")?,
455 timestamp: row.get::<_, u64>("timestamp")?,
456- datetime: row.get::<_, String>("datetime")?,
457+ datetime: row.get::<_, DateTime>("datetime")?,
458 },
459 pk,
460 ))
461 @@ -227,7 +227,7 @@ impl Connection {
462 message_id: row.get::<_, String>("message_id")?,
463 message: row.get::<_, Vec<u8>>("message")?,
464 timestamp: row.get::<_, u64>("timestamp")?,
465- datetime: row.get::<_, String>("datetime")?,
466+ datetime: row.get::<_, DateTime>("datetime")?,
467 })
468 };
469 let mut stmt = if index.is_empty() {
470 diff --git a/core/src/lib.rs b/core/src/lib.rs
471index 0139c2a..d0caca0 100644
472--- a/core/src/lib.rs
473+++ b/core/src/lib.rs
474 @@ -153,8 +153,12 @@
475 #[macro_use]
476 extern crate error_chain;
477 pub extern crate anyhow;
478+ pub extern crate chrono;
479 pub extern crate rusqlite;
480
481+ /// Alias for [`chrono::DateTime<chrono::Utc>`].
482+ pub type DateTime = chrono::DateTime<chrono::Utc>;
483+
484 #[macro_use]
485 pub extern crate serde;
486 pub extern crate log;
487 diff --git a/core/src/models.rs b/core/src/models.rs
488index 9cdcfc7..d743829 100644
489--- a/core/src/models.rs
490+++ b/core/src/models.rs
491 @@ -473,7 +473,7 @@ pub struct Post {
492 pub message: Vec<u8>,
493 /// Unix timestamp of date.
494 pub timestamp: u64,
495- /// Datetime as string.
496+ /// Date header as string.
497 pub datetime: String,
498 /// Month-year as a `YYYY-mm` formatted string, for use in archives.
499 pub month_year: String,
500 diff --git a/docs/mpot.1 b/docs/mpot.1
501index 18f4b91..834545d 100644
502--- a/docs/mpot.1
503+++ b/docs/mpot.1
504 @@ -891,5 +891,36 @@ Is account enabled.
505 [\fIpossible values: \fRtrue, false]
506 .ie \n(.g .ds Aq \(aq
507 .el .ds Aq '
508+ .\fB
509+ .SS mpot repair
510+ .\fR
511+ .br
512+
513+ .br
514+
515+ mpot repair [\-\-fix \fIFIX\fR] [\-\-all \fIALL\fR] [\-\-datetime\-header\-value \fIDATETIME_HEADER_VALUE\fR] [\-\-remove\-empty\-accounts \fIREMOVE_EMPTY_ACCOUNTS\fR] [\-\-remove\-accepted\-subscription\-requests \fIREMOVE_ACCEPTED_SUBSCRIPTION_REQUESTS\fR] [\-\-warn\-list\-no\-owner \fIWARN_LIST_NO_OWNER\fR]
516+ .br
517+
518+ Show and fix possible data mistakes or inconsistencies.
519+ .TP
520+ \-\-fix
521+ Fix errors (default: false).
522+ .TP
523+ \-\-all
524+ Select all tests (default: false).
525+ .TP
526+ \-\-datetime\-header\-value
527+ Post `datetime` column must have the Date: header value, in RFC2822 format.
528+ .TP
529+ \-\-remove\-empty\-accounts
530+ Remove accounts that have no matching subscriptions.
531+ .TP
532+ \-\-remove\-accepted\-subscription\-requests
533+ Remove subscription requests that have been accepted.
534+ .TP
535+ \-\-warn\-list\-no\-owner
536+ Warn if a list has no owners.
537+ .ie \n(.g .ds Aq \(aq
538+ .el .ds Aq '
539 .SH AUTHORS
540 Manos Pitsidianakis <el13635@mail.ntua.gr>
541 diff --git a/web/src/lists.rs b/web/src/lists.rs
542index 9e38d16..f148ab4 100644
543--- a/web/src/lists.rs
544+++ b/web/src/lists.rs
545 @@ -73,7 +73,11 @@ pub async fn list(
546 .map(|(thread, length, _timestamp)| {
547 let post = &post_map[&thread.message_id.as_str()];
548 //2019-07-14T14:21:02
549- if let Some(day) = post.datetime.get(8..10).and_then(|d| d.parse::<u64>().ok()) {
550+ if let Some(day) =
551+ chrono::DateTime::<chrono::FixedOffset>::parse_from_rfc2822(post.datetime.trim())
552+ .ok()
553+ .map(|d| d.day())
554+ {
555 hist.get_mut(&post.month_year).unwrap()[day.saturating_sub(1) as usize] += 1;
556 }
557 let envelope = melib::Envelope::from_bytes(post.message.as_slice(), None)