Author: Manos Pitsidianakis [manos@pitsidianak.is]
Hash: 657b58c4aed4cc89c4a1b6613ed955d4a4b70fcb
Timestamp: Thu, 18 May 2023 10:57:51 +0000 (1 year ago)

+412 -72 +/-5 browse
core: add migration test
1diff --git a/cli/src/args.rs b/cli/src/args.rs
2index 5cc26e8..98dc512 100644
3--- a/cli/src/args.rs
4+++ b/cli/src/args.rs
5 @@ -506,7 +506,6 @@ pub enum ListCommand {
6 pub struct QueueValueParser;
7
8 impl QueueValueParser {
9- /// Implementation for [`ValueParser::path_buf`]
10 pub fn new() -> Self {
11 Self
12 }
13 diff --git a/core/build.rs b/core/build.rs
14index 23a9bcd..c902648 100644
15--- a/core/build.rs
16+++ b/core/build.rs
17 @@ -18,10 +18,8 @@
18 */
19
20 use std::{
21- fs::{metadata, read_dir, OpenOptions},
22+ fs::{metadata, OpenOptions},
23 io,
24- io::Write,
25- path::Path,
26 process::{Command, Stdio},
27 };
28
29 @@ -46,7 +44,12 @@ where
30 }
31 }
32
33+ include!("make_migrations.rs");
34+
35+ const MIGRATION_RS: &str = "src/migrations.rs.inc";
36+
37 fn main() {
38+ println!("cargo:rerun-if-changed=src/migrations.rs.inc");
39 println!("cargo:rerun-if-changed=migrations");
40 println!("cargo:rerun-if-changed=src/schema.sql.m4");
41
42 @@ -88,71 +91,5 @@ fn main() {
43 file.write_all(&output.stdout).unwrap();
44 }
45
46- const MIGRATION_RS: &str = "src/migrations.rs.inc";
47-
48- let mut regen = false;
49- let mut paths = vec![];
50- let mut undo_paths = vec![];
51- for entry in read_dir("migrations").unwrap() {
52- let entry = entry.unwrap();
53- let path = entry.path();
54- if path.is_dir() || path.extension().map(|os| os.to_str().unwrap()) != Some("sql") {
55- continue;
56- }
57- if is_output_file_outdated(&path, MIGRATION_RS).unwrap() {
58- regen = true;
59- }
60- if path
61- .file_name()
62- .unwrap()
63- .to_str()
64- .unwrap()
65- .ends_with("undo.sql")
66- {
67- undo_paths.push(path);
68- } else {
69- paths.push(path);
70- }
71- }
72-
73- if regen {
74- paths.sort();
75- undo_paths.sort();
76- let mut migr_rs = OpenOptions::new()
77- .write(true)
78- .create(true)
79- .truncate(true)
80- .open(MIGRATION_RS)
81- .unwrap();
82- migr_rs
83- .write_all(b"\n//(user_version, redo sql, undo sql\n&[")
84- .unwrap();
85- for (p, u) in paths.iter().zip(undo_paths.iter()) {
86- // This should be a number string, padded with 2 zeros if it's less than 3
87- // digits. e.g. 001, \d{3}
88- let num = p.file_stem().unwrap().to_str().unwrap();
89- if !u.file_name().unwrap().to_str().unwrap().starts_with(num) {
90- panic!("Undo file {u:?} should match with {p:?}");
91- }
92- if num.parse::<u32>().is_err() {
93- panic!("Migration file {p:?} should start with a number");
94- }
95- migr_rs.write_all(b"(").unwrap();
96- migr_rs
97- .write_all(num.trim_start_matches('0').as_bytes())
98- .unwrap();
99- migr_rs.write_all(b",\"").unwrap();
100-
101- migr_rs
102- .write_all(std::fs::read_to_string(p).unwrap().as_bytes())
103- .unwrap();
104- migr_rs.write_all(b"\",\"").unwrap();
105- migr_rs
106- .write_all(std::fs::read_to_string(u).unwrap().as_bytes())
107- .unwrap();
108- migr_rs.write_all(b"\"),").unwrap();
109- }
110- migr_rs.write_all(b"]").unwrap();
111- migr_rs.flush().unwrap();
112- }
113+ make_migrations("migrations", MIGRATION_RS);
114 }
115 diff --git a/core/make_migrations.rs b/core/make_migrations.rs
116new file mode 100644
117index 0000000..8cf372d
118--- /dev/null
119+++ b/core/make_migrations.rs
120 @@ -0,0 +1,92 @@
121+ /*
122+ * This file is part of mailpot
123+ *
124+ * Copyright 2023 - Manos Pitsidianakis
125+ *
126+ * This program is free software: you can redistribute it and/or modify
127+ * it under the terms of the GNU Affero General Public License as
128+ * published by the Free Software Foundation, either version 3 of the
129+ * License, or (at your option) any later version.
130+ *
131+ * This program is distributed in the hope that it will be useful,
132+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
133+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
134+ * GNU Affero General Public License for more details.
135+ *
136+ * You should have received a copy of the GNU Affero General Public License
137+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
138+ */
139+
140+ use std::{fs::read_dir, io::Write, path::Path};
141+
142+ pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, output_file: O) {
143+ let migrations_folder_path = migrations_path.as_ref();
144+ let output_file_path = output_file.as_ref();
145+
146+ let mut regen = false;
147+ let mut paths = vec![];
148+ let mut undo_paths = vec![];
149+ for entry in read_dir(migrations_folder_path).unwrap() {
150+ let entry = entry.unwrap();
151+ let path = entry.path();
152+ if path.is_dir() || path.extension().map(|os| os.to_str().unwrap()) != Some("sql") {
153+ continue;
154+ }
155+ if is_output_file_outdated(&path, output_file_path).unwrap() {
156+ regen = true;
157+ }
158+ if path
159+ .file_name()
160+ .unwrap()
161+ .to_str()
162+ .unwrap()
163+ .ends_with("undo.sql")
164+ {
165+ undo_paths.push(path);
166+ } else {
167+ paths.push(path);
168+ }
169+ }
170+
171+ if regen {
172+ paths.sort();
173+ undo_paths.sort();
174+ let mut migr_rs = OpenOptions::new()
175+ .write(true)
176+ .create(true)
177+ .truncate(true)
178+ .open(output_file_path)
179+ .unwrap();
180+ migr_rs
181+ .write_all(b"\n//(user_version, redo sql, undo sql\n&[")
182+ .unwrap();
183+ for (i, (p, u)) in paths.iter().zip(undo_paths.iter()).enumerate() {
184+ // This should be a number string, padded with 2 zeros if it's less than 3
185+ // digits. e.g. 001, \d{3}
186+ let num = p.file_stem().unwrap().to_str().unwrap();
187+ if !u.file_name().unwrap().to_str().unwrap().starts_with(num) {
188+ panic!("Undo file {u:?} should match with {p:?}");
189+ }
190+ if num.parse::<u32>().is_err() {
191+ panic!("Migration file {p:?} should start with a number");
192+ }
193+ assert_eq!(num.parse::<usize>().unwrap(), i + 1, "migration sql files should start with 1, not zero, and no intermediate numbers should be missing. Panicked on file: {}", p.display());
194+ migr_rs.write_all(b"(").unwrap();
195+ migr_rs
196+ .write_all(num.trim_start_matches('0').as_bytes())
197+ .unwrap();
198+ migr_rs.write_all(b",\"").unwrap();
199+
200+ migr_rs
201+ .write_all(std::fs::read_to_string(p).unwrap().as_bytes())
202+ .unwrap();
203+ migr_rs.write_all(b"\",\"").unwrap();
204+ migr_rs
205+ .write_all(std::fs::read_to_string(u).unwrap().as_bytes())
206+ .unwrap();
207+ migr_rs.write_all(b"\"),").unwrap();
208+ }
209+ migr_rs.write_all(b"]").unwrap();
210+ migr_rs.flush().unwrap();
211+ }
212+ }
213 diff --git a/core/src/connection.rs b/core/src/connection.rs
214index 05ad84c..e6d6907 100644
215--- a/core/src/connection.rs
216+++ b/core/src/connection.rs
217 @@ -184,7 +184,7 @@ impl Connection {
218 return Err("Database doesn't exist".into());
219 }
220 INIT_SQLITE_LOGGING.call_once(|| {
221- unsafe { rusqlite::trace::config_log(Some(log_callback)).unwrap() };
222+ _ = unsafe { rusqlite::trace::config_log(Some(log_callback)) };
223 });
224 let conn = DbConnection::open(conf.db_path.to_str().unwrap())?;
225 rusqlite::vtab::array::load_module(&conn)?;
226 diff --git a/core/tests/migrations.rs b/core/tests/migrations.rs
227index f5464ec..841baf7 100644
228--- a/core/tests/migrations.rs
229+++ b/core/tests/migrations.rs
230 @@ -17,10 +17,34 @@
231 * along with this program. If not, see <https://www.gnu.org/licenses/>.
232 */
233
234+ use std::fs::{metadata, File, OpenOptions};
235+
236 use mailpot::{Configuration, Connection, SendMail};
237 use mailpot_tests::init_stderr_logging;
238 use tempfile::TempDir;
239
240+ // Source: https://stackoverflow.com/a/64535181
241+ fn is_output_file_outdated<P1, P2>(input: P1, output: P2) -> std::io::Result<bool>
242+ where
243+ P1: AsRef<Path>,
244+ P2: AsRef<Path>,
245+ {
246+ let out_meta = metadata(output);
247+ if let Ok(meta) = out_meta {
248+ let output_mtime = meta.modified()?;
249+
250+ // if input file is more recent than our output, we are outdated
251+ let input_meta = metadata(input)?;
252+ let input_mtime = input_meta.modified()?;
253+
254+ Ok(input_mtime > output_mtime)
255+ } else {
256+ // output file not found, we are outdated
257+ Ok(true)
258+ }
259+ }
260+ include!("../make_migrations.rs");
261+
262 #[test]
263 fn test_init_empty() {
264 init_stderr_logging();
265 @@ -49,3 +73,291 @@ fn test_init_empty() {
266
267 db.migrate(migrations[0].0, version).unwrap();
268 }
269+
270+ trait ConnectionExt {
271+ fn schema_version(&self) -> Result<u32, rusqlite::Error>;
272+ fn migrate(
273+ &mut self,
274+ from: u32,
275+ to: u32,
276+ migrations: &[(u32, &str, &str)],
277+ ) -> Result<(), rusqlite::Error>;
278+ }
279+
280+ impl ConnectionExt for rusqlite::Connection {
281+ fn schema_version(&self) -> Result<u32, rusqlite::Error> {
282+ self.prepare("SELECT user_version FROM pragma_user_version;")?
283+ .query_row([], |row| {
284+ let v: u32 = row.get(0)?;
285+ Ok(v)
286+ })
287+ }
288+
289+ fn migrate(
290+ &mut self,
291+ mut from: u32,
292+ to: u32,
293+ migrations: &[(u32, &str, &str)],
294+ ) -> Result<(), rusqlite::Error> {
295+ if from == to {
296+ return Ok(());
297+ }
298+
299+ let undo = from > to;
300+ let tx = self.transaction()?;
301+
302+ loop {
303+ log::trace!(
304+ "exec migration from {from} to {to}, type: {}do",
305+ if undo { "un" } else { "re" }
306+ );
307+ if undo {
308+ log::trace!("{}", migrations[from as usize - 1].2);
309+ tx.execute_batch(migrations[from as usize - 1].2)?;
310+ from -= 1;
311+ if from == to {
312+ break;
313+ }
314+ } else {
315+ if from != 0 {
316+ log::trace!("{}", migrations[from as usize - 1].1);
317+ tx.execute_batch(migrations[from as usize - 1].1)?;
318+ }
319+ from += 1;
320+ if from == to + 1 {
321+ break;
322+ }
323+ }
324+ }
325+ tx.pragma_update(
326+ None,
327+ "user_version",
328+ if to == 0 {
329+ 0
330+ } else {
331+ migrations[to as usize - 1].0
332+ },
333+ )?;
334+
335+ tx.commit()?;
336+ Ok(())
337+ }
338+ }
339+
340+ #[test]
341+ fn test_migration_gen() {
342+ init_stderr_logging();
343+ let tmp_dir = TempDir::new().unwrap();
344+ let in_path = tmp_dir.path().join("migrations");
345+ std::fs::create_dir(&in_path).unwrap();
346+ let out_path = tmp_dir.path().join("migrations.txt");
347+ for (num, redo, undo) in MIGRATIONS.iter() {
348+ let mut redo_file = File::options()
349+ .write(true)
350+ .create(true)
351+ .truncate(true)
352+ .open(&in_path.join(&format!("{num:03}.sql")))
353+ .unwrap();
354+ redo_file.write_all(redo.as_bytes()).unwrap();
355+ redo_file.flush().unwrap();
356+
357+ let mut undo_file = File::options()
358+ .write(true)
359+ .create(true)
360+ .truncate(true)
361+ .open(&in_path.join(&format!("{num:03}.undo.sql")))
362+ .unwrap();
363+ undo_file.write_all(undo.as_bytes()).unwrap();
364+ undo_file.flush().unwrap();
365+ }
366+
367+ make_migrations(&in_path, &out_path);
368+ let output = std::fs::read_to_string(&out_path).unwrap();
369+ assert_eq!(&output.replace([' ', '\n'], ""), &r#"//(user_version, redo sql, undo sql
370+ &[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], ""));
371+ }
372+
373+ #[test]
374+ #[should_panic]
375+ fn test_migration_gen_panic() {
376+ init_stderr_logging();
377+ let tmp_dir = TempDir::new().unwrap();
378+ let in_path = tmp_dir.path().join("migrations");
379+ std::fs::create_dir(&in_path).unwrap();
380+ let out_path = tmp_dir.path().join("migrations.txt");
381+ for (num, redo, undo) in MIGRATIONS.iter().skip(1) {
382+ let mut redo_file = File::options()
383+ .write(true)
384+ .create(true)
385+ .truncate(true)
386+ .open(&in_path.join(&format!("{num:03}.sql")))
387+ .unwrap();
388+ redo_file.write_all(redo.as_bytes()).unwrap();
389+ redo_file.flush().unwrap();
390+
391+ let mut undo_file = File::options()
392+ .write(true)
393+ .create(true)
394+ .truncate(true)
395+ .open(&in_path.join(&format!("{num:03}.undo.sql")))
396+ .unwrap();
397+ undo_file.write_all(undo.as_bytes()).unwrap();
398+ undo_file.flush().unwrap();
399+ }
400+
401+ make_migrations(&in_path, &out_path);
402+ let output = std::fs::read_to_string(&out_path).unwrap();
403+ assert_eq!(&output.replace([' ','\n'], ""), &r#"//(user_version, redo sql, undo sql
404+ &[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], ""));
405+ }
406+
407+ #[test]
408+ fn test_migration() {
409+ init_stderr_logging();
410+ let tmp_dir = TempDir::new().unwrap();
411+ let db_path = tmp_dir.path().join("migr.db");
412+
413+ let mut conn = rusqlite::Connection::open(db_path.to_str().unwrap()).unwrap();
414+ conn.execute_batch(FIRST_SCHEMA).unwrap();
415+
416+ conn.execute_batch(
417+ "INSERT INTO person(name,address) VALUES('John Doe', 'johndoe@example.com');",
418+ )
419+ .unwrap();
420+
421+ let version = conn.schema_version().unwrap();
422+ log::trace!("initial schema version is {}", version);
423+
424+ //assert_eq!(version, migrations[migrations.len() - 1].0);
425+
426+ conn.migrate(version, MIGRATIONS.last().unwrap().0, MIGRATIONS)
427+ .unwrap();
428+ /*
429+ * CREATE TABLE sqlite_schema (
430+ type text,
431+ name text,
432+ tbl_name text,
433+ rootpage integer,
434+ sql text
435+ );
436+ */
437+ let get_sql = |table: &str, conn: &rusqlite::Connection| -> String {
438+ conn.prepare("SELECT sql FROM sqlite_schema WHERE name = ?;")
439+ .unwrap()
440+ .query_row([table], |row| {
441+ let sql: String = row.get(0)?;
442+ Ok(sql)
443+ })
444+ .unwrap()
445+ };
446+
447+ let strip_ws = |sql: &str| -> String { sql.replace([' ', '\n'], "") };
448+
449+ let person_sql: String = get_sql("person", &conn);
450+ assert_eq!(
451+ &strip_ws(&person_sql),
452+ &strip_ws(
453+ r#"
454+ CREATE TABLE person (
455+ pk INTEGER PRIMARY KEY NOT NULL,
456+ name TEXT,
457+ address TEXT NOT NULL,
458+ created INTEGER NOT NULL DEFAULT (unixepoch()),
459+ last_modified INTEGER NOT NULL DEFAULT (unixepoch()),
460+ interests TEXT,
461+ main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL
462+ )"#
463+ )
464+ );
465+ let hobby_sql: String = get_sql("hobby", &conn);
466+ assert_eq!(
467+ &strip_ws(&hobby_sql),
468+ &strip_ws(
469+ r#"CREATE TABLE hobby (
470+ pk INTEGER PRIMARY KEY NOT NULL,
471+ title TEXT NOT NULL
472+ )"#
473+ )
474+ );
475+ conn.execute_batch(
476+ r#"
477+ INSERT INTO hobby(title) VALUES('fishing');
478+ INSERT INTO hobby(title) VALUES('reading books');
479+ INSERT INTO hobby(title) VALUES('running');
480+ INSERT INTO hobby(title) VALUES('forest walks');
481+ UPDATE person SET main_hobby = hpk FROM (SELECT pk AS hpk FROM hobby LIMIT 1) WHERE name = 'John Doe';
482+ "#
483+ )
484+ .unwrap();
485+ log::trace!(
486+ "John Doe's main hobby is {:?}",
487+ conn.prepare(
488+ "SELECT pk, title FROM hobby WHERE EXISTS (SELECT 1 FROM person AS p WHERE \
489+ p.main_hobby = pk);"
490+ )
491+ .unwrap()
492+ .query_row([], |row| {
493+ let pk: i64 = row.get(0)?;
494+ let title: String = row.get(1)?;
495+ Ok((pk, title))
496+ })
497+ .unwrap()
498+ );
499+
500+ conn.migrate(MIGRATIONS.last().unwrap().0, 0, MIGRATIONS)
501+ .unwrap();
502+
503+ assert_eq!(
504+ conn.prepare("SELECT sql FROM sqlite_schema WHERE name = 'hobby';")
505+ .unwrap()
506+ .query_row([], |row| { row.get::<_, String>(0) })
507+ .unwrap_err(),
508+ rusqlite::Error::QueryReturnedNoRows
509+ );
510+ let person_sql: String = get_sql("person", &conn);
511+ assert_eq!(
512+ &strip_ws(&person_sql),
513+ &strip_ws(
514+ r#"
515+ CREATE TABLE person (
516+ pk INTEGER PRIMARY KEY NOT NULL,
517+ name TEXT,
518+ address TEXT NOT NULL,
519+ created INTEGER NOT NULL DEFAULT (unixepoch()),
520+ last_modified INTEGER NOT NULL DEFAULT (unixepoch())
521+ )"#
522+ )
523+ );
524+ }
525+
526+ const FIRST_SCHEMA: &str = r#"
527+ PRAGMA foreign_keys = true;
528+ PRAGMA encoding = 'UTF-8';
529+ PRAGMA schema_version = 0;
530+
531+ CREATE TABLE IF NOT EXISTS person (
532+ pk INTEGER PRIMARY KEY NOT NULL,
533+ name TEXT,
534+ address TEXT NOT NULL,
535+ created INTEGER NOT NULL DEFAULT (unixepoch()),
536+ last_modified INTEGER NOT NULL DEFAULT (unixepoch())
537+ );
538+ "#;
539+
540+ const MIGRATIONS: &[(u32, &str, &str)] = &[
541+ (
542+ 1,
543+ "ALTER TABLE PERSON ADD COLUMN interests TEXT;",
544+ "ALTER TABLE PERSON DROP COLUMN interests;",
545+ ),
546+ (
547+ 2,
548+ "CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);",
549+ "DROP TABLE hobby;",
550+ ),
551+ (
552+ 3,
553+ "ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;",
554+ "ALTER TABLE PERSON DROP COLUMN main_hobby;",
555+ ),
556+ ];