Author: Manos Pitsidianakis [manos@pitsidianak.is]
Hash: 38ae3798f67c7aec00ca00b455de10179c619405
Timestamp: Fri, 09 Jun 2023 13:34:23 +0000 (1 year ago)

+138 -67 +/-6 browse
core: add data kind of migrations
1diff --git a/core/build.rs b/core/build.rs
2index c902648..fd9dc55 100644
3--- a/core/build.rs
4+++ b/core/build.rs
5 @@ -53,43 +53,39 @@ fn main() {
6 println!("cargo:rerun-if-changed=migrations");
7 println!("cargo:rerun-if-changed=src/schema.sql.m4");
8
9- if is_output_file_outdated("src/schema.sql.m4", "src/schema.sql").unwrap() {
10- let output = Command::new("m4")
11- .arg("./src/schema.sql.m4")
12- .output()
13- .unwrap();
14- if String::from_utf8_lossy(&output.stdout).trim().is_empty() {
15- panic!(
16- "m4 output is empty. stderr was {}",
17- String::from_utf8_lossy(&output.stderr)
18- );
19- }
20- let mut verify = Command::new("sqlite3")
21- .stdin(Stdio::piped())
22- .stdout(Stdio::piped())
23- .stderr(Stdio::piped())
24- .spawn()
25- .unwrap();
26- println!(
27- "Verifying by creating an in-memory database in sqlite3 and feeding it the output \
28- schema."
29+ let mut output = Command::new("m4")
30+ .arg("./src/schema.sql.m4")
31+ .output()
32+ .unwrap();
33+ if String::from_utf8_lossy(&output.stdout).trim().is_empty() {
34+ panic!(
35+ "m4 output is empty. stderr was {}",
36+ String::from_utf8_lossy(&output.stderr)
37 );
38- verify
39- .stdin
40- .take()
41- .unwrap()
42- .write_all(&output.stdout)
43- .unwrap();
44- let exit = verify.wait_with_output().unwrap();
45- if !exit.status.success() {
46- panic!(
47- "sqlite3 could not read SQL schema: {}",
48- String::from_utf8_lossy(&exit.stdout)
49- );
50- }
51- let mut file = std::fs::File::create("./src/schema.sql").unwrap();
52- file.write_all(&output.stdout).unwrap();
53 }
54-
55- make_migrations("migrations", MIGRATION_RS);
56+ make_migrations("migrations", MIGRATION_RS, &mut output.stdout);
57+ let mut verify = Command::new("sqlite3")
58+ .stdin(Stdio::piped())
59+ .stdout(Stdio::piped())
60+ .stderr(Stdio::piped())
61+ .spawn()
62+ .unwrap();
63+ println!(
64+ "Verifying by creating an in-memory database in sqlite3 and feeding it the output schema."
65+ );
66+ verify
67+ .stdin
68+ .take()
69+ .unwrap()
70+ .write_all(&output.stdout)
71+ .unwrap();
72+ let exit = verify.wait_with_output().unwrap();
73+ if !exit.status.success() {
74+ panic!(
75+ "sqlite3 could not read SQL schema: {}",
76+ String::from_utf8_lossy(&exit.stdout)
77+ );
78+ }
79+ let mut file = std::fs::File::create("./src/schema.sql").unwrap();
80+ file.write_all(&output.stdout).unwrap();
81 }
82 diff --git a/core/make_migrations.rs b/core/make_migrations.rs
83index 8cf372d..fa21a77 100644
84--- a/core/make_migrations.rs
85+++ b/core/make_migrations.rs
86 @@ -19,7 +19,17 @@
87
88 use std::{fs::read_dir, io::Write, path::Path};
89
90- pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, output_file: O) {
91+ /// Scans migrations directory for file entries, and creates a rust file with an array containing
92+ /// the migration slices.
93+ ///
94+ ///
95+ /// If a migration is a data migration (not a CREATE, DROP or ALTER statement) it is appended to
96+ /// the schema file.
97+ pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(
98+ migrations_path: M,
99+ output_file: O,
100+ schema_file: &mut Vec<u8>,
101+ ) {
102 let migrations_folder_path = migrations_path.as_ref();
103 let output_file_path = output_file.as_ref();
104
105 @@ -63,10 +73,16 @@ pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, outpu
106 for (i, (p, u)) in paths.iter().zip(undo_paths.iter()).enumerate() {
107 // This should be a number string, padded with 2 zeros if it's less than 3
108 // digits. e.g. 001, \d{3}
109- let num = p.file_stem().unwrap().to_str().unwrap();
110+ let mut num = p.file_stem().unwrap().to_str().unwrap();
111+ let is_data = num.ends_with(".data");
112+ if is_data {
113+ num = num.strip_suffix(".data").unwrap();
114+ }
115+
116 if !u.file_name().unwrap().to_str().unwrap().starts_with(num) {
117 panic!("Undo file {u:?} should match with {p:?}");
118 }
119+
120 if num.parse::<u32>().is_err() {
121 panic!("Migration file {p:?} should start with a number");
122 }
123 @@ -75,16 +91,21 @@ pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, outpu
124 migr_rs
125 .write_all(num.trim_start_matches('0').as_bytes())
126 .unwrap();
127- migr_rs.write_all(b",\"").unwrap();
128+ migr_rs.write_all(b",r###\"").unwrap();
129
130+ let redo = std::fs::read_to_string(p).unwrap();
131+ migr_rs.write_all(redo.trim().as_bytes()).unwrap();
132+ migr_rs.write_all(b"\"###,r###\"").unwrap();
133 migr_rs
134- .write_all(std::fs::read_to_string(p).unwrap().as_bytes())
135+ .write_all(std::fs::read_to_string(u).unwrap().trim().as_bytes())
136 .unwrap();
137- migr_rs.write_all(b"\",\"").unwrap();
138- migr_rs
139- .write_all(std::fs::read_to_string(u).unwrap().as_bytes())
140- .unwrap();
141- migr_rs.write_all(b"\"),").unwrap();
142+ migr_rs.write_all(b"\"###),").unwrap();
143+ if is_data {
144+ schema_file.extend(b"\n\n-- ".iter());
145+ schema_file.extend(num.as_bytes().iter());
146+ schema_file.extend(b".data.sql\n\n".iter());
147+ schema_file.extend(redo.into_bytes().into_iter());
148+ }
149 }
150 migr_rs.write_all(b"]").unwrap();
151 migr_rs.flush().unwrap();
152 diff --git a/core/migrations/005.data.sql b/core/migrations/005.data.sql
153new file mode 100644
154index 0000000..af28922
155--- /dev/null
156+++ b/core/migrations/005.data.sql
157 @@ -0,0 +1,31 @@
158+ INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('ArchivedAtLinkSettings', '{
159+ "$schema": "http://json-schema.org/draft-07/schema",
160+ "$ref": "#/$defs/ArchivedAtLinkSettings",
161+ "$defs": {
162+ "ArchivedAtLinkSettings": {
163+ "title": "ArchivedAtLinkSettings",
164+ "description": "Settings for ArchivedAtLink message filter",
165+ "type": "object",
166+ "properties": {
167+ "template": {
168+ "title": "Jinja template for header value",
169+ "description": "Template for\n `Archived-At` header value, as described in RFC 5064 \"The Archived-At\n Message Header Field\". The template receives only one string variable\n with the value of the mailing list post `Message-ID` header.\n\n For example, if:\n\n - the template is `http://www.example.com/mid/{{msg_id}}`\n - the `Message-ID` is `<0C2U00F01DFGCR@mailsj-v3.example.com>`\n\n The full header will be generated as:\n\n `Archived-At: <http://www.example.com/mid/0C2U00F01DFGCR@mailsj-v3.example.com>\n\n Note: Surrounding carets in the `Message-ID` value are not required. If\n you wish to preserve them in the URL, set option `preserve-carets` to\n true.\n ",
170+ "examples": [
171+ "https://www.example.com/{{msg_id}}",
172+ "https://www.example.com/{{msg_id}}.html"
173+ ],
174+ "type": "string",
175+ "pattern": ".+[{][{]msg_id[}][}].*"
176+ },
177+ "preserve_carets": {
178+ "title": "Preserve carets of `Message-ID` in generated value",
179+ "type": "boolean",
180+ "default": false
181+ }
182+ },
183+ "required": [
184+ "template"
185+ ]
186+ }
187+ }
188+ }');
189 diff --git a/core/migrations/005.data.undo.sql b/core/migrations/005.data.undo.sql
190new file mode 100644
191index 0000000..952d321
192--- /dev/null
193+++ b/core/migrations/005.data.undo.sql
194 @@ -0,0 +1 @@
195+ DELETE FROM settings_json_schema WHERE id = 'ArchivedAtLinkSettings';
196 diff --git a/core/src/migrations.rs.inc b/core/src/migrations.rs.inc
197index c70fe37..adfccf3 100644
198--- a/core/src/migrations.rs.inc
199+++ b/core/src/migrations.rs.inc
200 @@ -1,14 +1,10 @@
201
202 //(user_version, redo sql, undo sql
203- &[(1,"PRAGMA foreign_keys=ON;
204- ALTER TABLE templates RENAME TO template;
205- ","PRAGMA foreign_keys=ON;
206- ALTER TABLE template RENAME TO templates;
207- "),(2,"PRAGMA foreign_keys=ON;
208- ALTER TABLE list ADD COLUMN topics JSON NOT NULL CHECK (json_type(topics) == 'array') DEFAULT '[]';
209- ","PRAGMA foreign_keys=ON;
210- ALTER TABLE list DROP COLUMN topics;
211- "),(3,"PRAGMA foreign_keys=ON;
212+ &[(1,r###"PRAGMA foreign_keys=ON;
213+ ALTER TABLE templates RENAME TO template;"###,r###"PRAGMA foreign_keys=ON;
214+ ALTER TABLE template RENAME TO templates;"###),(2,r###"PRAGMA foreign_keys=ON;
215+ ALTER TABLE list ADD COLUMN topics JSON NOT NULL CHECK (json_type(topics) == 'array') DEFAULT '[]';"###,r###"PRAGMA foreign_keys=ON;
216+ ALTER TABLE list DROP COLUMN topics;"###),(3,r###"PRAGMA foreign_keys=ON;
217
218 UPDATE list SET topics = arr FROM (SELECT json_group_array(ord.val) AS arr, ord.pk AS pk FROM (SELECT json_each.value AS val, list.pk AS pk FROM list, json_each(list.topics) ORDER BY val ASC) AS ord GROUP BY pk) AS ord WHERE ord.pk = list.pk;
219
220 @@ -27,12 +23,10 @@ AFTER INSERT ON list
221 FOR EACH ROW
222 BEGIN
223 UPDATE list SET topics = arr FROM (SELECT json_group_array(ord.val) AS arr, ord.pk AS pk FROM (SELECT json_each.value AS val, list.pk AS pk FROM list, json_each(list.topics) ORDER BY val ASC) AS ord GROUP BY pk) AS ord WHERE ord.pk = list.pk AND list.pk = NEW.pk;
224- END;
225- ","PRAGMA foreign_keys=ON;
226+ END;"###,r###"PRAGMA foreign_keys=ON;
227
228 DROP TRIGGER sort_topics_update_trigger;
229- DROP TRIGGER sort_topics_new_trigger;
230- "),(4,"CREATE TABLE IF NOT EXISTS settings_json_schema (
231+ DROP TRIGGER sort_topics_new_trigger;"###),(4,r###"CREATE TABLE IF NOT EXISTS settings_json_schema (
232 pk INTEGER PRIMARY KEY NOT NULL,
233 id TEXT NOT NULL UNIQUE,
234 value JSON NOT NULL CHECK (json_type(value) = 'object'),
235 @@ -198,7 +192,35 @@ WHEN NEW.last_modified == OLD.last_modified
236 BEGIN
237 UPDATE list_settings_json SET last_modified = unixepoch()
238 WHERE pk = NEW.pk;
239- END;
240- ","DROP TABLE settings_json_schema;
241- DROP TABLE list_settings_json;
242- "),]
243\ No newline at end of file
244+ END;"###,r###"DROP TABLE settings_json_schema;
245+ DROP TABLE list_settings_json;"###),(5,r###"INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('ArchivedAtLinkSettings', '{
246+ "$schema": "http://json-schema.org/draft-07/schema",
247+ "$ref": "#/$defs/ArchivedAtLinkSettings",
248+ "$defs": {
249+ "ArchivedAtLinkSettings": {
250+ "title": "ArchivedAtLinkSettings",
251+ "description": "Settings for ArchivedAtLink message filter",
252+ "type": "object",
253+ "properties": {
254+ "template": {
255+ "title": "Jinja template for header value",
256+ "description": "Template for\n `Archived-At` header value, as described in RFC 5064 \"The Archived-At\n Message Header Field\". The template receives only one string variable\n with the value of the mailing list post `Message-ID` header.\n\n For example, if:\n\n - the template is `http://www.example.com/mid/{{msg_id}}`\n - the `Message-ID` is `<0C2U00F01DFGCR@mailsj-v3.example.com>`\n\n The full header will be generated as:\n\n `Archived-At: <http://www.example.com/mid/0C2U00F01DFGCR@mailsj-v3.example.com>\n\n Note: Surrounding carets in the `Message-ID` value are not required. If\n you wish to preserve them in the URL, set option `preserve-carets` to\n true.\n ",
257+ "examples": [
258+ "https://www.example.com/{{msg_id}}",
259+ "https://www.example.com/{{msg_id}}.html"
260+ ],
261+ "type": "string",
262+ "pattern": ".+[{][{]msg_id[}][}].*"
263+ },
264+ "preserve_carets": {
265+ "title": "Preserve carets of `Message-ID` in generated value",
266+ "type": "boolean",
267+ "default": false
268+ }
269+ },
270+ "required": [
271+ "template"
272+ ]
273+ }
274+ }
275+ }');"###,r###"DELETE FROM settings_json_schema WHERE id = 'ArchivedAtLinkSettings';"###),]
276\ No newline at end of file
277 diff --git a/core/tests/migrations.rs b/core/tests/migrations.rs
278index e05f9e7..0c8fa9a 100644
279--- a/core/tests/migrations.rs
280+++ b/core/tests/migrations.rs
281 @@ -203,10 +203,10 @@ fn test_migration_gen() {
282 undo_file.flush().unwrap();
283 }
284
285- make_migrations(&in_path, &out_path);
286+ make_migrations(&in_path, &out_path, &mut vec![]);
287 let output = std::fs::read_to_string(&out_path).unwrap();
288- assert_eq!(&output.replace([' ', '\n'], ""), &r#"//(user_version, redo sql, undo sql
289- &[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], ""));
290+ assert_eq!(&output.replace([' ', '\n'], ""), &r####"//(user_version, redo sql, undo sql
291+ &[(1,r###"ALTER TABLE PERSON ADD COLUMN interests TEXT;"###,r###"ALTER TABLE PERSON DROP COLUMN interests;"###),(2,r###"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);"###,r###"DROP TABLE hobby;"###),(3,r###"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;"###,r###"ALTER TABLE PERSON DROP COLUMN main_hobby;"###),]"####.replace([' ', '\n'], ""));
292 }
293
294 #[test]
295 @@ -237,7 +237,7 @@ fn test_migration_gen_panic() {
296 undo_file.flush().unwrap();
297 }
298
299- make_migrations(&in_path, &out_path);
300+ make_migrations(&in_path, &out_path, &mut vec![]);
301 let output = std::fs::read_to_string(&out_path).unwrap();
302 assert_eq!(&output.replace([' ','\n'], ""), &r#"//(user_version, redo sql, undo sql
303 &[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], ""));