+138 -67 +/-6 browse
1 | diff --git a/core/build.rs b/core/build.rs |
2 | index c902648..fd9dc55 100644 |
3 | --- a/core/build.rs |
4 | +++ b/core/build.rs |
5 | @@ -53,43 +53,39 @@ fn main() { |
6 | println!("cargo:rerun-if-changed=migrations"); |
7 | println!("cargo:rerun-if-changed=src/schema.sql.m4"); |
8 | |
9 | - if is_output_file_outdated("src/schema.sql.m4", "src/schema.sql").unwrap() { |
10 | - let output = Command::new("m4") |
11 | - .arg("./src/schema.sql.m4") |
12 | - .output() |
13 | - .unwrap(); |
14 | - if String::from_utf8_lossy(&output.stdout).trim().is_empty() { |
15 | - panic!( |
16 | - "m4 output is empty. stderr was {}", |
17 | - String::from_utf8_lossy(&output.stderr) |
18 | - ); |
19 | - } |
20 | - let mut verify = Command::new("sqlite3") |
21 | - .stdin(Stdio::piped()) |
22 | - .stdout(Stdio::piped()) |
23 | - .stderr(Stdio::piped()) |
24 | - .spawn() |
25 | - .unwrap(); |
26 | - println!( |
27 | - "Verifying by creating an in-memory database in sqlite3 and feeding it the output \ |
28 | - schema." |
29 | + let mut output = Command::new("m4") |
30 | + .arg("./src/schema.sql.m4") |
31 | + .output() |
32 | + .unwrap(); |
33 | + if String::from_utf8_lossy(&output.stdout).trim().is_empty() { |
34 | + panic!( |
35 | + "m4 output is empty. stderr was {}", |
36 | + String::from_utf8_lossy(&output.stderr) |
37 | ); |
38 | - verify |
39 | - .stdin |
40 | - .take() |
41 | - .unwrap() |
42 | - .write_all(&output.stdout) |
43 | - .unwrap(); |
44 | - let exit = verify.wait_with_output().unwrap(); |
45 | - if !exit.status.success() { |
46 | - panic!( |
47 | - "sqlite3 could not read SQL schema: {}", |
48 | - String::from_utf8_lossy(&exit.stdout) |
49 | - ); |
50 | - } |
51 | - let mut file = std::fs::File::create("./src/schema.sql").unwrap(); |
52 | - file.write_all(&output.stdout).unwrap(); |
53 | } |
54 | - |
55 | - make_migrations("migrations", MIGRATION_RS); |
56 | + make_migrations("migrations", MIGRATION_RS, &mut output.stdout); |
57 | + let mut verify = Command::new("sqlite3") |
58 | + .stdin(Stdio::piped()) |
59 | + .stdout(Stdio::piped()) |
60 | + .stderr(Stdio::piped()) |
61 | + .spawn() |
62 | + .unwrap(); |
63 | + println!( |
64 | + "Verifying by creating an in-memory database in sqlite3 and feeding it the output schema." |
65 | + ); |
66 | + verify |
67 | + .stdin |
68 | + .take() |
69 | + .unwrap() |
70 | + .write_all(&output.stdout) |
71 | + .unwrap(); |
72 | + let exit = verify.wait_with_output().unwrap(); |
73 | + if !exit.status.success() { |
74 | + panic!( |
75 | + "sqlite3 could not read SQL schema: {}", |
76 | + String::from_utf8_lossy(&exit.stdout) |
77 | + ); |
78 | + } |
79 | + let mut file = std::fs::File::create("./src/schema.sql").unwrap(); |
80 | + file.write_all(&output.stdout).unwrap(); |
81 | } |
82 | diff --git a/core/make_migrations.rs b/core/make_migrations.rs |
83 | index 8cf372d..fa21a77 100644 |
84 | --- a/core/make_migrations.rs |
85 | +++ b/core/make_migrations.rs |
86 | @@ -19,7 +19,17 @@ |
87 | |
88 | use std::{fs::read_dir, io::Write, path::Path}; |
89 | |
90 | - pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, output_file: O) { |
91 | + /// Scans migrations directory for file entries, and creates a rust file with an array containing |
92 | + /// the migration slices. |
93 | + /// |
94 | + /// |
95 | + /// If a migration is a data migration (not a CREATE, DROP or ALTER statement) it is appended to |
96 | + /// the schema file. |
97 | + pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>( |
98 | + migrations_path: M, |
99 | + output_file: O, |
100 | + schema_file: &mut Vec<u8>, |
101 | + ) { |
102 | let migrations_folder_path = migrations_path.as_ref(); |
103 | let output_file_path = output_file.as_ref(); |
104 | |
105 | @@ -63,10 +73,16 @@ pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, outpu |
106 | for (i, (p, u)) in paths.iter().zip(undo_paths.iter()).enumerate() { |
107 | // This should be a number string, padded with 2 zeros if it's less than 3 |
108 | // digits. e.g. 001, \d{3} |
109 | - let num = p.file_stem().unwrap().to_str().unwrap(); |
110 | + let mut num = p.file_stem().unwrap().to_str().unwrap(); |
111 | + let is_data = num.ends_with(".data"); |
112 | + if is_data { |
113 | + num = num.strip_suffix(".data").unwrap(); |
114 | + } |
115 | + |
116 | if !u.file_name().unwrap().to_str().unwrap().starts_with(num) { |
117 | panic!("Undo file {u:?} should match with {p:?}"); |
118 | } |
119 | + |
120 | if num.parse::<u32>().is_err() { |
121 | panic!("Migration file {p:?} should start with a number"); |
122 | } |
123 | @@ -75,16 +91,21 @@ pub fn make_migrations<M: AsRef<Path>, O: AsRef<Path>>(migrations_path: M, outpu |
124 | migr_rs |
125 | .write_all(num.trim_start_matches('0').as_bytes()) |
126 | .unwrap(); |
127 | - migr_rs.write_all(b",\"").unwrap(); |
128 | + migr_rs.write_all(b",r###\"").unwrap(); |
129 | |
130 | + let redo = std::fs::read_to_string(p).unwrap(); |
131 | + migr_rs.write_all(redo.trim().as_bytes()).unwrap(); |
132 | + migr_rs.write_all(b"\"###,r###\"").unwrap(); |
133 | migr_rs |
134 | - .write_all(std::fs::read_to_string(p).unwrap().as_bytes()) |
135 | + .write_all(std::fs::read_to_string(u).unwrap().trim().as_bytes()) |
136 | .unwrap(); |
137 | - migr_rs.write_all(b"\",\"").unwrap(); |
138 | - migr_rs |
139 | - .write_all(std::fs::read_to_string(u).unwrap().as_bytes()) |
140 | - .unwrap(); |
141 | - migr_rs.write_all(b"\"),").unwrap(); |
142 | + migr_rs.write_all(b"\"###),").unwrap(); |
143 | + if is_data { |
144 | + schema_file.extend(b"\n\n-- ".iter()); |
145 | + schema_file.extend(num.as_bytes().iter()); |
146 | + schema_file.extend(b".data.sql\n\n".iter()); |
147 | + schema_file.extend(redo.into_bytes().into_iter()); |
148 | + } |
149 | } |
150 | migr_rs.write_all(b"]").unwrap(); |
151 | migr_rs.flush().unwrap(); |
152 | diff --git a/core/migrations/005.data.sql b/core/migrations/005.data.sql |
153 | new file mode 100644 |
154 | index 0000000..af28922 |
155 | --- /dev/null |
156 | +++ b/core/migrations/005.data.sql |
157 | @@ -0,0 +1,31 @@ |
158 | + INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('ArchivedAtLinkSettings', '{ |
159 | + "$schema": "http://json-schema.org/draft-07/schema", |
160 | + "$ref": "#/$defs/ArchivedAtLinkSettings", |
161 | + "$defs": { |
162 | + "ArchivedAtLinkSettings": { |
163 | + "title": "ArchivedAtLinkSettings", |
164 | + "description": "Settings for ArchivedAtLink message filter", |
165 | + "type": "object", |
166 | + "properties": { |
167 | + "template": { |
168 | + "title": "Jinja template for header value", |
169 | + "description": "Template for\n `Archived-At` header value, as described in RFC 5064 \"The Archived-At\n Message Header Field\". The template receives only one string variable\n with the value of the mailing list post `Message-ID` header.\n\n For example, if:\n\n - the template is `http://www.example.com/mid/{{msg_id}}`\n - the `Message-ID` is `<0C2U00F01DFGCR@mailsj-v3.example.com>`\n\n The full header will be generated as:\n\n `Archived-At: <http://www.example.com/mid/0C2U00F01DFGCR@mailsj-v3.example.com>\n\n Note: Surrounding carets in the `Message-ID` value are not required. If\n you wish to preserve them in the URL, set option `preserve-carets` to\n true.\n ", |
170 | + "examples": [ |
171 | + "https://www.example.com/{{msg_id}}", |
172 | + "https://www.example.com/{{msg_id}}.html" |
173 | + ], |
174 | + "type": "string", |
175 | + "pattern": ".+[{][{]msg_id[}][}].*" |
176 | + }, |
177 | + "preserve_carets": { |
178 | + "title": "Preserve carets of `Message-ID` in generated value", |
179 | + "type": "boolean", |
180 | + "default": false |
181 | + } |
182 | + }, |
183 | + "required": [ |
184 | + "template" |
185 | + ] |
186 | + } |
187 | + } |
188 | + }'); |
189 | diff --git a/core/migrations/005.data.undo.sql b/core/migrations/005.data.undo.sql |
190 | new file mode 100644 |
191 | index 0000000..952d321 |
192 | --- /dev/null |
193 | +++ b/core/migrations/005.data.undo.sql |
194 | @@ -0,0 +1 @@ |
195 | + DELETE FROM settings_json_schema WHERE id = 'ArchivedAtLinkSettings'; |
196 | diff --git a/core/src/migrations.rs.inc b/core/src/migrations.rs.inc |
197 | index c70fe37..adfccf3 100644 |
198 | --- a/core/src/migrations.rs.inc |
199 | +++ b/core/src/migrations.rs.inc |
200 | @@ -1,14 +1,10 @@ |
201 | |
202 | //(user_version, redo sql, undo sql |
203 | - &[(1,"PRAGMA foreign_keys=ON; |
204 | - ALTER TABLE templates RENAME TO template; |
205 | - ","PRAGMA foreign_keys=ON; |
206 | - ALTER TABLE template RENAME TO templates; |
207 | - "),(2,"PRAGMA foreign_keys=ON; |
208 | - ALTER TABLE list ADD COLUMN topics JSON NOT NULL CHECK (json_type(topics) == 'array') DEFAULT '[]'; |
209 | - ","PRAGMA foreign_keys=ON; |
210 | - ALTER TABLE list DROP COLUMN topics; |
211 | - "),(3,"PRAGMA foreign_keys=ON; |
212 | + &[(1,r###"PRAGMA foreign_keys=ON; |
213 | + ALTER TABLE templates RENAME TO template;"###,r###"PRAGMA foreign_keys=ON; |
214 | + ALTER TABLE template RENAME TO templates;"###),(2,r###"PRAGMA foreign_keys=ON; |
215 | + ALTER TABLE list ADD COLUMN topics JSON NOT NULL CHECK (json_type(topics) == 'array') DEFAULT '[]';"###,r###"PRAGMA foreign_keys=ON; |
216 | + ALTER TABLE list DROP COLUMN topics;"###),(3,r###"PRAGMA foreign_keys=ON; |
217 | |
218 | UPDATE list SET topics = arr FROM (SELECT json_group_array(ord.val) AS arr, ord.pk AS pk FROM (SELECT json_each.value AS val, list.pk AS pk FROM list, json_each(list.topics) ORDER BY val ASC) AS ord GROUP BY pk) AS ord WHERE ord.pk = list.pk; |
219 | |
220 | @@ -27,12 +23,10 @@ AFTER INSERT ON list |
221 | FOR EACH ROW |
222 | BEGIN |
223 | UPDATE list SET topics = arr FROM (SELECT json_group_array(ord.val) AS arr, ord.pk AS pk FROM (SELECT json_each.value AS val, list.pk AS pk FROM list, json_each(list.topics) ORDER BY val ASC) AS ord GROUP BY pk) AS ord WHERE ord.pk = list.pk AND list.pk = NEW.pk; |
224 | - END; |
225 | - ","PRAGMA foreign_keys=ON; |
226 | + END;"###,r###"PRAGMA foreign_keys=ON; |
227 | |
228 | DROP TRIGGER sort_topics_update_trigger; |
229 | - DROP TRIGGER sort_topics_new_trigger; |
230 | - "),(4,"CREATE TABLE IF NOT EXISTS settings_json_schema ( |
231 | + DROP TRIGGER sort_topics_new_trigger;"###),(4,r###"CREATE TABLE IF NOT EXISTS settings_json_schema ( |
232 | pk INTEGER PRIMARY KEY NOT NULL, |
233 | id TEXT NOT NULL UNIQUE, |
234 | value JSON NOT NULL CHECK (json_type(value) = 'object'), |
235 | @@ -198,7 +192,35 @@ WHEN NEW.last_modified == OLD.last_modified |
236 | BEGIN |
237 | UPDATE list_settings_json SET last_modified = unixepoch() |
238 | WHERE pk = NEW.pk; |
239 | - END; |
240 | - ","DROP TABLE settings_json_schema; |
241 | - DROP TABLE list_settings_json; |
242 | - "),] |
243 | \ No newline at end of file |
244 | + END;"###,r###"DROP TABLE settings_json_schema; |
245 | + DROP TABLE list_settings_json;"###),(5,r###"INSERT OR REPLACE INTO settings_json_schema(id, value) VALUES('ArchivedAtLinkSettings', '{ |
246 | + "$schema": "http://json-schema.org/draft-07/schema", |
247 | + "$ref": "#/$defs/ArchivedAtLinkSettings", |
248 | + "$defs": { |
249 | + "ArchivedAtLinkSettings": { |
250 | + "title": "ArchivedAtLinkSettings", |
251 | + "description": "Settings for ArchivedAtLink message filter", |
252 | + "type": "object", |
253 | + "properties": { |
254 | + "template": { |
255 | + "title": "Jinja template for header value", |
256 | + "description": "Template for\n `Archived-At` header value, as described in RFC 5064 \"The Archived-At\n Message Header Field\". The template receives only one string variable\n with the value of the mailing list post `Message-ID` header.\n\n For example, if:\n\n - the template is `http://www.example.com/mid/{{msg_id}}`\n - the `Message-ID` is `<0C2U00F01DFGCR@mailsj-v3.example.com>`\n\n The full header will be generated as:\n\n `Archived-At: <http://www.example.com/mid/0C2U00F01DFGCR@mailsj-v3.example.com>\n\n Note: Surrounding carets in the `Message-ID` value are not required. If\n you wish to preserve them in the URL, set option `preserve-carets` to\n true.\n ", |
257 | + "examples": [ |
258 | + "https://www.example.com/{{msg_id}}", |
259 | + "https://www.example.com/{{msg_id}}.html" |
260 | + ], |
261 | + "type": "string", |
262 | + "pattern": ".+[{][{]msg_id[}][}].*" |
263 | + }, |
264 | + "preserve_carets": { |
265 | + "title": "Preserve carets of `Message-ID` in generated value", |
266 | + "type": "boolean", |
267 | + "default": false |
268 | + } |
269 | + }, |
270 | + "required": [ |
271 | + "template" |
272 | + ] |
273 | + } |
274 | + } |
275 | + }');"###,r###"DELETE FROM settings_json_schema WHERE id = 'ArchivedAtLinkSettings';"###),] |
276 | \ No newline at end of file |
277 | diff --git a/core/tests/migrations.rs b/core/tests/migrations.rs |
278 | index e05f9e7..0c8fa9a 100644 |
279 | --- a/core/tests/migrations.rs |
280 | +++ b/core/tests/migrations.rs |
281 | @@ -203,10 +203,10 @@ fn test_migration_gen() { |
282 | undo_file.flush().unwrap(); |
283 | } |
284 | |
285 | - make_migrations(&in_path, &out_path); |
286 | + make_migrations(&in_path, &out_path, &mut vec![]); |
287 | let output = std::fs::read_to_string(&out_path).unwrap(); |
288 | - assert_eq!(&output.replace([' ', '\n'], ""), &r#"//(user_version, redo sql, undo sql |
289 | - &[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], "")); |
290 | + assert_eq!(&output.replace([' ', '\n'], ""), &r####"//(user_version, redo sql, undo sql |
291 | + &[(1,r###"ALTER TABLE PERSON ADD COLUMN interests TEXT;"###,r###"ALTER TABLE PERSON DROP COLUMN interests;"###),(2,r###"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);"###,r###"DROP TABLE hobby;"###),(3,r###"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;"###,r###"ALTER TABLE PERSON DROP COLUMN main_hobby;"###),]"####.replace([' ', '\n'], "")); |
292 | } |
293 | |
294 | #[test] |
295 | @@ -237,7 +237,7 @@ fn test_migration_gen_panic() { |
296 | undo_file.flush().unwrap(); |
297 | } |
298 | |
299 | - make_migrations(&in_path, &out_path); |
300 | + make_migrations(&in_path, &out_path, &mut vec![]); |
301 | let output = std::fs::read_to_string(&out_path).unwrap(); |
302 | assert_eq!(&output.replace([' ','\n'], ""), &r#"//(user_version, redo sql, undo sql |
303 | &[(1,"ALTER TABLE PERSON ADD COLUMN interests TEXT;","ALTER TABLE PERSON DROP COLUMN interests;"),(2,"CREATE TABLE hobby ( pk INTEGER PRIMARY KEY NOT NULL,title TEXT NOT NULL);","DROP TABLE hobby;"),(3,"ALTER TABLE PERSON ADD COLUMN main_hobby INTEGER REFERENCES hobby(pk) ON DELETE SET NULL;","ALTER TABLE PERSON DROP COLUMN main_hobby;"),]"#.replace([' ', '\n'], "")); |