Commit
Author: Jason White [github@jasonwhite.io]
Hash: 263ff867565ef0cdc09a063bb6ced320d11196bd
Timestamp: Fri, 25 Jun 2021 04:14:27 +0000 (3 years ago)

+297 -1 +/-5 browse
Add integration test for disk backend
1diff --git a/.cirrus.yml b/.cirrus.yml
2index 57c18c1..bd2b6de 100644
3--- a/.cirrus.yml
4+++ b/.cirrus.yml
5 @@ -27,6 +27,7 @@ test_task:
6 cargo_cache:
7 folder: $CARGO_HOME/registry
8 fingerprint_script: cat Cargo.lock
9+ install_script: cd $(mktemp -d) && curl -Ls https://github.com/git-lfs/git-lfs/releases/download/v2.13.3/git-lfs-linux-amd64-v2.13.3.tar.gz | tar xvz && ./install.sh
10 build_script: cargo build
11 test_script: cargo test
12 before_cache_script: rm -rf $CARGO_HOME/registry/index
13 diff --git a/Cargo.lock b/Cargo.lock
14index d176643..9d03408 100644
15--- a/Cargo.lock
16+++ b/Cargo.lock
17 @@ -350,6 +350,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
18 checksum = "88d7ed2934d741c6b37e33e3832298e8850b53fd2d2bea03873375596c7cea4e"
19
20 [[package]]
21+ name = "duct"
22+ version = "0.13.5"
23+ source = "registry+https://github.com/rust-lang/crates.io-index"
24+ checksum = "0fc6a0a59ed0888e0041cf708e66357b7ae1a82f1c67247e1f93b5e0818f7d8d"
25+ dependencies = [
26+ "libc",
27+ "once_cell",
28+ "os_pipe",
29+ "shared_child",
30+ ]
31+
32+ [[package]]
33 name = "env_logger"
34 version = "0.7.1"
35 source = "registry+https://github.com/rust-lang/crates.io-index"
36 @@ -363,6 +375,19 @@ dependencies = [
37 ]
38
39 [[package]]
40+ name = "env_logger"
41+ version = "0.8.4"
42+ source = "registry+https://github.com/rust-lang/crates.io-index"
43+ checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"
44+ dependencies = [
45+ "atty",
46+ "humantime 2.1.0",
47+ "log",
48+ "regex",
49+ "termcolor",
50+ ]
51+
52+ [[package]]
53 name = "fnv"
54 version = "1.0.7"
55 source = "registry+https://github.com/rust-lang/crates.io-index"
56 @@ -882,6 +907,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
57 checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
58
59 [[package]]
60+ name = "os_pipe"
61+ version = "0.9.2"
62+ source = "registry+https://github.com/rust-lang/crates.io-index"
63+ checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213"
64+ dependencies = [
65+ "libc",
66+ "winapi",
67+ ]
68+
69+ [[package]]
70 name = "parking_lot"
71 version = "0.11.1"
72 source = "registry+https://github.com/rust-lang/crates.io-index"
73 @@ -976,7 +1011,7 @@ version = "0.4.0"
74 source = "registry+https://github.com/rust-lang/crates.io-index"
75 checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d"
76 dependencies = [
77- "env_logger",
78+ "env_logger 0.7.1",
79 "log",
80 ]
81
82 @@ -1130,6 +1165,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
83 checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
84
85 [[package]]
86+ name = "remove_dir_all"
87+ version = "0.5.3"
88+ source = "registry+https://github.com/rust-lang/crates.io-index"
89+ checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
90+ dependencies = [
91+ "winapi",
92+ ]
93+
94+ [[package]]
95 name = "ring"
96 version = "0.16.20"
97 source = "registry+https://github.com/rust-lang/crates.io-index"
98 @@ -1155,6 +1199,8 @@ dependencies = [
99 "bytes",
100 "chacha",
101 "derive_more",
102+ "duct",
103+ "env_logger 0.8.4",
104 "futures",
105 "generic-array",
106 "hex 0.3.2",
107 @@ -1175,6 +1221,7 @@ dependencies = [
108 "serde_json",
109 "sha2",
110 "structopt",
111+ "tempfile",
112 "tokio",
113 "tokio-util",
114 "url",
115 @@ -1453,6 +1500,16 @@ dependencies = [
116 ]
117
118 [[package]]
119+ name = "shared_child"
120+ version = "0.3.5"
121+ source = "registry+https://github.com/rust-lang/crates.io-index"
122+ checksum = "6be9f7d5565b1483af3e72975e2dee33879b3b86bd48c0929fccf6585d79e65a"
123+ dependencies = [
124+ "libc",
125+ "winapi",
126+ ]
127+
128+ [[package]]
129 name = "shlex"
130 version = "0.1.1"
131 source = "registry+https://github.com/rust-lang/crates.io-index"
132 @@ -1614,6 +1671,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
133 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
134
135 [[package]]
136+ name = "tempfile"
137+ version = "3.2.0"
138+ source = "registry+https://github.com/rust-lang/crates.io-index"
139+ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
140+ dependencies = [
141+ "cfg-if",
142+ "libc",
143+ "rand",
144+ "redox_syscall 0.2.4",
145+ "remove_dir_all",
146+ "winapi",
147+ ]
148+
149+ [[package]]
150 name = "termcolor"
151 version = "1.1.2"
152 source = "registry+https://github.com/rust-lang/crates.io-index"
153 diff --git a/Cargo.toml b/Cargo.toml
154index 63980a3..8d53523 100644
155--- a/Cargo.toml
156+++ b/Cargo.toml
157 @@ -43,6 +43,12 @@ tokio-util = { version = "0.6", features = ["full"] }
158 url = "2"
159 uuid = { version = "0.8", features = ["v4"] }
160
161+ [dev-dependencies]
162+ rand = "0.8"
163+ tempfile = "3"
164+ duct = "0.13"
165+ env_logger = "0.8"
166+
167 [dependencies.rusoto_core]
168 version = "0.46"
169 default_features = false
170 diff --git a/tests/common/mod.rs b/tests/common/mod.rs
171new file mode 100644
172index 0000000..5ff6fc8
173--- /dev/null
174+++ b/tests/common/mod.rs
175 @@ -0,0 +1,141 @@
176+ // Copyright (c) 2021 Jason White
177+ //
178+ // Permission is hereby granted, free of charge, to any person obtaining a copy
179+ // of this software and associated documentation files (the "Software"), to deal
180+ // in the Software without restriction, including without limitation the rights
181+ // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
182+ // copies of the Software, and to permit persons to whom the Software is
183+ // furnished to do so, subject to the following conditions:
184+ //
185+ // The above copyright notice and this permission notice shall be included in
186+ // all copies or substantial portions of the Software.
187+ //
188+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
189+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
190+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
191+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
192+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
193+ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
194+ // SOFTWARE.
195+ use std::fs::{self, File};
196+ use std::io;
197+ use std::net::SocketAddr;
198+ use std::path::Path;
199+
200+ use duct::cmd;
201+ use rand::Rng;
202+
203+ /// A temporary git repository.
204+ pub struct GitRepo {
205+ repo: tempfile::TempDir,
206+ }
207+
208+ impl GitRepo {
209+ /// Initialize a temporary synthetic git repository. It is set up to be
210+ /// connected to our LFS server.
211+ pub fn init(lfs_server: SocketAddr) -> io::Result<Self> {
212+ let repo = tempfile::TempDir::new()?;
213+ let path = repo.path();
214+
215+ cmd!("git", "init", ".").dir(path).run()?;
216+ cmd!("git", "lfs", "install").dir(path).run()?;
217+ cmd!("git", "remote", "add", "origin", "fake_remote")
218+ .dir(path)
219+ .run()?;
220+ cmd!(
221+ "git",
222+ "config",
223+ "lfs.url",
224+ format!("http://{}/api/test/test", lfs_server)
225+ )
226+ .dir(path)
227+ .run()?;
228+ cmd!("git", "config", "user.name", "Foo Bar")
229+ .dir(path)
230+ .run()?;
231+ cmd!("git", "config", "user.email", "foobar@example.com")
232+ .dir(path)
233+ .run()?;
234+ cmd!("git", "lfs", "track", "*.bin").dir(path).run()?;
235+ cmd!("git", "add", ".gitattributes").dir(path).run()?;
236+ cmd!("git", "commit", "-m", "Initial commit")
237+ .dir(path)
238+ .run()?;
239+
240+ Ok(Self { repo })
241+ }
242+
243+ /// Adds a random file with the given size and random number generator. The
244+ /// file is also staged with `git add`.
245+ pub fn add_random<R: Rng>(
246+ &self,
247+ path: &Path,
248+ size: usize,
249+ rng: &mut R,
250+ ) -> io::Result<()> {
251+ let mut file = File::create(self.repo.path().join(path))?;
252+ gen_file(&mut file, size, rng)?;
253+ cmd!("git", "add", path).dir(self.repo.path()).run()?;
254+ Ok(())
255+ }
256+
257+ /// Commits the currently staged files.
258+ pub fn commit(&self, message: &str) -> io::Result<()> {
259+ cmd!("git", "commit", "-m", message)
260+ .dir(self.repo.path())
261+ .run()?;
262+ Ok(())
263+ }
264+
265+ pub fn lfs_push(&self) -> io::Result<()> {
266+ cmd!("git", "lfs", "push", "origin", "master")
267+ .dir(self.repo.path())
268+ .run()?;
269+ Ok(())
270+ }
271+
272+ pub fn lfs_pull(&self) -> io::Result<()> {
273+ cmd!("git", "lfs", "pull").dir(self.repo.path()).run()?;
274+ Ok(())
275+ }
276+
277+ /// Deletes all cached LFS files in `.git/lfs/`. This will force a
278+ /// re-download from the server.
279+ pub fn clean_lfs(&self) -> io::Result<()> {
280+ fs::remove_dir_all(self.repo.path().join(".git/lfs"))
281+ }
282+ }
283+
284+ fn gen_file<W, R>(
285+ writer: &mut W,
286+ mut size: usize,
287+ rng: &mut R,
288+ ) -> io::Result<()>
289+ where
290+ W: io::Write,
291+ R: Rng,
292+ {
293+ let mut buf = [0u8; 4096];
294+
295+ while size > 0 {
296+ let to_write = buf.len().min(size);
297+
298+ let buf = &mut buf[..to_write];
299+ rng.fill(buf);
300+ writer.write_all(buf)?;
301+
302+ size -= to_write;
303+ }
304+
305+ Ok(())
306+ }
307+
308+ pub fn init_logger() {
309+ let _ = env_logger::builder()
310+ // Include all events in tests
311+ .filter_module("rudolfs", log::LevelFilter::max())
312+ // Ensure events are captured by `cargo test`
313+ .is_test(true)
314+ // Ignore errors initializing the logger if tests race to configure it
315+ .try_init();
316+ }
317 diff --git a/tests/test_local.rs b/tests/test_local.rs
318new file mode 100644
319index 0000000..58e9c39
320--- /dev/null
321+++ b/tests/test_local.rs
322 @@ -0,0 +1,77 @@
323+ // Copyright (c) 2021 Jason White
324+ //
325+ // Permission is hereby granted, free of charge, to any person obtaining a copy
326+ // of this software and associated documentation files (the "Software"), to deal
327+ // in the Software without restriction, including without limitation the rights
328+ // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
329+ // copies of the Software, and to permit persons to whom the Software is
330+ // furnished to do so, subject to the following conditions:
331+ //
332+ // The above copyright notice and this permission notice shall be included in
333+ // all copies or substantial portions of the Software.
334+ //
335+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
336+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
337+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
338+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
339+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
340+ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
341+ // SOFTWARE.
342+ mod common;
343+
344+ use std::net::SocketAddr;
345+ use std::path::Path;
346+
347+ use futures::future::Either;
348+ use rand::rngs::StdRng;
349+ use rand::Rng;
350+ use rand::SeedableRng;
351+ use rudolfs::{LocalServerBuilder, Server};
352+ use tokio::sync::oneshot;
353+
354+ use common::{init_logger, GitRepo};
355+
356+ #[tokio::test(flavor = "multi_thread")]
357+ async fn smoke_test() -> Result<(), Box<dyn std::error::Error>> {
358+ init_logger();
359+
360+ // Make sure our seed is deterministic. This makes it easier to reproduce
361+ // the same repo every time.
362+ let mut rng = StdRng::seed_from_u64(42);
363+
364+ let data = tempfile::TempDir::new()?;
365+ let key = rng.gen();
366+
367+ let server = LocalServerBuilder::new(data.path().into(), key);
368+ let server = server.spawn(SocketAddr::from(([0, 0, 0, 0], 0))).await?;
369+ let addr = server.addr();
370+
371+ let (shutdown_tx, shutdown_rx) = oneshot::channel();
372+
373+ let server = tokio::spawn(futures::future::select(shutdown_rx, server));
374+
375+ let repo = GitRepo::init(addr)?;
376+ repo.add_random(Path::new("4mb.bin"), 4 * 1024 * 1024, &mut rng)?;
377+ repo.add_random(Path::new("8mb.bin"), 8 * 1024 * 1024, &mut rng)?;
378+ repo.add_random(Path::new("16mb.bin"), 16 * 1024 * 1024, &mut rng)?;
379+ repo.commit("Add LFS objects")?;
380+
381+ // Make sure we can push LFS objects to the server.
382+ repo.lfs_push()?;
383+
384+ // Make sure we can re-download the same objects.
385+ repo.clean_lfs()?;
386+ repo.lfs_pull()?;
387+
388+ // Push again. This should be super fast.
389+ repo.lfs_push()?;
390+
391+ shutdown_tx.send(()).expect("server died too soon");
392+
393+ if let Either::Right((result, _)) = server.await? {
394+ // If the server exited first, then propagate the error.
395+ result?;
396+ }
397+
398+ Ok(())
399+ }