rusttests/test_s3.rs -rw-r--r-- 5.9 KiB
1// Copyright (c) 2021 Jason White
2//
3// Permission is hereby granted, free of charge, to any person obtaining a copy
4// of this software and associated documentation files (the "Software"), to deal
5// in the Software without restriction, including without limitation the rights
6// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7// copies of the Software, and to permit persons to whom the Software is
8// furnished to do so, subject to the following conditions:
9//
10// The above copyright notice and this permission notice shall be included in
11// all copies or substantial portions of the Software.
12//
13// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19// SOFTWARE.
20
21//! This integration test only runs if the file `.test_credentials.toml` is in
22//! the same directory. Otherwise, it succeeds and does nothing.
23//!
24//! To run this test, create `tests/.test_credentials.toml` with the following
25//! contents:
26//!
27//! ```toml
28//! access_key_id = "XXXXXXXXXXXXXXXXXXXX"
29//! secret_access_key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
30//! default_region = "us-east-1"
31//! bucket = "my-test-bucket"
32//! ```
33//!
34//! Be sure to *only* use non-production credentials for testing purposes. We
35//! intentionally do not load the credentials from the environment to avoid
36//! clobbering any existing S3 bucket.
37
38mod common;
39
40use std::fs;
41use std::io;
42use std::net::SocketAddr;
43use std::path::Path;
44
45use futures::future::Either;
46use rand::rngs::StdRng;
47use rand::Rng;
48use rand::SeedableRng;
49use rudolfs::S3ServerBuilder;
50use serde::{Deserialize, Serialize};
51use tokio::sync::oneshot;
52
53use common::{init_logger, GitRepo, SERVER_ADDR};
54
55#[derive(Debug, Serialize, Deserialize)]
56struct Credentials {
57 access_key_id: String,
58 secret_access_key: String,
59 default_region: String,
60 bucket: String,
61}
62
63fn load_s3_credentials() -> io::Result<Credentials> {
64 let config = fs::read("tests/.test_credentials.toml")?;
65
66 // Try to load S3 credentials `.test_credentials.toml`. If they don't exist,
67 // then we can't really run this test. Note that these should be completely
68 // separate credentials than what is used in production.
69 let creds: Credentials = toml::from_slice(&config)?;
70
71 std::env::set_var("AWS_ACCESS_KEY_ID", &creds.access_key_id);
72 std::env::set_var("AWS_SECRET_ACCESS_KEY", &creds.secret_access_key);
73 std::env::set_var("AWS_DEFAULT_REGION", &creds.default_region);
74
75 Ok(creds)
76}
77
78#[tokio::test(flavor = "multi_thread")]
79async fn s3_smoke_test_encrypted() -> Result<(), Box<dyn std::error::Error>> {
80 init_logger();
81
82 let creds = match load_s3_credentials() {
83 Ok(creds) => creds,
84 Err(err) => {
85 eprintln!("Skipping test. No S3 credentials available: {}", err);
86 return Ok(());
87 }
88 };
89
90 // Make sure our seed is deterministic. This prevents us from filling up our
91 // S3 bucket with a bunch of random files if this test gets ran a bunch of
92 // times.
93 let mut rng = StdRng::seed_from_u64(42);
94
95 let key = rng.gen();
96
97 let mut server = S3ServerBuilder::new(creds.bucket);
98 server.key(key);
99 server.prefix("test_lfs".into());
100
101 let server = server.spawn(SERVER_ADDR).await?;
102 let addr = server.addr();
103
104 let (shutdown_tx, shutdown_rx) = oneshot::channel();
105
106 let server = tokio::spawn(futures::future::select(shutdown_rx, server));
107
108 exercise_server(addr, &mut rng)?;
109
110 shutdown_tx.send(()).expect("server died too soon");
111
112 if let Either::Right((result, _)) = server.await.unwrap() {
113 // If the server exited first, then propagate the error.
114 result.expect("server failed unexpectedly");
115 }
116
117 Ok(())
118}
119
120#[tokio::test(flavor = "multi_thread")]
121async fn s3_smoke_test_unencrypted() -> Result<(), Box<dyn std::error::Error>> {
122 init_logger();
123
124 let creds = match load_s3_credentials() {
125 Ok(creds) => creds,
126 Err(err) => {
127 eprintln!("Skipping test. No S3 credentials available: {}", err);
128 return Ok(());
129 }
130 };
131
132 // Make sure our seed is deterministic. This prevents us from filling up our
133 // S3 bucket with a bunch of random files if this test gets ran a bunch of
134 // times.
135 let mut rng = StdRng::seed_from_u64(42);
136
137 let mut server = S3ServerBuilder::new(creds.bucket);
138 server.prefix("test_lfs".into());
139
140 let server = server.spawn(SERVER_ADDR).await?;
141 let addr = server.addr();
142
143 let (shutdown_tx, shutdown_rx) = oneshot::channel();
144
145 let server = tokio::spawn(futures::future::select(shutdown_rx, server));
146
147 exercise_server(addr, &mut rng)?;
148
149 shutdown_tx.send(()).expect("server died too soon");
150
151 if let Either::Right((result, _)) = server.await.unwrap() {
152 // If the server exited first, then propagate the error.
153 result.expect("server failed unexpectedly");
154 }
155
156 Ok(())
157}
158
159/// Creates a repository with a few LFS files in it to exercise the LFS server.
160fn exercise_server(addr: SocketAddr, rng: &mut impl Rng) -> io::Result<()> {
161 let repo = GitRepo::init(addr)?;
162 repo.add_random(Path::new("4mb.bin"), 4 * 1024 * 1024, rng)?;
163 repo.add_random(Path::new("8mb.bin"), 8 * 1024 * 1024, rng)?;
164 repo.add_random(Path::new("16mb.bin"), 16 * 1024 * 1024, rng)?;
165 repo.commit("Add LFS objects")?;
166
167 // Make sure we can push LFS objects to the server.
168 repo.lfs_push().unwrap();
169
170 // Make sure we can re-download the same objects.
171 repo.clean_lfs().unwrap();
172 repo.lfs_pull().unwrap();
173
174 // Push again. This should be super fast.
175 repo.lfs_push().unwrap();
176
177 Ok(())
178}