Author: Jason White [github@jasonwhite.io]
Committer: GitHub [noreply@github.com] Fri, 19 Apr 2024 04:56:52 +0000
Hash: 224ee837ef7602071ff1a7b21cd2b628cda37c55
Timestamp: Fri, 19 Apr 2024 04:56:52 +0000 (5 months ago)

+210 -75 +/-8 browse
Add optional encryption (#67) 
Add optional encryption (#67)
1diff --git a/CHANGELOG.md b/CHANGELOG.md
2index 7021380..adb16c8 100644
3--- a/CHANGELOG.md
4+++ b/CHANGELOG.md
5 @@ -1,11 +1,16 @@
6 # Changelog
7
8+ ## v0.3.7
9+
10+ - Made encryption optional. If `--key` is not specified, LFS objects are not
11+ encrypted.
12+
13 ## v0.3.6
14
15- - Bumped versions of various dependencies.
16- - Fixed Minio environment variables in `docker-compose.minio.yml`
17- * `MINIO_ACCESS_KEY` was renamed to `MINIO_ROOT_USER`
18- * `MINIO_SECRET_KEY` was renamed to `MINIO_ROOT_PASSWORD`
19+ - Bumped versions of various dependencies.
20+ - Fixed Minio environment variables in `docker-compose.minio.yml`
21+ * `MINIO_ACCESS_KEY` was renamed to `MINIO_ROOT_USER`
22+ * `MINIO_SECRET_KEY` was renamed to `MINIO_ROOT_PASSWORD`
23
24 ## v0.3.5
25
26 diff --git a/README.md b/README.md
27index 6cfb98a..2e3f6da 100644
28--- a/README.md
29+++ b/README.md
30 @@ -43,22 +43,25 @@ know by submitting an issue.
31
32 ## Running It
33
34- ### Generate an encryption key
35+ ### Generate an encryption key (optional)
36
37- All LFS objects are encrypted with the xchacha20 symmetric stream cipher. You
38- must generate a 32-byte encryption key before starting the server.
39+ If configured, all LFS objects are encrypted with the xchacha20 symmetric stream
40+ cipher. You must generate a 32-byte encryption key before starting the server.
41
42 Generating a random key is easy:
43
44 openssl rand -hex 32
45
46 Keep this secret and save it in a password manager so you don't lose it. We will
47- pass this to the server below.
48+ pass this to the server below via the `--key` option. If the `--key` option is
49+ **not** specified, then the LFS objects are **not** encrypted.
50
51 **Note**:
52- - If the key ever changes, all existing LFS objects will become garbage.
53- When the Git LFS client attempts to download them, the SHA256 verification
54- step will fail.
55+ - If the key ever changes (or if encryption is disabled), all existing LFS
56+ objects will become garbage. When the Git LFS client attempts to download
57+ them, the SHA256 verification step will fail.
58+ - Likewise, if encryption is later enabled after it has been disabled, all
59+ existing unencrypted LFS objects will be seen as garbage.
60 - LFS objects in both the cache and in permanent storage are encrypted.
61 However, objects are decrypted before being sent to the LFS client, so take
62 any necessary precautions to keep your intellectual property safe.
63 diff --git a/rustfmt.toml b/rustfmt.toml
64index 8118c57..5687b6a 100644
65--- a/rustfmt.toml
66+++ b/rustfmt.toml
67 @@ -1,7 +1,4 @@
68 max_width = 80
69 format_strings = true
70- error_on_line_overflow = true
71- error_on_unformatted = true
72 normalize_comments = true
73 wrap_comments = true
74- license_template_path = ".license_template"
75 diff --git a/src/lib.rs b/src/lib.rs
76index 8c0d2db..51a5c9f 100644
77--- a/src/lib.rs
78+++ b/src/lib.rs
79 @@ -82,19 +82,19 @@ impl Cache {
80 #[derive(Debug)]
81 pub struct S3ServerBuilder {
82 bucket: String,
83- key: [u8; 32],
84+ key: Option<[u8; 32]>,
85 prefix: Option<String>,
86 cdn: Option<String>,
87 cache: Option<Cache>,
88 }
89
90 impl S3ServerBuilder {
91- pub fn new(bucket: String, key: [u8; 32]) -> Self {
92+ pub fn new(bucket: String) -> Self {
93 Self {
94 bucket,
95 prefix: None,
96 cdn: None,
97- key,
98+ key: None,
99 cache: None,
100 }
101 }
102 @@ -107,7 +107,7 @@ impl S3ServerBuilder {
103
104 /// Sets the encryption key to use.
105 pub fn key(&mut self, key: [u8; 32]) -> &mut Self {
106- self.key = key;
107+ self.key = Some(key);
108 self
109 }
110
111 @@ -174,13 +174,28 @@ impl S3ServerBuilder {
112 let disk = Faulty::new(disk);
113
114 let cache = Cached::new(cache.max_size, disk, s3).await?;
115- let storage = Verify::new(Encrypted::new(self.key, cache));
116- Ok(Box::new(spawn_server(storage, &addr)))
117- }
118- None => {
119- let storage = Verify::new(Encrypted::new(self.key, s3));
120- Ok(Box::new(spawn_server(storage, &addr)))
121+
122+ match self.key {
123+ Some(key) => {
124+ let storage = Verify::new(Encrypted::new(key, cache));
125+ Ok(Box::new(spawn_server(storage, &addr)))
126+ }
127+ None => {
128+ let storage = Verify::new(cache);
129+ Ok(Box::new(spawn_server(storage, &addr)))
130+ }
131+ }
132 }
133+ None => match self.key {
134+ Some(key) => {
135+ let storage = Verify::new(Encrypted::new(key, s3));
136+ Ok(Box::new(spawn_server(storage, &addr)))
137+ }
138+ None => {
139+ let storage = Verify::new(s3);
140+ Ok(Box::new(spawn_server(storage, &addr)))
141+ }
142+ },
143 }
144 }
145
146 @@ -202,24 +217,24 @@ impl S3ServerBuilder {
147 #[derive(Debug)]
148 pub struct LocalServerBuilder {
149 path: PathBuf,
150- key: [u8; 32],
151+ key: Option<[u8; 32]>,
152 cache: Option<Cache>,
153 }
154
155 impl LocalServerBuilder {
156 /// Creates a local server builder. `path` is the path to the folder where
157 /// all of the LFS data will be stored.
158- pub fn new(path: PathBuf, key: [u8; 32]) -> Self {
159+ pub fn new(path: PathBuf) -> Self {
160 Self {
161 path,
162- key,
163+ key: None,
164 cache: None,
165 }
166 }
167
168 /// Sets the encryption key to use.
169 pub fn key(&mut self, key: [u8; 32]) -> &mut Self {
170- self.key = key;
171+ self.key = Some(key);
172 self
173 }
174
175 @@ -238,13 +253,24 @@ impl LocalServerBuilder {
176 pub async fn spawn(
177 self,
178 addr: SocketAddr,
179- ) -> Result<impl Server, Box<dyn std::error::Error>> {
180+ ) -> Result<Box<dyn Server + Unpin + Send>, Box<dyn std::error::Error>>
181+ {
182 let storage = Disk::new(self.path).map_err(Error::from).await?;
183- let storage = Verify::new(Encrypted::new(self.key, storage));
184-
185- log::info!("Local disk storage initialized.");
186
187- Ok(spawn_server(storage, &addr))
188+ match self.key {
189+ Some(key) => {
190+ let storage = Verify::new(Encrypted::new(key, storage));
191+ log::info!("Local disk storage initialized (with encryption).");
192+ Ok(Box::new(spawn_server(storage, &addr)))
193+ }
194+ None => {
195+ let storage = Verify::new(storage);
196+ log::info!(
197+ "Local disk storage initialized (without encryption)."
198+ );
199+ Ok(Box::new(spawn_server(storage, &addr)))
200+ }
201+ }
202 }
203
204 /// Spawns the server and runs it to completion. This will run forever
205 diff --git a/src/main.rs b/src/main.rs
206index c2b8436..7cfc6a6 100644
207--- a/src/main.rs
208+++ b/src/main.rs
209 @@ -67,7 +67,7 @@ struct GlobalArgs {
210 parse(try_from_str = FromHex::from_hex),
211 env = "RUDOLFS_KEY"
212 )]
213- key: [u8; 32],
214+ key: Option<[u8; 32]>,
215
216 /// Root directory of the object cache. If not specified or if the local
217 /// disk is the storage backend, then no local disk cache will be used.
218 @@ -156,9 +156,13 @@ impl S3Args {
219 addr: SocketAddr,
220 global_args: GlobalArgs,
221 ) -> Result<(), Box<dyn std::error::Error>> {
222- let mut builder = S3ServerBuilder::new(self.bucket, global_args.key);
223+ let mut builder = S3ServerBuilder::new(self.bucket);
224 builder.prefix(self.prefix);
225
226+ if let Some(key) = global_args.key {
227+ builder.key(key);
228+ }
229+
230 if let Some(cdn) = self.cdn {
231 builder.cdn(cdn);
232 }
233 @@ -181,7 +185,11 @@ impl LocalArgs {
234 addr: SocketAddr,
235 global_args: GlobalArgs,
236 ) -> Result<(), Box<dyn std::error::Error>> {
237- let mut builder = LocalServerBuilder::new(self.path, global_args.key);
238+ let mut builder = LocalServerBuilder::new(self.path);
239+
240+ if let Some(key) = global_args.key {
241+ builder.key(key);
242+ }
243
244 if let Some(cache_dir) = global_args.cache_dir {
245 let max_cache_size = global_args
246 diff --git a/tests/common/mod.rs b/tests/common/mod.rs
247index b8d17e7..1f32a64 100644
248--- a/tests/common/mod.rs
249+++ b/tests/common/mod.rs
250 @@ -17,6 +17,7 @@
251 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
252 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
253 // SOFTWARE.
254+ #![allow(unused)]
255 use std::fs::{self, File};
256 use std::io;
257 use std::net::SocketAddr;
258 diff --git a/tests/test_local.rs b/tests/test_local.rs
259index 646b7ee..5c9a51e 100644
260--- a/tests/test_local.rs
261+++ b/tests/test_local.rs
262 @@ -19,6 +19,7 @@
263 // SOFTWARE.
264 mod common;
265
266+ use std::io;
267 use std::net::SocketAddr;
268 use std::path::Path;
269
270 @@ -26,13 +27,14 @@ use futures::future::Either;
271 use rand::rngs::StdRng;
272 use rand::Rng;
273 use rand::SeedableRng;
274- use rudolfs::{LocalServerBuilder, Server};
275+ use rudolfs::LocalServerBuilder;
276 use tokio::sync::oneshot;
277
278 use common::{init_logger, GitRepo};
279
280 #[tokio::test(flavor = "multi_thread")]
281- async fn local_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
282+ async fn local_smoke_test_encrypted() -> Result<(), Box<dyn std::error::Error>>
283+ {
284 init_logger();
285
286 // Make sure our seed is deterministic. This makes it easier to reproduce
287 @@ -42,7 +44,8 @@ async fn local_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
288 let data = tempfile::TempDir::new()?;
289 let key = rng.gen();
290
291- let server = LocalServerBuilder::new(data.path().into(), key);
292+ let mut server = LocalServerBuilder::new(data.path().into());
293+ server.key(key);
294 let server = server.spawn(SocketAddr::from(([0, 0, 0, 0], 0))).await?;
295 let addr = server.addr();
296
297 @@ -50,10 +53,55 @@ async fn local_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
298
299 let server = tokio::spawn(futures::future::select(shutdown_rx, server));
300
301+ exercise_server(addr, &mut rng)?;
302+
303+ shutdown_tx.send(()).expect("server died too soon");
304+
305+ if let Either::Right((result, _)) = server.await? {
306+ // If the server exited first, then propagate the error.
307+ result?;
308+ }
309+
310+ Ok(())
311+ }
312+
313+ #[tokio::test(flavor = "multi_thread")]
314+ async fn local_smoke_test_unencrypted() -> Result<(), Box<dyn std::error::Error>>
315+ {
316+ init_logger();
317+
318+ // Make sure our seed is deterministic. This makes it easier to reproduce
319+ // the same repo every time.
320+ let mut rng = StdRng::seed_from_u64(42);
321+
322+ let data = tempfile::TempDir::new()?;
323+
324+ let server = LocalServerBuilder::new(data.path().into());
325+ let server = server.spawn(SocketAddr::from(([0, 0, 0, 0], 0))).await?;
326+ let addr = server.addr();
327+
328+ let (shutdown_tx, shutdown_rx) = oneshot::channel();
329+
330+ let server = tokio::spawn(futures::future::select(shutdown_rx, server));
331+
332+ exercise_server(addr, &mut rng)?;
333+
334+ shutdown_tx.send(()).expect("server died too soon");
335+
336+ if let Either::Right((result, _)) = server.await? {
337+ // If the server exited first, then propagate the error.
338+ result?;
339+ }
340+
341+ Ok(())
342+ }
343+
344+ /// Creates a repository with a few LFS files in it to exercise the LFS server.
345+ fn exercise_server(addr: SocketAddr, rng: &mut impl Rng) -> io::Result<()> {
346 let repo = GitRepo::init(addr)?;
347- repo.add_random(Path::new("4mb.bin"), 4 * 1024 * 1024, &mut rng)?;
348- repo.add_random(Path::new("8mb.bin"), 8 * 1024 * 1024, &mut rng)?;
349- repo.add_random(Path::new("16mb.bin"), 16 * 1024 * 1024, &mut rng)?;
350+ repo.add_random(Path::new("4mb.bin"), 4 * 1024 * 1024, rng)?;
351+ repo.add_random(Path::new("8mb.bin"), 8 * 1024 * 1024, rng)?;
352+ repo.add_random(Path::new("16mb.bin"), 16 * 1024 * 1024, rng)?;
353 repo.commit("Add LFS objects")?;
354
355 // Make sure we can push LFS objects to the server.
356 @@ -73,19 +121,12 @@ async fn local_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
357 repo_clone.lfs_pull()?;
358
359 // Add some more files and make sure you can pull those into the clone
360- repo.add_random(Path::new("4mb_2.bin"), 4 * 1024 * 1024, &mut rng)?;
361- repo.add_random(Path::new("8mb_2.bin"), 8 * 1024 * 1024, &mut rng)?;
362- repo.add_random(Path::new("16mb_2.bin"), 16 * 1024 * 1024, &mut rng)?;
363+ repo.add_random(Path::new("4mb_2.bin"), 4 * 1024 * 1024, rng)?;
364+ repo.add_random(Path::new("8mb_2.bin"), 8 * 1024 * 1024, rng)?;
365+ repo.add_random(Path::new("16mb_2.bin"), 16 * 1024 * 1024, rng)?;
366 repo.commit("Add LFS objects 2")?;
367
368 repo_clone.pull()?;
369
370- shutdown_tx.send(()).expect("server died too soon");
371-
372- if let Either::Right((result, _)) = server.await? {
373- // If the server exited first, then propagate the error.
374- result?;
375- }
376-
377 Ok(())
378 }
379 diff --git a/tests/test_s3.rs b/tests/test_s3.rs
380index c4c2e71..2cf9a8c 100644
381--- a/tests/test_s3.rs
382+++ b/tests/test_s3.rs
383 @@ -38,6 +38,7 @@
384 mod common;
385
386 use std::fs;
387+ use std::io;
388 use std::net::SocketAddr;
389 use std::path::Path;
390
391 @@ -59,27 +60,33 @@ struct Credentials {
392 bucket: String,
393 }
394
395+ fn load_s3_credentials() -> io::Result<Credentials> {
396+ let config = fs::read("tests/.test_credentials.toml")?;
397+
398+ // Try to load S3 credentials `.test_credentials.toml`. If they don't exist,
399+ // then we can't really run this test. Note that these should be completely
400+ // separate credentials than what is used in production.
401+ let creds: Credentials = toml::from_slice(&config)?;
402+
403+ std::env::set_var("AWS_ACCESS_KEY_ID", &creds.access_key_id);
404+ std::env::set_var("AWS_SECRET_ACCESS_KEY", &creds.secret_access_key);
405+ std::env::set_var("AWS_DEFAULT_REGION", &creds.default_region);
406+
407+ Ok(creds)
408+ }
409+
410 #[tokio::test(flavor = "multi_thread")]
411- async fn s3_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
412+ async fn s3_smoke_test_encrypted() -> Result<(), Box<dyn std::error::Error>> {
413 init_logger();
414
415- let config = match fs::read("tests/.test_credentials.toml") {
416- Ok(bytes) => bytes,
417+ let creds = match load_s3_credentials() {
418+ Ok(creds) => creds,
419 Err(err) => {
420 eprintln!("Skipping test. No S3 credentials available: {}", err);
421 return Ok(());
422 }
423 };
424
425- // Try to load S3 credentials `.test_credentials.toml`. If they don't exist,
426- // then we can't really run this test. Note that these should be completely
427- // separate credentials than what is used in production.
428- let creds: Credentials = toml::from_slice(&config)?;
429-
430- std::env::set_var("AWS_ACCESS_KEY_ID", creds.access_key_id);
431- std::env::set_var("AWS_SECRET_ACCESS_KEY", creds.secret_access_key);
432- std::env::set_var("AWS_DEFAULT_REGION", creds.default_region);
433-
434 // Make sure our seed is deterministic. This prevents us from filling up our
435 // S3 bucket with a bunch of random files if this test gets ran a bunch of
436 // times.
437 @@ -87,7 +94,8 @@ async fn s3_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
438
439 let key = rng.gen();
440
441- let mut server = S3ServerBuilder::new(creds.bucket, key);
442+ let mut server = S3ServerBuilder::new(creds.bucket);
443+ server.key(key);
444 server.prefix("test_lfs".into());
445
446 let server = server.spawn(SocketAddr::from(([0, 0, 0, 0], 0))).await?;
447 @@ -97,10 +105,63 @@ async fn s3_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
448
449 let server = tokio::spawn(futures::future::select(shutdown_rx, server));
450
451+ exercise_server(addr, &mut rng)?;
452+
453+ shutdown_tx.send(()).expect("server died too soon");
454+
455+ if let Either::Right((result, _)) = server.await.unwrap() {
456+ // If the server exited first, then propagate the error.
457+ result.expect("server failed unexpectedly");
458+ }
459+
460+ Ok(())
461+ }
462+
463+ #[tokio::test(flavor = "multi_thread")]
464+ async fn s3_smoke_test_unencrypted() -> Result<(), Box<dyn std::error::Error>> {
465+ init_logger();
466+
467+ let creds = match load_s3_credentials() {
468+ Ok(creds) => creds,
469+ Err(err) => {
470+ eprintln!("Skipping test. No S3 credentials available: {}", err);
471+ return Ok(());
472+ }
473+ };
474+
475+ // Make sure our seed is deterministic. This prevents us from filling up our
476+ // S3 bucket with a bunch of random files if this test gets ran a bunch of
477+ // times.
478+ let mut rng = StdRng::seed_from_u64(42);
479+
480+ let mut server = S3ServerBuilder::new(creds.bucket);
481+ server.prefix("test_lfs".into());
482+
483+ let server = server.spawn(SocketAddr::from(([0, 0, 0, 0], 0))).await?;
484+ let addr = server.addr();
485+
486+ let (shutdown_tx, shutdown_rx) = oneshot::channel();
487+
488+ let server = tokio::spawn(futures::future::select(shutdown_rx, server));
489+
490+ exercise_server(addr, &mut rng)?;
491+
492+ shutdown_tx.send(()).expect("server died too soon");
493+
494+ if let Either::Right((result, _)) = server.await.unwrap() {
495+ // If the server exited first, then propagate the error.
496+ result.expect("server failed unexpectedly");
497+ }
498+
499+ Ok(())
500+ }
501+
502+ /// Creates a repository with a few LFS files in it to exercise the LFS server.
503+ fn exercise_server(addr: SocketAddr, rng: &mut impl Rng) -> io::Result<()> {
504 let repo = GitRepo::init(addr)?;
505- repo.add_random(Path::new("4mb.bin"), 4 * 1024 * 1024, &mut rng)?;
506- repo.add_random(Path::new("8mb.bin"), 8 * 1024 * 1024, &mut rng)?;
507- repo.add_random(Path::new("16mb.bin"), 16 * 1024 * 1024, &mut rng)?;
508+ repo.add_random(Path::new("4mb.bin"), 4 * 1024 * 1024, rng)?;
509+ repo.add_random(Path::new("8mb.bin"), 8 * 1024 * 1024, rng)?;
510+ repo.add_random(Path::new("16mb.bin"), 16 * 1024 * 1024, rng)?;
511 repo.commit("Add LFS objects")?;
512
513 // Make sure we can push LFS objects to the server.
514 @@ -113,12 +174,5 @@ async fn s3_smoke_test() -> Result<(), Box<dyn std::error::Error>> {
515 // Push again. This should be super fast.
516 repo.lfs_push().unwrap();
517
518- shutdown_tx.send(()).expect("server died too soon");
519-
520- if let Either::Right((result, _)) = server.await.unwrap() {
521- // If the server exited first, then propagate the error.
522- result.expect("server failed unexpectedly");
523- }
524-
525 Ok(())
526 }