From 606f08648e3a711d33b7b54cd7d59e4d40c9d746 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 16 Oct 2022 01:12:39 +0200 Subject: [PATCH 01/10] Make backend sync --- Cargo.lock | 3 - Cargo.toml | 4 +- src/archiver/archiver_impl.rs | 8 +-- src/archiver/parent.rs | 8 +-- src/backend/cache.rs | 77 +++++++++++-------------- src/backend/choose.rs | 55 ++++++++---------- src/backend/decrypt.rs | 64 ++++++++++----------- src/backend/dry_run.rs | 35 +++++------- src/backend/hotcold.rs | 37 ++++++------ src/backend/local.rs | 23 +++----- src/backend/mod.rs | 33 +++++------ src/backend/rclone.rs | 32 +++++------ src/backend/rest.rs | 102 +++++++++++++--------------------- src/blob/packer.rs | 40 ++++++------- src/blob/tree.rs | 21 ++++--- src/commands/backup.rs | 4 +- src/commands/cat.rs | 19 +++---- src/commands/check.rs | 17 +++--- src/commands/config.rs | 6 +- src/commands/diff.rs | 8 +-- src/commands/helpers.rs | 7 +-- src/commands/init.rs | 15 +++-- src/commands/key.rs | 9 ++- src/commands/list.rs | 2 +- src/commands/ls.rs | 4 +- src/commands/mod.rs | 18 +++--- src/commands/prune.rs | 24 +++----- src/commands/repair.rs | 24 +++----- src/commands/repoinfo.rs | 2 +- src/commands/restore.rs | 7 +-- src/index/indexer.rs | 20 +++---- src/index/mod.rs | 24 ++++---- src/repo/keyfile.rs | 22 +++----- src/repo/packfile.rs | 9 +-- src/repo/snapshotfile.rs | 14 ++--- 35 files changed, 337 insertions(+), 460 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 042eb2f..93dcfea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,12 +119,9 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ - "futures-core", "getrandom", "instant", - "pin-project-lite", "rand", - "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 03f1915..a2f7599 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,8 +64,8 @@ ignore = "0.4" nix = "0.25" filetime = "0.2" # rest backend -reqwest = {version = "0.11", default-features = false, features = ["json", "rustls-tls", "stream"] } -backoff = { version = "0.4", features = ["tokio"] } +reqwest = {version = "0.11", default-features = false, features = ["json", "rustls-tls", "stream", "blocking"] } +backoff = "0.4" # rclone backend semver = "1" # cache diff --git a/src/archiver/archiver_impl.rs b/src/archiver/archiver_impl.rs index a1ddb60..7e6130e 100644 --- a/src/archiver/archiver_impl.rs +++ b/src/archiver/archiver_impl.rs @@ -122,13 +122,13 @@ impl Archiver { let tree = std::mem::replace(&mut self.tree, Tree::new()); if self.path == path { // use Node and return - let new_parent = self.parent.sub_parent(&node).await?; + let new_parent = self.parent.sub_parent(&node)?; let parent = std::mem::replace(&mut self.parent, new_parent); self.stack.push((node, tree, parent)); return Ok(()); } else { let node = Node::new_node(p, NodeType::Dir, Metadata::default()); - let new_parent = self.parent.sub_parent(&node).await?; + let new_parent = self.parent.sub_parent(&node)?; let parent = std::mem::replace(&mut self.parent, new_parent); self.stack.push((node, tree, parent)); }; @@ -294,7 +294,7 @@ impl Archiver { self.tree_packer.finalize().await?; { let indexer = self.indexer.write().await; - indexer.finalize().await?; + indexer.finalize()?; } let end_time = Local::now(); self.summary.backup_duration = (end_time - self.summary.backup_start) @@ -303,7 +303,7 @@ impl Archiver { self.summary.total_duration = (end_time - self.snap.time).to_std()?.as_secs_f64(); self.summary.backup_end = end_time; self.snap.summary = Some(self.summary); - let id = self.be.save_file(&self.snap).await?; + let id = self.be.save_file(&self.snap)?; self.snap.id = id; Ok(self.snap) diff --git a/src/archiver/parent.rs b/src/archiver/parent.rs index 1e06fd0..bdb2d00 100644 --- a/src/archiver/parent.rs +++ b/src/archiver/parent.rs @@ -21,12 +21,12 @@ pub enum ParentResult { } impl Parent { - pub async fn new(be: &BE, tree_id: Option, ignore_ctime: bool, ignore_inode: bool) -> Self { + pub fn new(be: &BE, tree_id: Option, ignore_ctime: bool, ignore_inode: bool) -> Self { // if tree_id is given, load tree from backend. Turn errors into None. // TODO: print warning when loading failed let tree = match tree_id { None => None, - Some(id) => Tree::from_backend(be, id).await.ok(), + Some(id) => Tree::from_backend(be, id).ok(), }; Self { tree, @@ -85,14 +85,14 @@ impl Parent { } } - pub async fn sub_parent(&mut self, node: &Node) -> Result { + pub fn sub_parent(&mut self, node: &Node) -> Result { let tree = match self.p_node(node) { None => None, Some(p_node) => { if p_node.node_type() == node.node_type() { // TODO: print warning when loading failed let tree = p_node.subtree.unwrap(); - Some(Tree::from_backend(&self.be, tree).await.ok()).flatten() + Some(Tree::from_backend(&self.be, tree).ok()).flatten() } else { None } diff --git a/src/backend/cache.rs b/src/backend/cache.rs index 18b4c70..fccb5cf 100644 --- a/src/backend/cache.rs +++ b/src/backend/cache.rs @@ -4,7 +4,6 @@ use std::io::{Read, Seek, SeekFrom, Write}; use std::path::PathBuf; use anyhow::{anyhow, Result}; -use async_trait::async_trait; use bytes::Bytes; use dirs::cache_dir; use log::*; @@ -24,7 +23,6 @@ impl CachedBackend { } } -#[async_trait] impl ReadBackend for CachedBackend { fn location(&self) -> &str { self.be.location() @@ -34,27 +32,27 @@ impl ReadBackend for CachedBackend { self.be.set_option(option, value) } - async fn list_with_size(&self, tpe: FileType) -> Result> { - let list = self.be.list_with_size(tpe).await?; + fn list_with_size(&self, tpe: FileType) -> Result> { + let list = self.be.list_with_size(tpe)?; if let Some(cache) = &self.cache { if tpe.is_cacheable() { - cache.remove_not_in_list(tpe, &list).await?; + cache.remove_not_in_list(tpe, &list)?; } } Ok(list) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { + fn read_full(&self, tpe: FileType, id: &Id) -> Result { match (&self.cache, tpe.is_cacheable()) { - (None, _) | (Some(_), false) => self.be.read_full(tpe, id).await, - (Some(cache), true) => match cache.read_full(tpe, id).await { + (None, _) | (Some(_), false) => self.be.read_full(tpe, id), + (Some(cache), true) => match cache.read_full(tpe, id) { Ok(res) => Ok(res), _ => { - let res = self.be.read_full(tpe, id).await; + let res = self.be.read_full(tpe, id); if let Ok(data) = &res { - let _ = cache.write_bytes(tpe, id, data.clone()).await; + let _ = cache.write_bytes(tpe, id, data.clone()); } res } @@ -62,7 +60,7 @@ impl ReadBackend for CachedBackend { } } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -72,21 +70,19 @@ impl ReadBackend for CachedBackend { ) -> Result { match (&self.cache, cacheable || tpe.is_cacheable()) { (None, _) | (Some(_), false) => { - self.be - .read_partial(tpe, id, cacheable, offset, length) - .await + self.be.read_partial(tpe, id, cacheable, offset, length) } - (Some(cache), true) => match cache.read_partial(tpe, id, offset, length).await { + (Some(cache), true) => match cache.read_partial(tpe, id, offset, length) { Ok(res) => Ok(res), - _ => match self.be.read_full(tpe, id).await { + _ => match self.be.read_full(tpe, id) { // read full file, save to cache and return partial content from cache - // TODO: - Do not read to memory, but use a (Async)Reader + // TODO: - Do not read to memory, but use a Reader // - Don't read from cache, but use the right part of the read content Ok(data) => { - if cache.write_bytes(tpe, id, data.clone()).await.is_ok() { - cache.read_partial(tpe, id, offset, length).await + if cache.write_bytes(tpe, id, data).is_ok() { + cache.read_partial(tpe, id, offset, length) } else { - self.be.read_partial(tpe, id, false, offset, length).await + self.be.read_partial(tpe, id, false, offset, length) } } error => error, @@ -96,28 +92,27 @@ impl ReadBackend for CachedBackend { } } -#[async_trait] impl WriteBackend for CachedBackend { - async fn create(&self) -> Result<()> { - self.be.create().await + fn create(&self) -> Result<()> { + self.be.create() } - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { if let Some(cache) = &self.cache { if cacheable || tpe.is_cacheable() { - let _ = cache.write_bytes(tpe, id, buf.clone()).await; + let _ = cache.write_bytes(tpe, id, buf.clone()); } } - self.be.write_bytes(tpe, id, cacheable, buf).await + self.be.write_bytes(tpe, id, cacheable, buf) } - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { if let Some(cache) = &self.cache { if cacheable || tpe.is_cacheable() { - let _ = cache.remove(tpe, id).await; + let _ = cache.remove(tpe, id); } } - self.be.remove(tpe, id, cacheable).await + self.be.remove(tpe, id, cacheable) } } @@ -154,7 +149,7 @@ impl Cache { self.path.join(tpe.name()).join(&hex_id[0..2]).join(&hex_id) } - pub async fn list_with_size(&self, tpe: FileType) -> Result> { + pub fn list_with_size(&self, tpe: FileType) -> Result> { let path = self.path.join(tpe.name()); let walker = WalkDir::new(path) @@ -183,38 +178,32 @@ impl Cache { Ok(walker.collect()) } - pub async fn remove_not_in_list(&self, tpe: FileType, list: &Vec<(Id, u32)>) -> Result<()> { - let mut list_cache = self.list_with_size(tpe).await?; + pub fn remove_not_in_list(&self, tpe: FileType, list: &Vec<(Id, u32)>) -> Result<()> { + let mut list_cache = self.list_with_size(tpe)?; // remove present files from the cache list for (id, size) in list { if let Some(cached_size) = list_cache.remove(id) { if &cached_size != size { // remove cache files with non-matching size - self.remove(tpe, id).await?; + self.remove(tpe, id)?; } } } // remove all remaining (i.e. not present in repo) cache files for id in list_cache.keys() { - self.remove(tpe, id).await?; + self.remove(tpe, id)?; } Ok(()) } - pub async fn read_full(&self, tpe: FileType, id: &Id) -> Result { + pub fn read_full(&self, tpe: FileType, id: &Id) -> Result { trace!("cache reading tpe: {:?}, id: {}", &tpe, &id); let data = fs::read(self.path(tpe, id))?; trace!("cache hit!"); Ok(data.into()) } - async fn read_partial( - &self, - tpe: FileType, - id: &Id, - offset: u32, - length: u32, - ) -> Result { + fn read_partial(&self, tpe: FileType, id: &Id, offset: u32, length: u32) -> Result { trace!( "cache reading tpe: {:?}, id: {}, offset: {}", &tpe, @@ -229,7 +218,7 @@ impl Cache { Ok(vec.into()) } - async fn write_bytes(&self, tpe: FileType, id: &Id, buf: Bytes) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, buf: Bytes) -> Result<()> { trace!("cache writing tpe: {:?}, id: {}", &tpe, &id); fs::create_dir_all(self.dir(tpe, id))?; let filename = self.path(tpe, id); @@ -241,7 +230,7 @@ impl Cache { Ok(()) } - async fn remove(&self, tpe: FileType, id: &Id) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id) -> Result<()> { trace!("cache writing tpe: {:?}, id: {}", &tpe, &id); let filename = self.path(tpe, id); fs::remove_file(filename)?; diff --git a/src/backend/choose.rs b/src/backend/choose.rs index d1c8a2f..503e018 100644 --- a/src/backend/choose.rs +++ b/src/backend/choose.rs @@ -1,5 +1,4 @@ use anyhow::{bail, Result}; -use async_trait::async_trait; use bytes::Bytes; use super::{FileType, Id, ReadBackend, WriteBackend}; @@ -26,7 +25,6 @@ impl ChooseBackend { } } -#[async_trait] impl ReadBackend for ChooseBackend { fn location(&self) -> &str { match self { @@ -44,23 +42,23 @@ impl ReadBackend for ChooseBackend { } } - async fn list_with_size(&self, tpe: FileType) -> Result> { + fn list_with_size(&self, tpe: FileType) -> Result> { match self { - Local(local) => local.list_with_size(tpe).await, - Rest(rest) => rest.list_with_size(tpe).await, - Rclone(rclone) => rclone.list_with_size(tpe).await, + Local(local) => local.list_with_size(tpe), + Rest(rest) => rest.list_with_size(tpe), + Rclone(rclone) => rclone.list_with_size(tpe), } } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { + fn read_full(&self, tpe: FileType, id: &Id) -> Result { match self { - Local(local) => local.read_full(tpe, id).await, - Rest(rest) => rest.read_full(tpe, id).await, - Rclone(rclone) => rclone.read_full(tpe, id).await, + Local(local) => local.read_full(tpe, id), + Rest(rest) => rest.read_full(tpe, id), + Rclone(rclone) => rclone.read_full(tpe, id), } } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -69,40 +67,35 @@ impl ReadBackend for ChooseBackend { length: u32, ) -> Result { match self { - Local(local) => local.read_partial(tpe, id, cacheable, offset, length).await, - Rest(rest) => rest.read_partial(tpe, id, cacheable, offset, length).await, - Rclone(rclone) => { - rclone - .read_partial(tpe, id, cacheable, offset, length) - .await - } + Local(local) => local.read_partial(tpe, id, cacheable, offset, length), + Rest(rest) => rest.read_partial(tpe, id, cacheable, offset, length), + Rclone(rclone) => rclone.read_partial(tpe, id, cacheable, offset, length), } } } -#[async_trait] impl WriteBackend for ChooseBackend { - async fn create(&self) -> Result<()> { + fn create(&self) -> Result<()> { match self { - Local(local) => local.create().await, - Rest(rest) => rest.create().await, - Rclone(rclone) => rclone.create().await, + Local(local) => local.create(), + Rest(rest) => rest.create(), + Rclone(rclone) => rclone.create(), } } - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { match self { - Local(local) => local.write_bytes(tpe, id, cacheable, buf).await, - Rest(rest) => rest.write_bytes(tpe, id, cacheable, buf).await, - Rclone(rclone) => rclone.write_bytes(tpe, id, cacheable, buf).await, + Local(local) => local.write_bytes(tpe, id, cacheable, buf), + Rest(rest) => rest.write_bytes(tpe, id, cacheable, buf), + Rclone(rclone) => rclone.write_bytes(tpe, id, cacheable, buf), } } - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { match self { - Local(local) => local.remove(tpe, id, cacheable).await, - Rest(rest) => rest.remove(tpe, id, cacheable).await, - Rclone(rclone) => rclone.remove(tpe, id, cacheable).await, + Local(local) => local.remove(tpe, id, cacheable), + Rest(rest) => rest.remove(tpe, id, cacheable), + Rclone(rclone) => rclone.remove(tpe, id, cacheable), } } } diff --git a/src/backend/decrypt.rs b/src/backend/decrypt.rs index 3d13ddd..c30fc1e 100644 --- a/src/backend/decrypt.rs +++ b/src/backend/decrypt.rs @@ -18,8 +18,8 @@ impl DecryptFullBackend for T {} pub trait DecryptReadBackend: ReadBackend { fn decrypt(&self, data: &[u8]) -> Result>; - async fn read_encrypted_full(&self, tpe: FileType, id: &Id) -> Result { - let decrypted = self.decrypt(&self.read_full(tpe, id).await?)?; + fn read_encrypted_full(&self, tpe: FileType, id: &Id) -> Result { + let decrypted = self.decrypt(&self.read_full(tpe, id)?)?; Ok(match decrypted[0] { b'{' | b'[' => decrypted, // not compressed 2 => decode_all(&decrypted[1..])?, // 2 indicates compressed data following @@ -28,7 +28,7 @@ pub trait DecryptReadBackend: ReadBackend { .into()) } - async fn read_encrypted_partial( + fn read_encrypted_partial( &self, tpe: FileType, id: &Id, @@ -37,11 +37,7 @@ pub trait DecryptReadBackend: ReadBackend { length: u32, uncompressed_length: Option, ) -> Result { - let mut data = self.decrypt( - &self - .read_partial(tpe, id, cacheable, offset, length) - .await?, - )?; + let mut data = self.decrypt(&self.read_partial(tpe, id, cacheable, offset, length)?)?; if let Some(length) = uncompressed_length { data = decode_all(&*data).unwrap(); if data.len() != length.get() as usize { @@ -51,8 +47,8 @@ pub trait DecryptReadBackend: ReadBackend { Ok(data.into()) } - async fn get_file(&self, id: &Id) -> Result { - let data = self.read_encrypted_full(F::TYPE, id).await?; + fn get_file(&self, id: &Id) -> Result { + let data = self.read_encrypted_full(F::TYPE, id)?; Ok(serde_json::from_slice(&data)?) } @@ -60,7 +56,7 @@ pub trait DecryptReadBackend: ReadBackend { &self, p: ProgressBar, ) -> Result>> { - let list = self.list(F::TYPE).await?; + let list = self.list(F::TYPE)?; self.stream_list(list, p).await } @@ -77,7 +73,7 @@ pub trait DecryptReadBackend: ReadBackend { let be = self.clone(); let p = p.clone(); spawn(async move { - let file = be.get_file::(&id).await.unwrap(); + let file = be.get_file::(&id).unwrap(); p.inc(1); (id, file) }) @@ -91,11 +87,11 @@ pub trait DecryptReadBackend: ReadBackend { pub trait DecryptWriteBackend: WriteBackend { type Key: CryptoKey; fn key(&self) -> &Self::Key; - async fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result; + fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result; - async fn save_file(&self, file: &F) -> Result { + fn save_file(&self, file: &F) -> Result { let data = serde_json::to_vec(file)?; - Ok(self.hash_write_full(F::TYPE, &data).await?) + Ok(self.hash_write_full(F::TYPE, &data)?) } async fn save_list(&self, list: Vec, p: ProgressBar) -> Result<()> { @@ -106,7 +102,7 @@ pub trait DecryptWriteBackend: WriteBackend { (file, be, p) })) .for_each_concurrent(5, |(file, be, p)| async move { - be.save_file(&file).await.unwrap(); + be.save_file(&file).unwrap(); p.inc(1); }) .await; @@ -128,7 +124,7 @@ pub trait DecryptWriteBackend: WriteBackend { (id, be, p) })) .for_each_concurrent(20, |(id, be, p)| async move { - be.remove(tpe, &id, cacheable).await.unwrap(); + be.remove(tpe, &id, cacheable).unwrap(); p.inc(1); }) .await; @@ -163,7 +159,8 @@ impl DecryptWriteBackend for DecryptBackend fn key(&self) -> &Self::Key { &self.key } - async fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result { + + fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result { let data = match self.zstd { Some(level) => { let mut out = vec![2_u8]; @@ -173,7 +170,7 @@ impl DecryptWriteBackend for DecryptBackend None => self.key().encrypt_data(data)?, }; let id = hash(&data); - self.write_bytes(tpe, &id, false, data.into()).await?; + self.write_bytes(tpe, &id, false, data.into())?; Ok(id) } @@ -189,7 +186,6 @@ impl DecryptReadBackend for DecryptBackend { } } -#[async_trait] impl ReadBackend for DecryptBackend { fn location(&self) -> &str { self.backend.location() @@ -199,19 +195,19 @@ impl ReadBackend for DecryptBackend { self.backend.set_option(option, value) } - async fn list(&self, tpe: FileType) -> Result> { - self.backend.list(tpe).await + fn list(&self, tpe: FileType) -> Result> { + self.backend.list(tpe) } - async fn list_with_size(&self, tpe: FileType) -> Result> { - self.backend.list_with_size(tpe).await + fn list_with_size(&self, tpe: FileType) -> Result> { + self.backend.list_with_size(tpe) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { - self.backend.read_full(tpe, id).await + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.backend.read_full(tpe, id) } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -221,21 +217,19 @@ impl ReadBackend for DecryptBackend { ) -> Result { self.backend .read_partial(tpe, id, cacheable, offset, length) - .await } } -#[async_trait] impl WriteBackend for DecryptBackend { - async fn create(&self) -> Result<()> { - self.backend.create().await + fn create(&self) -> Result<()> { + self.backend.create() } - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { - self.backend.write_bytes(tpe, id, cacheable, buf).await + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + self.backend.write_bytes(tpe, id, cacheable, buf) } - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { - self.backend.remove(tpe, id, cacheable).await + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + self.backend.remove(tpe, id, cacheable) } } diff --git a/src/backend/dry_run.rs b/src/backend/dry_run.rs index 4d57cab..24b8efd 100644 --- a/src/backend/dry_run.rs +++ b/src/backend/dry_run.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use super::{ @@ -19,14 +18,12 @@ impl DryRunBackend { } } -#[async_trait] impl DecryptReadBackend for DryRunBackend { fn decrypt(&self, data: &[u8]) -> Result> { self.be.decrypt(data) } } -#[async_trait] impl ReadBackend for DryRunBackend { fn location(&self) -> &str { self.be.location() @@ -36,15 +33,15 @@ impl ReadBackend for DryRunBackend { self.be.set_option(option, value) } - async fn list_with_size(&self, tpe: FileType) -> Result> { - self.be.list_with_size(tpe).await + fn list_with_size(&self, tpe: FileType) -> Result> { + self.be.list_with_size(tpe) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { - self.be.read_full(tpe, id).await + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.be.read_full(tpe, id) } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -52,13 +49,10 @@ impl ReadBackend for DryRunBackend { offset: u32, length: u32, ) -> Result { - self.be - .read_partial(tpe, id, cacheable, offset, length) - .await + self.be.read_partial(tpe, id, cacheable, offset, length) } } -#[async_trait] impl DecryptWriteBackend for DryRunBackend { type Key = ::Key; @@ -66,10 +60,10 @@ impl DecryptWriteBackend for DryRunBackend { self.be.key() } - async fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result { + fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result { match self.dry_run { true => Ok(Id::default()), - false => self.be.hash_write_full(tpe, data).await, + false => self.be.hash_write_full(tpe, data), } } @@ -81,26 +75,25 @@ impl DecryptWriteBackend for DryRunBackend { } } -#[async_trait] impl WriteBackend for DryRunBackend { - async fn create(&self) -> Result<()> { + fn create(&self) -> Result<()> { match self.dry_run { true => Ok(()), - false => self.be.create().await, + false => self.be.create(), } } - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { match self.dry_run { true => Ok(()), - false => self.be.write_bytes(tpe, id, cacheable, buf).await, + false => self.be.write_bytes(tpe, id, cacheable, buf), } } - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { match self.dry_run { true => Ok(()), - false => self.be.remove(tpe, id, cacheable).await, + false => self.be.remove(tpe, id, cacheable), } } } diff --git a/src/backend/hotcold.rs b/src/backend/hotcold.rs index 0c34b48..f4eff73 100644 --- a/src/backend/hotcold.rs +++ b/src/backend/hotcold.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use super::{FileType, Id, ReadBackend, WriteBackend}; @@ -16,7 +15,6 @@ impl HotColdBackend { } } -#[async_trait] impl ReadBackend for HotColdBackend { fn location(&self) -> &str { self.be.location() @@ -26,18 +24,18 @@ impl ReadBackend for HotColdBackend { self.be.set_option(option, value) } - async fn list_with_size(&self, tpe: FileType) -> Result> { - self.be.list_with_size(tpe).await + fn list_with_size(&self, tpe: FileType) -> Result> { + self.be.list_with_size(tpe) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { + fn read_full(&self, tpe: FileType, id: &Id) -> Result { match &self.hot_be { - None => self.be.read_full(tpe, id).await, - Some(be) => be.read_full(tpe, id).await, + None => self.be.read_full(tpe, id), + Some(be) => be.read_full(tpe, id), } } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -47,36 +45,33 @@ impl ReadBackend for HotColdBackend { ) -> Result { match (&self.hot_be, cacheable || tpe != FileType::Pack) { (None, _) | (Some(_), false) => { - self.be - .read_partial(tpe, id, cacheable, offset, length) - .await + self.be.read_partial(tpe, id, cacheable, offset, length) } - (Some(be), true) => be.read_partial(tpe, id, cacheable, offset, length).await, + (Some(be), true) => be.read_partial(tpe, id, cacheable, offset, length), } } } -#[async_trait] impl WriteBackend for HotColdBackend { - async fn create(&self) -> Result<()> { - self.be.create().await + fn create(&self) -> Result<()> { + self.be.create() } - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { if let Some(be) = &self.hot_be { if tpe != FileType::Config && (cacheable || tpe != FileType::Pack) { - be.write_bytes(tpe, id, cacheable, buf.clone()).await?; + be.write_bytes(tpe, id, cacheable, buf.clone())?; } } - self.be.write_bytes(tpe, id, cacheable, buf).await + self.be.write_bytes(tpe, id, cacheable, buf) } - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { // First remove cold file - self.be.remove(tpe, id, cacheable).await?; + self.be.remove(tpe, id, cacheable)?; if let Some(be) = &self.hot_be { if cacheable || tpe != FileType::Pack { - be.remove(tpe, id, cacheable).await?; + be.remove(tpe, id, cacheable)?; } } Ok(()) diff --git a/src/backend/local.rs b/src/backend/local.rs index 0fc258a..db16864 100644 --- a/src/backend/local.rs +++ b/src/backend/local.rs @@ -4,7 +4,6 @@ use std::os::unix::fs::{symlink, FileExt, PermissionsExt}; use std::path::{Path, PathBuf}; use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use filetime::{set_file_atime, set_file_mtime, FileTime}; use log::*; @@ -38,7 +37,6 @@ impl LocalBackend { } } -#[async_trait] impl ReadBackend for LocalBackend { fn location(&self) -> &str { self.path.to_str().unwrap() @@ -48,7 +46,7 @@ impl ReadBackend for LocalBackend { Ok(()) } - async fn list(&self, tpe: FileType) -> Result> { + fn list(&self, tpe: FileType) -> Result> { if tpe == FileType::Config { return Ok(match self.path.join("config").exists() { true => vec![Id::default()], @@ -65,7 +63,7 @@ impl ReadBackend for LocalBackend { Ok(walker.collect()) } - async fn list_with_size(&self, tpe: FileType) -> Result> { + fn list_with_size(&self, tpe: FileType) -> Result> { let path = self.path.join(tpe.name()); if tpe == FileType::Config { @@ -104,11 +102,11 @@ impl ReadBackend for LocalBackend { Ok(walker.collect()) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { + fn read_full(&self, tpe: FileType, id: &Id) -> Result { Ok(fs::read(self.path(tpe, id))?.into()) } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -124,9 +122,8 @@ impl ReadBackend for LocalBackend { } } -#[async_trait] impl WriteBackend for LocalBackend { - async fn create(&self) -> Result<()> { + fn create(&self) -> Result<()> { for tpe in ALL_FILE_TYPES { fs::create_dir_all(self.path.join(tpe.name()))?; } @@ -136,13 +133,7 @@ impl WriteBackend for LocalBackend { Ok(()) } - async fn write_bytes( - &self, - tpe: FileType, - id: &Id, - _cacheable: bool, - buf: Bytes, - ) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> { trace!("writing tpe: {:?}, id: {}", &tpe, &id); let filename = self.path(tpe, id); let mut file = fs::OpenOptions::new() @@ -155,7 +146,7 @@ impl WriteBackend for LocalBackend { Ok(()) } - async fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { trace!("writing tpe: {:?}, id: {}", &tpe, &id); let filename = self.path(tpe, id); fs::remove_file(filename)?; diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 578ca77..694fd06 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -2,7 +2,6 @@ use std::io::Read; use std::path::{Path, PathBuf}; use anyhow::{anyhow, Result}; -use async_trait::async_trait; use bytes::Bytes; use serde::{de::DeserializeOwned, Serialize}; @@ -70,25 +69,23 @@ pub trait RepoFile: Serialize + DeserializeOwned + Sized + Send + Sync + 'static const TYPE: FileType; } -#[async_trait] pub trait ReadBackend: Clone + Send + Sync + 'static { fn location(&self) -> &str; fn set_option(&mut self, option: &str, value: &str) -> Result<()>; - async fn list_with_size(&self, tpe: FileType) -> Result>; + fn list_with_size(&self, tpe: FileType) -> Result>; - async fn list(&self, tpe: FileType) -> Result> { + fn list(&self, tpe: FileType) -> Result> { Ok(self - .list_with_size(tpe) - .await? + .list_with_size(tpe)? .into_iter() .map(|(id, _)| id) .collect()) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result; - async fn read_partial( + fn read_full(&self, tpe: FileType, id: &Id) -> Result; + fn read_partial( &self, tpe: FileType, id: &Id, @@ -97,7 +94,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { length: u32, ) -> Result; - async fn find_starts_with(&self, tpe: FileType, vec: &[String]) -> Result>> { + fn find_starts_with(&self, tpe: FileType, vec: &[String]) -> Result>> { #[derive(Clone, Copy, PartialEq, Eq)] pub enum MapResult { None, @@ -105,7 +102,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { NonUnique, } let mut results = vec![MapResult::None; vec.len()]; - for id in self.list(tpe).await? { + for id in self.list(tpe)? { for (i, v) in vec.iter().enumerate() { if id.to_hex().starts_with(v) { if results[i] == MapResult::None { @@ -128,30 +125,28 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { .collect()) } - async fn find_id(&self, tpe: FileType, id: &str) -> Result { - Ok(self.find_ids(tpe, &[id.to_string()]).await?.remove(0)) + fn find_id(&self, tpe: FileType, id: &str) -> Result { + Ok(self.find_ids(tpe, &[id.to_string()])?.remove(0)) } - async fn find_ids(&self, tpe: FileType, ids: &[String]) -> Result> { + fn find_ids(&self, tpe: FileType, ids: &[String]) -> Result> { let long_ids: Vec<_> = ids.iter().map(|id| Id::from_hex(id)).collect(); Ok(match long_ids.iter().all(Result::is_ok) { true => long_ids.into_iter().map(Result::unwrap).collect(), // if the given id param are not full Ids, search for a suitable one false => self - .find_starts_with(tpe, ids) - .await? + .find_starts_with(tpe, ids)? .into_iter() .collect::>>()?, }) } } -#[async_trait] pub trait WriteBackend: ReadBackend { - async fn create(&self) -> Result<()>; - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()>; - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>; + fn create(&self) -> Result<()>; + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()>; + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>; } pub trait ReadSource: Iterator> { diff --git a/src/backend/rclone.rs b/src/backend/rclone.rs index 953fcf2..daa0761 100644 --- a/src/backend/rclone.rs +++ b/src/backend/rclone.rs @@ -4,12 +4,10 @@ use std::str; use std::sync::Arc; use anyhow::{anyhow, bail, Result}; -use async_trait::async_trait; use bytes::Bytes; use log::*; use rand::distributions::{Alphanumeric, DistString}; use rand::thread_rng; -use tokio::task::spawn_blocking; use super::{FileType, Id, ReadBackend, RestBackend, WriteBackend}; @@ -93,7 +91,7 @@ impl RcloneBackend { } }; - spawn_blocking(move || loop { + std::thread::spawn(move || loop { let mut line = String::new(); if stderr.read_line(&mut line).unwrap() == 0 { break; @@ -118,7 +116,6 @@ impl RcloneBackend { } } -#[async_trait] impl ReadBackend for RcloneBackend { fn location(&self) -> &str { self.rest.location() @@ -128,15 +125,15 @@ impl ReadBackend for RcloneBackend { self.rest.set_option(option, value) } - async fn list_with_size(&self, tpe: FileType) -> Result> { - self.rest.list_with_size(tpe).await + fn list_with_size(&self, tpe: FileType) -> Result> { + self.rest.list_with_size(tpe) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { - self.rest.read_full(tpe, id).await + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.rest.read_full(tpe, id) } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -144,23 +141,20 @@ impl ReadBackend for RcloneBackend { offset: u32, length: u32, ) -> Result { - self.rest - .read_partial(tpe, id, cacheable, offset, length) - .await + self.rest.read_partial(tpe, id, cacheable, offset, length) } } -#[async_trait] impl WriteBackend for RcloneBackend { - async fn create(&self) -> Result<()> { - self.rest.create().await + fn create(&self) -> Result<()> { + self.rest.create() } - async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { - self.rest.write_bytes(tpe, id, cacheable, buf).await + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + self.rest.write_bytes(tpe, id, cacheable, buf) } - async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { - self.rest.remove(tpe, id, cacheable).await + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + self.rest.remove(tpe, id, cacheable) } } diff --git a/src/backend/rest.rs b/src/backend/rest.rs index 37cf8c8..2826c36 100644 --- a/src/backend/rest.rs +++ b/src/backend/rest.rs @@ -1,11 +1,13 @@ use std::time::Duration; use anyhow::{bail, Result}; -use async_trait::async_trait; use backoff::{backoff::Backoff, Error, ExponentialBackoff, ExponentialBackoffBuilder}; use bytes::Bytes; use log::*; -use reqwest::{Client, Response, Url}; +use reqwest::{ + blocking::{Client, Response}, + Url, +}; use serde::Deserialize; use super::{FileType, Id, ReadBackend, WriteBackend}; @@ -92,7 +94,6 @@ impl RestBackend { } } -#[async_trait] impl ReadBackend for RestBackend { fn location(&self) -> &str { self.url.as_str() @@ -117,17 +118,16 @@ impl ReadBackend for RestBackend { Ok(()) } - async fn list_with_size(&self, tpe: FileType) -> Result> { - Ok(backoff::future::retry_notify( + fn list_with_size(&self, tpe: FileType) -> Result> { + Ok(backoff::retry_notify( self.backoff.clone(), - || async { + || { if tpe == FileType::Config { return Ok( match self .client .head(self.url.join("config").unwrap()) - .send() - .await? + .send()? .status() .is_success() { @@ -152,39 +152,33 @@ impl ReadBackend for RestBackend { .client .get(url) .header("Accept", "application/vnd.x.restic.rest.v2") - .send() - .await? + .send()? .check_error()? - .json::>() - .await?; + .json::>()?; Ok(list.into_iter().map(|i| (i.name, i.size)).collect()) }, notify, - ) - .await?) + )?) } - async fn read_full(&self, tpe: FileType, id: &Id) -> Result { - Ok(backoff::future::retry_notify( + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + Ok(backoff::retry_notify( self.backoff.clone(), - || async { + || { Ok(self .client .get(self.url(tpe, id)) - .send() - .await? + .send()? .check_error()? - .bytes() - .await? + .bytes()? .into_iter() .collect()) }, notify, - ) - .await?) + )?) } - async fn read_partial( + fn read_partial( &self, tpe: FileType, id: &Id, @@ -194,84 +188,64 @@ impl ReadBackend for RestBackend { ) -> Result { let offset2 = offset + length - 1; let header_value = format!("bytes={}-{}", offset, offset2); - Ok(backoff::future::retry_notify( + Ok(backoff::retry_notify( self.backoff.clone(), - || async { + || { Ok(self .client .get(self.url(tpe, id)) .header("Range", header_value.clone()) - .send() - .await? + .send()? .check_error()? - .bytes() - .await? + .bytes()? .into_iter() .collect()) }, notify, - ) - .await?) + )?) } } -#[async_trait] impl WriteBackend for RestBackend { - async fn create(&self) -> Result<()> { - Ok(backoff::future::retry_notify( + fn create(&self) -> Result<()> { + Ok(backoff::retry_notify( self.backoff.clone(), - || async { + || { self.client .post(self.url.join("?create=true").unwrap()) - .send() - .await? + .send()? .check_error()?; Ok(()) }, notify, - ) - .await?) + )?) } - async fn write_bytes( - &self, - tpe: FileType, - id: &Id, - _cacheable: bool, - buf: Bytes, - ) -> Result<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> { trace!("writing tpe: {:?}, id: {}", &tpe, &id); let req_builder = self.client.post(self.url(tpe, id)).body(buf); - Ok(backoff::future::retry_notify( + Ok(backoff::retry_notify( self.backoff.clone(), - || async { - req_builder - .try_clone() - .unwrap() - .send() - .await? - .check_error()?; + || { + req_builder.try_clone().unwrap().send()?.check_error()?; Ok(()) }, notify, - ) - .await?) + )?) } - async fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { + fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { trace!("removing tpe: {:?}, id: {}", &tpe, &id); - Ok(backoff::future::retry_notify( + Ok(backoff::retry_notify( self.backoff.clone(), - || async { + || { self.client .delete(self.url(tpe, id)) - .send() - .await? + .send()? .check_error()?; Ok(()) }, notify, - ) - .await?) + )?) } } diff --git a/src/blob/packer.rs b/src/blob/packer.rs index 311aac8..568f3e7 100644 --- a/src/blob/packer.rs +++ b/src/blob/packer.rs @@ -258,9 +258,9 @@ impl FileWriter { let indexer = self.indexer.clone(); let cacheable = self.cacheable; let new_future = spawn(async move { - be.write_bytes(FileType::Pack, &id, cacheable, file).await?; + be.write_bytes(FileType::Pack, &id, cacheable, file)?; index.time = Some(Local::now()); - indexer.write().await.add(index).await?; + indexer.write().await.add(index)?; Ok(()) }); @@ -302,16 +302,13 @@ impl Repacker { } pub async fn add_fast(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> { - let data = self - .be - .read_partial( - FileType::Pack, - pack_id, - blob.tpe.is_cacheable(), - blob.offset, - blob.length, - ) - .await?; + let data = self.be.read_partial( + FileType::Pack, + pack_id, + blob.tpe.is_cacheable(), + blob.offset, + blob.length, + )?; self.packer .add_raw(&data, &blob.id, blob.uncompressed_length, self.size_limit) .await?; @@ -319,17 +316,14 @@ impl Repacker { } pub async fn add(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> { - let data = self - .be - .read_encrypted_partial( - FileType::Pack, - pack_id, - blob.tpe.is_cacheable(), - blob.offset, - blob.length, - blob.uncompressed_length, - ) - .await?; + let data = self.be.read_encrypted_partial( + FileType::Pack, + pack_id, + blob.tpe.is_cacheable(), + blob.offset, + blob.length, + blob.uncompressed_length, + )?; self.packer .add_with_sizelimit(&data, &blob.id, self.size_limit) .await?; diff --git a/src/blob/tree.rs b/src/blob/tree.rs index 547ec5d..e3dfb85 100644 --- a/src/blob/tree.rs +++ b/src/blob/tree.rs @@ -51,24 +51,23 @@ impl Tree { Ok((chunk, id)) } - pub async fn from_backend(be: &impl IndexedBackend, id: Id) -> Result { + pub fn from_backend(be: &impl IndexedBackend, id: Id) -> Result { let data = be .get_tree(&id) .ok_or_else(|| anyhow!("blob {} not found in index", id.to_hex()))? - .read_data(be.be()) - .await?; + .read_data(be.be())?; Ok(serde_json::from_slice(&data)?) } - pub async fn subtree_id(be: &impl IndexedBackend, mut id: Id, path: &Path) -> Result { + pub fn subtree_id(be: &impl IndexedBackend, mut id: Id, path: &Path) -> Result { for p in path.iter() { let p = p.to_str().unwrap(); // TODO: check for root instead if p == "/" { continue; } - let tree = Tree::from_backend(be, id).await?; + let tree = Tree::from_backend(be, id)?; let node = tree .nodes() .iter() @@ -105,8 +104,8 @@ impl NodeStreamer where BE: IndexedBackend + Unpin, { - pub async fn new(be: BE, id: Id) -> Result { - let inner = Tree::from_backend(&be, id).await?.nodes.into_iter(); + pub fn new(be: BE, id: Id) -> Result { + let inner = Tree::from_backend(&be, id)?.nodes.into_iter(); Ok(Self { future: None, inner, @@ -150,7 +149,7 @@ where slf.path.push(node.name()); let be = slf.be.clone(); let id = *id; - slf.future = Some(spawn(async move { Tree::from_backend(&be, id).await })); + slf.future = Some(spawn(async move { Tree::from_backend(&be, id) })); } return Poll::Ready(Some(Ok((path, node)))); @@ -233,9 +232,9 @@ where while slf.futures.len() < MAX_TREE_LOADER && !slf.pending.is_empty() { let (path, id, count) = slf.pending.pop_front().unwrap(); let be = slf.be.clone(); - slf.futures.push(spawn(async move { - (path, Tree::from_backend(&be, id).await, count) - })); + slf.futures.push(spawn( + async move { (path, Tree::from_backend(&be, id), count) }, + )); } match Pin::new(&mut slf.futures).poll_next(cx) { diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 32d5377..7d8516f 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -163,7 +163,7 @@ pub(super) async fn execute( ) .await .ok(), - (false, false, Some(parent)) => SnapshotFile::from_id(&be, &parent).await.ok(), + (false, false, Some(parent)) => SnapshotFile::from_id(&be, &parent).ok(), }; let parent_tree = match &parent { @@ -197,7 +197,7 @@ pub(super) async fn execute( snap.paths.add(backup_path_str.clone()); snap.set_tags(opts.tag.clone()); - let parent = Parent::new(&index, parent_tree, opts.ignore_ctime, opts.ignore_inode).await; + let parent = Parent::new(&index, parent_tree, opts.ignore_ctime, opts.ignore_inode); let snap = if backup_stdin { let mut archiver = Archiver::new(be, index, &config, parent, snap)?; diff --git a/src/commands/cat.rs b/src/commands/cat.rs index 401f0d5..9d3c9bb 100644 --- a/src/commands/cat.rs +++ b/src/commands/cat.rs @@ -48,9 +48,9 @@ struct TreeOpts { pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> { match opts.command { - Command::Config => cat_file(be, FileType::Config, IdOpt::default()).await, - Command::Index(opt) => cat_file(be, FileType::Index, opt).await, - Command::Snapshot(opt) => cat_file(be, FileType::Snapshot, opt).await, + Command::Config => cat_file(be, FileType::Config, IdOpt::default()), + Command::Index(opt) => cat_file(be, FileType::Index, opt), + Command::Snapshot(opt) => cat_file(be, FileType::Snapshot, opt), // special treatment for catingg blobs: read the index and use it to locate the blob Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt).await, Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt).await, @@ -59,9 +59,9 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result< } } -async fn cat_file(be: &impl DecryptReadBackend, tpe: FileType, opt: IdOpt) -> Result<()> { - let id = be.find_id(tpe, &opt.id).await?; - let data = be.read_encrypted_full(tpe, &id).await?; +fn cat_file(be: &impl DecryptReadBackend, tpe: FileType, opt: IdOpt) -> Result<()> { + let id = be.find_id(tpe, &opt.id)?; + let data = be.read_encrypted_full(tpe, &id)?; println!("{}", String::from_utf8(data.to_vec())?); Ok(()) @@ -71,8 +71,7 @@ async fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Re let id = Id::from_hex(&opt.id)?; let data = IndexBackend::new(be, ProgressBar::hidden()) .await? - .blob_from_backend(&tpe, &id) - .await?; + .blob_from_backend(&tpe, &id)?; print!("{}", String::from_utf8(data.to_vec())?); Ok(()) @@ -82,8 +81,8 @@ async fn cat_tree(be: &impl DecryptReadBackend, opts: TreeOpts) -> Result<()> { let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, "")); let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?; let index = IndexBackend::new(be, progress_counter("")).await?; - let id = Tree::subtree_id(&index, snap.tree, Path::new(path)).await?; - let data = index.blob_from_backend(&BlobType::Tree, &id).await?; + let id = Tree::subtree_id(&index, snap.tree, Path::new(path))?; + let data = index.blob_from_backend(&BlobType::Tree, &id)?; println!("{}", String::from_utf8(data.to_vec())?); Ok(()) diff --git a/src/commands/check.rs b/src/commands/check.rs index f6b5a3c..18e77a1 100644 --- a/src/commands/check.rs +++ b/src/commands/check.rs @@ -45,7 +45,7 @@ pub(super) async fn execute( // // This lists files here and later when reading index / checking snapshots // TODO: Only list the files once... - let _ = be.list_with_size(file_type).await?; + let _ = be.list_with_size(file_type)?; let p = progress_bytes(format!("checking {} in cache...", file_type.name())); // TODO: Make concurrency (20) customizable @@ -84,7 +84,7 @@ pub(super) async fn execute( // TODO: Make concurrency (4) customizable .for_each_concurrent(4, |(pack, be, p)| async move { let id = pack.id; - let data = be.read_full(FileType::Pack, &id).await.unwrap(); + let data = be.read_full(FileType::Pack, &id).unwrap(); spawn_blocking(move || { match check_pack(&be, pack, data) { Ok(()) => {} @@ -109,12 +109,11 @@ async fn check_hot_files( ) -> Result<()> { let p = progress_spinner(format!("checking {} in hot repo...", file_type.name())); let mut files = be - .list_with_size(file_type) - .await? + .list_with_size(file_type)? .into_iter() .collect::>(); - let files_hot = be_hot.list_with_size(file_type).await?; + let files_hot = be_hot.list_with_size(file_type)?; for (id, size_hot) in files_hot { match files.remove(&id) { @@ -141,7 +140,7 @@ async fn check_cache_files( file_type: FileType, p: ProgressBar, ) -> Result<()> { - let files = cache.list_with_size(file_type).await?; + let files = cache.list_with_size(file_type)?; if files.is_empty() { return Ok(()); @@ -159,8 +158,8 @@ async fn check_cache_files( .for_each_concurrent(concurrency, |((id, size), cache, be, p)| async move { // Read file from cache and from backend and compare match ( - cache.read_full(file_type, &id).await, - be.read_full(file_type, &id).await, + cache.read_full(file_type, &id), + be.read_full(file_type, &id), ) { (Err(err), _) => { error!("Error reading cached file Type: {file_type:?}, Id: {id} : {err}",) @@ -252,7 +251,7 @@ async fn check_packs( } async fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap) -> Result<()> { - for (id, size) in be.list_with_size(FileType::Pack).await? { + for (id, size) in be.list_with_size(FileType::Pack)? { match packs.remove(&id) { None => warn!("pack {id} not referenced in index. Can be a parallel backup job. To repair: 'rustic repair index'."), Some(index_size) if index_size != size => { diff --git a/src/commands/config.rs b/src/commands/config.rs index dce182d..f2ffdfd 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -11,7 +11,7 @@ pub(super) struct Opts { config_opts: ConfigOpts, } -pub(super) async fn execute( +pub(super) fn execute( be: &impl DecryptFullBackend, hot_be: &Option, opts: Opts, @@ -22,13 +22,13 @@ pub(super) async fn execute( if new_config != config { new_config.is_hot = None; // for hot/cold backend, this only saves the config to the cold repo. - be.save_file(&new_config).await?; + be.save_file(&new_config)?; if let Some(hot_be) = hot_be { // save config to hot repo let dbe = DecryptBackend::new(hot_be, be.key().clone()); new_config.is_hot = Some(true); - dbe.save_file(&new_config).await?; + dbe.save_file(&new_config)?; } println!("saved new config"); diff --git a/src/commands/diff.rs b/src/commands/diff.rs index 52b7f63..8503a48 100644 --- a/src/commands/diff.rs +++ b/src/commands/diff.rs @@ -34,11 +34,11 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) let snap2 = &snaps[1]; let index = IndexBackend::new(be, progress_counter("")).await?; - let id1 = Tree::subtree_id(&index, snap1.tree, Path::new(path1)).await?; - let id2 = Tree::subtree_id(&index, snap2.tree, Path::new(path2)).await?; + let id1 = Tree::subtree_id(&index, snap1.tree, Path::new(path1))?; + let id2 = Tree::subtree_id(&index, snap2.tree, Path::new(path2))?; - let mut tree_streamer1 = NodeStreamer::new(index.clone(), id1).await?; - let mut tree_streamer2 = NodeStreamer::new(index, id2).await?; + let mut tree_streamer1 = NodeStreamer::new(index.clone(), id1)?; + let mut tree_streamer2 = NodeStreamer::new(index, id2)?; let mut item1 = tree_streamer1.next().await.transpose()?; let mut item2 = tree_streamer2.next().await.transpose()?; diff --git a/src/commands/helpers.rs b/src/commands/helpers.rs index 103a083..cad1d58 100644 --- a/src/commands/helpers.rs +++ b/src/commands/helpers.rs @@ -23,16 +23,15 @@ pub fn bytes(b: u64) -> String { ByteSize(b).to_string_as(true) } -pub async fn get_key(be: &impl ReadBackend, password: Option) -> Result { +pub fn get_key(be: &impl ReadBackend, password: Option) -> Result { for _ in 0..MAX_PASSWORD_RETRIES { match &password { // if password is given, directly return the result of find_key_in_backend and don't retry - Some(pass) => return find_key_in_backend(be, pass, None).await, + Some(pass) => return find_key_in_backend(be, pass, None), None => { // TODO: Differentiate between wrong password and other error! if let Ok(key) = find_key_in_backend(be, &prompt_password("enter repository password: ")?, None) - .await { return Ok(key); } @@ -125,7 +124,7 @@ pub async fn warm_up( let be = be.clone(); stream.push(spawn(async move { // ignore errors as they are expected from the warm-up - _ = be.read_partial(FileType::Pack, &pack, false, 0, 1).await; + _ = be.read_partial(FileType::Pack, &pack, false, 0, 1); p.inc(1); })) } diff --git a/src/commands/init.rs b/src/commands/init.rs index 3722956..78afc07 100644 --- a/src/commands/init.rs +++ b/src/commands/init.rs @@ -20,7 +20,7 @@ pub(super) struct Opts { config_opts: ConfigOpts, } -pub(super) async fn execute( +pub(super) fn execute( be: &impl WriteBackend, hot_be: &Option, opts: Opts, @@ -59,24 +59,23 @@ pub(super) async fn execute( )?; let data: Bytes = serde_json::to_vec(&keyfile)?.into(); let id = hash(&data); - be.create().await?; - be.write_bytes(FileType::Key, &id, false, data.clone()) - .await?; + be.create()?; + be.write_bytes(FileType::Key, &id, false, data.clone())?; if let Some(hot_be) = hot_be { - hot_be.create().await?; - hot_be.write_bytes(FileType::Key, &id, false, data).await?; + hot_be.create()?; + hot_be.write_bytes(FileType::Key, &id, false, data)?; } println!("key {} successfully added.", id); // save config let dbe = DecryptBackend::new(be, key.clone()); - dbe.save_file(&config).await?; + dbe.save_file(&config)?; if let Some(hot_be) = hot_be { let dbe = DecryptBackend::new(hot_be, key); config.is_hot = Some(true); - dbe.save_file(&config).await?; + dbe.save_file(&config)?; } println!("repository {} successfully created.", repo_id); diff --git a/src/commands/key.rs b/src/commands/key.rs index a5be97c..ee3e565 100644 --- a/src/commands/key.rs +++ b/src/commands/key.rs @@ -47,13 +47,13 @@ pub(crate) struct KeyOpts { pub(crate) with_created: bool, } -pub(super) async fn execute(be: &impl WriteBackend, key: Key, opts: Opts) -> Result<()> { +pub(super) fn execute(be: &impl WriteBackend, key: Key, opts: Opts) -> Result<()> { match opts.command { - Command::Add(opt) => add_key(be, key, opt).await, + Command::Add(opt) => add_key(be, key, opt), } } -async fn add_key(be: &impl WriteBackend, key: Key, opts: AddOpts) -> Result<()> { +fn add_key(be: &impl WriteBackend, key: Key, opts: AddOpts) -> Result<()> { let pass = match opts.new_password_file { Some(file) => { let mut file = BufReader::new(File::open(file)?); @@ -65,8 +65,7 @@ async fn add_key(be: &impl WriteBackend, key: Key, opts: AddOpts) -> Result<()> let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?; let data = serde_json::to_vec(&keyfile)?; let id = hash(&data); - be.write_bytes(FileType::Key, &id, false, data.into()) - .await?; + be.write_bytes(FileType::Key, &id, false, data.into())?; println!("key {} successfully added.", id); Ok(()) diff --git a/src/commands/list.rs b/src/commands/list.rs index e75acd7..1eca397 100644 --- a/src/commands/list.rs +++ b/src/commands/list.rs @@ -34,7 +34,7 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result< t => bail!("invalid type: {}", t), }; - for id in be.list(tpe).await? { + for id in be.list(tpe)? { println!("{}", id.to_hex()); } diff --git a/src/commands/ls.rs b/src/commands/ls.rs index 270a14f..477248a 100644 --- a/src/commands/ls.rs +++ b/src/commands/ls.rs @@ -20,9 +20,9 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, "")); let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?; let index = IndexBackend::new(be, progress_counter("")).await?; - let tree = Tree::subtree_id(&index, snap.tree, Path::new(path)).await?; + let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?; - let mut tree_streamer = NodeStreamer::new(index, tree).await?; + let mut tree_streamer = NodeStreamer::new(index, tree)?; while let Some(item) = tree_streamer.next().await { let (path, _) = item?; println!("{:?} ", path); diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 458462f..30c233e 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -263,29 +263,27 @@ pub async fn execute() -> Result<()> { (None, None, None) => None, }; - let config_ids = be.list(FileType::Config).await?; + let config_ids = be.list(FileType::Config)?; let (cmd, key, dbe, cache, be, be_hot, config) = match (args.command, config_ids.len()) { - (Command::Init(opts), _) => { - return init::execute(&be, &be_hot, opts, password, config_ids).await - } + (Command::Init(opts), _) => return init::execute(&be, &be_hot, opts, password, config_ids), (cmd, 1) => { let be = HotColdBackend::new(be, be_hot.clone()); if let Some(be_hot) = &be_hot { - let mut keys = be.list_with_size(FileType::Key).await?; + let mut keys = be.list_with_size(FileType::Key)?; keys.sort_unstable_by_key(|key| key.0); - let mut hot_keys = be_hot.list_with_size(FileType::Key).await?; + let mut hot_keys = be_hot.list_with_size(FileType::Key)?; hot_keys.sort_unstable_by_key(|key| key.0); if keys != hot_keys { bail!("keys from repo and repo-hot do not match. Aborting."); } } - let key = get_key(&be, password).await?; + let key = get_key(&be, password)?; info!("password is correct."); let dbe = DecryptBackend::new(&be, key.clone()); - let config: ConfigFile = dbe.get_file(&config_ids[0]).await?; + let config: ConfigFile = dbe.get_file(&config_ids[0])?; match (config.is_hot == Some(true), be_hot.is_some()) { (true, false) => bail!("repository is a hot repository!\nPlease use as --repo-hot in combination with the normal repo. Aborting."), (false, true) => bail!("repo-hot is not a hot repository! Aborting."), @@ -308,14 +306,14 @@ pub async fn execute() -> Result<()> { match cmd { Command::Backup(opts) => backup::execute(&dbe, opts, config, config_file, command).await?, - Command::Config(opts) => config::execute(&dbe, &be_hot, opts, config).await?, + Command::Config(opts) => config::execute(&dbe, &be_hot, opts, config)?, Command::Cat(opts) => cat::execute(&dbe, opts).await?, Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts).await?, Command::Completions(_) => {} // already handled above Command::Diff(opts) => diff::execute(&dbe, opts).await?, Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file).await?, Command::Init(_) => {} // already handled above - Command::Key(opts) => key::execute(&dbe, key, opts).await?, + Command::Key(opts) => key::execute(&dbe, key, opts)?, Command::List(opts) => list::execute(&dbe, opts).await?, Command::Ls(opts) => ls::execute(&dbe, opts).await?, Command::SelfUpdate(_) => {} // already handled above diff --git a/src/commands/prune.rs b/src/commands/prune.rs index d053ca0..cb8984c 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -113,9 +113,7 @@ pub(super) async fn execute( if let Some(cache) = &cache { let p = progress_spinner("cleaning up packs from cache..."); - cache - .remove_not_in_list(FileType::Pack, index_collector.tree_packs()) - .await?; + cache.remove_not_in_list(FileType::Pack, index_collector.tree_packs())?; p.finish(); } match (cache.is_some(), opts.cache_only) { @@ -137,11 +135,7 @@ pub(super) async fn execute( // list existing pack files let p = progress_spinner("geting packs from repository..."); - let existing_packs: HashMap<_, _> = be - .list_with_size(FileType::Pack) - .await? - .into_iter() - .collect(); + let existing_packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect(); p.finish(); let mut pruner = Pruner::new(used_ids, existing_packs, index_files); @@ -896,7 +890,7 @@ impl Pruner { time: Some(Local::now()), blobs: Vec::new(), }; - indexer.write().await.add_remove(pack).await?; + indexer.write().await.add_remove(pack)?; } } } @@ -933,7 +927,7 @@ impl Pruner { PackToDo::Keep => { // keep pack: add to new index let pack = pack.into_index_pack(); - indexer.write().await.add(pack).await?; + indexer.write().await.add(pack)?; } PackToDo::Repack => { // TODO: repack in parallel @@ -959,7 +953,7 @@ impl Pruner { } else { // mark pack for removal let pack = pack.into_index_pack_with_time(self.time); - indexer.write().await.add_remove(pack).await?; + indexer.write().await.add_remove(pack)?; } } PackToDo::MarkDelete => { @@ -968,7 +962,7 @@ impl Pruner { } else { // mark pack for removal let pack = pack.into_index_pack_with_time(self.time); - indexer.write().await.add_remove(pack).await?; + indexer.write().await.add_remove(pack)?; } } PackToDo::KeepMarked => { @@ -977,13 +971,13 @@ impl Pruner { } else { // keep pack: add to new index let pack = pack.into_index_pack(); - indexer.write().await.add_remove(pack).await?; + indexer.write().await.add_remove(pack)?; } } PackToDo::Recover => { // recover pack: add to new index in section packs let pack = pack.into_index_pack_with_time(self.time); - indexer.write().await.add(pack).await?; + indexer.write().await.add(pack)?; } PackToDo::Delete => delete_pack(pack), } @@ -992,7 +986,7 @@ impl Pruner { } tree_repacker.finalize().await?; data_repacker.finalize().await?; - indexer.write().await.finalize().await?; + indexer.write().await.finalize()?; p.finish(); if !data_packs_remove.is_empty() { diff --git a/src/commands/repair.rs b/src/commands/repair.rs index 1ed99a9..fac4446 100644 --- a/src/commands/repair.rs +++ b/src/commands/repair.rs @@ -96,11 +96,7 @@ pub(super) async fn execute( async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> { let p = progress_spinner("listing packs..."); - let mut packs: HashMap<_, _> = be - .list_with_size(FileType::Pack) - .await? - .into_iter() - .collect(); + let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect(); p.finish(); let mut pack_read_header = Vec::new(); @@ -162,9 +158,9 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<( (true, true) => info!("would have modified index file {index_id}"), (true, false) => { if !new_index.packs.is_empty() || !new_index.packs_to_delete.is_empty() { - be.save_file(&new_index).await?; + be.save_file(&new_index)?; } - be.remove(FileType::Index, &index_id, true).await?; + be.remove(FileType::Index, &index_id, true)?; } (false, _) => {} // nothing to do } @@ -197,15 +193,13 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<( debug!("reading pack {id}..."); let mut pack = IndexPack::default(); pack.set_id(id); - pack.blobs = PackHeader::from_file(be, id, size_hint, packsize) - .await? - .into_blobs(); + pack.blobs = PackHeader::from_file(be, id, size_hint, packsize)?.into_blobs(); if !opts.dry_run { - indexer.write().await.add_with(pack, to_delete).await?; + indexer.write().await.add_with(pack, to_delete)?; } p.inc(1); } - indexer.write().await.finalize().await?; + indexer.write().await.finalize()?; p.finish(); Ok(()) @@ -268,7 +262,7 @@ async fn repair_snaps( if opts.dry_run { info!("would have modified snapshot {snap_id}."); } else { - let new_id = be.save_file(&snap).await?; + let new_id = be.save_file(&snap)?; info!("saved modified snapshot as {new_id}."); } delete.push(snap_id); @@ -278,7 +272,7 @@ async fn repair_snaps( if !opts.dry_run { packer.finalize().await?; - indexer.write().await.finalize().await?; + indexer.write().await.finalize()?; } if opts.delete { @@ -324,7 +318,7 @@ async fn repair_tree( return Ok(*r); } - let (tree, mut changed) = match Tree::from_backend(be, id).await { + let (tree, mut changed) = match Tree::from_backend(be, id) { Ok(tree) => (tree, Changed::None), Err(_) => { warn!("tree {id} could not be loaded."); diff --git a/src/commands/repoinfo.rs b/src/commands/repoinfo.rs index 6bd0ec7..ebc32f5 100644 --- a/src/commands/repoinfo.rs +++ b/src/commands/repoinfo.rs @@ -116,7 +116,7 @@ async fn fileinfo(text: &str, be: &impl ReadBackend) -> Result<()> { let mut total_count = 0; let mut total_size = 0; for tpe in ALL_FILE_TYPES { - let list = be.list_with_size(tpe).await?; + let list = be.list_with_size(tpe)?; let count = list.len(); let size = list.iter().map(|f| f.1 as u64).sum(); table.add_row(row![format!("{:?}", tpe), r->count, r->bytes(size)]); diff --git a/src/commands/restore.rs b/src/commands/restore.rs index 2671532..e0ebc42 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -70,7 +70,7 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?; let index = IndexBackend::new(be, progress_counter("")).await?; - let tree = Tree::subtree_id(&index, snap.tree, Path::new(path)).await?; + let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?; let dest = LocalBackend::new(&opts.dest); @@ -222,7 +222,7 @@ async fn allocate_and_collect( .filter_map(Result::ok); // TODO: print out the ignored error let mut next_dst = dst_iter.next(); - let mut node_streamer = NodeStreamer::new(index.clone(), tree).await?; + let mut node_streamer = NodeStreamer::new(index.clone(), tree)?; let mut next_node = node_streamer.try_next().await?; loop { @@ -323,7 +323,6 @@ async fn restore_contents( bl.length, bl.uncompressed_length, ) - .await .unwrap() } }; @@ -351,7 +350,7 @@ async fn restore_metadata( opts: &Opts, ) -> Result<()> { // walk over tree in repository and compare with tree in dest - let mut node_streamer = NodeStreamer::new(index, tree).await?; + let mut node_streamer = NodeStreamer::new(index, tree)?; let mut dir_stack = Vec::new(); while let Some((path, node)) = node_streamer.try_next().await? { match node.node_type() { diff --git a/src/index/indexer.rs b/src/index/indexer.rs index 2778769..4aba144 100644 --- a/src/index/indexer.rs +++ b/src/index/indexer.rs @@ -53,26 +53,26 @@ impl Indexer { Arc::new(RwLock::new(self)) } - pub async fn finalize(&self) -> Result<()> { - self.save().await + pub fn finalize(&self) -> Result<()> { + self.save() } - pub async fn save(&self) -> Result<()> { + pub fn save(&self) -> Result<()> { if (self.file.packs.len() + self.file.packs_to_delete.len()) > 0 { - self.be.save_file(&self.file).await?; + self.be.save_file(&self.file)?; } Ok(()) } - pub async fn add(&mut self, pack: IndexPack) -> Result<()> { - self.add_with(pack, false).await + pub fn add(&mut self, pack: IndexPack) -> Result<()> { + self.add_with(pack, false) } - pub async fn add_remove(&mut self, pack: IndexPack) -> Result<()> { - self.add_with(pack, true).await + pub fn add_remove(&mut self, pack: IndexPack) -> Result<()> { + self.add_with(pack, true) } - pub async fn add_with(&mut self, pack: IndexPack, delete: bool) -> Result<()> { + pub fn add_with(&mut self, pack: IndexPack, delete: bool) -> Result<()> { self.count += pack.blobs.len(); if let Some(indexed) = &mut self.indexed { @@ -85,7 +85,7 @@ impl Indexer { // check if IndexFile needs to be saved if self.count >= MAX_COUNT || self.created.elapsed()? >= MAX_AGE { - self.save().await?; + self.save()?; self.reset(); } Ok(()) diff --git a/src/index/mod.rs b/src/index/mod.rs index 95a1362..dc5d4c5 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -41,17 +41,15 @@ impl IndexEntry { } /// Get a blob described by IndexEntry from the backend - pub async fn read_data(&self, be: &B) -> Result { - let data = be - .read_encrypted_partial( - FileType::Pack, - &self.pack, - self.blob_type.is_cacheable(), - self.offset, - self.length, - self.uncompressed_length, - ) - .await?; + pub fn read_data(&self, be: &B) -> Result { + let data = be.read_encrypted_partial( + FileType::Pack, + &self.pack, + self.blob_type.is_cacheable(), + self.offset, + self.length, + self.uncompressed_length, + )?; Ok(data) } @@ -91,10 +89,10 @@ pub trait IndexedBackend: ReadIndex + Clone + Sync + Send + 'static { fn be(&self) -> &Self::Backend; - async fn blob_from_backend(&self, tpe: &BlobType, id: &Id) -> Result { + fn blob_from_backend(&self, tpe: &BlobType, id: &Id) -> Result { match self.get_id(tpe, id) { None => Err(anyhow!("blob not found in index")), - Some(ie) => ie.read_data(self.be()).await, + Some(ie) => ie.read_data(self.be()), } } } diff --git a/src/repo/keyfile.rs b/src/repo/keyfile.rs index 46ce109..4c44fa0 100644 --- a/src/repo/keyfile.rs +++ b/src/repo/keyfile.rs @@ -88,8 +88,8 @@ impl KeyFile { impl KeyFile { /// Get a KeyFile from the backend - pub async fn from_backend(be: &B, id: &Id) -> Result { - let data = be.read_full(FileType::Key, id).await?; + pub fn from_backend(be: &B, id: &Id) -> Result { + let data = be.read_full(FileType::Key, id)?; Ok(serde_json::from_slice(&data)?) } } @@ -136,29 +136,23 @@ impl MasterKey { } } -async fn key_from_backend( - be: &B, - id: &Id, - passwd: &impl AsRef<[u8]>, -) -> Result { - KeyFile::from_backend(be, id) - .await? - .key_from_password(passwd) +fn key_from_backend(be: &B, id: &Id, passwd: &impl AsRef<[u8]>) -> Result { + KeyFile::from_backend(be, id)?.key_from_password(passwd) } /// Find a KeyFile in the backend that fits to the given password and return the contained key. /// If a key hint is given, only this key is tested. /// This is recommended for a large number of keys. -pub async fn find_key_in_backend( +pub fn find_key_in_backend( be: &B, passwd: &impl AsRef<[u8]>, hint: Option<&Id>, ) -> Result { match hint { - Some(id) => key_from_backend(be, id, passwd).await, + Some(id) => key_from_backend(be, id, passwd), None => { - for id in be.list(FileType::Key).await? { - if let Ok(key) = key_from_backend(be, &id, passwd).await { + for id in be.list(FileType::Key)? { + if let Ok(key) = key_from_backend(be, &id, passwd) { return Ok(key); } } diff --git a/src/repo/packfile.rs b/src/repo/packfile.rs index 8b6a4c1..cfdf7bb 100644 --- a/src/repo/packfile.rs +++ b/src/repo/packfile.rs @@ -156,7 +156,7 @@ impl PackHeader { } /// Read the pack header directly from a packfile using the backend - pub async fn from_file( + pub fn from_file( be: &impl DecryptReadBackend, id: Id, size_hint: Option, @@ -170,9 +170,7 @@ impl PackHeader { // read (guessed) header + length field let read_size = size_guess + LENGTH_LEN; let offset = pack_size - read_size; - let mut data = be - .read_partial(FileType::Pack, &id, false, offset, read_size) - .await?; + let mut data = be.read_partial(FileType::Pack, &id, false, offset, read_size)?; // get header length from the file let size_real = @@ -185,8 +183,7 @@ impl PackHeader { } else { // size_guess was too small; we have to read again let offset = pack_size - size_real - LENGTH_LEN; - be.read_partial(FileType::Pack, &id, false, offset, size_real) - .await? + be.read_partial(FileType::Pack, &id, false, offset, size_real)? }; Self::from_binary(&be.decrypt(&data)?) diff --git a/src/repo/snapshotfile.rs b/src/repo/snapshotfile.rs index f2d10fe..0788f46 100644 --- a/src/repo/snapshotfile.rs +++ b/src/repo/snapshotfile.rs @@ -108,8 +108,8 @@ impl SnapshotFile { } /// Get a SnapshotFile from the backend - pub async fn from_backend(be: &B, id: &Id) -> Result { - Ok(Self::set_id((*id, be.get_file(id).await?))) + pub fn from_backend(be: &B, id: &Id) -> Result { + Ok(Self::set_id((*id, be.get_file(id)?))) } pub async fn from_str( @@ -120,7 +120,7 @@ impl SnapshotFile { ) -> Result { match string { "latest" => Self::latest(be, predicate, p).await, - _ => Self::from_id(be, string).await, + _ => Self::from_id(be, string), } } @@ -152,15 +152,15 @@ impl SnapshotFile { } /// Get a SnapshotFile from the backend by (part of the) id - pub async fn from_id(be: &B, id: &str) -> Result { + pub fn from_id(be: &B, id: &str) -> Result { info!("getting snapshot..."); - let id = be.find_id(FileType::Snapshot, id).await?; - SnapshotFile::from_backend(be, &id).await + let id = be.find_id(FileType::Snapshot, id)?; + SnapshotFile::from_backend(be, &id) } /// Get a Vector of SnapshotFile from the backend by list of (parts of the) ids pub async fn from_ids(be: &B, ids: &[String]) -> Result> { - let ids = be.find_ids(FileType::Snapshot, ids).await?; + let ids = be.find_ids(FileType::Snapshot, ids)?; Ok(be .stream_list::(ids, ProgressBar::hidden()) .await? From 683f2bc26c6991afd3fa2cf75d6d124a0c5c4f29 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 16 Oct 2022 13:27:29 +0200 Subject: [PATCH 02/10] Parallelize backend operations --- Cargo.lock | 60 ++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++ src/backend/decrypt.rs | 73 ++++++++++++--------------------------- src/commands/backup.rs | 3 +- src/commands/cat.rs | 20 +++++------ src/commands/check.rs | 28 +++++++-------- src/commands/diff.rs | 4 +-- src/commands/forget.rs | 7 ++-- src/commands/list.rs | 8 ++--- src/commands/ls.rs | 4 +-- src/commands/mod.rs | 10 +++--- src/commands/prune.rs | 28 ++++++--------- src/commands/repair.rs | 15 +++----- src/commands/repoinfo.rs | 15 ++++---- src/commands/restore.rs | 4 +-- src/commands/snapshots.rs | 9 +++-- src/commands/tag.rs | 11 +++--- src/index/mod.rs | 27 +++++---------- src/repo/snapshotfile.rs | 43 +++++++++++------------ 19 files changed, 184 insertions(+), 188 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93dcfea..8d2e78c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -356,6 +356,40 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.11" @@ -1407,6 +1441,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -1570,6 +1628,7 @@ dependencies = [ "chrono", "clap", "clap_complete", + "crossbeam-channel", "derivative", "derive-getters", "derive_more", @@ -1595,6 +1654,7 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand", + "rayon", "reqwest", "rpassword", "rstest", diff --git a/Cargo.toml b/Cargo.toml index a2f7599..e9a602e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,9 @@ derivative = "2" derive-getters = "0.2" lazy_static = "1" log = "0.4" +# parallelize +crossbeam-channel = "0.5" +rayon = "1" # async tokio = { version = "1", features = ["full"] } futures = "0.3" diff --git a/src/backend/decrypt.rs b/src/backend/decrypt.rs index c30fc1e..6a90c75 100644 --- a/src/backend/decrypt.rs +++ b/src/backend/decrypt.rs @@ -1,11 +1,10 @@ use std::num::NonZeroU32; use anyhow::{bail, Result}; -use async_trait::async_trait; use bytes::Bytes; -use futures::{stream, stream::FuturesUnordered, StreamExt}; +use crossbeam_channel::{unbounded, Receiver}; use indicatif::ProgressBar; -use tokio::{spawn, task::JoinHandle}; +use rayon::prelude::*; use zstd::stream::{copy_encode, decode_all}; use super::{FileType, Id, ReadBackend, RepoFile, WriteBackend}; @@ -14,7 +13,6 @@ use crate::crypto::{hash, CryptoKey}; pub trait DecryptFullBackend: DecryptWriteBackend + DecryptReadBackend {} impl DecryptFullBackend for T {} -#[async_trait] pub trait DecryptReadBackend: ReadBackend { fn decrypt(&self, data: &[u8]) -> Result>; @@ -52,38 +50,25 @@ pub trait DecryptReadBackend: ReadBackend { Ok(serde_json::from_slice(&data)?) } - async fn stream_all( - &self, - p: ProgressBar, - ) -> Result>> { + fn stream_all(&self, p: ProgressBar) -> Result> { let list = self.list(F::TYPE)?; - self.stream_list(list, p).await + self.stream_list(list, p) } - async fn stream_list( - &self, - list: Vec, - p: ProgressBar, - ) -> Result>> { + fn stream_list(&self, list: Vec, p: ProgressBar) -> Result> { p.set_length(list.len() as u64); + let (tx, rx) = unbounded(); - let stream: FuturesUnordered<_> = list - .into_iter() - .map(|id| { - let be = self.clone(); - let p = p.clone(); - spawn(async move { - let file = be.get_file::(&id).unwrap(); - p.inc(1); - (id, file) - }) - }) - .collect(); - Ok(stream) + list.into_par_iter() + .for_each_with((self, p, tx), |(be, p, tx), id| { + let file = be.get_file::(&id).unwrap(); + p.inc(1); + tx.send((id, file)).unwrap(); + }); + Ok(rx) } } -#[async_trait] pub trait DecryptWriteBackend: WriteBackend { type Key: CryptoKey; fn key(&self) -> &Self::Key; @@ -91,26 +76,20 @@ pub trait DecryptWriteBackend: WriteBackend { fn save_file(&self, file: &F) -> Result { let data = serde_json::to_vec(file)?; - Ok(self.hash_write_full(F::TYPE, &data)?) + self.hash_write_full(F::TYPE, &data) } - async fn save_list(&self, list: Vec, p: ProgressBar) -> Result<()> { + fn save_list(&self, list: Vec, p: ProgressBar) -> Result<()> { p.set_length(list.len() as u64); - stream::iter(list.into_iter().map(|file| { - let be = self.clone(); - let p = p.clone(); - (file, be, p) - })) - .for_each_concurrent(5, |(file, be, p)| async move { - be.save_file(&file).unwrap(); + list.par_iter().for_each(|file| { + self.save_file(file).unwrap(); p.inc(1); - }) - .await; + }); p.finish(); Ok(()) } - async fn delete_list( + fn delete_list( &self, tpe: FileType, cacheable: bool, @@ -118,16 +97,10 @@ pub trait DecryptWriteBackend: WriteBackend { p: ProgressBar, ) -> Result<()> { p.set_length(list.len() as u64); - stream::iter(list.into_iter().map(|id| { - let be = self.clone(); - let p = p.clone(); - (id, be, p) - })) - .for_each_concurrent(20, |(id, be, p)| async move { - be.remove(tpe, &id, cacheable).unwrap(); + list.par_iter().for_each(|id| { + self.remove(tpe, id, cacheable).unwrap(); p.inc(1); - }) - .await; + }); p.finish(); Ok(()) @@ -153,7 +126,6 @@ impl DecryptBackend { } } -#[async_trait] impl DecryptWriteBackend for DecryptBackend { type Key = C; fn key(&self) -> &Self::Key { @@ -179,7 +151,6 @@ impl DecryptWriteBackend for DecryptBackend } } -#[async_trait] impl DecryptReadBackend for DecryptBackend { fn decrypt(&self, data: &[u8]) -> Result> { Ok(self.key.decrypt_data(data)?) diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 7d8516f..80922ea 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -116,7 +116,7 @@ pub(super) async fn execute( } }; - let index = IndexBackend::only_full_trees(&be.clone(), progress_counter("")).await?; + let index = IndexBackend::only_full_trees(&be.clone(), progress_counter(""))?; for source in sources { let mut opts = opts.clone(); @@ -161,7 +161,6 @@ pub(super) async fn execute( |snap| snap.hostname == hostname && snap.paths.contains(&backup_path_str), progress_counter(""), ) - .await .ok(), (false, false, Some(parent)) => SnapshotFile::from_id(&be, &parent).ok(), }; diff --git a/src/commands/cat.rs b/src/commands/cat.rs index 9d3c9bb..ab2c60e 100644 --- a/src/commands/cat.rs +++ b/src/commands/cat.rs @@ -46,16 +46,16 @@ struct TreeOpts { snap: String, } -pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> { +pub(super) fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> { match opts.command { Command::Config => cat_file(be, FileType::Config, IdOpt::default()), Command::Index(opt) => cat_file(be, FileType::Index, opt), Command::Snapshot(opt) => cat_file(be, FileType::Snapshot, opt), // special treatment for catingg blobs: read the index and use it to locate the blob - Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt).await, - Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt).await, + Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt), + Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt), // special treatment for cating a tree within a snapshot - Command::Tree(opts) => cat_tree(be, opts).await, + Command::Tree(opts) => cat_tree(be, opts), } } @@ -67,20 +67,18 @@ fn cat_file(be: &impl DecryptReadBackend, tpe: FileType, opt: IdOpt) -> Result<( Ok(()) } -async fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Result<()> { +fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Result<()> { let id = Id::from_hex(&opt.id)?; - let data = IndexBackend::new(be, ProgressBar::hidden()) - .await? - .blob_from_backend(&tpe, &id)?; + let data = IndexBackend::new(be, ProgressBar::hidden())?.blob_from_backend(&tpe, &id)?; print!("{}", String::from_utf8(data.to_vec())?); Ok(()) } -async fn cat_tree(be: &impl DecryptReadBackend, opts: TreeOpts) -> Result<()> { +fn cat_tree(be: &impl DecryptReadBackend, opts: TreeOpts) -> Result<()> { let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, "")); - let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?; - let index = IndexBackend::new(be, progress_counter("")).await?; + let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?; + let index = IndexBackend::new(be, progress_counter(""))?; let id = Tree::subtree_id(&index, snap.tree, Path::new(path))?; let data = index.blob_from_backend(&BlobType::Tree, &id)?; println!("{}", String::from_utf8(data.to_vec())?); diff --git a/src/commands/check.rs b/src/commands/check.rs index 18e77a1..1dbf85b 100644 --- a/src/commands/check.rs +++ b/src/commands/check.rs @@ -56,11 +56,11 @@ pub(super) async fn execute( if let Some(hot_be) = hot_be { for file_type in [FileType::Snapshot, FileType::Index] { - check_hot_files(raw_be, hot_be, file_type).await?; + check_hot_files(raw_be, hot_be, file_type)?; } } - let index_collector = check_packs(be, hot_be, opts.read_data).await?; + let index_collector = check_packs(be, hot_be, opts.read_data)?; if !opts.trust_cache { if let Some(cache) = &cache { @@ -102,7 +102,7 @@ pub(super) async fn execute( Ok(()) } -async fn check_hot_files( +fn check_hot_files( be: &impl ReadBackend, be_hot: &impl ReadBackend, file_type: FileType, @@ -180,7 +180,7 @@ async fn check_cache_files( } // check if packs correspond to index -async fn check_packs( +fn check_packs( be: &impl DecryptReadBackend, hot_be: &Option, read_data: bool, @@ -224,9 +224,7 @@ async fn check_packs( }; let p = progress_counter("reading index..."); - let mut stream = be.stream_all::(p.clone()).await?; - while let Some(index) = stream.try_next().await? { - let index = index.1; + for (_, index) in be.stream_all::(p.clone())? { index_collector.extend(index.packs.clone()); for p in index.packs { process_pack(p); @@ -235,22 +233,23 @@ async fn check_packs( process_pack(p); } } + p.finish(); if let Some(hot_be) = hot_be { let p = progress_spinner("listing packs in hot repo..."); - check_packs_list(hot_be, tree_packs).await?; + check_packs_list(hot_be, tree_packs)?; p.finish(); } let p = progress_spinner("listing packs..."); - check_packs_list(be, packs).await?; + check_packs_list(be, packs)?; p.finish(); Ok(index_collector) } -async fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap) -> Result<()> { +fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap) -> Result<()> { for (id, size) in be.list_with_size(FileType::Pack)? { match packs.remove(&id) { None => warn!("pack {id} not referenced in index. Can be a parallel backup job. To repair: 'rustic repair index'."), @@ -272,11 +271,10 @@ async fn check_snapshots(index: &(impl IndexedBackend + Unpin)) -> Result<()> { let p = progress_counter("reading snapshots..."); let snap_trees: Vec<_> = index .be() - .stream_all::(p.clone()) - .await? - .map_ok(|(_, snap)| snap.tree) - .try_collect() - .await?; + .stream_all::(p.clone())? + .iter() + .map(|(_, snap)| snap.tree) + .collect(); p.finish(); let p = progress_counter("checking trees..."); diff --git a/src/commands/diff.rs b/src/commands/diff.rs index 8503a48..612f51c 100644 --- a/src/commands/diff.rs +++ b/src/commands/diff.rs @@ -28,12 +28,12 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) let p = progress_spinner("getting snapshots..."); p.finish(); - let snaps = SnapshotFile::from_ids(be, &[id1.to_string(), id2.to_string()]).await?; + let snaps = SnapshotFile::from_ids(be, &[id1.to_string(), id2.to_string()])?; let snap1 = &snaps[0]; let snap2 = &snaps[1]; - let index = IndexBackend::new(be, progress_counter("")).await?; + let index = IndexBackend::new(be, progress_counter(""))?; let id1 = Tree::subtree_id(&index, snap1.tree, Path::new(path1))?; let id2 = Tree::subtree_id(&index, snap2.tree, Path::new(path2))?; diff --git a/src/commands/forget.rs b/src/commands/forget.rs index 5df976b..5201f64 100644 --- a/src/commands/forget.rs +++ b/src/commands/forget.rs @@ -74,10 +74,10 @@ pub(super) async fn execute( .unwrap_or_else(|| SnapshotGroupCriterion::from_str("host,paths").unwrap()); let groups = match opts.ids.is_empty() { - true => SnapshotFile::group_from_backend(be, &opts.config.filter, &group_by).await?, + true => SnapshotFile::group_from_backend(be, &opts.config.filter, &group_by)?, false => vec![( SnapshotGroup::default(), - SnapshotFile::from_ids(be, &opts.ids).await?, + SnapshotFile::from_ids(be, &opts.ids)?, )], }; let mut forget_snaps = Vec::new(); @@ -146,8 +146,7 @@ pub(super) async fn execute( ), (false, false) => { let p = progress_counter("removing snapshots..."); - be.delete_list(FileType::Snapshot, true, forget_snaps.clone(), p) - .await?; + be.delete_list(FileType::Snapshot, true, forget_snaps.clone(), p)?; } } diff --git a/src/commands/list.rs b/src/commands/list.rs index 1eca397..3b9e5f9 100644 --- a/src/commands/list.rs +++ b/src/commands/list.rs @@ -1,6 +1,5 @@ use anyhow::{bail, Result}; use clap::Parser; -use futures::StreamExt; use indicatif::ProgressBar; use crate::backend::{DecryptReadBackend, FileType}; @@ -13,13 +12,12 @@ pub(super) struct Opts { tpe: String, } -pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> { +pub(super) fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> { let tpe = match opts.tpe.as_str() { // special treatment for listing blobs: read the index and display it "blobs" => { - let mut stream = be.stream_all::(ProgressBar::hidden()).await?; - while let Some(index) = stream.next().await { - for pack in index?.1.packs { + for (_, index) in be.stream_all::(ProgressBar::hidden())? { + for pack in index.packs { for blob in pack.blobs { println!("{:?} {}", blob.tpe, blob.id.to_hex()); } diff --git a/src/commands/ls.rs b/src/commands/ls.rs index 477248a..f812eb4 100644 --- a/src/commands/ls.rs +++ b/src/commands/ls.rs @@ -18,8 +18,8 @@ pub(super) struct Opts { pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, "")); - let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?; - let index = IndexBackend::new(be, progress_counter("")).await?; + let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?; + let index = IndexBackend::new(be, progress_counter(""))?; let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?; let mut tree_streamer = NodeStreamer::new(index, tree)?; diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 30c233e..da4d6de 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -307,22 +307,22 @@ pub async fn execute() -> Result<()> { match cmd { Command::Backup(opts) => backup::execute(&dbe, opts, config, config_file, command).await?, Command::Config(opts) => config::execute(&dbe, &be_hot, opts, config)?, - Command::Cat(opts) => cat::execute(&dbe, opts).await?, + Command::Cat(opts) => cat::execute(&dbe, opts)?, Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts).await?, Command::Completions(_) => {} // already handled above Command::Diff(opts) => diff::execute(&dbe, opts).await?, Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file).await?, Command::Init(_) => {} // already handled above Command::Key(opts) => key::execute(&dbe, key, opts)?, - Command::List(opts) => list::execute(&dbe, opts).await?, + Command::List(opts) => list::execute(&dbe, opts)?, Command::Ls(opts) => ls::execute(&dbe, opts).await?, Command::SelfUpdate(_) => {} // already handled above - Command::Snapshots(opts) => snapshots::execute(&dbe, opts, config_file).await?, + Command::Snapshots(opts) => snapshots::execute(&dbe, opts, config_file)?, Command::Prune(opts) => prune::execute(&dbe, cache, opts, config, vec![]).await?, Command::Restore(opts) => restore::execute(&dbe, opts).await?, Command::Repair(opts) => repair::execute(&dbe, opts, config_file, &config).await?, - Command::Repoinfo(opts) => repoinfo::execute(&dbe, &be_hot, opts).await?, - Command::Tag(opts) => tag::execute(&dbe, opts, config_file).await?, + Command::Repoinfo(opts) => repoinfo::execute(&dbe, &be_hot, opts)?, + Command::Tag(opts) => tag::execute(&dbe, opts, config_file)?, }; Ok(()) diff --git a/src/commands/prune.rs b/src/commands/prune.rs index cb8984c..804b304 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -7,7 +7,7 @@ use bytesize::ByteSize; use chrono::{DateTime, Duration, Local}; use clap::{AppSettings, Parser}; use derive_more::Add; -use futures::{future, TryStreamExt}; +use futures::TryStreamExt; use log::*; use super::{bytes, no_progress, progress_bytes, progress_counter, wait, warm_up, warm_up_command}; @@ -98,10 +98,9 @@ pub(super) async fn execute( let mut index_files = Vec::new(); let p = progress_counter("reading index..."); - let mut stream = be.stream_all::(p.clone()).await?; let mut index_collector = IndexCollector::new(IndexType::OnlyTrees); - while let Some((id, index)) = stream.try_next().await? { + for (id, index) in be.stream_all::(p.clone())? { index_collector.extend(index.packs.clone()); // we add the trees from packs_to_delete to the index such that searching for // used blobs doesn't abort if they are already marked for deletion @@ -879,8 +878,7 @@ impl Pruner { let p = progress_counter("removing unindexed packs..."); let existing_packs: Vec<_> = self.existing_packs.into_iter().map(|(id, _)| id).collect(); - be.delete_list(FileType::Pack, true, existing_packs, p) - .await?; + be.delete_list(FileType::Pack, true, existing_packs, p)?; } else { info!("marking not needed unindexed pack files for deletion..."); for (id, size) in self.existing_packs { @@ -991,20 +989,17 @@ impl Pruner { if !data_packs_remove.is_empty() { let p = progress_counter("removing old data packs..."); - be.delete_list(FileType::Pack, false, data_packs_remove, p) - .await?; + be.delete_list(FileType::Pack, false, data_packs_remove, p)?; } if !tree_packs_remove.is_empty() { let p = progress_counter("removing old tree packs..."); - be.delete_list(FileType::Pack, true, tree_packs_remove, p) - .await?; + be.delete_list(FileType::Pack, true, tree_packs_remove, p)?; } if !indexes_remove.is_empty() { let p = progress_counter("removing old index files..."); - be.delete_list(FileType::Index, true, indexes_remove, p) - .await?; + be.delete_list(FileType::Index, true, indexes_remove, p)?; } Ok(()) @@ -1090,14 +1085,13 @@ async fn find_used_blobs( let p = progress_counter("reading snapshots..."); let snap_trees: Vec<_> = index .be() - .stream_all::(p.clone()) - .await? + .stream_all::(p.clone())? + .into_iter() // TODO: it would even better to give ignore_snaps to the streaming function instead // if reading and then filtering the snapshot - .try_filter(|(id, _)| future::ready(!ignore_snaps.contains(id))) - .map_ok(|(_, snap)| snap.tree) - .try_collect() - .await?; + .filter(|(id, _)| !ignore_snaps.contains(id)) + .map(|(_, snap)| snap.tree) + .collect(); p.finish(); let mut ids: HashMap<_, _> = snap_trees.iter().map(|id| (*id, 0)).collect(); diff --git a/src/commands/repair.rs b/src/commands/repair.rs index fac4446..d36bdcc 100644 --- a/src/commands/repair.rs +++ b/src/commands/repair.rs @@ -3,7 +3,6 @@ use std::collections::{HashMap, HashSet}; use anyhow::Result; use async_recursion::async_recursion; use clap::{AppSettings, Parser, Subcommand}; -use futures::TryStreamExt; use log::*; use crate::backend::{DecryptFullBackend, DecryptWriteBackend, FileType}; @@ -142,12 +141,9 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<( }; let p = progress_counter("reading index..."); - let mut stream = be.stream_all::(p.clone()).await?; - while let Some(index) = stream.try_next().await? { + for (index_id, index) in be.stream_all::(p.clone())? { let mut new_index = IndexFile::default(); let mut changed = false; - let index_id = index.0; - let index = index.1; for p in index.packs { process_pack(p, false, &mut new_index, &mut changed); } @@ -214,15 +210,15 @@ async fn repair_snaps( config_file.merge_into("snapshot-filter", &mut opts.filter)?; let snapshots = match opts.ids.is_empty() { - true => SnapshotFile::all_from_backend(be, &opts.filter).await?, - false => SnapshotFile::from_ids(be, &opts.ids).await?, + true => SnapshotFile::all_from_backend(be, &opts.filter)?, + false => SnapshotFile::from_ids(be, &opts.ids)?, }; let mut replaced = HashMap::new(); let mut seen = HashSet::new(); let mut delete = Vec::new(); - let index = IndexBackend::new(&be.clone(), progress_counter("")).await?; + let index = IndexBackend::new(&be.clone(), progress_counter(""))?; let indexer = Indexer::new(be.clone()).into_shared(); let mut packer = Packer::new( be.clone(), @@ -284,8 +280,7 @@ async fn repair_snaps( true, delete, progress_counter("remove defect snapshots"), - ) - .await?; + )?; } } diff --git a/src/commands/repoinfo.rs b/src/commands/repoinfo.rs index ebc32f5..c975cc1 100644 --- a/src/commands/repoinfo.rs +++ b/src/commands/repoinfo.rs @@ -1,7 +1,6 @@ use anyhow::Result; use clap::Parser; use derive_more::Add; -use futures::TryStreamExt; use log::*; use prettytable::{format, row, Table}; @@ -14,19 +13,16 @@ use crate::repo::{IndexFile, IndexPack}; #[derive(Parser)] pub(super) struct Opts; -pub(super) async fn execute( +pub(super) fn execute( be: &impl DecryptReadBackend, hot_be: &Option, _opts: Opts, ) -> Result<()> { - fileinfo("repository files", be).await?; + fileinfo("repository files", be)?; if let Some(hot_be) = hot_be { - fileinfo("hot repository files", hot_be).await?; + fileinfo("hot repository files", hot_be)?; } - let p = progress_counter("scanning index..."); - let mut stream = be.stream_all::(p.clone()).await?; - #[derive(Default, Clone, Copy, Add)] struct Info { count: u64, @@ -59,7 +55,8 @@ pub(super) async fn execute( info[BlobType::Data].min_pack_size = u64::MAX; let mut info_delete = BlobTypeMap::::default(); - while let Some((_, index)) = stream.try_next().await? { + let p = progress_counter("scanning index..."); + for (_, index) in be.stream_all::(p.clone())? { for pack in &index.packs { info[pack.blob_type()].add_pack(pack); @@ -109,7 +106,7 @@ pub(super) async fn execute( Ok(()) } -async fn fileinfo(text: &str, be: &impl ReadBackend) -> Result<()> { +fn fileinfo(text: &str, be: &impl ReadBackend) -> Result<()> { info!("scanning files..."); let mut table = Table::new(); diff --git a/src/commands/restore.rs b/src/commands/restore.rs index e0ebc42..812e513 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -67,9 +67,9 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) } let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, "")); - let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?; + let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?; - let index = IndexBackend::new(be, progress_counter("")).await?; + let index = IndexBackend::new(be, progress_counter(""))?; let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?; let dest = LocalBackend::new(&opts.dest); diff --git a/src/commands/snapshots.rs b/src/commands/snapshots.rs index fdcd70d..09c2a06 100644 --- a/src/commands/snapshots.rs +++ b/src/commands/snapshots.rs @@ -43,7 +43,7 @@ pub(super) struct Opts { ids: Vec, } -pub(super) async fn execute( +pub(super) fn execute( be: &impl DecryptReadBackend, mut opts: Opts, config_file: RusticConfig, @@ -51,10 +51,9 @@ pub(super) async fn execute( config_file.merge_into("snapshot-filter", &mut opts.filter)?; let groups = match &opts.ids[..] { - [] => SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by).await?, + [] => SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by)?, [id] if id == "latest" => { - SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by) - .await? + SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by)? .into_iter() .map(|(group, mut snaps)| { snaps.sort_unstable(); @@ -67,7 +66,7 @@ pub(super) async fn execute( } _ => vec![( SnapshotGroup::default(), - SnapshotFile::from_ids(be, &opts.ids).await?, + SnapshotFile::from_ids(be, &opts.ids)?, )], }; diff --git a/src/commands/tag.rs b/src/commands/tag.rs index 7758905..8e100a7 100644 --- a/src/commands/tag.rs +++ b/src/commands/tag.rs @@ -68,7 +68,7 @@ pub(super) struct Opts { ids: Vec, } -pub(super) async fn execute( +pub(super) fn execute( be: &impl DecryptFullBackend, mut opts: Opts, config_file: RusticConfig, @@ -76,8 +76,8 @@ pub(super) async fn execute( config_file.merge_into("snapshot-filter", &mut opts.filter)?; let snapshots = match opts.ids.is_empty() { - true => SnapshotFile::all_from_backend(be, &opts.filter).await?, - false => SnapshotFile::from_ids(be, &opts.ids).await?, + true => SnapshotFile::all_from_backend(be, &opts.filter)?, + false => SnapshotFile::from_ids(be, &opts.ids)?, }; let delete = match ( @@ -109,11 +109,10 @@ pub(super) async fn execute( ), (false, false) => { let p = progress_counter("saving new snapshots..."); - be.save_list(snapshots, p).await?; + be.save_list(snapshots, p)?; let p = progress_counter("deleting old snapshots..."); - be.delete_list(FileType::Snapshot, true, old_snap_ids, p) - .await?; + be.delete_list(FileType::Snapshot, true, old_snap_ids, p)?; } } Ok(()) diff --git a/src/index/mod.rs b/src/index/mod.rs index dc5d4c5..17035bb 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -2,11 +2,9 @@ use std::num::NonZeroU32; use std::sync::Arc; use anyhow::{anyhow, Result}; -use async_trait::async_trait; use bytes::Bytes; use derive_getters::Getters; use derive_more::Constructor; -use futures::StreamExt; use indicatif::ProgressBar; use crate::backend::{DecryptReadBackend, FileType}; @@ -83,7 +81,6 @@ pub trait ReadIndex { } } -#[async_trait] pub trait IndexedBackend: ReadIndex + Clone + Sync + Send + 'static { type Backend: DecryptReadBackend; @@ -124,31 +121,23 @@ impl IndexBackend { } } - async fn new_from_collector( - be: &BE, - p: ProgressBar, - mut collector: IndexCollector, - ) -> Result { + fn new_from_collector(be: &BE, p: ProgressBar, mut collector: IndexCollector) -> Result { p.set_prefix("reading index..."); - let mut stream = be - .stream_all::(p.clone()) - .await? - .map(|i| i.unwrap().1); - - while let Some(index) = stream.next().await { - collector.extend(index.packs); + for (_, i) in be.stream_all::(p.clone())? { + collector.extend(i.packs); } + p.finish(); Ok(Self::new_from_index(be, collector.into_index())) } - pub async fn new(be: &BE, p: ProgressBar) -> Result { - Self::new_from_collector(be, p, IndexCollector::new(IndexType::Full)).await + pub fn new(be: &BE, p: ProgressBar) -> Result { + Self::new_from_collector(be, p, IndexCollector::new(IndexType::Full)) } - pub async fn only_full_trees(be: &BE, p: ProgressBar) -> Result { - Self::new_from_collector(be, p, IndexCollector::new(IndexType::FullTrees)).await + pub fn only_full_trees(be: &BE, p: ProgressBar) -> Result { + Self::new_from_collector(be, p, IndexCollector::new(IndexType::FullTrees)) } pub fn into_index(self) -> Index { diff --git a/src/repo/snapshotfile.rs b/src/repo/snapshotfile.rs index 0788f46..b29eaea 100644 --- a/src/repo/snapshotfile.rs +++ b/src/repo/snapshotfile.rs @@ -6,7 +6,6 @@ use anyhow::{anyhow, bail, Result}; use chrono::{DateTime, Local}; use clap::Parser; use derivative::Derivative; -use futures::{future, TryStreamExt}; use indicatif::ProgressBar; use log::*; use merge::Merge; @@ -112,33 +111,33 @@ impl SnapshotFile { Ok(Self::set_id((*id, be.get_file(id)?))) } - pub async fn from_str( + pub fn from_str( be: &B, string: &str, - predicate: impl FnMut(&Self) -> bool, + predicate: impl FnMut(&Self) -> bool + Send + Sync, p: ProgressBar, ) -> Result { match string { - "latest" => Self::latest(be, predicate, p).await, + "latest" => Self::latest(be, predicate, p), _ => Self::from_id(be, string), } } /// Get the latest SnapshotFile from the backend - pub async fn latest( + pub fn latest( be: &B, - predicate: impl FnMut(&Self) -> bool, + predicate: impl FnMut(&Self) -> bool + Send + Sync, p: ProgressBar, ) -> Result { p.set_prefix("getting latest snapshot..."); let mut latest: Option = None; let mut pred = predicate; - let mut snaps = be.stream_all::(p.clone()).await?; - while let Some((id, mut snap)) = snaps.try_next().await? { + for (id, mut snap) in be.stream_all::(p.clone())? { if !pred(&snap) { continue; } + snap.id = id; match &latest { Some(l) if l.time > snap.time => {} @@ -159,14 +158,13 @@ impl SnapshotFile { } /// Get a Vector of SnapshotFile from the backend by list of (parts of the) ids - pub async fn from_ids(be: &B, ids: &[String]) -> Result> { + pub fn from_ids(be: &B, ids: &[String]) -> Result> { let ids = be.find_ids(FileType::Snapshot, ids)?; Ok(be - .stream_list::(ids, ProgressBar::hidden()) - .await? - .map_ok(Self::set_id) - .try_collect() - .await?) + .stream_list::(ids, ProgressBar::hidden())? + .into_iter() + .map(Self::set_id) + .collect()) } fn cmp_group(&self, crit: &SnapshotGroupCriterion, other: &Self) -> Ordering { @@ -199,12 +197,12 @@ impl SnapshotFile { /// Get SnapshotFiles which match the filter grouped by the group criterion /// from the backend - pub async fn group_from_backend( + pub fn group_from_backend( be: &B, filter: &SnapshotFilter, crit: &SnapshotGroupCriterion, ) -> Result)>> { - let mut snaps = Self::all_from_backend(be, filter).await?; + let mut snaps = Self::all_from_backend(be, filter)?; snaps.sort_unstable_by(|sn1, sn2| sn1.cmp_group(crit, sn2)); let mut result = Vec::new(); @@ -233,17 +231,16 @@ impl SnapshotFile { Ok(result) } - pub async fn all_from_backend( + pub fn all_from_backend( be: &B, filter: &SnapshotFilter, ) -> Result> { Ok(be - .stream_all::(ProgressBar::hidden()) - .await? - .map_ok(Self::set_id) - .try_filter(|sn| future::ready(sn.matches(filter))) - .try_collect() - .await?) + .stream_all::(ProgressBar::hidden())? + .into_iter() + .map(Self::set_id) + .filter(|sn| sn.matches(filter)) + .collect()) } pub fn matches(&self, filter: &SnapshotFilter) -> bool { From d8665fe794ee0827f3ad1d3db03efc2bf1c6a150 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Mon, 17 Oct 2022 15:41:17 +0200 Subject: [PATCH 03/10] Parallelize check --- src/commands/check.rs | 50 ++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/src/commands/check.rs b/src/commands/check.rs index 1dbf85b..1856dea 100644 --- a/src/commands/check.rs +++ b/src/commands/check.rs @@ -3,10 +3,10 @@ use std::collections::HashMap; use anyhow::Result; use bytes::Bytes; use clap::Parser; -use futures::{stream, StreamExt, TryStreamExt}; +use futures::TryStreamExt; use indicatif::ProgressBar; use log::*; -use tokio::task::spawn_blocking; +use rayon::prelude::*; use zstd::stream::decode_all; use super::{progress_bytes, progress_counter}; @@ -49,7 +49,7 @@ pub(super) async fn execute( let p = progress_bytes(format!("checking {} in cache...", file_type.name())); // TODO: Make concurrency (20) customizable - check_cache_files(20, cache, raw_be, file_type, p).await?; + check_cache_files(20, cache, raw_be, file_type, p)?; } } } @@ -66,7 +66,7 @@ pub(super) async fn execute( if let Some(cache) = &cache { let p = progress_bytes("checking packs in cache..."); // TODO: Make concurrency (5) customizable - check_cache_files(5, cache, raw_be, FileType::Pack, p).await?; + check_cache_files(5, cache, raw_be, FileType::Pack, p)?; } } @@ -76,26 +76,20 @@ pub(super) async fn execute( if opts.read_data { let p = progress_counter("reading pack data..."); - stream::iter(index_be.into_index().into_iter().map(|pack| { - let be = be.clone(); - let p = p.clone(); - (pack, be, p) - })) - // TODO: Make concurrency (4) customizable - .for_each_concurrent(4, |(pack, be, p)| async move { - let id = pack.id; - let data = be.read_full(FileType::Pack, &id).unwrap(); - spawn_blocking(move || { - match check_pack(&be, pack, data) { + + index_be + .into_index() + .into_iter() + .par_bridge() + .for_each_with((be.clone(), p.clone()), |(be, p), pack| { + let id = pack.id; + let data = be.read_full(FileType::Pack, &id).unwrap(); + match check_pack(be, pack, data) { Ok(()) => {} Err(err) => error!("Error reading pack {id} : {err}",), } p.inc(1); - }) - .await - .unwrap() - }) - .await; + }); p.finish(); } @@ -133,8 +127,8 @@ fn check_hot_files( Ok(()) } -async fn check_cache_files( - concurrency: usize, +fn check_cache_files( + _concurrency: usize, cache: &Cache, be: &impl ReadBackend, file_type: FileType, @@ -149,13 +143,8 @@ async fn check_cache_files( let total_size = files.iter().map(|(_, size)| *size as u64).sum(); p.set_length(total_size); - stream::iter(files.into_iter().map(|file| { - let cache = cache.clone(); - let be = be.clone(); - let p = p.clone(); - (file, cache, be, p) - })) - .for_each_concurrent(concurrency, |((id, size), cache, be, p)| async move { + files.into_par_iter() + .for_each_with((cache,be,p.clone()), |(cache, be, p),(id, size)| { // Read file from cache and from backend and compare match ( cache.read_full(file_type, &id), @@ -172,8 +161,7 @@ async fn check_cache_files( } p.inc(size as u64); - }) - .await; + }); p.finish(); Ok(()) From fa0ac71aec6e67ba245653e9b0ae89882f455412 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 20 Oct 2022 00:04:23 +0200 Subject: [PATCH 04/10] Make packer sync --- src/archiver/archiver_impl.rs | 18 ++--- src/blob/packer.rs | 148 ++++++++++++++++++++-------------- src/commands/prune.rs | 26 +++--- src/commands/repair.rs | 24 +++--- src/index/indexer.rs | 2 +- 5 files changed, 121 insertions(+), 97 deletions(-) diff --git a/src/archiver/archiver_impl.rs b/src/archiver/archiver_impl.rs index 7e6130e..b053e9d 100644 --- a/src/archiver/archiver_impl.rs +++ b/src/archiver/archiver_impl.rs @@ -188,7 +188,7 @@ impl Archiver { } if !self.index.has_tree(&id) { - match self.tree_packer.add(&chunk, &id).await? { + match self.tree_packer.add(&chunk, &id)? { 0 => {} packed_size => { self.summary.tree_blobs += 1; @@ -242,13 +242,13 @@ impl Archiver { if queue.len() > 8 { let (id, chunk, size) = queue.next().await.unwrap()?; - self.process_data_junk(id, &chunk, size, &p).await?; + self.process_data_junk(id, &chunk, size, &p)?; content.push(id); } } while let Some(Ok((id, chunk, size))) = queue.next().await { - self.process_data_junk(id, &chunk, size, &p).await?; + self.process_data_junk(id, &chunk, size, &p)?; content.push(id); } @@ -258,7 +258,7 @@ impl Archiver { Ok(()) } - async fn process_data_junk( + fn process_data_junk( &mut self, id: Id, chunk: &[u8], @@ -266,7 +266,7 @@ impl Archiver { p: &ProgressBar, ) -> Result<()> { if !self.index.has_data(&id) { - match self.data_packer.add(chunk, &id).await? { + match self.data_packer.add(chunk, &id)? { 0 => {} packed_size => { self.summary.data_blobs += 1; @@ -286,14 +286,14 @@ impl Archiver { let (chunk, id) = self.tree.serialize()?; if !self.index.has_tree(&id) { - self.tree_packer.add(&chunk, &id).await?; + self.tree_packer.add(&chunk, &id)?; } self.snap.tree = id; - self.data_packer.finalize().await?; - self.tree_packer.finalize().await?; + self.data_packer.finalize()?; + self.tree_packer.finalize()?; { - let indexer = self.indexer.write().await; + let indexer = self.indexer.write().unwrap(); indexer.finalize()?; } let end_time = Local::now(); diff --git a/src/blob/packer.rs b/src/blob/packer.rs index 568f3e7..a52c503 100644 --- a/src/blob/packer.rs +++ b/src/blob/packer.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime}; use anyhow::{anyhow, Result}; use bytes::{Bytes, BytesMut}; use chrono::Local; -use tokio::{spawn, task::JoinHandle}; +use crossbeam_channel::{bounded, Receiver, Sender}; use zstd::encode_all; use super::BlobType; @@ -74,7 +74,7 @@ pub struct Packer { index: IndexPack, indexer: SharedIndexer, hasher: Hasher, - file_writer: FileWriter, + file_writer: Actor<(Bytes, Id, IndexPack)>, zstd: Option, pack_sizer: PackSizer, } @@ -87,12 +87,15 @@ impl Packer { config: &ConfigFile, total_size: u64, ) -> Result { - let file_writer = FileWriter { - future: None, - be: be.clone(), - indexer: indexer.clone(), - cacheable: blob_type.is_cacheable(), - }; + let file_writer = Actor::new( + FileWriterHandle { + be: be.clone(), + indexer: indexer.clone(), + cacheable: blob_type.is_cacheable(), + }, + 1, + 1, + ); let zstd = config.zstd()?; let pack_sizer = PackSizer::from_config(config, blob_type, total_size); Ok(Self { @@ -111,12 +114,13 @@ impl Packer { }) } - pub async fn finalize(&mut self) -> Result<()> { - self.save().await?; - self.file_writer.finalize().await + pub fn finalize(mut self) -> Result<()> { + self.save()?; + self.file_writer.finalize()?; + Ok(()) } - pub async fn write_data(&mut self, data: &[u8]) -> Result { + pub fn write_data(&mut self, data: &[u8]) -> Result { self.hasher.update(data); let len = data.len().try_into()?; self.file.extend_from_slice(data); @@ -125,25 +129,20 @@ impl Packer { } // adds the blob to the packfile; returns the actually added size - pub async fn add(&mut self, data: &[u8], id: &Id) -> Result { + pub fn add(&mut self, data: &[u8], id: &Id) -> Result { // compute size limit based on total size and size bounds let size_limit = self.pack_sizer.pack_size(); - self.add_with_sizelimit(data, id, size_limit).await + self.add_with_sizelimit(data, id, size_limit) } // adds the blob to the packfile; returns the actually added size - pub async fn add_with_sizelimit( - &mut self, - data: &[u8], - id: &Id, - size_limit: u32, - ) -> Result { + pub fn add_with_sizelimit(&mut self, data: &[u8], id: &Id, size_limit: u32) -> Result { // only add if this blob is not present if self.has(id) { return Ok(0); } { - let indexer = self.indexer.read().await; + let indexer = self.indexer.read().unwrap(); if indexer.has(id) { return Ok(0); } @@ -167,13 +166,12 @@ impl Packer { }; // add using current total_size as repo_size - self.add_raw(&data, id, uncompressed_length, size_limit) - .await?; + self.add_raw(&data, id, uncompressed_length, size_limit)?; Ok(data.len().try_into()?) } // adds the already compressed/encrypted blob to the packfile without any check - pub async fn add_raw( + pub fn add_raw( &mut self, data: &[u8], id: &Id, @@ -181,7 +179,7 @@ impl Packer { size_limit: u32, ) -> Result<()> { let offset = self.size; - let len = self.write_data(data).await?; + let len = self.write_data(data)?; self.index .add(*id, self.blob_type, offset, len, uncompressed_length); self.count += 1; @@ -190,7 +188,7 @@ impl Packer { if self.count >= MAX_COUNT || self.size >= size_limit || self.created.elapsed()? >= MAX_AGE { self.pack_sizer.add_size(self.index.pack_size()); - self.save().await?; + self.save()?; self.size = 0; self.count = 0; self.created = SystemTime::now(); @@ -200,7 +198,7 @@ impl Packer { } /// writes header and length of header to packfile - pub async fn write_header(&mut self) -> Result<()> { + pub fn write_header(&mut self) -> Result<()> { // comput the pack header let data = PackHeaderRef::from_index_pack(&self.index).to_binary()?; @@ -212,21 +210,20 @@ impl Packer { .map_err(|_| anyhow!("crypto error"))?; let headerlen = data.len().try_into()?; - self.write_data(&data).await?; + self.write_data(&data)?; // finally write length of header unencrypted to pack file - self.write_data(&PackHeaderLength::from_u32(headerlen).to_binary()?) - .await?; + self.write_data(&PackHeaderLength::from_u32(headerlen).to_binary()?)?; Ok(()) } - pub async fn save(&mut self) -> Result<()> { + pub fn save(&mut self) -> Result<()> { if self.size == 0 { return Ok(()); } - self.write_header().await?; + self.write_header()?; // compute id of packfile let id = self.hasher.finalize(); @@ -235,7 +232,7 @@ impl Packer { // write file to backend let index = std::mem::take(&mut self.index); let file = std::mem::replace(&mut self.file, BytesMut::new()); - self.file_writer.add(index, file.into(), id).await?; + self.file_writer.send((file.into(), id, index))?; Ok(()) } @@ -245,36 +242,69 @@ impl Packer { } } -struct FileWriter { - future: Option>>, +#[derive(Clone)] +struct FileWriterHandle { be: BE, indexer: SharedIndexer, cacheable: bool, } -impl FileWriter { - async fn add(&mut self, mut index: IndexPack, file: Bytes, id: Id) -> Result<()> { - let be = self.be.clone(); - let indexer = self.indexer.clone(); - let cacheable = self.cacheable; - let new_future = spawn(async move { - be.write_bytes(FileType::Pack, &id, cacheable, file)?; - index.time = Some(Local::now()); - indexer.write().await.add(index)?; - Ok(()) +impl ActorHandle<(Bytes, Id, IndexPack)> for FileWriterHandle { + fn process(&self, load: (Bytes, Id, IndexPack)) -> Result<()> { + let (file, id, mut index) = load; + self.be + .write_bytes(FileType::Pack, &id, self.cacheable, file)?; + index.time = Some(Local::now()); + self.indexer.write().unwrap().add(index)?; + Ok(()) + } +} + +pub trait ActorHandle: Clone + Send + 'static { + fn process(&self, load: T) -> Result<()>; +} + +pub struct Actor { + sender: Sender, + finish: Receiver>, +} + +impl Actor { + pub fn new(fwh: impl ActorHandle, queue_len: usize, par: usize) -> Self { + let (tx, rx) = bounded(queue_len); + let (finish_tx, finish_rx) = bounded::>(0); + (0..par).for_each(|_| { + let rx = rx.clone(); + let finish_tx = finish_tx.clone(); + let fwh = fwh.clone(); + std::thread::spawn(move || { + let mut status = Ok(()); + for load in rx { + // only keep processing if there was no error + if status.is_ok() { + status = fwh.process(load); + } + } + let _ = finish_tx.send(status); + }); }); - if let Some(fut) = self.future.replace(new_future) { - fut.await??; + Self { + sender: tx, + finish: finish_rx, } + } + + pub fn send(&self, load: T) -> Result<()> { + self.sender.send(load)?; Ok(()) } - async fn finalize(&mut self) -> Result<()> { - if let Some(fut) = self.future.take() { - return fut.await?; - } - Ok(()) + pub fn finalize(self) -> Result<()> { + // cancel channel + drop(self.sender); + // wait for items in channel to be processed + self.finish.recv().unwrap() } } @@ -301,7 +331,7 @@ impl Repacker { }) } - pub async fn add_fast(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> { + pub fn add_fast(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> { let data = self.be.read_partial( FileType::Pack, pack_id, @@ -310,12 +340,11 @@ impl Repacker { blob.length, )?; self.packer - .add_raw(&data, &blob.id, blob.uncompressed_length, self.size_limit) - .await?; + .add_raw(&data, &blob.id, blob.uncompressed_length, self.size_limit)?; Ok(()) } - pub async fn add(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> { + pub fn add(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> { let data = self.be.read_encrypted_partial( FileType::Pack, pack_id, @@ -325,12 +354,11 @@ impl Repacker { blob.uncompressed_length, )?; self.packer - .add_with_sizelimit(&data, &blob.id, self.size_limit) - .await?; + .add_with_sizelimit(&data, &blob.id, self.size_limit)?; Ok(()) } - pub async fn finalize(&mut self) -> Result<()> { - self.packer.finalize().await + pub fn finalize(self) -> Result<()> { + self.packer.finalize() } } diff --git a/src/commands/prune.rs b/src/commands/prune.rs index 804b304..407827c 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -173,7 +173,7 @@ pub(super) async fn execute( wait(opts.warm_up_wait).await; if !opts.dry_run { - pruner.do_prune(be, opts, config).await?; + pruner.do_prune(be, opts, config)?; } Ok(()) } @@ -828,7 +828,7 @@ impl Pruner { .collect() } - async fn do_prune( + fn do_prune( mut self, be: &impl DecryptFullBackend, opts: Opts, @@ -888,7 +888,7 @@ impl Pruner { time: Some(Local::now()), blobs: Vec::new(), }; - indexer.write().await.add_remove(pack)?; + indexer.write().unwrap().add_remove(pack)?; } } } @@ -925,7 +925,7 @@ impl Pruner { PackToDo::Keep => { // keep pack: add to new index let pack = pack.into_index_pack(); - indexer.write().await.add(pack)?; + indexer.write().unwrap().add(pack)?; } PackToDo::Repack => { // TODO: repack in parallel @@ -940,9 +940,9 @@ impl Pruner { BlobType::Tree => &mut tree_repacker, }; if opts.fast_repack { - repacker.add_fast(&pack.id, blob).await?; + repacker.add_fast(&pack.id, blob)?; } else { - repacker.add(&pack.id, blob).await?; + repacker.add(&pack.id, blob)?; } p.inc(blob.length as u64); } @@ -951,7 +951,7 @@ impl Pruner { } else { // mark pack for removal let pack = pack.into_index_pack_with_time(self.time); - indexer.write().await.add_remove(pack)?; + indexer.write().unwrap().add_remove(pack)?; } } PackToDo::MarkDelete => { @@ -960,7 +960,7 @@ impl Pruner { } else { // mark pack for removal let pack = pack.into_index_pack_with_time(self.time); - indexer.write().await.add_remove(pack)?; + indexer.write().unwrap().add_remove(pack)?; } } PackToDo::KeepMarked => { @@ -969,22 +969,22 @@ impl Pruner { } else { // keep pack: add to new index let pack = pack.into_index_pack(); - indexer.write().await.add_remove(pack)?; + indexer.write().unwrap().add_remove(pack)?; } } PackToDo::Recover => { // recover pack: add to new index in section packs let pack = pack.into_index_pack_with_time(self.time); - indexer.write().await.add(pack)?; + indexer.write().unwrap().add(pack)?; } PackToDo::Delete => delete_pack(pack), } } indexes_remove.push(index.id); } - tree_repacker.finalize().await?; - data_repacker.finalize().await?; - indexer.write().await.finalize()?; + tree_repacker.finalize()?; + data_repacker.finalize()?; + indexer.write().unwrap().finalize()?; p.finish(); if !data_packs_remove.is_empty() { diff --git a/src/commands/repair.rs b/src/commands/repair.rs index d36bdcc..586d43d 100644 --- a/src/commands/repair.rs +++ b/src/commands/repair.rs @@ -1,7 +1,6 @@ use std::collections::{HashMap, HashSet}; use anyhow::Result; -use async_recursion::async_recursion; use clap::{AppSettings, Parser, Subcommand}; use log::*; @@ -89,7 +88,7 @@ pub(super) async fn execute( ) -> Result<()> { match opts.command { Command::Index(opt) => repair_index(be, opt).await, - Command::Snapshots(opt) => repair_snaps(be, opt, config_file, config).await, + Command::Snapshots(opt) => repair_snaps(be, opt, config_file, config), } } @@ -191,17 +190,17 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<( pack.set_id(id); pack.blobs = PackHeader::from_file(be, id, size_hint, packsize)?.into_blobs(); if !opts.dry_run { - indexer.write().await.add_with(pack, to_delete)?; + indexer.write().unwrap().add_with(pack, to_delete)?; } p.inc(1); } - indexer.write().await.finalize()?; + indexer.write().unwrap().finalize()?; p.finish(); Ok(()) } -async fn repair_snaps( +fn repair_snaps( be: &impl DecryptFullBackend, mut opts: SnapOpts, config_file: RusticConfig, @@ -238,9 +237,7 @@ async fn repair_snaps( &mut replaced, &mut seen, &opts, - ) - .await? - { + )? { (Changed::None, _) => { info!("snapshot {snap_id} is ok."); } @@ -267,8 +264,8 @@ async fn repair_snaps( } if !opts.dry_run { - packer.finalize().await?; - indexer.write().await.finalize()?; + packer.finalize()?; + indexer.write().unwrap().finalize()?; } if opts.delete { @@ -294,8 +291,7 @@ enum Changed { None, } -#[async_recursion] -async fn repair_tree( +fn repair_tree( be: &impl IndexedBackend, packer: &mut Packer, id: Option, @@ -352,7 +348,7 @@ async fn repair_tree( } NodeType::Dir {} => { let (c, tree_id) = - repair_tree(be, packer, node.subtree, replaced, seen, opts).await?; + repair_tree(be, packer, node.subtree, replaced, seen, opts)?; match c { Changed::None => {} Changed::This => { @@ -385,7 +381,7 @@ async fn repair_tree( // the tree has been changed => save it let (chunk, new_id) = tree.serialize()?; if !be.has_tree(&new_id) && !opts.dry_run { - packer.add(&chunk, &new_id).await?; + packer.add(&chunk, &new_id)?; } if let Some(id) = id { replaced.insert(id, (c, new_id)); diff --git a/src/index/indexer.rs b/src/index/indexer.rs index 4aba144..ca7d48b 100644 --- a/src/index/indexer.rs +++ b/src/index/indexer.rs @@ -1,9 +1,9 @@ use std::collections::HashSet; use std::sync::Arc; +use std::sync::RwLock; use std::time::{Duration, SystemTime}; use anyhow::Result; -use tokio::sync::RwLock; use crate::backend::DecryptWriteBackend; use crate::id::Id; From 8b87630e1bd14ae988c77d803d398396c0c7a8d0 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 20 Oct 2022 11:40:05 +0200 Subject: [PATCH 05/10] Make archiver async --- Cargo.lock | 36 +++++++++++++++++++ Cargo.toml | 1 + src/archiver/archiver_impl.rs | 65 +++++++++++++++++------------------ src/commands/backup.rs | 30 ++++++++-------- 4 files changed, 83 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d2e78c..4aac945 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -356,6 +356,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -390,6 +404,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.11" @@ -1240,6 +1264,17 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "pariter" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "324a62b9e7b5f270c0acc92a2040f8028bb643f959f9c068f11a7864f327e3d9" +dependencies = [ + "crossbeam", + "crossbeam-channel", + "num_cpus", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -1649,6 +1684,7 @@ dependencies = [ "log", "merge", "nix", + "pariter", "path-absolutize", "prettytable-rs", "quickcheck", diff --git a/Cargo.toml b/Cargo.toml index e9a602e..f4d509d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ log = "0.4" # parallelize crossbeam-channel = "0.5" rayon = "1" +pariter = "0.5" # async tokio = { version = "1", features = ["full"] } futures = "0.3" diff --git a/src/archiver/archiver_impl.rs b/src/archiver/archiver_impl.rs index b053e9d..4961f98 100644 --- a/src/archiver/archiver_impl.rs +++ b/src/archiver/archiver_impl.rs @@ -5,10 +5,9 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, Result}; use bytesize::ByteSize; use chrono::Local; -use futures::{stream::FuturesOrdered, StreamExt}; use indicatif::ProgressBar; use log::*; -use tokio::spawn; +use pariter::IteratorExt; use crate::backend::DecryptWriteBackend; use crate::blob::{BlobType, Metadata, Node, NodeType, Packer, Tree}; @@ -105,7 +104,7 @@ impl Archiver { self.summary.total_dirsize_processed += size; } - pub async fn add_entry(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> { + pub fn add_entry(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> { let basepath = if node.is_dir() { path } else { @@ -113,7 +112,7 @@ impl Archiver { .ok_or_else(|| anyhow!("file path should have a parent!"))? }; - self.finish_trees(basepath).await?; + self.finish_trees(basepath)?; let missing_dirs = basepath.strip_prefix(&self.path)?; for p in missing_dirs.iter() { @@ -136,7 +135,7 @@ impl Archiver { match node.node_type() { NodeType::File => { - self.backup_file(path, node, p).await?; + self.backup_file(path, node, p)?; } NodeType::Dir => {} // is already handled, see above _ => self.add_file(node, 0), // all other cases: just save the given node @@ -144,7 +143,7 @@ impl Archiver { Ok(()) } - pub async fn finish_trees(&mut self, path: &Path) -> Result<()> { + pub fn finish_trees(&mut self, path: &Path) -> Result<()> { while !path.starts_with(&self.path) { // save tree and go back to parent dir let (chunk, id) = self.tree.serialize()?; @@ -158,13 +157,13 @@ impl Archiver { self.tree = tree; self.parent = parent; - self.backup_tree(node, chunk).await?; + self.backup_tree(node, chunk)?; self.path.pop(); } Ok(()) } - pub async fn backup_tree(&mut self, node: Node, chunk: Vec) -> Result<()> { + pub fn backup_tree(&mut self, node: Node, chunk: Vec) -> Result<()> { let dirsize = chunk.len() as u64; let dirsize_bytes = ByteSize(dirsize).to_string_as(true); let id = node.subtree().unwrap(); @@ -203,7 +202,7 @@ impl Archiver { Ok(()) } - pub async fn backup_file(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> { + pub fn backup_file(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> { if let ParentResult::Matched(p_node) = self.parent.is_parent(&node) { if p_node.content().iter().all(|id| self.index.has_data(id)) { let size = *p_node.meta().size(); @@ -220,37 +219,37 @@ impl Archiver { } } let f = File::open(path)?; - self.backup_reader(f, node, p).await + self.backup_reader(f, node, p) } - pub async fn backup_reader(&mut self, r: impl Read, node: Node, p: ProgressBar) -> Result<()> { + pub fn backup_reader( + &mut self, + r: impl Read + 'static, + node: Node, + p: ProgressBar, + ) -> Result<()> { let chunk_iter = ChunkIter::new(r, *node.meta().size() as usize, &self.poly); let mut content = Vec::new(); let mut filesize: u64 = 0; - let mut queue = FuturesOrdered::new(); - - for chunk in chunk_iter { - let chunk = chunk?; - let size = chunk.len() as u64; - filesize += size; - - queue.push_back(spawn(async move { + chunk_iter + .into_iter() + // TODO: This parallelization works pretty well for big files. For small files this produces a lot of + // unneccessary overhead. Maybe use a parallel hashing actor? + .parallel_map(|chunk| { + let chunk = chunk?; let id = hash(&chunk); - (id, chunk, size) - })); + Ok((chunk, id)) + }) + .try_for_each(|data: Result<_>| -> Result<_> { + let (chunk, id) = data?; + let size = chunk.len() as u64; + filesize += size; - if queue.len() > 8 { - let (id, chunk, size) = queue.next().await.unwrap()?; - self.process_data_junk(id, &chunk, size, &p)?; content.push(id); - } - } - - while let Some(Ok((id, chunk, size))) = queue.next().await { - self.process_data_junk(id, &chunk, size, &p)?; - content.push(id); - } + self.process_data_junk(id, &chunk, size, &p)?; + Ok(()) + })?; let mut node = node; node.set_content(content); @@ -281,8 +280,8 @@ impl Archiver { Ok(()) } - pub async fn finalize_snapshot(mut self) -> Result { - self.finish_trees(&PathBuf::from("/")).await?; + pub fn finalize_snapshot(mut self) -> Result { + self.finish_trees(&PathBuf::from("/"))?; let (chunk, id) = self.tree.serialize()?; if !self.index.has_tree(&id) { diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 80922ea..a0b557c 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -201,21 +201,19 @@ pub(super) async fn execute( let snap = if backup_stdin { let mut archiver = Archiver::new(be, index, &config, parent, snap)?; let p = progress_bytes("starting backup from stdin..."); - archiver - .backup_reader( - std::io::stdin(), - Node::new( - backup_path_str, - NodeType::File, - Metadata::default(), - None, - None, - ), - p.clone(), - ) - .await?; + archiver.backup_reader( + std::io::stdin(), + Node::new( + backup_path_str, + NodeType::File, + Metadata::default(), + None, + None, + ), + p.clone(), + )?; - let snap = archiver.finalize_snapshot().await?; + let snap = archiver.finalize_snapshot()?; p.finish_with_message("done"); snap } else { @@ -234,13 +232,13 @@ pub(super) async fn execute( warn!("ignoring error {}\n", e) } Ok((path, node)) => { - if let Err(e) = archiver.add_entry(&path, node, p.clone()).await { + if let Err(e) = archiver.add_entry(&path, node, p.clone()) { warn!("ignoring error {} for {:?}\n", e, path); } } } } - let snap = archiver.finalize_snapshot().await?; + let snap = archiver.finalize_snapshot()?; p.finish_with_message("done"); snap }; From 0bbe5bc7b30a59f4628d90f0202a8577f5dd27a1 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 20 Oct 2022 19:33:57 +0200 Subject: [PATCH 06/10] Make tree streamer sync --- src/blob/tree.rs | 179 ++++++++++++++++-------------------- src/commands/backup.rs | 2 +- src/commands/check.rs | 11 +-- src/commands/diff.rs | 19 ++-- src/commands/ls.rs | 6 +- src/commands/mod.rs | 10 +- src/commands/prune.rs | 22 +++-- src/commands/restore.rs | 10 +- src/commands/self_update.rs | 2 +- 9 files changed, 121 insertions(+), 140 deletions(-) diff --git a/src/blob/tree.rs b/src/blob/tree.rs index e3dfb85..be18131 100644 --- a/src/blob/tree.rs +++ b/src/blob/tree.rs @@ -1,18 +1,12 @@ -use std::collections::{HashSet, VecDeque}; +use std::collections::HashSet; use std::mem; use std::path::{Path, PathBuf}; -use std::pin::Pin; use anyhow::{anyhow, Result}; +use crossbeam_channel::{bounded, unbounded, Receiver, Sender}; use derive_getters::Getters; -use futures::{ - stream::FuturesUnordered, - task::{Context, Poll}, - Future, Stream, -}; use indicatif::ProgressBar; use serde::{Deserialize, Deserializer, Serialize}; -use tokio::{spawn, task::JoinHandle}; use crate::crypto::hash; use crate::id::Id; @@ -91,9 +85,8 @@ impl IntoIterator for Tree { /// NodeStreamer recursively streams all nodes of a given tree including all subtrees in-order pub struct NodeStreamer where - BE: IndexedBackend + Unpin, + BE: IndexedBackend, { - future: Option>>, open_iterators: Vec>, inner: std::vec::IntoIter, path: PathBuf, @@ -102,12 +95,11 @@ where impl NodeStreamer where - BE: IndexedBackend + Unpin, + BE: IndexedBackend, { pub fn new(be: BE, id: Id) -> Result { let inner = Tree::from_backend(&be, id)?.nodes.into_iter(); Ok(Self { - future: None, inner, open_iterators: Vec::new(), path: PathBuf::new(), @@ -118,48 +110,37 @@ where type NodeStreamItem = Result<(PathBuf, Node)>; -// TODO: This is not really parallel at the moment... -impl Stream for NodeStreamer +// TODO: This is not parallel at the moment... +impl Iterator for NodeStreamer where - BE: IndexedBackend + Unpin, + BE: IndexedBackend, { type Item = NodeStreamItem; - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let slf = self.get_mut(); - if let Some(mut future) = slf.future.as_mut() { - match Pin::new(&mut future).poll(cx) { - Poll::Pending => { - return Poll::Pending; - } - Poll::Ready(tree) => { - let old_inner = - mem::replace(&mut slf.inner, tree.unwrap().unwrap().nodes.into_iter()); - slf.open_iterators.push(old_inner); - slf.future = None; - } - } - } - + fn next(&mut self) -> Option { loop { - match slf.inner.next() { + match self.inner.next() { Some(node) => { - let path = slf.path.join(node.name()); + let path = self.path.join(node.name()); if let Some(id) = node.subtree() { - slf.path.push(node.name()); - let be = slf.be.clone(); - let id = *id; - slf.future = Some(spawn(async move { Tree::from_backend(&be, id) })); + self.path.push(node.name()); + let be = self.be.clone(); + let tree = match Tree::from_backend(&be, *id) { + Ok(tree) => tree, + Err(err) => return Some(Err(err)), + }; + let old_inner = mem::replace(&mut self.inner, tree.nodes.into_iter()); + self.open_iterators.push(old_inner); } - return Poll::Ready(Some(Ok((path, node)))); + return Some(Ok((path, node))); } - None => match slf.open_iterators.pop() { + None => match self.open_iterators.pop() { Some(it) => { - slf.inner = it; - slf.path.pop(); + self.inner = it; + self.path.pop(); } - None => return Poll::Ready(None), + None => return None, }, } } @@ -167,98 +148,100 @@ where } /// TreeStreamerOnce recursively visits all trees and subtrees, but each tree ID only once -pub struct TreeStreamerOnce -where - BE: IndexedBackend + Unpin, -{ - futures: FuturesUnordered, usize)>>, +pub struct TreeStreamerOnce { visited: HashSet, - pending: VecDeque<(PathBuf, Id, usize)>, - be: BE, + queue_in: Option>, + queue_out: Receiver>, p: ProgressBar, counter: Vec, + finished_ids: usize, } -impl TreeStreamerOnce -where - BE: IndexedBackend + Unpin, -{ - pub async fn new(be: BE, ids: Vec, p: ProgressBar) -> Result { +const MAX_TREE_LOADER: usize = 4; + +impl TreeStreamerOnce { + pub fn new(be: BE, ids: Vec, p: ProgressBar) -> Result { p.set_length(ids.len() as u64); + + let (out_tx, out_rx) = bounded(MAX_TREE_LOADER); + let (in_tx, in_rx) = unbounded(); + + for _ in 0..MAX_TREE_LOADER { + let be = be.clone(); + let in_rx = in_rx.clone(); + let out_tx = out_tx.clone(); + std::thread::spawn(move || { + for (path, id, count) in in_rx { + out_tx + .send(Tree::from_backend(&be, id).map(|tree| (path, tree, count))) + .unwrap(); + } + }); + } + let counter = vec![0; ids.len()]; let mut streamer = Self { - futures: FuturesUnordered::new(), visited: HashSet::new(), - pending: VecDeque::new(), - be, + queue_in: Some(in_tx), + queue_out: out_rx, p, counter, + finished_ids: 0, }; for (count, id) in ids.into_iter().enumerate() { - if !streamer.add_pending(PathBuf::new(), id, count) { + if !streamer.add_pending(PathBuf::new(), id, count)? { streamer.p.inc(1); + streamer.finished_ids += 1; } } Ok(streamer) } - fn add_pending(&mut self, path: PathBuf, id: Id, count: usize) -> bool { + fn add_pending(&mut self, path: PathBuf, id: Id, count: usize) -> Result { if self.visited.insert(id) { - self.pending.push_back((path, id, count)); + self.queue_in.as_ref().unwrap().send((path, id, count))?; self.counter[count] += 1; - true + Ok(true) } else { - false + Ok(false) } } } type TreeStreamItem = Result<(PathBuf, Tree)>; -const MAX_TREE_LOADER: usize = 20; - -impl Stream for TreeStreamerOnce -where - BE: IndexedBackend + Unpin, -{ +impl Iterator for TreeStreamerOnce { type Item = TreeStreamItem; - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let slf = self.get_mut(); - - // fill futures queue if there is space - while slf.futures.len() < MAX_TREE_LOADER && !slf.pending.is_empty() { - let (path, id, count) = slf.pending.pop_front().unwrap(); - let be = slf.be.clone(); - slf.futures.push(spawn( - async move { (path, Tree::from_backend(&be, id), count) }, - )); + fn next(&mut self) -> Option { + if self.counter.len() == self.finished_ids { + drop(self.queue_in.take()); + self.p.finish(); + return None; } + let (path, tree, count) = match self.queue_out.recv() { + Ok(Ok(res)) => res, + Err(err) => return Some(Err(err.into())), + Ok(Err(err)) => return Some(Err(err)), + }; - match Pin::new(&mut slf.futures).poll_next(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(Some(Ok((path, tree, count)))) => { - let tree = tree.unwrap(); - for node in tree.nodes() { - if let Some(id) = node.subtree() { - let mut path = path.clone(); - path.push(node.name()); - slf.add_pending(path, *id, count); - } + for node in tree.nodes() { + if let Some(id) = node.subtree() { + let mut path = path.clone(); + path.push(node.name()); + match self.add_pending(path, *id, count) { + Ok(_) => {} + Err(err) => return Some(Err(err)), } - slf.counter[count] -= 1; - if slf.counter[count] == 0 { - slf.p.inc(1); - } - Poll::Ready(Some(Ok((path, tree)))) } - Poll::Ready(None) => { - slf.p.finish(); - Poll::Ready(None) - } - Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err.into()))), } + self.counter[count] -= 1; + if self.counter[count] == 0 { + self.p.inc(1); + self.finished_ids += 1; + } + Some(Ok((path, tree))) } } diff --git a/src/commands/backup.rs b/src/commands/backup.rs index a0b557c..6d56e1a 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -91,7 +91,7 @@ pub(super) struct Opts { source: String, } -pub(super) async fn execute( +pub(super) fn execute( be: &impl DecryptFullBackend, opts: Opts, config: ConfigFile, diff --git a/src/commands/check.rs b/src/commands/check.rs index 1856dea..aa3ec9f 100644 --- a/src/commands/check.rs +++ b/src/commands/check.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use anyhow::Result; use bytes::Bytes; use clap::Parser; -use futures::TryStreamExt; use indicatif::ProgressBar; use log::*; use rayon::prelude::*; @@ -31,7 +30,7 @@ pub(super) struct Opts { read_data: bool, } -pub(super) async fn execute( +pub(super) fn execute( be: &(impl DecryptReadBackend + Unpin), cache: &Option, hot_be: &Option, @@ -72,7 +71,7 @@ pub(super) async fn execute( let index_be = IndexBackend::new_from_index(be, index_collector.into_index()); - check_snapshots(&index_be).await?; + check_snapshots(&index_be)?; if opts.read_data { let p = progress_counter("reading pack data..."); @@ -255,7 +254,7 @@ fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap) -> Resul } // check if all snapshots and contained trees can be loaded and contents exist in the index -async fn check_snapshots(index: &(impl IndexedBackend + Unpin)) -> Result<()> { +fn check_snapshots(index: &(impl IndexedBackend + Unpin)) -> Result<()> { let p = progress_counter("reading snapshots..."); let snap_trees: Vec<_> = index .be() @@ -266,8 +265,8 @@ async fn check_snapshots(index: &(impl IndexedBackend + Unpin)) -> Result<()> { p.finish(); let p = progress_counter("checking trees..."); - let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p).await?; - while let Some(item) = tree_streamer.try_next().await? { + let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?; + while let Some(item) = tree_streamer.next().transpose()? { let (path, tree) = item; for node in tree.nodes() { match node.node_type() { diff --git a/src/commands/diff.rs b/src/commands/diff.rs index 612f51c..2be3657 100644 --- a/src/commands/diff.rs +++ b/src/commands/diff.rs @@ -2,7 +2,6 @@ use std::path::Path; use anyhow::Result; use clap::Parser; -use futures::StreamExt; use super::progress_counter; use crate::backend::DecryptReadBackend; @@ -22,7 +21,7 @@ pub(super) struct Opts { snap2: String, } -pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { +pub(super) fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { let (id1, path1) = opts.snap1.split_once(':').unwrap_or((&opts.snap1, "")); let (id2, path2) = opts.snap2.split_once(':').unwrap_or((&opts.snap2, path1)); @@ -40,27 +39,27 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) let mut tree_streamer1 = NodeStreamer::new(index.clone(), id1)?; let mut tree_streamer2 = NodeStreamer::new(index, id2)?; - let mut item1 = tree_streamer1.next().await.transpose()?; - let mut item2 = tree_streamer2.next().await.transpose()?; + let mut item1 = tree_streamer1.next().transpose()?; + let mut item2 = tree_streamer2.next().transpose()?; loop { match (&item1, &item2) { (None, None) => break, (Some(i1), None) => { println!("- {:?}", i1.0); - item1 = tree_streamer1.next().await.transpose()?; + item1 = tree_streamer1.next().transpose()?; } (None, Some(i2)) => { println!("+ {:?}", i2.0); - item2 = tree_streamer2.next().await.transpose()?; + item2 = tree_streamer2.next().transpose()?; } (Some(i1), Some(i2)) if i1.0 < i2.0 => { println!("- {:?}", i1.0); - item1 = tree_streamer1.next().await.transpose()?; + item1 = tree_streamer1.next().transpose()?; } (Some(i1), Some(i2)) if i1.0 > i2.0 => { println!("+ {:?}", i2.0); - item2 = tree_streamer2.next().await.transpose()?; + item2 = tree_streamer2.next().transpose()?; } (Some(i1), Some(i2)) => { let path = &i1.0; @@ -83,8 +82,8 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) } _ => {} // no difference to show } - item1 = tree_streamer1.next().await.transpose()?; - item2 = tree_streamer2.next().await.transpose()?; + item1 = tree_streamer1.next().transpose()?; + item2 = tree_streamer2.next().transpose()?; } } } diff --git a/src/commands/ls.rs b/src/commands/ls.rs index f812eb4..b87d07a 100644 --- a/src/commands/ls.rs +++ b/src/commands/ls.rs @@ -1,6 +1,5 @@ use anyhow::Result; use clap::Parser; -use futures::StreamExt; use std::path::Path; use super::progress_counter; @@ -16,14 +15,13 @@ pub(super) struct Opts { snap: String, } -pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { +pub(super) fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, "")); let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?; let index = IndexBackend::new(be, progress_counter(""))?; let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?; - let mut tree_streamer = NodeStreamer::new(index, tree)?; - while let Some(item) = tree_streamer.next().await { + for item in NodeStreamer::new(index, tree)? { let (path, _) = item?; println!("{:?} ", path); } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index da4d6de..529f5df 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -220,7 +220,7 @@ pub async fn execute() -> Result<()> { } if let Command::SelfUpdate(opts) = args.command { - self_update::execute(opts).await?; + self_update::execute(opts)?; return Ok(()); } @@ -305,17 +305,17 @@ pub async fn execute() -> Result<()> { }; match cmd { - Command::Backup(opts) => backup::execute(&dbe, opts, config, config_file, command).await?, + Command::Backup(opts) => backup::execute(&dbe, opts, config, config_file, command)?, Command::Config(opts) => config::execute(&dbe, &be_hot, opts, config)?, Command::Cat(opts) => cat::execute(&dbe, opts)?, - Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts).await?, + Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts)?, Command::Completions(_) => {} // already handled above - Command::Diff(opts) => diff::execute(&dbe, opts).await?, + Command::Diff(opts) => diff::execute(&dbe, opts)?, Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file).await?, Command::Init(_) => {} // already handled above Command::Key(opts) => key::execute(&dbe, key, opts)?, Command::List(opts) => list::execute(&dbe, opts)?, - Command::Ls(opts) => ls::execute(&dbe, opts).await?, + Command::Ls(opts) => ls::execute(&dbe, opts)?, Command::SelfUpdate(_) => {} // already handled above Command::Snapshots(opts) => snapshots::execute(&dbe, opts, config_file)?, Command::Prune(opts) => prune::execute(&dbe, cache, opts, config, vec![]).await?, diff --git a/src/commands/prune.rs b/src/commands/prune.rs index 407827c..80e6784 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -7,11 +7,10 @@ use bytesize::ByteSize; use chrono::{DateTime, Duration, Local}; use clap::{AppSettings, Parser}; use derive_more::Add; -use futures::TryStreamExt; use log::*; use super::{bytes, no_progress, progress_bytes, progress_counter, wait, warm_up, warm_up_command}; -use crate::backend::{Cache, DecryptFullBackend, DecryptReadBackend, FileType}; +use crate::backend::{Cache, DecryptFullBackend, DecryptReadBackend, FileType, ReadBackend}; use crate::blob::{ BlobType, BlobTypeMap, Initialize, NodeType, PackSizer, Repacker, Sum, TreeStreamerOnce, }; @@ -128,7 +127,7 @@ pub(super) async fn execute( let index = index_collector.into_index(); let total_size = BlobTypeMap::init(|blob_type| index.total_size(&blob_type)); let indexed_be = IndexBackend::new_from_index(&be.clone(), index); - let used_ids = find_used_blobs(&indexed_be, ignore_snaps).await?; + let used_ids = find_used_blobs(&indexed_be, ignore_snaps)?; (used_ids, total_size) }; @@ -1076,20 +1075,23 @@ impl PackInfo { } // find used blobs in repo -async fn find_used_blobs( +fn find_used_blobs( index: &(impl IndexedBackend + Unpin), ignore_snaps: Vec, ) -> Result> { let ignore_snaps: HashSet<_> = ignore_snaps.into_iter().collect(); let p = progress_counter("reading snapshots..."); + let list = index + .be() + .list(FileType::Snapshot)? + .into_iter() + .filter(|id| !ignore_snaps.contains(id)) + .collect(); let snap_trees: Vec<_> = index .be() - .stream_all::(p.clone())? + .stream_list::(list, p.clone())? .into_iter() - // TODO: it would even better to give ignore_snaps to the streaming function instead - // if reading and then filtering the snapshot - .filter(|(id, _)| !ignore_snaps.contains(id)) .map(|(_, snap)| snap.tree) .collect(); p.finish(); @@ -1097,8 +1099,8 @@ async fn find_used_blobs( let mut ids: HashMap<_, _> = snap_trees.iter().map(|id| (*id, 0)).collect(); let p = progress_counter("finding used blobs..."); - let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p).await?; - while let Some(item) = tree_streamer.try_next().await? { + let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?; + while let Some(item) = tree_streamer.next().transpose()? { let (_, tree) = item; for node in tree.nodes() { match node.node_type() { diff --git a/src/commands/restore.rs b/src/commands/restore.rs index 812e513..fff8f77 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -223,7 +223,7 @@ async fn allocate_and_collect( let mut next_dst = dst_iter.next(); let mut node_streamer = NodeStreamer::new(index.clone(), tree)?; - let mut next_node = node_streamer.try_next().await?; + let mut next_node = node_streamer.next().transpose()?; loop { match (&next_dst, &next_node) { @@ -244,16 +244,16 @@ async fn allocate_and_collect( // does not match the type of the node in the snapshot! process_node(path, node, true)?; next_dst = dst_iter.next(); - next_node = node_streamer.try_next().await?; + next_node = node_streamer.next().transpose()?; } Ordering::Greater => { process_node(path, node, false)?; - next_node = node_streamer.try_next().await?; + next_node = node_streamer.next().transpose()?; } }, (None, Some((path, node))) => { process_node(path, node, false)?; - next_node = node_streamer.try_next().await?; + next_node = node_streamer.next().transpose()?; } } } @@ -352,7 +352,7 @@ async fn restore_metadata( // walk over tree in repository and compare with tree in dest let mut node_streamer = NodeStreamer::new(index, tree)?; let mut dir_stack = Vec::new(); - while let Some((path, node)) = node_streamer.try_next().await? { + while let Some((path, node)) = node_streamer.next().transpose()? { match node.node_type() { NodeType::Dir => { // set metadata for all non-parent paths in stack diff --git a/src/commands/self_update.rs b/src/commands/self_update.rs index 3a9c150..0697309 100644 --- a/src/commands/self_update.rs +++ b/src/commands/self_update.rs @@ -9,7 +9,7 @@ pub(super) struct Opts { force: bool, } -pub(super) async fn execute(opts: Opts) -> Result<()> { +pub(super) fn execute(opts: Opts) -> Result<()> { let status = self_update::backends::github::Update::configure() .repo_owner("rustic-rs") .repo_name("rustic") From 181ab5da2fec923dcbc85c8cb075b413e0ea41f2 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 20 Oct 2022 19:42:13 +0200 Subject: [PATCH 07/10] index: Parallelize sorting the index --- src/index/binarysorted.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/index/binarysorted.rs b/src/index/binarysorted.rs index 7ce978a..6b93596 100644 --- a/src/index/binarysorted.rs +++ b/src/index/binarysorted.rs @@ -4,6 +4,7 @@ use super::{BlobType, IndexEntry, ReadIndex}; use crate::blob::BlobTypeMap; use crate::id::Id; use crate::repo::{IndexBlob, IndexPack}; +use rayon::prelude::*; #[derive(Debug, PartialEq, Eq)] struct SortedEntry { @@ -82,8 +83,8 @@ impl IndexCollector { Index(self.0.map(|_, mut tc| { match &mut tc.entries { EntriesVariants::None => {} - EntriesVariants::Ids(ids) => ids.sort_unstable(), - EntriesVariants::FullEntries(entries) => entries.sort_unstable_by_key(|e| e.id), + EntriesVariants::Ids(ids) => ids.par_sort_unstable(), + EntriesVariants::FullEntries(entries) => entries.par_sort_unstable_by_key(|e| e.id), }; let packs = tc.packs.into_iter().map(|(id, _)| id).collect(); @@ -182,13 +183,13 @@ impl IntoIterator for Index { fn into_iter(mut self) -> Self::IntoIter { for (_, tc) in self.0.iter_mut() { if let EntriesVariants::FullEntries(entries) = &mut tc.entries { - entries.sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx)) + entries.par_sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx)) } } PackIndexes { c: Index(self.0.map(|_, mut tc| { if let EntriesVariants::FullEntries(entries) = &mut tc.entries { - entries.sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx)) + entries.par_sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx)) } TypeIndex { From de0bd189a8ab4267f3e75cb09fe2932645113d75 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 23 Oct 2022 08:14:20 +0200 Subject: [PATCH 08/10] Make restore sync --- src/commands/restore.rs | 109 +++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 56 deletions(-) diff --git a/src/commands/restore.rs b/src/commands/restore.rs index fff8f77..432918c 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -7,10 +7,9 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, Result}; use clap::{AppSettings, Parser}; use derive_getters::Dissolve; -use futures::{stream::FuturesUnordered, TryStreamExt}; use ignore::{DirEntry, WalkBuilder}; use log::*; -use tokio::spawn; +use rayon::ThreadPoolBuilder; use super::{bytes, progress_bytes, progress_counter, wait, warm_up, warm_up_command}; use crate::backend::{DecryptReadBackend, FileType, LocalBackend}; @@ -75,7 +74,7 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) let dest = LocalBackend::new(&opts.dest); let p = progress_spinner("collecting file information..."); - let file_infos = allocate_and_collect(&dest, index.clone(), tree, &opts).await?; + let file_infos = allocate_and_collect(&dest, index.clone(), tree, &opts)?; p.finish(); info!("total restore size: {}", bytes(file_infos.total_size)); if file_infos.matched_size > 0 { @@ -98,13 +97,13 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) } wait(opts.warm_up_wait).await; if !opts.dry_run { - restore_contents(be, &dest, file_infos).await?; + restore_contents(be, &dest, file_infos)?; } } if !opts.dry_run { let p = progress_spinner("setting metadata..."); - restore_metadata(&dest, index, tree, &opts).await?; + restore_metadata(&dest, index, tree, &opts)?; p.finish(); } @@ -113,7 +112,7 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) } /// collect restore information, scan existing files and allocate non-existing files -async fn allocate_and_collect( +fn allocate_and_collect( dest: &LocalBackend, index: impl IndexedBackend + Unpin, tree: Id, @@ -272,7 +271,7 @@ async fn allocate_and_collect( /// restore_contents restores all files contents as described by file_infos /// using the ReadBackend be and writing them into the LocalBackend dest. -async fn restore_contents( +fn restore_contents( be: &impl DecryptReadBackend, dest: &LocalBackend, file_infos: FileInfos, @@ -281,69 +280,67 @@ async fn restore_contents( let p = progress_bytes("restoring file contents..."); p.set_length(total_size - matched_size); - let mut stream = FuturesUnordered::new(); const MAX_READER: usize = 20; - for (pack, blob) in restore_info { - for (bl, fls) in blob { - let p = p.clone(); - let be = be.clone(); - let dest = dest.clone(); + let pool = ThreadPoolBuilder::new().num_threads(MAX_READER).build()?; + pool.in_place_scope(|s| { + for (pack, blob) in restore_info { + for (bl, fls) in blob { + let from_file = fls + .iter() + .find(|fl| fl.matches) + .map(|fl| (filenames[fl.file_idx].clone(), fl.file_start)); - let from_file = fls - .iter() - .find(|fl| fl.matches) - .map(|fl| (filenames[fl.file_idx].clone(), fl.file_start)); + let name_dests: Vec<_> = fls + .iter() + .filter(|fl| !fl.matches) + .map(|fl| (filenames[fl.file_idx].clone(), fl.file_start)) + .collect(); + let p = &p; - let name_dests: Vec<_> = fls - .iter() - .filter(|fl| !fl.matches) - .map(|fl| (filenames[fl.file_idx].clone(), fl.file_start)) - .collect(); + if !name_dests.is_empty() { + // TODO: error handling! + s.spawn(move |s1| { + let data = match from_file { + Some((filename, start)) => { + // read from existing file + dest.read_at(filename, start, bl.data_length()).unwrap() + } + None => { + // read pack at blob_offset with length blob_length + be.read_encrypted_partial( + FileType::Pack, + &pack, + false, + bl.offset, + bl.length, + bl.uncompressed_length, + ) + .unwrap() + } + }; + let size = bl.data_length(); - if !name_dests.is_empty() { - while stream.len() > MAX_READER { - stream.try_next().await?; + // save into needed files in parallel + for (name, start) in name_dests { + let data = data.clone(); + s1.spawn(move |_| { + dest.write_at(&name, start, &data).unwrap(); + p.inc(size); + }); + } + }) } - - // TODO: error handling! - stream.push(spawn(async move { - let data = match from_file { - Some((filename, start)) => { - // read from existing file - dest.read_at(filename, start, bl.data_length()).unwrap() - } - None => { - // read pack at blob_offset with length blob_length - be.read_encrypted_partial( - FileType::Pack, - &pack, - false, - bl.offset, - bl.length, - bl.uncompressed_length, - ) - .unwrap() - } - }; - - // save into needed files in parallel - for (name, start) in name_dests { - dest.write_at(&name, start, &data).unwrap(); - p.inc(bl.data_length()); - } - })) } } - } + }); - stream.try_collect().await?; p.finish(); Ok(()) } -async fn restore_metadata( +fn restore_metadata( dest: &LocalBackend, index: impl IndexedBackend + Unpin, tree: Id, From 3b6f608e979c417776757dab5c183c5ad45821db Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 23 Oct 2022 07:52:53 +0200 Subject: [PATCH 09/10] Make warmup sync --- src/commands/forget.rs | 4 ++-- src/commands/helpers.rs | 35 +++++++++++++++-------------------- src/commands/mod.rs | 8 ++++---- src/commands/prune.rs | 6 +++--- src/commands/repair.rs | 10 +++++----- src/commands/restore.rs | 6 +++--- 6 files changed, 32 insertions(+), 37 deletions(-) diff --git a/src/commands/forget.rs b/src/commands/forget.rs index 5201f64..186ac71 100644 --- a/src/commands/forget.rs +++ b/src/commands/forget.rs @@ -55,7 +55,7 @@ struct ConfigOpts { keep: KeepOptions, } -pub(super) async fn execute( +pub(super) fn execute( be: &(impl DecryptFullBackend + Unpin), cache: Option, mut opts: Opts, @@ -151,7 +151,7 @@ pub(super) async fn execute( } if opts.prune { - prune::execute(be, cache, opts.prune_opts, config, forget_snaps).await?; + prune::execute(be, cache, opts.prune_opts, config, forget_snaps)?; } Ok(()) diff --git a/src/commands/helpers.rs b/src/commands/helpers.rs index cad1d58..56eec31 100644 --- a/src/commands/helpers.rs +++ b/src/commands/helpers.rs @@ -5,13 +5,11 @@ use std::time::Duration; use anyhow::{bail, Result}; use bytesize::ByteSize; -use futures::{stream::FuturesUnordered, TryStreamExt}; use indicatif::HumanDuration; use indicatif::{ProgressBar, ProgressState, ProgressStyle}; use log::*; +use rayon::ThreadPoolBuilder; use rpassword::prompt_password; -use tokio::spawn; -use tokio::time::sleep; use crate::backend::{DecryptReadBackend, FileType, ReadBackend}; use crate::crypto::Key; @@ -103,7 +101,7 @@ pub fn warm_up_command(packs: impl ExactSizeIterator, command: &str) Ok(()) } -pub async fn warm_up( +pub fn warm_up( be: &impl DecryptReadBackend, packs: impl ExactSizeIterator, ) -> Result<()> { @@ -112,33 +110,30 @@ pub async fn warm_up( let p = progress_counter("warming up packs..."); p.set_length(packs.len() as u64); - let mut stream = FuturesUnordered::new(); const MAX_READER: usize = 20; - for pack in packs { - while stream.len() > MAX_READER { - stream.try_next().await?; + let pool = ThreadPoolBuilder::new().num_threads(MAX_READER).build()?; + let p = &p; + let be = &be; + pool.in_place_scope(|s| { + for pack in packs { + s.spawn(move |_| { + // ignore errors as they are expected from the warm-up + _ = be.read_partial(FileType::Pack, &pack, false, 0, 1); + p.inc(1); + }); } + }); - let p = p.clone(); - let be = be.clone(); - stream.push(spawn(async move { - // ignore errors as they are expected from the warm-up - _ = be.read_partial(FileType::Pack, &pack, false, 0, 1); - p.inc(1); - })) - } - - stream.try_collect().await?; p.finish(); Ok(()) } -pub async fn wait(d: Option) { +pub fn wait(d: Option) { if let Some(wait) = d { let p = progress_spinner(format!("waiting {}...", wait)); - sleep(*wait).await; + std::thread::sleep(*wait); p.finish(); } } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 529f5df..5741208 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -311,16 +311,16 @@ pub async fn execute() -> Result<()> { Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts)?, Command::Completions(_) => {} // already handled above Command::Diff(opts) => diff::execute(&dbe, opts)?, - Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file).await?, + Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file)?, Command::Init(_) => {} // already handled above Command::Key(opts) => key::execute(&dbe, key, opts)?, Command::List(opts) => list::execute(&dbe, opts)?, Command::Ls(opts) => ls::execute(&dbe, opts)?, Command::SelfUpdate(_) => {} // already handled above Command::Snapshots(opts) => snapshots::execute(&dbe, opts, config_file)?, - Command::Prune(opts) => prune::execute(&dbe, cache, opts, config, vec![]).await?, - Command::Restore(opts) => restore::execute(&dbe, opts).await?, - Command::Repair(opts) => repair::execute(&dbe, opts, config_file, &config).await?, + Command::Prune(opts) => prune::execute(&dbe, cache, opts, config, vec![])?, + Command::Restore(opts) => restore::execute(&dbe, opts)?, + Command::Repair(opts) => repair::execute(&dbe, opts, config_file, &config)?, Command::Repoinfo(opts) => repoinfo::execute(&dbe, &be_hot, opts)?, Command::Tag(opts) => tag::execute(&dbe, opts, config_file)?, }; diff --git a/src/commands/prune.rs b/src/commands/prune.rs index 80e6784..0d8e7a3 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -83,7 +83,7 @@ pub(super) struct Opts { warm_up_wait: Option, } -pub(super) async fn execute( +pub(super) fn execute( be: &(impl DecryptFullBackend + Unpin), cache: Option, opts: Opts, @@ -162,14 +162,14 @@ pub(super) async fn execute( pruner.print_stats(); if opts.warm_up { - warm_up(be, pruner.repack_packs().into_iter()).await?; + warm_up(be, pruner.repack_packs().into_iter())?; } else if opts.warm_up_command.is_some() { warm_up_command( pruner.repack_packs().into_iter(), opts.warm_up_command.as_ref().unwrap(), )?; } - wait(opts.warm_up_wait).await; + wait(opts.warm_up_wait); if !opts.dry_run { pruner.do_prune(be, opts, config)?; diff --git a/src/commands/repair.rs b/src/commands/repair.rs index 586d43d..0f51748 100644 --- a/src/commands/repair.rs +++ b/src/commands/repair.rs @@ -80,19 +80,19 @@ struct SnapOpts { ids: Vec, } -pub(super) async fn execute( +pub(super) fn execute( be: &impl DecryptFullBackend, opts: Opts, config_file: RusticConfig, config: &ConfigFile, ) -> Result<()> { match opts.command { - Command::Index(opt) => repair_index(be, opt).await, + Command::Index(opt) => repair_index(be, opt), Command::Snapshots(opt) => repair_snaps(be, opt, config_file, config), } } -async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> { +fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> { let p = progress_spinner("listing packs..."); let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect(); p.finish(); @@ -166,7 +166,7 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<( pack_read_header.extend(packs.into_iter().map(|(id, size)| (id, false, None, size))); if opts.warm_up { - warm_up(be, pack_read_header.iter().map(|(id, _, _, _)| *id)).await?; + warm_up(be, pack_read_header.iter().map(|(id, _, _, _)| *id))?; if opts.dry_run { return Ok(()); } @@ -179,7 +179,7 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<( return Ok(()); } } - wait(opts.warm_up_wait).await; + wait(opts.warm_up_wait); let indexer = Indexer::new(be.clone()).into_shared(); let p = progress_counter("reading pack headers"); diff --git a/src/commands/restore.rs b/src/commands/restore.rs index 432918c..df88a8a 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -57,7 +57,7 @@ pub(super) struct Opts { dest: String, } -pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { +pub(super) fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> { if let Some(command) = &opts.warm_up_command { if !command.contains("%id") { bail!("warm-up command must contain %id!") @@ -88,14 +88,14 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) info!("all file contents are fine."); } else { if opts.warm_up { - warm_up(be, file_infos.to_packs().into_iter()).await?; + warm_up(be, file_infos.to_packs().into_iter())?; } else if opts.warm_up_command.is_some() { warm_up_command( file_infos.to_packs().into_iter(), opts.warm_up_command.as_ref().unwrap(), )?; } - wait(opts.warm_up_wait).await; + wait(opts.warm_up_wait); if !opts.dry_run { restore_contents(be, &dest, file_infos)?; } From 7e457a082f4dd1b8af3bd7bed5296f888d69ff9f Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 23 Oct 2022 07:58:50 +0200 Subject: [PATCH 10/10] Completely remove async --- Cargo.lock | 276 +++++++++++++++++++++----------------------- Cargo.toml | 5 - src/commands/mod.rs | 2 +- src/main.rs | 5 +- 4 files changed, 136 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4aac945..39da45b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,28 +74,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfb6d71005dc22a708c7496eee5c8dc0300ee47355de6256c3b35b12b5fef596" -[[package]] -name = "async-recursion" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "async-trait" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "atty" version = "0.2.14" @@ -180,9 +158,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "bytemuck" @@ -313,14 +291,24 @@ dependencies = [ ] [[package]] -name = "console" -version = "0.15.1" +name = "codespan-reporting" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" dependencies = [ "encode_unicode 0.3.6", + "lazy_static", "libc", - "once_cell", "terminal_size", "unicode-width", "winapi", @@ -416,12 +404,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -443,6 +430,50 @@ dependencies = [ "cipher 0.3.0", ] +[[package]] +name = "cxx" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.14.1" @@ -679,9 +710,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", @@ -694,9 +725,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", @@ -704,15 +735,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-executor" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ "futures-core", "futures-task", @@ -721,15 +752,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-macro" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", @@ -738,15 +769,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-timer" @@ -756,9 +787,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -794,9 +825,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", @@ -953,18 +984,28 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.48" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0" +checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" dependencies = [ "android_system_properties", "core-foundation-sys", + "iana-time-zone-haiku", "js-sys", - "once_cell", "wasm-bindgen", "winapi", ] +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1065,15 +1106,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jobserver" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" dependencies = [ "libc", ] @@ -1095,18 +1136,17 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.132" +version = "0.2.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" [[package]] -name = "lock_api" -version = "0.4.8" +name = "link-cplusplus" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" dependencies = [ - "autocfg", - "scopeguard", + "cc", ] [[package]] @@ -1242,9 +1282,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "opaque-debug" @@ -1275,29 +1315,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys 0.36.1", -] - [[package]] name = "path-absolutize" version = "3.0.14" @@ -1399,9 +1416,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.43" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -1469,9 +1486,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] @@ -1651,8 +1668,6 @@ version = "0.3.2-dev" dependencies = [ "aes256ctr_poly1305aes", "anyhow", - "async-recursion", - "async-trait", "backoff", "base64", "binrw", @@ -1672,7 +1687,6 @@ dependencies = [ "enum-map", "enum-map-derive", "filetime", - "futures", "gethostname", "hex", "humantime", @@ -1704,7 +1718,6 @@ dependencies = [ "sha2", "simplelog", "thiserror", - "tokio", "toml", "users", "walkdir", @@ -1713,9 +1726,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.6" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ "log", "ring", @@ -1768,6 +1781,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + [[package]] name = "scrypt" version = "0.10.0" @@ -1909,15 +1928,6 @@ dependencies = [ "digest", ] -[[package]] -name = "signal-hook-registry" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" -dependencies = [ - "libc", -] - [[package]] name = "simplelog" version = "0.12.0" @@ -1938,12 +1948,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" - [[package]] name = "socket2" version = "0.4.7" @@ -1974,9 +1978,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.99" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ "proc-macro2", "quote", @@ -2075,9 +2079,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" +checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c" dependencies = [ "itoa", "libc", @@ -2119,25 +2123,11 @@ dependencies = [ "memchr", "mio", "num_cpus", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", - "tokio-macros", "winapi", ] -[[package]] -name = "tokio-macros" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-rustls" version = "0.23.4" @@ -2180,9 +2170,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", @@ -2191,9 +2181,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", ] @@ -2218,15 +2208,15 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] @@ -2395,9 +2385,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" +checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" dependencies = [ "webpki", ] diff --git a/Cargo.toml b/Cargo.toml index f4d509d..5567905 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,8 +23,6 @@ strip = true [dependencies] # macros -async-trait = "0.1" -async-recursion = "1" anyhow = "1" thiserror = "1" derive_more = "0.99" @@ -36,9 +34,6 @@ log = "0.4" crossbeam-channel = "0.5" rayon = "1" pariter = "0.5" -# async -tokio = { version = "1", features = ["full"] } -futures = "0.3" #crypto aes256ctr_poly1305aes = "0.1" sha2 = "0.10" diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 5741208..11ab908 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -181,7 +181,7 @@ enum Command { Tag(tag::Opts), } -pub async fn execute() -> Result<()> { +pub fn execute() -> Result<()> { let command: Vec<_> = std::env::args_os().into_iter().collect(); let args = Opts::parse_from(&command); diff --git a/src/main.rs b/src/main.rs index c990639..729d28d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,7 +40,6 @@ mod id; mod index; mod repo; -#[tokio::main] -async fn main() -> Result<()> { - commands::execute().await +fn main() -> Result<()> { + commands::execute() }