Rework backend

src/archiver/archiver_impl.rs

@@ -6,7 +6,7 @@ use chrono::Local;
 use indicatif::ProgressBar;
 use log::*;
 
-use crate::backend::DecryptWriteBackend;
+use crate::backend::{DecryptWriteBackend, ReadSource, ReadSourceEntry};
 use crate::blob::{BlobType, Node};
 use crate::index::{IndexedBackend, Indexer, SharedIndexer};
 use crate::repofile::{ConfigFile, SnapshotFile};
@@ -48,35 +48,35 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
 
     pub fn archive(
         mut self,
-        src: impl Iterator<Item = Result<(PathBuf, Node)>>,
+        src: impl ReadSource,
         backup_path: &Path,
         as_path: Option<&PathBuf>,
         p: &ProgressBar,
     ) -> Result<SnapshotFile> {
         // filter out errors and handle as_path
-        let iter = src.filter_map(|item| match item {
+        let iter = src.entries().filter_map(|item| match item {
             Err(e) => {
                 warn!("ignoring error {}\n", e);
                 None
             }
-            Ok((path, node)) => {
+            Ok(ReadSourceEntry { path, node, open }) => {
                 let snapshot_path = if let Some(as_path) = as_path {
                     as_path
                         .clone()
                         .join(path.strip_prefix(backup_path).unwrap())
                 } else {
-                    path.clone()
+                    path
                 };
                 Some(if node.is_dir() {
-                    (snapshot_path, path, node)
+                    (snapshot_path, node, open)
                 } else {
                     (
                         snapshot_path
                             .parent()
                             .expect("file path should have a parent!")
                             .to_path_buf(),
-                        path,
                         node,
+                        open,
                     )
                 })
             }

src/archiver/file_archiver.rs

@@ -1,11 +1,10 @@
-use std::fs::File;
 use std::io::Read;
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use indicatif::ProgressBar;
 use rayon::prelude::*;
 
-use crate::backend::DecryptWriteBackend;
+use crate::backend::{DecryptWriteBackend, ReadSourceOpen};
 use crate::blob::{BlobType, Node, NodeType, Packer, PackerStats};
 use crate::chunker::ChunkIter;
 use crate::crypto::hash;
@@ -39,18 +38,22 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> FileArchiver<BE, I> {
         })
     }
 
-    pub fn process(&self, item: ItemWithParent, p: ProgressBar) -> Result<TreeItem> {
+    pub fn process<O: ReadSourceOpen>(
+        &self,
+        item: ItemWithParent<Option<O>>,
+        p: ProgressBar,
+    ) -> Result<TreeItem> {
         Ok(match item {
             TreeType::NewTree(item) => TreeType::NewTree(item),
             TreeType::EndTree => TreeType::EndTree,
-            TreeType::Other((path, real_path, node, parent)) => {
+            TreeType::Other((path, node, open, parent)) => {
                 let (node, filesize) = if let ParentResult::Matched(()) = parent {
                     let size = node.meta.size;
                     p.inc(size);
                     (node, size)
                 } else if let NodeType::File = node.node_type() {
-                    let r = File::open(real_path)?;
-                    self.backup_reader(r, node, p.clone())?
+                    let r = open.ok_or(anyhow!("cannot open file"))?.open()?;
+                    self.backup_reader(r, node, p)?
                 } else {
                     (node, 0)
                 };

src/archiver/parent.rs

@@ -47,8 +47,8 @@ impl ParentResult<&Node> {
     }
 }
 
-pub type ItemWithParent =
-    TreeType<(PathBuf, PathBuf, Node, ParentResult<()>), (PathBuf, Node, ParentResult<Id>)>;
+pub type ItemWithParent<O> =
+    TreeType<(PathBuf, Node, O, ParentResult<()>), (PathBuf, Node, ParentResult<Id>)>;
 
 impl<BE: IndexedBackend> Parent<BE> {
     pub fn new(be: &BE, tree_id: Option<Id>, ignore_ctime: bool, ignore_inode: bool) -> Self {
@@ -155,10 +155,10 @@ impl<BE: IndexedBackend> Parent<BE> {
         Ok(())
     }
 
-    pub fn process(
+    pub fn process<O>(
         &mut self,
-        item: TreeType<(PathBuf, PathBuf, Node), (PathBuf, Node, OsString)>,
-    ) -> Result<ItemWithParent> {
+        item: TreeType<(PathBuf, Node, O), (PathBuf, Node, OsString)>,
+    ) -> Result<ItemWithParent<O>> {
         let result = match item {
             TreeType::NewTree((path, node, tree)) => {
                 let parent_result = self.is_parent(&node, &tree).into_tree_id();
@@ -169,7 +169,7 @@ impl<BE: IndexedBackend> Parent<BE> {
                 self.finish_dir()?;
                 TreeType::EndTree
             }
-            TreeType::Other((path, real_path, mut node)) => TreeType::Other({
+            TreeType::Other((path, mut node, open)) => TreeType::Other({
                 let be = self.be.clone();
                 let parent = self.is_parent(&node, &node.name());
                 let parent = match parent {
@@ -186,7 +186,7 @@ impl<BE: IndexedBackend> Parent<BE> {
                     }
                     parent_result => parent_result.into_type_only(),
                 };
-                (path, real_path, node, parent)
+                (path, node, open, parent)
             }),
         };
         Ok(result)

src/archiver/tree.rs

@@ -30,11 +30,11 @@ pub enum TreeType<T, U> {
     Other(T),
 }
 
-impl<I> Iterator for TreeIterator<(PathBuf, PathBuf, Node), I>
+impl<I, O> Iterator for TreeIterator<(PathBuf, Node, O), I>
 where
-    I: Iterator<Item = (PathBuf, PathBuf, Node)>,
+    I: Iterator<Item = (PathBuf, Node, O)>,
 {
-    type Item = TreeType<(PathBuf, PathBuf, Node), (PathBuf, Node, OsString)>;
+    type Item = TreeType<(PathBuf, Node, O), (PathBuf, Node, OsString)>;
     fn next(&mut self) -> Option<Self::Item> {
         match &self.item {
             None => {
@@ -44,7 +44,7 @@ where
                     None
                 }
             }
-            Some((path, _, node)) => {
+            Some((path, node, _)) => {
                 match path.strip_prefix(&self.path) {
                     Err(_) => {
                         self.path.pop();
@@ -56,7 +56,7 @@ where
                             // process next normal path component - other components are simply ignored
                             if let Component::Normal(p) = comp {
                                 if node.is_dir() && path == &self.path {
-                                    let (path, _, node) = self.item.take().unwrap();
+                                    let (path, node, _) = self.item.take().unwrap();
                                     self.item = self.iter.next();
                                     let name = node.name();
                                     return Some(TreeType::NewTree((path, node, name)));

src/archiver/tree_archiver.rs

@@ -46,6 +46,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> TreeArchiver<BE, I> {
             summary,
         })
     }
+
     pub fn add(&mut self, item: TreeItem) -> Result<()> {
         match item {
             TreeType::NewTree((path, node, parent)) => {
@@ -97,10 +98,6 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> TreeArchiver<BE, I> {
         self.tree.add(node);
     }
 
-    pub fn finish_tree(&mut self) -> Result<()> {
-        Ok(())
-    }
-
     pub fn backup_tree(&mut self, path: &Path, parent: ParentResult<Id>) -> Result<Id> {
         let (chunk, id) = self.tree.serialize()?;
         let dirsize = chunk.len() as u64;

src/backend/ignore.rs

@@ -3,7 +3,7 @@ use std::fs::{read_link, File};
 use std::os::unix::fs::{FileTypeExt, MetadataExt};
 use std::path::{Path, PathBuf};
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use bytesize::ByteSize;
 #[cfg(not(windows))]
 use chrono::TimeZone;
@@ -18,6 +18,7 @@ use serde_with::{serde_as, DisplayFromStr};
 use users::{Groups, Users, UsersCache};
 
 use super::{node::Metadata, node::NodeType, Node, ReadSource};
+use super::{ReadSourceEntry, ReadSourceOpen};
 
 pub struct LocalSource {
     builder: WalkBuilder,
@@ -152,12 +153,22 @@ impl LocalSource {
     }
 }
 
-impl ReadSource for LocalSource {
+pub struct OpenFile(PathBuf);
+
+impl ReadSourceOpen for OpenFile {
     type Reader = File;
-    fn read(path: &Path) -> Result<Self::Reader> {
-        Ok(File::open(path)?)
+
+    fn open(self) -> Result<Self::Reader> {
+        let path = self.0;
+        File::open(&path).with_context(|| format!("Unable to open {}", path.display()))
     }
-    fn size(&self) -> Result<u64> {
+}
+
+impl ReadSource for LocalSource {
+    type Open = OpenFile;
+    type Iter = Self;
+
+    fn size(&self) -> Result<Option<u64>> {
         let mut size = 0;
         for entry in self.builder.build() {
             if let Err(e) = entry.and_then(|e| e.metadata()).map(|m| {
@@ -166,12 +177,16 @@ impl ReadSource for LocalSource {
                 warn!("ignoring error {}", e);
             }
         }
-        Ok(size)
+        Ok(Some(size))
+    }
+
+    fn entries(self) -> Self::Iter {
+        self
     }
 }
 
 impl Iterator for LocalSource {
-    type Item = Result<(PathBuf, Node)>;
+    type Item = Result<ReadSourceEntry<OpenFile>>;
 
     fn next(&mut self) -> Option<Self::Item> {
         match self.walker.next() {
@@ -194,7 +209,11 @@ impl Iterator for LocalSource {
 }
 
 #[cfg(windows)]
-fn map_entry(entry: DirEntry, with_atime: bool, _ignore_devid: bool) -> Result<(PathBuf, Node)> {
+fn map_entry(
+    entry: DirEntry,
+    with_atime: bool,
+    _ignore_devid: bool,
+) -> Result<ReadSourceEntry<OpenFile>> {
     let name = entry.file_name();
     let m = entry.metadata()?;
 
@@ -253,7 +272,10 @@ fn map_entry(entry: DirEntry, with_atime: bool, _ignore_devid: bool) -> Result<(
     } else {
         Node::new_node(name, NodeType::File, meta)
     };
-    Ok((entry.path().to_path_buf(), node))
+
+    let path = entry.into_path();
+    let open = Some(OpenFile(path.clone()));
+    Ok(ReadSourceEntry { path, node, open })
 }
 
 #[cfg(not(windows))]
@@ -263,7 +285,7 @@ fn map_entry(
     with_atime: bool,
     ignore_devid: bool,
     cache: &UsersCache,
-) -> Result<(PathBuf, Node)> {
+) -> Result<ReadSourceEntry<OpenFile>> {
     let name = entry.file_name();
     let m = entry.metadata()?;
 
@@ -337,7 +359,9 @@ fn map_entry(
     } else {
         Node::new_node(name, NodeType::File, meta)
     };
-    Ok((entry.path().to_path_buf(), node))
+    let path = entry.into_path();
+    let open = Some(OpenFile(path.clone()));
+    Ok(ReadSourceEntry { path, node, open })
 }
 
 #[cfg(not(windows))]

src/backend/mod.rs

@@ -1,5 +1,5 @@
 use std::io::Read;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 use anyhow::{anyhow, Result};
 use bytes::Bytes;
@@ -144,10 +144,24 @@ pub trait WriteBackend: ReadBackend {
     fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>;
 }
 
-pub trait ReadSource: Iterator<Item = Result<(PathBuf, Node)>> {
-    type Reader: Read;
-    fn read(path: &Path) -> Result<Self::Reader>;
-    fn size(&self) -> Result<u64>;
+pub struct ReadSourceEntry<O> {
+    pub path: PathBuf,
+    pub node: Node,
+    pub open: Option<O>,
+}
+
+pub trait ReadSourceOpen {
+    type Reader: Read + Send + 'static;
+
+    fn open(self) -> Result<Self::Reader>;
+}
+
+pub trait ReadSource {
+    type Open: ReadSourceOpen;
+    type Iter: Iterator<Item = Result<ReadSourceEntry<Self::Open>>>;
+
+    fn size(&self) -> Result<Option<u64>>;
+    fn entries(self) -> Self::Iter;
 }
 
 pub trait WriteSource: Clone {

src/commands/backup.rs

@@ -234,8 +234,9 @@ pub(super) fn execute(
 
             let p = progress_bytes("determining size...");
             if !p.is_hidden() {
-                let size = src.size()?;
-                p.set_length(size);
+                if let Some(size) = src.size()? {
+                    p.set_length(size);
+                }
             };
             p.set_prefix("backing up...");
             let archiver = Archiver::new(be, index.clone(), &repo.config, parent, snap)?;

src/commands/diff.rs

@@ -5,7 +5,7 @@ use anyhow::{anyhow, bail, Context, Result};
 use clap::Parser;
 
 use super::{progress_counter, RusticConfig};
-use crate::backend::{LocalDestination, LocalSource, LocalSourceOptions};
+use crate::backend::{LocalDestination, LocalSource, LocalSourceOptions, ReadSourceEntry};
 use crate::blob::{Node, NodeStreamer, NodeType, Tree};
 use crate::commands::helpers::progress_spinner;
 use crate::crypto::hash;
@@ -86,7 +86,7 @@ pub(super) fn execute(
                 .with_context(|| format!("Error accessing {path2:?}"))?
                 .is_dir();
             let src = LocalSource::new(opts.ignore_opts, &[&path2])?.map(|item| {
-                let (path, node) = item?;
+                let ReadSourceEntry { path, node, .. } = item?;
                 let path = if is_dir {
                     // remove given path prefix for dirs as local path
                     path.strip_prefix(&path2)?.to_path_buf()