From 8ca3a99ab24cd66eba24724ec605b85e504eade1 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Wed, 15 Feb 2023 16:43:06 +0100 Subject: [PATCH] Rework backend --- src/archiver/archiver_impl.rs | 14 +++++------ src/archiver/file_archiver.rs | 17 +++++++------ src/archiver/parent.rs | 14 +++++------ src/archiver/tree.rs | 10 ++++---- src/archiver/tree_archiver.rs | 5 +--- src/backend/ignore.rs | 46 ++++++++++++++++++++++++++--------- src/backend/mod.rs | 24 ++++++++++++++---- src/commands/backup.rs | 5 ++-- src/commands/diff.rs | 4 +-- 9 files changed, 89 insertions(+), 50 deletions(-) diff --git a/src/archiver/archiver_impl.rs b/src/archiver/archiver_impl.rs index 01a0f3e..258713c 100644 --- a/src/archiver/archiver_impl.rs +++ b/src/archiver/archiver_impl.rs @@ -6,7 +6,7 @@ use chrono::Local; use indicatif::ProgressBar; use log::*; -use crate::backend::DecryptWriteBackend; +use crate::backend::{DecryptWriteBackend, ReadSource, ReadSourceEntry}; use crate::blob::{BlobType, Node}; use crate::index::{IndexedBackend, Indexer, SharedIndexer}; use crate::repofile::{ConfigFile, SnapshotFile}; @@ -48,35 +48,35 @@ impl Archiver { pub fn archive( mut self, - src: impl Iterator>, + src: impl ReadSource, backup_path: &Path, as_path: Option<&PathBuf>, p: &ProgressBar, ) -> Result { // filter out errors and handle as_path - let iter = src.filter_map(|item| match item { + let iter = src.entries().filter_map(|item| match item { Err(e) => { warn!("ignoring error {}\n", e); None } - Ok((path, node)) => { + Ok(ReadSourceEntry { path, node, open }) => { let snapshot_path = if let Some(as_path) = as_path { as_path .clone() .join(path.strip_prefix(backup_path).unwrap()) } else { - path.clone() + path }; Some(if node.is_dir() { - (snapshot_path, path, node) + (snapshot_path, node, open) } else { ( snapshot_path .parent() .expect("file path should have a parent!") .to_path_buf(), - path, node, + open, ) }) } diff --git a/src/archiver/file_archiver.rs b/src/archiver/file_archiver.rs index ce568e9..486431d 100644 --- a/src/archiver/file_archiver.rs +++ b/src/archiver/file_archiver.rs @@ -1,11 +1,10 @@ -use std::fs::File; use std::io::Read; -use anyhow::Result; +use anyhow::{anyhow, Result}; use indicatif::ProgressBar; use rayon::prelude::*; -use crate::backend::DecryptWriteBackend; +use crate::backend::{DecryptWriteBackend, ReadSourceOpen}; use crate::blob::{BlobType, Node, NodeType, Packer, PackerStats}; use crate::chunker::ChunkIter; use crate::crypto::hash; @@ -39,18 +38,22 @@ impl FileArchiver { }) } - pub fn process(&self, item: ItemWithParent, p: ProgressBar) -> Result { + pub fn process( + &self, + item: ItemWithParent>, + p: ProgressBar, + ) -> Result { Ok(match item { TreeType::NewTree(item) => TreeType::NewTree(item), TreeType::EndTree => TreeType::EndTree, - TreeType::Other((path, real_path, node, parent)) => { + TreeType::Other((path, node, open, parent)) => { let (node, filesize) = if let ParentResult::Matched(()) = parent { let size = node.meta.size; p.inc(size); (node, size) } else if let NodeType::File = node.node_type() { - let r = File::open(real_path)?; - self.backup_reader(r, node, p.clone())? + let r = open.ok_or(anyhow!("cannot open file"))?.open()?; + self.backup_reader(r, node, p)? } else { (node, 0) }; diff --git a/src/archiver/parent.rs b/src/archiver/parent.rs index 8ca5098..4d84d9b 100644 --- a/src/archiver/parent.rs +++ b/src/archiver/parent.rs @@ -47,8 +47,8 @@ impl ParentResult<&Node> { } } -pub type ItemWithParent = - TreeType<(PathBuf, PathBuf, Node, ParentResult<()>), (PathBuf, Node, ParentResult)>; +pub type ItemWithParent = + TreeType<(PathBuf, Node, O, ParentResult<()>), (PathBuf, Node, ParentResult)>; impl Parent { pub fn new(be: &BE, tree_id: Option, ignore_ctime: bool, ignore_inode: bool) -> Self { @@ -155,10 +155,10 @@ impl Parent { Ok(()) } - pub fn process( + pub fn process( &mut self, - item: TreeType<(PathBuf, PathBuf, Node), (PathBuf, Node, OsString)>, - ) -> Result { + item: TreeType<(PathBuf, Node, O), (PathBuf, Node, OsString)>, + ) -> Result> { let result = match item { TreeType::NewTree((path, node, tree)) => { let parent_result = self.is_parent(&node, &tree).into_tree_id(); @@ -169,7 +169,7 @@ impl Parent { self.finish_dir()?; TreeType::EndTree } - TreeType::Other((path, real_path, mut node)) => TreeType::Other({ + TreeType::Other((path, mut node, open)) => TreeType::Other({ let be = self.be.clone(); let parent = self.is_parent(&node, &node.name()); let parent = match parent { @@ -186,7 +186,7 @@ impl Parent { } parent_result => parent_result.into_type_only(), }; - (path, real_path, node, parent) + (path, node, open, parent) }), }; Ok(result) diff --git a/src/archiver/tree.rs b/src/archiver/tree.rs index f58c5dc..bc6f5b4 100644 --- a/src/archiver/tree.rs +++ b/src/archiver/tree.rs @@ -30,11 +30,11 @@ pub enum TreeType { Other(T), } -impl Iterator for TreeIterator<(PathBuf, PathBuf, Node), I> +impl Iterator for TreeIterator<(PathBuf, Node, O), I> where - I: Iterator, + I: Iterator, { - type Item = TreeType<(PathBuf, PathBuf, Node), (PathBuf, Node, OsString)>; + type Item = TreeType<(PathBuf, Node, O), (PathBuf, Node, OsString)>; fn next(&mut self) -> Option { match &self.item { None => { @@ -44,7 +44,7 @@ where None } } - Some((path, _, node)) => { + Some((path, node, _)) => { match path.strip_prefix(&self.path) { Err(_) => { self.path.pop(); @@ -56,7 +56,7 @@ where // process next normal path component - other components are simply ignored if let Component::Normal(p) = comp { if node.is_dir() && path == &self.path { - let (path, _, node) = self.item.take().unwrap(); + let (path, node, _) = self.item.take().unwrap(); self.item = self.iter.next(); let name = node.name(); return Some(TreeType::NewTree((path, node, name))); diff --git a/src/archiver/tree_archiver.rs b/src/archiver/tree_archiver.rs index 6a01f74..3d13cd3 100644 --- a/src/archiver/tree_archiver.rs +++ b/src/archiver/tree_archiver.rs @@ -46,6 +46,7 @@ impl TreeArchiver { summary, }) } + pub fn add(&mut self, item: TreeItem) -> Result<()> { match item { TreeType::NewTree((path, node, parent)) => { @@ -97,10 +98,6 @@ impl TreeArchiver { self.tree.add(node); } - pub fn finish_tree(&mut self) -> Result<()> { - Ok(()) - } - pub fn backup_tree(&mut self, path: &Path, parent: ParentResult) -> Result { let (chunk, id) = self.tree.serialize()?; let dirsize = chunk.len() as u64; diff --git a/src/backend/ignore.rs b/src/backend/ignore.rs index 05addab..1e9b7b7 100644 --- a/src/backend/ignore.rs +++ b/src/backend/ignore.rs @@ -3,7 +3,7 @@ use std::fs::{read_link, File}; use std::os::unix::fs::{FileTypeExt, MetadataExt}; use std::path::{Path, PathBuf}; -use anyhow::Result; +use anyhow::{Context, Result}; use bytesize::ByteSize; #[cfg(not(windows))] use chrono::TimeZone; @@ -18,6 +18,7 @@ use serde_with::{serde_as, DisplayFromStr}; use users::{Groups, Users, UsersCache}; use super::{node::Metadata, node::NodeType, Node, ReadSource}; +use super::{ReadSourceEntry, ReadSourceOpen}; pub struct LocalSource { builder: WalkBuilder, @@ -152,12 +153,22 @@ impl LocalSource { } } -impl ReadSource for LocalSource { +pub struct OpenFile(PathBuf); + +impl ReadSourceOpen for OpenFile { type Reader = File; - fn read(path: &Path) -> Result { - Ok(File::open(path)?) + + fn open(self) -> Result { + let path = self.0; + File::open(&path).with_context(|| format!("Unable to open {}", path.display())) } - fn size(&self) -> Result { +} + +impl ReadSource for LocalSource { + type Open = OpenFile; + type Iter = Self; + + fn size(&self) -> Result> { let mut size = 0; for entry in self.builder.build() { if let Err(e) = entry.and_then(|e| e.metadata()).map(|m| { @@ -166,12 +177,16 @@ impl ReadSource for LocalSource { warn!("ignoring error {}", e); } } - Ok(size) + Ok(Some(size)) + } + + fn entries(self) -> Self::Iter { + self } } impl Iterator for LocalSource { - type Item = Result<(PathBuf, Node)>; + type Item = Result>; fn next(&mut self) -> Option { match self.walker.next() { @@ -194,7 +209,11 @@ impl Iterator for LocalSource { } #[cfg(windows)] -fn map_entry(entry: DirEntry, with_atime: bool, _ignore_devid: bool) -> Result<(PathBuf, Node)> { +fn map_entry( + entry: DirEntry, + with_atime: bool, + _ignore_devid: bool, +) -> Result> { let name = entry.file_name(); let m = entry.metadata()?; @@ -253,7 +272,10 @@ fn map_entry(entry: DirEntry, with_atime: bool, _ignore_devid: bool) -> Result<( } else { Node::new_node(name, NodeType::File, meta) }; - Ok((entry.path().to_path_buf(), node)) + + let path = entry.into_path(); + let open = Some(OpenFile(path.clone())); + Ok(ReadSourceEntry { path, node, open }) } #[cfg(not(windows))] @@ -263,7 +285,7 @@ fn map_entry( with_atime: bool, ignore_devid: bool, cache: &UsersCache, -) -> Result<(PathBuf, Node)> { +) -> Result> { let name = entry.file_name(); let m = entry.metadata()?; @@ -337,7 +359,9 @@ fn map_entry( } else { Node::new_node(name, NodeType::File, meta) }; - Ok((entry.path().to_path_buf(), node)) + let path = entry.into_path(); + let open = Some(OpenFile(path.clone())); + Ok(ReadSourceEntry { path, node, open }) } #[cfg(not(windows))] diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 6bfc252..b00858b 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -1,5 +1,5 @@ use std::io::Read; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use anyhow::{anyhow, Result}; use bytes::Bytes; @@ -144,10 +144,24 @@ pub trait WriteBackend: ReadBackend { fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>; } -pub trait ReadSource: Iterator> { - type Reader: Read; - fn read(path: &Path) -> Result; - fn size(&self) -> Result; +pub struct ReadSourceEntry { + pub path: PathBuf, + pub node: Node, + pub open: Option, +} + +pub trait ReadSourceOpen { + type Reader: Read + Send + 'static; + + fn open(self) -> Result; +} + +pub trait ReadSource { + type Open: ReadSourceOpen; + type Iter: Iterator>>; + + fn size(&self) -> Result>; + fn entries(self) -> Self::Iter; } pub trait WriteSource: Clone { diff --git a/src/commands/backup.rs b/src/commands/backup.rs index c1811c9..73ea623 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -234,8 +234,9 @@ pub(super) fn execute( let p = progress_bytes("determining size..."); if !p.is_hidden() { - let size = src.size()?; - p.set_length(size); + if let Some(size) = src.size()? { + p.set_length(size); + } }; p.set_prefix("backing up..."); let archiver = Archiver::new(be, index.clone(), &repo.config, parent, snap)?; diff --git a/src/commands/diff.rs b/src/commands/diff.rs index 237908d..801fc78 100644 --- a/src/commands/diff.rs +++ b/src/commands/diff.rs @@ -5,7 +5,7 @@ use anyhow::{anyhow, bail, Context, Result}; use clap::Parser; use super::{progress_counter, RusticConfig}; -use crate::backend::{LocalDestination, LocalSource, LocalSourceOptions}; +use crate::backend::{LocalDestination, LocalSource, LocalSourceOptions, ReadSourceEntry}; use crate::blob::{Node, NodeStreamer, NodeType, Tree}; use crate::commands::helpers::progress_spinner; use crate::crypto::hash; @@ -86,7 +86,7 @@ pub(super) fn execute( .with_context(|| format!("Error accessing {path2:?}"))? .is_dir(); let src = LocalSource::new(opts.ignore_opts, &[&path2])?.map(|item| { - let (path, node) = item?; + let ReadSourceEntry { path, node, .. } = item?; let path = if is_dir { // remove given path prefix for dirs as local path path.strip_prefix(&path2)?.to_path_buf()