From 3fa3e03cf6daab444f848a7ad42c6847beeeaf74 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Tue, 1 Mar 2022 12:49:09 +0100 Subject: [PATCH] refactor archiver: Add support for parent snapshot --- src/{ => archiver}/archiver.rs | 40 ++++++++++++++++------- src/archiver/mod.rs | 5 +++ src/archiver/parent.rs | 60 ++++++++++++++++++++++++++++++++++ src/commands/backup.rs | 43 ++++++++++++++---------- 4 files changed, 119 insertions(+), 29 deletions(-) rename src/{ => archiver}/archiver.rs (77%) create mode 100644 src/archiver/mod.rs create mode 100644 src/archiver/parent.rs diff --git a/src/archiver.rs b/src/archiver/archiver.rs similarity index 77% rename from src/archiver.rs rename to src/archiver/archiver.rs index 6239a8e..ed24519 100644 --- a/src/archiver.rs +++ b/src/archiver/archiver.rs @@ -9,15 +9,18 @@ use crate::backend::DecryptWriteBackend; use crate::blob::{BlobType, Metadata, Node, NodeType, Packer, Tree}; use crate::chunker::ChunkIter; use crate::crypto::hash; -use crate::index::{Indexer, ReadIndex}; +use crate::index::{IndexedBackend, Indexer}; use crate::repo::{SnapshotFile, TagList}; -pub type SharedIndexer = Rc>>; +use super::Parent; -pub struct Archiver { +type SharedIndexer = Rc>>; + +pub struct Archiver { path: PathBuf, tree: Tree, - stack: Vec<(Node, Tree)>, + parent: Parent, + stack: Vec<(Node, Tree, Parent)>, size: u64, count: u64, be: BE, @@ -28,12 +31,13 @@ pub struct Archiver { poly: u64, } -impl Archiver { - pub fn new(be: BE, index: I, poly: u64) -> Result { +impl Archiver { + pub fn new(be: BE, index: I, poly: u64, parent: Parent) -> Result { let indexer = Rc::new(RefCell::new(Indexer::new(be.clone()))); Ok(Self { path: PathBuf::from("/"), tree: Tree::new(), + parent: parent, stack: Vec::new(), size: 0, count: 0, @@ -68,13 +72,17 @@ impl Archiver { self.path.push(p); let tree = std::mem::replace(&mut self.tree, Tree::new()); if self.path == path { - self.stack.push((node, tree)); + // use Node and return + let new_parent = self.parent.sub_parent(&node); + let parent = std::mem::replace(&mut self.parent, new_parent); + self.stack.push((node, tree, parent)); return Ok(()); } else { - self.stack - .push((Node::new_dir(p.to_os_string(), Metadata::default()), tree)); - } - + let node = Node::new_dir(p.to_os_string(), Metadata::default()); + let new_parent = self.parent.sub_parent(&node); + let parent = std::mem::replace(&mut self.parent, new_parent); + self.stack.push((node, tree, parent)); + }; println!("add tree {:?}, path: {:?}", p, self.path); } @@ -99,10 +107,11 @@ impl Archiver { self.tree_packer.add(&chunk, &id, BlobType::Tree)?; } - let (mut node, tree) = self.stack.pop().ok_or(anyhow!("tree stack empty??"))?; + let (mut node, tree, parent) = self.stack.pop().ok_or(anyhow!("tree stack empty??"))?; println!("finishing tree: {:?}", node.name()); node.set_subtree(id); self.tree = tree; + self.parent = parent; self.add_node(node, dirsize); self.path.pop(); } @@ -110,6 +119,13 @@ impl Archiver { } pub fn backup_file(&mut self, node: Node, reader: impl BufRead) -> Result<()> { + if let Some(p_node) = self.parent.is_parent(&node) { + println!("using parent!"); + let size = *p_node.meta().size(); + let node = p_node.clone(); + self.add_node(node, size); + return Ok(()); + } let chunk_iter = ChunkIter::new(reader, &self.poly); let mut content = Vec::new(); let mut filesize: u64 = 0; diff --git a/src/archiver/mod.rs b/src/archiver/mod.rs new file mode 100644 index 0000000..3d7a6ac --- /dev/null +++ b/src/archiver/mod.rs @@ -0,0 +1,5 @@ +mod archiver; +mod parent; + +pub use archiver::*; +pub use parent::*; diff --git a/src/archiver/parent.rs b/src/archiver/parent.rs new file mode 100644 index 0000000..e0b54dd --- /dev/null +++ b/src/archiver/parent.rs @@ -0,0 +1,60 @@ +use crate::blob::{Node, Tree}; +use crate::id::Id; +use crate::index::IndexedBackend; + +pub struct Parent { + tree: Option, + be: BE, +} + +impl Parent { + pub fn new(be: &BE, tree_id: Option<&Id>) -> Self { + let tree = tree_id.map(|id| Tree::from_backend(be, id).unwrap()); + Self { + tree, + be: be.clone(), + } + } + + pub fn p_node(&self, node: &Node) -> Option<&Node> { + match &self.tree { + None => None, + Some(tree) => tree + .nodes() + .iter() + .find(|p_node| p_node.name() == node.name()), + } + } + + pub fn is_parent(&self, node: &Node) -> Option<&Node> { + match self.p_node(node) { + None => None, + Some(p_node) => { + if p_node.node_type() == node.node_type() + && p_node.meta().ctime() == node.meta().ctime() + && p_node.meta().inode() > &0 + && p_node.meta().inode() == node.meta().inode() + && p_node.content().iter().all(|id| self.be.has(id)) + { + Some(p_node) + } else { + None + } + } + } + } + + pub fn sub_parent(&self, node: &Node) -> Self { + let tree = match self.p_node(node) { + None => None, + Some(p_node) if p_node.node_type() == node.node_type() => { + Some(Tree::from_backend(&self.be, &p_node.subtree().unwrap()).unwrap()) + } + Some(..) => None, + }; + Self { + tree, + be: self.be.clone(), + } + } +} diff --git a/src/commands/backup.rs b/src/commands/backup.rs index bddbd84..dbfca01 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -9,12 +9,12 @@ use clap::Parser; use ignore::{DirEntry, WalkBuilder}; use path_absolutize::*; -use crate::archiver::Archiver; -use crate::backend::DecryptFullBackend; +use crate::archiver::{Archiver, Parent}; +use crate::backend::{DecryptFullBackend, FileType}; use crate::blob::{Metadata, Node}; +use crate::id::Id; use crate::index::IndexBackend; -use crate::repo::ConfigFile; - +use crate::repo::{ConfigFile, SnapshotFile}; #[derive(Parser)] pub(super) struct Opts { /// save access time for files and directories @@ -23,27 +23,35 @@ pub(super) struct Opts { /// backup sources sources: Vec, + + /// snapshot to use as parent + #[clap(long)] + parent: Option, } pub(super) fn execute(opts: Opts, be: &impl DecryptFullBackend) -> Result<()> { let config = ConfigFile::from_backend_no_id(be)?; let poly = u64::from_str_radix(config.chunker_polynomial(), 16)?; - let path = PathBuf::from(&opts.sources[0]); - let path = path.absolutize()?; - backup(path.into(), &poly, be, opts)?; - Ok(()) -} + let backup_path = PathBuf::from(&opts.sources[0]); + let backup_path = backup_path.absolutize()?; -fn backup( - backup_path: PathBuf, - poly: &u64, - be: &impl DecryptFullBackend, - opts: Opts, -) -> Result<()> { println! {"reading index..."} let index = IndexBackend::new(be)?; - let mut archiver = Archiver::new(be.clone(), index, *poly)?; + let parent_tree = match opts.parent { + None => None, + Some(parent) => { + let id = Id::from_hex(&parent).or_else(|_| { + // if the given id param is not a full Id, search for a suitable one + be.find_starts_with(FileType::Snapshot, &[&parent])? + .remove(0) + })?; + let snap = SnapshotFile::from_backend(be, &id)?; + Some(snap.tree) + } + }; + let parent = Parent::new(&index, parent_tree.as_ref()); + let mut archiver = Archiver::new(be.clone(), index, poly, parent)?; let mut wb = WalkBuilder::new(backup_path.clone()); /* @@ -59,7 +67,7 @@ fn backup( let (path, node, r) = res?; archiver.add_entry(&path, node, r)?; } - archiver.finalize_snapshot(backup_path)?; + archiver.finalize_snapshot(backup_path.to_path_buf())?; Ok(()) } @@ -93,6 +101,7 @@ fn map_entry(entry: DirEntry, with_atime: bool) -> Result<(PathBuf, Node, Option let target = read_link(entry.path())?; (Node::new_symlink(name, target, meta), None) } else { + // TODO: lazily open file! - might be contained in parent let f = File::open(&entry.path())?; (Node::new_file(name, meta), Some(BufReader::new(f))) };