diff --git a/crates/rustic_core/examples/merge.rs b/crates/rustic_core/examples/merge.rs new file mode 100644 index 0000000..40321fe --- /dev/null +++ b/crates/rustic_core/examples/merge.rs @@ -0,0 +1,23 @@ +//! `merge` example +use rustic_core::{latest_node, Repository, RepositoryOptions, SnapshotFile}; +use simplelog::{Config, LevelFilter, SimpleLogger}; +use std::error::Error; + +fn main() -> Result<(), Box> { + // Display info logs + let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); + + // Open repository + let repo_opts = RepositoryOptions::default() + .repository("/tmp/repo") + .password("test"); + let repo = Repository::new(&repo_opts)?.open()?.to_indexed_ids()?; + + // Merge all snapshots using the latest entry for duplicate entries + let snaps = repo.get_all_snapshots()?; + // This creates a new snapshot without removing the used ones + let snap = repo.merge_snapshots(&snaps, &latest_node, SnapshotFile::default())?; + + println!("successfully created snapshot:\n{snap:#?}"); + Ok(()) +} diff --git a/crates/rustic_core/src/backend/node.rs b/crates/rustic_core/src/backend/node.rs index 4fb0b26..e50a5ad 100644 --- a/crates/rustic_core/src/backend/node.rs +++ b/crates/rustic_core/src/backend/node.rs @@ -1,4 +1,5 @@ use std::{ + cmp::Ordering, ffi::{OsStr, OsString}, fmt::Debug, str::FromStr, @@ -149,6 +150,10 @@ impl Node { } } +pub fn latest_node(n1: &Node, n2: &Node) -> Ordering { + n1.meta.mtime.cmp(&n2.meta.mtime) +} + // TODO(Windows): This is not able to handle non-unicode filenames and // doesn't treat filenames which need and escape (like `\`, `"`, ...) correctly #[cfg(windows)] diff --git a/crates/rustic_core/src/blob/tree.rs b/crates/rustic_core/src/blob/tree.rs index f327fec..db71f39 100644 --- a/crates/rustic_core/src/blob/tree.rs +++ b/crates/rustic_core/src/blob/tree.rs @@ -396,7 +396,7 @@ impl Iterator for TreeStreamerOnce

{ } } -pub fn merge_trees( +pub(crate) fn merge_trees( be: &impl IndexedBackend, trees: &[Id], cmp: &impl Fn(&Node, &Node) -> Ordering, diff --git a/crates/rustic_core/src/commands.rs b/crates/rustic_core/src/commands.rs index 0354610..5befb1f 100644 --- a/crates/rustic_core/src/commands.rs +++ b/crates/rustic_core/src/commands.rs @@ -7,6 +7,7 @@ pub mod dump; pub mod forget; pub mod init; pub mod key; +pub mod merge; pub mod prune; pub mod repoinfo; pub mod restore; diff --git a/crates/rustic_core/src/commands/merge.rs b/crates/rustic_core/src/commands/merge.rs new file mode 100644 index 0000000..3c0422c --- /dev/null +++ b/crates/rustic_core/src/commands/merge.rs @@ -0,0 +1,76 @@ +//! `merge` subcommand + +use std::cmp::Ordering; + +use chrono::Local; + +use crate::{ + blob::tree, error::CommandErrorKind, repofile::snapshotfile::SnapshotSummary, + repository::IndexedTree, BlobType, DecryptWriteBackend, Id, Indexer, Node, Open, Packer, + PathList, Progress, ProgressBars, ReadIndex, Repository, RusticResult, SnapshotFile, Tree, +}; + +pub(crate) fn merge_snapshots( + repo: &Repository, + snapshots: &[SnapshotFile], + cmp: &impl Fn(&Node, &Node) -> Ordering, + mut snap: SnapshotFile, +) -> RusticResult { + let now = Local::now(); + + let paths = PathList::from_strings(snapshots.iter().flat_map(|snap| snap.paths.iter()), false)?; + snap.paths.set_paths(&paths.paths())?; + + // set snapshot time to time of latest snapshot to be merged + snap.time = snapshots + .iter() + .max_by_key(|sn| sn.time) + .map_or(now, |sn| sn.time); + + let mut summary = snap.summary.take().unwrap_or_default(); + summary.backup_start = Local::now(); + + let trees: Vec = snapshots.iter().map(|sn| sn.tree).collect(); + snap.tree = merge_trees(repo, &trees, cmp, &mut summary)?; + + summary.finalize(now)?; + snap.summary = Some(summary); + + snap.id = repo.dbe().save_file(&snap)?; + Ok(snap) +} + +pub(crate) fn merge_trees( + repo: &Repository, + trees: &[Id], + cmp: &impl Fn(&Node, &Node) -> Ordering, + summary: &mut SnapshotSummary, +) -> RusticResult { + let index = repo.index(); + let indexer = Indexer::new(repo.dbe().clone()).into_shared(); + let packer = Packer::new( + repo.dbe().clone(), + BlobType::Tree, + indexer.clone(), + repo.config(), + index.total_size(BlobType::Tree), + )?; + let save = |tree: Tree| { + let (chunk, new_id) = tree.serialize()?; + let size = u64::try_from(chunk.len()).map_err(CommandErrorKind::ConversionToU64Failed)?; + if !index.has_tree(&new_id) { + packer.add(chunk.into(), new_id)?; + } + Ok((new_id, size)) + }; + + let p = repo.pb.progress_spinner("merging snapshots..."); + let tree_merged = tree::merge_trees(index, trees, cmp, &save, summary)?; + let stats = packer.finalize()?; + indexer.write().unwrap().finalize()?; + p.finish(); + + stats.apply(summary, BlobType::Tree); + + Ok(tree_merged) +} diff --git a/crates/rustic_core/src/error.rs b/crates/rustic_core/src/error.rs index 4b055c6..0053bc1 100644 --- a/crates/rustic_core/src/error.rs +++ b/crates/rustic_core/src/error.rs @@ -189,6 +189,8 @@ pub enum CommandErrorKind { IdNotFound(Id), /// {0:?} FromRayonError(#[from] rayon::ThreadPoolBuildError), + /// conversion to `u64` failed: `{0:?}` + ConversionToU64Failed(TryFromIntError), } /// [`CryptoErrorKind`] describes the errors that can happen while dealing with Cryptographic functions diff --git a/crates/rustic_core/src/lib.rs b/crates/rustic_core/src/lib.rs index 4ca2f15..5760f82 100644 --- a/crates/rustic_core/src/lib.rs +++ b/crates/rustic_core/src/lib.rs @@ -106,13 +106,13 @@ pub use crate::{ decrypt::{DecryptReadBackend, DecryptWriteBackend}, ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions}, local::LocalDestination, - node::{Node, NodeType}, + node::{latest_node, Node, NodeType}, stdin::StdinSource, FileType, ReadBackend, ReadSourceEntry, WriteBackend, ALL_FILE_TYPES, }, blob::{ packer::Packer, - tree::{merge_trees, NodeStreamer, Tree, TreeStreamerOnce, TreeStreamerOptions}, + tree::{NodeStreamer, Tree, TreeStreamerOnce, TreeStreamerOptions}, BlobType, BlobTypeMap, Initialize, Sum, }, commands::{ diff --git a/crates/rustic_core/src/repository.rs b/crates/rustic_core/src/repository.rs index d8a2711..8e9c1cb 100644 --- a/crates/rustic_core/src/repository.rs +++ b/crates/rustic_core/src/repository.rs @@ -1,4 +1,5 @@ use std::{ + cmp::Ordering, collections::HashMap, fs::File, io::{BufRead, BufReader, Write}, @@ -44,8 +45,10 @@ use crate::{ }, crypto::aespoly1305::Key, error::{KeyFileErrorKind, RepositoryErrorKind, RusticErrorKind}, - repofile::RepoFile, - repofile::{configfile::ConfigFile, keyfile::find_key_in_backend}, + repofile::{ + configfile::ConfigFile, keyfile::find_key_in_backend, snapshotfile::SnapshotSummary, + RepoFile, + }, BlobType, Id, IndexBackend, IndexedBackend, LocalDestination, NoProgressBars, Node, NodeStreamer, PathList, ProgressBars, PruneOpts, PrunePlan, RusticResult, SnapshotFile, SnapshotGroup, SnapshotGroupCriterion, Tree, TreeStreamerOptions, @@ -532,6 +535,8 @@ impl Repository { commands::copy::relevant_snapshots(snaps, self, filter) } + // TODO: Maybe only offer a method to remove &[Snapshotfile] and check if they must be kept. + // See e.g. the merge command of the CLI pub fn delete_snapshots(&self, ids: &[Id]) -> RusticResult<()> { let p = self.pb.progress_counter("removing snapshots..."); self.dbe() @@ -700,6 +705,24 @@ impl Repository { ) -> RusticResult<()> { opts.restore(restore_infos, self, node_streamer, dest) } + + pub fn merge_trees( + &self, + trees: &[Id], + cmp: &impl Fn(&Node, &Node) -> Ordering, + summary: &mut SnapshotSummary, + ) -> RusticResult { + commands::merge::merge_trees(self, trees, cmp, summary) + } + + pub fn merge_snapshots( + &self, + snaps: &[SnapshotFile], + cmp: &impl Fn(&Node, &Node) -> Ordering, + snap: SnapshotFile, + ) -> RusticResult { + commands::merge::merge_snapshots(self, snaps, cmp, snap) + } } impl Repository { diff --git a/src/commands/merge.rs b/src/commands/merge.rs index 2a4c0de..b286f13 100644 --- a/src/commands/merge.rs +++ b/src/commands/merge.rs @@ -9,10 +9,7 @@ use log::info; use chrono::Local; -use rustic_core::{ - merge_trees, BlobType, DecryptWriteBackend, FileType, Id, IndexBackend, Indexer, Node, Open, - Packer, PathList, Progress, ProgressBars, ReadIndex, SnapshotFile, SnapshotOptions, Tree, -}; +use rustic_core::{Node, SnapshotFile, SnapshotOptions}; /// `merge` subcommand #[derive(clap::Parser, Default, Command, Debug)] @@ -52,95 +49,35 @@ impl MergeCmd { .join(" "); let config = RUSTIC_APP.config(); - let progress_options = &config.global.progress_options; + let repo = open_repository(&config)?.to_indexed_ids()?; - let repo = open_repository(&config)?; - - let be = repo.dbe(); - - let p = progress_options.progress_hidden(); let snapshots = if self.ids.is_empty() { - SnapshotFile::all_from_backend(be, |sn| config.snapshot_filter.matches(sn), &p)? + repo.get_matching_snapshots(|sn| config.snapshot_filter.matches(sn))? } else { - SnapshotFile::from_ids(be, &self.ids, &p)? + repo.get_snapshots(&self.ids)? }; - let index = - IndexBackend::only_full_trees(&be.clone(), &progress_options.progress_counter(""))?; - let indexer = Indexer::new(be.clone()).into_shared(); - let packer = Packer::new( - be.clone(), - BlobType::Tree, - indexer.clone(), - repo.config(), - index.total_size(BlobType::Tree), - )?; - - let mut snap = SnapshotFile::new_from_options(&self.snap_opts, now, command)?; - - let paths = - PathList::from_strings(snapshots.iter().flat_map(|snap| snap.paths.iter()), false)?; - snap.paths.set_paths(&paths.paths())?; - - // set snapshot time to time of latest snapshot to be merged - snap.time = snapshots - .iter() - .max_by_key(|sn| sn.time) - .map_or(now, |sn| sn.time); - - let mut summary = snap.summary.take().unwrap(); - summary.backup_start = Local::now(); - - let p = progress_options.progress_spinner("merging snapshots..."); - let trees: Vec = snapshots.iter().map(|sn| sn.tree).collect(); + let snap = SnapshotFile::new_from_options(&self.snap_opts, now, command)?; let cmp = |n1: &Node, n2: &Node| n1.meta.mtime.cmp(&n2.meta.mtime); - let save = |tree: Tree| { - let (chunk, new_id) = tree.serialize()?; - let size = match u64::try_from(chunk.len()) { - Ok(it) => it, - Err(err) => { - status_err!("{}", err); - RUSTIC_APP.shutdown(Shutdown::Crash); - } - }; - if !index.has_tree(&new_id) { - packer.add(chunk.into(), new_id)?; - } - Ok((new_id, size)) - }; - let tree_merged = merge_trees(&index, &trees, &cmp, &save, &mut summary)?; - snap.tree = tree_merged; - - let stats = packer.finalize()?; - stats.apply(&mut summary, BlobType::Tree); - - indexer.write().unwrap().finalize()?; - - p.finish(); - - summary.finalize(now)?; - snap.summary = Some(summary); - - let new_id = be.save_file(&snap)?; - snap.id = new_id; + let snap = repo.merge_snapshots(&snapshots, &cmp, snap)?; if self.json { let mut stdout = std::io::stdout(); serde_json::to_writer_pretty(&mut stdout, &snap)?; } - info!("saved new snapshot as {new_id}."); + info!("saved new snapshot as {}.", snap.id); if self.delete { let now = Local::now(); - let p = progress_options.progress_counter("deleting old snapshots..."); + // TODO: Maybe use this check in repo.delete_snapshots? let snap_ids: Vec<_> = snapshots .iter() .filter(|sn| !sn.must_keep(now)) .map(|sn| sn.id) .collect(); - be.delete_list(FileType::Snapshot, true, snap_ids.iter(), p)?; + repo.delete_snapshots(&snap_ids)?; } Ok(())