Merge pull request #764 from rustic-rs/refactor-merge

refactor merge command

crates/rustic_core/examples/merge.rs

@@ -0,0 +1,23 @@
+//! `merge` example
+use rustic_core::{latest_node, Repository, RepositoryOptions, SnapshotFile};
+use simplelog::{Config, LevelFilter, SimpleLogger};
+use std::error::Error;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    // Display info logs
+    let _ = SimpleLogger::init(LevelFilter::Info, Config::default());
+
+    // Open repository
+    let repo_opts = RepositoryOptions::default()
+        .repository("/tmp/repo")
+        .password("test");
+    let repo = Repository::new(&repo_opts)?.open()?.to_indexed_ids()?;
+
+    // Merge all snapshots using the latest entry for duplicate entries
+    let snaps = repo.get_all_snapshots()?;
+    // This creates a new snapshot without removing the used ones
+    let snap = repo.merge_snapshots(&snaps, &latest_node, SnapshotFile::default())?;
+
+    println!("successfully created snapshot:\n{snap:#?}");
+    Ok(())
+}

crates/rustic_core/src/backend/node.rs

@@ -1,4 +1,5 @@
 use std::{
+    cmp::Ordering,
     ffi::{OsStr, OsString},
     fmt::Debug,
     str::FromStr,
@@ -149,6 +150,10 @@ impl Node {
     }
 }
 
+pub fn latest_node(n1: &Node, n2: &Node) -> Ordering {
+    n1.meta.mtime.cmp(&n2.meta.mtime)
+}
+
 // TODO(Windows): This is not able to handle non-unicode filenames and
 // doesn't treat filenames which need and escape (like `\`, `"`, ...) correctly
 #[cfg(windows)]

crates/rustic_core/src/blob/tree.rs

@@ -396,7 +396,7 @@ impl<P: Progress> Iterator for TreeStreamerOnce<P> {
     }
 }
 
-pub fn merge_trees(
+pub(crate) fn merge_trees(
     be: &impl IndexedBackend,
     trees: &[Id],
     cmp: &impl Fn(&Node, &Node) -> Ordering,

crates/rustic_core/src/commands.rs

@@ -7,6 +7,7 @@ pub mod dump;
 pub mod forget;
 pub mod init;
 pub mod key;
+pub mod merge;
 pub mod prune;
 pub mod repoinfo;
 pub mod restore;

crates/rustic_core/src/commands/merge.rs

@@ -0,0 +1,76 @@
+//! `merge` subcommand
+
+use std::cmp::Ordering;
+
+use chrono::Local;
+
+use crate::{
+    blob::tree, error::CommandErrorKind, repofile::snapshotfile::SnapshotSummary,
+    repository::IndexedTree, BlobType, DecryptWriteBackend, Id, Indexer, Node, Open, Packer,
+    PathList, Progress, ProgressBars, ReadIndex, Repository, RusticResult, SnapshotFile, Tree,
+};
+
+pub(crate) fn merge_snapshots<P: ProgressBars, S: IndexedTree>(
+    repo: &Repository<P, S>,
+    snapshots: &[SnapshotFile],
+    cmp: &impl Fn(&Node, &Node) -> Ordering,
+    mut snap: SnapshotFile,
+) -> RusticResult<SnapshotFile> {
+    let now = Local::now();
+
+    let paths = PathList::from_strings(snapshots.iter().flat_map(|snap| snap.paths.iter()), false)?;
+    snap.paths.set_paths(&paths.paths())?;
+
+    // set snapshot time to time of latest snapshot to be merged
+    snap.time = snapshots
+        .iter()
+        .max_by_key(|sn| sn.time)
+        .map_or(now, |sn| sn.time);
+
+    let mut summary = snap.summary.take().unwrap_or_default();
+    summary.backup_start = Local::now();
+
+    let trees: Vec<Id> = snapshots.iter().map(|sn| sn.tree).collect();
+    snap.tree = merge_trees(repo, &trees, cmp, &mut summary)?;
+
+    summary.finalize(now)?;
+    snap.summary = Some(summary);
+
+    snap.id = repo.dbe().save_file(&snap)?;
+    Ok(snap)
+}
+
+pub(crate) fn merge_trees<P: ProgressBars, S: IndexedTree>(
+    repo: &Repository<P, S>,
+    trees: &[Id],
+    cmp: &impl Fn(&Node, &Node) -> Ordering,
+    summary: &mut SnapshotSummary,
+) -> RusticResult<Id> {
+    let index = repo.index();
+    let indexer = Indexer::new(repo.dbe().clone()).into_shared();
+    let packer = Packer::new(
+        repo.dbe().clone(),
+        BlobType::Tree,
+        indexer.clone(),
+        repo.config(),
+        index.total_size(BlobType::Tree),
+    )?;
+    let save = |tree: Tree| {
+        let (chunk, new_id) = tree.serialize()?;
+        let size = u64::try_from(chunk.len()).map_err(CommandErrorKind::ConversionToU64Failed)?;
+        if !index.has_tree(&new_id) {
+            packer.add(chunk.into(), new_id)?;
+        }
+        Ok((new_id, size))
+    };
+
+    let p = repo.pb.progress_spinner("merging snapshots...");
+    let tree_merged = tree::merge_trees(index, trees, cmp, &save, summary)?;
+    let stats = packer.finalize()?;
+    indexer.write().unwrap().finalize()?;
+    p.finish();
+
+    stats.apply(summary, BlobType::Tree);
+
+    Ok(tree_merged)
+}

crates/rustic_core/src/error.rs

@@ -189,6 +189,8 @@ pub enum CommandErrorKind {
     IdNotFound(Id),
     /// {0:?}
     FromRayonError(#[from] rayon::ThreadPoolBuildError),
+    /// conversion to `u64` failed: `{0:?}`
+    ConversionToU64Failed(TryFromIntError),
 }
 
 /// [`CryptoErrorKind`] describes the errors that can happen while dealing with Cryptographic functions

crates/rustic_core/src/lib.rs

@@ -106,13 +106,13 @@ pub use crate::{
         decrypt::{DecryptReadBackend, DecryptWriteBackend},
         ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions},
         local::LocalDestination,
-        node::{Node, NodeType},
+        node::{latest_node, Node, NodeType},
         stdin::StdinSource,
         FileType, ReadBackend, ReadSourceEntry, WriteBackend, ALL_FILE_TYPES,
     },
     blob::{
         packer::Packer,
-        tree::{merge_trees, NodeStreamer, Tree, TreeStreamerOnce, TreeStreamerOptions},
+        tree::{NodeStreamer, Tree, TreeStreamerOnce, TreeStreamerOptions},
         BlobType, BlobTypeMap, Initialize, Sum,
     },
     commands::{

crates/rustic_core/src/repository.rs

@@ -1,4 +1,5 @@
 use std::{
+    cmp::Ordering,
     collections::HashMap,
     fs::File,
     io::{BufRead, BufReader, Write},
@@ -44,8 +45,10 @@ use crate::{
     },
     crypto::aespoly1305::Key,
     error::{KeyFileErrorKind, RepositoryErrorKind, RusticErrorKind},
-    repofile::RepoFile,
-    repofile::{configfile::ConfigFile, keyfile::find_key_in_backend},
+    repofile::{
+        configfile::ConfigFile, keyfile::find_key_in_backend, snapshotfile::SnapshotSummary,
+        RepoFile,
+    },
     BlobType, Id, IndexBackend, IndexedBackend, LocalDestination, NoProgressBars, Node,
     NodeStreamer, PathList, ProgressBars, PruneOpts, PrunePlan, RusticResult, SnapshotFile,
     SnapshotGroup, SnapshotGroupCriterion, Tree, TreeStreamerOptions,
@@ -532,6 +535,8 @@ impl<P: ProgressBars, S: Open> Repository<P, S> {
         commands::copy::relevant_snapshots(snaps, self, filter)
     }
 
+    // TODO: Maybe only offer a method to remove &[Snapshotfile] and check if they must be kept.
+    // See e.g. the merge command of the CLI
     pub fn delete_snapshots(&self, ids: &[Id]) -> RusticResult<()> {
         let p = self.pb.progress_counter("removing snapshots...");
         self.dbe()
@@ -700,6 +705,24 @@ impl<P: ProgressBars, S: IndexedTree> Repository<P, S> {
     ) -> RusticResult<()> {
         opts.restore(restore_infos, self, node_streamer, dest)
     }
+
+    pub fn merge_trees(
+        &self,
+        trees: &[Id],
+        cmp: &impl Fn(&Node, &Node) -> Ordering,
+        summary: &mut SnapshotSummary,
+    ) -> RusticResult<Id> {
+        commands::merge::merge_trees(self, trees, cmp, summary)
+    }
+
+    pub fn merge_snapshots(
+        &self,
+        snaps: &[SnapshotFile],
+        cmp: &impl Fn(&Node, &Node) -> Ordering,
+        snap: SnapshotFile,
+    ) -> RusticResult<SnapshotFile> {
+        commands::merge::merge_snapshots(self, snaps, cmp, snap)
+    }
 }
 
 impl<P: ProgressBars, S: IndexedIds> Repository<P, S> {

src/commands/merge.rs

@@ -9,10 +9,7 @@ use log::info;
 
 use chrono::Local;
 
-use rustic_core::{
-    merge_trees, BlobType, DecryptWriteBackend, FileType, Id, IndexBackend, Indexer, Node, Open,
-    Packer, PathList, Progress, ProgressBars, ReadIndex, SnapshotFile, SnapshotOptions, Tree,
-};
+use rustic_core::{Node, SnapshotFile, SnapshotOptions};
 
 /// `merge` subcommand
 #[derive(clap::Parser, Default, Command, Debug)]
@@ -52,95 +49,35 @@ impl MergeCmd {
             .join(" ");
 
         let config = RUSTIC_APP.config();
-        let progress_options = &config.global.progress_options;
+        let repo = open_repository(&config)?.to_indexed_ids()?;
 
-        let repo = open_repository(&config)?;
-
-        let be = repo.dbe();
-
-        let p = progress_options.progress_hidden();
         let snapshots = if self.ids.is_empty() {
-            SnapshotFile::all_from_backend(be, |sn| config.snapshot_filter.matches(sn), &p)?
+            repo.get_matching_snapshots(|sn| config.snapshot_filter.matches(sn))?
         } else {
-            SnapshotFile::from_ids(be, &self.ids, &p)?
+            repo.get_snapshots(&self.ids)?
         };
-        let index =
-            IndexBackend::only_full_trees(&be.clone(), &progress_options.progress_counter(""))?;
 
-        let indexer = Indexer::new(be.clone()).into_shared();
-        let packer = Packer::new(
-            be.clone(),
-            BlobType::Tree,
-            indexer.clone(),
-            repo.config(),
-            index.total_size(BlobType::Tree),
-        )?;
-
-        let mut snap = SnapshotFile::new_from_options(&self.snap_opts, now, command)?;
-
-        let paths =
-            PathList::from_strings(snapshots.iter().flat_map(|snap| snap.paths.iter()), false)?;
-        snap.paths.set_paths(&paths.paths())?;
-
-        // set snapshot time to time of latest snapshot to be merged
-        snap.time = snapshots
-            .iter()
-            .max_by_key(|sn| sn.time)
-            .map_or(now, |sn| sn.time);
-
-        let mut summary = snap.summary.take().unwrap();
-        summary.backup_start = Local::now();
-
-        let p = progress_options.progress_spinner("merging snapshots...");
-        let trees: Vec<Id> = snapshots.iter().map(|sn| sn.tree).collect();
+        let snap = SnapshotFile::new_from_options(&self.snap_opts, now, command)?;
 
         let cmp = |n1: &Node, n2: &Node| n1.meta.mtime.cmp(&n2.meta.mtime);
-        let save = |tree: Tree| {
-            let (chunk, new_id) = tree.serialize()?;
-            let size = match u64::try_from(chunk.len()) {
-                Ok(it) => it,
-                Err(err) => {
-                    status_err!("{}", err);
-                    RUSTIC_APP.shutdown(Shutdown::Crash);
-                }
-            };
-            if !index.has_tree(&new_id) {
-                packer.add(chunk.into(), new_id)?;
-            }
-            Ok((new_id, size))
-        };
 
-        let tree_merged = merge_trees(&index, &trees, &cmp, &save, &mut summary)?;
-        snap.tree = tree_merged;
-
-        let stats = packer.finalize()?;
-        stats.apply(&mut summary, BlobType::Tree);
-
-        indexer.write().unwrap().finalize()?;
-
-        p.finish();
-
-        summary.finalize(now)?;
-        snap.summary = Some(summary);
-
-        let new_id = be.save_file(&snap)?;
-        snap.id = new_id;
+        let snap = repo.merge_snapshots(&snapshots, &cmp, snap)?;
 
         if self.json {
             let mut stdout = std::io::stdout();
             serde_json::to_writer_pretty(&mut stdout, &snap)?;
         }
-        info!("saved new snapshot as {new_id}.");
+        info!("saved new snapshot as {}.", snap.id);
 
         if self.delete {
             let now = Local::now();
-            let p = progress_options.progress_counter("deleting old snapshots...");
+            // TODO: Maybe use this check in repo.delete_snapshots?
             let snap_ids: Vec<_> = snapshots
                 .iter()
                 .filter(|sn| !sn.must_keep(now))
                 .map(|sn| sn.id)
                 .collect();
-            be.delete_list(FileType::Snapshot, true, snap_ids.iter(), p)?;
+            repo.delete_snapshots(&snap_ids)?;
         }
 
         Ok(())