diff --git a/README.md b/README.md index f05a647..78b928b 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Current limitations: ## Open points: * [ ] Add tests and benchmarks - * [ ] Add missing commands: copy, dump, find, mount + * [ ] Add missing commands: dump, find, mount * [ ] Improve error handling * [ ] Parallelize the code even more and optimize for speed where useful diff --git a/examples/copy_example.toml b/examples/copy_example.toml new file mode 100644 index 0000000..fe5a855 --- /dev/null +++ b/examples/copy_example.toml @@ -0,0 +1,21 @@ +# This is an example how to configure the copy command to copy snapshots from one repository to another +# The targets of the copy command cannot be specified on the command line, but must be in a config file like this. +# If the config file is named "copy_example.toml", run "rustic -P copy_example copy" to copy all snapshots. +# See "rustic copy --help" for options how to select or filter snapshots to copy. + +# [repository] specified the source repository +[repository] +repository = "/tmp/repo" +password = "test" + +# you can specify multiple targets +[[copy.targets]] +repository = "/tmp/repo2" +password = "test" +no-cache = true + +[[copy.targets]] +repository = "rclone:ovh:backup" +repo-hot = "clone:ovh:backup-hot" +password-file = "/root/key-rustic-ovh" +cache-dir = "/var/lib/cache/rustic" # explicitely specify cache dir for remote repository diff --git a/src/commands/copy.rs b/src/commands/copy.rs new file mode 100644 index 0000000..98e4012 --- /dev/null +++ b/src/commands/copy.rs @@ -0,0 +1,206 @@ +use std::collections::BTreeSet; + +use anyhow::{bail, Result}; +use clap::{AppSettings, Parser}; +use log::*; +use rayon::prelude::*; + +use super::{progress_counter, table_with_titles, RusticConfig}; +use crate::backend::DecryptWriteBackend; +use crate::blob::{BlobType, NodeType, Packer, TreeStreamerOnce}; +use crate::index::{IndexBackend, IndexedBackend, Indexer, ReadIndex}; +use crate::repofile::{Id, SnapshotFile, SnapshotFilter}; +use crate::repository::{OpenRepository, Repository, RepositoryOptions}; + +#[derive(Parser)] +#[clap(global_setting(AppSettings::DeriveDisplayOrder))] +pub(super) struct Opts { + /// Don't copy any snapshot, only show what would be done + #[clap(long, short = 'n')] + dry_run: bool, + + #[clap( + flatten, + help_heading = "SNAPSHOT FILTER OPTIONS (if no snapshot is given)" + )] + filter: SnapshotFilter, + + /// Snapshots to copy. If none is given, use filter to filter from all + /// snapshots. + #[clap(value_name = "ID")] + ids: Vec, +} + +pub(super) fn execute( + repo: OpenRepository, + mut opts: Opts, + config_file: RusticConfig, +) -> Result<()> { + config_file.merge_into("snapshot-filter", &mut opts.filter)?; + + let target_opts: Vec = config_file.get("copy.targets")?; + if target_opts.is_empty() { + bail!("no [[copy.targets]] section in config file found!"); + } + + let be = &repo.dbe; + let mut snapshots = match opts.ids.is_empty() { + true => SnapshotFile::all_from_backend(be, &opts.filter)?, + false => SnapshotFile::from_ids(be, &opts.ids)?, + }; + // sort for nicer output + snapshots.sort_unstable(); + + let be = &repo.dbe; + let index = IndexBackend::new(be, progress_counter(""))?; + + let poly = repo.config.poly()?; + + for target_opt in target_opts { + let repo_dest = Repository::new(target_opt)?.open()?; + info!("copying to target {}...", repo_dest.name); + if poly != repo_dest.config.poly()? { + bail!("cannot copy to repository with different chunker parameter (re-chunking not implemented)!"); + } + copy(&snapshots, index.clone(), repo_dest, &opts)?; + } + Ok(()) +} + +fn copy( + snapshots: &[SnapshotFile], + index: impl IndexedBackend, + repo_dest: OpenRepository, + opts: &Opts, +) -> Result<()> { + let be_dest = &repo_dest.dbe; + + let snapshots = relevant_snapshots(snapshots, &repo_dest, &opts.filter)?; + match (snapshots.len(), opts.dry_run) { + (count, true) => { + info!("would have copied {count} snapshots"); + return Ok(()); + } + (0, false) => { + info!("no snapshot to copy."); + return Ok(()); + } + _ => {} // continue + } + + let snap_trees: Vec<_> = snapshots.iter().map(|sn| sn.tree).collect(); + + let index_dest = IndexBackend::new(be_dest, progress_counter(""))?; + let indexer = Indexer::new(be_dest.clone()).into_shared(); + + let data_packer = Packer::new( + be_dest.clone(), + BlobType::Data, + indexer.clone(), + &repo_dest.config, + index.total_size(&BlobType::Data), + )?; + let tree_packer = Packer::new( + be_dest.clone(), + BlobType::Tree, + indexer.clone(), + &repo_dest.config, + index.total_size(&BlobType::Tree), + )?; + + let p = progress_counter("copying blobs in snapshots..."); + + snap_trees.par_iter().try_for_each(|id| -> Result<_> { + trace!("copy tree blob {id}"); + if !index_dest.has_tree(id) { + let data = index.get_tree(id).unwrap().read_data(index.be())?; + tree_packer.add(&data, id)?; + } + Ok(()) + })?; + + let tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?; + tree_streamer + .par_bridge() + .try_for_each(|item| -> Result<_> { + let (_, tree) = item?; + tree.nodes().par_iter().try_for_each(|node| { + match node.node_type() { + NodeType::File => { + node.content().par_iter().try_for_each(|id| -> Result<_> { + trace!("copy data blob {id}"); + if !index_dest.has_data(id) { + let data = index.get_data(id).unwrap().read_data(index.be())?; + data_packer.add(&data, id)?; + } + Ok(()) + })?; + } + + NodeType::Dir => { + let id = node.subtree().unwrap(); + trace!("copy tree blob {id}"); + if !index_dest.has_tree(&id) { + let data = index.get_tree(&id).unwrap().read_data(index.be())?; + tree_packer.add(&data, &id)?; + } + } + + _ => {} // nothing to copy + } + Ok(()) + }) + })?; + + data_packer.finalize()?; + tree_packer.finalize()?; + indexer.write().unwrap().finalize()?; + + let p = progress_counter("saving snapshots..."); + be_dest.save_list(snapshots, p)?; + Ok(()) +} + +fn relevant_snapshots( + snaps: &[SnapshotFile], + dest_repo: &OpenRepository, + filter: &SnapshotFilter, +) -> Result> { + // save snapshots in destination in BTreeSet, as we want to efficiently search within to filter out already exising snapshots before copying. + let snapshots_dest: BTreeSet<_> = SnapshotFile::all_from_backend(&dest_repo.dbe, filter)? + .into_iter() + .map(remove_ids) + .collect(); + let mut table = table_with_titles(["ID", "Time", "Host", "Label", "Tags", "Paths", "Status"]); + let snaps = snaps + .iter() + .cloned() + .map(|sn| (sn.id, remove_ids(sn))) + .filter_map(|(id, sn)| { + let relevant = !snapshots_dest.contains(&sn); + let tags = sn.tags.formatln(); + let paths = sn.paths.formatln(); + let time = sn.time.format("%Y-%m-%d %H:%M:%S").to_string(); + table.add_row([ + &id.to_string(), + &time, + &sn.hostname, + &sn.label, + &tags, + &paths, + &(if relevant { "to copy" } else { "existing" }).to_string(), + ]); + relevant.then_some(sn) + }) + .collect(); + println!("{table}"); + + Ok(snaps) +} + +// remove ids which are not saved by the copy command (and not compared when checking if snapshots already exist in the copy target) +fn remove_ids(mut sn: SnapshotFile) -> SnapshotFile { + sn.id = Id::default(); + sn.parent = None; + sn +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 639ec47..ee834ef 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -18,6 +18,7 @@ mod cat; mod check; mod completions; mod config; +mod copy; mod diff; mod forget; mod helpers; @@ -91,6 +92,9 @@ enum Command { /// Check the repository Check(check::Opts), + /// Copy snapshots to another repository + Copy(copy::Opts), + /// Compare two snapshots/paths /// /// Note that the exclude options only apply for comparison with a local path @@ -205,6 +209,7 @@ pub fn execute() -> Result<()> { Command::Cat(opts) => cat::execute(repo, opts, config_file)?, Command::Check(opts) => check::execute(repo, opts)?, Command::Completions(_) => {} // already handled above + Command::Copy(opts) => copy::execute(repo, opts, config_file)?, Command::Diff(opts) => diff::execute(repo, opts, config_file)?, Command::Forget(opts) => forget::execute(repo, opts, config_file)?, Command::Init(_) => {} // already handled above