From b7b776b47214b752fc4c1f054cfcc5a4667eeab2 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Fri, 3 Feb 2023 12:11:08 +0100 Subject: [PATCH] Allow multiple sources within one snapshot --- changelog/new.txt | 4 +- src/backend/ignore.rs | 11 +++--- src/commands/backup.rs | 71 +++++++++++++++++++++++++----------- src/commands/diff.rs | 2 +- src/repofile/snapshotfile.rs | 71 ++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 30 deletions(-) diff --git a/changelog/new.txt b/changelog/new.txt index 593f45a..9e062b6 100644 --- a/changelog/new.txt +++ b/changelog/new.txt @@ -2,8 +2,10 @@ Changes in version x.x.x: Breaking changes: - Repository options in the config file must now be given under the `[repository]` section. +- Backing up multiple sources on the command line now results in one instead of several snapshots. Bugs fixed: New features: -- Extra or wrong fields in the config file now lead to rustic complaining and aborting. \ No newline at end of file +- Extra or wrong fields in the config file now lead to rustic complaining and aborting. +- backup: Paths are now sanitized from command arguments and config file before matching and applying the configuration. diff --git a/src/backend/ignore.rs b/src/backend/ignore.rs index 475b57c..12af56e 100644 --- a/src/backend/ignore.rs +++ b/src/backend/ignore.rs @@ -79,13 +79,12 @@ pub struct LocalSourceOptions { } impl LocalSource { - pub fn new(opts: LocalSourceOptions, backup_path: PathBuf) -> Result { - let mut walk_builder = WalkBuilder::new(backup_path); - /* - for path in &paths[1..] { - wb.add(path); + pub fn new(opts: LocalSourceOptions, backup_paths: &[impl AsRef]) -> Result { + let mut walk_builder = WalkBuilder::new(&backup_paths[0]); + + for path in &backup_paths[1..] { + walk_builder.add(path); } - */ let mut override_builder = OverrideBuilder::new("/"); diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 9271105..9be1212 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use std::str::FromStr; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use chrono::{Duration, Local}; use clap::{AppSettings, Parser}; use gethostname::gethostname; @@ -18,7 +18,8 @@ use crate::backend::{DryRunBackend, LocalSource, LocalSourceOptions, ReadSource} use crate::blob::{Metadata, Node, NodeType}; use crate::index::IndexBackend; use crate::repofile::{ - DeleteOption, SnapshotFile, SnapshotGroup, SnapshotGroupCriterion, SnapshotSummary, StringList, + DeleteOption, PathList, SnapshotFile, SnapshotGroup, SnapshotGroupCriterion, SnapshotSummary, + StringList, }; use crate::repository::OpenRepository; @@ -138,11 +139,25 @@ pub(super) fn execute( let mut config_opts: Vec = config_file.get("backup.sources")?; + let config_sources: Vec<_> = config_opts + .iter() + .filter_map(|opt| match PathList::from_string(&opt.source) { + Ok(paths) => Some(paths), + Err(err) => { + warn!( + "error sanitizing source=\"{}\" in config file: {err}", + opt.source + ); + None + } + }) + .collect(); + let sources = match (opts.cli_sources.is_empty(), config_opts.is_empty()) { - (false, _) => opts.cli_sources.clone(), + (false, _) => vec![PathList::from_strings(&opts.cli_sources)?], (true, false) => { info!("using all backup sources from config file."); - config_opts.iter().map(|opt| opt.source.clone()).collect() + config_sources.clone() } (true, true) => { warn!("no backup source given."); @@ -154,14 +169,25 @@ pub(super) fn execute( for source in sources { let mut opts = opts.clone(); + let index = index.clone(); + let backup_stdin = source == PathList::from_string("-")?; + let backup_path = if backup_stdin { + vec![PathBuf::from(&opts.stdin_filename)] + } else { + source.paths() + }; // merge Options from config file, if given - if let Some(idx) = config_opts.iter().position(|opt| opt.source == *source) { - info!("merging source=\"{source}\" section from config file"); + if let Some(idx) = config_sources.iter().position(|s| s == &source) { + info!("merging source={source} section from config file"); opts.merge(config_opts.remove(idx)); } - // merge Options from config file using as_path, if given if let Some(path) = &opts.as_path { + // as_path only works in combination with a single target + if source.len() > 1 { + bail!("as-path only works with a single target!"); + } + // merge Options from config file using as_path, if given if let Some(path) = path.as_os_str().to_str() { if let Some(idx) = config_opts.iter().position(|opt| opt.source == path) { info!("merging source=\"{path}\" section from config file"); @@ -173,23 +199,24 @@ pub(super) fn execute( config_file.merge_into("backup", &mut opts)?; let be = DryRunBackend::new(repo.dbe.clone(), opts.dry_run); - info!("starting to backup \"{source}\"..."); - let index = index.clone(); - let backup_stdin = source == "-"; - let backup_path = if backup_stdin { - PathBuf::from(&opts.stdin_filename) - } else { - PathBuf::from(&source).parse_dot()?.to_path_buf() - }; + info!("starting to backup {source}..."); let as_path = match opts.as_path { None => None, Some(p) => Some(p.parse_dot()?.to_path_buf()), }; - let backup_path_str = as_path.as_ref().unwrap_or(&backup_path); + let backup_path_str = match &as_path { + Some(as_path) => vec![as_path], + None => backup_path.iter().collect(), + }; let backup_path_str = backup_path_str - .to_str() - .ok_or_else(|| anyhow!("non-unicode path {:?}", backup_path_str))? - .to_string(); + .iter() + .map(|p| { + Ok(p.to_str() + .ok_or_else(|| anyhow!("non-unicode path {:?}", backup_path_str))? + .to_string()) + }) + .collect::>>()? + .join("\n"); let hostname = match opts.host { Some(host) => host, @@ -278,7 +305,7 @@ pub(super) fn execute( p.finish_with_message("done"); snap } else { - let src = LocalSource::new(opts.ignore_opts.clone(), backup_path.clone())?; + let src = LocalSource::new(opts.ignore_opts.clone(), &backup_path)?; let p = progress_bytes("determining size..."); if !p.is_hidden() { @@ -296,7 +323,7 @@ pub(super) fn execute( let snapshot_path = if let Some(as_path) = &as_path { as_path .clone() - .join(path.strip_prefix(&backup_path).unwrap()) + .join(path.strip_prefix(&backup_path[0]).unwrap()) } else { path.clone() }; @@ -340,7 +367,7 @@ pub(super) fn execute( println!("snapshot {} successfully saved.", snap.id); } - info!("backup of \"{source}\" done."); + info!("backup of {source} done."); } Ok(()) diff --git a/src/commands/diff.rs b/src/commands/diff.rs index 26bcb47..237908d 100644 --- a/src/commands/diff.rs +++ b/src/commands/diff.rs @@ -85,7 +85,7 @@ pub(super) fn execute( .metadata() .with_context(|| format!("Error accessing {path2:?}"))? .is_dir(); - let src = LocalSource::new(opts.ignore_opts, path2.clone())?.map(|item| { + let src = LocalSource::new(opts.ignore_opts, &[&path2])?.map(|item| { let (path, node) = item?; let path = if is_dir { // remove given path prefix for dirs as local path diff --git a/src/repofile/snapshotfile.rs b/src/repofile/snapshotfile.rs index 6be8687..0d0e53d 100644 --- a/src/repofile/snapshotfile.rs +++ b/src/repofile/snapshotfile.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::path::PathBuf; use std::str::FromStr; use std::{cmp::Ordering, fmt::Display}; @@ -10,6 +11,7 @@ use indicatif::ProgressBar; use itertools::Itertools; use log::*; use merge::Merge; +use path_dedot::ParseDot; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; @@ -467,3 +469,72 @@ impl StringList { self.0.join("\n") } } + +#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub struct PathList(Vec); + +impl Display for PathList { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if !self.0.is_empty() { + write!(f, "{:?}", self.0[0])?; + } + for p in &self.0[1..] { + write!(f, ",{:?}", p)?; + } + Ok(()) + } +} + +impl PathList { + pub fn from_strings(source: I) -> Result + where + I: IntoIterator, + I::Item: AsRef, + { + let mut paths = PathList( + source + .into_iter() + .map(|source| Ok(PathBuf::from(source.as_ref()).parse_dot()?.to_path_buf())) + .collect::>()?, + ); + paths.merge_paths()?; + Ok(paths) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn from_string(sources: &str) -> Result { + Self::from_strings(sources.split_whitespace()) + } + + pub fn paths(&self) -> Vec { + self.0.clone() + } + + // sort paths and filters out subpaths of already existing paths + fn merge_paths(&mut self) -> Result<()> { + if self.0.iter().any(|p| p.is_absolute()) { + for path in &mut self.0 { + *path = path.canonicalize()?; + } + } + + // sort paths + self.0.sort_unstable(); + + let mut root_path = None; + + // filter out subpaths + self.0.retain(|path| match &root_path { + Some(root_path) if path.starts_with(root_path) => false, + _ => { + root_path = Some(path.to_path_buf()); + true + } + }); + + Ok(()) + } +}