diff --git a/examples/ovh-hot-cold.toml b/examples/ovh-hot-cold.toml index e25f78f..2ce2489 100644 --- a/examples/ovh-hot-cold.toml +++ b/examples/ovh-hot-cold.toml @@ -1,4 +1,5 @@ # rustic config file to backup /home, /etc and /root to a hot/cold repository hosted by OVH +# using OVH cloud archive and OVH object storage # # backup usage: "rustic -P ovh-hot-cold backup # cleanup: "rustic -P ovh-hot-cold forget --prune @@ -8,6 +9,8 @@ repository = "rclone:ovh:backup-home" repo-hot = "rclone:ovh:backup-home-hot" password-file = "/root/key-rustic-ovh" cache-dir = "/var/lib/cache/rustic" # explicitely specify cache dir for remote repository +warm-up = true # cold storage needs warm-up, just trying to access a file is sufficient to start the warm-up +warm-up-wait = "10m" # in my examples, 10 minutes wait-time was sufficient, according to docu it can be up to 12h [forget] keep-daily = 8 diff --git a/src/commands/helpers.rs b/src/commands/helpers.rs index 0d2fdc8..0e77751 100644 --- a/src/commands/helpers.rs +++ b/src/commands/helpers.rs @@ -4,7 +4,7 @@ use std::process::Command; use std::str::FromStr; use std::time::Duration; -use anyhow::{bail, Result}; +use anyhow::Result; use bytesize::ByteSize; use comfy_table::{ presets::ASCII_MARKDOWN, Attribute, Cell, CellAlignment, ContentArrangement, Table, @@ -14,8 +14,9 @@ use indicatif::{ProgressBar, ProgressState, ProgressStyle}; use log::*; use rayon::ThreadPoolBuilder; -use crate::backend::{DecryptReadBackend, FileType}; +use crate::backend::{FileType, ReadBackend}; use crate::repofile::Id; +use crate::repository::{parse_command, OpenRepository}; pub fn bytes(b: u64) -> String { ByteSize(b).to_string_as(true) @@ -74,28 +75,45 @@ pub fn progress_bytes(prefix: impl Into>) -> ProgressBar { p } +pub fn warm_up_wait( + repo: &OpenRepository, + packs: impl ExactSizeIterator, + wait: bool, +) -> Result<()> { + if repo.opts.warm_up_command.is_some() { + warm_up_command(packs, repo.opts.warm_up_command.as_ref().unwrap())?; + } else if repo.opts.warm_up { + warm_up(&repo.be, packs)?; + } + if wait { + if let Some(wait) = repo.opts.warm_up_wait { + let p = progress_spinner(format!("waiting {}...", wait)); + std::thread::sleep(*wait); + p.finish(); + } + } + Ok(()) +} + pub fn warm_up_command(packs: impl ExactSizeIterator, command: &str) -> Result<()> { let p = progress_counter("warming up packs..."); p.set_length(packs.len() as u64); for pack in packs { let actual_command = command.replace("%id", &pack.to_hex()); debug!("calling {actual_command}..."); - let mut commands: Vec<_> = actual_command.split(' ').collect(); + let mut commands = parse_command::<()>(&actual_command)?.1; let status = Command::new(commands[0]) .args(&mut commands[1..]) .status()?; if !status.success() { - bail!("warm-up command was not successful for pack {pack:?}. {status}"); + warn!("warm-up command was not successful for pack {pack:?}. {status}"); } } p.finish(); Ok(()) } -pub fn warm_up( - be: &impl DecryptReadBackend, - packs: impl ExactSizeIterator, -) -> Result<()> { +pub fn warm_up(be: &impl ReadBackend, packs: impl ExactSizeIterator) -> Result<()> { let mut be = be.clone(); be.set_option("retry", "false")?; @@ -121,14 +139,6 @@ pub fn warm_up( Ok(()) } -pub fn wait(d: Option) { - if let Some(wait) = d { - let p = progress_spinner(format!("waiting {}...", wait)); - std::thread::sleep(*wait); - p.finish(); - } -} - // Helpers for table output pub fn bold_cell(s: T) -> Cell { diff --git a/src/commands/prune.rs b/src/commands/prune.rs index f4ad6fb..083b1b4 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -12,7 +12,7 @@ use itertools::Itertools; use log::*; use rayon::prelude::*; -use super::{bytes, no_progress, progress_bytes, progress_counter, wait, warm_up, warm_up_command}; +use super::{bytes, no_progress, progress_bytes, progress_counter, warm_up_wait}; use crate::backend::{DecryptReadBackend, DecryptWriteBackend, FileType, ReadBackend}; use crate::blob::{ BlobType, BlobTypeMap, Initialize, NodeType, PackSizer, Repacker, Sum, TreeStreamerOnce, @@ -69,18 +69,6 @@ pub(super) struct Opts { /// Do not repack packs which only needs to be resized #[clap(long)] no_resize: bool, - - /// Warm up needed data pack files by only requesting them without processing - #[clap(long)] - warm_up: bool, - - /// Warm up needed data pack files by running the command with %id replaced by pack id - #[clap(long, conflicts_with = "warm-up")] - warm_up_command: Option, - - /// Duration (e.g. 10m) to wait after warm up before doing the actual restore - #[clap(long, value_name = "DURATION", conflicts_with = "dry-run")] - warm_up_wait: Option, } pub(super) fn execute(repo: OpenRepository, opts: Opts, ignore_snaps: Vec) -> Result<()> { @@ -143,15 +131,7 @@ pub(super) fn execute(repo: OpenRepository, opts: Opts, ignore_snaps: Vec) - pruner.filter_index_files(opts.instant_delete); pruner.print_stats(); - if opts.warm_up { - warm_up(be, pruner.repack_packs().into_iter())?; - } else if opts.warm_up_command.is_some() { - warm_up_command( - pruner.repack_packs().into_iter(), - opts.warm_up_command.as_ref().unwrap(), - )?; - } - wait(opts.warm_up_wait); + warm_up_wait(&repo, pruner.repack_packs().into_iter(), !opts.dry_run)?; if !opts.dry_run { pruner.do_prune(repo, opts)?; diff --git a/src/commands/repair.rs b/src/commands/repair.rs index 2e22d9b..ff2a987 100644 --- a/src/commands/repair.rs +++ b/src/commands/repair.rs @@ -4,7 +4,10 @@ use anyhow::Result; use clap::{AppSettings, Parser, Subcommand}; use log::*; -use crate::backend::{DecryptFullBackend, DecryptWriteBackend, FileType}; +use crate::backend::{ + DecryptFullBackend, DecryptReadBackend, DecryptWriteBackend, FileType, ReadBackend, + WriteBackend, +}; use crate::blob::{BlobType, NodeType, Packer, Tree}; use crate::id::Id; use crate::index::{IndexBackend, IndexedBackend, Indexer, ReadIndex}; @@ -15,7 +18,7 @@ use crate::repofile::{ use crate::repository::OpenRepository; use super::rustic_config::RusticConfig; -use super::{progress_counter, progress_spinner, wait, warm_up, warm_up_command}; +use super::{progress_counter, progress_spinner, warm_up_wait}; #[derive(Parser)] pub(super) struct Opts { @@ -40,18 +43,6 @@ struct IndexOpts { // Read all data packs, i.e. completely re-create the index #[clap(long)] read_all: bool, - - /// Warm up needed data pack files by only requesting them without processing - #[clap(long)] - warm_up: bool, - - /// Warm up needed data pack files by running the command with %id replaced by pack id - #[clap(long, conflicts_with = "warm-up")] - warm_up_command: Option, - - /// Duration (e.g. 10m) to wait after warm up before doing the actual restore - #[clap(long, value_name = "DURATION", conflicts_with = "dry-run")] - warm_up_wait: Option, } #[derive(Default, Parser)] @@ -83,12 +74,13 @@ struct SnapOpts { pub(super) fn execute(repo: OpenRepository, opts: Opts, config_file: RusticConfig) -> Result<()> { match opts.command { - Command::Index(opt) => repair_index(&repo.dbe, opt), + Command::Index(opt) => repair_index(&repo, opt), Command::Snapshots(opt) => repair_snaps(&repo.dbe, opt, config_file, &repo.config), } } -fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> { +fn repair_index(repo: &OpenRepository, opts: IndexOpts) -> Result<()> { + let be = &repo.dbe; let p = progress_spinner("listing packs..."); let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect(); p.finish(); @@ -162,21 +154,7 @@ fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> { // process packs which are listed but not contained in the index pack_read_header.extend(packs.into_iter().map(|(id, size)| (id, false, None, size))); - if opts.warm_up { - warm_up(be, pack_read_header.iter().map(|(id, _, _, _)| *id))?; - if opts.dry_run { - return Ok(()); - } - } else if opts.warm_up_command.is_some() { - warm_up_command( - pack_read_header.iter().map(|(id, _, _, _)| *id), - opts.warm_up_command.as_ref().unwrap(), - )?; - if opts.dry_run { - return Ok(()); - } - } - wait(opts.warm_up_wait); + warm_up_wait(repo, pack_read_header.iter().map(|(id, _, _, _)| *id), true)?; let indexer = Indexer::new(be.clone()).into_shared(); let p = progress_counter("reading pack headers"); diff --git a/src/commands/restore.rs b/src/commands/restore.rs index b6797c7..d847519 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -14,7 +14,7 @@ use log::*; use rayon::ThreadPoolBuilder; use super::rustic_config::RusticConfig; -use super::{bytes, progress_bytes, progress_counter, wait, warm_up, warm_up_command}; +use super::{bytes, progress_bytes, progress_counter, warm_up_wait}; use crate::backend::{DecryptReadBackend, FileType, LocalBackend}; use crate::blob::{Node, NodeStreamer, NodeType, Tree}; use crate::commands::helpers::progress_spinner; @@ -117,15 +117,7 @@ pub(super) fn execute( if file_infos.restore_size == 0 { info!("all file contents are fine."); } else { - if opts.warm_up { - warm_up(be, file_infos.to_packs().into_iter())?; - } else if opts.warm_up_command.is_some() { - warm_up_command( - file_infos.to_packs().into_iter(), - opts.warm_up_command.as_ref().unwrap(), - )?; - } - wait(opts.warm_up_wait); + warm_up_wait(&repo, file_infos.to_packs().into_iter(), !opts.dry_run)?; if !opts.dry_run { restore_contents(be, &dest, file_infos)?; } diff --git a/src/repository/mod.rs b/src/repository/mod.rs index fcfa6d2..65f564d 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -18,6 +18,7 @@ use nom::{ }; use rpassword::{prompt_password, read_password_from_bufread}; use serde::Deserialize; +use serde_with::{serde_as, DisplayFromStr}; use crate::backend::{ Cache, CachedBackend, ChooseBackend, DecryptBackend, DecryptReadBackend, DecryptWriteBackend, @@ -26,6 +27,7 @@ use crate::backend::{ use crate::crypto::Key; use crate::repofile::{find_key_in_backend, ConfigFile}; +#[serde_as] #[derive(Default, Parser, Deserialize, Merge)] #[serde(default, rename_all = "kebab-case")] pub struct RepositoryOptions { @@ -75,10 +77,26 @@ pub struct RepositoryOptions { env = "RUSTIC_CACHE_DIR" )] cache_dir: Option, + + /// Warm up needed data pack files by only requesting them without processing + #[clap(long, global = true)] + #[merge(strategy = merge::bool::overwrite_false)] + pub(crate) warm_up: bool, + + /// Warm up needed data pack files by running the command with %id replaced by pack id + #[clap(long, global = true, conflicts_with = "warm-up")] + pub(crate) warm_up_command: Option, + + /// Duration (e.g. 10m) to wait after warm up + #[clap(long, global = true, value_name = "DURATION")] + #[serde_as(as = "Option")] + pub(crate) warm_up_wait: Option, } // parse a command -fn parse_command<'a, E: ParseError<&'a str>>(input: &'a str) -> IResult<&'a str, Vec<&'a str>, E> { +pub(crate) fn parse_command<'a, E: ParseError<&'a str>>( + input: &'a str, +) -> IResult<&'a str, Vec<&'a str>, E> { separated_list0( // a command is a list multispace1, // separated by one or more spaces @@ -209,7 +227,7 @@ impl Repository { _ => {} } let cache = (!self.opts.no_cache) - .then(|| Cache::new(config.id, self.opts.cache_dir).ok()) + .then(|| Cache::new(config.id, self.opts.cache_dir.clone()).ok()) .flatten(); match &cache { None => info!("using no cache"), @@ -228,6 +246,7 @@ impl Repository { be: self.be, be_hot: self.be_hot, config, + opts: self.opts, }) } } @@ -240,6 +259,7 @@ pub struct OpenRepository { pub(crate) cache: Option, pub(crate) dbe: DecryptBackend>, Key>, pub(crate) config: ConfigFile, + pub(crate) opts: RepositoryOptions, } const MAX_PASSWORD_RETRIES: usize = 5;