Merge pull request #374 from rustic-rs/repo-warm-up

repository: integrate warm-up options

examples/ovh-hot-cold.toml

@@ -1,4 +1,5 @@
 # rustic config file to backup /home, /etc and /root to a hot/cold repository hosted by OVH
+# using OVH cloud archive and OVH object storage
 #
 # backup usage: "rustic -P ovh-hot-cold backup
 # cleanup:      "rustic -P ovh-hot-cold forget --prune
@@ -8,6 +9,8 @@ repository = "rclone:ovh:backup-home"
 repo-hot = "rclone:ovh:backup-home-hot"
 password-file =  "/root/key-rustic-ovh"
 cache-dir = "/var/lib/cache/rustic" # explicitely specify cache dir for remote repository 
+warm-up = true # cold storage needs warm-up, just trying to access a file is sufficient to start the warm-up
+warm-up-wait = "10m" # in my examples, 10 minutes wait-time was sufficient, according to docu it can be up to 12h
 
 [forget]
 keep-daily = 8

src/commands/helpers.rs

@@ -4,7 +4,7 @@ use std::process::Command;
 use std::str::FromStr;
 use std::time::Duration;
 
-use anyhow::{bail, Result};
+use anyhow::Result;
 use bytesize::ByteSize;
 use comfy_table::{
     presets::ASCII_MARKDOWN, Attribute, Cell, CellAlignment, ContentArrangement, Table,
@@ -14,8 +14,9 @@ use indicatif::{ProgressBar, ProgressState, ProgressStyle};
 use log::*;
 use rayon::ThreadPoolBuilder;
 
-use crate::backend::{DecryptReadBackend, FileType};
+use crate::backend::{FileType, ReadBackend};
 use crate::repofile::Id;
+use crate::repository::{parse_command, OpenRepository};
 
 pub fn bytes(b: u64) -> String {
     ByteSize(b).to_string_as(true)
@@ -74,28 +75,45 @@ pub fn progress_bytes(prefix: impl Into<Cow<'static, str>>) -> ProgressBar {
     p
 }
 
+pub fn warm_up_wait(
+    repo: &OpenRepository,
+    packs: impl ExactSizeIterator<Item = Id>,
+    wait: bool,
+) -> Result<()> {
+    if repo.opts.warm_up_command.is_some() {
+        warm_up_command(packs, repo.opts.warm_up_command.as_ref().unwrap())?;
+    } else if repo.opts.warm_up {
+        warm_up(&repo.be, packs)?;
+    }
+    if wait {
+        if let Some(wait) = repo.opts.warm_up_wait {
+            let p = progress_spinner(format!("waiting {}...", wait));
+            std::thread::sleep(*wait);
+            p.finish();
+        }
+    }
+    Ok(())
+}
+
 pub fn warm_up_command(packs: impl ExactSizeIterator<Item = Id>, command: &str) -> Result<()> {
     let p = progress_counter("warming up packs...");
     p.set_length(packs.len() as u64);
     for pack in packs {
         let actual_command = command.replace("%id", &pack.to_hex());
         debug!("calling {actual_command}...");
-        let mut commands: Vec<_> = actual_command.split(' ').collect();
+        let mut commands = parse_command::<()>(&actual_command)?.1;
         let status = Command::new(commands[0])
             .args(&mut commands[1..])
             .status()?;
         if !status.success() {
-            bail!("warm-up command was not successful for pack {pack:?}. {status}");
+            warn!("warm-up command was not successful for pack {pack:?}. {status}");
         }
     }
     p.finish();
     Ok(())
 }
 
-pub fn warm_up(
-    be: &impl DecryptReadBackend,
-    packs: impl ExactSizeIterator<Item = Id>,
-) -> Result<()> {
+pub fn warm_up(be: &impl ReadBackend, packs: impl ExactSizeIterator<Item = Id>) -> Result<()> {
     let mut be = be.clone();
     be.set_option("retry", "false")?;
 
@@ -121,14 +139,6 @@ pub fn warm_up(
     Ok(())
 }
 
-pub fn wait(d: Option<humantime::Duration>) {
-    if let Some(wait) = d {
-        let p = progress_spinner(format!("waiting {}...", wait));
-        std::thread::sleep(*wait);
-        p.finish();
-    }
-}
-
 // Helpers for table output
 
 pub fn bold_cell<T: ToString>(s: T) -> Cell {

src/commands/prune.rs

@@ -12,7 +12,7 @@ use itertools::Itertools;
 use log::*;
 use rayon::prelude::*;
 
-use super::{bytes, no_progress, progress_bytes, progress_counter, wait, warm_up, warm_up_command};
+use super::{bytes, no_progress, progress_bytes, progress_counter, warm_up_wait};
 use crate::backend::{DecryptReadBackend, DecryptWriteBackend, FileType, ReadBackend};
 use crate::blob::{
     BlobType, BlobTypeMap, Initialize, NodeType, PackSizer, Repacker, Sum, TreeStreamerOnce,
@@ -69,18 +69,6 @@ pub(super) struct Opts {
     /// Do not repack packs which only needs to be resized
     #[clap(long)]
     no_resize: bool,
-
-    /// Warm up needed data pack files by only requesting them without processing
-    #[clap(long)]
-    warm_up: bool,
-
-    /// Warm up needed data pack files by running the command with %id replaced by pack id
-    #[clap(long, conflicts_with = "warm-up")]
-    warm_up_command: Option<String>,
-
-    /// Duration (e.g. 10m) to wait after warm up before doing the actual restore
-    #[clap(long, value_name = "DURATION", conflicts_with = "dry-run")]
-    warm_up_wait: Option<humantime::Duration>,
 }
 
 pub(super) fn execute(repo: OpenRepository, opts: Opts, ignore_snaps: Vec<Id>) -> Result<()> {
@@ -143,15 +131,7 @@ pub(super) fn execute(repo: OpenRepository, opts: Opts, ignore_snaps: Vec<Id>) -
     pruner.filter_index_files(opts.instant_delete);
     pruner.print_stats();
 
-    if opts.warm_up {
-        warm_up(be, pruner.repack_packs().into_iter())?;
-    } else if opts.warm_up_command.is_some() {
-        warm_up_command(
-            pruner.repack_packs().into_iter(),
-            opts.warm_up_command.as_ref().unwrap(),
-        )?;
-    }
-    wait(opts.warm_up_wait);
+    warm_up_wait(&repo, pruner.repack_packs().into_iter(), !opts.dry_run)?;
 
     if !opts.dry_run {
         pruner.do_prune(repo, opts)?;

src/commands/repair.rs

@@ -4,7 +4,10 @@ use anyhow::Result;
 use clap::{AppSettings, Parser, Subcommand};
 use log::*;
 
-use crate::backend::{DecryptFullBackend, DecryptWriteBackend, FileType};
+use crate::backend::{
+    DecryptFullBackend, DecryptReadBackend, DecryptWriteBackend, FileType, ReadBackend,
+    WriteBackend,
+};
 use crate::blob::{BlobType, NodeType, Packer, Tree};
 use crate::id::Id;
 use crate::index::{IndexBackend, IndexedBackend, Indexer, ReadIndex};
@@ -15,7 +18,7 @@ use crate::repofile::{
 use crate::repository::OpenRepository;
 
 use super::rustic_config::RusticConfig;
-use super::{progress_counter, progress_spinner, wait, warm_up, warm_up_command};
+use super::{progress_counter, progress_spinner, warm_up_wait};
 
 #[derive(Parser)]
 pub(super) struct Opts {
@@ -40,18 +43,6 @@ struct IndexOpts {
     // Read all data packs, i.e. completely re-create the index
     #[clap(long)]
     read_all: bool,
-
-    /// Warm up needed data pack files by only requesting them without processing
-    #[clap(long)]
-    warm_up: bool,
-
-    /// Warm up needed data pack files by running the command with %id replaced by pack id
-    #[clap(long, conflicts_with = "warm-up")]
-    warm_up_command: Option<String>,
-
-    /// Duration (e.g. 10m) to wait after warm up before doing the actual restore
-    #[clap(long, value_name = "DURATION", conflicts_with = "dry-run")]
-    warm_up_wait: Option<humantime::Duration>,
 }
 
 #[derive(Default, Parser)]
@@ -83,12 +74,13 @@ struct SnapOpts {
 
 pub(super) fn execute(repo: OpenRepository, opts: Opts, config_file: RusticConfig) -> Result<()> {
     match opts.command {
-        Command::Index(opt) => repair_index(&repo.dbe, opt),
+        Command::Index(opt) => repair_index(&repo, opt),
         Command::Snapshots(opt) => repair_snaps(&repo.dbe, opt, config_file, &repo.config),
     }
 }
 
-fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> {
+fn repair_index(repo: &OpenRepository, opts: IndexOpts) -> Result<()> {
+    let be = &repo.dbe;
     let p = progress_spinner("listing packs...");
     let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect();
     p.finish();
@@ -162,21 +154,7 @@ fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> {
     // process packs which are listed but not contained in the index
     pack_read_header.extend(packs.into_iter().map(|(id, size)| (id, false, None, size)));
 
-    if opts.warm_up {
-        warm_up(be, pack_read_header.iter().map(|(id, _, _, _)| *id))?;
-        if opts.dry_run {
-            return Ok(());
-        }
-    } else if opts.warm_up_command.is_some() {
-        warm_up_command(
-            pack_read_header.iter().map(|(id, _, _, _)| *id),
-            opts.warm_up_command.as_ref().unwrap(),
-        )?;
-        if opts.dry_run {
-            return Ok(());
-        }
-    }
-    wait(opts.warm_up_wait);
+    warm_up_wait(repo, pack_read_header.iter().map(|(id, _, _, _)| *id), true)?;
 
     let indexer = Indexer::new(be.clone()).into_shared();
     let p = progress_counter("reading pack headers");

src/commands/restore.rs

@@ -14,7 +14,7 @@ use log::*;
 use rayon::ThreadPoolBuilder;
 
 use super::rustic_config::RusticConfig;
-use super::{bytes, progress_bytes, progress_counter, wait, warm_up, warm_up_command};
+use super::{bytes, progress_bytes, progress_counter, warm_up_wait};
 use crate::backend::{DecryptReadBackend, FileType, LocalBackend};
 use crate::blob::{Node, NodeStreamer, NodeType, Tree};
 use crate::commands::helpers::progress_spinner;
@@ -117,15 +117,7 @@ pub(super) fn execute(
     if file_infos.restore_size == 0 {
         info!("all file contents are fine.");
     } else {
-        if opts.warm_up {
-            warm_up(be, file_infos.to_packs().into_iter())?;
-        } else if opts.warm_up_command.is_some() {
-            warm_up_command(
-                file_infos.to_packs().into_iter(),
-                opts.warm_up_command.as_ref().unwrap(),
-            )?;
-        }
-        wait(opts.warm_up_wait);
+        warm_up_wait(&repo, file_infos.to_packs().into_iter(), !opts.dry_run)?;
         if !opts.dry_run {
             restore_contents(be, &dest, file_infos)?;
         }

src/repository/mod.rs

@@ -18,6 +18,7 @@ use nom::{
 };
 use rpassword::{prompt_password, read_password_from_bufread};
 use serde::Deserialize;
+use serde_with::{serde_as, DisplayFromStr};
 
 use crate::backend::{
     Cache, CachedBackend, ChooseBackend, DecryptBackend, DecryptReadBackend, DecryptWriteBackend,
@@ -26,6 +27,7 @@ use crate::backend::{
 use crate::crypto::Key;
 use crate::repofile::{find_key_in_backend, ConfigFile};
 
+#[serde_as]
 #[derive(Default, Parser, Deserialize, Merge)]
 #[serde(default, rename_all = "kebab-case")]
 pub struct RepositoryOptions {
@@ -75,10 +77,26 @@ pub struct RepositoryOptions {
         env = "RUSTIC_CACHE_DIR"
     )]
     cache_dir: Option<PathBuf>,
+
+    /// Warm up needed data pack files by only requesting them without processing
+    #[clap(long, global = true)]
+    #[merge(strategy = merge::bool::overwrite_false)]
+    pub(crate) warm_up: bool,
+
+    /// Warm up needed data pack files by running the command with %id replaced by pack id
+    #[clap(long, global = true, conflicts_with = "warm-up")]
+    pub(crate) warm_up_command: Option<String>,
+
+    /// Duration (e.g. 10m) to wait after warm up
+    #[clap(long, global = true, value_name = "DURATION")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub(crate) warm_up_wait: Option<humantime::Duration>,
 }
 
 // parse a command
-fn parse_command<'a, E: ParseError<&'a str>>(input: &'a str) -> IResult<&'a str, Vec<&'a str>, E> {
+pub(crate) fn parse_command<'a, E: ParseError<&'a str>>(
+    input: &'a str,
+) -> IResult<&'a str, Vec<&'a str>, E> {
     separated_list0(
         // a command is a list
         multispace1, // separated by one or more spaces
@@ -209,7 +227,7 @@ impl Repository {
                 _ => {}
             }
         let cache = (!self.opts.no_cache)
-            .then(|| Cache::new(config.id, self.opts.cache_dir).ok())
+            .then(|| Cache::new(config.id, self.opts.cache_dir.clone()).ok())
             .flatten();
         match &cache {
             None => info!("using no cache"),
@@ -228,6 +246,7 @@ impl Repository {
             be: self.be,
             be_hot: self.be_hot,
             config,
+            opts: self.opts,
         })
     }
 }
@@ -240,6 +259,7 @@ pub struct OpenRepository {
     pub(crate) cache: Option<Cache>,
     pub(crate) dbe: DecryptBackend<CachedBackend<HotColdBackend<ChooseBackend>>, Key>,
     pub(crate) config: ConfigFile,
+    pub(crate) opts: RepositoryOptions,
 }
 
 const MAX_PASSWORD_RETRIES: usize = 5;