Merge pull request #604 from rustic-rs/rework-config

rework config implementation
This commit is contained in:
aawsome 2023-04-24 13:47:05 +02:00 committed by GitHub
commit 6a627aed25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 271 additions and 343 deletions

View File

@ -1,13 +1,16 @@
Changes in version x.x.x:
Breaking changes:
- The option `--config-profile` was renamed into `--use-profile`
Bugs fixed:
- restore: Warm-up options given by the command line didn't work. This has been fixed.
- backup showed 1 dir as changed when backing up without parent. This has been fixed.
- diff: The options --no-atime and --ignore-devid had no effect and are now removed.
- Rustic's check of additional fields in the config file didn't work in edge cases. This has been fixed.
New features:
- config file: New field use-profile allows to merge options from other config profiles
- backup: Backing up (small) files is now much more parallelized.
- forget: Using "-1" as value for --keep-* options will keep all snapshots of that interval
- prune: Added option --repack-all

View File

@ -7,6 +7,7 @@
# Global options: These options are used for all commands.
[global]
use-profile = ""
log-level = "info" # any of "off", "error", "warn", "info", "debug", "trace"; default: "info"
log-file = "/path/to/rustic.log" # Default: not set
no-progress = false
@ -32,7 +33,7 @@ warm-up-wait = "10min" # Default: not set
[repository.options]
post-create-command = "par2create -qq -n1 -r5 %file" # Only local backend; Default: not set
post-delete-command = "sh -c \"rm -f %file*.par2\"" # Only local backend; Default: not set
retry = true # Only rest/rclone backend
retry = "true" # Only rest/rclone backend
timeout = "2min" # Ony rest/rclone backend
# Snapshot-filter options: These options apply to all commands that use snapshot filters
@ -114,7 +115,7 @@ filter-host = ["host2", "host2"] # Default: no host filter
filter-label = ["label1", "label2"] # Default: no label filter
filter-tags = ["tag1,tag2", "tag3"] # Default: no tags filger
filter-paths = ["path1", "path2,path3"] # Default: no paths filter
filter-fn = '|sn| {sn.host == "host1" || sn.description.contains("test")}'' # Default: no filter function
filter-fn = '|sn| {sn.host == "host1" || sn.description.contains("test")}' # Default: no filter function
# The retention options follow. All of these are not set by default.
keep-tags = ["tag1", "tag2,tag3"]
keep-ids = ["123abc", "11122233"] # Keep all snapshots whose ID starts with any of these strings

View File

@ -8,9 +8,8 @@ use log::*;
use merge::Merge;
use path_dedot::ParseDot;
use serde::Deserialize;
use toml::Value;
use super::{bytes, progress_bytes, progress_counter, GlobalOpts, RusticConfig};
use super::{bytes, progress_bytes, progress_counter, Config};
use crate::archiver::Archiver;
use crate::backend::{
DryRunBackend, LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions, StdinSource,
@ -22,8 +21,13 @@ use crate::repofile::{
use crate::repository::OpenRepository;
#[derive(Clone, Default, Parser, Deserialize, Merge)]
#[serde(default, rename_all = "kebab-case")]
pub(super) struct Opts {
#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
// Note: using cli_sources, sources and source within this strict is a hack to support serde(deny_unknown_fields)
// for deserializing the backup options from TOML
// Unfortunately we cannot work with nested flattened structures, see
// https://github.com/serde-rs/serde/issues/1547
// A drawback is that a wrongly set "source(s) = ..." won't get correct error handling and need to be manually checked, see below.
pub struct Opts {
/// Backup source (can be specified multiple times), use - for stdin. If no source is given, uses all
/// sources defined in the config file
#[clap(value_name = "SOURCE")]
@ -103,15 +107,9 @@ pub(super) struct Opts {
#[merge(strategy = merge::bool::overwrite_false)]
json: bool,
// This is a hack to support serde(deny_unknown_fields) for deserializing the backup options from TOML
// while still being able to use [[backup.sources]] in the config file.
// A drawback is that a unkowen "sources = ..." won't be bailed...
// Note that unfortunately we cannot work with nested flattened structures, see
// https://github.com/serde-rs/serde/issues/1547
#[clap(skip)]
#[merge(skip)]
#[serde(rename = "sources")]
config_sources: Option<Value>,
sources: Vec<Opts>,
/// Backup source, used within config file
#[clap(skip)]
@ -121,14 +119,24 @@ pub(super) struct Opts {
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
mut config: Config,
opts: Opts,
config_file: RusticConfig,
command: String,
) -> Result<()> {
let time = Local::now();
let config_opts: Vec<Opts> = config_file.get("backup.sources")?;
// manually check for a "source" field, check is not done by serde, see above.
if !config.backup.source.is_empty() {
bail!("key \"source\" is not valid in the [backup] section!");
}
let config_opts = config.backup.sources;
config.backup.sources = Vec::new();
// manually check for a "sources" field, check is not done by serde, see above.
if config_opts.iter().any(|opt| !opt.sources.is_empty()) {
bail!("key \"sources\" is not valid in a [[backup.sources]] section!");
}
let config_sources: Vec<_> = config_opts
.iter()
@ -186,10 +194,11 @@ pub(super) fn execute(
}
}
}
// merge "backup" section from config file, if given
config_file.merge_into("backup", &mut opts)?;
let be = DryRunBackend::new(repo.dbe.clone(), gopts.dry_run);
// merge "backup" section from config file, if given
opts.merge(config.backup.clone());
let be = DryRunBackend::new(repo.dbe.clone(), config.global.dry_run);
info!("starting to backup {source}...");
let as_path = match opts.as_path {
None => None,

View File

@ -4,13 +4,12 @@ use anyhow::{anyhow, Result};
use clap::{Parser, Subcommand};
use indicatif::ProgressBar;
use super::progress_counter;
use super::rustic_config::RusticConfig;
use super::{progress_counter, Config};
use crate::backend::{DecryptReadBackend, FileType};
use crate::blob::{BlobType, Tree};
use crate::id::Id;
use crate::index::{IndexBackend, IndexedBackend};
use crate::repofile::{SnapshotFile, SnapshotFilter};
use crate::repofile::SnapshotFile;
use crate::repository::OpenRepository;
#[derive(Parser)]
@ -46,15 +45,9 @@ struct TreeOpts {
/// Snapshot/path of the tree to display
#[clap(value_name = "SNAPSHOT[:PATH]")]
snap: String,
#[clap(
flatten,
next_help_heading = "Snapshot filter options (when using latest)"
)]
filter: SnapshotFilter,
}
pub(super) fn execute(repo: OpenRepository, opts: Opts, config_file: RusticConfig) -> Result<()> {
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let be = &repo.dbe;
match opts.command {
Command::Config => cat_file(be, FileType::Config, IdOpt::default()),
@ -64,7 +57,7 @@ pub(super) fn execute(repo: OpenRepository, opts: Opts, config_file: RusticConfi
Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt),
Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt),
// special treatment for cating a tree within a snapshot
Command::Tree(opts) => cat_tree(be, opts, config_file),
Command::Tree(opts) => cat_tree(be, config, opts),
}
}
@ -84,15 +77,14 @@ fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Result<(
Ok(())
}
fn cat_tree(
be: &impl DecryptReadBackend,
mut opts: TreeOpts,
config_file: RusticConfig,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
fn cat_tree(be: &impl DecryptReadBackend, config: Config, opts: TreeOpts) -> Result<()> {
let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
let snap = SnapshotFile::from_str(be, id, |sn| sn.matches(&opts.filter), progress_counter(""))?;
let snap = SnapshotFile::from_str(
be,
id,
|sn| sn.matches(&config.snapshot_filter),
progress_counter(""),
)?;
let index = IndexBackend::new(be, progress_counter(""))?;
let node = Tree::node_from_path(&index, snap.tree, Path::new(path))?;
let id = node.subtree.ok_or_else(|| anyhow!("{path} is no dir"))?;

View File

@ -26,7 +26,7 @@ pub(super) fn execute(opts: Opts) {
}
fn generate_completion<G: Generator>(shell: G, buf: &mut dyn Write) {
let mut command = super::Opts::command();
let mut command = super::Args::command();
generate(shell, &mut command, env!("CARGO_BIN_NAME"), buf);
}

View File

@ -0,0 +1,64 @@
use std::path::Path;
use anyhow::{Context, Result};
use clap::Parser;
use directories::ProjectDirs;
use merge::Merge;
use serde::Deserialize;
use crate::{repofile::SnapshotFilter, repository::RepositoryOptions};
use super::{backup, copy, forget, GlobalOpts};
#[derive(Default, Parser, Deserialize, Merge)]
#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
pub struct Config {
#[clap(flatten, next_help_heading = "Global options")]
pub global: GlobalOpts,
#[clap(flatten, next_help_heading = "Repository options")]
pub repository: RepositoryOptions,
#[clap(flatten, next_help_heading = "Snapshot filter options")]
pub snapshot_filter: SnapshotFilter,
#[clap(skip)]
pub backup: backup::Opts,
#[clap(skip)]
pub copy: copy::Targets,
#[clap(skip)]
pub forget: forget::ConfigOpts,
}
impl Config {
pub fn merge_profile(&mut self, profile: &str) -> Result<()> {
let mut path = match ProjectDirs::from("", "", "rustic") {
Some(path) => path.config_dir().to_path_buf(),
None => Path::new(".").to_path_buf(),
};
if !path.exists() {
path = Path::new(".").to_path_buf();
};
let path = path.join(profile.to_string() + ".toml");
if path.exists() {
// TODO: This should be log::info! - however, the logging config
// can be stored in the config file and is needed to initialize the logger
eprintln!("using config {}", path.display());
let data = std::fs::read_to_string(path).context("error reading config file")?;
let mut config: Self =
toml::from_str(&data).context("error reading TOML from config file")?;
// if "use_profile" is defined in config file, merge this referenced profile first
if !config.global.use_profile.is_empty() {
let profile = config.global.use_profile.clone();
config.merge_profile(&profile)?;
}
self.merge(config);
} else {
eprintln!("using no config file ({} doesn't exist)", path.display());
};
Ok(())
}
}

View File

@ -3,9 +3,11 @@ use std::collections::BTreeSet;
use anyhow::{bail, Result};
use clap::Parser;
use log::*;
use merge::Merge;
use rayon::prelude::*;
use serde::Deserialize;
use super::{progress_counter, table_with_titles, GlobalOpts, RusticConfig};
use super::{progress_counter, table_with_titles, Config};
use crate::backend::DecryptWriteBackend;
use crate::blob::{BlobType, NodeType, Packer, TreeStreamerOnce};
use crate::index::{IndexBackend, IndexedBackend, Indexer, ReadIndex};
@ -17,30 +19,22 @@ pub(super) struct Opts {
/// Snapshots to copy. If none is given, use filter options to filter from all snapshots.
#[clap(value_name = "ID")]
ids: Vec<String>,
#[clap(
flatten,
next_help_heading = "Snapshot filter options (if no snapshot is given)"
)]
filter: SnapshotFilter,
}
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
#[derive(Default, Deserialize, Merge)]
pub struct Targets {
#[merge(strategy = merge::vec::overwrite_empty)]
targets: Vec<RepositoryOptions>,
}
let target_opts: Vec<RepositoryOptions> = config_file.get("copy.targets")?;
if target_opts.is_empty() {
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
if config.copy.targets.is_empty() {
bail!("no [[copy.targets]] section in config file found!");
}
let be = &repo.dbe;
let mut snapshots = match opts.ids.is_empty() {
true => SnapshotFile::all_from_backend(be, &opts.filter)?,
true => SnapshotFile::all_from_backend(be, &config.snapshot_filter)?,
false => SnapshotFile::from_ids(be, &opts.ids)?,
};
// sort for nicer output
@ -51,13 +45,13 @@ pub(super) fn execute(
let poly = repo.config.poly()?;
for target_opt in target_opts {
let repo_dest = Repository::new(target_opt)?.open()?;
for target_opt in &config.copy.targets {
let repo_dest = Repository::new(target_opt.clone())?.open()?;
info!("copying to target {}...", repo_dest.name);
if poly != repo_dest.config.poly()? {
bail!("cannot copy to repository with different chunker parameter (re-chunking not implemented)!");
}
copy(&snapshots, index.clone(), repo_dest, &gopts, &opts)?;
copy(&snapshots, index.clone(), repo_dest, &config)?;
}
Ok(())
}
@ -66,13 +60,12 @@ fn copy(
snapshots: &[SnapshotFile],
index: impl IndexedBackend,
repo_dest: OpenRepository,
gopts: &GlobalOpts,
opts: &Opts,
config: &Config,
) -> Result<()> {
let be_dest = &repo_dest.dbe;
let snapshots = relevant_snapshots(snapshots, &repo_dest, &opts.filter)?;
match (snapshots.len(), gopts.dry_run) {
let snapshots = relevant_snapshots(snapshots, &repo_dest, &config.snapshot_filter)?;
match (snapshots.len(), config.global.dry_run) {
(count, true) => {
info!("would have copied {count} snapshots");
return Ok(());

View File

@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
use anyhow::{anyhow, bail, Context, Result};
use clap::Parser;
use super::{progress_counter, RusticConfig};
use super::{progress_counter, Config};
use crate::backend::{
LocalDestination, LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions,
ReadSourceEntry,
@ -13,7 +13,7 @@ use crate::blob::{Node, NodeStreamer, NodeType, Tree};
use crate::commands::helpers::progress_spinner;
use crate::crypto::hash;
use crate::index::{IndexBackend, ReadIndex};
use crate::repofile::{SnapshotFile, SnapshotFilter};
use crate::repofile::SnapshotFile;
use crate::repository::OpenRepository;
#[derive(Parser)]
@ -36,19 +36,9 @@ pub(super) struct Opts {
#[clap(flatten)]
ignore_opts: LocalSourceFilterOptions,
#[clap(
flatten,
next_help_heading = "Snapshot filter options (when using latest)"
)]
filter: SnapshotFilter,
}
pub(super) fn execute(
repo: OpenRepository,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let be = &repo.dbe;
let (id1, path1) = arg_to_snap_path(&opts.snap1, "");
let (id2, path2) = arg_to_snap_path(&opts.snap2, path1);
@ -77,10 +67,13 @@ pub(super) fn execute(
}
(Some(id1), None) => {
// diff between snapshot and local path
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let p = progress_spinner("getting snapshot...");
let snap1 = SnapshotFile::from_str(be, id1, |sn| sn.matches(&opts.filter), p.clone())?;
let snap1 = SnapshotFile::from_str(
be,
id1,
|sn| sn.matches(&config.snapshot_filter),
p.clone(),
)?;
p.finish();
let index = IndexBackend::new(be, progress_counter(""))?;

View File

@ -5,34 +5,28 @@ use std::path::Path;
use crate::blob::{BlobType, NodeType, Tree};
use crate::index::{IndexBackend, IndexedBackend};
use crate::repofile::{SnapshotFile, SnapshotFilter};
use crate::repofile::SnapshotFile;
use crate::repository::OpenRepository;
use super::{progress_counter, RusticConfig};
use super::{progress_counter, Config};
#[derive(Parser)]
pub(super) struct Opts {
/// file from snapshot to dump
#[clap(value_name = "SNAPSHOT[:PATH]")]
snap: String,
#[clap(
flatten,
next_help_heading = "Snapshot filter options (when using latest)"
)]
filter: SnapshotFilter,
}
pub(super) fn execute(
repo: OpenRepository,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let be = &repo.dbe;
let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
let snap = SnapshotFile::from_str(be, id, |sn| sn.matches(&opts.filter), progress_counter(""))?;
let snap = SnapshotFile::from_str(
be,
id,
|sn| sn.matches(&config.snapshot_filter),
progress_counter(""),
)?;
let index = IndexBackend::new(be, progress_counter(""))?;
let node = Tree::node_from_path(&index, snap.tree, Path::new(path))?;

View File

@ -8,7 +8,7 @@ use merge::Merge;
use serde::Deserialize;
use serde_with::{serde_as, DisplayFromStr};
use super::{progress_counter, prune, table_with_titles, GlobalOpts, RusticConfig};
use super::{progress_counter, prune, table_with_titles, Config};
use crate::backend::{DecryptWriteBackend, FileType};
use crate::repofile::{
SnapshotFile, SnapshotFilter, SnapshotGroup, SnapshotGroupCriterion, StringList,
@ -32,9 +32,9 @@ pub(super) struct Opts {
}
#[serde_as]
#[derive(Default, Parser, Deserialize, Merge)]
#[derive(Clone, Default, Parser, Deserialize, Merge)]
#[serde(default, rename_all = "kebab-case")]
struct ConfigOpts {
pub struct ConfigOpts {
/// Group snapshots by any combination of host,label,paths,tags (default: "host,label,paths")
#[clap(long, short = 'g', value_name = "CRITERION")]
#[serde_as(as = "Option<DisplayFromStr>")]
@ -54,17 +54,12 @@ struct ConfigOpts {
keep: KeepOptions,
}
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
pub(super) fn execute(repo: OpenRepository, config: Config, mut opts: Opts) -> Result<()> {
let be = &repo.dbe;
// merge "forget" section from config file, if given
config_file.merge_into("forget", &mut opts.config)?;
opts.config.merge(config.forget.clone());
// merge "snapshot-filter" section from config file, if given
config_file.merge_into("snapshot-filter", &mut opts.config.filter)?;
opts.config.filter.merge(config.snapshot_filter.clone());
let group_by = opts
.config
@ -143,7 +138,7 @@ pub(super) fn execute(
println!();
}
match (forget_snaps.is_empty(), gopts.dry_run) {
match (forget_snaps.is_empty(), config.global.dry_run) {
(true, _) => println!("nothing to remove"),
(false, true) => println!("would have removed the following snapshots:\n {forget_snaps:?}"),
(false, false) => {
@ -153,7 +148,7 @@ pub(super) fn execute(
}
if opts.config.prune {
prune::execute(repo, gopts, opts.prune_opts, forget_snaps)?;
prune::execute(repo, config, opts.prune_opts, forget_snaps)?;
}
Ok(())

View File

@ -3,10 +3,10 @@ use clap::Parser;
use std::path::Path;
use super::progress_counter;
use super::rustic_config::RusticConfig;
use super::Config;
use crate::blob::{NodeStreamer, Tree, TreeStreamerOptions};
use crate::index::IndexBackend;
use crate::repofile::{SnapshotFile, SnapshotFilter};
use crate::repofile::SnapshotFile;
use crate::repository::OpenRepository;
#[derive(Parser)]
@ -19,22 +19,11 @@ pub(super) struct Opts {
#[clap(long)]
recursive: bool,
#[clap(
flatten,
next_help_heading = "Snapshot filter options (when using latest)"
)]
filter: SnapshotFilter,
#[clap(flatten)]
streamer_opts: TreeStreamerOptions,
}
pub(super) fn execute(
repo: OpenRepository,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let be = &repo.dbe;
let mut recursive = opts.recursive;
@ -42,7 +31,12 @@ pub(super) fn execute(
recursive = true;
(&opts.snap, "")
});
let snap = SnapshotFile::from_str(be, id, |sn| sn.matches(&opts.filter), progress_counter(""))?;
let snap = SnapshotFile::from_str(
be,
id,
|sn| sn.matches(&config.snapshot_filter),
progress_counter(""),
)?;
let index = IndexBackend::new(be, progress_counter(""))?;
let node = Tree::node_from_path(&index, snap.tree, Path::new(path))?;

View File

@ -6,11 +6,11 @@ use log::*;
use crate::backend::{DecryptWriteBackend, FileType};
use crate::blob::{merge_trees, BlobType, Node, Packer, Tree};
use crate::index::{IndexBackend, Indexer, ReadIndex};
use crate::repofile::{PathList, SnapshotFile, SnapshotFilter, SnapshotOptions};
use crate::repofile::{PathList, SnapshotFile, SnapshotOptions};
use crate::repository::OpenRepository;
use super::helpers::{progress_counter, progress_spinner};
use super::rustic_config::RusticConfig;
use super::Config;
#[derive(Default, Parser)]
pub(super) struct Opts {
@ -28,24 +28,19 @@ pub(super) struct Opts {
#[clap(flatten, next_help_heading = "Snapshot options")]
snap_opts: SnapshotOptions,
#[clap(flatten, next_help_heading = "Snapshot filter options")]
filter: SnapshotFilter,
}
pub(super) fn execute(
repo: OpenRepository,
mut opts: Opts,
config_file: RusticConfig,
config: Config,
opts: Opts,
command: String,
) -> Result<()> {
let now = Local::now();
let be = &repo.dbe;
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let snapshots = match opts.ids.is_empty() {
true => SnapshotFile::all_from_backend(be, &opts.filter)?,
true => SnapshotFile::all_from_backend(be, &config.snapshot_filter)?,
false => SnapshotFile::from_ids(be, &opts.ids)?,
};
let index = IndexBackend::only_full_trees(&be.clone(), progress_counter(""))?;

View File

@ -6,10 +6,10 @@ use clap::{Parser, Subcommand};
use merge::Merge;
use serde::Deserialize;
use serde_with::{serde_as, DisplayFromStr};
use simplelog::*;
use simplelog::{ColorChoice, CombinedLogger, LevelFilter, TermLogger, TerminalMode, WriteLogger};
use crate::backend::{FileType, ReadBackend};
use crate::repository::{Repository, RepositoryOptions};
use crate::repository::Repository;
use helpers::*;
@ -18,6 +18,7 @@ mod cat;
mod check;
mod completions;
mod config;
mod configfile;
mod copy;
mod diff;
mod dump;
@ -32,32 +33,17 @@ mod prune;
mod repair;
mod repoinfo;
mod restore;
mod rustic_config;
mod self_update;
mod snapshots;
mod tag;
use rustic_config::RusticConfig;
use configfile::Config;
#[derive(Parser)]
#[clap(about, version, name="rustic", version = option_env!("PROJECT_VERSION").unwrap_or(env!("CARGO_PKG_VERSION")))]
struct Opts {
/// Config profile to use. This parses the file `<PROFILE>.toml` in the config directory.
#[clap(
short = 'P',
long,
value_name = "PROFILE",
global = true,
default_value = "rustic",
help_heading = "Global options"
)]
config_profile: String,
#[clap(flatten, next_help_heading = "Global options")]
global: GlobalOpts,
#[clap(flatten, next_help_heading = "Repository options")]
repository: RepositoryOptions,
struct Args {
#[clap(flatten)]
config: Config,
#[clap(subcommand)]
command: Command,
@ -66,7 +52,19 @@ struct Opts {
#[serde_as]
#[derive(Default, Parser, Deserialize, Merge)]
#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
struct GlobalOpts {
pub struct GlobalOpts {
/// Config profile to use. This parses the file `<PROFILE>.toml` in the config directory.
#[clap(
short = 'P',
long,
global = true,
value_name = "PROFILE",
default_value = "rustic",
env = "RUSTIC_USE_PROFILE"
)]
#[merge(skip)]
use_profile: String,
/// Only show what would be done without modifying anything. Does not affect read-only commands
#[clap(long, short = 'n', global = true, env = "RUSTIC_DRY_RUN")]
#[merge(strategy = merge::bool::overwrite_false)]
@ -168,19 +166,19 @@ enum Command {
pub fn execute() -> Result<()> {
let command: Vec<_> = std::env::args_os().collect();
let args = Opts::parse_from(&command);
let args = Args::parse_from(&command);
let mut config = args.config;
// get global options from command line / env and config file
let config_file = RusticConfig::new(&args.config_profile)?;
let mut gopts = args.global;
config_file.merge_into("global", &mut gopts)?;
let profile = config.global.use_profile.clone();
config.merge_profile(&profile)?;
// start logger
let level_filter = gopts.log_level.unwrap_or(LevelFilter::Info);
match &gopts.log_file {
let level_filter = config.global.log_level.unwrap_or(LevelFilter::Info);
match &config.global.log_file {
None => TermLogger::init(
level_filter,
ConfigBuilder::new()
simplelog::ConfigBuilder::new()
.set_time_level(LevelFilter::Off)
.build(),
TerminalMode::Stderr,
@ -190,7 +188,7 @@ pub fn execute() -> Result<()> {
Some(file) => CombinedLogger::init(vec![
TermLogger::new(
level_filter.max(LevelFilter::Warn),
ConfigBuilder::new()
simplelog::ConfigBuilder::new()
.set_time_level(LevelFilter::Off)
.build(),
TerminalMode::Stderr,
@ -198,18 +196,18 @@ pub fn execute() -> Result<()> {
),
WriteLogger::new(
level_filter,
Config::default(),
simplelog::Config::default(),
File::options().create(true).append(true).open(file)?,
),
])?,
}
if gopts.no_progress {
if config.global.no_progress {
let mut no_progress = NO_PROGRESS.lock().unwrap();
*no_progress = true;
}
if let Some(duration) = gopts.progress_interval {
if let Some(duration) = config.global.progress_interval {
let mut interval = PROGRESS_INTERVAL.lock().unwrap();
*interval = *duration;
}
@ -230,9 +228,7 @@ pub fn execute() -> Result<()> {
.collect::<Vec<_>>()
.join(" ");
let mut repo_opts = args.repository;
config_file.merge_into("repository", &mut repo_opts)?;
let repo = Repository::new(repo_opts)?;
let repo = Repository::new(config.repository.clone())?;
if let Command::Init(opts) = args.command {
let config_ids = repo.be.list(FileType::Config)?;
@ -243,27 +239,27 @@ pub fn execute() -> Result<()> {
#[allow(clippy::match_same_arms)]
match args.command {
Command::Backup(opts) => backup::execute(repo, gopts, opts, config_file, command)?,
Command::Backup(opts) => backup::execute(repo, config, opts, command)?,
Command::Config(opts) => config::execute(repo, opts)?,
Command::Cat(opts) => cat::execute(repo, opts, config_file)?,
Command::Cat(opts) => cat::execute(repo, config, opts)?,
Command::Check(opts) => check::execute(repo, opts)?,
Command::Completions(_) => {} // already handled above
Command::Copy(opts) => copy::execute(repo, gopts, opts, config_file)?,
Command::Diff(opts) => diff::execute(repo, opts, config_file)?,
Command::Dump(opts) => dump::execute(repo, opts, config_file)?,
Command::Forget(opts) => forget::execute(repo, gopts, opts, config_file)?,
Command::Copy(opts) => copy::execute(repo, config, opts)?,
Command::Diff(opts) => diff::execute(repo, config, opts)?,
Command::Dump(opts) => dump::execute(repo, config, opts)?,
Command::Forget(opts) => forget::execute(repo, config, opts)?,
Command::Init(_) => {} // already handled above
Command::Key(opts) => key::execute(repo, opts)?,
Command::List(opts) => list::execute(repo, opts)?,
Command::Ls(opts) => ls::execute(repo, opts, config_file)?,
Command::Merge(opts) => merge_cmd::execute(repo, opts, config_file, command)?,
Command::Ls(opts) => ls::execute(repo, config, opts)?,
Command::Merge(opts) => merge_cmd::execute(repo, config, opts, command)?,
Command::SelfUpdate(_) => {} // already handled above
Command::Snapshots(opts) => snapshots::execute(repo, opts, config_file)?,
Command::Prune(opts) => prune::execute(repo, gopts, opts, vec![])?,
Command::Restore(opts) => restore::execute(repo, gopts, opts, config_file)?,
Command::Repair(opts) => repair::execute(repo, gopts, opts, config_file)?,
Command::Snapshots(opts) => snapshots::execute(repo, config, opts)?,
Command::Prune(opts) => prune::execute(repo, config, opts, vec![])?,
Command::Restore(opts) => restore::execute(repo, config, opts)?,
Command::Repair(opts) => repair::execute(repo, config, opts)?,
Command::Repoinfo(opts) => repoinfo::execute(repo, opts)?,
Command::Tag(opts) => tag::execute(repo, gopts, opts, config_file)?,
Command::Tag(opts) => tag::execute(repo, config, opts)?,
};
Ok(())
@ -272,5 +268,5 @@ pub fn execute() -> Result<()> {
#[test]
fn verify_cli() {
use clap::CommandFactory;
Opts::command().debug_assert()
Args::command().debug_assert()
}

View File

@ -12,7 +12,7 @@ use itertools::Itertools;
use log::*;
use rayon::prelude::*;
use super::{bytes, no_progress, progress_bytes, progress_counter, warm_up_wait, GlobalOpts};
use super::{bytes, no_progress, progress_bytes, progress_counter, warm_up_wait, Config};
use crate::backend::{DecryptReadBackend, DecryptWriteBackend, FileType, ReadBackend};
use crate::blob::{
BlobType, BlobTypeMap, Initialize, NodeType, PackSizer, Repacker, Sum, TreeStreamerOnce,
@ -73,7 +73,7 @@ pub(super) struct Opts {
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
config: Config,
opts: Opts,
ignore_snaps: Vec<Id>,
) -> Result<()> {
@ -137,9 +137,10 @@ pub(super) fn execute(
pruner.filter_index_files(opts.instant_delete);
pruner.print_stats();
warm_up_wait(&repo, pruner.repack_packs().into_iter(), !gopts.dry_run)?;
let dry_run = config.global.dry_run;
warm_up_wait(&repo, pruner.repack_packs().into_iter(), !dry_run)?;
if !gopts.dry_run {
if !dry_run {
pruner.do_prune(repo, opts)?;
}
Ok(())

View File

@ -12,13 +12,11 @@ use crate::blob::{BlobType, NodeType, Packer, Tree};
use crate::id::Id;
use crate::index::{IndexBackend, IndexedBackend, Indexer, ReadIndex};
use crate::repofile::{
ConfigFile, IndexFile, IndexPack, PackHeader, PackHeaderRef, SnapshotFile, SnapshotFilter,
StringList,
ConfigFile, IndexFile, IndexPack, PackHeader, PackHeaderRef, SnapshotFile, StringList,
};
use crate::repository::OpenRepository;
use super::rustic_config::RusticConfig;
use super::{progress_counter, progress_spinner, warm_up_wait, GlobalOpts};
use super::{progress_counter, progress_spinner, warm_up_wait, Config};
#[derive(Parser)]
pub(super) struct Opts {
@ -43,9 +41,6 @@ struct IndexOpts {
#[derive(Default, Parser)]
struct SnapOpts {
#[clap(flatten, next_help_heading = "Snapshot filter options")]
filter: SnapshotFilter,
/// Also remove defect snapshots - WARNING: This can result in data loss!
#[clap(long)]
delete: bool,
@ -63,19 +58,14 @@ struct SnapOpts {
ids: Vec<String>,
}
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
match opts.command {
Command::Index(opt) => repair_index(&repo, gopts, opt),
Command::Snapshots(opt) => repair_snaps(&repo.dbe, gopts, opt, config_file, &repo.config),
Command::Index(opt) => repair_index(&repo, config, opt),
Command::Snapshots(opt) => repair_snaps(&repo.dbe, config, opt, &repo.config),
}
}
fn repair_index(repo: &OpenRepository, gopts: GlobalOpts, opts: IndexOpts) -> Result<()> {
fn repair_index(repo: &OpenRepository, config: Config, opts: IndexOpts) -> Result<()> {
let be = &repo.dbe;
let p = progress_spinner("listing packs...");
let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect();
@ -134,7 +124,7 @@ fn repair_index(repo: &OpenRepository, gopts: GlobalOpts, opts: IndexOpts) -> Re
for p in index.packs_to_delete {
process_pack(p, true, &mut new_index, &mut changed);
}
match (changed, gopts.dry_run) {
match (changed, config.global.dry_run) {
(true, true) => info!("would have modified index file {index_id}"),
(true, false) => {
if !new_index.packs.is_empty() || !new_index.packs_to_delete.is_empty() {
@ -169,7 +159,7 @@ fn repair_index(repo: &OpenRepository, gopts: GlobalOpts, opts: IndexOpts) -> Re
..Default::default()
};
if !gopts.dry_run {
if !config.global.dry_run {
indexer.write().unwrap().add_with(pack, to_delete)?;
}
p.inc(1);
@ -182,15 +172,12 @@ fn repair_index(repo: &OpenRepository, gopts: GlobalOpts, opts: IndexOpts) -> Re
fn repair_snaps(
be: &impl DecryptFullBackend,
gopts: GlobalOpts,
mut opts: SnapOpts,
config_file: RusticConfig,
config: &ConfigFile,
config: Config,
opts: SnapOpts,
config_file: &ConfigFile,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let snapshots = match opts.ids.is_empty() {
true => SnapshotFile::all_from_backend(be, &opts.filter)?,
true => SnapshotFile::all_from_backend(be, &config.snapshot_filter)?,
false => SnapshotFile::from_ids(be, &opts.ids)?,
};
@ -204,7 +191,7 @@ fn repair_snaps(
be.clone(),
BlobType::Tree,
indexer.clone(),
config,
config_file,
index.total_size(BlobType::Tree),
)?;
@ -217,7 +204,7 @@ fn repair_snaps(
Some(snap.tree),
&mut replaced,
&mut seen,
&gopts,
&config,
&opts,
)? {
(Changed::None, _) => {
@ -234,7 +221,7 @@ fn repair_snaps(
}
snap.set_tags(opts.tag.clone());
snap.tree = id;
if gopts.dry_run {
if config.global.dry_run {
info!("would have modified snapshot {snap_id}.");
} else {
let new_id = be.save_file(&snap)?;
@ -245,13 +232,13 @@ fn repair_snaps(
}
}
if !gopts.dry_run {
if !config.global.dry_run {
packer.finalize()?;
indexer.write().unwrap().finalize()?;
}
if opts.delete {
if gopts.dry_run {
if config.global.dry_run {
info!("would have removed {} snapshots.", delete.len());
} else {
be.delete_list(
@ -279,7 +266,7 @@ fn repair_tree<BE: DecryptWriteBackend>(
id: Option<Id>,
replaced: &mut HashMap<Id, (Changed, Id)>,
seen: &mut HashSet<Id>,
gopts: &GlobalOpts,
config: &Config,
opts: &SnapOpts,
) -> Result<(Changed, Id)> {
let (tree, changed) = match id {
@ -331,7 +318,7 @@ fn repair_tree<BE: DecryptWriteBackend>(
}
NodeType::Dir {} => {
let (c, tree_id) =
repair_tree(be, packer, node.subtree, replaced, seen, gopts, opts)?;
repair_tree(be, packer, node.subtree, replaced, seen, config, opts)?;
match c {
Changed::None => {}
Changed::This => {
@ -363,7 +350,7 @@ fn repair_tree<BE: DecryptWriteBackend>(
(_, c) => {
// the tree has been changed => save it
let (chunk, new_id) = tree.serialize()?;
if !be.has_tree(&new_id) && !gopts.dry_run {
if !be.has_tree(&new_id) && !config.global.dry_run {
packer.add(chunk.into(), new_id)?;
}
if let Some(id) = id {

View File

@ -11,8 +11,7 @@ use ignore::{DirEntry, WalkBuilder};
use log::*;
use rayon::ThreadPoolBuilder;
use super::rustic_config::RusticConfig;
use super::{bytes, progress_bytes, progress_counter, warm_up_wait, GlobalOpts};
use super::{bytes, progress_bytes, progress_counter, warm_up_wait, Config};
use crate::backend::{DecryptReadBackend, FileType, LocalDestination};
use crate::blob::{Node, NodeStreamer, NodeType, Tree, TreeStreamerOptions};
use crate::commands::helpers::progress_spinner;
@ -59,17 +58,16 @@ pub(super) struct Opts {
filter: SnapshotFilter,
}
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let be = &repo.dbe;
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
let snap = SnapshotFile::from_str(be, id, |sn| sn.matches(&opts.filter), progress_counter(""))?;
let snap = SnapshotFile::from_str(
be,
id,
|sn| sn.matches(&config.snapshot_filter),
progress_counter(""),
)?;
let index = IndexBackend::new(be, progress_counter(""))?;
let node = Tree::node_from_path(&index, snap.tree, Path::new(path))?;
@ -77,7 +75,7 @@ pub(super) fn execute(
let dest = LocalDestination::new(&opts.dest, true, !node.is_dir())?;
let p = progress_spinner("collecting file information...");
let (file_infos, stats) = allocate_and_collect(&dest, index.clone(), &node, &gopts, &opts)?;
let (file_infos, stats) = allocate_and_collect(&dest, index.clone(), &node, &config, &opts)?;
p.finish();
let fs = stats.file;
@ -102,13 +100,17 @@ pub(super) fn execute(
if file_infos.restore_size == 0 {
info!("all file contents are fine.");
} else {
warm_up_wait(&repo, file_infos.to_packs().into_iter(), !gopts.dry_run)?;
if !gopts.dry_run {
warm_up_wait(
&repo,
file_infos.to_packs().into_iter(),
!config.global.dry_run,
)?;
if !config.global.dry_run {
restore_contents(be, &dest, file_infos)?;
}
}
if !gopts.dry_run {
if !config.global.dry_run {
let p = progress_spinner("setting metadata...");
restore_metadata(&dest, index, &node, &opts)?;
p.finish();
@ -138,7 +140,7 @@ fn allocate_and_collect(
dest: &LocalDestination,
index: impl IndexedBackend + Unpin,
node: &Node,
gopts: &GlobalOpts,
config: &Config,
opts: &Opts,
) -> Result<(FileInfos, RestoreStats)> {
let dest_path = Path::new(&opts.dest);
@ -162,7 +164,7 @@ fn allocate_and_collect(
}
match (
opts.delete,
gopts.dry_run,
config.global.dry_run,
entry.file_type().unwrap().is_dir(),
) {
(true, true, true) => {
@ -207,7 +209,7 @@ fn allocate_and_collect(
} else {
stats.dir.restore += 1;
debug!("to restore: {path:?}");
if !gopts.dry_run {
if !config.global.dry_run {
dest.create_dir(path)
.with_context(|| format!("error creating {path:?}"))?;
}
@ -236,7 +238,7 @@ fn allocate_and_collect(
(true, AddFileResult::New(size) | AddFileResult::Modify(size)) => {
stats.file.modify += 1;
debug!("to modify: {path:?}");
if !gopts.dry_run {
if !config.global.dry_run {
// set the right file size
dest.set_length(path, size)
.with_context(|| format!("error setting length for {path:?}"))?;
@ -245,7 +247,7 @@ fn allocate_and_collect(
(false, AddFileResult::New(size) | AddFileResult::Modify(size)) => {
stats.file.restore += 1;
debug!("to restore: {path:?}");
if !gopts.dry_run {
if !config.global.dry_run {
// create the file as it doesn't exist
dest.set_length(path, size)
.with_context(|| format!("error creating {path:?}"))?;

View File

@ -1,68 +0,0 @@
use std::path::Path;
use anyhow::{Context, Result};
use directories::ProjectDirs;
use merge::Merge;
use serde::Deserialize;
use toml::Value;
pub struct RusticConfig(Value);
impl RusticConfig {
pub fn new(profile: &str) -> Result<Self> {
let mut path = match ProjectDirs::from("", "", "rustic") {
Some(path) => path.config_dir().to_path_buf(),
None => Path::new(".").to_path_buf(),
};
if !path.exists() {
path = Path::new(".").to_path_buf();
};
let path = path.join(profile.to_string() + ".toml");
let config = if path.exists() {
// TODO: This should be log::info! - however, the logging config
// can be stored in the config file and is needed to initialize the logger
eprintln!("using config {}", path.display());
let data = std::fs::read_to_string(path).context("error reading config file")?;
toml::from_str(&data).context("error reading TOML from config file")?
} else {
eprintln!("using no config file ({} doesn't exist)", path.display());
Value::Array(Vec::new())
};
Ok(RusticConfig(
config.try_into().context("reading config file")?,
))
}
fn get_value(&self, section: &str) -> Option<&Value> {
// loop over subsections separated by '.'
section
.split('.')
.fold(Some(&self.0), |acc, x| acc.and_then(|value| value.get(x)))
}
pub fn merge_into<'de, Opts>(&self, section: &str, opts: &mut Opts) -> Result<()>
where
Opts: Merge + Deserialize<'de>,
{
if let Some(value) = self.get_value(section) {
let config: Opts = value
.clone()
.try_into()
.with_context(|| format!("reading section [{section}] in config file"))?;
opts.merge(config);
}
Ok(())
}
pub fn get<'de, Opts>(&self, section: &str) -> Result<Opts>
where
Opts: Default + Deserialize<'de>,
{
match self.get_value(section) {
Some(value) => Ok(value.clone().try_into()?),
None => Ok(Opts::default()),
}
}
}

View File

@ -6,10 +6,8 @@ use comfy_table::Cell;
use humantime::format_duration;
use itertools::Itertools;
use super::{bold_cell, bytes, table, table_right_from, RusticConfig};
use crate::repofile::{
DeleteOption, SnapshotFile, SnapshotFilter, SnapshotGroup, SnapshotGroupCriterion,
};
use super::{bold_cell, bytes, table, table_right_from, Config};
use crate::repofile::{DeleteOption, SnapshotFile, SnapshotGroup, SnapshotGroupCriterion};
use crate::repository::OpenRepository;
#[derive(Parser)]
@ -38,22 +36,13 @@ pub(super) struct Opts {
/// Show all snapshots instead of summarizing identical follow-up snapshots
#[clap(long, conflicts_with_all = &["long", "json"])]
all: bool,
#[clap(flatten, next_help_heading = "Snapshot filter options")]
filter: SnapshotFilter,
}
pub(super) fn execute(
repo: OpenRepository,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let groups = match &opts.ids[..] {
[] => SnapshotFile::group_from_backend(&repo.dbe, &opts.filter, &opts.group_by)?,
[] => SnapshotFile::group_from_backend(&repo.dbe, &config.snapshot_filter, &opts.group_by)?,
[id] if id == "latest" => {
SnapshotFile::group_from_backend(&repo.dbe, &opts.filter, &opts.group_by)?
SnapshotFile::group_from_backend(&repo.dbe, &config.snapshot_filter, &opts.group_by)?
.into_iter()
.map(|(group, mut snaps)| {
snaps.sort_unstable();

View File

@ -2,10 +2,10 @@ use anyhow::Result;
use chrono::{Duration, Local};
use clap::Parser;
use super::{progress_counter, GlobalOpts, RusticConfig};
use super::{progress_counter, Config};
use crate::backend::{DecryptWriteBackend, FileType};
use crate::id::Id;
use crate::repofile::{DeleteOption, SnapshotFile, SnapshotFilter, StringList};
use crate::repofile::{DeleteOption, SnapshotFile, StringList};
use crate::repository::OpenRepository;
#[derive(Parser)]
@ -15,12 +15,6 @@ pub(super) struct Opts {
#[clap(value_name = "ID")]
ids: Vec<String>,
#[clap(
flatten,
next_help_heading = "Snapshot filter options (if no snapshot is given)"
)]
filter: SnapshotFilter,
/// Tags to add (can be specified multiple times)
#[clap(
long,
@ -64,17 +58,11 @@ pub(super) struct Opts {
set_delete_after: Option<humantime::Duration>,
}
pub(super) fn execute(
repo: OpenRepository,
gopts: GlobalOpts,
mut opts: Opts,
config_file: RusticConfig,
) -> Result<()> {
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
let be = &repo.dbe;
let snapshots = match opts.ids.is_empty() {
true => SnapshotFile::all_from_backend(be, &opts.filter)?,
true => SnapshotFile::all_from_backend(be, &config.snapshot_filter)?,
false => SnapshotFile::from_ids(be, &opts.ids)?,
};
@ -99,7 +87,7 @@ pub(super) fn execute(
snap.id = Id::default();
}
match (old_snap_ids.is_empty(), gopts.dry_run) {
match (old_snap_ids.is_empty(), config.global.dry_run) {
(true, _) => println!("no snapshot changed."),
(false, true) => {
println!("would have modified the following snapshots:\n {old_snap_ids:?}");

View File

@ -17,7 +17,7 @@ use path_dedot::ParseDot;
use rhai::serde::to_dynamic;
use rhai::{Dynamic, Engine, FnPtr, AST};
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, DisplayFromStr};
use serde_with::{serde_as, DeserializeFromStr, DisplayFromStr};
use super::Id;
use crate::backend::{DecryptReadBackend, FileType, RepoFile};
@ -440,7 +440,7 @@ impl SnapshotFn {
}
#[serde_as]
#[derive(Default, Parser, Deserialize, Merge)]
#[derive(Clone, Default, Parser, Deserialize, Merge)]
#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
pub struct SnapshotFilter {
/// Hostname to filter (can be specified multiple times)
@ -471,7 +471,7 @@ pub struct SnapshotFilter {
filter_fn: Option<SnapshotFn>,
}
#[derive(Clone, Default, Deserialize)]
#[derive(Clone, Default, DeserializeFromStr)]
pub struct SnapshotGroupCriterion {
hostname: bool,
label: bool,

View File

@ -29,7 +29,7 @@ use crate::crypto::Key;
use crate::repofile::{find_key_in_backend, ConfigFile};
#[serde_as]
#[derive(Default, Parser, Deserialize, Merge)]
#[derive(Clone, Default, Parser, Deserialize, Merge)]
#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
pub struct RepositoryOptions {
/// Repository to use