From fffa7d1c05f12e8813fe5acf16c18aab22e19bc9 Mon Sep 17 00:00:00 2001 From: aawsome <37850842+aawsome@users.noreply.github.com> Date: Sun, 31 Aug 2025 23:33:33 +0200 Subject: [PATCH] feat(commands): make group_by a global option (#1545) `group-by` can now be given as a global option. This also allows to set it in the config profiles or use the env variable `RUSTIC_GROUP_BY`. --- config/README.md | 147 +++++++++--------- src/commands/backup.rs | 7 +- src/commands/find.rs | 19 +-- src/commands/forget.rs | 6 +- src/commands/snapshots.rs | 21 +-- src/config.rs | 13 ++ ..._config__tests__default_config_passes.snap | 1 + ..._tests__global_env_roundtrip_passes-3.snap | 1 + 8 files changed, 113 insertions(+), 102 deletions(-) diff --git a/config/README.md b/config/README.md index ae141db..95759de 100644 --- a/config/README.md +++ b/config/README.md @@ -75,19 +75,20 @@ If you want to contribute your own configuration, please ### Global Options `[global]` -| Attribute | Description | Default Value | Example Value | Environment Variable | CLI Option | -| ----------------- | --------------------------------------------------------------------------------- | ------------- | ------------------------ | ------------------------------------------------ | ------------------- | -| check-index | If true, check the index and read pack headers if index information is missing. | false | | RUSTIC_CHECK_INDEX | --check-index | -| dry-run | If true, performs a dry run without making any changes. | false | | RUSTIC_DRY_RUN | --dry-run, -n | -| log-level | Logging level. Possible values: "off", "error", "warn", "info", "debug", "trace". | "info" | | RUSTIC_LOG_LEVEL | --log-level | -| log-file | Path to the log file. | No log file | "/log/rustic.log" | RUSTIC_LOG_FILE | --log-file | -| no-progress | If true, disables progress indicators. | false | | RUSTIC_NO_PROGRESS | --no-progress | -| progress-interval | The interval at which progress indicators are shown. | "100ms" | "1m" | RUSTIC_PROGRESS_INTERVAL | --progress-interval | -| use-profiles | Array of profiles to use. Allows to recursively use other profiles. | Empty array | ["2nd", "3rd"] | RUSTIC_USE_PROFILE | --use-profile, -P | -| prometheus | URL of a Prometheus Pushgateway to push metrics to. | Not set | "http://gateway/" | RUSTIC_PROMETHEUS | --prometheus | -| prometheus-user | Username to authenticate to the Prometheus Pushgateway | Not set | "myuser" | RUSTIC_PROMETHEUS_USER | --prometheus-user | -| prometheus-pass | Password to authenticate to the Prometheus Pushgateway | Not set | "secret" | RUSTIC_PROMETHEUS_PASS | --prometheus-pass | -| opentelemetry | OpenTelemetry metrics endpoint (HTTP Protobuf) | Not set | "http://otel/v1/metrics" | RUSTIC_OTEL, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT | --opentelemetry | +| Attribute | Description | Default Value | Example Value | Environment Variable | CLI Option | +| ----------------- | --------------------------------------------------------------------------------- | ------------------ | ------------------------ | ------------------------------------------------ | ------------------- | +| check-index | If true, check the index and read pack headers if index information is missing. | false | | RUSTIC_CHECK_INDEX | --check-index | +| dry-run | If true, performs a dry run without making any changes. | false | | RUSTIC_DRY_RUN | --dry-run, -n | +| group-by | Group snapshots by any combination of host,label,paths,tags e.g. for "latest" | "host,label,paths" | | RUSTIC_GROUP_BY | --group-by, -g | +| log-level | Logging level. Possible values: "off", "error", "warn", "info", "debug", "trace". | "info" | | RUSTIC_LOG_LEVEL | --log-level | +| log-file | Path to the log file. | No log file | "/log/rustic.log" | RUSTIC_LOG_FILE | --log-file | +| no-progress | If true, disables progress indicators. | false | | RUSTIC_NO_PROGRESS | --no-progress | +| progress-interval | The interval at which progress indicators are shown. | "100ms" | "1m" | RUSTIC_PROGRESS_INTERVAL | --progress-interval | +| use-profiles | Array of profiles to use. Allows to recursively use other profiles. | Empty array | ["2nd", "3rd"] | RUSTIC_USE_PROFILE | --use-profile, -P | +| prometheus | URL of a Prometheus Pushgateway to push metrics to. | Not set | "http://gateway/" | RUSTIC_PROMETHEUS | --prometheus | +| prometheus-user | Username to authenticate to the Prometheus Pushgateway | Not set | "myuser" | RUSTIC_PROMETHEUS_USER | --prometheus-user | +| prometheus-pass | Password to authenticate to the Prometheus Pushgateway | Not set | "secret" | RUSTIC_PROMETHEUS_PASS | --prometheus-pass | +| opentelemetry | OpenTelemetry metrics endpoint (HTTP Protobuf) | Not set | "http://otel/v1/metrics" | RUSTIC_OTEL, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT | --opentelemetry | ### Global Hooks `[global.hooks]` @@ -190,41 +191,41 @@ See [Global Hooks](#global-hooks-globalhooks). **Note**: If set here, the backup options apply for all sources, although they can be overwritten in the source-specific configuration, see below. -| Attribute | Description | Default Value | Example Value | CLI Option | -| ------------------ | ----------------------------------------------------------------------------------------------- | --------------------- | ------------- | ----------------------- | -| as-path | Specifies the path for the backup when the source contains a single path. | Not set | | --as-path | -| command | Set the command saved in the snapshot. | The full command used | | --command | -| custom-ignorefiles | Array of names of custom ignorefiles which will be used to exclude files. | [] | | --custom-ignorefile | -| description | Description for the snapshot. | Not set | | --description | -| description-from | Path to a file containing the description for the snapshot. | Not set | | --description-from | -| delete-never | If true, never delete the snapshot. | false | | --delete-never | -| delete-after | Time duration after which the snapshot be deleted. | Not set | | --delete-after | -| exclude-if-present | Array of filenames which will exclude its parent directory from the backup if they are present. | [] | | --exclude-if-present | -| force | If true, forces the backup even if no changes are detected. | false | | --force | -| git-ignore | If true, use .gitignore rules to exclude files from the backup in the source directory. | false | | --git-ignore | -| globs | Array of globs specifying what to include/exclude in the backup. | [] | | --glob | -| glob-files | Array or string of glob files specifying what to include/exclude in the backup. | [] | | --glob-file | -| group-by | Grouping strategy to find parent snapshot. | "host,label,paths" | | --group-by | -| host | Host name used in the snapshot. | local hostname | | --host | -| iglobs | Like glob, but apply case-insensitive | [] | | --iglob | -| iglob-files | Like glob-file, but apply case-insensitive | [] | | --iglob-file | -| ignore-devid | If true, don't save device ID. | false | | --ignore-devid | -| ignore-ctime | If true, ignore file change time (ctime). | false | | --ignore-ctime | -| ignore-inode | If true, ignore file inode for the backup. | false | | --ignore-inode | -| init | If true, initialize repository if it doesn't exist, yet. | false | | --init | -| json | If true, returns output of the command as json. | false | | --json | -| label | Set label for the snapshot. | Not set | | --label | -| no-require-git | (with git-ignore:) Apply .git-ignore files even if they are not in a git repository. | false | | --no-require-git | -| no-scan | Don't scan the backup source for its size (disables ETA). | false | | --no-scan | -| one-file-system | If true, only backs up files from the same filesystem as the source. | false | | --one-file-system | -| parent | Parent snapshot ID for the backup. | Not set | | --parent | -| quiet | Don't output backup summary. | false | | --quiet | -| skip-if-unchanged | Skip saving of the snapshot if it is identical to the parent. | false | | --skip-identical-parent | -| stdin-filename | File name to be used when reading from stdin. | Not set | | --stdin-filename | -| tags | Array of tags for the backup. | [] | | --tag | -| time | Set the time saved in the snapshot. | current time | | --time | -| with-atime | If true, includes file access time (atime) in the backup. | false | | --with-atime | -| metrics-job | jobname used when pushing metrics (if global prometheus or opentelemetry option is set) | "rustic-backup" | "myjob" | --metrics-job | +| Attribute | Description | Default Value | Example Value | CLI Option | +| ------------------ | ----------------------------------------------------------------------------------------------- | ------------------------ | ------------- | ----------------------- | +| as-path | Specifies the path for the backup when the source contains a single path. | Not set | | --as-path | +| command | Set the command saved in the snapshot. | The full command used | | --command | +| custom-ignorefiles | Array of names of custom ignorefiles which will be used to exclude files. | [] | | --custom-ignorefile | +| description | Description for the snapshot. | Not set | | --description | +| description-from | Path to a file containing the description for the snapshot. | Not set | | --description-from | +| delete-never | If true, never delete the snapshot. | false | | --delete-never | +| delete-after | Time duration after which the snapshot be deleted. | Not set | | --delete-after | +| exclude-if-present | Array of filenames which will exclude its parent directory from the backup if they are present. | [] | | --exclude-if-present | +| force | If true, forces the backup even if no changes are detected. | false | | --force | +| git-ignore | If true, use .gitignore rules to exclude files from the backup in the source directory. | false | | --git-ignore | +| globs | Array of globs specifying what to include/exclude in the backup. | [] | | --glob | +| glob-files | Array or string of glob files specifying what to include/exclude in the backup. | [] | | --glob-file | +| group-by | Grouping strategy to find parent snapshot. | global grouping strategy | | --group-by | +| host | Host name used in the snapshot. | local hostname | | --host | +| iglobs | Like glob, but apply case-insensitive | [] | | --iglob | +| iglob-files | Like glob-file, but apply case-insensitive | [] | | --iglob-file | +| ignore-devid | If true, don't save device ID. | false | | --ignore-devid | +| ignore-ctime | If true, ignore file change time (ctime). | false | | --ignore-ctime | +| ignore-inode | If true, ignore file inode for the backup. | false | | --ignore-inode | +| init | If true, initialize repository if it doesn't exist, yet. | false | | --init | +| json | If true, returns output of the command as json. | false | | --json | +| label | Set label for the snapshot. | Not set | | --label | +| no-require-git | (with git-ignore:) Apply .git-ignore files even if they are not in a git repository. | false | | --no-require-git | +| no-scan | Don't scan the backup source for its size (disables ETA). | false | | --no-scan | +| one-file-system | If true, only backs up files from the same filesystem as the source. | false | | --one-file-system | +| parent | Parent snapshot ID for the backup. | Not set | | --parent | +| quiet | Don't output backup summary. | false | | --quiet | +| skip-if-unchanged | Skip saving of the snapshot if it is identical to the parent. | false | | --skip-identical-parent | +| stdin-filename | File name to be used when reading from stdin. | Not set | | --stdin-filename | +| tags | Array of tags for the backup. | [] | | --tag | +| time | Set the time saved in the snapshot. | current time | | --time | +| with-atime | If true, includes file access time (atime) in the backup. | false | | --with-atime | +| metrics-job | jobname used when pushing metrics (if global prometheus or opentelemetry option is set) | "rustic-backup" | "myjob" | --metrics-job | ### Backup Hooks `[backup.hooks]` @@ -256,31 +257,31 @@ hooks when backing up the defined sources into a snapshot. **Note**: At lest on of the `keep-*` options must be given. Use `keep-none = true` if you want to remove all snapshots. -| Attribute | Description | Default Value | Example Value | CLI Option | -| -------------------------- | ----------------------------------------------------------------------- | ------------------ | ---------------------- | ---------------------------- | -| group-by | Group snapshots by given criteria before applying keep policies. | "host,label,paths" | | --group-by | -| keep-last | Number of most recent snapshots to keep. | Not set | 15 | --keep-last, -l | -| keep-minutely, -M | Number of minutely snapshots to keep. | Not set | | --keep-minutely | -| keep-hourly, -H | Number of hourly snapshots to keep. | Not set | | --keep-hourly | -| keep-daily, -d | Number of daily snapshots to keep. | Not set | 8 | --keep-daily | -| keep-weekly, -w | Number of weekly snapshots to keep. | Not set | | --keep-weekly | -| keep-monthly, -m | Number of monthly snapshots to keep. | Not set | | --keep-monthly | -| keep-quarter-yearly | Number of quarter-yearly snapshots to keep. | Not set | | --keep-quarter-yearly | -| keep-half-yearly | Number of half-yearly snapshots to keep. | Not set | | --keep-half-yearly | -| keep-yearly, -y | Number of yearly snapshots to keep. | Not set | | --keep-yearly | -| keep-within-minutely | The time duration within which minutely snapshots will be kept. | Not set | "2 hours" | --keep-within-minutely | -| keep-within-hourly | The time duration within which hourly snapshots will be kept. | Not set | "1 day" | --keep-within-hourly | -| keep-within-daily | The time duration within which daily snapshots will be kept. | Not set | "7 days" | --keep-within-daily | -| keep-within-weekly | The time duration within which weekly snapshots will be kept. | Not set | | --keep-within-weekly | -| keep-within-monthly | The time duration within which monthly snapshots will be kept. | Not set | | --keep-within-monthly | -| keep-within-quarter-yearly | The time duration within which quarter-yearly snapshots will be kept. | Not set | | --keep-within-quarter-yearly | -| keep-within-half-yearly | The time duration within which half-yearly snapshots will be kept. | Not set | | --keep-within-half-yearly | -| keep-within-yearly | The time duration within which yearly snapshots will be kept. | Not set | | --keep-within-yearly | -| keep-tags | Keep snapshots containing one of these taglists. | [] | ["keep", "important" ] | --keep-tags | -| keep-ids | Keep snapshots containing one of these IDs. | [] | ["6e58f3d32" ] | --keep-id | -| keep-none | Allow to keep no snapshots. | false | true | --keep-none | -| delete-unchanged | Remove snapshots which are unchanged w.r.t their parent. | false | true | --delete-unchanged | -| prune | If set to true, prune the repository after snapshots have been removed. | false | | --prune | +| Attribute | Description | Default Value | Example Value | CLI Option | +| -------------------------- | ----------------------------------------------------------------------- | ------------------------ | ---------------------- | ---------------------------- | +| group-by | Group snapshots by given criteria before applying keep policies. | global grouping strategy | | --group-by | +| keep-last | Number of most recent snapshots to keep. | Not set | 15 | --keep-last, -l | +| keep-minutely, -M | Number of minutely snapshots to keep. | Not set | | --keep-minutely | +| keep-hourly, -H | Number of hourly snapshots to keep. | Not set | | --keep-hourly | +| keep-daily, -d | Number of daily snapshots to keep. | Not set | 8 | --keep-daily | +| keep-weekly, -w | Number of weekly snapshots to keep. | Not set | | --keep-weekly | +| keep-monthly, -m | Number of monthly snapshots to keep. | Not set | | --keep-monthly | +| keep-quarter-yearly | Number of quarter-yearly snapshots to keep. | Not set | | --keep-quarter-yearly | +| keep-half-yearly | Number of half-yearly snapshots to keep. | Not set | | --keep-half-yearly | +| keep-yearly, -y | Number of yearly snapshots to keep. | Not set | | --keep-yearly | +| keep-within-minutely | The time duration within which minutely snapshots will be kept. | Not set | "2 hours" | --keep-within-minutely | +| keep-within-hourly | The time duration within which hourly snapshots will be kept. | Not set | "1 day" | --keep-within-hourly | +| keep-within-daily | The time duration within which daily snapshots will be kept. | Not set | "7 days" | --keep-within-daily | +| keep-within-weekly | The time duration within which weekly snapshots will be kept. | Not set | | --keep-within-weekly | +| keep-within-monthly | The time duration within which monthly snapshots will be kept. | Not set | | --keep-within-monthly | +| keep-within-quarter-yearly | The time duration within which quarter-yearly snapshots will be kept. | Not set | | --keep-within-quarter-yearly | +| keep-within-half-yearly | The time duration within which half-yearly snapshots will be kept. | Not set | | --keep-within-half-yearly | +| keep-within-yearly | The time duration within which yearly snapshots will be kept. | Not set | | --keep-within-yearly | +| keep-tags | Keep snapshots containing one of these taglists. | [] | ["keep", "important" ] | --keep-tags | +| keep-ids | Keep snapshots containing one of these IDs. | [] | ["6e58f3d32" ] | --keep-id | +| keep-none | Allow to keep no snapshots. | false | true | --keep-none | +| delete-unchanged | Remove snapshots which are unchanged w.r.t their parent. | false | true | --delete-unchanged | +| prune | If set to true, prune the repository after snapshots have been removed. | false | | --prune | Additionally extra snapshot filter options can be given for the `forget` command here, see Snapshot-Filter options. diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 3e2d8fe..ac0f470 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -174,6 +174,7 @@ impl Runnable for BackupCmd { status_err!("key \"snapshots\" is not valid in a [[backup.snapshots]] section!"); RUSTIC_APP.shutdown(Shutdown::Crash); } + if let Err(err) = config.repository.run(|repo| self.inner_run(repo)) { status_err!("{}", err); RUSTIC_APP.shutdown(Shutdown::Crash); @@ -329,11 +330,15 @@ impl BackupCmd { let hooks = self.hooks(&hooks, "source-specific-backup", &source); + // use global group-by if not set + let mut parent_opts = self.parent_opts; + parent_opts.group_by = parent_opts.group_by.or(config.global.group_by); + let backup_opts = BackupOptions::default() .stdin_filename(self.stdin_filename) .stdin_command(self.stdin_command) .as_path(self.as_path) - .parent_opts(self.parent_opts) + .parent_opts(parent_opts) .ignore_save_opts(self.ignore_save_opts) .ignore_filter_opts(self.ignore_filter_opts) .no_scan(self.no_scan) diff --git a/src/commands/find.rs b/src/commands/find.rs index 66eb5b0..c8ee13d 100644 --- a/src/commands/find.rs +++ b/src/commands/find.rs @@ -11,7 +11,7 @@ use globset::{Glob, GlobBuilder, GlobSetBuilder}; use itertools::Itertools; use rustic_core::{ - FindMatches, FindNode, SnapshotGroupCriterion, + FindMatches, FindNode, repofile::{Node, SnapshotFile}, }; @@ -36,15 +36,6 @@ pub(crate) struct FindCmd { #[clap(value_name = "ID")] ids: Vec, - /// Group snapshots by any combination of host,label,paths,tags - #[clap( - long, - short = 'g', - value_name = "CRITERION", - default_value = "host,label,paths" - )] - group_by: SnapshotGroupCriterion, - /// Show all snapshots instead of summarizing snapshots with identical search results #[clap(long)] all: bool, @@ -75,9 +66,11 @@ impl FindCmd { fn inner_run(&self, repo: CliIndexedRepo) -> Result<()> { let config = RUSTIC_APP.config(); - let groups = repo.get_snapshot_group(&self.ids, self.group_by, |sn| { - config.snapshot_filter.matches(sn) - })?; + let groups = repo.get_snapshot_group( + &self.ids, + config.global.group_by.unwrap_or_default(), + |sn| config.snapshot_filter.matches(sn), + )?; for (group, mut snapshots) in groups { snapshots.sort_unstable(); if !group.is_empty() { diff --git a/src/commands/forget.rs b/src/commands/forget.rs index 1cf9884..1124371 100644 --- a/src/commands/forget.rs +++ b/src/commands/forget.rs @@ -107,7 +107,11 @@ impl ForgetCmd { fn inner_run(&self, repo: CliOpenRepo) -> Result<()> { let config = RUSTIC_APP.config(); - let group_by = config.forget.group_by.unwrap_or_default(); + let group_by = config + .forget + .group_by + .or(config.global.group_by) + .unwrap_or_default(); let groups = if self.ids.is_empty() { repo.get_forget_snapshots(&config.forget.keep, group_by, |sn| { diff --git a/src/commands/snapshots.rs b/src/commands/snapshots.rs index 2c03fbc..3202082 100644 --- a/src/commands/snapshots.rs +++ b/src/commands/snapshots.rs @@ -16,7 +16,7 @@ use itertools::Itertools; use serde::Serialize; use rustic_core::{ - Progress, ProgressBars, SnapshotGroup, SnapshotGroupCriterion, + Progress, ProgressBars, SnapshotGroup, repofile::{DeleteOption, SnapshotFile}, }; @@ -30,15 +30,6 @@ pub(crate) struct SnapshotCmd { #[clap(value_name = "ID")] ids: Vec, - /// Group snapshots by any combination of host,label,paths,tags - #[clap( - long, - short = 'g', - value_name = "CRITERION", - default_value = "host,label,paths" - )] - group_by: SnapshotGroupCriterion, - /// Show detailed information about snapshots #[arg(long)] long: bool, @@ -85,7 +76,7 @@ impl SnapshotCmd { let snapshots = tui::Snapshots::new( &repo, config.snapshot_filter.clone(), - self.group_by, + config.global.group_by.unwrap_or_default(), )?; tui::run_app(progress.terminal, snapshots) }) @@ -94,9 +85,11 @@ impl SnapshotCmd { let config = RUSTIC_APP.config(); - let groups = repo.get_snapshot_group(&self.ids, self.group_by, |sn| { - config.snapshot_filter.matches(sn) - })?; + let groups = repo.get_snapshot_group( + &self.ids, + config.global.group_by.unwrap_or_default(), + |sn| config.snapshot_filter.matches(sn), + )?; if self.json { #[derive(Serialize, From)] diff --git a/src/config.rs b/src/config.rs index 6c15438..bd39cdb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -21,6 +21,7 @@ use directories::ProjectDirs; use itertools::Itertools; use log::Level; use reqwest::Url; +use rustic_core::SnapshotGroupCriterion; use serde::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; #[cfg(not(all(feature = "mount", feature = "webdav")))] @@ -158,6 +159,18 @@ pub struct GlobalOptions { #[merge(strategy=conflate::vec::append)] pub use_profiles: Vec, + /// Group snapshots by any combination of host,label,paths,tags, e.g. to find the latest snapshot [default: "host,label,paths"] + #[clap( + long, + short = 'g', + global = true, + value_name = "CRITERION", + env = "RUSTIC_USE_PROFILE" + )] + #[serde_as(as = "Option")] + #[merge(strategy=conflate::option::overwrite_none)] + pub group_by: Option, + /// Only show what would be done without modifying anything. Does not affect read-only commands. #[clap(long, short = 'n', global = true, env = "RUSTIC_DRY_RUN")] #[merge(strategy=conflate::bool::overwrite_false)] diff --git a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap index 0ba0f21..92b8728 100644 --- a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap @@ -5,6 +5,7 @@ expression: config RusticConfig { global: GlobalOptions { use_profiles: [], + group_by: None, dry_run: false, check_index: false, log_level: None, diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap index 14c431f..2a1fa2b 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap @@ -5,6 +5,7 @@ expression: deserialized RusticConfig { global: GlobalOptions { use_profiles: [], + group_by: None, dry_run: false, check_index: false, log_level: None,