From ff1efc991127fdea5bbffa33f183ff004a617a5d Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Tue, 31 May 2022 00:41:42 +0200 Subject: [PATCH] Use summary structure in snapshot files --- src/archiver/archiver_impl.rs | 67 ++++++---- src/commands/backup.rs | 36 +++--- src/commands/snapshots.rs | 136 +++++++++----------- src/repo/snapshotfile.rs | 229 ++++++++++++++++++---------------- 4 files changed, 243 insertions(+), 225 deletions(-) diff --git a/src/archiver/archiver_impl.rs b/src/archiver/archiver_impl.rs index 5d3c54d..cea4e1c 100644 --- a/src/archiver/archiver_impl.rs +++ b/src/archiver/archiver_impl.rs @@ -15,7 +15,7 @@ use crate::chunker::ChunkIter; use crate::crypto::hash; use crate::id::Id; use crate::index::{IndexedBackend, Indexer, SharedIndexer}; -use crate::repo::SnapshotFile; +use crate::repo::{SnapshotFile, SnapshotSummary}; use super::{Parent, ParentResult}; @@ -31,6 +31,7 @@ pub struct Archiver { be: BE, poly: u64, snap: SnapshotFile, + summary: SnapshotSummary, } impl Archiver { @@ -43,7 +44,9 @@ impl Archiver { zstd: Option, ) -> Result { let indexer = Indexer::new(be.clone()).into_shared(); - snap.backup_start = Some(Local::now()); + let mut summary = snap.summary.take().unwrap(); + summary.backup_start = Local::now(); + Ok(Self { path: PathBuf::from("/"), tree: Tree::new(), @@ -56,13 +59,20 @@ impl Archiver { poly, indexer, snap, + summary, }) } - pub fn add_node(&mut self, node: Node, size: u64) { + pub fn add_file(&mut self, node: Node, size: u64) { self.tree.add(node); - *self.snap.node_count.get_or_insert(0) += 1; - *self.snap.size.get_or_insert(0) += size; + self.summary.total_files_processed += 1; + self.summary.total_bytes_processed += size; + } + + pub fn add_dir(&mut self, node: Node, size: u64) { + self.tree.add(node); + self.summary.total_dirs_processed += 1; + self.summary.total_dirsize_processed += size; } pub async fn add_entry(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> { @@ -99,7 +109,7 @@ impl Archiver { self.backup_file(path, node, p).await?; } NodeType::Dir => {} // is already handled, see above - _ => self.add_node(node, 0), // all other cases: just save the given node + _ => self.add_file(node, 0), // all other cases: just save the given node } Ok(()) } @@ -134,39 +144,41 @@ impl Archiver { match self.parent.is_parent(&node) { ParentResult::Matched(p_node) if node.subtree() == p_node.subtree() => { v2!("unchanged tree: {:?}", self.path); - self.add_node(node, dirsize); - *self.snap.trees_unchanged.get_or_insert(0) += 1; + self.add_dir(node, dirsize); + self.summary.dirs_unmodified += 1; return Ok(()); } ParentResult::NotFound => { v2!("new tree: {:?} {}", self.path, dirsize_bytes); - *self.snap.trees_new.get_or_insert(0) += 1; + self.summary.dirs_new += 1; } _ => { // "Matched" trees where the subree id does not match or unmach v2!("changed tree: {:?} {}", self.path, dirsize_bytes); - *self.snap.trees_changed.get_or_insert(0) += 1; + self.summary.dirs_changed += 1; } } if !self.index.has_tree(&id) && self.tree_packer.add(&chunk, &id, BlobType::Tree).await? { - *self.snap.tree_blobs_written.get_or_insert(0) += 1; - *self.snap.data_added.get_or_insert(0) += dirsize; + self.summary.tree_blobs += 1; + self.summary.data_added += dirsize; + self.summary.data_trees_added += dirsize; } - self.add_node(node, dirsize); + self.add_dir(node, dirsize); Ok(()) } pub async fn backup_file(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> { + let filename = self.path.join(node.name()); match self.parent.is_parent(&node) { ParentResult::Matched(p_node) => { - v2!("unchanged file: {:?}", self.path.join(node.name())); - *self.snap.files_unchanged.get_or_insert(0) += 1; + v2!("unchanged file: {:?}", filename); + self.summary.files_unmodified += 1; if p_node.content().iter().all(|id| self.index.has_data(id)) { let size = *p_node.meta().size(); let mut node = node; node.set_content(p_node.content().to_vec()); - self.add_node(node, size); + self.add_file(node, size); p.inc(size); return Ok(()); } else { @@ -177,12 +189,12 @@ impl Archiver { } } ParentResult::NotMatched => { - v2!("changed file: {:?}", self.path.join(node.name())); - *self.snap.files_changed.get_or_insert(0) += 1; + v2!("changed file: {:?}", filename); + self.summary.files_changed += 1; } ParentResult::NotFound => { - v2!("new file: {:?}", self.path.join(node.name())); - *self.snap.files_new.get_or_insert(0) += 1; + v2!("new file: {:?}", filename); + self.summary.files_new += 1; } } let f = File::open(path)?; @@ -216,7 +228,7 @@ impl Archiver { let mut node = node; node.set_content(content); - self.add_node(node, filesize); + self.add_file(node, filesize); Ok(()) } @@ -228,8 +240,9 @@ impl Archiver { p: &ProgressBar, ) -> Result<()> { if !self.index.has_data(&id) && self.data_packer.add(chunk, &id, BlobType::Data).await? { - *self.snap.data_blobs_written.get_or_insert(0) += 1; - *self.snap.data_added.get_or_insert(0) += size; + self.summary.data_blobs += 1; + self.summary.data_added += size; + self.summary.data_files_added += size; } p.inc(size); Ok(()) @@ -251,7 +264,13 @@ impl Archiver { let indexer = self.indexer.write().await; indexer.finalize().await?; } - self.snap.backup_end = Some(Local::now()); + let end_time = Local::now(); + self.summary.backup_duration = (end_time - self.summary.backup_start) + .to_std()? + .as_secs_f64(); + self.summary.total_duration = (end_time - self.snap.time).to_std()?.as_secs_f64(); + self.summary.backup_end = end_time; + self.snap.summary = Some(self.summary); let id = self.be.save_file(&self.snap).await?; self.snap.id = id; diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 553834c..5078117 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -16,7 +16,7 @@ use crate::backend::{ }; use crate::id::Id; use crate::index::IndexBackend; -use crate::repo::{ConfigFile, SnapshotFile, StringList}; +use crate::repo::{ConfigFile, SnapshotFile, SnapshotSummary, StringList}; #[derive(Parser)] pub(super) struct Opts { @@ -50,7 +50,10 @@ pub(super) async fn execute( command: String, ) -> Result<()> { let mut snap = SnapshotFile { - command: Some(command), + summary: Some(SnapshotSummary { + command, + ..Default::default() + }), ..Default::default() }; @@ -135,30 +138,29 @@ pub(super) async fn execute( } p.finish_with_message("done"); let snap = archiver.finalize_snapshot().await?; + let summary = snap.summary.unwrap(); + let bytes = |b| ByteSize(b).to_string_as(true); v1!( "Files: {} new, {} changed, {} unchanged", - snap.files_new.unwrap(), - snap.files_changed.unwrap(), - snap.files_unchanged.unwrap() + summary.files_new, + summary.files_changed, + summary.files_unmodified ); v1!( "Dirs: {} new, {} changed, {} unchanged", - snap.trees_new.unwrap(), - snap.trees_changed.unwrap(), - snap.trees_unchanged.unwrap() - ); - v2!("Data Blobs: {} new", snap.data_blobs_written.unwrap()); - v2!("Tree Blobs: {} new", snap.tree_blobs_written.unwrap()); - v1!( - "Added to the repo: {}", - ByteSize(snap.data_added.unwrap()).to_string_as(true) + summary.dirs_new, + summary.dirs_changed, + summary.dirs_unmodified ); + v2!("Data Blobs: {} new", summary.data_blobs); + v2!("Tree Blobs: {} new", summary.tree_blobs); + v1!("Added to the repo: {}", bytes(summary.data_added)); v1!( - "processed {} nodes, {}", - snap.node_count.unwrap(), - ByteSize(snap.size.unwrap()).to_string_as(true) + "processed {} files, {}", + summary.total_files_processed, + bytes(summary.total_bytes_processed) ); v1!("snapshot {} successfully saved.", snap.id); diff --git a/src/commands/snapshots.rs b/src/commands/snapshots.rs index 8c8981c..4071af2 100644 --- a/src/commands/snapshots.rs +++ b/src/commands/snapshots.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use anyhow::Result; use bytesize::ByteSize; use clap::Parser; @@ -33,6 +35,7 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result< SnapshotFile::from_ids(be, &opts.ids).await?, )], }; + let bytes = |b| ByteSize(b).to_string_as(true); for (group, mut snapshots) in groups { if !group.is_empty() { @@ -52,19 +55,21 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result< let tags = sn.tags.formatln(); let paths = sn.paths.formatln(); let time = sn.time.format("%Y-%m-%d %H:%M:%S"); - let nodes = sn - .node_count - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()); - let size = sn - .size - .map(|b| ByteSize(b).to_string_as(true)) - .unwrap_or_else(|| "?".to_string()); - row![sn.id, time, sn.hostname, tags, paths, r->nodes, r->size] + let (files, dirs, size) = sn + .summary + .map(|s| { + ( + s.total_files_processed.to_string(), + s.total_dirs_processed.to_string(), + bytes(s.total_bytes_processed), + ) + }) + .unwrap_or_else(|| ("?".to_string(), "?".to_string(), "?".to_string())); + row![sn.id, time, sn.hostname, tags, paths, r->files, r->dirs, r->size] }) .collect(); table.set_titles( - row![b->"ID", b->"Time", b->"Host", b->"Tags", b->"Paths", br->"Nodes", br->"Size"], + row![b->"ID", b->"Time", b->"Host", b->"Tags", b->"Paths", br->"Files",br->"Dirs", br->"Size"], ); table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); table.printstd(); @@ -77,88 +82,61 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result< fn display_snap(sn: SnapshotFile) { let mut table = Table::new(); + let bytes = |b| ByteSize(b).to_string_as(true); table.add_row(row![b->"Snapshot", b->sn.id.to_hex()]); table.add_row(row![b->"Time", sn.time.format("%Y-%m-%d %H:%M:%S")]); table.add_row(row![b->"Host", sn.hostname]); table.add_row(row![b->"Tags", sn.tags.formatln()]); table.add_row(row![b->"Paths", sn.paths.formatln()]); - table.add_row(row![]); - table.add_row(row![b->"Command", sn.command.unwrap_or_else(|| "?".to_string())]); + if let Some(summary) = sn.summary { + table.add_row(row![]); + table.add_row(row![b->"Command", summary.command]); - let source = format!( - "size: {} / nodes: {}", - sn.size - .map(|b| ByteSize(b).to_string_as(true)) - .unwrap_or_else(|| "?".to_string()), - sn.node_count - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - ); - table.add_row(row![b->"Source", source]); + let source = format!( + "files: {} / dirs: {} / size: {}", + summary.total_files_processed, + summary.total_dirs_processed, + bytes(summary.total_bytes_processed) + ); + table.add_row(row![b->"Source", source]); - table.add_row(row![]); + table.add_row(row![]); - let files = format!( - "new: {:>10} / changed: {:>10} / unchanged: {:>10}", - sn.files_new - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - sn.files_changed - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - sn.files_unchanged - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - ); - table.add_row(row![b->"Files", files]); + let files = format!( + "new: {:>10} / changed: {:>10} / unchanged: {:>10}", + summary.files_new, summary.files_changed, summary.files_unmodified, + ); + table.add_row(row![b->"Files", files]); - let trees = format!( - "new: {:>10} / changed: {:>10} / unchanged: {:>10}", - sn.trees_new - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - sn.trees_changed - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - sn.trees_unchanged - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - ); - table.add_row(row![b->"Trees", trees]); + let trees = format!( + "new: {:>10} / changed: {:>10} / unchanged: {:>10}", + summary.dirs_new, summary.dirs_changed, summary.dirs_unmodified, + ); + table.add_row(row![b->"Dirs", trees]); - table.add_row(row![]); + table.add_row(row![]); - let written = format!( - "total: {} / tree blobs: {} / data blobs: {}", - sn.data_added - .map(|b| ByteSize(b).to_string_as(true)) - .unwrap_or_else(|| "?".to_string()), - sn.tree_blobs_written - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - sn.data_blobs_written - .map(|c| c.to_string()) - .unwrap_or_else(|| "?".to_string()), - ); - table.add_row(row![b->"Added to repo", written]); - - let duration = format!( - "Start: {} / End: {} / Duration: {}", - sn.backup_start - .map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string()) - .unwrap_or_else(|| "?".to_string()), - sn.backup_end - .map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string()) - .unwrap_or_else(|| "?".to_string()), - match (sn.backup_start, sn.backup_end) { - (Some(start), Some(end)) => - format_duration((end - start).to_std().unwrap()).to_string(), - _ => "?".to_string(), - }, - ); - table.add_row(row![b->"Duration", duration]); + let written = format!( + "data: {:>10} blobs / {}\ntree: {:>10} blobs / {}\ntotal: {:>10} blobs / {}", + summary.data_blobs, + bytes(summary.data_files_added), + summary.tree_blobs, + bytes(summary.data_trees_added), + summary.tree_blobs + summary.data_blobs, + bytes(summary.data_added), + ); + table.add_row(row![b->"Added to repo", written]); + let duration = format!( + "backup start: {} / backup end: {} / backup duration: {}\ntotal duration: {}", + summary.backup_start.format("%Y-%m-%d %H:%M:%S"), + summary.backup_end.format("%Y-%m-%d %H:%M:%S"), + format_duration(Duration::from_secs_f64(summary.backup_duration)), + format_duration(Duration::from_secs_f64(summary.total_duration)) + ); + table.add_row(row![b->"Duration", duration]); + } table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); table.printstd(); println!(); diff --git a/src/repo/snapshotfile.rs b/src/repo/snapshotfile.rs index ca1c826..01528e6 100644 --- a/src/repo/snapshotfile.rs +++ b/src/repo/snapshotfile.rs @@ -12,7 +12,61 @@ use vlog::*; use super::Id; use crate::backend::{DecryptReadBackend, FileType, RepoFile}; -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +/// This is an extended version of the summaryOutput structure of restic in +/// restic/internal/ui/backup$/json.go +#[derive(Debug, Serialize, Deserialize)] +pub struct SnapshotSummary { + pub files_new: u64, + pub files_changed: u64, + pub files_unmodified: u64, + pub dirs_new: u64, + pub dirs_changed: u64, + pub dirs_unmodified: u64, + pub data_blobs: u64, + pub tree_blobs: u64, + pub data_added: u64, + pub data_files_added: u64, + pub data_trees_added: u64, + pub total_files_processed: u64, + pub total_dirs_processed: u64, + pub total_bytes_processed: u64, + pub total_dirsize_processed: u64, + pub total_duration: f64, // in seconds + + pub command: String, + pub backup_start: DateTime, + pub backup_end: DateTime, + pub backup_duration: f64, // in seconds +} + +impl Default for SnapshotSummary { + fn default() -> Self { + Self { + files_new: 0, + files_changed: 0, + files_unmodified: 0, + dirs_new: 0, + dirs_changed: 0, + dirs_unmodified: 0, + data_blobs: 0, + tree_blobs: 0, + data_added: 0, + data_files_added: 0, + data_trees_added: 0, + total_files_processed: 0, + total_dirs_processed: 0, + total_bytes_processed: 0, + total_dirsize_processed: 0, + total_duration: 0.0, + command: String::new(), + backup_start: Local::now(), + backup_end: Local::now(), + backup_duration: 0.0, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] pub struct SnapshotFile { pub time: DateTime, pub tree: Id, @@ -29,35 +83,7 @@ pub struct SnapshotFile { pub tags: StringList, #[serde(default, skip_serializing_if = "Option::is_none")] - pub command: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub backup_start: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub backup_end: Option>, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub files_new: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub files_changed: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub files_unchanged: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trees_new: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trees_changed: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trees_unchanged: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub data_blobs_written: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tree_blobs_written: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub data_added: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub node_count: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub size: Option, + pub summary: Option, #[serde(skip)] pub id: Id, @@ -78,86 +104,12 @@ impl Default for SnapshotFile { uid: 0, gid: 0, tags: StringList::default(), - node_count: None, - size: None, - command: None, - backup_start: None, - backup_end: None, - files_new: Some(0), - files_changed: Some(0), - files_unchanged: Some(0), - trees_new: Some(0), - trees_changed: Some(0), - trees_unchanged: Some(0), - data_blobs_written: Some(0), - tree_blobs_written: Some(0), - data_added: Some(0), - + summary: None, id: Id::default(), } } } -#[derive(Parser)] -pub struct SnapshotFilter { - /// Path list to filter (can be specified multiple times) - #[clap(long = "filter-paths")] - paths: Vec, - - /// Tag list to filter (can be specified multiple times) - #[clap(long = "filter-tags")] - tags: Vec, - - /// Hostname to filter (can be specified multiple times) - #[clap(long = "filter-host", value_name = "HOSTNAME")] - hostnames: Vec, -} - -#[derive(Default)] -pub struct SnapshotGroupCriterion { - hostname: bool, - paths: bool, - tags: bool, -} - -impl FromStr for SnapshotGroupCriterion { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let mut crit = SnapshotGroupCriterion::default(); - for val in s.split(',') { - match val { - "host" => crit.hostname = true, - "paths" => crit.paths = true, - "tags" => crit.tags = true, - "" => continue, - v => bail!("{} not allowed", v), - } - } - Ok(crit) - } -} - -#[derive(Default, Debug)] -pub struct SnapshotGroup { - hostname: Option, - paths: Option, - tags: Option, -} - -impl SnapshotGroup { - pub fn from_sn(sn: &SnapshotFile, crit: &SnapshotGroupCriterion) -> Self { - Self { - hostname: crit.hostname.then(|| sn.hostname.clone()), - paths: crit.paths.then(|| sn.paths.clone()), - tags: crit.tags.then(|| sn.tags.clone()), - } - } - - pub fn is_empty(&self) -> bool { - self.hostname.is_none() && self.paths.is_none() && self.tags.is_none() - } -} - impl SnapshotFile { fn set_id(tuple: (Id, Self)) -> Self { let (id, mut snap) = tuple; @@ -337,6 +289,13 @@ impl SnapshotFile { } } +impl PartialEq for SnapshotFile { + fn eq(&self, other: &SnapshotFile) -> bool { + self.time.eq(&other.time) + } +} +impl Eq for SnapshotFile {} + impl PartialOrd for SnapshotFile { fn partial_cmp(&self, other: &Self) -> Option { self.time.partial_cmp(&other.time) @@ -348,6 +307,66 @@ impl Ord for SnapshotFile { } } +#[derive(Parser)] +pub struct SnapshotFilter { + /// Path list to filter (can be specified multiple times) + #[clap(long = "filter-paths")] + paths: Vec, + + /// Tag list to filter (can be specified multiple times) + #[clap(long = "filter-tags")] + tags: Vec, + + /// Hostname to filter (can be specified multiple times) + #[clap(long = "filter-host", value_name = "HOSTNAME")] + hostnames: Vec, +} + +#[derive(Default)] +pub struct SnapshotGroupCriterion { + hostname: bool, + paths: bool, + tags: bool, +} + +impl FromStr for SnapshotGroupCriterion { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let mut crit = SnapshotGroupCriterion::default(); + for val in s.split(',') { + match val { + "host" => crit.hostname = true, + "paths" => crit.paths = true, + "tags" => crit.tags = true, + "" => continue, + v => bail!("{} not allowed", v), + } + } + Ok(crit) + } +} + +#[derive(Default, Debug)] +pub struct SnapshotGroup { + hostname: Option, + paths: Option, + tags: Option, +} + +impl SnapshotGroup { + pub fn from_sn(sn: &SnapshotFile, crit: &SnapshotGroupCriterion) -> Self { + Self { + hostname: crit.hostname.then(|| sn.hostname.clone()), + paths: crit.paths.then(|| sn.paths.clone()), + tags: crit.tags.then(|| sn.tags.clone()), + } + } + + pub fn is_empty(&self) -> bool { + self.hostname.is_none() && self.paths.is_none() && self.tags.is_none() + } +} + #[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)] pub struct StringList(Vec);