mirrors/rustic
2
0
Fork 0
mirror of https://github.com/rustic-rs/rustic.git synced 2025-10-26 11:18:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use summary structure in snapshot files

Browse Source
This commit is contained in:
Alexander Weiss 2022-05-31 00:41:42 +02:00
parent d6b523b688
commit ff1efc9911
4 changed files with 243 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
src/archiver/archiver_impl.rs
View File

@ -15,7 +15,7 @@ use crate::chunker::ChunkIter;
use crate::crypto::hash;
use crate::id::Id;
use crate::index::{IndexedBackend, Indexer, SharedIndexer};
use crate::repo::SnapshotFile;
use crate::repo::{SnapshotFile, SnapshotSummary};
use super::{Parent, ParentResult};
@ -31,6 +31,7 @@ pub struct Archiver<BE: DecryptWriteBackend, I: IndexedBackend> {
be: BE,
poly: u64,
snap: SnapshotFile,
summary: SnapshotSummary,
}
impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
@ -43,7 +44,9 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
zstd: Option<i32>,
) -> Result<Self> {
let indexer = Indexer::new(be.clone()).into_shared();
snap.backup_start = Some(Local::now());
let mut summary = snap.summary.take().unwrap();
summary.backup_start = Local::now();
Ok(Self {
path: PathBuf::from("/"),
tree: Tree::new(),
@ -56,13 +59,20 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
poly,
indexer,
snap,
summary,
})
}
pub fn add_node(&mut self, node: Node, size: u64) {
pub fn add_file(&mut self, node: Node, size: u64) {
self.tree.add(node);
*self.snap.node_count.get_or_insert(0) += 1;
*self.snap.size.get_or_insert(0) += size;
self.summary.total_files_processed += 1;
self.summary.total_bytes_processed += size;
}
pub fn add_dir(&mut self, node: Node, size: u64) {
self.tree.add(node);
self.summary.total_dirs_processed += 1;
self.summary.total_dirsize_processed += size;
}
pub async fn add_entry(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> {
@ -99,7 +109,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
self.backup_file(path, node, p).await?;
}
NodeType::Dir => {} // is already handled, see above
_ => self.add_node(node, 0), // all other cases: just save the given node
_ => self.add_file(node, 0), // all other cases: just save the given node
}
Ok(())
}
@ -134,39 +144,41 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
match self.parent.is_parent(&node) {
ParentResult::Matched(p_node) if node.subtree() == p_node.subtree() => {
v2!("unchanged tree: {:?}", self.path);
self.add_node(node, dirsize);
*self.snap.trees_unchanged.get_or_insert(0) += 1;
self.add_dir(node, dirsize);
self.summary.dirs_unmodified += 1;
return Ok(());
}
ParentResult::NotFound => {
v2!("new tree: {:?} {}", self.path, dirsize_bytes);
*self.snap.trees_new.get_or_insert(0) += 1;
self.summary.dirs_new += 1;
}
_ => {
// "Matched" trees where the subree id does not match or unmach
v2!("changed tree: {:?} {}", self.path, dirsize_bytes);
*self.snap.trees_changed.get_or_insert(0) += 1;
self.summary.dirs_changed += 1;
}
}
if !self.index.has_tree(&id) && self.tree_packer.add(&chunk, &id, BlobType::Tree).await? {
*self.snap.tree_blobs_written.get_or_insert(0) += 1;
*self.snap.data_added.get_or_insert(0) += dirsize;
self.summary.tree_blobs += 1;
self.summary.data_added += dirsize;
self.summary.data_trees_added += dirsize;
}
self.add_node(node, dirsize);
self.add_dir(node, dirsize);
Ok(())
}
pub async fn backup_file(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> {
let filename = self.path.join(node.name());
match self.parent.is_parent(&node) {
ParentResult::Matched(p_node) => {
v2!("unchanged file: {:?}", self.path.join(node.name()));
*self.snap.files_unchanged.get_or_insert(0) += 1;
v2!("unchanged file: {:?}", filename);
self.summary.files_unmodified += 1;
if p_node.content().iter().all(|id| self.index.has_data(id)) {
let size = *p_node.meta().size();
let mut node = node;
node.set_content(p_node.content().to_vec());
self.add_node(node, size);
self.add_file(node, size);
p.inc(size);
return Ok(());
} else {
@ -177,12 +189,12 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
}
}
ParentResult::NotMatched => {
v2!("changed file: {:?}", self.path.join(node.name()));
*self.snap.files_changed.get_or_insert(0) += 1;
v2!("changed file: {:?}", filename);
self.summary.files_changed += 1;
}
ParentResult::NotFound => {
v2!("new file: {:?}", self.path.join(node.name()));
*self.snap.files_new.get_or_insert(0) += 1;
v2!("new file: {:?}", filename);
self.summary.files_new += 1;
}
}
let f = File::open(path)?;
@ -216,7 +228,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
let mut node = node;
node.set_content(content);
self.add_node(node, filesize);
self.add_file(node, filesize);
Ok(())
}
@ -228,8 +240,9 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
p: &ProgressBar,
) -> Result<()> {
if !self.index.has_data(&id) && self.data_packer.add(chunk, &id, BlobType::Data).await? {
*self.snap.data_blobs_written.get_or_insert(0) += 1;
*self.snap.data_added.get_or_insert(0) += size;
self.summary.data_blobs += 1;
self.summary.data_added += size;
self.summary.data_files_added += size;
}
p.inc(size);
Ok(())
@ -251,7 +264,13 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
let indexer = self.indexer.write().await;
indexer.finalize().await?;
}
self.snap.backup_end = Some(Local::now());
let end_time = Local::now();
self.summary.backup_duration = (end_time - self.summary.backup_start)
.to_std()?
.as_secs_f64();
self.summary.total_duration = (end_time - self.snap.time).to_std()?.as_secs_f64();
self.summary.backup_end = end_time;
self.snap.summary = Some(self.summary);
let id = self.be.save_file(&self.snap).await?;
self.snap.id = id;

36
src/commands/backup.rs
View File

@ -16,7 +16,7 @@ use crate::backend::{
};
use crate::id::Id;
use crate::index::IndexBackend;
use crate::repo::{ConfigFile, SnapshotFile, StringList};
use crate::repo::{ConfigFile, SnapshotFile, SnapshotSummary, StringList};
#[derive(Parser)]
pub(super) struct Opts {
@ -50,7 +50,10 @@ pub(super) async fn execute(
command: String,
) -> Result<()> {
let mut snap = SnapshotFile {
command: Some(command),
summary: Some(SnapshotSummary {
command,
..Default::default()
}),
..Default::default()
};
@ -135,30 +138,29 @@ pub(super) async fn execute(
}
p.finish_with_message("done");
let snap = archiver.finalize_snapshot().await?;
let summary = snap.summary.unwrap();
let bytes = |b| ByteSize(b).to_string_as(true);
v1!(
"Files: {} new, {} changed, {} unchanged",
snap.files_new.unwrap(),
snap.files_changed.unwrap(),
snap.files_unchanged.unwrap()
summary.files_new,
summary.files_changed,
summary.files_unmodified
);
v1!(
"Dirs: {} new, {} changed, {} unchanged",
snap.trees_new.unwrap(),
snap.trees_changed.unwrap(),
snap.trees_unchanged.unwrap()
);
v2!("Data Blobs: {} new", snap.data_blobs_written.unwrap());
v2!("Tree Blobs: {} new", snap.tree_blobs_written.unwrap());
v1!(
"Added to the repo: {}",
ByteSize(snap.data_added.unwrap()).to_string_as(true)
summary.dirs_new,
summary.dirs_changed,
summary.dirs_unmodified
);
v2!("Data Blobs: {} new", summary.data_blobs);
v2!("Tree Blobs: {} new", summary.tree_blobs);
v1!("Added to the repo: {}", bytes(summary.data_added));
v1!(
"processed {} nodes, {}",
snap.node_count.unwrap(),
ByteSize(snap.size.unwrap()).to_string_as(true)
"processed {} files, {}",
summary.total_files_processed,
bytes(summary.total_bytes_processed)
);
v1!("snapshot {} successfully saved.", snap.id);

136
src/commands/snapshots.rs
View File

@ -1,3 +1,5 @@
use std::time::Duration;
use anyhow::Result;
use bytesize::ByteSize;
use clap::Parser;
@ -33,6 +35,7 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<
SnapshotFile::from_ids(be, &opts.ids).await?,
)],
};
let bytes = |b| ByteSize(b).to_string_as(true);
for (group, mut snapshots) in groups {
if !group.is_empty() {
@ -52,19 +55,21 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<
let tags = sn.tags.formatln();
let paths = sn.paths.formatln();
let time = sn.time.format("%Y-%m-%d %H:%M:%S");
let nodes = sn
.node_count
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string());
let size = sn
.size
.map(|b| ByteSize(b).to_string_as(true))
.unwrap_or_else(|| "?".to_string());
row![sn.id, time, sn.hostname, tags, paths, r->nodes, r->size]
let (files, dirs, size) = sn
.summary
.map(|s| {
(
s.total_files_processed.to_string(),
s.total_dirs_processed.to_string(),
bytes(s.total_bytes_processed),
)
})
.unwrap_or_else(|| ("?".to_string(), "?".to_string(), "?".to_string()));
row![sn.id, time, sn.hostname, tags, paths, r->files, r->dirs, r->size]
})
.collect();
table.set_titles(
row![b->"ID", b->"Time", b->"Host", b->"Tags", b->"Paths", br->"Nodes", br->"Size"],
row![b->"ID", b->"Time", b->"Host", b->"Tags", b->"Paths", br->"Files",br->"Dirs", br->"Size"],
);
table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
table.printstd();
@ -77,88 +82,61 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<
fn display_snap(sn: SnapshotFile) {
let mut table = Table::new();
let bytes = |b| ByteSize(b).to_string_as(true);
table.add_row(row![b->"Snapshot", b->sn.id.to_hex()]);
table.add_row(row![b->"Time", sn.time.format("%Y-%m-%d %H:%M:%S")]);
table.add_row(row![b->"Host", sn.hostname]);
table.add_row(row![b->"Tags", sn.tags.formatln()]);
table.add_row(row![b->"Paths", sn.paths.formatln()]);
table.add_row(row![]);
table.add_row(row![b->"Command", sn.command.unwrap_or_else(|| "?".to_string())]);
if let Some(summary) = sn.summary {
table.add_row(row![]);
table.add_row(row![b->"Command", summary.command]);
let source = format!(
"size: {} / nodes: {}",
sn.size
.map(|b| ByteSize(b).to_string_as(true))
.unwrap_or_else(|| "?".to_string()),
sn.node_count
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
);
table.add_row(row![b->"Source", source]);
let source = format!(
"files: {} / dirs: {} / size: {}",
summary.total_files_processed,
summary.total_dirs_processed,
bytes(summary.total_bytes_processed)
);
table.add_row(row![b->"Source", source]);
table.add_row(row![]);
table.add_row(row![]);
let files = format!(
"new: {:>10} / changed: {:>10} / unchanged: {:>10}",
sn.files_new
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
sn.files_changed
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
sn.files_unchanged
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
);
table.add_row(row![b->"Files", files]);
let files = format!(
"new: {:>10} / changed: {:>10} / unchanged: {:>10}",
summary.files_new, summary.files_changed, summary.files_unmodified,
);
table.add_row(row![b->"Files", files]);
let trees = format!(
"new: {:>10} / changed: {:>10} / unchanged: {:>10}",
sn.trees_new
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
sn.trees_changed
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
sn.trees_unchanged
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
);
table.add_row(row![b->"Trees", trees]);
let trees = format!(
"new: {:>10} / changed: {:>10} / unchanged: {:>10}",
summary.dirs_new, summary.dirs_changed, summary.dirs_unmodified,
);
table.add_row(row![b->"Dirs", trees]);
table.add_row(row![]);
table.add_row(row![]);
let written = format!(
"total: {} / tree blobs: {} / data blobs: {}",
sn.data_added
.map(|b| ByteSize(b).to_string_as(true))
.unwrap_or_else(|| "?".to_string()),
sn.tree_blobs_written
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
sn.data_blobs_written
.map(|c| c.to_string())
.unwrap_or_else(|| "?".to_string()),
);
table.add_row(row![b->"Added to repo", written]);
let duration = format!(
"Start: {} / End: {} / Duration: {}",
sn.backup_start
.map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "?".to_string()),
sn.backup_end
.map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "?".to_string()),
match (sn.backup_start, sn.backup_end) {
(Some(start), Some(end)) =>
format_duration((end - start).to_std().unwrap()).to_string(),
_ => "?".to_string(),
},
);
table.add_row(row![b->"Duration", duration]);
let written = format!(
"data: {:>10} blobs / {}\ntree: {:>10} blobs / {}\ntotal: {:>10} blobs / {}",
summary.data_blobs,
bytes(summary.data_files_added),
summary.tree_blobs,
bytes(summary.data_trees_added),
summary.tree_blobs + summary.data_blobs,
bytes(summary.data_added),
);
table.add_row(row![b->"Added to repo", written]);
let duration = format!(
"backup start: {} / backup end: {} / backup duration: {}\ntotal duration: {}",
summary.backup_start.format("%Y-%m-%d %H:%M:%S"),
summary.backup_end.format("%Y-%m-%d %H:%M:%S"),
format_duration(Duration::from_secs_f64(summary.backup_duration)),
format_duration(Duration::from_secs_f64(summary.total_duration))
);
table.add_row(row![b->"Duration", duration]);
}
table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
table.printstd();
println!();

229
src/repo/snapshotfile.rs
View File

@ -12,7 +12,61 @@ use vlog::*;
use super::Id;
use crate::backend::{DecryptReadBackend, FileType, RepoFile};
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
/// This is an extended version of the summaryOutput structure of restic in
/// restic/internal/ui/backup$/json.go
#[derive(Debug, Serialize, Deserialize)]
pub struct SnapshotSummary {
pub files_new: u64,
pub files_changed: u64,
pub files_unmodified: u64,
pub dirs_new: u64,
pub dirs_changed: u64,
pub dirs_unmodified: u64,
pub data_blobs: u64,
pub tree_blobs: u64,
pub data_added: u64,
pub data_files_added: u64,
pub data_trees_added: u64,
pub total_files_processed: u64,
pub total_dirs_processed: u64,
pub total_bytes_processed: u64,
pub total_dirsize_processed: u64,
pub total_duration: f64, // in seconds
pub command: String,
pub backup_start: DateTime<Local>,
pub backup_end: DateTime<Local>,
pub backup_duration: f64, // in seconds
}
impl Default for SnapshotSummary {
fn default() -> Self {
Self {
files_new: 0,
files_changed: 0,
files_unmodified: 0,
dirs_new: 0,
dirs_changed: 0,
dirs_unmodified: 0,
data_blobs: 0,
tree_blobs: 0,
data_added: 0,
data_files_added: 0,
data_trees_added: 0,
total_files_processed: 0,
total_dirs_processed: 0,
total_bytes_processed: 0,
total_dirsize_processed: 0,
total_duration: 0.0,
command: String::new(),
backup_start: Local::now(),
backup_end: Local::now(),
backup_duration: 0.0,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SnapshotFile {
pub time: DateTime<Local>,
pub tree: Id,
@ -29,35 +83,7 @@ pub struct SnapshotFile {
pub tags: StringList,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub command: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub backup_start: Option<DateTime<Local>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub backup_end: Option<DateTime<Local>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub files_new: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub files_changed: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub files_unchanged: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trees_new: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trees_changed: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trees_unchanged: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_blobs_written: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tree_blobs_written: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_added: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub node_count: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub size: Option<u64>,
pub summary: Option<SnapshotSummary>,
#[serde(skip)]
pub id: Id,
@ -78,86 +104,12 @@ impl Default for SnapshotFile {
uid: 0,
gid: 0,
tags: StringList::default(),
node_count: None,
size: None,
command: None,
backup_start: None,
backup_end: None,
files_new: Some(0),
files_changed: Some(0),
files_unchanged: Some(0),
trees_new: Some(0),
trees_changed: Some(0),
trees_unchanged: Some(0),
data_blobs_written: Some(0),
tree_blobs_written: Some(0),
data_added: Some(0),
summary: None,
id: Id::default(),
}
}
}
#[derive(Parser)]
pub struct SnapshotFilter {
/// Path list to filter (can be specified multiple times)
#[clap(long = "filter-paths")]
paths: Vec<StringList>,
/// Tag list to filter (can be specified multiple times)
#[clap(long = "filter-tags")]
tags: Vec<StringList>,
/// Hostname to filter (can be specified multiple times)
#[clap(long = "filter-host", value_name = "HOSTNAME")]
hostnames: Vec<String>,
}
#[derive(Default)]
pub struct SnapshotGroupCriterion {
hostname: bool,
paths: bool,
tags: bool,
}
impl FromStr for SnapshotGroupCriterion {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
let mut crit = SnapshotGroupCriterion::default();
for val in s.split(',') {
match val {
"host" => crit.hostname = true,
"paths" => crit.paths = true,
"tags" => crit.tags = true,
"" => continue,
v => bail!("{} not allowed", v),
}
}
Ok(crit)
}
}
#[derive(Default, Debug)]
pub struct SnapshotGroup {
hostname: Option<String>,
paths: Option<StringList>,
tags: Option<StringList>,
}
impl SnapshotGroup {
pub fn from_sn(sn: &SnapshotFile, crit: &SnapshotGroupCriterion) -> Self {
Self {
hostname: crit.hostname.then(|| sn.hostname.clone()),
paths: crit.paths.then(|| sn.paths.clone()),
tags: crit.tags.then(|| sn.tags.clone()),
}
}
pub fn is_empty(&self) -> bool {
self.hostname.is_none() && self.paths.is_none() && self.tags.is_none()
}
}
impl SnapshotFile {
fn set_id(tuple: (Id, Self)) -> Self {
let (id, mut snap) = tuple;
@ -337,6 +289,13 @@ impl SnapshotFile {
}
}
impl PartialEq<SnapshotFile> for SnapshotFile {
fn eq(&self, other: &SnapshotFile) -> bool {
self.time.eq(&other.time)
}
}
impl Eq for SnapshotFile {}
impl PartialOrd for SnapshotFile {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.time.partial_cmp(&other.time)
@ -348,6 +307,66 @@ impl Ord for SnapshotFile {
}
}
#[derive(Parser)]
pub struct SnapshotFilter {
/// Path list to filter (can be specified multiple times)
#[clap(long = "filter-paths")]
paths: Vec<StringList>,
/// Tag list to filter (can be specified multiple times)
#[clap(long = "filter-tags")]
tags: Vec<StringList>,
/// Hostname to filter (can be specified multiple times)
#[clap(long = "filter-host", value_name = "HOSTNAME")]
hostnames: Vec<String>,
}
#[derive(Default)]
pub struct SnapshotGroupCriterion {
hostname: bool,
paths: bool,
tags: bool,
}
impl FromStr for SnapshotGroupCriterion {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
let mut crit = SnapshotGroupCriterion::default();
for val in s.split(',') {
match val {
"host" => crit.hostname = true,
"paths" => crit.paths = true,
"tags" => crit.tags = true,
"" => continue,
v => bail!("{} not allowed", v),
}
}
Ok(crit)
}
}
#[derive(Default, Debug)]
pub struct SnapshotGroup {
hostname: Option<String>,
paths: Option<StringList>,
tags: Option<StringList>,
}
impl SnapshotGroup {
pub fn from_sn(sn: &SnapshotFile, crit: &SnapshotGroupCriterion) -> Self {
Self {
hostname: crit.hostname.then(|| sn.hostname.clone()),
paths: crit.paths.then(|| sn.paths.clone()),
tags: crit.tags.then(|| sn.tags.clone()),
}
}
pub fn is_empty(&self) -> bool {
self.hostname.is_none() && self.paths.is_none() && self.tags.is_none()
}
}
#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]
pub struct StringList(Vec<String>);