Merge pull request #701 from rustic-rs/refactor-repoinfo

Refactor repoinfo

changelog/new.txt

@@ -11,3 +11,4 @@ New features:
 - restore: Files are now allocated just before being first processed. This allows easier resumed restores.
 - New option: `no-require-git` for backup - if enabled, a git repository is not required to apply `git-ignore` rule.
 - fix: wait for password-command to successfully exit, allowing to input something into the command, and read password from stdout.
+- repoinfo: Added new options --json, --only-files, --only-index

crates/rustic_core/src/backend.rs

@@ -15,6 +15,7 @@ use std::{io::Read, path::PathBuf};
 use bytes::Bytes;
 use displaydoc::Display;
 use log::trace;
+use serde::{Deserialize, Serialize};
 
 use crate::{backend::node::Node, error::BackendErrorKind, id::Id, RusticResult};
 
@@ -27,17 +28,22 @@ pub const ALL_FILE_TYPES: [FileType; 4] = [
 ];
 
 /// Type for describing the kind of a file that can occur.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Display)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Display, Serialize, Deserialize)]
 pub enum FileType {
     /// config
+    #[serde(rename = "config")]
     Config,
     /// index
+    #[serde(rename = "index")]
     Index,
     /// keys
+    #[serde(rename = "key")]
     Key,
     /// snapshots
+    #[serde(rename = "snapshot")]
     Snapshot,
     /// data
+    #[serde(rename = "pack")]
     Pack,
 }
 

crates/rustic_core/src/commands.rs

@@ -1,2 +1,3 @@
 pub mod cat;
 pub mod check;
+pub mod repoinfo;

crates/rustic_core/src/commands/repoinfo.rs

@@ -0,0 +1,143 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    index::IndexEntry,
+    repofile::indexfile::{IndexFile, IndexPack},
+    BlobType, BlobTypeMap, DecryptReadBackend, FileType, OpenRepository, Progress, ProgressBars,
+    ReadBackend, Repository, RusticResult, ALL_FILE_TYPES,
+};
+
+#[derive(Default, Clone, Debug, Serialize, Deserialize)]
+pub struct IndexInfos {
+    pub blobs: Vec<BlobInfo>,
+    pub blobs_delete: Vec<BlobInfo>,
+    pub packs: Vec<PackInfo>,
+    pub packs_delete: Vec<PackInfo>,
+}
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct BlobInfo {
+    pub blob_type: BlobType,
+    pub count: u64,
+    pub size: u64,
+    pub data_size: u64,
+}
+
+impl BlobInfo {
+    pub fn add(&mut self, ie: IndexEntry) {
+        self.count += 1;
+        self.size += u64::from(ie.length);
+        self.data_size += u64::from(ie.data_length());
+    }
+}
+
+#[serde_with::apply(Option => #[serde(default, skip_serializing_if = "Option::is_none")])]
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct PackInfo {
+    pub blob_type: BlobType,
+    pub count: u64,
+    pub min_size: Option<u64>,
+    pub max_size: Option<u64>,
+}
+
+impl PackInfo {
+    pub fn add(&mut self, ip: &IndexPack) {
+        self.count += 1;
+        let size = u64::from(ip.pack_size());
+        self.min_size = self
+            .min_size
+            .map_or(Some(size), |min_size| Some(min_size.min(size)));
+        self.max_size = self
+            .max_size
+            .map_or(Some(size), |max_size| Some(max_size.max(size)));
+    }
+}
+
+pub(crate) fn collect_index_infos<P: ProgressBars>(
+    repo: &OpenRepository<P>,
+) -> RusticResult<IndexInfos> {
+    let mut blob_info = BlobTypeMap::<()>::default().map(|blob_type, _| BlobInfo {
+        blob_type,
+        count: 0,
+        size: 0,
+        data_size: 0,
+    });
+    let mut blob_info_delete = blob_info;
+    let mut pack_info = BlobTypeMap::<()>::default().map(|blob_type, _| PackInfo {
+        blob_type,
+        count: 0,
+        min_size: None,
+        max_size: None,
+    });
+    let mut pack_info_delete = pack_info;
+
+    let p = repo.pb.progress_counter("scanning index...");
+    for index in repo.dbe.stream_all::<IndexFile>(&p)? {
+        let index = index?.1;
+        for pack in &index.packs {
+            let tpe = pack.blob_type();
+            pack_info[tpe].add(pack);
+
+            for blob in &pack.blobs {
+                let ie = IndexEntry::from_index_blob(blob, pack.id);
+                blob_info[tpe].add(ie);
+            }
+        }
+
+        for pack in &index.packs_to_delete {
+            let tpe = pack.blob_type();
+            pack_info_delete[tpe].add(pack);
+            for blob in &pack.blobs {
+                let ie = IndexEntry::from_index_blob(blob, pack.id);
+                blob_info_delete[tpe].add(ie);
+            }
+        }
+    }
+    p.finish();
+
+    let info = IndexInfos {
+        blobs: blob_info.into_values().collect(),
+        blobs_delete: blob_info_delete.into_values().collect(),
+        packs: pack_info.into_values().collect(),
+        packs_delete: pack_info_delete.into_values().collect(),
+    };
+
+    Ok(info)
+}
+
+#[serde_with::apply(Option => #[serde(default, skip_serializing_if = "Option::is_none")])]
+#[derive(Default, Clone, Debug, Serialize, Deserialize)]
+pub struct RepoFileInfos {
+    pub repo: Vec<RepoFileInfo>,
+    pub repo_hot: Option<Vec<RepoFileInfo>>,
+}
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct RepoFileInfo {
+    pub tpe: FileType,
+    pub count: u64,
+    pub size: u64,
+}
+
+pub(crate) fn collect_file_info(be: &impl ReadBackend) -> RusticResult<Vec<RepoFileInfo>> {
+    let mut files = Vec::with_capacity(ALL_FILE_TYPES.len());
+    for tpe in ALL_FILE_TYPES {
+        let list = be.list_with_size(tpe)?;
+        let count = list.len() as u64;
+        let size = list.iter().map(|f| u64::from(f.1)).sum();
+        files.push(RepoFileInfo { tpe, count, size });
+    }
+    Ok(files)
+}
+
+pub fn collect_file_infos<P: ProgressBars>(repo: &Repository<P>) -> RusticResult<RepoFileInfos> {
+    let p = repo.pb.progress_spinner("scanning files...");
+    let files = collect_file_info(&repo.be)?;
+    let files_hot = repo.be_hot.as_ref().map(collect_file_info).transpose()?;
+    p.finish();
+
+    Ok(RepoFileInfos {
+        repo: files,
+        repo_hot: files_hot,
+    })
+}

crates/rustic_core/src/lib.rs

@@ -120,7 +120,10 @@ pub use crate::{
         BlobLocation, BlobType, BlobTypeMap, Initialize, Sum,
     },
     chunker::random_poly,
-    commands::check::CheckOpts,
+    commands::{
+        check::CheckOpts,
+        repoinfo::{BlobInfo, IndexInfos, PackInfo, RepoFileInfo, RepoFileInfos},
+    },
     crypto::{aespoly1305::Key, hasher::hash},
     error::{RusticError, RusticResult},
     file::{AddFileResult, FileInfos, RestoreStats},
@@ -142,5 +145,5 @@ pub use crate::{
         },
         RepoFile,
     },
-    repository::{parse_command, OpenRepository, RepoInfo, Repository, RepositoryOptions},
+    repository::{parse_command, OpenRepository, Repository, RepositoryOptions},
 };

crates/rustic_core/src/repository.rs

@@ -7,7 +7,6 @@ use std::{
 };
 
 use bytes::Bytes;
-use derive_more::Add;
 use log::{debug, error, info};
 
 use nom::{
@@ -29,11 +28,14 @@ use crate::{
         decrypt::DecryptReadBackend, decrypt::DecryptWriteBackend, hotcold::HotColdBackend,
         FileType, ReadBackend,
     },
-    commands::{self, check::CheckOpts},
+    commands::{
+        self,
+        check::CheckOpts,
+        repoinfo::{IndexInfos, RepoFileInfos},
+    },
     crypto::aespoly1305::Key,
     error::RepositoryErrorKind,
-    index::IndexEntry,
-    repofile::{configfile::ConfigFile, indexfile::IndexPack, keyfile::find_key_in_backend},
+    repofile::{configfile::ConfigFile, keyfile::find_key_in_backend},
     BlobType, IndexBackend, NoProgressBars, ProgressBars, RusticResult, SnapshotFile,
 };
 
@@ -41,33 +43,6 @@ pub(super) mod constants {
     pub(super) const MAX_PASSWORD_RETRIES: usize = 5;
 }
 
-#[derive(Default, Clone, Copy, Add, Debug)]
-pub struct RepoInfo {
-    pub count: u64,
-    pub size: u64,
-    pub data_size: u64,
-    pub pack_count: u64,
-    pub total_pack_size: u64,
-    pub min_pack_size: u64,
-    pub max_pack_size: u64,
-}
-
-impl RepoInfo {
-    pub fn add(&mut self, ie: IndexEntry) {
-        self.count += 1;
-        self.size += u64::from(ie.length);
-        self.data_size += u64::from(ie.data_length());
-    }
-
-    pub fn add_pack(&mut self, ip: &IndexPack) {
-        self.pack_count += 1;
-        let size = u64::from(ip.pack_size());
-        self.total_pack_size += size;
-        self.min_pack_size = self.min_pack_size.min(size);
-        self.max_pack_size = self.max_pack_size.max(size);
-    }
-}
-
 #[serde_as]
 #[cfg_attr(feature = "clap", derive(clap::Parser))]
 #[cfg_attr(feature = "merge", derive(merge::Merge))]
@@ -201,7 +176,7 @@ pub struct Repository<P> {
     pub be: HotColdBackend<ChooseBackend>,
     pub be_hot: Option<ChooseBackend>,
     opts: RepositoryOptions,
-    pb: P,
+    pub(crate) pb: P,
 }
 
 impl Repository<NoProgressBars> {
@@ -354,6 +329,12 @@ impl<P> Repository<P> {
     }
 }
 
+impl<P: ProgressBars> Repository<P> {
+    pub fn infos_files(&self) -> RusticResult<RepoFileInfos> {
+        commands::repoinfo::collect_file_infos(self)
+    }
+}
+
 pub(crate) fn get_key(be: &impl ReadBackend, password: Option<String>) -> RusticResult<Key> {
     for _ in 0..constants::MAX_PASSWORD_RETRIES {
         match password {
@@ -403,6 +384,10 @@ impl<P: ProgressBars> OpenRepository<P> {
         let index = IndexBackend::new(&self.dbe, &self.pb.progress_counter(""))?;
         Ok(IndexedRepository { repo: self, index })
     }
+
+    pub fn infos_index(&self) -> RusticResult<IndexInfos> {
+        commands::repoinfo::collect_index_infos(self)
+    }
 }
 
 #[derive(Debug)]

src/commands/repoinfo.rs

@@ -3,23 +3,31 @@
 /// App-local prelude includes `app_reader()`/`app_writer()`/`app_config()`
 /// accessors along with logging macros. Customize as you see fit.
 use crate::{
-    commands::{get_repository, open_repository},
-    helpers::bytes_size_to_string,
-    status_err, Application, RUSTIC_APP,
+    commands::get_repository, helpers::bytes_size_to_string, status_err, Application, RUSTIC_APP,
 };
 
 use abscissa_core::{Command, Runnable, Shutdown};
+use serde::Serialize;
 
-use crate::helpers::{print_file_info, table_right_from};
+use crate::helpers::table_right_from;
 use anyhow::Result;
-use rustic_core::{
-    BlobType, BlobTypeMap, DecryptReadBackend, IndexEntry, IndexFile, Progress, ProgressBars,
-    RepoInfo, Sum,
-};
+use rustic_core::{IndexInfos, RepoFileInfo, RepoFileInfos};
 
 /// `repoinfo` subcommand
 #[derive(clap::Parser, Command, Debug)]
-pub(crate) struct RepoInfoCmd;
+pub(crate) struct RepoInfoCmd {
+    /// Only scan repository files (doesn't need repository password)
+    #[clap(long)]
+    only_files: bool,
+
+    /// Only scan index
+    #[clap(long)]
+    only_index: bool,
+
+    /// Show infos in json format
+    #[clap(long)]
+    json: bool,
+}
 
 impl Runnable for RepoInfoCmd {
     fn run(&self) {
@@ -30,107 +38,142 @@ impl Runnable for RepoInfoCmd {
     }
 }
 
+#[serde_with::apply(Option => #[serde(default, skip_serializing_if = "Option::is_none")])]
+#[derive(Serialize)]
+struct Infos {
+    files: Option<RepoFileInfos>,
+    index: Option<IndexInfos>,
+}
+
 impl RepoInfoCmd {
     fn inner_run(&self) -> Result<()> {
         let config = RUSTIC_APP.config();
-        let repo = open_repository(get_repository(&config));
+        let repo = get_repository(&config);
 
-        print_file_info("repository files", &repo.be)?;
+        let infos = Infos {
+            files: (!self.only_index).then(|| repo.infos_files()).transpose()?,
+            index: (!self.only_files)
+                .then(|| -> Result<_> {
+                    let repo = repo.open()?;
+                    let info_index = repo.infos_index()?;
+                    Ok(info_index)
+                })
+                .transpose()?,
+        };
 
-        if let Some(hot_be) = &repo.be_hot {
-            print_file_info("hot repository files", hot_be)?;
+        if self.json {
+            let mut stdout = std::io::stdout();
+            serde_json::to_writer_pretty(&mut stdout, &infos)?;
+            return Ok(());
         }
 
-        let mut info = BlobTypeMap::<RepoInfo>::default();
-        info[BlobType::Tree].min_pack_size = u64::MAX;
-        info[BlobType::Data].min_pack_size = u64::MAX;
-        let mut info_delete = BlobTypeMap::<RepoInfo>::default();
-
-        let p = config
-            .global
-            .progress_options
-            .progress_counter("scanning index...");
-        repo.dbe
-            .stream_all::<IndexFile>(&p)?
-            .into_iter()
-            .for_each(|index| {
-                let index = match index {
-                    Ok(it) => it,
-                    Err(err) => {
-                        status_err!("{}", err);
-                        RUSTIC_APP.shutdown(Shutdown::Crash);
-                    }
-                }
-                .1;
-                for pack in &index.packs {
-                    info[pack.blob_type()].add_pack(pack);
-
-                    for blob in &pack.blobs {
-                        let ie = IndexEntry::from_index_blob(blob, pack.id);
-                        info[pack.blob_type()].add(ie);
-                    }
-                }
-
-                for pack in &index.packs_to_delete {
-                    for blob in &pack.blobs {
-                        let ie = IndexEntry::from_index_blob(blob, pack.id);
-                        info_delete[pack.blob_type()].add(ie);
-                    }
-                }
-            });
-        p.finish();
-
-        let mut table = table_right_from(
-            1,
-            ["Blob type", "Count", "Total Size", "Total Size in Packs"],
-        );
-
-        for (blob_type, info) in &info {
-            _ = table.add_row([
-                format!("{blob_type:?}"),
-                info.count.to_string(),
-                bytes_size_to_string(info.data_size),
-                bytes_size_to_string(info.size),
-            ]);
-        }
-
-        for (blob_type, info_delete) in &info_delete {
-            if info_delete.count > 0 {
-                _ = table.add_row([
-                    format!("{blob_type:?} to delete"),
-                    info_delete.count.to_string(),
-                    bytes_size_to_string(info_delete.data_size),
-                    bytes_size_to_string(info_delete.size),
-                ]);
+        if let Some(file_info) = infos.files {
+            print_file_info("repository files", file_info.repo);
+            if let Some(info) = file_info.repo_hot {
+                print_file_info("hot repository files", info);
             }
         }
-        let total = info.sum() + info_delete.sum();
-        _ = table.add_row([
-            "Total".to_string(),
-            total.count.to_string(),
-            bytes_size_to_string(total.data_size),
-            bytes_size_to_string(total.size),
-        ]);
 
-        println!();
-        println!("{table}");
-
-        let mut table = table_right_from(
-            1,
-            ["Blob type", "Pack Count", "Minimum Size", "Maximum Size"],
-        );
-
-        for (blob_type, info) in info {
-            _ = table.add_row([
-                format!("{blob_type:?} packs"),
-                info.pack_count.to_string(),
-                bytes_size_to_string(info.min_pack_size),
-                bytes_size_to_string(info.max_pack_size),
-            ]);
+        if let Some(index_info) = infos.index {
+            print_index_info(index_info);
         }
-        println!();
-        println!("{table}");
-
         Ok(())
     }
 }
+
+pub fn print_file_info(text: &str, info: Vec<RepoFileInfo>) {
+    let mut table = table_right_from(1, ["File type", "Count", "Total Size"]);
+    let mut total_count = 0;
+    let mut total_size = 0;
+    for row in info {
+        _ = table.add_row([
+            format!("{:?}", row.tpe),
+            row.count.to_string(),
+            bytes_size_to_string(row.size),
+        ]);
+        total_count += row.count;
+        total_size += row.size;
+    }
+    println!("{text}");
+    _ = table.add_row([
+        "Total".to_string(),
+        total_count.to_string(),
+        bytes_size_to_string(total_size),
+    ]);
+
+    println!();
+    println!("{table}");
+    println!();
+}
+
+pub fn print_index_info(index_info: IndexInfos) {
+    let mut table = table_right_from(
+        1,
+        ["Blob type", "Count", "Total Size", "Total Size in Packs"],
+    );
+
+    let mut total_count = 0;
+    let mut total_data_size = 0;
+    let mut total_size = 0;
+
+    for blobs in &index_info.blobs {
+        _ = table.add_row([
+            format!("{:?}", blobs.blob_type),
+            blobs.count.to_string(),
+            bytes_size_to_string(blobs.data_size),
+            bytes_size_to_string(blobs.size),
+        ]);
+        total_count += blobs.count;
+        total_data_size += blobs.data_size;
+        total_size += blobs.size;
+    }
+    for blobs in &index_info.blobs_delete {
+        if blobs.count > 0 {
+            _ = table.add_row([
+                format!("{:?} to delete", blobs.blob_type),
+                blobs.count.to_string(),
+                bytes_size_to_string(blobs.data_size),
+                bytes_size_to_string(blobs.size),
+            ]);
+            total_count += blobs.count;
+            total_data_size += blobs.data_size;
+            total_size += blobs.size;
+        }
+    }
+
+    _ = table.add_row([
+        "Total".to_string(),
+        total_count.to_string(),
+        bytes_size_to_string(total_data_size),
+        bytes_size_to_string(total_size),
+    ]);
+
+    println!();
+    println!("{table}");
+
+    let mut table = table_right_from(
+        1,
+        ["Blob type", "Pack Count", "Minimum Size", "Maximum Size"],
+    );
+
+    for packs in index_info.packs {
+        _ = table.add_row([
+            format!("{:?} packs", packs.blob_type),
+            packs.count.to_string(),
+            packs.min_size.map_or("-".to_string(), bytes_size_to_string),
+            packs.max_size.map_or("-".to_string(), bytes_size_to_string),
+        ]);
+    }
+    for packs in index_info.packs_delete {
+        if packs.count > 0 {
+            _ = table.add_row([
+                format!("{:?} packs to delete", packs.blob_type),
+                packs.count.to_string(),
+                packs.min_size.map_or("-".to_string(), bytes_size_to_string),
+                packs.max_size.map_or("-".to_string(), bytes_size_to_string),
+            ]);
+        }
+    }
+    println!();
+    println!("{table}");
+}

src/helpers.rs

@@ -16,7 +16,7 @@ use rayon::{
 use rustic_core::{
     parse_command, BlobType, DecryptWriteBackend, FileType, Id, IndexBackend, IndexedBackend,
     Indexer, NodeType, OpenRepository, Packer, Progress, ProgressBars, ReadBackend, ReadIndex,
-    RusticResult, SnapshotFile, TreeStreamerOnce, ALL_FILE_TYPES,
+    SnapshotFile, TreeStreamerOnce,
 };
 
 use crate::{application::RUSTIC_APP, config::progress_options::ProgressOptions};
@@ -274,37 +274,6 @@ pub fn table_right_from<I: IntoIterator<Item = T>, T: ToString>(start: usize, ti
     table
 }
 
-pub fn print_file_info(text: &str, be: &impl ReadBackend) -> RusticResult<()> {
-    info!("scanning files...");
-
-    let mut table = table_right_from(1, ["File type", "Count", "Total Size"]);
-    let mut total_count = 0;
-    let mut total_size = 0;
-    for tpe in ALL_FILE_TYPES {
-        let list = be.list_with_size(tpe)?;
-        let count = list.len();
-        let size = list.iter().map(|f| u64::from(f.1)).sum();
-        _ = table.add_row([
-            format!("{tpe:?}"),
-            count.to_string(),
-            bytes_size_to_string(size),
-        ]);
-        total_count += count;
-        total_size += size;
-    }
-    println!("{text}");
-    _ = table.add_row([
-        "Total".to_string(),
-        total_count.to_string(),
-        bytes_size_to_string(total_size),
-    ]);
-
-    println!();
-    println!("{table}");
-    println!();
-    Ok(())
-}
-
 #[must_use]
 pub fn bytes_size_to_string(b: u64) -> String {
     ByteSize(b).to_string_as(true)