This commit is contained in:
Alexander Weiss 2022-07-23 07:41:45 +02:00
parent 43c3ca11fc
commit 32bd2fb98d
4 changed files with 16 additions and 34 deletions

View File

@ -10,7 +10,7 @@ use tokio::spawn;
use vlog::*;
use crate::backend::DecryptWriteBackend;
use crate::blob::{BlobType, Metadata, Node, NodeType, Packer, Tree, DEFAULT_TREE_SIZE};
use crate::blob::{BlobType, Metadata, Node, NodeType, Packer, Tree};
use crate::chunker::ChunkIter;
use crate::crypto::hash;
use crate::id::Id;
@ -42,7 +42,6 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
parent: Parent<I>,
mut snap: SnapshotFile,
zstd: Option<i32>,
default_data_size: u32,
) -> Result<Self> {
let indexer = Indexer::new(be.clone()).into_shared();
let mut summary = snap.summary.take().unwrap();
@ -53,7 +52,6 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
BlobType::Data,
indexer.clone(),
zstd,
default_data_size,
index.total_size(&BlobType::Data),
)?;
let tree_packer = Packer::new(
@ -61,7 +59,6 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
BlobType::Tree,
indexer.clone(),
zstd,
DEFAULT_TREE_SIZE,
index.total_size(&BlobType::Tree),
)?;
Ok(Self {

View File

@ -20,8 +20,9 @@ use crate::repo::{IndexBlob, IndexPack};
const KB: u32 = 1024;
const MB: u32 = 1024 * KB;
// default pack size for tree packs
// default pack size
pub const DEFAULT_TREE_SIZE: u32 = 4 * MB;
pub const DEFAULT_DATA_SIZE: u32 = 50 * MB;
// the absolute maximum size of a pack: including headers it should not exceed 4 GB
const MAX_SIZE: u32 = 4076 * MB;
// the factor used for repo-size dependent pack size.
@ -56,7 +57,6 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
blob_type: BlobType,
indexer: SharedIndexer<BE>,
zstd: Option<i32>,
default_size: u32,
total_size: u64,
) -> Result<Self> {
let file_writer = FileWriter {
@ -65,6 +65,10 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
indexer: indexer.clone(),
cacheable: blob_type.is_cacheable(),
};
let default_size = match blob_type {
BlobType::Tree => DEFAULT_TREE_SIZE,
BlobType::Data => DEFAULT_DATA_SIZE,
};
Ok(Self {
be,
blob_type,
@ -311,10 +315,13 @@ impl<BE: DecryptFullBackend> Repacker<BE> {
blob_type: BlobType,
indexer: SharedIndexer<BE>,
zstd: Option<i32>,
default_size: u32,
total_size: u64,
) -> Result<Self> {
let packer = Packer::new(be.clone(), blob_type, indexer, zstd, 0, 0)?;
let default_size = match blob_type {
BlobType::Tree => DEFAULT_TREE_SIZE,
BlobType::Data => DEFAULT_DATA_SIZE,
};
let packer = Packer::new(be.clone(), blob_type, indexer, zstd, 0)?;
let size_limit = Self::size_limit_from_size(total_size, default_size);
Ok(Self {
be,

View File

@ -1,7 +1,6 @@
use std::path::PathBuf;
use anyhow::{anyhow, Result};
use bytesize::ByteSize;
use chrono::{Duration, Local};
use clap::Parser;
use gethostname::gethostname;
@ -43,11 +42,6 @@ pub(super) struct Opts {
#[clap(long, value_name = "DURATION")]
delete_after: Option<humantime::Duration>,
/// Default packsize. rustic tries to always produce packs greater than this value.
/// Note that for large repos, packs can get even larger. Does only apply to data packs.
#[clap(long, value_name = "SIZE", default_value = "50M")]
default_packsize: ByteSize,
#[clap(flatten)]
ignore_opts: LocalSourceOptions,
@ -135,10 +129,8 @@ pub(super) async fn execute(
} else {
0
};
let default_packsize = opts.default_packsize.as_u64().try_into()?;
v1!("starting backup...");
let mut archiver = Archiver::new(be, index, poly, parent, snap, zstd, default_packsize)?;
let mut archiver = Archiver::new(be, index, poly, parent, snap, zstd)?;
let p = progress_bytes();
p.set_length(size);
for item in src {

View File

@ -11,7 +11,7 @@ use vlog::*;
use super::{bytes, progress_counter};
use crate::backend::{DecryptFullBackend, DecryptReadBackend, FileType};
use crate::blob::{BlobType, NodeType, Repacker, TreeStreamerOnce, DEFAULT_TREE_SIZE};
use crate::blob::{BlobType, NodeType, Repacker, TreeStreamerOnce};
use crate::id::Id;
use crate::index::{IndexBackend, IndexCollector, IndexType, IndexedBackend, Indexer};
use crate::repo::{ConfigFile, IndexBlob, IndexFile, IndexPack, SnapshotFile};
@ -52,11 +52,6 @@ pub(super) struct Opts {
#[clap(long, conflicts_with = "fast-repack")]
repack_uncompressed: bool,
/// Default packsize. rustic tries to always produce packs greater than this value.
/// Note that for large repos, packs can get even larger. Does only apply to data packs.
#[clap(long, value_name = "SIZE", default_value = "50M")]
default_packsize: ByteSize,
/// don't remove anything, only show what would be done
#[clap(long, short = 'n')]
pub(crate) dry_run: bool,
@ -726,22 +721,13 @@ impl Pruner {
let indexer = Indexer::new_unindexed(be.clone()).into_shared();
let default_packsize: u32 = opts.default_packsize.as_u64().try_into()?;
// TODO: use size of data/tree blobs after prune here
let mut tree_repacker = Repacker::new(
be.clone(),
BlobType::Tree,
indexer.clone(),
zstd,
DEFAULT_TREE_SIZE,
0,
)?;
let mut tree_repacker =
Repacker::new(be.clone(), BlobType::Tree, indexer.clone(), zstd, 0)?;
let mut data_repacker = Repacker::new(
be.clone(),
BlobType::Data,
indexer.clone(),
zstd,
default_packsize,
self.stats.size.total_after_prune(),
)?;