mirrors/rustic
2
0
Fork 0
mirror of https://github.com/rustic-rs/rustic.git synced 2025-10-26 11:18:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #268 from rustic-rs/no-async

Browse Source 
WIP: remove async code and parallelize using threads
This commit is contained in:
aawsome 2022-10-26 13:57:10 +02:00 committed by GitHub
commit e5a31550c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 1057 additions and 1180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

367
Cargo.lock generated
View File

@ -74,28 +74,6 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfb6d71005dc22a708c7496eee5c8dc0300ee47355de6256c3b35b12b5fef596"
[[package]]
name = "async-recursion"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "async-trait"
version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "atty"
version = "0.2.14"
@ -119,12 +97,9 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
dependencies = [
"futures-core",
"getrandom",
"instant",
"pin-project-lite",
"rand",
"tokio",
]
[[package]]
@ -183,9 +158,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.11.0"
version = "3.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
[[package]]
name = "bytemuck"
@ -316,14 +291,24 @@ dependencies = [
]
[[package]]
name = "console"
version = "0.15.1"
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847"
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
dependencies = [
"termcolor",
"unicode-width",
]
[[package]]
name = "console"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c"
dependencies = [
"encode_unicode 0.3.6",
"lazy_static",
"libc",
"once_cell",
"terminal_size",
"unicode-width",
"winapi",
@ -360,13 +345,70 @@ dependencies = [
]
[[package]]
name = "crossbeam-utils"
version = "0.8.11"
name = "crossbeam"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c"
dependencies = [
"cfg-if",
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
@ -388,6 +430,50 @@ dependencies = [
"cipher 0.3.0",
]
[[package]]
name = "cxx"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a"
dependencies = [
"cc",
"cxxbridge-flags",
"cxxbridge-macro",
"link-cplusplus",
]
[[package]]
name = "cxx-build"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827"
dependencies = [
"cc",
"codespan-reporting",
"once_cell",
"proc-macro2",
"quote",
"scratch",
"syn",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a"
[[package]]
name = "cxxbridge-macro"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "darling"
version = "0.14.1"
@ -624,9 +710,9 @@ dependencies = [
[[package]]
name = "futures"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c"
checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
dependencies = [
"futures-channel",
"futures-core",
@ -639,9 +725,9 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050"
checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
dependencies = [
"futures-core",
"futures-sink",
@ -649,15 +735,15 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf"
checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
[[package]]
name = "futures-executor"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab"
checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"
dependencies = [
"futures-core",
"futures-task",
@ -666,15 +752,15 @@ dependencies = [
[[package]]
name = "futures-io"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68"
checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
[[package]]
name = "futures-macro"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17"
checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
dependencies = [
"proc-macro2",
"quote",
@ -683,15 +769,15 @@ dependencies = [
[[package]]
name = "futures-sink"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56"
checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
[[package]]
name = "futures-task"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1"
checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
[[package]]
name = "futures-timer"
@ -701,9 +787,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
[[package]]
name = "futures-util"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90"
checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
dependencies = [
"futures-channel",
"futures-core",
@ -739,9 +825,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.7"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
"libc",
@ -898,18 +984,28 @@ dependencies = [
[[package]]
name = "iana-time-zone"
version = "0.1.48"
version = "0.1.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0"
checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"once_cell",
"wasm-bindgen",
"winapi",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
dependencies = [
"cxx",
"cxx-build",
]
[[package]]
name = "ident_case"
version = "1.0.1"
@ -1010,15 +1106,15 @@ dependencies = [
[[package]]
name = "itoa"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
[[package]]
name = "jobserver"
version = "0.1.24"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b"
dependencies = [
"libc",
]
@ -1040,18 +1136,17 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.132"
version = "0.2.135"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
[[package]]
name = "lock_api"
version = "0.4.8"
name = "link-cplusplus"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390"
checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369"
dependencies = [
"autocfg",
"scopeguard",
"cc",
]
[[package]]
@ -1187,9 +1282,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.14.0"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0"
checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
[[package]]
name = "opaque-debug"
@ -1210,26 +1305,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "parking_lot"
version = "0.12.1"
name = "pariter"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
checksum = "324a62b9e7b5f270c0acc92a2040f8028bb643f959f9c068f11a7864f327e3d9"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-sys 0.36.1",
"crossbeam",
"crossbeam-channel",
"num_cpus",
]
[[package]]
@ -1333,9 +1416,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.43"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
dependencies = [
"unicode-ident",
]
@ -1403,13 +1486,37 @@ dependencies = [
[[package]]
name = "rand_core"
version = "0.6.3"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
@ -1561,8 +1668,6 @@ version = "0.3.2-dev"
dependencies = [
"aes256ctr_poly1305aes",
"anyhow",
"async-recursion",
"async-trait",
"backoff",
"base64",
"binrw",
@ -1573,6 +1678,7 @@ dependencies = [
"chrono",
"clap",
"clap_complete",
"crossbeam-channel",
"derivative",
"derive-getters",
"derive_more",
@ -1581,7 +1687,6 @@ dependencies = [
"enum-map",
"enum-map-derive",
"filetime",
"futures",
"gethostname",
"hex",
"humantime",
@ -1593,11 +1698,13 @@ dependencies = [
"log",
"merge",
"nix",
"pariter",
"path-absolutize",
"prettytable-rs",
"quickcheck",
"quickcheck_macros",
"rand",
"rayon",
"reqwest",
"rpassword",
"rstest",
@ -1611,7 +1718,6 @@ dependencies = [
"sha2",
"simplelog",
"thiserror",
"tokio",
"toml",
"users",
"walkdir",
@ -1620,9 +1726,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.20.6"
version = "0.20.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033"
checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c"
dependencies = [
"log",
"ring",
@ -1675,6 +1781,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "scratch"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
[[package]]
name = "scrypt"
version = "0.10.0"
@ -1816,15 +1928,6 @@ dependencies = [
"digest",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
dependencies = [
"libc",
]
[[package]]
name = "simplelog"
version = "0.12.0"
@ -1845,12 +1948,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
[[package]]
name = "socket2"
version = "0.4.7"
@ -1881,9 +1978,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.99"
version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d"
dependencies = [
"proc-macro2",
"quote",
@ -1982,9 +2079,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b"
checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c"
dependencies = [
"itoa",
"libc",
@ -2026,25 +2123,11 @@ dependencies = [
"memchr",
"mio",
"num_cpus",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"winapi",
]
[[package]]
name = "tokio-macros"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-rustls"
version = "0.23.4"
@ -2087,9 +2170,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
[[package]]
name = "tracing"
version = "0.1.36"
version = "0.1.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307"
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if",
"pin-project-lite",
@ -2098,9 +2181,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.29"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7"
checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
dependencies = [
"once_cell",
]
@ -2125,15 +2208,15 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
[[package]]
name = "unicode-ident"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
[[package]]
name = "unicode-normalization"
version = "0.1.21"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6"
checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
dependencies = [
"tinyvec",
]
@ -2302,9 +2385,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
version = "0.22.4"
version = "0.22.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf"
checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be"
dependencies = [
"webpki",
]

13
Cargo.toml
View File

@ -23,8 +23,6 @@ strip = true
[dependencies]
# macros
async-trait = "0.1"
async-recursion = "1"
anyhow = "1"
thiserror = "1"
derive_more = "0.99"
@ -32,9 +30,10 @@ derivative = "2"
derive-getters = "0.2"
lazy_static = "1"
log = "0.4"
# async
tokio = { version = "1", features = ["full"] }
futures = "0.3"
# parallelize
crossbeam-channel = "0.5"
rayon = "1"
pariter = "0.5"
#crypto
aes256ctr_poly1305aes = "0.1"
sha2 = "0.10"
@ -64,8 +63,8 @@ ignore = "0.4"
nix = "0.25"
filetime = "0.2"
# rest backend
reqwest = {version = "0.11", default-features = false, features = ["json", "rustls-tls", "stream"] }
backoff = { version = "0.4", features = ["tokio"] }
reqwest = {version = "0.11", default-features = false, features = ["json", "rustls-tls", "stream", "blocking"] }
backoff = "0.4"
# rclone backend
semver = "1"
# cache

87
src/archiver/archiver_impl.rs
View File

@ -5,10 +5,9 @@ use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use bytesize::ByteSize;
use chrono::Local;
use futures::{stream::FuturesOrdered, StreamExt};
use indicatif::ProgressBar;
use log::*;
use tokio::spawn;
use pariter::IteratorExt;
use crate::backend::DecryptWriteBackend;
use crate::blob::{BlobType, Metadata, Node, NodeType, Packer, Tree};
@ -105,7 +104,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
self.summary.total_dirsize_processed += size;
}
pub async fn add_entry(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> {
pub fn add_entry(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> {
let basepath = if node.is_dir() {
path
} else {
@ -113,7 +112,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
.ok_or_else(|| anyhow!("file path should have a parent!"))?
};
self.finish_trees(basepath).await?;
self.finish_trees(basepath)?;
let missing_dirs = basepath.strip_prefix(&self.path)?;
for p in missing_dirs.iter() {
@ -122,13 +121,13 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
let tree = std::mem::replace(&mut self.tree, Tree::new());
if self.path == path {
// use Node and return
let new_parent = self.parent.sub_parent(&node).await?;
let new_parent = self.parent.sub_parent(&node)?;
let parent = std::mem::replace(&mut self.parent, new_parent);
self.stack.push((node, tree, parent));
return Ok(());
} else {
let node = Node::new_node(p, NodeType::Dir, Metadata::default());
let new_parent = self.parent.sub_parent(&node).await?;
let new_parent = self.parent.sub_parent(&node)?;
let parent = std::mem::replace(&mut self.parent, new_parent);
self.stack.push((node, tree, parent));
};
@ -136,7 +135,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
match node.node_type() {
NodeType::File => {
self.backup_file(path, node, p).await?;
self.backup_file(path, node, p)?;
}
NodeType::Dir => {} // is already handled, see above
_ => self.add_file(node, 0), // all other cases: just save the given node
@ -144,7 +143,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
Ok(())
}
pub async fn finish_trees(&mut self, path: &Path) -> Result<()> {
pub fn finish_trees(&mut self, path: &Path) -> Result<()> {
while !path.starts_with(&self.path) {
// save tree and go back to parent dir
let (chunk, id) = self.tree.serialize()?;
@ -158,13 +157,13 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
self.tree = tree;
self.parent = parent;
self.backup_tree(node, chunk).await?;
self.backup_tree(node, chunk)?;
self.path.pop();
}
Ok(())
}
pub async fn backup_tree(&mut self, node: Node, chunk: Vec<u8>) -> Result<()> {
pub fn backup_tree(&mut self, node: Node, chunk: Vec<u8>) -> Result<()> {
let dirsize = chunk.len() as u64;
let dirsize_bytes = ByteSize(dirsize).to_string_as(true);
let id = node.subtree().unwrap();
@ -188,7 +187,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
}
if !self.index.has_tree(&id) {
match self.tree_packer.add(&chunk, &id).await? {
match self.tree_packer.add(&chunk, &id)? {
0 => {}
packed_size => {
self.summary.tree_blobs += 1;
@ -203,7 +202,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
Ok(())
}
pub async fn backup_file(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> {
pub fn backup_file(&mut self, path: &Path, node: Node, p: ProgressBar) -> Result<()> {
if let ParentResult::Matched(p_node) = self.parent.is_parent(&node) {
if p_node.content().iter().all(|id| self.index.has_data(id)) {
let size = *p_node.meta().size();
@ -220,37 +219,37 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
}
}
let f = File::open(path)?;
self.backup_reader(f, node, p).await
self.backup_reader(f, node, p)
}
pub async fn backup_reader(&mut self, r: impl Read, node: Node, p: ProgressBar) -> Result<()> {
pub fn backup_reader(
&mut self,
r: impl Read + 'static,
node: Node,
p: ProgressBar,
) -> Result<()> {
let chunk_iter = ChunkIter::new(r, *node.meta().size() as usize, &self.poly);
let mut content = Vec::new();
let mut filesize: u64 = 0;
let mut queue = FuturesOrdered::new();
for chunk in chunk_iter {
let chunk = chunk?;
let size = chunk.len() as u64;
filesize += size;
queue.push_back(spawn(async move {
chunk_iter
.into_iter()
// TODO: This parallelization works pretty well for big files. For small files this produces a lot of
// unneccessary overhead. Maybe use a parallel hashing actor?
.parallel_map(|chunk| {
let chunk = chunk?;
let id = hash(&chunk);
(id, chunk, size)
}));
Ok((chunk, id))
})
.try_for_each(|data: Result<_>| -> Result<_> {
let (chunk, id) = data?;
let size = chunk.len() as u64;
filesize += size;
if queue.len() > 8 {
let (id, chunk, size) = queue.next().await.unwrap()?;
self.process_data_junk(id, &chunk, size, &p).await?;
content.push(id);
}
}
while let Some(Ok((id, chunk, size))) = queue.next().await {
self.process_data_junk(id, &chunk, size, &p).await?;
content.push(id);
}
self.process_data_junk(id, &chunk, size, &p)?;
Ok(())
})?;
let mut node = node;
node.set_content(content);
@ -258,7 +257,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
Ok(())
}
async fn process_data_junk(
fn process_data_junk(
&mut self,
id: Id,
chunk: &[u8],
@ -266,7 +265,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
p: &ProgressBar,
) -> Result<()> {
if !self.index.has_data(&id) {
match self.data_packer.add(chunk, &id).await? {
match self.data_packer.add(chunk, &id)? {
0 => {}
packed_size => {
self.summary.data_blobs += 1;
@ -281,20 +280,20 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
Ok(())
}
pub async fn finalize_snapshot(mut self) -> Result<SnapshotFile> {
self.finish_trees(&PathBuf::from("/")).await?;
pub fn finalize_snapshot(mut self) -> Result<SnapshotFile> {
self.finish_trees(&PathBuf::from("/"))?;
let (chunk, id) = self.tree.serialize()?;
if !self.index.has_tree(&id) {
self.tree_packer.add(&chunk, &id).await?;
self.tree_packer.add(&chunk, &id)?;
}
self.snap.tree = id;
self.data_packer.finalize().await?;
self.tree_packer.finalize().await?;
self.data_packer.finalize()?;
self.tree_packer.finalize()?;
{
let indexer = self.indexer.write().await;
indexer.finalize().await?;
let indexer = self.indexer.write().unwrap();
indexer.finalize()?;
}
let end_time = Local::now();
self.summary.backup_duration = (end_time - self.summary.backup_start)
@ -303,7 +302,7 @@ impl<BE: DecryptWriteBackend, I: IndexedBackend> Archiver<BE, I> {
self.summary.total_duration = (end_time - self.snap.time).to_std()?.as_secs_f64();
self.summary.backup_end = end_time;
self.snap.summary = Some(self.summary);
let id = self.be.save_file(&self.snap).await?;
let id = self.be.save_file(&self.snap)?;
self.snap.id = id;
Ok(self.snap)

8
src/archiver/parent.rs
View File

@ -21,12 +21,12 @@ pub enum ParentResult<T> {
}
impl<BE: IndexedBackend> Parent<BE> {
pub async fn new(be: &BE, tree_id: Option<Id>, ignore_ctime: bool, ignore_inode: bool) -> Self {
pub fn new(be: &BE, tree_id: Option<Id>, ignore_ctime: bool, ignore_inode: bool) -> Self {
// if tree_id is given, load tree from backend. Turn errors into None.
// TODO: print warning when loading failed
let tree = match tree_id {
None => None,
Some(id) => Tree::from_backend(be, id).await.ok(),
Some(id) => Tree::from_backend(be, id).ok(),
};
Self {
tree,
@ -85,14 +85,14 @@ impl<BE: IndexedBackend> Parent<BE> {
}
}
pub async fn sub_parent(&mut self, node: &Node) -> Result<Self> {
pub fn sub_parent(&mut self, node: &Node) -> Result<Self> {
let tree = match self.p_node(node) {
None => None,
Some(p_node) => {
if p_node.node_type() == node.node_type() {
// TODO: print warning when loading failed
let tree = p_node.subtree.unwrap();
Some(Tree::from_backend(&self.be, tree).await.ok()).flatten()
Some(Tree::from_backend(&self.be, tree).ok()).flatten()
} else {
None
}

77
src/backend/cache.rs
View File

@ -4,7 +4,6 @@ use std::io::{Read, Seek, SeekFrom, Write};
use std::path::PathBuf;
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use bytes::Bytes;
use dirs::cache_dir;
use log::*;
@ -24,7 +23,6 @@ impl<BE: WriteBackend> CachedBackend<BE> {
}
}
#[async_trait]
impl<BE: WriteBackend> ReadBackend for CachedBackend<BE> {
fn location(&self) -> &str {
self.be.location()
@ -34,27 +32,27 @@ impl<BE: WriteBackend> ReadBackend for CachedBackend<BE> {
self.be.set_option(option, value)
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
let list = self.be.list_with_size(tpe).await?;
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
let list = self.be.list_with_size(tpe)?;
if let Some(cache) = &self.cache {
if tpe.is_cacheable() {
cache.remove_not_in_list(tpe, &list).await?;
cache.remove_not_in_list(tpe, &list)?;
}
}
Ok(list)
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
match (&self.cache, tpe.is_cacheable()) {
(None, _) | (Some(_), false) => self.be.read_full(tpe, id).await,
(Some(cache), true) => match cache.read_full(tpe, id).await {
(None, _) | (Some(_), false) => self.be.read_full(tpe, id),
(Some(cache), true) => match cache.read_full(tpe, id) {
Ok(res) => Ok(res),
_ => {
let res = self.be.read_full(tpe, id).await;
let res = self.be.read_full(tpe, id);
if let Ok(data) = &res {
let _ = cache.write_bytes(tpe, id, data.clone()).await;
let _ = cache.write_bytes(tpe, id, data.clone());
}
res
}
@ -62,7 +60,7 @@ impl<BE: WriteBackend> ReadBackend for CachedBackend<BE> {
}
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -72,21 +70,19 @@ impl<BE: WriteBackend> ReadBackend for CachedBackend<BE> {
) -> Result<Bytes> {
match (&self.cache, cacheable || tpe.is_cacheable()) {
(None, _) | (Some(_), false) => {
self.be
.read_partial(tpe, id, cacheable, offset, length)
.await
self.be.read_partial(tpe, id, cacheable, offset, length)
}
(Some(cache), true) => match cache.read_partial(tpe, id, offset, length).await {
(Some(cache), true) => match cache.read_partial(tpe, id, offset, length) {
Ok(res) => Ok(res),
_ => match self.be.read_full(tpe, id).await {
_ => match self.be.read_full(tpe, id) {
// read full file, save to cache and return partial content from cache
// TODO: - Do not read to memory, but use a (Async)Reader
// TODO: - Do not read to memory, but use a Reader
// - Don't read from cache, but use the right part of the read content
Ok(data) => {
if cache.write_bytes(tpe, id, data.clone()).await.is_ok() {
cache.read_partial(tpe, id, offset, length).await
if cache.write_bytes(tpe, id, data).is_ok() {
cache.read_partial(tpe, id, offset, length)
} else {
self.be.read_partial(tpe, id, false, offset, length).await
self.be.read_partial(tpe, id, false, offset, length)
}
}
error => error,
@ -96,28 +92,27 @@ impl<BE: WriteBackend> ReadBackend for CachedBackend<BE> {
}
}
#[async_trait]
impl<BE: WriteBackend> WriteBackend for CachedBackend<BE> {
async fn create(&self) -> Result<()> {
self.be.create().await
fn create(&self) -> Result<()> {
self.be.create()
}
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
if let Some(cache) = &self.cache {
if cacheable || tpe.is_cacheable() {
let _ = cache.write_bytes(tpe, id, buf.clone()).await;
let _ = cache.write_bytes(tpe, id, buf.clone());
}
}
self.be.write_bytes(tpe, id, cacheable, buf).await
self.be.write_bytes(tpe, id, cacheable, buf)
}
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
if let Some(cache) = &self.cache {
if cacheable || tpe.is_cacheable() {
let _ = cache.remove(tpe, id).await;
let _ = cache.remove(tpe, id);
}
}
self.be.remove(tpe, id, cacheable).await
self.be.remove(tpe, id, cacheable)
}
}
@ -154,7 +149,7 @@ impl Cache {
self.path.join(tpe.name()).join(&hex_id[0..2]).join(&hex_id)
}
pub async fn list_with_size(&self, tpe: FileType) -> Result<HashMap<Id, u32>> {
pub fn list_with_size(&self, tpe: FileType) -> Result<HashMap<Id, u32>> {
let path = self.path.join(tpe.name());
let walker = WalkDir::new(path)
@ -183,38 +178,32 @@ impl Cache {
Ok(walker.collect())
}
pub async fn remove_not_in_list(&self, tpe: FileType, list: &Vec<(Id, u32)>) -> Result<()> {
let mut list_cache = self.list_with_size(tpe).await?;
pub fn remove_not_in_list(&self, tpe: FileType, list: &Vec<(Id, u32)>) -> Result<()> {
let mut list_cache = self.list_with_size(tpe)?;
// remove present files from the cache list
for (id, size) in list {
if let Some(cached_size) = list_cache.remove(id) {
if &cached_size != size {
// remove cache files with non-matching size
self.remove(tpe, id).await?;
self.remove(tpe, id)?;
}
}
}
// remove all remaining (i.e. not present in repo) cache files
for id in list_cache.keys() {
self.remove(tpe, id).await?;
self.remove(tpe, id)?;
}
Ok(())
}
pub async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
pub fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
trace!("cache reading tpe: {:?}, id: {}", &tpe, &id);
let data = fs::read(self.path(tpe, id))?;
trace!("cache hit!");
Ok(data.into())
}
async fn read_partial(
&self,
tpe: FileType,
id: &Id,
offset: u32,
length: u32,
) -> Result<Bytes> {
fn read_partial(&self, tpe: FileType, id: &Id, offset: u32, length: u32) -> Result<Bytes> {
trace!(
"cache reading tpe: {:?}, id: {}, offset: {}",
&tpe,
@ -229,7 +218,7 @@ impl Cache {
Ok(vec.into())
}
async fn write_bytes(&self, tpe: FileType, id: &Id, buf: Bytes) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, buf: Bytes) -> Result<()> {
trace!("cache writing tpe: {:?}, id: {}", &tpe, &id);
fs::create_dir_all(self.dir(tpe, id))?;
let filename = self.path(tpe, id);
@ -241,7 +230,7 @@ impl Cache {
Ok(())
}
async fn remove(&self, tpe: FileType, id: &Id) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id) -> Result<()> {
trace!("cache writing tpe: {:?}, id: {}", &tpe, &id);
let filename = self.path(tpe, id);
fs::remove_file(filename)?;

55
src/backend/choose.rs
View File

@ -1,5 +1,4 @@
use anyhow::{bail, Result};
use async_trait::async_trait;
use bytes::Bytes;
use super::{FileType, Id, ReadBackend, WriteBackend};
@ -26,7 +25,6 @@ impl ChooseBackend {
}
}
#[async_trait]
impl ReadBackend for ChooseBackend {
fn location(&self) -> &str {
match self {
@ -44,23 +42,23 @@ impl ReadBackend for ChooseBackend {
}
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
match self {
Local(local) => local.list_with_size(tpe).await,
Rest(rest) => rest.list_with_size(tpe).await,
Rclone(rclone) => rclone.list_with_size(tpe).await,
Local(local) => local.list_with_size(tpe),
Rest(rest) => rest.list_with_size(tpe),
Rclone(rclone) => rclone.list_with_size(tpe),
}
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
match self {
Local(local) => local.read_full(tpe, id).await,
Rest(rest) => rest.read_full(tpe, id).await,
Rclone(rclone) => rclone.read_full(tpe, id).await,
Local(local) => local.read_full(tpe, id),
Rest(rest) => rest.read_full(tpe, id),
Rclone(rclone) => rclone.read_full(tpe, id),
}
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -69,40 +67,35 @@ impl ReadBackend for ChooseBackend {
length: u32,
) -> Result<Bytes> {
match self {
Local(local) => local.read_partial(tpe, id, cacheable, offset, length).await,
Rest(rest) => rest.read_partial(tpe, id, cacheable, offset, length).await,
Rclone(rclone) => {
rclone
.read_partial(tpe, id, cacheable, offset, length)
.await
}
Local(local) => local.read_partial(tpe, id, cacheable, offset, length),
Rest(rest) => rest.read_partial(tpe, id, cacheable, offset, length),
Rclone(rclone) => rclone.read_partial(tpe, id, cacheable, offset, length),
}
}
}
#[async_trait]
impl WriteBackend for ChooseBackend {
async fn create(&self) -> Result<()> {
fn create(&self) -> Result<()> {
match self {
Local(local) => local.create().await,
Rest(rest) => rest.create().await,
Rclone(rclone) => rclone.create().await,
Local(local) => local.create(),
Rest(rest) => rest.create(),
Rclone(rclone) => rclone.create(),
}
}
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
match self {
Local(local) => local.write_bytes(tpe, id, cacheable, buf).await,
Rest(rest) => rest.write_bytes(tpe, id, cacheable, buf).await,
Rclone(rclone) => rclone.write_bytes(tpe, id, cacheable, buf).await,
Local(local) => local.write_bytes(tpe, id, cacheable, buf),
Rest(rest) => rest.write_bytes(tpe, id, cacheable, buf),
Rclone(rclone) => rclone.write_bytes(tpe, id, cacheable, buf),
}
}
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
match self {
Local(local) => local.remove(tpe, id, cacheable).await,
Rest(rest) => rest.remove(tpe, id, cacheable).await,
Rclone(rclone) => rclone.remove(tpe, id, cacheable).await,
Local(local) => local.remove(tpe, id, cacheable),
Rest(rest) => rest.remove(tpe, id, cacheable),
Rclone(rclone) => rclone.remove(tpe, id, cacheable),
}
}
}

129
src/backend/decrypt.rs
View File

@ -1,11 +1,10 @@
use std::num::NonZeroU32;
use anyhow::{bail, Result};
use async_trait::async_trait;
use bytes::Bytes;
use futures::{stream, stream::FuturesUnordered, StreamExt};
use crossbeam_channel::{unbounded, Receiver};
use indicatif::ProgressBar;
use tokio::{spawn, task::JoinHandle};
use rayon::prelude::*;
use zstd::stream::{copy_encode, decode_all};
use super::{FileType, Id, ReadBackend, RepoFile, WriteBackend};
@ -14,12 +13,11 @@ use crate::crypto::{hash, CryptoKey};
pub trait DecryptFullBackend: DecryptWriteBackend + DecryptReadBackend {}
impl<T: DecryptWriteBackend + DecryptReadBackend> DecryptFullBackend for T {}
#[async_trait]
pub trait DecryptReadBackend: ReadBackend {
fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>>;
async fn read_encrypted_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
let decrypted = self.decrypt(&self.read_full(tpe, id).await?)?;
fn read_encrypted_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
let decrypted = self.decrypt(&self.read_full(tpe, id)?)?;
Ok(match decrypted[0] {
b'{' | b'[' => decrypted, // not compressed
2 => decode_all(&decrypted[1..])?, // 2 indicates compressed data following
@ -28,7 +26,7 @@ pub trait DecryptReadBackend: ReadBackend {
.into())
}
async fn read_encrypted_partial(
fn read_encrypted_partial(
&self,
tpe: FileType,
id: &Id,
@ -37,11 +35,7 @@ pub trait DecryptReadBackend: ReadBackend {
length: u32,
uncompressed_length: Option<NonZeroU32>,
) -> Result<Bytes> {
let mut data = self.decrypt(
&self
.read_partial(tpe, id, cacheable, offset, length)
.await?,
)?;
let mut data = self.decrypt(&self.read_partial(tpe, id, cacheable, offset, length)?)?;
if let Some(length) = uncompressed_length {
data = decode_all(&*data).unwrap();
if data.len() != length.get() as usize {
@ -51,70 +45,51 @@ pub trait DecryptReadBackend: ReadBackend {
Ok(data.into())
}
async fn get_file<F: RepoFile>(&self, id: &Id) -> Result<F> {
let data = self.read_encrypted_full(F::TYPE, id).await?;
fn get_file<F: RepoFile>(&self, id: &Id) -> Result<F> {
let data = self.read_encrypted_full(F::TYPE, id)?;
Ok(serde_json::from_slice(&data)?)
}
async fn stream_all<F: RepoFile>(
&self,
p: ProgressBar,
) -> Result<FuturesUnordered<JoinHandle<(Id, F)>>> {
let list = self.list(F::TYPE).await?;
self.stream_list(list, p).await
fn stream_all<F: RepoFile>(&self, p: ProgressBar) -> Result<Receiver<(Id, F)>> {
let list = self.list(F::TYPE)?;
self.stream_list(list, p)
}
async fn stream_list<F: RepoFile>(
&self,
list: Vec<Id>,
p: ProgressBar,
) -> Result<FuturesUnordered<JoinHandle<(Id, F)>>> {
fn stream_list<F: RepoFile>(&self, list: Vec<Id>, p: ProgressBar) -> Result<Receiver<(Id, F)>> {
p.set_length(list.len() as u64);
let (tx, rx) = unbounded();
let stream: FuturesUnordered<_> = list
.into_iter()
.map(|id| {
let be = self.clone();
let p = p.clone();
spawn(async move {
let file = be.get_file::<F>(&id).await.unwrap();
p.inc(1);
(id, file)
})
})
.collect();
Ok(stream)
list.into_par_iter()
.for_each_with((self, p, tx), |(be, p, tx), id| {
let file = be.get_file::<F>(&id).unwrap();
p.inc(1);
tx.send((id, file)).unwrap();
});
Ok(rx)
}
}
#[async_trait]
pub trait DecryptWriteBackend: WriteBackend {
type Key: CryptoKey;
fn key(&self) -> &Self::Key;
async fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result<Id>;
fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result<Id>;
async fn save_file<F: RepoFile>(&self, file: &F) -> Result<Id> {
fn save_file<F: RepoFile>(&self, file: &F) -> Result<Id> {
let data = serde_json::to_vec(file)?;
Ok(self.hash_write_full(F::TYPE, &data).await?)
self.hash_write_full(F::TYPE, &data)
}
async fn save_list<F: RepoFile>(&self, list: Vec<F>, p: ProgressBar) -> Result<()> {
fn save_list<F: RepoFile>(&self, list: Vec<F>, p: ProgressBar) -> Result<()> {
p.set_length(list.len() as u64);
stream::iter(list.into_iter().map(|file| {
let be = self.clone();
let p = p.clone();
(file, be, p)
}))
.for_each_concurrent(5, |(file, be, p)| async move {
be.save_file(&file).await.unwrap();
list.par_iter().for_each(|file| {
self.save_file(file).unwrap();
p.inc(1);
})
.await;
});
p.finish();
Ok(())
}
async fn delete_list(
fn delete_list(
&self,
tpe: FileType,
cacheable: bool,
@ -122,16 +97,10 @@ pub trait DecryptWriteBackend: WriteBackend {
p: ProgressBar,
) -> Result<()> {
p.set_length(list.len() as u64);
stream::iter(list.into_iter().map(|id| {
let be = self.clone();
let p = p.clone();
(id, be, p)
}))
.for_each_concurrent(20, |(id, be, p)| async move {
be.remove(tpe, &id, cacheable).await.unwrap();
list.par_iter().for_each(|id| {
self.remove(tpe, id, cacheable).unwrap();
p.inc(1);
})
.await;
});
p.finish();
Ok(())
@ -157,13 +126,13 @@ impl<R: ReadBackend, C: CryptoKey> DecryptBackend<R, C> {
}
}
#[async_trait]
impl<R: WriteBackend, C: CryptoKey> DecryptWriteBackend for DecryptBackend<R, C> {
type Key = C;
fn key(&self) -> &Self::Key {
&self.key
}
async fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result<Id> {
fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result<Id> {
let data = match self.zstd {
Some(level) => {
let mut out = vec![2_u8];
@ -173,7 +142,7 @@ impl<R: WriteBackend, C: CryptoKey> DecryptWriteBackend for DecryptBackend<R, C>
None => self.key().encrypt_data(data)?,
};
let id = hash(&data);
self.write_bytes(tpe, &id, false, data.into()).await?;
self.write_bytes(tpe, &id, false, data.into())?;
Ok(id)
}
@ -182,14 +151,12 @@ impl<R: WriteBackend, C: CryptoKey> DecryptWriteBackend for DecryptBackend<R, C>
}
}
#[async_trait]
impl<R: ReadBackend, C: CryptoKey> DecryptReadBackend for DecryptBackend<R, C> {
fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>> {
Ok(self.key.decrypt_data(data)?)
}
}
#[async_trait]
impl<R: ReadBackend, C: CryptoKey> ReadBackend for DecryptBackend<R, C> {
fn location(&self) -> &str {
self.backend.location()
@ -199,19 +166,19 @@ impl<R: ReadBackend, C: CryptoKey> ReadBackend for DecryptBackend<R, C> {
self.backend.set_option(option, value)
}
async fn list(&self, tpe: FileType) -> Result<Vec<Id>> {
self.backend.list(tpe).await
fn list(&self, tpe: FileType) -> Result<Vec<Id>> {
self.backend.list(tpe)
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.backend.list_with_size(tpe).await
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.backend.list_with_size(tpe)
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
self.backend.read_full(tpe, id).await
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
self.backend.read_full(tpe, id)
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -221,21 +188,19 @@ impl<R: ReadBackend, C: CryptoKey> ReadBackend for DecryptBackend<R, C> {
) -> Result<Bytes> {
self.backend
.read_partial(tpe, id, cacheable, offset, length)
.await
}
}
#[async_trait]
impl<R: WriteBackend, C: CryptoKey> WriteBackend for DecryptBackend<R, C> {
async fn create(&self) -> Result<()> {
self.backend.create().await
fn create(&self) -> Result<()> {
self.backend.create()
}
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
self.backend.write_bytes(tpe, id, cacheable, buf).await
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
self.backend.write_bytes(tpe, id, cacheable, buf)
}
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
self.backend.remove(tpe, id, cacheable).await
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
self.backend.remove(tpe, id, cacheable)
}
}

35
src/backend/dry_run.rs
View File

@ -1,5 +1,4 @@
use anyhow::Result;
use async_trait::async_trait;
use bytes::Bytes;
use super::{
@ -19,14 +18,12 @@ impl<BE: DecryptFullBackend> DryRunBackend<BE> {
}
}
#[async_trait]
impl<BE: DecryptFullBackend> DecryptReadBackend for DryRunBackend<BE> {
fn decrypt(&self, data: &[u8]) -> Result<Vec<u8>> {
self.be.decrypt(data)
}
}
#[async_trait]
impl<BE: DecryptFullBackend> ReadBackend for DryRunBackend<BE> {
fn location(&self) -> &str {
self.be.location()
@ -36,15 +33,15 @@ impl<BE: DecryptFullBackend> ReadBackend for DryRunBackend<BE> {
self.be.set_option(option, value)
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.be.list_with_size(tpe).await
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.be.list_with_size(tpe)
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
self.be.read_full(tpe, id).await
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
self.be.read_full(tpe, id)
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -52,13 +49,10 @@ impl<BE: DecryptFullBackend> ReadBackend for DryRunBackend<BE> {
offset: u32,
length: u32,
) -> Result<Bytes> {
self.be
.read_partial(tpe, id, cacheable, offset, length)
.await
self.be.read_partial(tpe, id, cacheable, offset, length)
}
}
#[async_trait]
impl<BE: DecryptFullBackend> DecryptWriteBackend for DryRunBackend<BE> {
type Key = <BE as DecryptWriteBackend>::Key;
@ -66,10 +60,10 @@ impl<BE: DecryptFullBackend> DecryptWriteBackend for DryRunBackend<BE> {
self.be.key()
}
async fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result<Id> {
fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> Result<Id> {
match self.dry_run {
true => Ok(Id::default()),
false => self.be.hash_write_full(tpe, data).await,
false => self.be.hash_write_full(tpe, data),
}
}
@ -81,26 +75,25 @@ impl<BE: DecryptFullBackend> DecryptWriteBackend for DryRunBackend<BE> {
}
}
#[async_trait]
impl<BE: DecryptFullBackend> WriteBackend for DryRunBackend<BE> {
async fn create(&self) -> Result<()> {
fn create(&self) -> Result<()> {
match self.dry_run {
true => Ok(()),
false => self.be.create().await,
false => self.be.create(),
}
}
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
match self.dry_run {
true => Ok(()),
false => self.be.write_bytes(tpe, id, cacheable, buf).await,
false => self.be.write_bytes(tpe, id, cacheable, buf),
}
}
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
match self.dry_run {
true => Ok(()),
false => self.be.remove(tpe, id, cacheable).await,
false => self.be.remove(tpe, id, cacheable),
}
}
}

37
src/backend/hotcold.rs
View File

@ -1,5 +1,4 @@
use anyhow::Result;
use async_trait::async_trait;
use bytes::Bytes;
use super::{FileType, Id, ReadBackend, WriteBackend};
@ -16,7 +15,6 @@ impl<BE: WriteBackend> HotColdBackend<BE> {
}
}
#[async_trait]
impl<BE: WriteBackend> ReadBackend for HotColdBackend<BE> {
fn location(&self) -> &str {
self.be.location()
@ -26,18 +24,18 @@ impl<BE: WriteBackend> ReadBackend for HotColdBackend<BE> {
self.be.set_option(option, value)
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.be.list_with_size(tpe).await
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.be.list_with_size(tpe)
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
match &self.hot_be {
None => self.be.read_full(tpe, id).await,
Some(be) => be.read_full(tpe, id).await,
None => self.be.read_full(tpe, id),
Some(be) => be.read_full(tpe, id),
}
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -47,36 +45,33 @@ impl<BE: WriteBackend> ReadBackend for HotColdBackend<BE> {
) -> Result<Bytes> {
match (&self.hot_be, cacheable || tpe != FileType::Pack) {
(None, _) | (Some(_), false) => {
self.be
.read_partial(tpe, id, cacheable, offset, length)
.await
self.be.read_partial(tpe, id, cacheable, offset, length)
}
(Some(be), true) => be.read_partial(tpe, id, cacheable, offset, length).await,
(Some(be), true) => be.read_partial(tpe, id, cacheable, offset, length),
}
}
}
#[async_trait]
impl<BE: WriteBackend> WriteBackend for HotColdBackend<BE> {
async fn create(&self) -> Result<()> {
self.be.create().await
fn create(&self) -> Result<()> {
self.be.create()
}
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
if let Some(be) = &self.hot_be {
if tpe != FileType::Config && (cacheable || tpe != FileType::Pack) {
be.write_bytes(tpe, id, cacheable, buf.clone()).await?;
be.write_bytes(tpe, id, cacheable, buf.clone())?;
}
}
self.be.write_bytes(tpe, id, cacheable, buf).await
self.be.write_bytes(tpe, id, cacheable, buf)
}
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
// First remove cold file
self.be.remove(tpe, id, cacheable).await?;
self.be.remove(tpe, id, cacheable)?;
if let Some(be) = &self.hot_be {
if cacheable || tpe != FileType::Pack {
be.remove(tpe, id, cacheable).await?;
be.remove(tpe, id, cacheable)?;
}
}
Ok(())

23
src/backend/local.rs
View File

@ -4,7 +4,6 @@ use std::os::unix::fs::{symlink, FileExt, PermissionsExt};
use std::path::{Path, PathBuf};
use anyhow::Result;
use async_trait::async_trait;
use bytes::Bytes;
use filetime::{set_file_atime, set_file_mtime, FileTime};
use log::*;
@ -38,7 +37,6 @@ impl LocalBackend {
}
}
#[async_trait]
impl ReadBackend for LocalBackend {
fn location(&self) -> &str {
self.path.to_str().unwrap()
@ -48,7 +46,7 @@ impl ReadBackend for LocalBackend {
Ok(())
}
async fn list(&self, tpe: FileType) -> Result<Vec<Id>> {
fn list(&self, tpe: FileType) -> Result<Vec<Id>> {
if tpe == FileType::Config {
return Ok(match self.path.join("config").exists() {
true => vec![Id::default()],
@ -65,7 +63,7 @@ impl ReadBackend for LocalBackend {
Ok(walker.collect())
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
let path = self.path.join(tpe.name());
if tpe == FileType::Config {
@ -104,11 +102,11 @@ impl ReadBackend for LocalBackend {
Ok(walker.collect())
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
Ok(fs::read(self.path(tpe, id))?.into())
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -124,9 +122,8 @@ impl ReadBackend for LocalBackend {
}
}
#[async_trait]
impl WriteBackend for LocalBackend {
async fn create(&self) -> Result<()> {
fn create(&self) -> Result<()> {
for tpe in ALL_FILE_TYPES {
fs::create_dir_all(self.path.join(tpe.name()))?;
}
@ -136,13 +133,7 @@ impl WriteBackend for LocalBackend {
Ok(())
}
async fn write_bytes(
&self,
tpe: FileType,
id: &Id,
_cacheable: bool,
buf: Bytes,
) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> {
trace!("writing tpe: {:?}, id: {}", &tpe, &id);
let filename = self.path(tpe, id);
let mut file = fs::OpenOptions::new()
@ -155,7 +146,7 @@ impl WriteBackend for LocalBackend {
Ok(())
}
async fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> {
trace!("writing tpe: {:?}, id: {}", &tpe, &id);
let filename = self.path(tpe, id);
fs::remove_file(filename)?;

33
src/backend/mod.rs
View File

@ -2,7 +2,6 @@ use std::io::Read;
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use bytes::Bytes;
use serde::{de::DeserializeOwned, Serialize};
@ -70,25 +69,23 @@ pub trait RepoFile: Serialize + DeserializeOwned + Sized + Send + Sync + 'static
const TYPE: FileType;
}
#[async_trait]
pub trait ReadBackend: Clone + Send + Sync + 'static {
fn location(&self) -> &str;
fn set_option(&mut self, option: &str, value: &str) -> Result<()>;
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>>;
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>>;
async fn list(&self, tpe: FileType) -> Result<Vec<Id>> {
fn list(&self, tpe: FileType) -> Result<Vec<Id>> {
Ok(self
.list_with_size(tpe)
.await?
.list_with_size(tpe)?
.into_iter()
.map(|(id, _)| id)
.collect())
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes>;
async fn read_partial(
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes>;
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -97,7 +94,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static {
length: u32,
) -> Result<Bytes>;
async fn find_starts_with(&self, tpe: FileType, vec: &[String]) -> Result<Vec<Result<Id>>> {
fn find_starts_with(&self, tpe: FileType, vec: &[String]) -> Result<Vec<Result<Id>>> {
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum MapResult<T> {
None,
@ -105,7 +102,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static {
NonUnique,
}
let mut results = vec![MapResult::None; vec.len()];
for id in self.list(tpe).await? {
for id in self.list(tpe)? {
for (i, v) in vec.iter().enumerate() {
if id.to_hex().starts_with(v) {
if results[i] == MapResult::None {
@ -128,30 +125,28 @@ pub trait ReadBackend: Clone + Send + Sync + 'static {
.collect())
}
async fn find_id(&self, tpe: FileType, id: &str) -> Result<Id> {
Ok(self.find_ids(tpe, &[id.to_string()]).await?.remove(0))
fn find_id(&self, tpe: FileType, id: &str) -> Result<Id> {
Ok(self.find_ids(tpe, &[id.to_string()])?.remove(0))
}
async fn find_ids(&self, tpe: FileType, ids: &[String]) -> Result<Vec<Id>> {
fn find_ids(&self, tpe: FileType, ids: &[String]) -> Result<Vec<Id>> {
let long_ids: Vec<_> = ids.iter().map(|id| Id::from_hex(id)).collect();
Ok(match long_ids.iter().all(Result::is_ok) {
true => long_ids.into_iter().map(Result::unwrap).collect(),
// if the given id param are not full Ids, search for a suitable one
false => self
.find_starts_with(tpe, ids)
.await?
.find_starts_with(tpe, ids)?
.into_iter()
.collect::<Result<Vec<_>>>()?,
})
}
}
#[async_trait]
pub trait WriteBackend: ReadBackend {
async fn create(&self) -> Result<()>;
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()>;
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>;
fn create(&self) -> Result<()>;
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()>;
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>;
}
pub trait ReadSource: Iterator<Item = Result<(PathBuf, Node)>> {

32
src/backend/rclone.rs
View File

@ -4,12 +4,10 @@ use std::str;
use std::sync::Arc;
use anyhow::{anyhow, bail, Result};
use async_trait::async_trait;
use bytes::Bytes;
use log::*;
use rand::distributions::{Alphanumeric, DistString};
use rand::thread_rng;
use tokio::task::spawn_blocking;
use super::{FileType, Id, ReadBackend, RestBackend, WriteBackend};
@ -93,7 +91,7 @@ impl RcloneBackend {
}
};
spawn_blocking(move || loop {
std::thread::spawn(move || loop {
let mut line = String::new();
if stderr.read_line(&mut line).unwrap() == 0 {
break;
@ -118,7 +116,6 @@ impl RcloneBackend {
}
}
#[async_trait]
impl ReadBackend for RcloneBackend {
fn location(&self) -> &str {
self.rest.location()
@ -128,15 +125,15 @@ impl ReadBackend for RcloneBackend {
self.rest.set_option(option, value)
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.rest.list_with_size(tpe).await
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
self.rest.list_with_size(tpe)
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
self.rest.read_full(tpe, id).await
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
self.rest.read_full(tpe, id)
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -144,23 +141,20 @@ impl ReadBackend for RcloneBackend {
offset: u32,
length: u32,
) -> Result<Bytes> {
self.rest
.read_partial(tpe, id, cacheable, offset, length)
.await
self.rest.read_partial(tpe, id, cacheable, offset, length)
}
}
#[async_trait]
impl WriteBackend for RcloneBackend {
async fn create(&self) -> Result<()> {
self.rest.create().await
fn create(&self) -> Result<()> {
self.rest.create()
}
async fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
self.rest.write_bytes(tpe, id, cacheable, buf).await
fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> {
self.rest.write_bytes(tpe, id, cacheable, buf)
}
async fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
self.rest.remove(tpe, id, cacheable).await
fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> {
self.rest.remove(tpe, id, cacheable)
}
}

102
src/backend/rest.rs
View File

@ -1,11 +1,13 @@
use std::time::Duration;
use anyhow::{bail, Result};
use async_trait::async_trait;
use backoff::{backoff::Backoff, Error, ExponentialBackoff, ExponentialBackoffBuilder};
use bytes::Bytes;
use log::*;
use reqwest::{Client, Response, Url};
use reqwest::{
blocking::{Client, Response},
Url,
};
use serde::Deserialize;
use super::{FileType, Id, ReadBackend, WriteBackend};
@ -92,7 +94,6 @@ impl RestBackend {
}
}
#[async_trait]
impl ReadBackend for RestBackend {
fn location(&self) -> &str {
self.url.as_str()
@ -117,17 +118,16 @@ impl ReadBackend for RestBackend {
Ok(())
}
async fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
Ok(backoff::future::retry_notify(
fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
Ok(backoff::retry_notify(
self.backoff.clone(),
|| async {
|| {
if tpe == FileType::Config {
return Ok(
match self
.client
.head(self.url.join("config").unwrap())
.send()
.await?
.send()?
.status()
.is_success()
{
@ -152,39 +152,33 @@ impl ReadBackend for RestBackend {
.client
.get(url)
.header("Accept", "application/vnd.x.restic.rest.v2")
.send()
.await?
.send()?
.check_error()?
.json::<Vec<ListEntry>>()
.await?;
.json::<Vec<ListEntry>>()?;
Ok(list.into_iter().map(|i| (i.name, i.size)).collect())
},
notify,
)
.await?)
)?)
}
async fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
Ok(backoff::future::retry_notify(
fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
Ok(backoff::retry_notify(
self.backoff.clone(),
|| async {
|| {
Ok(self
.client
.get(self.url(tpe, id))
.send()
.await?
.send()?
.check_error()?
.bytes()
.await?
.bytes()?
.into_iter()
.collect())
},
notify,
)
.await?)
)?)
}
async fn read_partial(
fn read_partial(
&self,
tpe: FileType,
id: &Id,
@ -194,84 +188,64 @@ impl ReadBackend for RestBackend {
) -> Result<Bytes> {
let offset2 = offset + length - 1;
let header_value = format!("bytes={}-{}", offset, offset2);
Ok(backoff::future::retry_notify(
Ok(backoff::retry_notify(
self.backoff.clone(),
|| async {
|| {
Ok(self
.client
.get(self.url(tpe, id))
.header("Range", header_value.clone())
.send()
.await?
.send()?
.check_error()?
.bytes()
.await?
.bytes()?
.into_iter()
.collect())
},
notify,
)
.await?)
)?)
}
}
#[async_trait]
impl WriteBackend for RestBackend {
async fn create(&self) -> Result<()> {
Ok(backoff::future::retry_notify(
fn create(&self) -> Result<()> {
Ok(backoff::retry_notify(
self.backoff.clone(),
|| async {
|| {
self.client
.post(self.url.join("?create=true").unwrap())
.send()
.await?
.send()?
.check_error()?;
Ok(())
},
notify,
)
.await?)
)?)
}
async fn write_bytes(
&self,
tpe: FileType,
id: &Id,
_cacheable: bool,
buf: Bytes,
) -> Result<()> {
fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> {
trace!("writing tpe: {:?}, id: {}", &tpe, &id);
let req_builder = self.client.post(self.url(tpe, id)).body(buf);
Ok(backoff::future::retry_notify(
Ok(backoff::retry_notify(
self.backoff.clone(),
|| async {
req_builder
.try_clone()
.unwrap()
.send()
.await?
.check_error()?;
|| {
req_builder.try_clone().unwrap().send()?.check_error()?;
Ok(())
},
notify,
)
.await?)
)?)
}
async fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> {
fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> {
trace!("removing tpe: {:?}, id: {}", &tpe, &id);
Ok(backoff::future::retry_notify(
Ok(backoff::retry_notify(
self.backoff.clone(),
|| async {
|| {
self.client
.delete(self.url(tpe, id))
.send()
.await?
.send()?
.check_error()?;
Ok(())
},
notify,
)
.await?)
)?)
}
}

184
src/blob/packer.rs
View File

@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime};
use anyhow::{anyhow, Result};
use bytes::{Bytes, BytesMut};
use chrono::Local;
use tokio::{spawn, task::JoinHandle};
use crossbeam_channel::{bounded, Receiver, Sender};
use zstd::encode_all;
use super::BlobType;
@ -74,7 +74,7 @@ pub struct Packer<BE: DecryptWriteBackend> {
index: IndexPack,
indexer: SharedIndexer<BE>,
hasher: Hasher,
file_writer: FileWriter<BE>,
file_writer: Actor<(Bytes, Id, IndexPack)>,
zstd: Option<i32>,
pack_sizer: PackSizer,
}
@ -87,12 +87,15 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
config: &ConfigFile,
total_size: u64,
) -> Result<Self> {
let file_writer = FileWriter {
future: None,
be: be.clone(),
indexer: indexer.clone(),
cacheable: blob_type.is_cacheable(),
};
let file_writer = Actor::new(
FileWriterHandle {
be: be.clone(),
indexer: indexer.clone(),
cacheable: blob_type.is_cacheable(),
},
1,
1,
);
let zstd = config.zstd()?;
let pack_sizer = PackSizer::from_config(config, blob_type, total_size);
Ok(Self {
@ -111,12 +114,13 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
})
}
pub async fn finalize(&mut self) -> Result<()> {
self.save().await?;
self.file_writer.finalize().await
pub fn finalize(mut self) -> Result<()> {
self.save()?;
self.file_writer.finalize()?;
Ok(())
}
pub async fn write_data(&mut self, data: &[u8]) -> Result<u32> {
pub fn write_data(&mut self, data: &[u8]) -> Result<u32> {
self.hasher.update(data);
let len = data.len().try_into()?;
self.file.extend_from_slice(data);
@ -125,25 +129,20 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
}
// adds the blob to the packfile; returns the actually added size
pub async fn add(&mut self, data: &[u8], id: &Id) -> Result<u64> {
pub fn add(&mut self, data: &[u8], id: &Id) -> Result<u64> {
// compute size limit based on total size and size bounds
let size_limit = self.pack_sizer.pack_size();
self.add_with_sizelimit(data, id, size_limit).await
self.add_with_sizelimit(data, id, size_limit)
}
// adds the blob to the packfile; returns the actually added size
pub async fn add_with_sizelimit(
&mut self,
data: &[u8],
id: &Id,
size_limit: u32,
) -> Result<u64> {
pub fn add_with_sizelimit(&mut self, data: &[u8], id: &Id, size_limit: u32) -> Result<u64> {
// only add if this blob is not present
if self.has(id) {
return Ok(0);
}
{
let indexer = self.indexer.read().await;
let indexer = self.indexer.read().unwrap();
if indexer.has(id) {
return Ok(0);
}
@ -167,13 +166,12 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
};
// add using current total_size as repo_size
self.add_raw(&data, id, uncompressed_length, size_limit)
.await?;
self.add_raw(&data, id, uncompressed_length, size_limit)?;
Ok(data.len().try_into()?)
}
// adds the already compressed/encrypted blob to the packfile without any check
pub async fn add_raw(
pub fn add_raw(
&mut self,
data: &[u8],
id: &Id,
@ -181,7 +179,7 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
size_limit: u32,
) -> Result<()> {
let offset = self.size;
let len = self.write_data(data).await?;
let len = self.write_data(data)?;
self.index
.add(*id, self.blob_type, offset, len, uncompressed_length);
self.count += 1;
@ -190,7 +188,7 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
if self.count >= MAX_COUNT || self.size >= size_limit || self.created.elapsed()? >= MAX_AGE
{
self.pack_sizer.add_size(self.index.pack_size());
self.save().await?;
self.save()?;
self.size = 0;
self.count = 0;
self.created = SystemTime::now();
@ -200,7 +198,7 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
}
/// writes header and length of header to packfile
pub async fn write_header(&mut self) -> Result<()> {
pub fn write_header(&mut self) -> Result<()> {
// comput the pack header
let data = PackHeaderRef::from_index_pack(&self.index).to_binary()?;
@ -212,21 +210,20 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
.map_err(|_| anyhow!("crypto error"))?;
let headerlen = data.len().try_into()?;
self.write_data(&data).await?;
self.write_data(&data)?;
// finally write length of header unencrypted to pack file
self.write_data(&PackHeaderLength::from_u32(headerlen).to_binary()?)
.await?;
self.write_data(&PackHeaderLength::from_u32(headerlen).to_binary()?)?;
Ok(())
}
pub async fn save(&mut self) -> Result<()> {
pub fn save(&mut self) -> Result<()> {
if self.size == 0 {
return Ok(());
}
self.write_header().await?;
self.write_header()?;
// compute id of packfile
let id = self.hasher.finalize();
@ -235,7 +232,7 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
// write file to backend
let index = std::mem::take(&mut self.index);
let file = std::mem::replace(&mut self.file, BytesMut::new());
self.file_writer.add(index, file.into(), id).await?;
self.file_writer.send((file.into(), id, index))?;
Ok(())
}
@ -245,36 +242,69 @@ impl<BE: DecryptWriteBackend> Packer<BE> {
}
}
struct FileWriter<BE: DecryptWriteBackend> {
future: Option<JoinHandle<Result<()>>>,
#[derive(Clone)]
struct FileWriterHandle<BE: DecryptWriteBackend> {
be: BE,
indexer: SharedIndexer<BE>,
cacheable: bool,
}
impl<BE: DecryptWriteBackend> FileWriter<BE> {
async fn add(&mut self, mut index: IndexPack, file: Bytes, id: Id) -> Result<()> {
let be = self.be.clone();
let indexer = self.indexer.clone();
let cacheable = self.cacheable;
let new_future = spawn(async move {
be.write_bytes(FileType::Pack, &id, cacheable, file).await?;
index.time = Some(Local::now());
indexer.write().await.add(index).await?;
Ok(())
impl<BE: DecryptWriteBackend> ActorHandle<(Bytes, Id, IndexPack)> for FileWriterHandle<BE> {
fn process(&self, load: (Bytes, Id, IndexPack)) -> Result<()> {
let (file, id, mut index) = load;
self.be
.write_bytes(FileType::Pack, &id, self.cacheable, file)?;
index.time = Some(Local::now());
self.indexer.write().unwrap().add(index)?;
Ok(())
}
}
pub trait ActorHandle<T>: Clone + Send + 'static {
fn process(&self, load: T) -> Result<()>;
}
pub struct Actor<T> {
sender: Sender<T>,
finish: Receiver<Result<()>>,
}
impl<T: Send + Sync + 'static> Actor<T> {
pub fn new(fwh: impl ActorHandle<T>, queue_len: usize, par: usize) -> Self {
let (tx, rx) = bounded(queue_len);
let (finish_tx, finish_rx) = bounded::<Result<()>>(0);
(0..par).for_each(|_| {
let rx = rx.clone();
let finish_tx = finish_tx.clone();
let fwh = fwh.clone();
std::thread::spawn(move || {
let mut status = Ok(());
for load in rx {
// only keep processing if there was no error
if status.is_ok() {
status = fwh.process(load);
}
}
let _ = finish_tx.send(status);
});
});
if let Some(fut) = self.future.replace(new_future) {
fut.await??;
Self {
sender: tx,
finish: finish_rx,
}
}
pub fn send(&self, load: T) -> Result<()> {
self.sender.send(load)?;
Ok(())
}
async fn finalize(&mut self) -> Result<()> {
if let Some(fut) = self.future.take() {
return fut.await?;
}
Ok(())
pub fn finalize(self) -> Result<()> {
// cancel channel
drop(self.sender);
// wait for items in channel to be processed
self.finish.recv().unwrap()
}
}
@ -301,42 +331,34 @@ impl<BE: DecryptFullBackend> Repacker<BE> {
})
}
pub async fn add_fast(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> {
let data = self
.be
.read_partial(
FileType::Pack,
pack_id,
blob.tpe.is_cacheable(),
blob.offset,
blob.length,
)
.await?;
pub fn add_fast(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> {
let data = self.be.read_partial(
FileType::Pack,
pack_id,
blob.tpe.is_cacheable(),
blob.offset,
blob.length,
)?;
self.packer
.add_raw(&data, &blob.id, blob.uncompressed_length, self.size_limit)
.await?;
.add_raw(&data, &blob.id, blob.uncompressed_length, self.size_limit)?;
Ok(())
}
pub async fn add(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> {
let data = self
.be
.read_encrypted_partial(
FileType::Pack,
pack_id,
blob.tpe.is_cacheable(),
blob.offset,
blob.length,
blob.uncompressed_length,
)
.await?;
pub fn add(&mut self, pack_id: &Id, blob: &IndexBlob) -> Result<()> {
let data = self.be.read_encrypted_partial(
FileType::Pack,
pack_id,
blob.tpe.is_cacheable(),
blob.offset,
blob.length,
blob.uncompressed_length,
)?;
self.packer
.add_with_sizelimit(&data, &blob.id, self.size_limit)
.await?;
.add_with_sizelimit(&data, &blob.id, self.size_limit)?;
Ok(())
}
pub async fn finalize(&mut self) -> Result<()> {
self.packer.finalize().await
pub fn finalize(self) -> Result<()> {
self.packer.finalize()
}
}

192
src/blob/tree.rs
View File

@ -1,18 +1,12 @@
use std::collections::{HashSet, VecDeque};
use std::collections::HashSet;
use std::mem;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use anyhow::{anyhow, Result};
use crossbeam_channel::{bounded, unbounded, Receiver, Sender};
use derive_getters::Getters;
use futures::{
stream::FuturesUnordered,
task::{Context, Poll},
Future, Stream,
};
use indicatif::ProgressBar;
use serde::{Deserialize, Deserializer, Serialize};
use tokio::{spawn, task::JoinHandle};
use crate::crypto::hash;
use crate::id::Id;
@ -51,24 +45,23 @@ impl Tree {
Ok((chunk, id))
}
pub async fn from_backend(be: &impl IndexedBackend, id: Id) -> Result<Self> {
pub fn from_backend(be: &impl IndexedBackend, id: Id) -> Result<Self> {
let data = be
.get_tree(&id)
.ok_or_else(|| anyhow!("blob {} not found in index", id.to_hex()))?
.read_data(be.be())
.await?;
.read_data(be.be())?;
Ok(serde_json::from_slice(&data)?)
}
pub async fn subtree_id(be: &impl IndexedBackend, mut id: Id, path: &Path) -> Result<Id> {
pub fn subtree_id(be: &impl IndexedBackend, mut id: Id, path: &Path) -> Result<Id> {
for p in path.iter() {
let p = p.to_str().unwrap();
// TODO: check for root instead
if p == "/" {
continue;
}
let tree = Tree::from_backend(be, id).await?;
let tree = Tree::from_backend(be, id)?;
let node = tree
.nodes()
.iter()
@ -92,9 +85,8 @@ impl IntoIterator for Tree {
/// NodeStreamer recursively streams all nodes of a given tree including all subtrees in-order
pub struct NodeStreamer<BE>
where
BE: IndexedBackend + Unpin,
BE: IndexedBackend,
{
future: Option<JoinHandle<Result<Tree>>>,
open_iterators: Vec<std::vec::IntoIter<Node>>,
inner: std::vec::IntoIter<Node>,
path: PathBuf,
@ -103,12 +95,11 @@ where
impl<BE> NodeStreamer<BE>
where
BE: IndexedBackend + Unpin,
BE: IndexedBackend,
{
pub async fn new(be: BE, id: Id) -> Result<Self> {
let inner = Tree::from_backend(&be, id).await?.nodes.into_iter();
pub fn new(be: BE, id: Id) -> Result<Self> {
let inner = Tree::from_backend(&be, id)?.nodes.into_iter();
Ok(Self {
future: None,
inner,
open_iterators: Vec::new(),
path: PathBuf::new(),
@ -119,48 +110,37 @@ where
type NodeStreamItem = Result<(PathBuf, Node)>;
// TODO: This is not really parallel at the moment...
impl<BE> Stream for NodeStreamer<BE>
// TODO: This is not parallel at the moment...
impl<BE> Iterator for NodeStreamer<BE>
where
BE: IndexedBackend + Unpin,
BE: IndexedBackend,
{
type Item = NodeStreamItem;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let slf = self.get_mut();
if let Some(mut future) = slf.future.as_mut() {
match Pin::new(&mut future).poll(cx) {
Poll::Pending => {
return Poll::Pending;
}
Poll::Ready(tree) => {
let old_inner =
mem::replace(&mut slf.inner, tree.unwrap().unwrap().nodes.into_iter());
slf.open_iterators.push(old_inner);
slf.future = None;
}
}
}
fn next(&mut self) -> Option<Self::Item> {
loop {
match slf.inner.next() {
match self.inner.next() {
Some(node) => {
let path = slf.path.join(node.name());
let path = self.path.join(node.name());
if let Some(id) = node.subtree() {
slf.path.push(node.name());
let be = slf.be.clone();
let id = *id;
slf.future = Some(spawn(async move { Tree::from_backend(&be, id).await }));
self.path.push(node.name());
let be = self.be.clone();
let tree = match Tree::from_backend(&be, *id) {
Ok(tree) => tree,
Err(err) => return Some(Err(err)),
};
let old_inner = mem::replace(&mut self.inner, tree.nodes.into_iter());
self.open_iterators.push(old_inner);
}
return Poll::Ready(Some(Ok((path, node))));
return Some(Ok((path, node)));
}
None => match slf.open_iterators.pop() {
None => match self.open_iterators.pop() {
Some(it) => {
slf.inner = it;
slf.path.pop();
self.inner = it;
self.path.pop();
}
None => return Poll::Ready(None),
None => return None,
},
}
}
@ -168,98 +148,100 @@ where
}
/// TreeStreamerOnce recursively visits all trees and subtrees, but each tree ID only once
pub struct TreeStreamerOnce<BE>
where
BE: IndexedBackend + Unpin,
{
futures: FuturesUnordered<JoinHandle<(PathBuf, Result<Tree>, usize)>>,
pub struct TreeStreamerOnce {
visited: HashSet<Id>,
pending: VecDeque<(PathBuf, Id, usize)>,
be: BE,
queue_in: Option<Sender<(PathBuf, Id, usize)>>,
queue_out: Receiver<Result<(PathBuf, Tree, usize)>>,
p: ProgressBar,
counter: Vec<usize>,
finished_ids: usize,
}
impl<BE> TreeStreamerOnce<BE>
where
BE: IndexedBackend + Unpin,
{
pub async fn new(be: BE, ids: Vec<Id>, p: ProgressBar) -> Result<Self> {
const MAX_TREE_LOADER: usize = 4;
impl TreeStreamerOnce {
pub fn new<BE: IndexedBackend>(be: BE, ids: Vec<Id>, p: ProgressBar) -> Result<Self> {
p.set_length(ids.len() as u64);
let (out_tx, out_rx) = bounded(MAX_TREE_LOADER);
let (in_tx, in_rx) = unbounded();
for _ in 0..MAX_TREE_LOADER {
let be = be.clone();
let in_rx = in_rx.clone();
let out_tx = out_tx.clone();
std::thread::spawn(move || {
for (path, id, count) in in_rx {
out_tx
.send(Tree::from_backend(&be, id).map(|tree| (path, tree, count)))
.unwrap();
}
});
}
let counter = vec![0; ids.len()];
let mut streamer = Self {
futures: FuturesUnordered::new(),
visited: HashSet::new(),
pending: VecDeque::new(),
be,
queue_in: Some(in_tx),
queue_out: out_rx,
p,
counter,
finished_ids: 0,
};
for (count, id) in ids.into_iter().enumerate() {
if !streamer.add_pending(PathBuf::new(), id, count) {
if !streamer.add_pending(PathBuf::new(), id, count)? {
streamer.p.inc(1);
streamer.finished_ids += 1;
}
}
Ok(streamer)
}
fn add_pending(&mut self, path: PathBuf, id: Id, count: usize) -> bool {
fn add_pending(&mut self, path: PathBuf, id: Id, count: usize) -> Result<bool> {
if self.visited.insert(id) {
self.pending.push_back((path, id, count));
self.queue_in.as_ref().unwrap().send((path, id, count))?;
self.counter[count] += 1;
true
Ok(true)
} else {
false
Ok(false)
}
}
}
type TreeStreamItem = Result<(PathBuf, Tree)>;
const MAX_TREE_LOADER: usize = 20;
impl<BE> Stream for TreeStreamerOnce<BE>
where
BE: IndexedBackend + Unpin,
{
impl Iterator for TreeStreamerOnce {
type Item = TreeStreamItem;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let slf = self.get_mut();
// fill futures queue if there is space
while slf.futures.len() < MAX_TREE_LOADER && !slf.pending.is_empty() {
let (path, id, count) = slf.pending.pop_front().unwrap();
let be = slf.be.clone();
slf.futures.push(spawn(async move {
(path, Tree::from_backend(&be, id).await, count)
}));
fn next(&mut self) -> Option<Self::Item> {
if self.counter.len() == self.finished_ids {
drop(self.queue_in.take());
self.p.finish();
return None;
}
let (path, tree, count) = match self.queue_out.recv() {
Ok(Ok(res)) => res,
Err(err) => return Some(Err(err.into())),
Ok(Err(err)) => return Some(Err(err)),
};
match Pin::new(&mut slf.futures).poll_next(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Some(Ok((path, tree, count)))) => {
let tree = tree.unwrap();
for node in tree.nodes() {
if let Some(id) = node.subtree() {
let mut path = path.clone();
path.push(node.name());
slf.add_pending(path, *id, count);
}
for node in tree.nodes() {
if let Some(id) = node.subtree() {
let mut path = path.clone();
path.push(node.name());
match self.add_pending(path, *id, count) {
Ok(_) => {}
Err(err) => return Some(Err(err)),
}
slf.counter[count] -= 1;
if slf.counter[count] == 0 {
slf.p.inc(1);
}
Poll::Ready(Some(Ok((path, tree))))
}
Poll::Ready(None) => {
slf.p.finish();
Poll::Ready(None)
}
Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err.into()))),
}
self.counter[count] -= 1;
if self.counter[count] == 0 {
self.p.inc(1);
self.finished_ids += 1;
}
Some(Ok((path, tree)))
}
}

39
src/commands/backup.rs
View File

@ -91,7 +91,7 @@ pub(super) struct Opts {
source: String,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &impl DecryptFullBackend,
opts: Opts,
config: ConfigFile,
@ -116,7 +116,7 @@ pub(super) async fn execute(
}
};
let index = IndexBackend::only_full_trees(&be.clone(), progress_counter("")).await?;
let index = IndexBackend::only_full_trees(&be.clone(), progress_counter(""))?;
for source in sources {
let mut opts = opts.clone();
@ -161,9 +161,8 @@ pub(super) async fn execute(
|snap| snap.hostname == hostname && snap.paths.contains(&backup_path_str),
progress_counter(""),
)
.await
.ok(),
(false, false, Some(parent)) => SnapshotFile::from_id(&be, &parent).await.ok(),
(false, false, Some(parent)) => SnapshotFile::from_id(&be, &parent).ok(),
};
let parent_tree = match &parent {
@ -197,26 +196,24 @@ pub(super) async fn execute(
snap.paths.add(backup_path_str.clone());
snap.set_tags(opts.tag.clone());
let parent = Parent::new(&index, parent_tree, opts.ignore_ctime, opts.ignore_inode).await;
let parent = Parent::new(&index, parent_tree, opts.ignore_ctime, opts.ignore_inode);
let snap = if backup_stdin {
let mut archiver = Archiver::new(be, index, &config, parent, snap)?;
let p = progress_bytes("starting backup from stdin...");
archiver
.backup_reader(
std::io::stdin(),
Node::new(
backup_path_str,
NodeType::File,
Metadata::default(),
None,
None,
),
p.clone(),
)
.await?;
archiver.backup_reader(
std::io::stdin(),
Node::new(
backup_path_str,
NodeType::File,
Metadata::default(),
None,
None,
),
p.clone(),
)?;
let snap = archiver.finalize_snapshot().await?;
let snap = archiver.finalize_snapshot()?;
p.finish_with_message("done");
snap
} else {
@ -235,13 +232,13 @@ pub(super) async fn execute(
warn!("ignoring error {}\n", e)
}
Ok((path, node)) => {
if let Err(e) = archiver.add_entry(&path, node, p.clone()).await {
if let Err(e) = archiver.add_entry(&path, node, p.clone()) {
warn!("ignoring error {} for {:?}\n", e, path);
}
}
}
}
let snap = archiver.finalize_snapshot().await?;
let snap = archiver.finalize_snapshot()?;
p.finish_with_message("done");
snap
};

37
src/commands/cat.rs
View File

@ -46,44 +46,41 @@ struct TreeOpts {
snap: String,
}
pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> {
pub(super) fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> {
match opts.command {
Command::Config => cat_file(be, FileType::Config, IdOpt::default()).await,
Command::Index(opt) => cat_file(be, FileType::Index, opt).await,
Command::Snapshot(opt) => cat_file(be, FileType::Snapshot, opt).await,
Command::Config => cat_file(be, FileType::Config, IdOpt::default()),
Command::Index(opt) => cat_file(be, FileType::Index, opt),
Command::Snapshot(opt) => cat_file(be, FileType::Snapshot, opt),
// special treatment for catingg blobs: read the index and use it to locate the blob
Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt).await,
Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt).await,
Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt),
Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt),
// special treatment for cating a tree within a snapshot
Command::Tree(opts) => cat_tree(be, opts).await,
Command::Tree(opts) => cat_tree(be, opts),
}
}
async fn cat_file(be: &impl DecryptReadBackend, tpe: FileType, opt: IdOpt) -> Result<()> {
let id = be.find_id(tpe, &opt.id).await?;
let data = be.read_encrypted_full(tpe, &id).await?;
fn cat_file(be: &impl DecryptReadBackend, tpe: FileType, opt: IdOpt) -> Result<()> {
let id = be.find_id(tpe, &opt.id)?;
let data = be.read_encrypted_full(tpe, &id)?;
println!("{}", String::from_utf8(data.to_vec())?);
Ok(())
}
async fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Result<()> {
fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Result<()> {
let id = Id::from_hex(&opt.id)?;
let data = IndexBackend::new(be, ProgressBar::hidden())
.await?
.blob_from_backend(&tpe, &id)
.await?;
let data = IndexBackend::new(be, ProgressBar::hidden())?.blob_from_backend(&tpe, &id)?;
print!("{}", String::from_utf8(data.to_vec())?);
Ok(())
}
async fn cat_tree(be: &impl DecryptReadBackend, opts: TreeOpts) -> Result<()> {
fn cat_tree(be: &impl DecryptReadBackend, opts: TreeOpts) -> Result<()> {
let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?;
let index = IndexBackend::new(be, progress_counter("")).await?;
let id = Tree::subtree_id(&index, snap.tree, Path::new(path)).await?;
let data = index.blob_from_backend(&BlobType::Tree, &id).await?;
let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?;
let index = IndexBackend::new(be, progress_counter(""))?;
let id = Tree::subtree_id(&index, snap.tree, Path::new(path))?;
let data = index.blob_from_backend(&BlobType::Tree, &id)?;
println!("{}", String::from_utf8(data.to_vec())?);
Ok(())

102
src/commands/check.rs
View File

@ -3,10 +3,9 @@ use std::collections::HashMap;
use anyhow::Result;
use bytes::Bytes;
use clap::Parser;
use futures::{stream, StreamExt, TryStreamExt};
use indicatif::ProgressBar;
use log::*;
use tokio::task::spawn_blocking;
use rayon::prelude::*;
use zstd::stream::decode_all;
use super::{progress_bytes, progress_counter};
@ -31,7 +30,7 @@ pub(super) struct Opts {
read_data: bool,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &(impl DecryptReadBackend + Unpin),
cache: &Option<Cache>,
hot_be: &Option<impl ReadBackend>,
@ -45,76 +44,69 @@ pub(super) async fn execute(
//
// This lists files here and later when reading index / checking snapshots
// TODO: Only list the files once...
let _ = be.list_with_size(file_type).await?;
let _ = be.list_with_size(file_type)?;
let p = progress_bytes(format!("checking {} in cache...", file_type.name()));
// TODO: Make concurrency (20) customizable
check_cache_files(20, cache, raw_be, file_type, p).await?;
check_cache_files(20, cache, raw_be, file_type, p)?;
}
}
}
if let Some(hot_be) = hot_be {
for file_type in [FileType::Snapshot, FileType::Index] {
check_hot_files(raw_be, hot_be, file_type).await?;
check_hot_files(raw_be, hot_be, file_type)?;
}
}
let index_collector = check_packs(be, hot_be, opts.read_data).await?;
let index_collector = check_packs(be, hot_be, opts.read_data)?;
if !opts.trust_cache {
if let Some(cache) = &cache {
let p = progress_bytes("checking packs in cache...");
// TODO: Make concurrency (5) customizable
check_cache_files(5, cache, raw_be, FileType::Pack, p).await?;
check_cache_files(5, cache, raw_be, FileType::Pack, p)?;
}
}
let index_be = IndexBackend::new_from_index(be, index_collector.into_index());
check_snapshots(&index_be).await?;
check_snapshots(&index_be)?;
if opts.read_data {
let p = progress_counter("reading pack data...");
stream::iter(index_be.into_index().into_iter().map(|pack| {
let be = be.clone();
let p = p.clone();
(pack, be, p)
}))
// TODO: Make concurrency (4) customizable
.for_each_concurrent(4, |(pack, be, p)| async move {
let id = pack.id;
let data = be.read_full(FileType::Pack, &id).await.unwrap();
spawn_blocking(move || {
match check_pack(&be, pack, data) {
index_be
.into_index()
.into_iter()
.par_bridge()
.for_each_with((be.clone(), p.clone()), |(be, p), pack| {
let id = pack.id;
let data = be.read_full(FileType::Pack, &id).unwrap();
match check_pack(be, pack, data) {
Ok(()) => {}
Err(err) => error!("Error reading pack {id} : {err}",),
}
p.inc(1);
})
.await
.unwrap()
})
.await;
});
p.finish();
}
Ok(())
}
async fn check_hot_files(
fn check_hot_files(
be: &impl ReadBackend,
be_hot: &impl ReadBackend,
file_type: FileType,
) -> Result<()> {
let p = progress_spinner(format!("checking {} in hot repo...", file_type.name()));
let mut files = be
.list_with_size(file_type)
.await?
.list_with_size(file_type)?
.into_iter()
.collect::<HashMap<_, _>>();
let files_hot = be_hot.list_with_size(file_type).await?;
let files_hot = be_hot.list_with_size(file_type)?;
for (id, size_hot) in files_hot {
match files.remove(&id) {
@ -134,14 +126,14 @@ async fn check_hot_files(
Ok(())
}
async fn check_cache_files(
concurrency: usize,
fn check_cache_files(
_concurrency: usize,
cache: &Cache,
be: &impl ReadBackend,
file_type: FileType,
p: ProgressBar,
) -> Result<()> {
let files = cache.list_with_size(file_type).await?;
let files = cache.list_with_size(file_type)?;
if files.is_empty() {
return Ok(());
@ -150,17 +142,12 @@ async fn check_cache_files(
let total_size = files.iter().map(|(_, size)| *size as u64).sum();
p.set_length(total_size);
stream::iter(files.into_iter().map(|file| {
let cache = cache.clone();
let be = be.clone();
let p = p.clone();
(file, cache, be, p)
}))
.for_each_concurrent(concurrency, |((id, size), cache, be, p)| async move {
files.into_par_iter()
.for_each_with((cache,be,p.clone()), |(cache, be, p),(id, size)| {
// Read file from cache and from backend and compare
match (
cache.read_full(file_type, &id).await,
be.read_full(file_type, &id).await,
cache.read_full(file_type, &id),
be.read_full(file_type, &id),
) {
(Err(err), _) => {
error!("Error reading cached file Type: {file_type:?}, Id: {id} : {err}",)
@ -173,15 +160,14 @@ async fn check_cache_files(
}
p.inc(size as u64);
})
.await;
});
p.finish();
Ok(())
}
// check if packs correspond to index
async fn check_packs(
fn check_packs(
be: &impl DecryptReadBackend,
hot_be: &Option<impl ReadBackend>,
read_data: bool,
@ -225,9 +211,7 @@ async fn check_packs(
};
let p = progress_counter("reading index...");
let mut stream = be.stream_all::<IndexFile>(p.clone()).await?;
while let Some(index) = stream.try_next().await? {
let index = index.1;
for (_, index) in be.stream_all::<IndexFile>(p.clone())? {
index_collector.extend(index.packs.clone());
for p in index.packs {
process_pack(p);
@ -236,23 +220,24 @@ async fn check_packs(
process_pack(p);
}
}
p.finish();
if let Some(hot_be) = hot_be {
let p = progress_spinner("listing packs in hot repo...");
check_packs_list(hot_be, tree_packs).await?;
check_packs_list(hot_be, tree_packs)?;
p.finish();
}
let p = progress_spinner("listing packs...");
check_packs_list(be, packs).await?;
check_packs_list(be, packs)?;
p.finish();
Ok(index_collector)
}
async fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap<Id, u32>) -> Result<()> {
for (id, size) in be.list_with_size(FileType::Pack).await? {
fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap<Id, u32>) -> Result<()> {
for (id, size) in be.list_with_size(FileType::Pack)? {
match packs.remove(&id) {
None => warn!("pack {id} not referenced in index. Can be a parallel backup job. To repair: 'rustic repair index'."),
Some(index_size) if index_size != size => {
@ -269,20 +254,19 @@ async fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap<Id, u32>) ->
}
// check if all snapshots and contained trees can be loaded and contents exist in the index
async fn check_snapshots(index: &(impl IndexedBackend + Unpin)) -> Result<()> {
fn check_snapshots(index: &(impl IndexedBackend + Unpin)) -> Result<()> {
let p = progress_counter("reading snapshots...");
let snap_trees: Vec<_> = index
.be()
.stream_all::<SnapshotFile>(p.clone())
.await?
.map_ok(|(_, snap)| snap.tree)
.try_collect()
.await?;
.stream_all::<SnapshotFile>(p.clone())?
.iter()
.map(|(_, snap)| snap.tree)
.collect();
p.finish();
let p = progress_counter("checking trees...");
let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p).await?;
while let Some(item) = tree_streamer.try_next().await? {
let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?;
while let Some(item) = tree_streamer.next().transpose()? {
let (path, tree) = item;
for node in tree.nodes() {
match node.node_type() {

6
src/commands/config.rs
View File

@ -11,7 +11,7 @@ pub(super) struct Opts {
config_opts: ConfigOpts,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &impl DecryptFullBackend,
hot_be: &Option<impl WriteBackend>,
opts: Opts,
@ -22,13 +22,13 @@ pub(super) async fn execute(
if new_config != config {
new_config.is_hot = None;
// for hot/cold backend, this only saves the config to the cold repo.
be.save_file(&new_config).await?;
be.save_file(&new_config)?;
if let Some(hot_be) = hot_be {
// save config to hot repo
let dbe = DecryptBackend::new(hot_be, be.key().clone());
new_config.is_hot = Some(true);
dbe.save_file(&new_config).await?;
dbe.save_file(&new_config)?;
}
println!("saved new config");

31
src/commands/diff.rs
View File

@ -2,7 +2,6 @@ use std::path::Path;
use anyhow::Result;
use clap::Parser;
use futures::StreamExt;
use super::progress_counter;
use crate::backend::DecryptReadBackend;
@ -22,45 +21,45 @@ pub(super) struct Opts {
snap2: String,
}
pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> {
pub(super) fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> {
let (id1, path1) = opts.snap1.split_once(':').unwrap_or((&opts.snap1, ""));
let (id2, path2) = opts.snap2.split_once(':').unwrap_or((&opts.snap2, path1));
let p = progress_spinner("getting snapshots...");
p.finish();
let snaps = SnapshotFile::from_ids(be, &[id1.to_string(), id2.to_string()]).await?;
let snaps = SnapshotFile::from_ids(be, &[id1.to_string(), id2.to_string()])?;
let snap1 = &snaps[0];
let snap2 = &snaps[1];
let index = IndexBackend::new(be, progress_counter("")).await?;
let id1 = Tree::subtree_id(&index, snap1.tree, Path::new(path1)).await?;
let id2 = Tree::subtree_id(&index, snap2.tree, Path::new(path2)).await?;
let index = IndexBackend::new(be, progress_counter(""))?;
let id1 = Tree::subtree_id(&index, snap1.tree, Path::new(path1))?;
let id2 = Tree::subtree_id(&index, snap2.tree, Path::new(path2))?;
let mut tree_streamer1 = NodeStreamer::new(index.clone(), id1).await?;
let mut tree_streamer2 = NodeStreamer::new(index, id2).await?;
let mut tree_streamer1 = NodeStreamer::new(index.clone(), id1)?;
let mut tree_streamer2 = NodeStreamer::new(index, id2)?;
let mut item1 = tree_streamer1.next().await.transpose()?;
let mut item2 = tree_streamer2.next().await.transpose()?;
let mut item1 = tree_streamer1.next().transpose()?;
let mut item2 = tree_streamer2.next().transpose()?;
loop {
match (&item1, &item2) {
(None, None) => break,
(Some(i1), None) => {
println!("- {:?}", i1.0);
item1 = tree_streamer1.next().await.transpose()?;
item1 = tree_streamer1.next().transpose()?;
}
(None, Some(i2)) => {
println!("+ {:?}", i2.0);
item2 = tree_streamer2.next().await.transpose()?;
item2 = tree_streamer2.next().transpose()?;
}
(Some(i1), Some(i2)) if i1.0 < i2.0 => {
println!("- {:?}", i1.0);
item1 = tree_streamer1.next().await.transpose()?;
item1 = tree_streamer1.next().transpose()?;
}
(Some(i1), Some(i2)) if i1.0 > i2.0 => {
println!("+ {:?}", i2.0);
item2 = tree_streamer2.next().await.transpose()?;
item2 = tree_streamer2.next().transpose()?;
}
(Some(i1), Some(i2)) => {
let path = &i1.0;
@ -83,8 +82,8 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts)
}
_ => {} // no difference to show
}
item1 = tree_streamer1.next().await.transpose()?;
item2 = tree_streamer2.next().await.transpose()?;
item1 = tree_streamer1.next().transpose()?;
item2 = tree_streamer2.next().transpose()?;
}
}
}

11
src/commands/forget.rs
View File

@ -55,7 +55,7 @@ struct ConfigOpts {
keep: KeepOptions,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &(impl DecryptFullBackend + Unpin),
cache: Option<Cache>,
mut opts: Opts,
@ -74,10 +74,10 @@ pub(super) async fn execute(
.unwrap_or_else(|| SnapshotGroupCriterion::from_str("host,paths").unwrap());
let groups = match opts.ids.is_empty() {
true => SnapshotFile::group_from_backend(be, &opts.config.filter, &group_by).await?,
true => SnapshotFile::group_from_backend(be, &opts.config.filter, &group_by)?,
false => vec![(
SnapshotGroup::default(),
SnapshotFile::from_ids(be, &opts.ids).await?,
SnapshotFile::from_ids(be, &opts.ids)?,
)],
};
let mut forget_snaps = Vec::new();
@ -146,13 +146,12 @@ pub(super) async fn execute(
),
(false, false) => {
let p = progress_counter("removing snapshots...");
be.delete_list(FileType::Snapshot, true, forget_snaps.clone(), p)
.await?;
be.delete_list(FileType::Snapshot, true, forget_snaps.clone(), p)?;
}
}
if opts.prune {
prune::execute(be, cache, opts.prune_opts, config, forget_snaps).await?;
prune::execute(be, cache, opts.prune_opts, config, forget_snaps)?;
}
Ok(())

40
src/commands/helpers.rs
View File

@ -5,13 +5,11 @@ use std::time::Duration;
use anyhow::{bail, Result};
use bytesize::ByteSize;
use futures::{stream::FuturesUnordered, TryStreamExt};
use indicatif::HumanDuration;
use indicatif::{ProgressBar, ProgressState, ProgressStyle};
use log::*;
use rayon::ThreadPoolBuilder;
use rpassword::prompt_password;
use tokio::spawn;
use tokio::time::sleep;
use crate::backend::{DecryptReadBackend, FileType, ReadBackend};
use crate::crypto::Key;
@ -23,16 +21,15 @@ pub fn bytes(b: u64) -> String {
ByteSize(b).to_string_as(true)
}
pub async fn get_key(be: &impl ReadBackend, password: Option<String>) -> Result<Key> {
pub fn get_key(be: &impl ReadBackend, password: Option<String>) -> Result<Key> {
for _ in 0..MAX_PASSWORD_RETRIES {
match &password {
// if password is given, directly return the result of find_key_in_backend and don't retry
Some(pass) => return find_key_in_backend(be, pass, None).await,
Some(pass) => return find_key_in_backend(be, pass, None),
None => {
// TODO: Differentiate between wrong password and other error!
if let Ok(key) =
find_key_in_backend(be, &prompt_password("enter repository password: ")?, None)
.await
{
return Ok(key);
}
@ -104,7 +101,7 @@ pub fn warm_up_command(packs: impl ExactSizeIterator<Item = Id>, command: &str)
Ok(())
}
pub async fn warm_up(
pub fn warm_up(
be: &impl DecryptReadBackend,
packs: impl ExactSizeIterator<Item = Id>,
) -> Result<()> {
@ -113,33 +110,30 @@ pub async fn warm_up(
let p = progress_counter("warming up packs...");
p.set_length(packs.len() as u64);
let mut stream = FuturesUnordered::new();
const MAX_READER: usize = 20;
for pack in packs {
while stream.len() > MAX_READER {
stream.try_next().await?;
let pool = ThreadPoolBuilder::new().num_threads(MAX_READER).build()?;
let p = &p;
let be = &be;
pool.in_place_scope(|s| {
for pack in packs {
s.spawn(move |_| {
// ignore errors as they are expected from the warm-up
_ = be.read_partial(FileType::Pack, &pack, false, 0, 1);
p.inc(1);
});
}
});
let p = p.clone();
let be = be.clone();
stream.push(spawn(async move {
// ignore errors as they are expected from the warm-up
_ = be.read_partial(FileType::Pack, &pack, false, 0, 1).await;
p.inc(1);
}))
}
stream.try_collect().await?;
p.finish();
Ok(())
}
pub async fn wait(d: Option<humantime::Duration>) {
pub fn wait(d: Option<humantime::Duration>) {
if let Some(wait) = d {
let p = progress_spinner(format!("waiting {}...", wait));
sleep(*wait).await;
std::thread::sleep(*wait);
p.finish();
}
}

15
src/commands/init.rs
View File

@ -20,7 +20,7 @@ pub(super) struct Opts {
config_opts: ConfigOpts,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &impl WriteBackend,
hot_be: &Option<impl WriteBackend>,
opts: Opts,
@ -59,24 +59,23 @@ pub(super) async fn execute(
)?;
let data: Bytes = serde_json::to_vec(&keyfile)?.into();
let id = hash(&data);
be.create().await?;
be.write_bytes(FileType::Key, &id, false, data.clone())
.await?;
be.create()?;
be.write_bytes(FileType::Key, &id, false, data.clone())?;
if let Some(hot_be) = hot_be {
hot_be.create().await?;
hot_be.write_bytes(FileType::Key, &id, false, data).await?;
hot_be.create()?;
hot_be.write_bytes(FileType::Key, &id, false, data)?;
}
println!("key {} successfully added.", id);
// save config
let dbe = DecryptBackend::new(be, key.clone());
dbe.save_file(&config).await?;
dbe.save_file(&config)?;
if let Some(hot_be) = hot_be {
let dbe = DecryptBackend::new(hot_be, key);
config.is_hot = Some(true);
dbe.save_file(&config).await?;
dbe.save_file(&config)?;
}
println!("repository {} successfully created.", repo_id);

9
src/commands/key.rs
View File

@ -47,13 +47,13 @@ pub(crate) struct KeyOpts {
pub(crate) with_created: bool,
}
pub(super) async fn execute(be: &impl WriteBackend, key: Key, opts: Opts) -> Result<()> {
pub(super) fn execute(be: &impl WriteBackend, key: Key, opts: Opts) -> Result<()> {
match opts.command {
Command::Add(opt) => add_key(be, key, opt).await,
Command::Add(opt) => add_key(be, key, opt),
}
}
async fn add_key(be: &impl WriteBackend, key: Key, opts: AddOpts) -> Result<()> {
fn add_key(be: &impl WriteBackend, key: Key, opts: AddOpts) -> Result<()> {
let pass = match opts.new_password_file {
Some(file) => {
let mut file = BufReader::new(File::open(file)?);
@ -65,8 +65,7 @@ async fn add_key(be: &impl WriteBackend, key: Key, opts: AddOpts) -> Result<()>
let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?;
let data = serde_json::to_vec(&keyfile)?;
let id = hash(&data);
be.write_bytes(FileType::Key, &id, false, data.into())
.await?;
be.write_bytes(FileType::Key, &id, false, data.into())?;
println!("key {} successfully added.", id);
Ok(())

10
src/commands/list.rs
View File

@ -1,6 +1,5 @@
use anyhow::{bail, Result};
use clap::Parser;
use futures::StreamExt;
use indicatif::ProgressBar;
use crate::backend::{DecryptReadBackend, FileType};
@ -13,13 +12,12 @@ pub(super) struct Opts {
tpe: String,
}
pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> {
pub(super) fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<()> {
let tpe = match opts.tpe.as_str() {
// special treatment for listing blobs: read the index and display it
"blobs" => {
let mut stream = be.stream_all::<IndexFile>(ProgressBar::hidden()).await?;
while let Some(index) = stream.next().await {
for pack in index?.1.packs {
for (_, index) in be.stream_all::<IndexFile>(ProgressBar::hidden())? {
for pack in index.packs {
for blob in pack.blobs {
println!("{:?} {}", blob.tpe, blob.id.to_hex());
}
@ -34,7 +32,7 @@ pub(super) async fn execute(be: &impl DecryptReadBackend, opts: Opts) -> Result<
t => bail!("invalid type: {}", t),
};
for id in be.list(tpe).await? {
for id in be.list(tpe)? {
println!("{}", id.to_hex());
}

12
src/commands/ls.rs
View File

@ -1,6 +1,5 @@
use anyhow::Result;
use clap::Parser;
use futures::StreamExt;
use std::path::Path;
use super::progress_counter;
@ -16,14 +15,13 @@ pub(super) struct Opts {
snap: String,
}
pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> {
pub(super) fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> {
let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?;
let index = IndexBackend::new(be, progress_counter("")).await?;
let tree = Tree::subtree_id(&index, snap.tree, Path::new(path)).await?;
let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?;
let index = IndexBackend::new(be, progress_counter(""))?;
let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?;
let mut tree_streamer = NodeStreamer::new(index, tree).await?;
while let Some(item) = tree_streamer.next().await {
for item in NodeStreamer::new(index, tree)? {
let (path, _) = item?;
println!("{:?} ", path);
}

48
src/commands/mod.rs
View File

@ -181,7 +181,7 @@ enum Command {
Tag(tag::Opts),
}
pub async fn execute() -> Result<()> {
pub fn execute() -> Result<()> {
let command: Vec<_> = std::env::args_os().into_iter().collect();
let args = Opts::parse_from(&command);
@ -220,7 +220,7 @@ pub async fn execute() -> Result<()> {
}
if let Command::SelfUpdate(opts) = args.command {
self_update::execute(opts).await?;
self_update::execute(opts)?;
return Ok(());
}
@ -263,29 +263,27 @@ pub async fn execute() -> Result<()> {
(None, None, None) => None,
};
let config_ids = be.list(FileType::Config).await?;
let config_ids = be.list(FileType::Config)?;
let (cmd, key, dbe, cache, be, be_hot, config) = match (args.command, config_ids.len()) {
(Command::Init(opts), _) => {
return init::execute(&be, &be_hot, opts, password, config_ids).await
}
(Command::Init(opts), _) => return init::execute(&be, &be_hot, opts, password, config_ids),
(cmd, 1) => {
let be = HotColdBackend::new(be, be_hot.clone());
if let Some(be_hot) = &be_hot {
let mut keys = be.list_with_size(FileType::Key).await?;
let mut keys = be.list_with_size(FileType::Key)?;
keys.sort_unstable_by_key(|key| key.0);
let mut hot_keys = be_hot.list_with_size(FileType::Key).await?;
let mut hot_keys = be_hot.list_with_size(FileType::Key)?;
hot_keys.sort_unstable_by_key(|key| key.0);
if keys != hot_keys {
bail!("keys from repo and repo-hot do not match. Aborting.");
}
}
let key = get_key(&be, password).await?;
let key = get_key(&be, password)?;
info!("password is correct.");
let dbe = DecryptBackend::new(&be, key.clone());
let config: ConfigFile = dbe.get_file(&config_ids[0]).await?;
let config: ConfigFile = dbe.get_file(&config_ids[0])?;
match (config.is_hot == Some(true), be_hot.is_some()) {
(true, false) => bail!("repository is a hot repository!\nPlease use as --repo-hot in combination with the normal repo. Aborting."),
(false, true) => bail!("repo-hot is not a hot repository! Aborting."),
@ -307,24 +305,24 @@ pub async fn execute() -> Result<()> {
};
match cmd {
Command::Backup(opts) => backup::execute(&dbe, opts, config, config_file, command).await?,
Command::Config(opts) => config::execute(&dbe, &be_hot, opts, config).await?,
Command::Cat(opts) => cat::execute(&dbe, opts).await?,
Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts).await?,
Command::Backup(opts) => backup::execute(&dbe, opts, config, config_file, command)?,
Command::Config(opts) => config::execute(&dbe, &be_hot, opts, config)?,
Command::Cat(opts) => cat::execute(&dbe, opts)?,
Command::Check(opts) => check::execute(&dbe, &cache, &be_hot, &be, opts)?,
Command::Completions(_) => {} // already handled above
Command::Diff(opts) => diff::execute(&dbe, opts).await?,
Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file).await?,
Command::Diff(opts) => diff::execute(&dbe, opts)?,
Command::Forget(opts) => forget::execute(&dbe, cache, opts, config, config_file)?,
Command::Init(_) => {} // already handled above
Command::Key(opts) => key::execute(&dbe, key, opts).await?,
Command::List(opts) => list::execute(&dbe, opts).await?,
Command::Ls(opts) => ls::execute(&dbe, opts).await?,
Command::Key(opts) => key::execute(&dbe, key, opts)?,
Command::List(opts) => list::execute(&dbe, opts)?,
Command::Ls(opts) => ls::execute(&dbe, opts)?,
Command::SelfUpdate(_) => {} // already handled above
Command::Snapshots(opts) => snapshots::execute(&dbe, opts, config_file).await?,
Command::Prune(opts) => prune::execute(&dbe, cache, opts, config, vec![]).await?,
Command::Restore(opts) => restore::execute(&dbe, opts).await?,
Command::Repair(opts) => repair::execute(&dbe, opts, config_file, &config).await?,
Command::Repoinfo(opts) => repoinfo::execute(&dbe, &be_hot, opts).await?,
Command::Tag(opts) => tag::execute(&dbe, opts, config_file).await?,
Command::Snapshots(opts) => snapshots::execute(&dbe, opts, config_file)?,
Command::Prune(opts) => prune::execute(&dbe, cache, opts, config, vec![])?,
Command::Restore(opts) => restore::execute(&dbe, opts)?,
Command::Repair(opts) => repair::execute(&dbe, opts, config_file, &config)?,
Command::Repoinfo(opts) => repoinfo::execute(&dbe, &be_hot, opts)?,
Command::Tag(opts) => tag::execute(&dbe, opts, config_file)?,
};
Ok(())

86
src/commands/prune.rs
View File

@ -7,11 +7,10 @@ use bytesize::ByteSize;
use chrono::{DateTime, Duration, Local};
use clap::{AppSettings, Parser};
use derive_more::Add;
use futures::{future, TryStreamExt};
use log::*;
use super::{bytes, no_progress, progress_bytes, progress_counter, wait, warm_up, warm_up_command};
use crate::backend::{Cache, DecryptFullBackend, DecryptReadBackend, FileType};
use crate::backend::{Cache, DecryptFullBackend, DecryptReadBackend, FileType, ReadBackend};
use crate::blob::{
BlobType, BlobTypeMap, Initialize, NodeType, PackSizer, Repacker, Sum, TreeStreamerOnce,
};
@ -84,7 +83,7 @@ pub(super) struct Opts {
warm_up_wait: Option<humantime::Duration>,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &(impl DecryptFullBackend + Unpin),
cache: Option<Cache>,
opts: Opts,
@ -98,10 +97,9 @@ pub(super) async fn execute(
let mut index_files = Vec::new();
let p = progress_counter("reading index...");
let mut stream = be.stream_all::<IndexFile>(p.clone()).await?;
let mut index_collector = IndexCollector::new(IndexType::OnlyTrees);
while let Some((id, index)) = stream.try_next().await? {
for (id, index) in be.stream_all::<IndexFile>(p.clone())? {
index_collector.extend(index.packs.clone());
// we add the trees from packs_to_delete to the index such that searching for
// used blobs doesn't abort if they are already marked for deletion
@ -113,9 +111,7 @@ pub(super) async fn execute(
if let Some(cache) = &cache {
let p = progress_spinner("cleaning up packs from cache...");
cache
.remove_not_in_list(FileType::Pack, index_collector.tree_packs())
.await?;
cache.remove_not_in_list(FileType::Pack, index_collector.tree_packs())?;
p.finish();
}
match (cache.is_some(), opts.cache_only) {
@ -131,17 +127,13 @@ pub(super) async fn execute(
let index = index_collector.into_index();
let total_size = BlobTypeMap::init(|blob_type| index.total_size(&blob_type));
let indexed_be = IndexBackend::new_from_index(&be.clone(), index);
let used_ids = find_used_blobs(&indexed_be, ignore_snaps).await?;
let used_ids = find_used_blobs(&indexed_be, ignore_snaps)?;
(used_ids, total_size)
};
// list existing pack files
let p = progress_spinner("geting packs from repository...");
let existing_packs: HashMap<_, _> = be
.list_with_size(FileType::Pack)
.await?
.into_iter()
.collect();
let existing_packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect();
p.finish();
let mut pruner = Pruner::new(used_ids, existing_packs, index_files);
@ -170,17 +162,17 @@ pub(super) async fn execute(
pruner.print_stats();
if opts.warm_up {
warm_up(be, pruner.repack_packs().into_iter()).await?;
warm_up(be, pruner.repack_packs().into_iter())?;
} else if opts.warm_up_command.is_some() {
warm_up_command(
pruner.repack_packs().into_iter(),
opts.warm_up_command.as_ref().unwrap(),
)?;
}
wait(opts.warm_up_wait).await;
wait(opts.warm_up_wait);
if !opts.dry_run {
pruner.do_prune(be, opts, config).await?;
pruner.do_prune(be, opts, config)?;
}
Ok(())
}
@ -835,7 +827,7 @@ impl Pruner {
.collect()
}
async fn do_prune(
fn do_prune(
mut self,
be: &impl DecryptFullBackend,
opts: Opts,
@ -885,8 +877,7 @@ impl Pruner {
let p = progress_counter("removing unindexed packs...");
let existing_packs: Vec<_> =
self.existing_packs.into_iter().map(|(id, _)| id).collect();
be.delete_list(FileType::Pack, true, existing_packs, p)
.await?;
be.delete_list(FileType::Pack, true, existing_packs, p)?;
} else {
info!("marking not needed unindexed pack files for deletion...");
for (id, size) in self.existing_packs {
@ -896,7 +887,7 @@ impl Pruner {
time: Some(Local::now()),
blobs: Vec::new(),
};
indexer.write().await.add_remove(pack).await?;
indexer.write().unwrap().add_remove(pack)?;
}
}
}
@ -933,7 +924,7 @@ impl Pruner {
PackToDo::Keep => {
// keep pack: add to new index
let pack = pack.into_index_pack();
indexer.write().await.add(pack).await?;
indexer.write().unwrap().add(pack)?;
}
PackToDo::Repack => {
// TODO: repack in parallel
@ -948,9 +939,9 @@ impl Pruner {
BlobType::Tree => &mut tree_repacker,
};
if opts.fast_repack {
repacker.add_fast(&pack.id, blob).await?;
repacker.add_fast(&pack.id, blob)?;
} else {
repacker.add(&pack.id, blob).await?;
repacker.add(&pack.id, blob)?;
}
p.inc(blob.length as u64);
}
@ -959,7 +950,7 @@ impl Pruner {
} else {
// mark pack for removal
let pack = pack.into_index_pack_with_time(self.time);
indexer.write().await.add_remove(pack).await?;
indexer.write().unwrap().add_remove(pack)?;
}
}
PackToDo::MarkDelete => {
@ -968,7 +959,7 @@ impl Pruner {
} else {
// mark pack for removal
let pack = pack.into_index_pack_with_time(self.time);
indexer.write().await.add_remove(pack).await?;
indexer.write().unwrap().add_remove(pack)?;
}
}
PackToDo::KeepMarked => {
@ -977,40 +968,37 @@ impl Pruner {
} else {
// keep pack: add to new index
let pack = pack.into_index_pack();
indexer.write().await.add_remove(pack).await?;
indexer.write().unwrap().add_remove(pack)?;
}
}
PackToDo::Recover => {
// recover pack: add to new index in section packs
let pack = pack.into_index_pack_with_time(self.time);
indexer.write().await.add(pack).await?;
indexer.write().unwrap().add(pack)?;
}
PackToDo::Delete => delete_pack(pack),
}
}
indexes_remove.push(index.id);
}
tree_repacker.finalize().await?;
data_repacker.finalize().await?;
indexer.write().await.finalize().await?;
tree_repacker.finalize()?;
data_repacker.finalize()?;
indexer.write().unwrap().finalize()?;
p.finish();
if !data_packs_remove.is_empty() {
let p = progress_counter("removing old data packs...");
be.delete_list(FileType::Pack, false, data_packs_remove, p)
.await?;
be.delete_list(FileType::Pack, false, data_packs_remove, p)?;
}
if !tree_packs_remove.is_empty() {
let p = progress_counter("removing old tree packs...");
be.delete_list(FileType::Pack, true, tree_packs_remove, p)
.await?;
be.delete_list(FileType::Pack, true, tree_packs_remove, p)?;
}
if !indexes_remove.is_empty() {
let p = progress_counter("removing old index files...");
be.delete_list(FileType::Index, true, indexes_remove, p)
.await?;
be.delete_list(FileType::Index, true, indexes_remove, p)?;
}
Ok(())
@ -1087,30 +1075,32 @@ impl PackInfo {
}
// find used blobs in repo
async fn find_used_blobs(
fn find_used_blobs(
index: &(impl IndexedBackend + Unpin),
ignore_snaps: Vec<Id>,
) -> Result<HashMap<Id, u8>> {
let ignore_snaps: HashSet<_> = ignore_snaps.into_iter().collect();
let p = progress_counter("reading snapshots...");
let list = index
.be()
.list(FileType::Snapshot)?
.into_iter()
.filter(|id| !ignore_snaps.contains(id))
.collect();
let snap_trees: Vec<_> = index
.be()
.stream_all::<SnapshotFile>(p.clone())
.await?
// TODO: it would even better to give ignore_snaps to the streaming function instead
// if reading and then filtering the snapshot
.try_filter(|(id, _)| future::ready(!ignore_snaps.contains(id)))
.map_ok(|(_, snap)| snap.tree)
.try_collect()
.await?;
.stream_list::<SnapshotFile>(list, p.clone())?
.into_iter()
.map(|(_, snap)| snap.tree)
.collect();
p.finish();
let mut ids: HashMap<_, _> = snap_trees.iter().map(|id| (*id, 0)).collect();
let p = progress_counter("finding used blobs...");
let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p).await?;
while let Some(item) = tree_streamer.try_next().await? {
let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?;
while let Some(item) = tree_streamer.next().transpose()? {
let (_, tree) = item;
for node in tree.nodes() {
match node.node_type() {

67
src/commands/repair.rs
View File

@ -1,9 +1,7 @@
use std::collections::{HashMap, HashSet};
use anyhow::Result;
use async_recursion::async_recursion;
use clap::{AppSettings, Parser, Subcommand};
use futures::TryStreamExt;
use log::*;
use crate::backend::{DecryptFullBackend, DecryptWriteBackend, FileType};
@ -82,25 +80,21 @@ struct SnapOpts {
ids: Vec<String>,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &impl DecryptFullBackend,
opts: Opts,
config_file: RusticConfig,
config: &ConfigFile,
) -> Result<()> {
match opts.command {
Command::Index(opt) => repair_index(be, opt).await,
Command::Snapshots(opt) => repair_snaps(be, opt, config_file, config).await,
Command::Index(opt) => repair_index(be, opt),
Command::Snapshots(opt) => repair_snaps(be, opt, config_file, config),
}
}
async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> {
fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<()> {
let p = progress_spinner("listing packs...");
let mut packs: HashMap<_, _> = be
.list_with_size(FileType::Pack)
.await?
.into_iter()
.collect();
let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect();
p.finish();
let mut pack_read_header = Vec::new();
@ -146,12 +140,9 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<(
};
let p = progress_counter("reading index...");
let mut stream = be.stream_all::<IndexFile>(p.clone()).await?;
while let Some(index) = stream.try_next().await? {
for (index_id, index) in be.stream_all::<IndexFile>(p.clone())? {
let mut new_index = IndexFile::default();
let mut changed = false;
let index_id = index.0;
let index = index.1;
for p in index.packs {
process_pack(p, false, &mut new_index, &mut changed);
}
@ -162,9 +153,9 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<(
(true, true) => info!("would have modified index file {index_id}"),
(true, false) => {
if !new_index.packs.is_empty() || !new_index.packs_to_delete.is_empty() {
be.save_file(&new_index).await?;
be.save_file(&new_index)?;
}
be.remove(FileType::Index, &index_id, true).await?;
be.remove(FileType::Index, &index_id, true)?;
}
(false, _) => {} // nothing to do
}
@ -175,7 +166,7 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<(
pack_read_header.extend(packs.into_iter().map(|(id, size)| (id, false, None, size)));
if opts.warm_up {
warm_up(be, pack_read_header.iter().map(|(id, _, _, _)| *id)).await?;
warm_up(be, pack_read_header.iter().map(|(id, _, _, _)| *id))?;
if opts.dry_run {
return Ok(());
}
@ -188,7 +179,7 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<(
return Ok(());
}
}
wait(opts.warm_up_wait).await;
wait(opts.warm_up_wait);
let indexer = Indexer::new(be.clone()).into_shared();
let p = progress_counter("reading pack headers");
@ -197,21 +188,19 @@ async fn repair_index(be: &impl DecryptFullBackend, opts: IndexOpts) -> Result<(
debug!("reading pack {id}...");
let mut pack = IndexPack::default();
pack.set_id(id);
pack.blobs = PackHeader::from_file(be, id, size_hint, packsize)
.await?
.into_blobs();
pack.blobs = PackHeader::from_file(be, id, size_hint, packsize)?.into_blobs();
if !opts.dry_run {
indexer.write().await.add_with(pack, to_delete).await?;
indexer.write().unwrap().add_with(pack, to_delete)?;
}
p.inc(1);
}
indexer.write().await.finalize().await?;
indexer.write().unwrap().finalize()?;
p.finish();
Ok(())
}
async fn repair_snaps(
fn repair_snaps(
be: &impl DecryptFullBackend,
mut opts: SnapOpts,
config_file: RusticConfig,
@ -220,15 +209,15 @@ async fn repair_snaps(
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let snapshots = match opts.ids.is_empty() {
true => SnapshotFile::all_from_backend(be, &opts.filter).await?,
false => SnapshotFile::from_ids(be, &opts.ids).await?,
true => SnapshotFile::all_from_backend(be, &opts.filter)?,
false => SnapshotFile::from_ids(be, &opts.ids)?,
};
let mut replaced = HashMap::new();
let mut seen = HashSet::new();
let mut delete = Vec::new();
let index = IndexBackend::new(&be.clone(), progress_counter("")).await?;
let index = IndexBackend::new(&be.clone(), progress_counter(""))?;
let indexer = Indexer::new(be.clone()).into_shared();
let mut packer = Packer::new(
be.clone(),
@ -248,9 +237,7 @@ async fn repair_snaps(
&mut replaced,
&mut seen,
&opts,
)
.await?
{
)? {
(Changed::None, _) => {
info!("snapshot {snap_id} is ok.");
}
@ -268,7 +255,7 @@ async fn repair_snaps(
if opts.dry_run {
info!("would have modified snapshot {snap_id}.");
} else {
let new_id = be.save_file(&snap).await?;
let new_id = be.save_file(&snap)?;
info!("saved modified snapshot as {new_id}.");
}
delete.push(snap_id);
@ -277,8 +264,8 @@ async fn repair_snaps(
}
if !opts.dry_run {
packer.finalize().await?;
indexer.write().await.finalize().await?;
packer.finalize()?;
indexer.write().unwrap().finalize()?;
}
if opts.delete {
@ -290,8 +277,7 @@ async fn repair_snaps(
true,
delete,
progress_counter("remove defect snapshots"),
)
.await?;
)?;
}
}
@ -305,8 +291,7 @@ enum Changed {
None,
}
#[async_recursion]
async fn repair_tree<BE: DecryptWriteBackend>(
fn repair_tree<BE: DecryptWriteBackend>(
be: &impl IndexedBackend,
packer: &mut Packer<BE>,
id: Option<Id>,
@ -324,7 +309,7 @@ async fn repair_tree<BE: DecryptWriteBackend>(
return Ok(*r);
}
let (tree, mut changed) = match Tree::from_backend(be, id).await {
let (tree, mut changed) = match Tree::from_backend(be, id) {
Ok(tree) => (tree, Changed::None),
Err(_) => {
warn!("tree {id} could not be loaded.");
@ -363,7 +348,7 @@ async fn repair_tree<BE: DecryptWriteBackend>(
}
NodeType::Dir {} => {
let (c, tree_id) =
repair_tree(be, packer, node.subtree, replaced, seen, opts).await?;
repair_tree(be, packer, node.subtree, replaced, seen, opts)?;
match c {
Changed::None => {}
Changed::This => {
@ -396,7 +381,7 @@ async fn repair_tree<BE: DecryptWriteBackend>(
// the tree has been changed => save it
let (chunk, new_id) = tree.serialize()?;
if !be.has_tree(&new_id) && !opts.dry_run {
packer.add(&chunk, &new_id).await?;
packer.add(&chunk, &new_id)?;
}
if let Some(id) = id {
replaced.insert(id, (c, new_id));

17
src/commands/repoinfo.rs
View File

@ -1,7 +1,6 @@
use anyhow::Result;
use clap::Parser;
use derive_more::Add;
use futures::TryStreamExt;
use log::*;
use prettytable::{format, row, Table};
@ -14,19 +13,16 @@ use crate::repo::{IndexFile, IndexPack};
#[derive(Parser)]
pub(super) struct Opts;
pub(super) async fn execute(
pub(super) fn execute(
be: &impl DecryptReadBackend,
hot_be: &Option<impl ReadBackend>,
_opts: Opts,
) -> Result<()> {
fileinfo("repository files", be).await?;
fileinfo("repository files", be)?;
if let Some(hot_be) = hot_be {
fileinfo("hot repository files", hot_be).await?;
fileinfo("hot repository files", hot_be)?;
}
let p = progress_counter("scanning index...");
let mut stream = be.stream_all::<IndexFile>(p.clone()).await?;
#[derive(Default, Clone, Copy, Add)]
struct Info {
count: u64,
@ -59,7 +55,8 @@ pub(super) async fn execute(
info[BlobType::Data].min_pack_size = u64::MAX;
let mut info_delete = BlobTypeMap::<Info>::default();
while let Some((_, index)) = stream.try_next().await? {
let p = progress_counter("scanning index...");
for (_, index) in be.stream_all::<IndexFile>(p.clone())? {
for pack in &index.packs {
info[pack.blob_type()].add_pack(pack);
@ -109,14 +106,14 @@ pub(super) async fn execute(
Ok(())
}
async fn fileinfo(text: &str, be: &impl ReadBackend) -> Result<()> {
fn fileinfo(text: &str, be: &impl ReadBackend) -> Result<()> {
info!("scanning files...");
let mut table = Table::new();
let mut total_count = 0;
let mut total_size = 0;
for tpe in ALL_FILE_TYPES {
let list = be.list_with_size(tpe).await?;
let list = be.list_with_size(tpe)?;
let count = list.len();
let size = list.iter().map(|f| f.1 as u64).sum();
table.add_row(row![format!("{:?}", tpe), r->count, r->bytes(size)]);

136
src/commands/restore.rs
View File

@ -7,10 +7,9 @@ use std::path::{Path, PathBuf};
use anyhow::{anyhow, bail, Result};
use clap::{AppSettings, Parser};
use derive_getters::Dissolve;
use futures::{stream::FuturesUnordered, TryStreamExt};
use ignore::{DirEntry, WalkBuilder};
use log::*;
use tokio::spawn;
use rayon::ThreadPoolBuilder;
use super::{bytes, progress_bytes, progress_counter, wait, warm_up, warm_up_command};
use crate::backend::{DecryptReadBackend, FileType, LocalBackend};
@ -58,7 +57,7 @@ pub(super) struct Opts {
dest: String,
}
pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> {
pub(super) fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts) -> Result<()> {
if let Some(command) = &opts.warm_up_command {
if !command.contains("%id") {
bail!("warm-up command must contain %id!")
@ -67,15 +66,15 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts)
}
let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter("")).await?;
let snap = SnapshotFile::from_str(be, id, |_| true, progress_counter(""))?;
let index = IndexBackend::new(be, progress_counter("")).await?;
let tree = Tree::subtree_id(&index, snap.tree, Path::new(path)).await?;
let index = IndexBackend::new(be, progress_counter(""))?;
let tree = Tree::subtree_id(&index, snap.tree, Path::new(path))?;
let dest = LocalBackend::new(&opts.dest);
let p = progress_spinner("collecting file information...");
let file_infos = allocate_and_collect(&dest, index.clone(), tree, &opts).await?;
let file_infos = allocate_and_collect(&dest, index.clone(), tree, &opts)?;
p.finish();
info!("total restore size: {}", bytes(file_infos.total_size));
if file_infos.matched_size > 0 {
@ -89,22 +88,22 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts)
info!("all file contents are fine.");
} else {
if opts.warm_up {
warm_up(be, file_infos.to_packs().into_iter()).await?;
warm_up(be, file_infos.to_packs().into_iter())?;
} else if opts.warm_up_command.is_some() {
warm_up_command(
file_infos.to_packs().into_iter(),
opts.warm_up_command.as_ref().unwrap(),
)?;
}
wait(opts.warm_up_wait).await;
wait(opts.warm_up_wait);
if !opts.dry_run {
restore_contents(be, &dest, file_infos).await?;
restore_contents(be, &dest, file_infos)?;
}
}
if !opts.dry_run {
let p = progress_spinner("setting metadata...");
restore_metadata(&dest, index, tree, &opts).await?;
restore_metadata(&dest, index, tree, &opts)?;
p.finish();
}
@ -113,7 +112,7 @@ pub(super) async fn execute(be: &(impl DecryptReadBackend + Unpin), opts: Opts)
}
/// collect restore information, scan existing files and allocate non-existing files
async fn allocate_and_collect(
fn allocate_and_collect(
dest: &LocalBackend,
index: impl IndexedBackend + Unpin,
tree: Id,
@ -222,8 +221,8 @@ async fn allocate_and_collect(
.filter_map(Result::ok); // TODO: print out the ignored error
let mut next_dst = dst_iter.next();
let mut node_streamer = NodeStreamer::new(index.clone(), tree).await?;
let mut next_node = node_streamer.try_next().await?;
let mut node_streamer = NodeStreamer::new(index.clone(), tree)?;
let mut next_node = node_streamer.next().transpose()?;
loop {
match (&next_dst, &next_node) {
@ -244,16 +243,16 @@ async fn allocate_and_collect(
// does not match the type of the node in the snapshot!
process_node(path, node, true)?;
next_dst = dst_iter.next();
next_node = node_streamer.try_next().await?;
next_node = node_streamer.next().transpose()?;
}
Ordering::Greater => {
process_node(path, node, false)?;
next_node = node_streamer.try_next().await?;
next_node = node_streamer.next().transpose()?;
}
},
(None, Some((path, node))) => {
process_node(path, node, false)?;
next_node = node_streamer.try_next().await?;
next_node = node_streamer.next().transpose()?;
}
}
}
@ -272,7 +271,7 @@ async fn allocate_and_collect(
/// restore_contents restores all files contents as described by file_infos
/// using the ReadBackend be and writing them into the LocalBackend dest.
async fn restore_contents(
fn restore_contents(
be: &impl DecryptReadBackend,
dest: &LocalBackend,
file_infos: FileInfos,
@ -281,79 +280,76 @@ async fn restore_contents(
let p = progress_bytes("restoring file contents...");
p.set_length(total_size - matched_size);
let mut stream = FuturesUnordered::new();
const MAX_READER: usize = 20;
for (pack, blob) in restore_info {
for (bl, fls) in blob {
let p = p.clone();
let be = be.clone();
let dest = dest.clone();
let pool = ThreadPoolBuilder::new().num_threads(MAX_READER).build()?;
pool.in_place_scope(|s| {
for (pack, blob) in restore_info {
for (bl, fls) in blob {
let from_file = fls
.iter()
.find(|fl| fl.matches)
.map(|fl| (filenames[fl.file_idx].clone(), fl.file_start));
let from_file = fls
.iter()
.find(|fl| fl.matches)
.map(|fl| (filenames[fl.file_idx].clone(), fl.file_start));
let name_dests: Vec<_> = fls
.iter()
.filter(|fl| !fl.matches)
.map(|fl| (filenames[fl.file_idx].clone(), fl.file_start))
.collect();
let p = &p;
let name_dests: Vec<_> = fls
.iter()
.filter(|fl| !fl.matches)
.map(|fl| (filenames[fl.file_idx].clone(), fl.file_start))
.collect();
if !name_dests.is_empty() {
// TODO: error handling!
s.spawn(move |s1| {
let data = match from_file {
Some((filename, start)) => {
// read from existing file
dest.read_at(filename, start, bl.data_length()).unwrap()
}
None => {
// read pack at blob_offset with length blob_length
be.read_encrypted_partial(
FileType::Pack,
&pack,
false,
bl.offset,
bl.length,
bl.uncompressed_length,
)
.unwrap()
}
};
let size = bl.data_length();
if !name_dests.is_empty() {
while stream.len() > MAX_READER {
stream.try_next().await?;
// save into needed files in parallel
for (name, start) in name_dests {
let data = data.clone();
s1.spawn(move |_| {
dest.write_at(&name, start, &data).unwrap();
p.inc(size);
});
}
})
}
// TODO: error handling!
stream.push(spawn(async move {
let data = match from_file {
Some((filename, start)) => {
// read from existing file
dest.read_at(filename, start, bl.data_length()).unwrap()
}
None => {
// read pack at blob_offset with length blob_length
be.read_encrypted_partial(
FileType::Pack,
&pack,
false,
bl.offset,
bl.length,
bl.uncompressed_length,
)
.await
.unwrap()
}
};
// save into needed files in parallel
for (name, start) in name_dests {
dest.write_at(&name, start, &data).unwrap();
p.inc(bl.data_length());
}
}))
}
}
}
});
stream.try_collect().await?;
p.finish();
Ok(())
}
async fn restore_metadata(
fn restore_metadata(
dest: &LocalBackend,
index: impl IndexedBackend + Unpin,
tree: Id,
opts: &Opts,
) -> Result<()> {
// walk over tree in repository and compare with tree in dest
let mut node_streamer = NodeStreamer::new(index, tree).await?;
let mut node_streamer = NodeStreamer::new(index, tree)?;
let mut dir_stack = Vec::new();
while let Some((path, node)) = node_streamer.try_next().await? {
while let Some((path, node)) = node_streamer.next().transpose()? {
match node.node_type() {
NodeType::Dir => {
// set metadata for all non-parent paths in stack

2
src/commands/self_update.rs
View File

@ -9,7 +9,7 @@ pub(super) struct Opts {
force: bool,
}
pub(super) async fn execute(opts: Opts) -> Result<()> {
pub(super) fn execute(opts: Opts) -> Result<()> {
let status = self_update::backends::github::Update::configure()
.repo_owner("rustic-rs")
.repo_name("rustic")

9
src/commands/snapshots.rs
View File

@ -43,7 +43,7 @@ pub(super) struct Opts {
ids: Vec<String>,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &impl DecryptReadBackend,
mut opts: Opts,
config_file: RusticConfig,
@ -51,10 +51,9 @@ pub(super) async fn execute(
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let groups = match &opts.ids[..] {
[] => SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by).await?,
[] => SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by)?,
[id] if id == "latest" => {
SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by)
.await?
SnapshotFile::group_from_backend(be, &opts.filter, &opts.group_by)?
.into_iter()
.map(|(group, mut snaps)| {
snaps.sort_unstable();
@ -67,7 +66,7 @@ pub(super) async fn execute(
}
_ => vec![(
SnapshotGroup::default(),
SnapshotFile::from_ids(be, &opts.ids).await?,
SnapshotFile::from_ids(be, &opts.ids)?,
)],
};

11
src/commands/tag.rs
View File

@ -68,7 +68,7 @@ pub(super) struct Opts {
ids: Vec<String>,
}
pub(super) async fn execute(
pub(super) fn execute(
be: &impl DecryptFullBackend,
mut opts: Opts,
config_file: RusticConfig,
@ -76,8 +76,8 @@ pub(super) async fn execute(
config_file.merge_into("snapshot-filter", &mut opts.filter)?;
let snapshots = match opts.ids.is_empty() {
true => SnapshotFile::all_from_backend(be, &opts.filter).await?,
false => SnapshotFile::from_ids(be, &opts.ids).await?,
true => SnapshotFile::all_from_backend(be, &opts.filter)?,
false => SnapshotFile::from_ids(be, &opts.ids)?,
};
let delete = match (
@ -109,11 +109,10 @@ pub(super) async fn execute(
),
(false, false) => {
let p = progress_counter("saving new snapshots...");
be.save_list(snapshots, p).await?;
be.save_list(snapshots, p)?;
let p = progress_counter("deleting old snapshots...");
be.delete_list(FileType::Snapshot, true, old_snap_ids, p)
.await?;
be.delete_list(FileType::Snapshot, true, old_snap_ids, p)?;
}
}
Ok(())

9
src/index/binarysorted.rs
View File

@ -4,6 +4,7 @@ use super::{BlobType, IndexEntry, ReadIndex};
use crate::blob::BlobTypeMap;
use crate::id::Id;
use crate::repo::{IndexBlob, IndexPack};
use rayon::prelude::*;
#[derive(Debug, PartialEq, Eq)]
struct SortedEntry {
@ -82,8 +83,8 @@ impl IndexCollector {
Index(self.0.map(|_, mut tc| {
match &mut tc.entries {
EntriesVariants::None => {}
EntriesVariants::Ids(ids) => ids.sort_unstable(),
EntriesVariants::FullEntries(entries) => entries.sort_unstable_by_key(|e| e.id),
EntriesVariants::Ids(ids) => ids.par_sort_unstable(),
EntriesVariants::FullEntries(entries) => entries.par_sort_unstable_by_key(|e| e.id),
};
let packs = tc.packs.into_iter().map(|(id, _)| id).collect();
@ -182,13 +183,13 @@ impl IntoIterator for Index {
fn into_iter(mut self) -> Self::IntoIter {
for (_, tc) in self.0.iter_mut() {
if let EntriesVariants::FullEntries(entries) = &mut tc.entries {
entries.sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx))
entries.par_sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx))
}
}
PackIndexes {
c: Index(self.0.map(|_, mut tc| {
if let EntriesVariants::FullEntries(entries) = &mut tc.entries {
entries.sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx))
entries.par_sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx))
}
TypeIndex {

22
src/index/indexer.rs
View File

@ -1,9 +1,9 @@
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::RwLock;
use std::time::{Duration, SystemTime};
use anyhow::Result;
use tokio::sync::RwLock;
use crate::backend::DecryptWriteBackend;
use crate::id::Id;
@ -53,26 +53,26 @@ impl<BE: DecryptWriteBackend> Indexer<BE> {
Arc::new(RwLock::new(self))
}
pub async fn finalize(&self) -> Result<()> {
self.save().await
pub fn finalize(&self) -> Result<()> {
self.save()
}
pub async fn save(&self) -> Result<()> {
pub fn save(&self) -> Result<()> {
if (self.file.packs.len() + self.file.packs_to_delete.len()) > 0 {
self.be.save_file(&self.file).await?;
self.be.save_file(&self.file)?;
}
Ok(())
}
pub async fn add(&mut self, pack: IndexPack) -> Result<()> {
self.add_with(pack, false).await
pub fn add(&mut self, pack: IndexPack) -> Result<()> {
self.add_with(pack, false)
}
pub async fn add_remove(&mut self, pack: IndexPack) -> Result<()> {
self.add_with(pack, true).await
pub fn add_remove(&mut self, pack: IndexPack) -> Result<()> {
self.add_with(pack, true)
}
pub async fn add_with(&mut self, pack: IndexPack, delete: bool) -> Result<()> {
pub fn add_with(&mut self, pack: IndexPack, delete: bool) -> Result<()> {
self.count += pack.blobs.len();
if let Some(indexed) = &mut self.indexed {
@ -85,7 +85,7 @@ impl<BE: DecryptWriteBackend> Indexer<BE> {
// check if IndexFile needs to be saved
if self.count >= MAX_COUNT || self.created.elapsed()? >= MAX_AGE {
self.save().await?;
self.save()?;
self.reset();
}
Ok(())

51
src/index/mod.rs
View File

@ -2,11 +2,9 @@ use std::num::NonZeroU32;
use std::sync::Arc;
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use bytes::Bytes;
use derive_getters::Getters;
use derive_more::Constructor;
use futures::StreamExt;
use indicatif::ProgressBar;
use crate::backend::{DecryptReadBackend, FileType};
@ -41,17 +39,15 @@ impl IndexEntry {
}
/// Get a blob described by IndexEntry from the backend
pub async fn read_data<B: DecryptReadBackend>(&self, be: &B) -> Result<Bytes> {
let data = be
.read_encrypted_partial(
FileType::Pack,
&self.pack,
self.blob_type.is_cacheable(),
self.offset,
self.length,
self.uncompressed_length,
)
.await?;
pub fn read_data<B: DecryptReadBackend>(&self, be: &B) -> Result<Bytes> {
let data = be.read_encrypted_partial(
FileType::Pack,
&self.pack,
self.blob_type.is_cacheable(),
self.offset,
self.length,
self.uncompressed_length,
)?;
Ok(data)
}
@ -85,16 +81,15 @@ pub trait ReadIndex {
}
}
#[async_trait]
pub trait IndexedBackend: ReadIndex + Clone + Sync + Send + 'static {
type Backend: DecryptReadBackend;
fn be(&self) -> &Self::Backend;
async fn blob_from_backend(&self, tpe: &BlobType, id: &Id) -> Result<Bytes> {
fn blob_from_backend(&self, tpe: &BlobType, id: &Id) -> Result<Bytes> {
match self.get_id(tpe, id) {
None => Err(anyhow!("blob not found in index")),
Some(ie) => ie.read_data(self.be()).await,
Some(ie) => ie.read_data(self.be()),
}
}
}
@ -126,31 +121,23 @@ impl<BE: DecryptReadBackend> IndexBackend<BE> {
}
}
async fn new_from_collector(
be: &BE,
p: ProgressBar,
mut collector: IndexCollector,
) -> Result<Self> {
fn new_from_collector(be: &BE, p: ProgressBar, mut collector: IndexCollector) -> Result<Self> {
p.set_prefix("reading index...");
let mut stream = be
.stream_all::<IndexFile>(p.clone())
.await?
.map(|i| i.unwrap().1);
while let Some(index) = stream.next().await {
collector.extend(index.packs);
for (_, i) in be.stream_all::<IndexFile>(p.clone())? {
collector.extend(i.packs);
}
p.finish();
Ok(Self::new_from_index(be, collector.into_index()))
}
pub async fn new(be: &BE, p: ProgressBar) -> Result<Self> {
Self::new_from_collector(be, p, IndexCollector::new(IndexType::Full)).await
pub fn new(be: &BE, p: ProgressBar) -> Result<Self> {
Self::new_from_collector(be, p, IndexCollector::new(IndexType::Full))
}
pub async fn only_full_trees(be: &BE, p: ProgressBar) -> Result<Self> {
Self::new_from_collector(be, p, IndexCollector::new(IndexType::FullTrees)).await
pub fn only_full_trees(be: &BE, p: ProgressBar) -> Result<Self> {
Self::new_from_collector(be, p, IndexCollector::new(IndexType::FullTrees))
}
pub fn into_index(self) -> Index {

5
src/main.rs
View File

@ -40,7 +40,6 @@ mod id;
mod index;
mod repo;
#[tokio::main]
async fn main() -> Result<()> {
commands::execute().await
fn main() -> Result<()> {
commands::execute()
}

22
src/repo/keyfile.rs
View File

@ -88,8 +88,8 @@ impl KeyFile {
impl KeyFile {
/// Get a KeyFile from the backend
pub async fn from_backend<B: ReadBackend>(be: &B, id: &Id) -> Result<Self> {
let data = be.read_full(FileType::Key, id).await?;
pub fn from_backend<B: ReadBackend>(be: &B, id: &Id) -> Result<Self> {
let data = be.read_full(FileType::Key, id)?;
Ok(serde_json::from_slice(&data)?)
}
}
@ -136,29 +136,23 @@ impl MasterKey {
}
}
async fn key_from_backend<B: ReadBackend>(
be: &B,
id: &Id,
passwd: &impl AsRef<[u8]>,
) -> Result<Key> {
KeyFile::from_backend(be, id)
.await?
.key_from_password(passwd)
fn key_from_backend<B: ReadBackend>(be: &B, id: &Id, passwd: &impl AsRef<[u8]>) -> Result<Key> {
KeyFile::from_backend(be, id)?.key_from_password(passwd)
}
/// Find a KeyFile in the backend that fits to the given password and return the contained key.
/// If a key hint is given, only this key is tested.
/// This is recommended for a large number of keys.
pub async fn find_key_in_backend<B: ReadBackend>(
pub fn find_key_in_backend<B: ReadBackend>(
be: &B,
passwd: &impl AsRef<[u8]>,
hint: Option<&Id>,
) -> Result<Key> {
match hint {
Some(id) => key_from_backend(be, id, passwd).await,
Some(id) => key_from_backend(be, id, passwd),
None => {
for id in be.list(FileType::Key).await? {
if let Ok(key) = key_from_backend(be, &id, passwd).await {
for id in be.list(FileType::Key)? {
if let Ok(key) = key_from_backend(be, &id, passwd) {
return Ok(key);
}
}

9
src/repo/packfile.rs
View File

@ -156,7 +156,7 @@ impl PackHeader {
}
/// Read the pack header directly from a packfile using the backend
pub async fn from_file(
pub fn from_file(
be: &impl DecryptReadBackend,
id: Id,
size_hint: Option<u32>,
@ -170,9 +170,7 @@ impl PackHeader {
// read (guessed) header + length field
let read_size = size_guess + LENGTH_LEN;
let offset = pack_size - read_size;
let mut data = be
.read_partial(FileType::Pack, &id, false, offset, read_size)
.await?;
let mut data = be.read_partial(FileType::Pack, &id, false, offset, read_size)?;
// get header length from the file
let size_real =
@ -185,8 +183,7 @@ impl PackHeader {
} else {
// size_guess was too small; we have to read again
let offset = pack_size - size_real - LENGTH_LEN;
be.read_partial(FileType::Pack, &id, false, offset, size_real)
.await?
be.read_partial(FileType::Pack, &id, false, offset, size_real)?
};
Self::from_binary(&be.decrypt(&data)?)

57
src/repo/snapshotfile.rs
View File

@ -6,7 +6,6 @@ use anyhow::{anyhow, bail, Result};
use chrono::{DateTime, Local};
use clap::Parser;
use derivative::Derivative;
use futures::{future, TryStreamExt};
use indicatif::ProgressBar;
use log::*;
use merge::Merge;
@ -108,37 +107,37 @@ impl SnapshotFile {
}
/// Get a SnapshotFile from the backend
pub async fn from_backend<B: DecryptReadBackend>(be: &B, id: &Id) -> Result<Self> {
Ok(Self::set_id((*id, be.get_file(id).await?)))
pub fn from_backend<B: DecryptReadBackend>(be: &B, id: &Id) -> Result<Self> {
Ok(Self::set_id((*id, be.get_file(id)?)))
}
pub async fn from_str<B: DecryptReadBackend>(
pub fn from_str<B: DecryptReadBackend>(
be: &B,
string: &str,
predicate: impl FnMut(&Self) -> bool,
predicate: impl FnMut(&Self) -> bool + Send + Sync,
p: ProgressBar,
) -> Result<Self> {
match string {
"latest" => Self::latest(be, predicate, p).await,
_ => Self::from_id(be, string).await,
"latest" => Self::latest(be, predicate, p),
_ => Self::from_id(be, string),
}
}
/// Get the latest SnapshotFile from the backend
pub async fn latest<B: DecryptReadBackend>(
pub fn latest<B: DecryptReadBackend>(
be: &B,
predicate: impl FnMut(&Self) -> bool,
predicate: impl FnMut(&Self) -> bool + Send + Sync,
p: ProgressBar,
) -> Result<Self> {
p.set_prefix("getting latest snapshot...");
let mut latest: Option<Self> = None;
let mut pred = predicate;
let mut snaps = be.stream_all::<SnapshotFile>(p.clone()).await?;
while let Some((id, mut snap)) = snaps.try_next().await? {
for (id, mut snap) in be.stream_all::<SnapshotFile>(p.clone())? {
if !pred(&snap) {
continue;
}
snap.id = id;
match &latest {
Some(l) if l.time > snap.time => {}
@ -152,21 +151,20 @@ impl SnapshotFile {
}
/// Get a SnapshotFile from the backend by (part of the) id
pub async fn from_id<B: DecryptReadBackend>(be: &B, id: &str) -> Result<Self> {
pub fn from_id<B: DecryptReadBackend>(be: &B, id: &str) -> Result<Self> {
info!("getting snapshot...");
let id = be.find_id(FileType::Snapshot, id).await?;
SnapshotFile::from_backend(be, &id).await
let id = be.find_id(FileType::Snapshot, id)?;
SnapshotFile::from_backend(be, &id)
}
/// Get a Vector of SnapshotFile from the backend by list of (parts of the) ids
pub async fn from_ids<B: DecryptReadBackend>(be: &B, ids: &[String]) -> Result<Vec<Self>> {
let ids = be.find_ids(FileType::Snapshot, ids).await?;
pub fn from_ids<B: DecryptReadBackend>(be: &B, ids: &[String]) -> Result<Vec<Self>> {
let ids = be.find_ids(FileType::Snapshot, ids)?;
Ok(be
.stream_list::<Self>(ids, ProgressBar::hidden())
.await?
.map_ok(Self::set_id)
.try_collect()
.await?)
.stream_list::<Self>(ids, ProgressBar::hidden())?
.into_iter()
.map(Self::set_id)
.collect())
}
fn cmp_group(&self, crit: &SnapshotGroupCriterion, other: &Self) -> Ordering {
@ -199,12 +197,12 @@ impl SnapshotFile {
/// Get SnapshotFiles which match the filter grouped by the group criterion
/// from the backend
pub async fn group_from_backend<B: DecryptReadBackend>(
pub fn group_from_backend<B: DecryptReadBackend>(
be: &B,
filter: &SnapshotFilter,
crit: &SnapshotGroupCriterion,
) -> Result<Vec<(SnapshotGroup, Vec<Self>)>> {
let mut snaps = Self::all_from_backend(be, filter).await?;
let mut snaps = Self::all_from_backend(be, filter)?;
snaps.sort_unstable_by(|sn1, sn2| sn1.cmp_group(crit, sn2));
let mut result = Vec::new();
@ -233,17 +231,16 @@ impl SnapshotFile {
Ok(result)
}
pub async fn all_from_backend<B: DecryptReadBackend>(
pub fn all_from_backend<B: DecryptReadBackend>(
be: &B,
filter: &SnapshotFilter,
) -> Result<Vec<Self>> {
Ok(be
.stream_all::<SnapshotFile>(ProgressBar::hidden())
.await?
.map_ok(Self::set_id)
.try_filter(|sn| future::ready(sn.matches(filter)))
.try_collect()
.await?)
.stream_all::<SnapshotFile>(ProgressBar::hidden())?
.into_iter()
.map(Self::set_id)
.filter(|sn| sn.matches(filter))
.collect())
}
pub fn matches(&self, filter: &SnapshotFilter) -> bool {