]> git.proxmox.com Git - proxmox-backup.git/blame - proxmox-backup-client/src/main.rs
move data_blob encode/decode from crypt_config.rs to data_blob.rs
[proxmox-backup.git] / proxmox-backup-client / src / main.rs
CommitLineData
f1a83e97 1use std::collections::HashSet;
0351f23b 2use std::io::{self, Read, Write, Seek, SeekFrom};
c443f58b
WB
3use std::path::{Path, PathBuf};
4use std::pin::Pin;
5use std::sync::{Arc, Mutex};
a6f87283 6use std::task::Context;
c443f58b
WB
7
8use anyhow::{bail, format_err, Error};
c443f58b 9use futures::stream::{StreamExt, TryStreamExt};
c443f58b 10use serde_json::{json, Value};
c443f58b 11use tokio::sync::mpsc;
7c667013 12use tokio_stream::wrappers::ReceiverStream;
c443f58b 13use xdg::BaseDirectories;
2761d6a4 14
c443f58b 15use pathpatterns::{MatchEntry, MatchType, PatternFlag};
6a7be83e
DM
16use proxmox::{
17 tools::{
18 time::{strftime_local, epoch_i64},
ff8945fd 19 fs::{file_get_json, replace_file, CreateOptions, image_size},
6a7be83e
DM
20 },
21 api::{
22 api,
6a7be83e
DM
23 ApiMethod,
24 RpcEnvironment,
6a7be83e
DM
25 cli::*,
26 },
27};
a6f87283 28use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
ff5d3707 29
51ec8a3c
WB
30use pbs_api_types::{
31 BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem,
32 PruneListItem, SnapshotListItem, StorageStatus,
33};
2b7f8dd5
WB
34use pbs_client::{
35 BACKUP_SOURCE_SCHEMA,
36 BackupReader,
37 BackupRepository,
38 BackupSpecificationType,
39 BackupStats,
40 BackupWriter,
38629c39
WB
41 ChunkStream,
42 FixedChunkStream,
2b7f8dd5
WB
43 HttpClient,
44 PxarBackupStream,
45 RemoteChunkReader,
46 UploadOptions,
47 delete_ticket_info,
48 parse_backup_specification,
49 view_task_result,
50};
51use pbs_client::catalog_shell::Shell;
52use pbs_client::tools::{
53 complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
54 complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
55 complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
56 extract_repository_from_value,
57 key_source::{
58 crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
59 KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
60 },
61 CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
62};
51ec8a3c 63use pbs_datastore::{CATALOG_NAME, CryptConfig, KeyConfig, decrypt_key, rsa_encrypt_key_config};
2b7f8dd5 64use pbs_datastore::backup_info::{BackupDir, BackupGroup};
51ec8a3c
WB
65use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter};
66use pbs_datastore::chunk_store::verify_chunk_size;
eb5e0ae6 67use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader};
2b7f8dd5
WB
68use pbs_datastore::fixed_index::FixedIndexReader;
69use pbs_datastore::index::IndexFile;
51ec8a3c
WB
70use pbs_datastore::manifest::{
71 ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type,
72};
2b7f8dd5 73use pbs_datastore::read_chunk::AsyncReadChunk;
51ec8a3c 74use pbs_datastore::prune::PruneOptions;
4805edc4
WB
75use pbs_tools::sync::StdChannelWriter;
76use pbs_tools::tokio::TokioWriterAdapter;
3c8c2827 77use pbs_tools::json;
f323e906 78
e351ac78
WB
79mod benchmark;
80pub use benchmark::*;
81mod mount;
82pub use mount::*;
83mod task;
84pub use task::*;
85mod catalog;
86pub use catalog::*;
87mod snapshot;
88pub use snapshot::*;
89pub mod key;
caea8d61 90
d0a03d40
DM
91fn record_repository(repo: &BackupRepository) {
92
93 let base = match BaseDirectories::with_prefix("proxmox-backup") {
94 Ok(v) => v,
95 _ => return,
96 };
97
98 // usually $HOME/.cache/proxmox-backup/repo-list
99 let path = match base.place_cache_file("repo-list") {
100 Ok(v) => v,
101 _ => return,
102 };
103
11377a47 104 let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
d0a03d40
DM
105
106 let repo = repo.to_string();
107
108 data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
109
110 let mut map = serde_json::map::Map::new();
111
112 loop {
113 let mut max_used = 0;
114 let mut max_repo = None;
115 for (repo, count) in data.as_object().unwrap() {
116 if map.contains_key(repo) { continue; }
117 if let Some(count) = count.as_i64() {
118 if count > max_used {
119 max_used = count;
120 max_repo = Some(repo);
121 }
122 }
123 }
124 if let Some(repo) = max_repo {
125 map.insert(repo.to_owned(), json!(max_used));
126 } else {
127 break;
128 }
129 if map.len() > 10 { // store max. 10 repos
130 break;
131 }
132 }
133
134 let new_data = json!(map);
135
feaa1ad3 136 let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
d0a03d40
DM
137}
138
42af4b8f
DM
139async fn api_datastore_list_snapshots(
140 client: &HttpClient,
141 store: &str,
142 group: Option<BackupGroup>,
f24fc116 143) -> Result<Value, Error> {
42af4b8f
DM
144
145 let path = format!("api2/json/admin/datastore/{}/snapshots", store);
146
147 let mut args = json!({});
148 if let Some(group) = group {
149 args["backup-type"] = group.backup_type().into();
150 args["backup-id"] = group.backup_id().into();
151 }
152
153 let mut result = client.get(&path, Some(args)).await?;
154
f24fc116 155 Ok(result["data"].take())
42af4b8f
DM
156}
157
43abba4b 158pub async fn api_datastore_latest_snapshot(
27c9affb
DM
159 client: &HttpClient,
160 store: &str,
161 group: BackupGroup,
6a7be83e 162) -> Result<(String, String, i64), Error> {
27c9affb 163
f24fc116
DM
164 let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
165 let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
27c9affb
DM
166
167 if list.is_empty() {
168 bail!("backup group {:?} does not contain any snapshots.", group.group_path());
169 }
170
171 list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
172
6a7be83e 173 let backup_time = list[0].backup_time;
27c9affb
DM
174
175 Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
176}
177
e9722f8b 178async fn backup_directory<P: AsRef<Path>>(
cf9271e2 179 client: &BackupWriter,
17d6979a 180 dir_path: P,
247cdbce 181 archive_name: &str,
36898ffc 182 chunk_size: Option<usize>,
f1d76ecf 183 catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
2b7f8dd5 184 pxar_create_options: pbs_client::pxar::PxarCreateOptions,
e43b9175 185 upload_options: UploadOptions,
2c3891d1 186) -> Result<BackupStats, Error> {
33d64b81 187
6fc053ed
CE
188 let pxar_stream = PxarBackupStream::open(
189 dir_path.as_ref(),
6fc053ed 190 catalog,
77486a60 191 pxar_create_options,
6fc053ed 192 )?;
e9722f8b 193 let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
ff3d3100 194
0bfcea6a 195 let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
5e7a09be 196
7c667013 197 let stream = ReceiverStream::new(rx)
e9722f8b 198 .map_err(Error::from);
17d6979a 199
c4ff3dce 200 // spawn chunker inside a separate task so that it can run parallel
e9722f8b 201 tokio::spawn(async move {
db0cb9ce
WB
202 while let Some(v) = chunk_stream.next().await {
203 let _ = tx.send(v).await;
204 }
e9722f8b 205 });
17d6979a 206
e43b9175
FG
207 if upload_options.fixed_size.is_some() {
208 bail!("cannot backup directory with fixed chunk size!");
209 }
210
e9722f8b 211 let stats = client
e43b9175 212 .upload_stream(archive_name, stream, upload_options)
e9722f8b 213 .await?;
bcd879cf 214
2c3891d1 215 Ok(stats)
bcd879cf
DM
216}
217
e9722f8b 218async fn backup_image<P: AsRef<Path>>(
cf9271e2 219 client: &BackupWriter,
6af905c1
DM
220 image_path: P,
221 archive_name: &str,
36898ffc 222 chunk_size: Option<usize>,
e43b9175 223 upload_options: UploadOptions,
2c3891d1 224) -> Result<BackupStats, Error> {
6af905c1 225
6af905c1
DM
226 let path = image_path.as_ref().to_owned();
227
e9722f8b 228 let file = tokio::fs::File::open(path).await?;
6af905c1 229
db0cb9ce 230 let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
6af905c1
DM
231 .map_err(Error::from);
232
36898ffc 233 let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
6af905c1 234
e43b9175
FG
235 if upload_options.fixed_size.is_none() {
236 bail!("cannot backup image with dynamic chunk size!");
237 }
238
e9722f8b 239 let stats = client
e43b9175 240 .upload_stream(archive_name, stream, upload_options)
e9722f8b 241 .await?;
6af905c1 242
2c3891d1 243 Ok(stats)
6af905c1
DM
244}
245
a47a02ae
DM
246#[api(
247 input: {
248 properties: {
249 repository: {
250 schema: REPO_URL_SCHEMA,
251 optional: true,
252 },
253 "output-format": {
254 schema: OUTPUT_FORMAT,
255 optional: true,
256 },
257 }
258 }
259)]
260/// List backup groups.
261async fn list_backup_groups(param: Value) -> Result<Value, Error> {
812c6f87 262
c81b2b7c
DM
263 let output_format = get_output_format(&param);
264
2665cef7 265 let repo = extract_repository_from_value(&param)?;
812c6f87 266
f3fde36b 267 let client = connect(&repo)?;
812c6f87 268
d0a03d40 269 let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
812c6f87 270
8a8a4703 271 let mut result = client.get(&path, None).await?;
812c6f87 272
d0a03d40
DM
273 record_repository(&repo);
274
c81b2b7c
DM
275 let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
276 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
277 let group = BackupGroup::new(item.backup_type, item.backup_id);
278 Ok(group.group_path().to_str().unwrap().to_owned())
279 };
812c6f87 280
18deda40
DM
281 let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
282 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 283 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
18deda40 284 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
c81b2b7c 285 };
812c6f87 286
c81b2b7c
DM
287 let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
288 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
770a36e5 289 Ok(pbs_tools::format::render_backup_file_list(&item.files))
c81b2b7c 290 };
812c6f87 291
c81b2b7c
DM
292 let options = default_table_format_options()
293 .sortby("backup-type", false)
294 .sortby("backup-id", false)
295 .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
18deda40
DM
296 .column(
297 ColumnConfig::new("last-backup")
298 .renderer(render_last_backup)
299 .header("last snapshot")
300 .right_align(false)
301 )
c81b2b7c
DM
302 .column(ColumnConfig::new("backup-count"))
303 .column(ColumnConfig::new("files").renderer(render_files));
ad20d198 304
c81b2b7c 305 let mut data: Value = result["data"].take();
ad20d198 306
e351ac78 307 let return_type = &pbs_api_types::ADMIN_DATASTORE_LIST_GROUPS_RETURN_TYPE;
812c6f87 308
b2362a12 309 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc 310
812c6f87
DM
311 Ok(Value::Null)
312}
313
344add38
DW
314#[api(
315 input: {
316 properties: {
317 repository: {
318 schema: REPO_URL_SCHEMA,
319 optional: true,
320 },
321 group: {
322 type: String,
323 description: "Backup group.",
324 },
325 "new-owner": {
e6dc35ac 326 type: Authid,
344add38
DW
327 },
328 }
329 }
330)]
331/// Change owner of a backup group
332async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
333
334 let repo = extract_repository_from_value(&param)?;
335
f3fde36b 336 let mut client = connect(&repo)?;
344add38
DW
337
338 param.as_object_mut().unwrap().remove("repository");
339
340 let group: BackupGroup = group.parse()?;
341
342 param["backup-type"] = group.backup_type().into();
343 param["backup-id"] = group.backup_id().into();
344
345 let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
346 client.post(&path, Some(param)).await?;
347
348 record_repository(&repo);
349
350 Ok(())
351}
352
a47a02ae
DM
353#[api(
354 input: {
355 properties: {
356 repository: {
357 schema: REPO_URL_SCHEMA,
358 optional: true,
359 },
360 }
361 }
362)]
363/// Try to login. If successful, store ticket.
364async fn api_login(param: Value) -> Result<Value, Error> {
e240d8be
DM
365
366 let repo = extract_repository_from_value(&param)?;
367
f3fde36b 368 let client = connect(&repo)?;
8a8a4703 369 client.login().await?;
e240d8be
DM
370
371 record_repository(&repo);
372
373 Ok(Value::Null)
374}
375
a47a02ae
DM
376#[api(
377 input: {
378 properties: {
379 repository: {
380 schema: REPO_URL_SCHEMA,
381 optional: true,
382 },
383 }
384 }
385)]
386/// Logout (delete stored ticket).
387fn api_logout(param: Value) -> Result<Value, Error> {
e240d8be
DM
388
389 let repo = extract_repository_from_value(&param)?;
390
5030b7ce 391 delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
e240d8be
DM
392
393 Ok(Value::Null)
394}
395
e39974af
TL
396#[api(
397 input: {
398 properties: {
399 repository: {
400 schema: REPO_URL_SCHEMA,
401 optional: true,
402 },
403 "output-format": {
404 schema: OUTPUT_FORMAT,
405 optional: true,
406 },
407 }
408 }
409)]
410/// Show client and optional server version
411async fn api_version(param: Value) -> Result<(), Error> {
412
413 let output_format = get_output_format(&param);
414
415 let mut version_info = json!({
416 "client": {
a12b1be7
WB
417 "version": pbs_buildcfg::PROXMOX_PKG_VERSION,
418 "release": pbs_buildcfg::PROXMOX_PKG_RELEASE,
419 "repoid": pbs_buildcfg::PROXMOX_PKG_REPOID,
e39974af
TL
420 }
421 });
422
423 let repo = extract_repository_from_value(&param);
424 if let Ok(repo) = repo {
f3fde36b 425 let client = connect(&repo)?;
e39974af
TL
426
427 match client.get("api2/json/version", None).await {
428 Ok(mut result) => version_info["server"] = result["data"].take(),
429 Err(e) => eprintln!("could not connect to server - {}", e),
430 }
431 }
432 if output_format == "text" {
a12b1be7
WB
433 println!(
434 "client version: {}.{}",
435 pbs_buildcfg::PROXMOX_PKG_VERSION,
436 pbs_buildcfg::PROXMOX_PKG_RELEASE,
437 );
e39974af
TL
438 if let Some(server) = version_info["server"].as_object() {
439 let server_version = server["version"].as_str().unwrap();
440 let server_release = server["release"].as_str().unwrap();
441 println!("server version: {}.{}", server_version, server_release);
442 }
443 } else {
444 format_and_print_result(&version_info, &output_format);
445 }
446
447 Ok(())
448}
449
a47a02ae 450#[api(
94913f35 451 input: {
a47a02ae
DM
452 properties: {
453 repository: {
454 schema: REPO_URL_SCHEMA,
455 optional: true,
456 },
94913f35
DM
457 "output-format": {
458 schema: OUTPUT_FORMAT,
459 optional: true,
460 },
461 },
462 },
a47a02ae
DM
463)]
464/// Start garbage collection for a specific repository.
465async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
8cc0d6af 466
2665cef7 467 let repo = extract_repository_from_value(&param)?;
c2043614
DM
468
469 let output_format = get_output_format(&param);
8cc0d6af 470
f3fde36b 471 let mut client = connect(&repo)?;
8cc0d6af 472
d0a03d40 473 let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
8cc0d6af 474
8a8a4703 475 let result = client.post(&path, None).await?;
8cc0d6af 476
8a8a4703 477 record_repository(&repo);
d0a03d40 478
e68269fc 479 view_task_result(&mut client, result, &output_format).await?;
e5f7def4 480
e5f7def4 481 Ok(Value::Null)
8cc0d6af 482}
33d64b81 483
6d233161 484struct CatalogUploadResult {
f1d76ecf 485 catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
6d233161
FG
486 result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
487}
488
bf6e3217 489fn spawn_catalog_upload(
3bad3e6e 490 client: Arc<BackupWriter>,
3638341a 491 encrypt: bool,
6d233161 492) -> Result<CatalogUploadResult, Error> {
f1d99e3f 493 let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
fc5870be 494 let catalog_stream = pbs_tools::blocking::StdChannelStream(catalog_rx);
bf6e3217
DM
495 let catalog_chunk_size = 512*1024;
496 let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
497
f1d76ecf 498 let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)))?));
bf6e3217
DM
499
500 let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
501
e43b9175
FG
502 let upload_options = UploadOptions {
503 encrypt,
504 compress: true,
505 ..UploadOptions::default()
506 };
507
bf6e3217
DM
508 tokio::spawn(async move {
509 let catalog_upload_result = client
e43b9175 510 .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
bf6e3217
DM
511 .await;
512
513 if let Err(ref err) = catalog_upload_result {
514 eprintln!("catalog upload error - {}", err);
515 client.cancel();
516 }
517
518 let _ = catalog_result_tx.send(catalog_upload_result);
519 });
520
6d233161 521 Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
bf6e3217
DM
522}
523
a47a02ae
DM
524#[api(
525 input: {
526 properties: {
527 backupspec: {
528 type: Array,
529 description: "List of backup source specifications ([<label.ext>:<path>] ...)",
530 items: {
531 schema: BACKUP_SOURCE_SCHEMA,
532 }
533 },
534 repository: {
535 schema: REPO_URL_SCHEMA,
536 optional: true,
537 },
538 "include-dev": {
539 description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
540 optional: true,
541 items: {
542 type: String,
543 description: "Path to file.",
544 }
545 },
58fcbf5a
FE
546 "all-file-systems": {
547 type: Boolean,
548 description: "Include all mounted subdirectories.",
549 optional: true,
550 },
a47a02ae
DM
551 keyfile: {
552 schema: KEYFILE_SCHEMA,
553 optional: true,
554 },
0351f23b
WB
555 "keyfd": {
556 schema: KEYFD_SCHEMA,
557 optional: true,
558 },
c0a87c12
FG
559 "master-pubkey-file": {
560 schema: MASTER_PUBKEY_FILE_SCHEMA,
561 optional: true,
562 },
563 "master-pubkey-fd": {
564 schema: MASTER_PUBKEY_FD_SCHEMA,
565 optional: true,
566 },
24be37e3
WB
567 "crypt-mode": {
568 type: CryptMode,
96ee8577
WB
569 optional: true,
570 },
a47a02ae
DM
571 "skip-lost-and-found": {
572 type: Boolean,
573 description: "Skip lost+found directory.",
574 optional: true,
575 },
576 "backup-type": {
577 schema: BACKUP_TYPE_SCHEMA,
578 optional: true,
579 },
580 "backup-id": {
581 schema: BACKUP_ID_SCHEMA,
582 optional: true,
583 },
584 "backup-time": {
585 schema: BACKUP_TIME_SCHEMA,
586 optional: true,
587 },
588 "chunk-size": {
589 schema: CHUNK_SIZE_SCHEMA,
590 optional: true,
591 },
189996cf
CE
592 "exclude": {
593 type: Array,
594 description: "List of paths or patterns for matching files to exclude.",
595 optional: true,
596 items: {
597 type: String,
598 description: "Path or match pattern.",
599 }
600 },
6fc053ed
CE
601 "entries-max": {
602 type: Integer,
603 description: "Max number of entries to hold in memory.",
604 optional: true,
2b7f8dd5 605 default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
6fc053ed 606 },
e02c3d46
DM
607 "verbose": {
608 type: Boolean,
609 description: "Verbose output.",
610 optional: true,
611 },
a47a02ae
DM
612 }
613 }
614)]
615/// Create (host) backup.
616async fn create_backup(
6049b71f
DM
617 param: Value,
618 _info: &ApiMethod,
dd5495d6 619 _rpcenv: &mut dyn RpcEnvironment,
6049b71f 620) -> Result<Value, Error> {
ff5d3707 621
2665cef7 622 let repo = extract_repository_from_value(&param)?;
ae0be2dd 623
3c8c2827 624 let backupspec_list = json::required_array_param(&param, "backupspec")?;
a914a774 625
eed6db39
DM
626 let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
627
5b72c9b4
DM
628 let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
629
219ef0e6
DM
630 let verbose = param["verbose"].as_bool().unwrap_or(false);
631
ca5d0b61
DM
632 let backup_time_opt = param["backup-time"].as_i64();
633
36898ffc 634 let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
2d9d143a 635
247cdbce
DM
636 if let Some(size) = chunk_size_opt {
637 verify_chunk_size(size)?;
2d9d143a
DM
638 }
639
c6a7ea0a 640 let crypto = crypto_parameters(&param)?;
6d0983db 641
f69adc81 642 let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
fba30411 643
bbf9e7e9 644 let backup_type = param["backup-type"].as_str().unwrap_or("host");
ca5d0b61 645
2eeaacb9
DM
646 let include_dev = param["include-dev"].as_array();
647
c443f58b 648 let entries_max = param["entries-max"].as_u64()
2b7f8dd5 649 .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
6fc053ed 650
189996cf 651 let empty = Vec::new();
c443f58b
WB
652 let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
653
239e49f9 654 let mut pattern_list = Vec::with_capacity(exclude_args.len());
c443f58b
WB
655 for entry in exclude_args {
656 let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
239e49f9 657 pattern_list.push(
c443f58b
WB
658 MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
659 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
660 );
189996cf
CE
661 }
662
2eeaacb9
DM
663 let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
664
665 if let Some(include_dev) = include_dev {
666 if all_file_systems {
667 bail!("option 'all-file-systems' conflicts with option 'include-dev'");
668 }
669
670 let mut set = HashSet::new();
671 for path in include_dev {
672 let path = path.as_str().unwrap();
673 let stat = nix::sys::stat::stat(path)
674 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
675 set.insert(stat.st_dev);
676 }
677 devices = Some(set);
678 }
679
ae0be2dd 680 let mut upload_list = vec![];
f2b4b4b9 681 let mut target_set = HashSet::new();
a914a774 682
ae0be2dd 683 for backupspec in backupspec_list {
7cc3473a
DM
684 let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
685 let filename = &spec.config_string;
686 let target = &spec.archive_name;
bcd879cf 687
f2b4b4b9
SI
688 if target_set.contains(target) {
689 bail!("got target twice: '{}'", target);
690 }
691 target_set.insert(target.to_string());
692
eb1804c5
DM
693 use std::os::unix::fs::FileTypeExt;
694
3fa71727
CE
695 let metadata = std::fs::metadata(filename)
696 .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
eb1804c5 697 let file_type = metadata.file_type();
23bb8780 698
7cc3473a
DM
699 match spec.spec_type {
700 BackupSpecificationType::PXAR => {
ec8a9bb9
DM
701 if !file_type.is_dir() {
702 bail!("got unexpected file type (expected directory)");
703 }
7cc3473a 704 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
ec8a9bb9 705 }
7cc3473a 706 BackupSpecificationType::IMAGE => {
ec8a9bb9
DM
707 if !(file_type.is_file() || file_type.is_block_device()) {
708 bail!("got unexpected file type (expected file or block device)");
709 }
eb1804c5 710
e18a6c9e 711 let size = image_size(&PathBuf::from(filename))?;
23bb8780 712
ec8a9bb9 713 if size == 0 { bail!("got zero-sized file '{}'", filename); }
ae0be2dd 714
7cc3473a 715 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
ec8a9bb9 716 }
7cc3473a 717 BackupSpecificationType::CONFIG => {
ec8a9bb9
DM
718 if !file_type.is_file() {
719 bail!("got unexpected file type (expected regular file)");
720 }
7cc3473a 721 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 722 }
7cc3473a 723 BackupSpecificationType::LOGFILE => {
79679c2d
DM
724 if !file_type.is_file() {
725 bail!("got unexpected file type (expected regular file)");
726 }
7cc3473a 727 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 728 }
ae0be2dd
DM
729 }
730 }
731
22a9189e 732 let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
ae0be2dd 733
f3fde36b 734 let client = connect(&repo)?;
d0a03d40
DM
735 record_repository(&repo);
736
6a7be83e 737 println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
ca5d0b61 738
f69adc81 739 println!("Client name: {}", proxmox::tools::nodename());
ca5d0b61 740
6a7be83e 741 let start_time = std::time::Instant::now();
ca5d0b61 742
6a7be83e 743 println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
51144821 744
c6a7ea0a 745 let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
bb823140 746 None => (None, None),
2f26b866
FG
747 Some(key_with_source) => {
748 println!(
749 "{}",
750 format_key_source(&key_with_source.source, "encryption")
751 );
752
753 let (key, created, fingerprint) =
ff8945fd 754 decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
6f2626ae 755 println!("Encryption key fingerprint: {}", fingerprint);
bb823140 756
44288184 757 let crypt_config = CryptConfig::new(key)?;
bb823140 758
c0a87c12 759 match crypto.master_pubkey {
2f26b866
FG
760 Some(pem_with_source) => {
761 println!("{}", format_key_source(&pem_with_source.source, "master"));
762
763 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
82a103c8 764
1c86893d 765 let mut key_config = KeyConfig::without_password(key)?;
82a103c8 766 key_config.created = created; // keep original value
82a103c8 767
8acfd15d 768 let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
6f2626ae 769
05389a01 770 (Some(Arc::new(crypt_config)), Some(enc_key))
2f26b866 771 },
05389a01 772 _ => (Some(Arc::new(crypt_config)), None),
bb823140 773 }
6d0983db
DM
774 }
775 };
f98ac774 776
8a8a4703
DM
777 let client = BackupWriter::start(
778 client,
b957aa81 779 crypt_config.clone(),
8a8a4703
DM
780 repo.store(),
781 backup_type,
782 &backup_id,
783 backup_time,
784 verbose,
61d7b501 785 false
8a8a4703
DM
786 ).await?;
787
8b7f8d3f
FG
788 let download_previous_manifest = match client.previous_backup_time().await {
789 Ok(Some(backup_time)) => {
790 println!(
791 "Downloading previous manifest ({})",
792 strftime_local("%c", backup_time)?
793 );
794 true
795 }
796 Ok(None) => {
797 println!("No previous manifest available.");
798 false
799 }
800 Err(_) => {
801 // Fallback for outdated server, TODO remove/bubble up with 2.0
802 true
803 }
804 };
805
806 let previous_manifest = if download_previous_manifest {
807 match client.download_previous_manifest().await {
808 Ok(previous_manifest) => {
809 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
810 Ok(()) => Some(Arc::new(previous_manifest)),
811 Err(err) => {
812 println!("Couldn't re-use previous manifest - {}", err);
813 None
814 }
815 }
23f9503a 816 }
8b7f8d3f
FG
817 Err(err) => {
818 println!("Couldn't download previous manifest - {}", err);
819 None
820 }
821 }
822 } else {
823 None
b957aa81
DM
824 };
825
6a7be83e 826 let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
8a8a4703
DM
827 let mut manifest = BackupManifest::new(snapshot);
828
5d85847f 829 let mut catalog = None;
6d233161 830 let mut catalog_result_rx = None;
8a8a4703
DM
831
832 for (backup_type, filename, target, size) in upload_list {
833 match backup_type {
7cc3473a 834 BackupSpecificationType::CONFIG => {
e43b9175
FG
835 let upload_options = UploadOptions {
836 compress: true,
c6a7ea0a 837 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
838 ..UploadOptions::default()
839 };
840
5b32820e 841 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 842 let stats = client
e43b9175 843 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 844 .await?;
c6a7ea0a 845 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 846 }
7cc3473a 847 BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
e43b9175
FG
848 let upload_options = UploadOptions {
849 compress: true,
c6a7ea0a 850 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
851 ..UploadOptions::default()
852 };
853
5b32820e 854 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 855 let stats = client
e43b9175 856 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 857 .await?;
c6a7ea0a 858 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 859 }
7cc3473a 860 BackupSpecificationType::PXAR => {
5d85847f
DC
861 // start catalog upload on first use
862 if catalog.is_none() {
c6a7ea0a 863 let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
6d233161
FG
864 catalog = Some(catalog_upload_res.catalog_writer);
865 catalog_result_rx = Some(catalog_upload_res.result);
5d85847f
DC
866 }
867 let catalog = catalog.as_ref().unwrap();
868
5b32820e 869 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
8a8a4703 870 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
77486a60 871
2b7f8dd5 872 let pxar_options = pbs_client::pxar::PxarCreateOptions {
77486a60
FG
873 device_set: devices.clone(),
874 patterns: pattern_list.clone(),
875 entries_max: entries_max as usize,
876 skip_lost_and_found,
877 verbose,
878 };
879
e43b9175
FG
880 let upload_options = UploadOptions {
881 previous_manifest: previous_manifest.clone(),
882 compress: true,
c6a7ea0a 883 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
884 ..UploadOptions::default()
885 };
886
8a8a4703
DM
887 let stats = backup_directory(
888 &client,
889 &filename,
890 &target,
891 chunk_size_opt,
8a8a4703 892 catalog.clone(),
77486a60 893 pxar_options,
e43b9175 894 upload_options,
8a8a4703 895 ).await?;
c6a7ea0a 896 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703
DM
897 catalog.lock().unwrap().end_directory()?;
898 }
7cc3473a 899 BackupSpecificationType::IMAGE => {
8a8a4703 900 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
e43b9175
FG
901
902 let upload_options = UploadOptions {
903 previous_manifest: previous_manifest.clone(),
904 fixed_size: Some(size),
905 compress: true,
c6a7ea0a 906 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
907 };
908
8a8a4703
DM
909 let stats = backup_image(
910 &client,
e43b9175 911 &filename,
8a8a4703 912 &target,
8a8a4703 913 chunk_size_opt,
e43b9175 914 upload_options,
8a8a4703 915 ).await?;
c6a7ea0a 916 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
6af905c1
DM
917 }
918 }
8a8a4703 919 }
4818c8b6 920
8a8a4703 921 // finalize and upload catalog
5d85847f 922 if let Some(catalog) = catalog {
8a8a4703
DM
923 let mutex = Arc::try_unwrap(catalog)
924 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
925 let mut catalog = mutex.into_inner().unwrap();
bf6e3217 926
8a8a4703 927 catalog.finish()?;
2761d6a4 928
8a8a4703 929 drop(catalog); // close upload stream
2761d6a4 930
6d233161 931 if let Some(catalog_result_rx) = catalog_result_rx {
5d85847f 932 let stats = catalog_result_rx.await??;
c6a7ea0a 933 manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
5d85847f 934 }
8a8a4703 935 }
2761d6a4 936
8a8a4703 937 if let Some(rsa_encrypted_key) = rsa_encrypted_key {
9990af30 938 let target = ENCRYPTED_KEY_BLOB_NAME;
8a8a4703 939 println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
e43b9175 940 let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
8a8a4703 941 let stats = client
e43b9175 942 .upload_blob_from_data(rsa_encrypted_key, target, options)
8a8a4703 943 .await?;
c6a7ea0a 944 manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
8a8a4703 945
8a8a4703 946 }
8a8a4703 947 // create manifest (index.json)
3638341a 948 // manifests are never encrypted, but include a signature
dfa517ad 949 let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
b53f6379 950 .map_err(|err| format_err!("unable to format manifest - {}", err))?;
3638341a 951
b53f6379 952
9688f6de 953 if verbose { println!("Upload index.json to '{}'", repo) };
e43b9175 954 let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
8a8a4703 955 client
e43b9175 956 .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
8a8a4703 957 .await?;
2c3891d1 958
8a8a4703 959 client.finish().await?;
c4ff3dce 960
6a7be83e
DM
961 let end_time = std::time::Instant::now();
962 let elapsed = end_time.duration_since(start_time);
963 println!("Duration: {:.2}s", elapsed.as_secs_f64());
3ec3ec3f 964
6a7be83e 965 println!("End Time: {}", strftime_local("%c", epoch_i64())?);
3d5c11e5 966
8a8a4703 967 Ok(Value::Null)
f98ea63d
DM
968}
969
8e6e18b7 970async fn dump_image<W: Write>(
88892ea8
DM
971 client: Arc<BackupReader>,
972 crypt_config: Option<Arc<CryptConfig>>,
14f6c9cb 973 crypt_mode: CryptMode,
88892ea8
DM
974 index: FixedIndexReader,
975 mut writer: W,
fd04ca7a 976 verbose: bool,
88892ea8
DM
977) -> Result<(), Error> {
978
979 let most_used = index.find_most_used_chunks(8);
980
14f6c9cb 981 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
88892ea8
DM
982
983 // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
984 // and thus slows down reading. Instead, directly use RemoteChunkReader
fd04ca7a
DM
985 let mut per = 0;
986 let mut bytes = 0;
987 let start_time = std::time::Instant::now();
988
88892ea8
DM
989 for pos in 0..index.index_count() {
990 let digest = index.index_digest(pos).unwrap();
8e6e18b7 991 let raw_data = chunk_reader.read_chunk(&digest).await?;
88892ea8 992 writer.write_all(&raw_data)?;
fd04ca7a
DM
993 bytes += raw_data.len();
994 if verbose {
995 let next_per = ((pos+1)*100)/index.index_count();
996 if per != next_per {
997 eprintln!("progress {}% (read {} bytes, duration {} sec)",
998 next_per, bytes, start_time.elapsed().as_secs());
999 per = next_per;
1000 }
1001 }
88892ea8
DM
1002 }
1003
fd04ca7a
DM
1004 let end_time = std::time::Instant::now();
1005 let elapsed = end_time.duration_since(start_time);
1006 eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1007 bytes,
1008 elapsed.as_secs_f64(),
1009 bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1010 );
1011
1012
88892ea8
DM
1013 Ok(())
1014}
1015
dc155e9b 1016fn parse_archive_type(name: &str) -> (String, ArchiveType) {
2d32fe2c
TL
1017 if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1018 (name.into(), archive_type(name).unwrap())
1019 } else if name.ends_with(".pxar") {
dc155e9b
TL
1020 (format!("{}.didx", name), ArchiveType::DynamicIndex)
1021 } else if name.ends_with(".img") {
1022 (format!("{}.fidx", name), ArchiveType::FixedIndex)
1023 } else {
1024 (format!("{}.blob", name), ArchiveType::Blob)
1025 }
1026}
1027
a47a02ae
DM
1028#[api(
1029 input: {
1030 properties: {
1031 repository: {
1032 schema: REPO_URL_SCHEMA,
1033 optional: true,
1034 },
1035 snapshot: {
1036 type: String,
1037 description: "Group/Snapshot path.",
1038 },
1039 "archive-name": {
1040 description: "Backup archive name.",
1041 type: String,
1042 },
1043 target: {
1044 type: String,
90c815bf 1045 description: r###"Target directory path. Use '-' to write to standard output.
8a8a4703 1046
d1d74c43 1047We do not extract '.pxar' archives when writing to standard output.
8a8a4703 1048
a47a02ae
DM
1049"###
1050 },
1051 "allow-existing-dirs": {
1052 type: Boolean,
1053 description: "Do not fail if directories already exists.",
1054 optional: true,
1055 },
1056 keyfile: {
1057 schema: KEYFILE_SCHEMA,
1058 optional: true,
1059 },
0351f23b
WB
1060 "keyfd": {
1061 schema: KEYFD_SCHEMA,
1062 optional: true,
1063 },
24be37e3
WB
1064 "crypt-mode": {
1065 type: CryptMode,
96ee8577
WB
1066 optional: true,
1067 },
a47a02ae
DM
1068 }
1069 }
1070)]
1071/// Restore backup repository.
1072async fn restore(param: Value) -> Result<Value, Error> {
2665cef7 1073 let repo = extract_repository_from_value(&param)?;
9f912493 1074
86eda3eb
DM
1075 let verbose = param["verbose"].as_bool().unwrap_or(false);
1076
46d5aa0a
DM
1077 let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1078
3c8c2827 1079 let archive_name = json::required_string_param(&param, "archive-name")?;
d5c34d98 1080
f3fde36b 1081 let client = connect(&repo)?;
d0a03d40 1082
d0a03d40 1083 record_repository(&repo);
d5c34d98 1084
3c8c2827 1085 let path = json::required_string_param(&param, "snapshot")?;
9f912493 1086
86eda3eb 1087 let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
d6d3b353 1088 let group: BackupGroup = path.parse()?;
27c9affb 1089 api_datastore_latest_snapshot(&client, repo.store(), group).await?
d5c34d98 1090 } else {
a67f7d0a 1091 let snapshot: BackupDir = path.parse()?;
86eda3eb
DM
1092 (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1093 };
9f912493 1094
3c8c2827 1095 let target = json::required_string_param(&param, "target")?;
bf125261 1096 let target = if target == "-" { None } else { Some(target) };
2ae7d196 1097
c6a7ea0a 1098 let crypto = crypto_parameters(&param)?;
2ae7d196 1099
c6a7ea0a 1100 let crypt_config = match crypto.enc_key {
86eda3eb 1101 None => None,
2f26b866
FG
1102 Some(ref key) => {
1103 let (key, _, _) =
ff8945fd 1104 decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
2f26b866
FG
1105 eprintln!("{}", format_key_source(&key.source, "encryption"));
1106 err
1107 })?;
86eda3eb
DM
1108 Some(Arc::new(CryptConfig::new(key)?))
1109 }
1110 };
d5c34d98 1111
296c50ba
DM
1112 let client = BackupReader::start(
1113 client,
1114 crypt_config.clone(),
1115 repo.store(),
1116 &backup_type,
1117 &backup_id,
1118 backup_time,
1119 true,
1120 ).await?;
86eda3eb 1121
48fbbfeb
FG
1122 let (archive_name, archive_type) = parse_archive_type(archive_name);
1123
2107a5ae 1124 let (manifest, backup_index_data) = client.download_manifest().await?;
02fcf372 1125
48fbbfeb
FG
1126 if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1127 eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1128 } else {
2f26b866
FG
1129 if manifest.signature.is_some() {
1130 if let Some(key) = &crypto.enc_key {
1131 eprintln!("{}", format_key_source(&key.source, "encryption"));
1132 }
1133 if let Some(config) = &crypt_config {
1134 eprintln!("Fingerprint: {}", config.fingerprint());
1135 }
1136 }
48fbbfeb
FG
1137 manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1138 }
dc155e9b
TL
1139
1140 if archive_name == MANIFEST_BLOB_NAME {
02fcf372 1141 if let Some(target) = target {
2107a5ae 1142 replace_file(target, &backup_index_data, CreateOptions::new())?;
02fcf372
DM
1143 } else {
1144 let stdout = std::io::stdout();
1145 let mut writer = stdout.lock();
2107a5ae 1146 writer.write_all(&backup_index_data)
02fcf372
DM
1147 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1148 }
1149
14f6c9cb
FG
1150 return Ok(Value::Null);
1151 }
1152
1153 let file_info = manifest.lookup_file_info(&archive_name)?;
1154
1155 if archive_type == ArchiveType::Blob {
d2267b11 1156
dc155e9b 1157 let mut reader = client.download_blob(&manifest, &archive_name).await?;
f8100e96 1158
bf125261 1159 if let Some(target) = target {
0d986280
DM
1160 let mut writer = std::fs::OpenOptions::new()
1161 .write(true)
1162 .create(true)
1163 .create_new(true)
1164 .open(target)
1165 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1166 std::io::copy(&mut reader, &mut writer)?;
bf125261
DM
1167 } else {
1168 let stdout = std::io::stdout();
1169 let mut writer = stdout.lock();
0d986280 1170 std::io::copy(&mut reader, &mut writer)
bf125261
DM
1171 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1172 }
f8100e96 1173
dc155e9b 1174 } else if archive_type == ArchiveType::DynamicIndex {
86eda3eb 1175
dc155e9b 1176 let index = client.download_dynamic_index(&manifest, &archive_name).await?;
df65bd3d 1177
f4bf7dfc
DM
1178 let most_used = index.find_most_used_chunks(8);
1179
14f6c9cb 1180 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
f4bf7dfc 1181
afb4cd28 1182 let mut reader = BufferedDynamicReader::new(index, chunk_reader);
86eda3eb 1183
2b7f8dd5 1184 let options = pbs_client::pxar::PxarExtractOptions {
72064fd0
FG
1185 match_list: &[],
1186 extract_match_default: true,
1187 allow_existing_dirs,
1188 on_error: None,
1189 };
1190
bf125261 1191 if let Some(target) = target {
2b7f8dd5 1192 pbs_client::pxar::extract_archive(
c443f58b
WB
1193 pxar::decoder::Decoder::from_std(reader)?,
1194 Path::new(target),
2b7f8dd5 1195 pbs_client::pxar::Flags::DEFAULT,
c443f58b
WB
1196 |path| {
1197 if verbose {
1198 println!("{:?}", path);
1199 }
1200 },
72064fd0 1201 options,
c443f58b
WB
1202 )
1203 .map_err(|err| format_err!("error extracting archive - {}", err))?;
bf125261 1204 } else {
88892ea8
DM
1205 let mut writer = std::fs::OpenOptions::new()
1206 .write(true)
1207 .open("/dev/stdout")
1208 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
afb4cd28 1209
bf125261
DM
1210 std::io::copy(&mut reader, &mut writer)
1211 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1212 }
dc155e9b 1213 } else if archive_type == ArchiveType::FixedIndex {
afb4cd28 1214
dc155e9b 1215 let index = client.download_fixed_index(&manifest, &archive_name).await?;
df65bd3d 1216
88892ea8
DM
1217 let mut writer = if let Some(target) = target {
1218 std::fs::OpenOptions::new()
bf125261
DM
1219 .write(true)
1220 .create(true)
1221 .create_new(true)
1222 .open(target)
88892ea8 1223 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
bf125261 1224 } else {
88892ea8
DM
1225 std::fs::OpenOptions::new()
1226 .write(true)
1227 .open("/dev/stdout")
1228 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1229 };
afb4cd28 1230
14f6c9cb 1231 dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
3031e44c 1232 }
fef44d4f
DM
1233
1234 Ok(Value::Null)
45db6f89
DM
1235}
1236
e0665a64
DC
1237#[api(
1238 input: {
1239 properties: {
1240 "dry-run": {
1241 type: bool,
1242 optional: true,
1243 description: "Just show what prune would do, but do not delete anything.",
1244 },
1245 group: {
1246 type: String,
1247 description: "Backup group",
1248 },
1249 "prune-options": {
1250 type: PruneOptions,
1251 flatten: true,
1252 },
1253 "output-format": {
1254 schema: OUTPUT_FORMAT,
1255 optional: true,
1256 },
1257 quiet: {
1258 type: bool,
1259 optional: true,
1260 default: false,
1261 description: "Minimal output - only show removals.",
1262 },
1263 repository: {
1264 schema: REPO_URL_SCHEMA,
1265 optional: true,
1266 },
1267 },
1268 },
1269)]
1270/// Prune a backup repository.
1271async fn prune(
1272 dry_run: Option<bool>,
1273 group: String,
1274 prune_options: PruneOptions,
1275 quiet: bool,
1276 mut param: Value
1277) -> Result<Value, Error> {
2665cef7 1278 let repo = extract_repository_from_value(&param)?;
83b7db02 1279
f3fde36b 1280 let mut client = connect(&repo)?;
83b7db02 1281
d0a03d40 1282 let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
83b7db02 1283
d6d3b353 1284 let group: BackupGroup = group.parse()?;
c2043614 1285
671c6a96 1286 let output_format = extract_output_format(&mut param);
9fdc3ef4 1287
e0665a64
DC
1288 let mut api_param = serde_json::to_value(prune_options)?;
1289 if let Some(dry_run) = dry_run {
1290 api_param["dry-run"] = dry_run.into();
1291 }
1292 api_param["backup-type"] = group.backup_type().into();
1293 api_param["backup-id"] = group.backup_id().into();
83b7db02 1294
e0665a64 1295 let mut result = client.post(&path, Some(api_param)).await?;
74fa81b8 1296
87c42375 1297 record_repository(&repo);
3b03abfe 1298
db1e061d
DM
1299 let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1300 let item: PruneListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 1301 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
db1e061d
DM
1302 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1303 };
1304
c48aa39f
DM
1305 let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1306 Ok(match v.as_bool() {
1307 Some(true) => "keep",
1308 Some(false) => "remove",
1309 None => "unknown",
1310 }.to_string())
1311 };
1312
db1e061d
DM
1313 let options = default_table_format_options()
1314 .sortby("backup-type", false)
1315 .sortby("backup-id", false)
1316 .sortby("backup-time", false)
1317 .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
770a36e5 1318 .column(ColumnConfig::new("backup-time").renderer(pbs_tools::format::render_epoch).header("date"))
c48aa39f 1319 .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
db1e061d
DM
1320 ;
1321
e351ac78 1322 let return_type = &pbs_api_types::ADMIN_DATASTORE_PRUNE_RETURN_TYPE;
db1e061d
DM
1323
1324 let mut data = result["data"].take();
1325
c48aa39f
DM
1326 if quiet {
1327 let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1328 item["keep"].as_bool() == Some(false)
a375df6f 1329 }).cloned().collect();
c48aa39f
DM
1330 data = list.into();
1331 }
1332
b2362a12 1333 format_and_print_result_full(&mut data, return_type, &output_format, &options);
d0a03d40 1334
43a406fd 1335 Ok(Value::Null)
83b7db02
DM
1336}
1337
a47a02ae
DM
1338#[api(
1339 input: {
1340 properties: {
1341 repository: {
1342 schema: REPO_URL_SCHEMA,
1343 optional: true,
1344 },
1345 "output-format": {
1346 schema: OUTPUT_FORMAT,
1347 optional: true,
1348 },
1349 }
f9beae9c
TL
1350 },
1351 returns: {
1352 type: StorageStatus,
1353 },
a47a02ae
DM
1354)]
1355/// Get repository status.
1356async fn status(param: Value) -> Result<Value, Error> {
34a816cc
DM
1357
1358 let repo = extract_repository_from_value(&param)?;
1359
c2043614 1360 let output_format = get_output_format(&param);
34a816cc 1361
f3fde36b 1362 let client = connect(&repo)?;
34a816cc
DM
1363
1364 let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1365
1dc117bb 1366 let mut result = client.get(&path, None).await?;
14e08625 1367 let mut data = result["data"].take();
34a816cc
DM
1368
1369 record_repository(&repo);
1370
390c5bdd
DM
1371 let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1372 let v = v.as_u64().unwrap();
1373 let total = record["total"].as_u64().unwrap();
1374 let roundup = total/200;
1375 let per = ((v+roundup)*100)/total;
e23f5863
DM
1376 let info = format!(" ({} %)", per);
1377 Ok(format!("{} {:>8}", v, info))
390c5bdd 1378 };
1dc117bb 1379
c2043614 1380 let options = default_table_format_options()
be2425ff 1381 .noheader(true)
e23f5863 1382 .column(ColumnConfig::new("total").renderer(render_total_percentage))
390c5bdd
DM
1383 .column(ColumnConfig::new("used").renderer(render_total_percentage))
1384 .column(ColumnConfig::new("avail").renderer(render_total_percentage));
34a816cc 1385
b2362a12 1386 let return_type = &API_METHOD_STATUS.returns;
390c5bdd 1387
b2362a12 1388 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc
DM
1389
1390 Ok(Value::Null)
1391}
1392
c443f58b
WB
1393/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1394/// async use!
1395///
1396/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1397/// so that we can properly access it from multiple threads simultaneously while not issuing
1398/// duplicate simultaneous reads over http.
43abba4b 1399pub struct BufferedDynamicReadAt {
c443f58b
WB
1400 inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1401}
1402
1403impl BufferedDynamicReadAt {
1404 fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
1405 Self {
1406 inner: Mutex::new(inner),
1407 }
1408 }
1409}
1410
a6f87283
WB
1411impl ReadAt for BufferedDynamicReadAt {
1412 fn start_read_at<'a>(
1413 self: Pin<&'a Self>,
c443f58b 1414 _cx: &mut Context,
a6f87283 1415 buf: &'a mut [u8],
c443f58b 1416 offset: u64,
a6f87283 1417 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
a6f87283 1418 MaybeReady::Ready(tokio::task::block_in_place(move || {
c443f58b
WB
1419 let mut reader = self.inner.lock().unwrap();
1420 reader.seek(SeekFrom::Start(offset))?;
a6f87283
WB
1421 Ok(reader.read(buf)?)
1422 }))
1423 }
1424
1425 fn poll_complete<'a>(
1426 self: Pin<&'a Self>,
1427 _op: ReadAtOperation<'a>,
1428 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
bbc71e3b 1429 panic!("BufferedDynamicReadAt::start_read_at returned Pending");
c443f58b
WB
1430 }
1431}
1432
f2401311 1433fn main() {
33d64b81 1434
255f378a 1435 let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
49fddd98 1436 .arg_param(&["backupspec"])
d0a03d40 1437 .completion_cb("repository", complete_repository)
49811347 1438 .completion_cb("backupspec", complete_backup_source)
2b7f8dd5
WB
1439 .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
1440 .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
49811347 1441 .completion_cb("chunk-size", complete_chunk_size);
f8838fe9 1442
caea8d61
DM
1443 let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
1444 .completion_cb("repository", complete_repository)
2b7f8dd5 1445 .completion_cb("keyfile", pbs_tools::fs::complete_file_name);
caea8d61 1446
255f378a 1447 let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
d0a03d40 1448 .completion_cb("repository", complete_repository);
41c039e1 1449
255f378a 1450 let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
d0a03d40 1451 .completion_cb("repository", complete_repository);
8cc0d6af 1452
255f378a 1453 let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
49fddd98 1454 .arg_param(&["snapshot", "archive-name", "target"])
b2388518 1455 .completion_cb("repository", complete_repository)
08dc340a
DM
1456 .completion_cb("snapshot", complete_group_or_snapshot)
1457 .completion_cb("archive-name", complete_archive_name)
2b7f8dd5 1458 .completion_cb("target", pbs_tools::fs::complete_file_name);
9f912493 1459
255f378a 1460 let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
49fddd98 1461 .arg_param(&["group"])
9fdc3ef4 1462 .completion_cb("group", complete_backup_group)
d0a03d40 1463 .completion_cb("repository", complete_repository);
9f912493 1464
255f378a 1465 let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
34a816cc
DM
1466 .completion_cb("repository", complete_repository);
1467
255f378a 1468 let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
e240d8be
DM
1469 .completion_cb("repository", complete_repository);
1470
255f378a 1471 let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
e240d8be 1472 .completion_cb("repository", complete_repository);
32efac1c 1473
e39974af
TL
1474 let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
1475 .completion_cb("repository", complete_repository);
1476
344add38
DW
1477 let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
1478 .arg_param(&["group", "new-owner"])
1479 .completion_cb("group", complete_backup_group)
0224c3c2 1480 .completion_cb("new-owner", complete_auth_id)
344add38
DW
1481 .completion_cb("repository", complete_repository);
1482
41c039e1 1483 let cmd_def = CliCommandMap::new()
48ef3c33 1484 .insert("backup", backup_cmd_def)
48ef3c33
DM
1485 .insert("garbage-collect", garbage_collect_cmd_def)
1486 .insert("list", list_cmd_def)
1487 .insert("login", login_cmd_def)
1488 .insert("logout", logout_cmd_def)
1489 .insert("prune", prune_cmd_def)
1490 .insert("restore", restore_cmd_def)
a65e3e4b 1491 .insert("snapshot", snapshot_mgtm_cli())
48ef3c33 1492 .insert("status", status_cmd_def)
9696f519 1493 .insert("key", key::cli())
43abba4b 1494 .insert("mount", mount_cmd_def())
45f9b32e
SR
1495 .insert("map", map_cmd_def())
1496 .insert("unmap", unmap_cmd_def())
5830c205 1497 .insert("catalog", catalog_mgmt_cli())
caea8d61 1498 .insert("task", task_mgmt_cli())
e39974af 1499 .insert("version", version_cmd_def)
344add38 1500 .insert("benchmark", benchmark_cmd_def)
731eeef2
DM
1501 .insert("change-owner", change_owner_cmd_def)
1502
61205f00 1503 .alias(&["files"], &["snapshot", "files"])
edebd523 1504 .alias(&["forget"], &["snapshot", "forget"])
0c9209b0 1505 .alias(&["upload-log"], &["snapshot", "upload-log"])
731eeef2
DM
1506 .alias(&["snapshots"], &["snapshot", "list"])
1507 ;
48ef3c33 1508
7b22acd0
DM
1509 let rpcenv = CliEnvironment::new();
1510 run_cli_command(cmd_def, rpcenv, Some(|future| {
d420962f 1511 pbs_runtime::main(future)
d08bc483 1512 }));
ff5d3707 1513}