]> git.proxmox.com Git - proxmox-backup.git/blame - src/bin/proxmox-backup-client.rs
client: fix panic message
[proxmox-backup.git] / src / bin / proxmox-backup-client.rs
CommitLineData
f1a83e97 1use std::collections::HashSet;
0351f23b 2use std::io::{self, Read, Write, Seek, SeekFrom};
c443f58b
WB
3use std::path::{Path, PathBuf};
4use std::pin::Pin;
5use std::sync::{Arc, Mutex};
a6f87283 6use std::task::Context;
c443f58b
WB
7
8use anyhow::{bail, format_err, Error};
c443f58b 9use futures::stream::{StreamExt, TryStreamExt};
c443f58b 10use serde_json::{json, Value};
c443f58b 11use tokio::sync::mpsc;
7c667013 12use tokio_stream::wrappers::ReceiverStream;
c443f58b 13use xdg::BaseDirectories;
2761d6a4 14
c443f58b 15use pathpatterns::{MatchEntry, MatchType, PatternFlag};
6a7be83e
DM
16use proxmox::{
17 tools::{
18 time::{strftime_local, epoch_i64},
ff8945fd 19 fs::{file_get_json, replace_file, CreateOptions, image_size},
6a7be83e
DM
20 },
21 api::{
22 api,
6a7be83e
DM
23 ApiMethod,
24 RpcEnvironment,
6a7be83e
DM
25 cli::*,
26 },
27};
a6f87283 28use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
ff5d3707 29
51ec8a3c
WB
30use pbs_api_types::{
31 BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem,
32 PruneListItem, SnapshotListItem, StorageStatus,
33};
2b7f8dd5
WB
34use pbs_client::{
35 BACKUP_SOURCE_SCHEMA,
36 BackupReader,
37 BackupRepository,
38 BackupSpecificationType,
39 BackupStats,
40 BackupWriter,
38629c39
WB
41 ChunkStream,
42 FixedChunkStream,
2b7f8dd5
WB
43 HttpClient,
44 PxarBackupStream,
45 RemoteChunkReader,
46 UploadOptions,
47 delete_ticket_info,
48 parse_backup_specification,
49 view_task_result,
50};
51use pbs_client::catalog_shell::Shell;
52use pbs_client::tools::{
53 complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
54 complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
55 complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
56 extract_repository_from_value,
57 key_source::{
58 crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
59 KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
60 },
61 CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
62};
51ec8a3c 63use pbs_datastore::{CATALOG_NAME, CryptConfig, KeyConfig, decrypt_key, rsa_encrypt_key_config};
2b7f8dd5 64use pbs_datastore::backup_info::{BackupDir, BackupGroup};
51ec8a3c
WB
65use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter};
66use pbs_datastore::chunk_store::verify_chunk_size;
2b7f8dd5
WB
67use pbs_datastore::dynamic_index::DynamicIndexReader;
68use pbs_datastore::fixed_index::FixedIndexReader;
69use pbs_datastore::index::IndexFile;
51ec8a3c
WB
70use pbs_datastore::manifest::{
71 ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type,
72};
2b7f8dd5 73use pbs_datastore::read_chunk::AsyncReadChunk;
51ec8a3c 74use pbs_datastore::prune::PruneOptions;
4805edc4
WB
75use pbs_tools::sync::StdChannelWriter;
76use pbs_tools::tokio::TokioWriterAdapter;
3c8c2827 77use pbs_tools::json;
f323e906 78
4d16badf 79use proxmox_backup::backup::{
4d16badf 80 BufferedDynamicReader,
4d16badf 81};
ae0be2dd 82
caea8d61
DM
83mod proxmox_backup_client;
84use proxmox_backup_client::*;
85
d0a03d40
DM
86fn record_repository(repo: &BackupRepository) {
87
88 let base = match BaseDirectories::with_prefix("proxmox-backup") {
89 Ok(v) => v,
90 _ => return,
91 };
92
93 // usually $HOME/.cache/proxmox-backup/repo-list
94 let path = match base.place_cache_file("repo-list") {
95 Ok(v) => v,
96 _ => return,
97 };
98
11377a47 99 let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
d0a03d40
DM
100
101 let repo = repo.to_string();
102
103 data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
104
105 let mut map = serde_json::map::Map::new();
106
107 loop {
108 let mut max_used = 0;
109 let mut max_repo = None;
110 for (repo, count) in data.as_object().unwrap() {
111 if map.contains_key(repo) { continue; }
112 if let Some(count) = count.as_i64() {
113 if count > max_used {
114 max_used = count;
115 max_repo = Some(repo);
116 }
117 }
118 }
119 if let Some(repo) = max_repo {
120 map.insert(repo.to_owned(), json!(max_used));
121 } else {
122 break;
123 }
124 if map.len() > 10 { // store max. 10 repos
125 break;
126 }
127 }
128
129 let new_data = json!(map);
130
feaa1ad3 131 let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
d0a03d40
DM
132}
133
42af4b8f
DM
134async fn api_datastore_list_snapshots(
135 client: &HttpClient,
136 store: &str,
137 group: Option<BackupGroup>,
f24fc116 138) -> Result<Value, Error> {
42af4b8f
DM
139
140 let path = format!("api2/json/admin/datastore/{}/snapshots", store);
141
142 let mut args = json!({});
143 if let Some(group) = group {
144 args["backup-type"] = group.backup_type().into();
145 args["backup-id"] = group.backup_id().into();
146 }
147
148 let mut result = client.get(&path, Some(args)).await?;
149
f24fc116 150 Ok(result["data"].take())
42af4b8f
DM
151}
152
43abba4b 153pub async fn api_datastore_latest_snapshot(
27c9affb
DM
154 client: &HttpClient,
155 store: &str,
156 group: BackupGroup,
6a7be83e 157) -> Result<(String, String, i64), Error> {
27c9affb 158
f24fc116
DM
159 let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
160 let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
27c9affb
DM
161
162 if list.is_empty() {
163 bail!("backup group {:?} does not contain any snapshots.", group.group_path());
164 }
165
166 list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
167
6a7be83e 168 let backup_time = list[0].backup_time;
27c9affb
DM
169
170 Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
171}
172
e9722f8b 173async fn backup_directory<P: AsRef<Path>>(
cf9271e2 174 client: &BackupWriter,
17d6979a 175 dir_path: P,
247cdbce 176 archive_name: &str,
36898ffc 177 chunk_size: Option<usize>,
f1d76ecf 178 catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
2b7f8dd5 179 pxar_create_options: pbs_client::pxar::PxarCreateOptions,
e43b9175 180 upload_options: UploadOptions,
2c3891d1 181) -> Result<BackupStats, Error> {
33d64b81 182
6fc053ed
CE
183 let pxar_stream = PxarBackupStream::open(
184 dir_path.as_ref(),
6fc053ed 185 catalog,
77486a60 186 pxar_create_options,
6fc053ed 187 )?;
e9722f8b 188 let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
ff3d3100 189
0bfcea6a 190 let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
5e7a09be 191
7c667013 192 let stream = ReceiverStream::new(rx)
e9722f8b 193 .map_err(Error::from);
17d6979a 194
c4ff3dce 195 // spawn chunker inside a separate task so that it can run parallel
e9722f8b 196 tokio::spawn(async move {
db0cb9ce
WB
197 while let Some(v) = chunk_stream.next().await {
198 let _ = tx.send(v).await;
199 }
e9722f8b 200 });
17d6979a 201
e43b9175
FG
202 if upload_options.fixed_size.is_some() {
203 bail!("cannot backup directory with fixed chunk size!");
204 }
205
e9722f8b 206 let stats = client
e43b9175 207 .upload_stream(archive_name, stream, upload_options)
e9722f8b 208 .await?;
bcd879cf 209
2c3891d1 210 Ok(stats)
bcd879cf
DM
211}
212
e9722f8b 213async fn backup_image<P: AsRef<Path>>(
cf9271e2 214 client: &BackupWriter,
6af905c1
DM
215 image_path: P,
216 archive_name: &str,
36898ffc 217 chunk_size: Option<usize>,
e43b9175 218 upload_options: UploadOptions,
2c3891d1 219) -> Result<BackupStats, Error> {
6af905c1 220
6af905c1
DM
221 let path = image_path.as_ref().to_owned();
222
e9722f8b 223 let file = tokio::fs::File::open(path).await?;
6af905c1 224
db0cb9ce 225 let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
6af905c1
DM
226 .map_err(Error::from);
227
36898ffc 228 let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
6af905c1 229
e43b9175
FG
230 if upload_options.fixed_size.is_none() {
231 bail!("cannot backup image with dynamic chunk size!");
232 }
233
e9722f8b 234 let stats = client
e43b9175 235 .upload_stream(archive_name, stream, upload_options)
e9722f8b 236 .await?;
6af905c1 237
2c3891d1 238 Ok(stats)
6af905c1
DM
239}
240
a47a02ae
DM
241#[api(
242 input: {
243 properties: {
244 repository: {
245 schema: REPO_URL_SCHEMA,
246 optional: true,
247 },
248 "output-format": {
249 schema: OUTPUT_FORMAT,
250 optional: true,
251 },
252 }
253 }
254)]
255/// List backup groups.
256async fn list_backup_groups(param: Value) -> Result<Value, Error> {
812c6f87 257
c81b2b7c
DM
258 let output_format = get_output_format(&param);
259
2665cef7 260 let repo = extract_repository_from_value(&param)?;
812c6f87 261
f3fde36b 262 let client = connect(&repo)?;
812c6f87 263
d0a03d40 264 let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
812c6f87 265
8a8a4703 266 let mut result = client.get(&path, None).await?;
812c6f87 267
d0a03d40
DM
268 record_repository(&repo);
269
c81b2b7c
DM
270 let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
271 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
272 let group = BackupGroup::new(item.backup_type, item.backup_id);
273 Ok(group.group_path().to_str().unwrap().to_owned())
274 };
812c6f87 275
18deda40
DM
276 let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
277 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 278 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
18deda40 279 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
c81b2b7c 280 };
812c6f87 281
c81b2b7c
DM
282 let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
283 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
770a36e5 284 Ok(pbs_tools::format::render_backup_file_list(&item.files))
c81b2b7c 285 };
812c6f87 286
c81b2b7c
DM
287 let options = default_table_format_options()
288 .sortby("backup-type", false)
289 .sortby("backup-id", false)
290 .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
18deda40
DM
291 .column(
292 ColumnConfig::new("last-backup")
293 .renderer(render_last_backup)
294 .header("last snapshot")
295 .right_align(false)
296 )
c81b2b7c
DM
297 .column(ColumnConfig::new("backup-count"))
298 .column(ColumnConfig::new("files").renderer(render_files));
ad20d198 299
c81b2b7c 300 let mut data: Value = result["data"].take();
ad20d198 301
b2362a12 302 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_LIST_GROUPS.returns;
812c6f87 303
b2362a12 304 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc 305
812c6f87
DM
306 Ok(Value::Null)
307}
308
344add38
DW
309#[api(
310 input: {
311 properties: {
312 repository: {
313 schema: REPO_URL_SCHEMA,
314 optional: true,
315 },
316 group: {
317 type: String,
318 description: "Backup group.",
319 },
320 "new-owner": {
e6dc35ac 321 type: Authid,
344add38
DW
322 },
323 }
324 }
325)]
326/// Change owner of a backup group
327async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
328
329 let repo = extract_repository_from_value(&param)?;
330
f3fde36b 331 let mut client = connect(&repo)?;
344add38
DW
332
333 param.as_object_mut().unwrap().remove("repository");
334
335 let group: BackupGroup = group.parse()?;
336
337 param["backup-type"] = group.backup_type().into();
338 param["backup-id"] = group.backup_id().into();
339
340 let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
341 client.post(&path, Some(param)).await?;
342
343 record_repository(&repo);
344
345 Ok(())
346}
347
a47a02ae
DM
348#[api(
349 input: {
350 properties: {
351 repository: {
352 schema: REPO_URL_SCHEMA,
353 optional: true,
354 },
355 }
356 }
357)]
358/// Try to login. If successful, store ticket.
359async fn api_login(param: Value) -> Result<Value, Error> {
e240d8be
DM
360
361 let repo = extract_repository_from_value(&param)?;
362
f3fde36b 363 let client = connect(&repo)?;
8a8a4703 364 client.login().await?;
e240d8be
DM
365
366 record_repository(&repo);
367
368 Ok(Value::Null)
369}
370
a47a02ae
DM
371#[api(
372 input: {
373 properties: {
374 repository: {
375 schema: REPO_URL_SCHEMA,
376 optional: true,
377 },
378 }
379 }
380)]
381/// Logout (delete stored ticket).
382fn api_logout(param: Value) -> Result<Value, Error> {
e240d8be
DM
383
384 let repo = extract_repository_from_value(&param)?;
385
5030b7ce 386 delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
e240d8be
DM
387
388 Ok(Value::Null)
389}
390
e39974af
TL
391#[api(
392 input: {
393 properties: {
394 repository: {
395 schema: REPO_URL_SCHEMA,
396 optional: true,
397 },
398 "output-format": {
399 schema: OUTPUT_FORMAT,
400 optional: true,
401 },
402 }
403 }
404)]
405/// Show client and optional server version
406async fn api_version(param: Value) -> Result<(), Error> {
407
408 let output_format = get_output_format(&param);
409
410 let mut version_info = json!({
411 "client": {
a12b1be7
WB
412 "version": pbs_buildcfg::PROXMOX_PKG_VERSION,
413 "release": pbs_buildcfg::PROXMOX_PKG_RELEASE,
414 "repoid": pbs_buildcfg::PROXMOX_PKG_REPOID,
e39974af
TL
415 }
416 });
417
418 let repo = extract_repository_from_value(&param);
419 if let Ok(repo) = repo {
f3fde36b 420 let client = connect(&repo)?;
e39974af
TL
421
422 match client.get("api2/json/version", None).await {
423 Ok(mut result) => version_info["server"] = result["data"].take(),
424 Err(e) => eprintln!("could not connect to server - {}", e),
425 }
426 }
427 if output_format == "text" {
a12b1be7
WB
428 println!(
429 "client version: {}.{}",
430 pbs_buildcfg::PROXMOX_PKG_VERSION,
431 pbs_buildcfg::PROXMOX_PKG_RELEASE,
432 );
e39974af
TL
433 if let Some(server) = version_info["server"].as_object() {
434 let server_version = server["version"].as_str().unwrap();
435 let server_release = server["release"].as_str().unwrap();
436 println!("server version: {}.{}", server_version, server_release);
437 }
438 } else {
439 format_and_print_result(&version_info, &output_format);
440 }
441
442 Ok(())
443}
444
a47a02ae 445#[api(
94913f35 446 input: {
a47a02ae
DM
447 properties: {
448 repository: {
449 schema: REPO_URL_SCHEMA,
450 optional: true,
451 },
94913f35
DM
452 "output-format": {
453 schema: OUTPUT_FORMAT,
454 optional: true,
455 },
456 },
457 },
a47a02ae
DM
458)]
459/// Start garbage collection for a specific repository.
460async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
8cc0d6af 461
2665cef7 462 let repo = extract_repository_from_value(&param)?;
c2043614
DM
463
464 let output_format = get_output_format(&param);
8cc0d6af 465
f3fde36b 466 let mut client = connect(&repo)?;
8cc0d6af 467
d0a03d40 468 let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
8cc0d6af 469
8a8a4703 470 let result = client.post(&path, None).await?;
8cc0d6af 471
8a8a4703 472 record_repository(&repo);
d0a03d40 473
e68269fc 474 view_task_result(&mut client, result, &output_format).await?;
e5f7def4 475
e5f7def4 476 Ok(Value::Null)
8cc0d6af 477}
33d64b81 478
6d233161 479struct CatalogUploadResult {
f1d76ecf 480 catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
6d233161
FG
481 result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
482}
483
bf6e3217 484fn spawn_catalog_upload(
3bad3e6e 485 client: Arc<BackupWriter>,
3638341a 486 encrypt: bool,
6d233161 487) -> Result<CatalogUploadResult, Error> {
f1d99e3f 488 let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
fc5870be 489 let catalog_stream = pbs_tools::blocking::StdChannelStream(catalog_rx);
bf6e3217
DM
490 let catalog_chunk_size = 512*1024;
491 let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
492
f1d76ecf 493 let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)))?));
bf6e3217
DM
494
495 let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
496
e43b9175
FG
497 let upload_options = UploadOptions {
498 encrypt,
499 compress: true,
500 ..UploadOptions::default()
501 };
502
bf6e3217
DM
503 tokio::spawn(async move {
504 let catalog_upload_result = client
e43b9175 505 .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
bf6e3217
DM
506 .await;
507
508 if let Err(ref err) = catalog_upload_result {
509 eprintln!("catalog upload error - {}", err);
510 client.cancel();
511 }
512
513 let _ = catalog_result_tx.send(catalog_upload_result);
514 });
515
6d233161 516 Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
bf6e3217
DM
517}
518
a47a02ae
DM
519#[api(
520 input: {
521 properties: {
522 backupspec: {
523 type: Array,
524 description: "List of backup source specifications ([<label.ext>:<path>] ...)",
525 items: {
526 schema: BACKUP_SOURCE_SCHEMA,
527 }
528 },
529 repository: {
530 schema: REPO_URL_SCHEMA,
531 optional: true,
532 },
533 "include-dev": {
534 description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
535 optional: true,
536 items: {
537 type: String,
538 description: "Path to file.",
539 }
540 },
58fcbf5a
FE
541 "all-file-systems": {
542 type: Boolean,
543 description: "Include all mounted subdirectories.",
544 optional: true,
545 },
a47a02ae
DM
546 keyfile: {
547 schema: KEYFILE_SCHEMA,
548 optional: true,
549 },
0351f23b
WB
550 "keyfd": {
551 schema: KEYFD_SCHEMA,
552 optional: true,
553 },
c0a87c12
FG
554 "master-pubkey-file": {
555 schema: MASTER_PUBKEY_FILE_SCHEMA,
556 optional: true,
557 },
558 "master-pubkey-fd": {
559 schema: MASTER_PUBKEY_FD_SCHEMA,
560 optional: true,
561 },
24be37e3
WB
562 "crypt-mode": {
563 type: CryptMode,
96ee8577
WB
564 optional: true,
565 },
a47a02ae
DM
566 "skip-lost-and-found": {
567 type: Boolean,
568 description: "Skip lost+found directory.",
569 optional: true,
570 },
571 "backup-type": {
572 schema: BACKUP_TYPE_SCHEMA,
573 optional: true,
574 },
575 "backup-id": {
576 schema: BACKUP_ID_SCHEMA,
577 optional: true,
578 },
579 "backup-time": {
580 schema: BACKUP_TIME_SCHEMA,
581 optional: true,
582 },
583 "chunk-size": {
584 schema: CHUNK_SIZE_SCHEMA,
585 optional: true,
586 },
189996cf
CE
587 "exclude": {
588 type: Array,
589 description: "List of paths or patterns for matching files to exclude.",
590 optional: true,
591 items: {
592 type: String,
593 description: "Path or match pattern.",
594 }
595 },
6fc053ed
CE
596 "entries-max": {
597 type: Integer,
598 description: "Max number of entries to hold in memory.",
599 optional: true,
2b7f8dd5 600 default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
6fc053ed 601 },
e02c3d46
DM
602 "verbose": {
603 type: Boolean,
604 description: "Verbose output.",
605 optional: true,
606 },
a47a02ae
DM
607 }
608 }
609)]
610/// Create (host) backup.
611async fn create_backup(
6049b71f
DM
612 param: Value,
613 _info: &ApiMethod,
dd5495d6 614 _rpcenv: &mut dyn RpcEnvironment,
6049b71f 615) -> Result<Value, Error> {
ff5d3707 616
2665cef7 617 let repo = extract_repository_from_value(&param)?;
ae0be2dd 618
3c8c2827 619 let backupspec_list = json::required_array_param(&param, "backupspec")?;
a914a774 620
eed6db39
DM
621 let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
622
5b72c9b4
DM
623 let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
624
219ef0e6
DM
625 let verbose = param["verbose"].as_bool().unwrap_or(false);
626
ca5d0b61
DM
627 let backup_time_opt = param["backup-time"].as_i64();
628
36898ffc 629 let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
2d9d143a 630
247cdbce
DM
631 if let Some(size) = chunk_size_opt {
632 verify_chunk_size(size)?;
2d9d143a
DM
633 }
634
c6a7ea0a 635 let crypto = crypto_parameters(&param)?;
6d0983db 636
f69adc81 637 let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
fba30411 638
bbf9e7e9 639 let backup_type = param["backup-type"].as_str().unwrap_or("host");
ca5d0b61 640
2eeaacb9
DM
641 let include_dev = param["include-dev"].as_array();
642
c443f58b 643 let entries_max = param["entries-max"].as_u64()
2b7f8dd5 644 .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
6fc053ed 645
189996cf 646 let empty = Vec::new();
c443f58b
WB
647 let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
648
239e49f9 649 let mut pattern_list = Vec::with_capacity(exclude_args.len());
c443f58b
WB
650 for entry in exclude_args {
651 let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
239e49f9 652 pattern_list.push(
c443f58b
WB
653 MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
654 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
655 );
189996cf
CE
656 }
657
2eeaacb9
DM
658 let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
659
660 if let Some(include_dev) = include_dev {
661 if all_file_systems {
662 bail!("option 'all-file-systems' conflicts with option 'include-dev'");
663 }
664
665 let mut set = HashSet::new();
666 for path in include_dev {
667 let path = path.as_str().unwrap();
668 let stat = nix::sys::stat::stat(path)
669 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
670 set.insert(stat.st_dev);
671 }
672 devices = Some(set);
673 }
674
ae0be2dd 675 let mut upload_list = vec![];
f2b4b4b9 676 let mut target_set = HashSet::new();
a914a774 677
ae0be2dd 678 for backupspec in backupspec_list {
7cc3473a
DM
679 let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
680 let filename = &spec.config_string;
681 let target = &spec.archive_name;
bcd879cf 682
f2b4b4b9
SI
683 if target_set.contains(target) {
684 bail!("got target twice: '{}'", target);
685 }
686 target_set.insert(target.to_string());
687
eb1804c5
DM
688 use std::os::unix::fs::FileTypeExt;
689
3fa71727
CE
690 let metadata = std::fs::metadata(filename)
691 .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
eb1804c5 692 let file_type = metadata.file_type();
23bb8780 693
7cc3473a
DM
694 match spec.spec_type {
695 BackupSpecificationType::PXAR => {
ec8a9bb9
DM
696 if !file_type.is_dir() {
697 bail!("got unexpected file type (expected directory)");
698 }
7cc3473a 699 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
ec8a9bb9 700 }
7cc3473a 701 BackupSpecificationType::IMAGE => {
ec8a9bb9
DM
702 if !(file_type.is_file() || file_type.is_block_device()) {
703 bail!("got unexpected file type (expected file or block device)");
704 }
eb1804c5 705
e18a6c9e 706 let size = image_size(&PathBuf::from(filename))?;
23bb8780 707
ec8a9bb9 708 if size == 0 { bail!("got zero-sized file '{}'", filename); }
ae0be2dd 709
7cc3473a 710 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
ec8a9bb9 711 }
7cc3473a 712 BackupSpecificationType::CONFIG => {
ec8a9bb9
DM
713 if !file_type.is_file() {
714 bail!("got unexpected file type (expected regular file)");
715 }
7cc3473a 716 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 717 }
7cc3473a 718 BackupSpecificationType::LOGFILE => {
79679c2d
DM
719 if !file_type.is_file() {
720 bail!("got unexpected file type (expected regular file)");
721 }
7cc3473a 722 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 723 }
ae0be2dd
DM
724 }
725 }
726
22a9189e 727 let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
ae0be2dd 728
f3fde36b 729 let client = connect(&repo)?;
d0a03d40
DM
730 record_repository(&repo);
731
6a7be83e 732 println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
ca5d0b61 733
f69adc81 734 println!("Client name: {}", proxmox::tools::nodename());
ca5d0b61 735
6a7be83e 736 let start_time = std::time::Instant::now();
ca5d0b61 737
6a7be83e 738 println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
51144821 739
c6a7ea0a 740 let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
bb823140 741 None => (None, None),
2f26b866
FG
742 Some(key_with_source) => {
743 println!(
744 "{}",
745 format_key_source(&key_with_source.source, "encryption")
746 );
747
748 let (key, created, fingerprint) =
ff8945fd 749 decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
6f2626ae 750 println!("Encryption key fingerprint: {}", fingerprint);
bb823140 751
44288184 752 let crypt_config = CryptConfig::new(key)?;
bb823140 753
c0a87c12 754 match crypto.master_pubkey {
2f26b866
FG
755 Some(pem_with_source) => {
756 println!("{}", format_key_source(&pem_with_source.source, "master"));
757
758 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
82a103c8 759
1c86893d 760 let mut key_config = KeyConfig::without_password(key)?;
82a103c8 761 key_config.created = created; // keep original value
82a103c8 762
8acfd15d 763 let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
6f2626ae 764
05389a01 765 (Some(Arc::new(crypt_config)), Some(enc_key))
2f26b866 766 },
05389a01 767 _ => (Some(Arc::new(crypt_config)), None),
bb823140 768 }
6d0983db
DM
769 }
770 };
f98ac774 771
8a8a4703
DM
772 let client = BackupWriter::start(
773 client,
b957aa81 774 crypt_config.clone(),
8a8a4703
DM
775 repo.store(),
776 backup_type,
777 &backup_id,
778 backup_time,
779 verbose,
61d7b501 780 false
8a8a4703
DM
781 ).await?;
782
8b7f8d3f
FG
783 let download_previous_manifest = match client.previous_backup_time().await {
784 Ok(Some(backup_time)) => {
785 println!(
786 "Downloading previous manifest ({})",
787 strftime_local("%c", backup_time)?
788 );
789 true
790 }
791 Ok(None) => {
792 println!("No previous manifest available.");
793 false
794 }
795 Err(_) => {
796 // Fallback for outdated server, TODO remove/bubble up with 2.0
797 true
798 }
799 };
800
801 let previous_manifest = if download_previous_manifest {
802 match client.download_previous_manifest().await {
803 Ok(previous_manifest) => {
804 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
805 Ok(()) => Some(Arc::new(previous_manifest)),
806 Err(err) => {
807 println!("Couldn't re-use previous manifest - {}", err);
808 None
809 }
810 }
23f9503a 811 }
8b7f8d3f
FG
812 Err(err) => {
813 println!("Couldn't download previous manifest - {}", err);
814 None
815 }
816 }
817 } else {
818 None
b957aa81
DM
819 };
820
6a7be83e 821 let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
8a8a4703
DM
822 let mut manifest = BackupManifest::new(snapshot);
823
5d85847f 824 let mut catalog = None;
6d233161 825 let mut catalog_result_rx = None;
8a8a4703
DM
826
827 for (backup_type, filename, target, size) in upload_list {
828 match backup_type {
7cc3473a 829 BackupSpecificationType::CONFIG => {
e43b9175
FG
830 let upload_options = UploadOptions {
831 compress: true,
c6a7ea0a 832 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
833 ..UploadOptions::default()
834 };
835
5b32820e 836 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 837 let stats = client
e43b9175 838 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 839 .await?;
c6a7ea0a 840 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 841 }
7cc3473a 842 BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
e43b9175
FG
843 let upload_options = UploadOptions {
844 compress: true,
c6a7ea0a 845 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
846 ..UploadOptions::default()
847 };
848
5b32820e 849 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 850 let stats = client
e43b9175 851 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 852 .await?;
c6a7ea0a 853 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 854 }
7cc3473a 855 BackupSpecificationType::PXAR => {
5d85847f
DC
856 // start catalog upload on first use
857 if catalog.is_none() {
c6a7ea0a 858 let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
6d233161
FG
859 catalog = Some(catalog_upload_res.catalog_writer);
860 catalog_result_rx = Some(catalog_upload_res.result);
5d85847f
DC
861 }
862 let catalog = catalog.as_ref().unwrap();
863
5b32820e 864 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
8a8a4703 865 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
77486a60 866
2b7f8dd5 867 let pxar_options = pbs_client::pxar::PxarCreateOptions {
77486a60
FG
868 device_set: devices.clone(),
869 patterns: pattern_list.clone(),
870 entries_max: entries_max as usize,
871 skip_lost_and_found,
872 verbose,
873 };
874
e43b9175
FG
875 let upload_options = UploadOptions {
876 previous_manifest: previous_manifest.clone(),
877 compress: true,
c6a7ea0a 878 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
879 ..UploadOptions::default()
880 };
881
8a8a4703
DM
882 let stats = backup_directory(
883 &client,
884 &filename,
885 &target,
886 chunk_size_opt,
8a8a4703 887 catalog.clone(),
77486a60 888 pxar_options,
e43b9175 889 upload_options,
8a8a4703 890 ).await?;
c6a7ea0a 891 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703
DM
892 catalog.lock().unwrap().end_directory()?;
893 }
7cc3473a 894 BackupSpecificationType::IMAGE => {
8a8a4703 895 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
e43b9175
FG
896
897 let upload_options = UploadOptions {
898 previous_manifest: previous_manifest.clone(),
899 fixed_size: Some(size),
900 compress: true,
c6a7ea0a 901 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
902 };
903
8a8a4703
DM
904 let stats = backup_image(
905 &client,
e43b9175 906 &filename,
8a8a4703 907 &target,
8a8a4703 908 chunk_size_opt,
e43b9175 909 upload_options,
8a8a4703 910 ).await?;
c6a7ea0a 911 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
6af905c1
DM
912 }
913 }
8a8a4703 914 }
4818c8b6 915
8a8a4703 916 // finalize and upload catalog
5d85847f 917 if let Some(catalog) = catalog {
8a8a4703
DM
918 let mutex = Arc::try_unwrap(catalog)
919 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
920 let mut catalog = mutex.into_inner().unwrap();
bf6e3217 921
8a8a4703 922 catalog.finish()?;
2761d6a4 923
8a8a4703 924 drop(catalog); // close upload stream
2761d6a4 925
6d233161 926 if let Some(catalog_result_rx) = catalog_result_rx {
5d85847f 927 let stats = catalog_result_rx.await??;
c6a7ea0a 928 manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
5d85847f 929 }
8a8a4703 930 }
2761d6a4 931
8a8a4703 932 if let Some(rsa_encrypted_key) = rsa_encrypted_key {
9990af30 933 let target = ENCRYPTED_KEY_BLOB_NAME;
8a8a4703 934 println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
e43b9175 935 let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
8a8a4703 936 let stats = client
e43b9175 937 .upload_blob_from_data(rsa_encrypted_key, target, options)
8a8a4703 938 .await?;
c6a7ea0a 939 manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
8a8a4703 940
8a8a4703 941 }
8a8a4703 942 // create manifest (index.json)
3638341a 943 // manifests are never encrypted, but include a signature
dfa517ad 944 let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
b53f6379 945 .map_err(|err| format_err!("unable to format manifest - {}", err))?;
3638341a 946
b53f6379 947
9688f6de 948 if verbose { println!("Upload index.json to '{}'", repo) };
e43b9175 949 let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
8a8a4703 950 client
e43b9175 951 .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
8a8a4703 952 .await?;
2c3891d1 953
8a8a4703 954 client.finish().await?;
c4ff3dce 955
6a7be83e
DM
956 let end_time = std::time::Instant::now();
957 let elapsed = end_time.duration_since(start_time);
958 println!("Duration: {:.2}s", elapsed.as_secs_f64());
3ec3ec3f 959
6a7be83e 960 println!("End Time: {}", strftime_local("%c", epoch_i64())?);
3d5c11e5 961
8a8a4703 962 Ok(Value::Null)
f98ea63d
DM
963}
964
8e6e18b7 965async fn dump_image<W: Write>(
88892ea8
DM
966 client: Arc<BackupReader>,
967 crypt_config: Option<Arc<CryptConfig>>,
14f6c9cb 968 crypt_mode: CryptMode,
88892ea8
DM
969 index: FixedIndexReader,
970 mut writer: W,
fd04ca7a 971 verbose: bool,
88892ea8
DM
972) -> Result<(), Error> {
973
974 let most_used = index.find_most_used_chunks(8);
975
14f6c9cb 976 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
88892ea8
DM
977
978 // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
979 // and thus slows down reading. Instead, directly use RemoteChunkReader
fd04ca7a
DM
980 let mut per = 0;
981 let mut bytes = 0;
982 let start_time = std::time::Instant::now();
983
88892ea8
DM
984 for pos in 0..index.index_count() {
985 let digest = index.index_digest(pos).unwrap();
8e6e18b7 986 let raw_data = chunk_reader.read_chunk(&digest).await?;
88892ea8 987 writer.write_all(&raw_data)?;
fd04ca7a
DM
988 bytes += raw_data.len();
989 if verbose {
990 let next_per = ((pos+1)*100)/index.index_count();
991 if per != next_per {
992 eprintln!("progress {}% (read {} bytes, duration {} sec)",
993 next_per, bytes, start_time.elapsed().as_secs());
994 per = next_per;
995 }
996 }
88892ea8
DM
997 }
998
fd04ca7a
DM
999 let end_time = std::time::Instant::now();
1000 let elapsed = end_time.duration_since(start_time);
1001 eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1002 bytes,
1003 elapsed.as_secs_f64(),
1004 bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1005 );
1006
1007
88892ea8
DM
1008 Ok(())
1009}
1010
dc155e9b 1011fn parse_archive_type(name: &str) -> (String, ArchiveType) {
2d32fe2c
TL
1012 if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1013 (name.into(), archive_type(name).unwrap())
1014 } else if name.ends_with(".pxar") {
dc155e9b
TL
1015 (format!("{}.didx", name), ArchiveType::DynamicIndex)
1016 } else if name.ends_with(".img") {
1017 (format!("{}.fidx", name), ArchiveType::FixedIndex)
1018 } else {
1019 (format!("{}.blob", name), ArchiveType::Blob)
1020 }
1021}
1022
a47a02ae
DM
1023#[api(
1024 input: {
1025 properties: {
1026 repository: {
1027 schema: REPO_URL_SCHEMA,
1028 optional: true,
1029 },
1030 snapshot: {
1031 type: String,
1032 description: "Group/Snapshot path.",
1033 },
1034 "archive-name": {
1035 description: "Backup archive name.",
1036 type: String,
1037 },
1038 target: {
1039 type: String,
90c815bf 1040 description: r###"Target directory path. Use '-' to write to standard output.
8a8a4703 1041
d1d74c43 1042We do not extract '.pxar' archives when writing to standard output.
8a8a4703 1043
a47a02ae
DM
1044"###
1045 },
1046 "allow-existing-dirs": {
1047 type: Boolean,
1048 description: "Do not fail if directories already exists.",
1049 optional: true,
1050 },
1051 keyfile: {
1052 schema: KEYFILE_SCHEMA,
1053 optional: true,
1054 },
0351f23b
WB
1055 "keyfd": {
1056 schema: KEYFD_SCHEMA,
1057 optional: true,
1058 },
24be37e3
WB
1059 "crypt-mode": {
1060 type: CryptMode,
96ee8577
WB
1061 optional: true,
1062 },
a47a02ae
DM
1063 }
1064 }
1065)]
1066/// Restore backup repository.
1067async fn restore(param: Value) -> Result<Value, Error> {
2665cef7 1068 let repo = extract_repository_from_value(&param)?;
9f912493 1069
86eda3eb
DM
1070 let verbose = param["verbose"].as_bool().unwrap_or(false);
1071
46d5aa0a
DM
1072 let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1073
3c8c2827 1074 let archive_name = json::required_string_param(&param, "archive-name")?;
d5c34d98 1075
f3fde36b 1076 let client = connect(&repo)?;
d0a03d40 1077
d0a03d40 1078 record_repository(&repo);
d5c34d98 1079
3c8c2827 1080 let path = json::required_string_param(&param, "snapshot")?;
9f912493 1081
86eda3eb 1082 let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
d6d3b353 1083 let group: BackupGroup = path.parse()?;
27c9affb 1084 api_datastore_latest_snapshot(&client, repo.store(), group).await?
d5c34d98 1085 } else {
a67f7d0a 1086 let snapshot: BackupDir = path.parse()?;
86eda3eb
DM
1087 (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1088 };
9f912493 1089
3c8c2827 1090 let target = json::required_string_param(&param, "target")?;
bf125261 1091 let target = if target == "-" { None } else { Some(target) };
2ae7d196 1092
c6a7ea0a 1093 let crypto = crypto_parameters(&param)?;
2ae7d196 1094
c6a7ea0a 1095 let crypt_config = match crypto.enc_key {
86eda3eb 1096 None => None,
2f26b866
FG
1097 Some(ref key) => {
1098 let (key, _, _) =
ff8945fd 1099 decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
2f26b866
FG
1100 eprintln!("{}", format_key_source(&key.source, "encryption"));
1101 err
1102 })?;
86eda3eb
DM
1103 Some(Arc::new(CryptConfig::new(key)?))
1104 }
1105 };
d5c34d98 1106
296c50ba
DM
1107 let client = BackupReader::start(
1108 client,
1109 crypt_config.clone(),
1110 repo.store(),
1111 &backup_type,
1112 &backup_id,
1113 backup_time,
1114 true,
1115 ).await?;
86eda3eb 1116
48fbbfeb
FG
1117 let (archive_name, archive_type) = parse_archive_type(archive_name);
1118
2107a5ae 1119 let (manifest, backup_index_data) = client.download_manifest().await?;
02fcf372 1120
48fbbfeb
FG
1121 if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1122 eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1123 } else {
2f26b866
FG
1124 if manifest.signature.is_some() {
1125 if let Some(key) = &crypto.enc_key {
1126 eprintln!("{}", format_key_source(&key.source, "encryption"));
1127 }
1128 if let Some(config) = &crypt_config {
1129 eprintln!("Fingerprint: {}", config.fingerprint());
1130 }
1131 }
48fbbfeb
FG
1132 manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1133 }
dc155e9b
TL
1134
1135 if archive_name == MANIFEST_BLOB_NAME {
02fcf372 1136 if let Some(target) = target {
2107a5ae 1137 replace_file(target, &backup_index_data, CreateOptions::new())?;
02fcf372
DM
1138 } else {
1139 let stdout = std::io::stdout();
1140 let mut writer = stdout.lock();
2107a5ae 1141 writer.write_all(&backup_index_data)
02fcf372
DM
1142 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1143 }
1144
14f6c9cb
FG
1145 return Ok(Value::Null);
1146 }
1147
1148 let file_info = manifest.lookup_file_info(&archive_name)?;
1149
1150 if archive_type == ArchiveType::Blob {
d2267b11 1151
dc155e9b 1152 let mut reader = client.download_blob(&manifest, &archive_name).await?;
f8100e96 1153
bf125261 1154 if let Some(target) = target {
0d986280
DM
1155 let mut writer = std::fs::OpenOptions::new()
1156 .write(true)
1157 .create(true)
1158 .create_new(true)
1159 .open(target)
1160 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1161 std::io::copy(&mut reader, &mut writer)?;
bf125261
DM
1162 } else {
1163 let stdout = std::io::stdout();
1164 let mut writer = stdout.lock();
0d986280 1165 std::io::copy(&mut reader, &mut writer)
bf125261
DM
1166 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1167 }
f8100e96 1168
dc155e9b 1169 } else if archive_type == ArchiveType::DynamicIndex {
86eda3eb 1170
dc155e9b 1171 let index = client.download_dynamic_index(&manifest, &archive_name).await?;
df65bd3d 1172
f4bf7dfc
DM
1173 let most_used = index.find_most_used_chunks(8);
1174
14f6c9cb 1175 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
f4bf7dfc 1176
afb4cd28 1177 let mut reader = BufferedDynamicReader::new(index, chunk_reader);
86eda3eb 1178
2b7f8dd5 1179 let options = pbs_client::pxar::PxarExtractOptions {
72064fd0
FG
1180 match_list: &[],
1181 extract_match_default: true,
1182 allow_existing_dirs,
1183 on_error: None,
1184 };
1185
bf125261 1186 if let Some(target) = target {
2b7f8dd5 1187 pbs_client::pxar::extract_archive(
c443f58b
WB
1188 pxar::decoder::Decoder::from_std(reader)?,
1189 Path::new(target),
2b7f8dd5 1190 pbs_client::pxar::Flags::DEFAULT,
c443f58b
WB
1191 |path| {
1192 if verbose {
1193 println!("{:?}", path);
1194 }
1195 },
72064fd0 1196 options,
c443f58b
WB
1197 )
1198 .map_err(|err| format_err!("error extracting archive - {}", err))?;
bf125261 1199 } else {
88892ea8
DM
1200 let mut writer = std::fs::OpenOptions::new()
1201 .write(true)
1202 .open("/dev/stdout")
1203 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
afb4cd28 1204
bf125261
DM
1205 std::io::copy(&mut reader, &mut writer)
1206 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1207 }
dc155e9b 1208 } else if archive_type == ArchiveType::FixedIndex {
afb4cd28 1209
dc155e9b 1210 let index = client.download_fixed_index(&manifest, &archive_name).await?;
df65bd3d 1211
88892ea8
DM
1212 let mut writer = if let Some(target) = target {
1213 std::fs::OpenOptions::new()
bf125261
DM
1214 .write(true)
1215 .create(true)
1216 .create_new(true)
1217 .open(target)
88892ea8 1218 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
bf125261 1219 } else {
88892ea8
DM
1220 std::fs::OpenOptions::new()
1221 .write(true)
1222 .open("/dev/stdout")
1223 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1224 };
afb4cd28 1225
14f6c9cb 1226 dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
3031e44c 1227 }
fef44d4f
DM
1228
1229 Ok(Value::Null)
45db6f89
DM
1230}
1231
e0665a64
DC
1232#[api(
1233 input: {
1234 properties: {
1235 "dry-run": {
1236 type: bool,
1237 optional: true,
1238 description: "Just show what prune would do, but do not delete anything.",
1239 },
1240 group: {
1241 type: String,
1242 description: "Backup group",
1243 },
1244 "prune-options": {
1245 type: PruneOptions,
1246 flatten: true,
1247 },
1248 "output-format": {
1249 schema: OUTPUT_FORMAT,
1250 optional: true,
1251 },
1252 quiet: {
1253 type: bool,
1254 optional: true,
1255 default: false,
1256 description: "Minimal output - only show removals.",
1257 },
1258 repository: {
1259 schema: REPO_URL_SCHEMA,
1260 optional: true,
1261 },
1262 },
1263 },
1264)]
1265/// Prune a backup repository.
1266async fn prune(
1267 dry_run: Option<bool>,
1268 group: String,
1269 prune_options: PruneOptions,
1270 quiet: bool,
1271 mut param: Value
1272) -> Result<Value, Error> {
2665cef7 1273 let repo = extract_repository_from_value(&param)?;
83b7db02 1274
f3fde36b 1275 let mut client = connect(&repo)?;
83b7db02 1276
d0a03d40 1277 let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
83b7db02 1278
d6d3b353 1279 let group: BackupGroup = group.parse()?;
c2043614 1280
671c6a96 1281 let output_format = extract_output_format(&mut param);
9fdc3ef4 1282
e0665a64
DC
1283 let mut api_param = serde_json::to_value(prune_options)?;
1284 if let Some(dry_run) = dry_run {
1285 api_param["dry-run"] = dry_run.into();
1286 }
1287 api_param["backup-type"] = group.backup_type().into();
1288 api_param["backup-id"] = group.backup_id().into();
83b7db02 1289
e0665a64 1290 let mut result = client.post(&path, Some(api_param)).await?;
74fa81b8 1291
87c42375 1292 record_repository(&repo);
3b03abfe 1293
db1e061d
DM
1294 let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1295 let item: PruneListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 1296 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
db1e061d
DM
1297 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1298 };
1299
c48aa39f
DM
1300 let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1301 Ok(match v.as_bool() {
1302 Some(true) => "keep",
1303 Some(false) => "remove",
1304 None => "unknown",
1305 }.to_string())
1306 };
1307
db1e061d
DM
1308 let options = default_table_format_options()
1309 .sortby("backup-type", false)
1310 .sortby("backup-id", false)
1311 .sortby("backup-time", false)
1312 .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
770a36e5 1313 .column(ColumnConfig::new("backup-time").renderer(pbs_tools::format::render_epoch).header("date"))
c48aa39f 1314 .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
db1e061d
DM
1315 ;
1316
b2362a12 1317 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_PRUNE.returns;
db1e061d
DM
1318
1319 let mut data = result["data"].take();
1320
c48aa39f
DM
1321 if quiet {
1322 let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1323 item["keep"].as_bool() == Some(false)
a375df6f 1324 }).cloned().collect();
c48aa39f
DM
1325 data = list.into();
1326 }
1327
b2362a12 1328 format_and_print_result_full(&mut data, return_type, &output_format, &options);
d0a03d40 1329
43a406fd 1330 Ok(Value::Null)
83b7db02
DM
1331}
1332
a47a02ae
DM
1333#[api(
1334 input: {
1335 properties: {
1336 repository: {
1337 schema: REPO_URL_SCHEMA,
1338 optional: true,
1339 },
1340 "output-format": {
1341 schema: OUTPUT_FORMAT,
1342 optional: true,
1343 },
1344 }
f9beae9c
TL
1345 },
1346 returns: {
1347 type: StorageStatus,
1348 },
a47a02ae
DM
1349)]
1350/// Get repository status.
1351async fn status(param: Value) -> Result<Value, Error> {
34a816cc
DM
1352
1353 let repo = extract_repository_from_value(&param)?;
1354
c2043614 1355 let output_format = get_output_format(&param);
34a816cc 1356
f3fde36b 1357 let client = connect(&repo)?;
34a816cc
DM
1358
1359 let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1360
1dc117bb 1361 let mut result = client.get(&path, None).await?;
14e08625 1362 let mut data = result["data"].take();
34a816cc
DM
1363
1364 record_repository(&repo);
1365
390c5bdd
DM
1366 let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1367 let v = v.as_u64().unwrap();
1368 let total = record["total"].as_u64().unwrap();
1369 let roundup = total/200;
1370 let per = ((v+roundup)*100)/total;
e23f5863
DM
1371 let info = format!(" ({} %)", per);
1372 Ok(format!("{} {:>8}", v, info))
390c5bdd 1373 };
1dc117bb 1374
c2043614 1375 let options = default_table_format_options()
be2425ff 1376 .noheader(true)
e23f5863 1377 .column(ColumnConfig::new("total").renderer(render_total_percentage))
390c5bdd
DM
1378 .column(ColumnConfig::new("used").renderer(render_total_percentage))
1379 .column(ColumnConfig::new("avail").renderer(render_total_percentage));
34a816cc 1380
b2362a12 1381 let return_type = &API_METHOD_STATUS.returns;
390c5bdd 1382
b2362a12 1383 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc
DM
1384
1385 Ok(Value::Null)
1386}
1387
c443f58b
WB
1388/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1389/// async use!
1390///
1391/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1392/// so that we can properly access it from multiple threads simultaneously while not issuing
1393/// duplicate simultaneous reads over http.
43abba4b 1394pub struct BufferedDynamicReadAt {
c443f58b
WB
1395 inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1396}
1397
1398impl BufferedDynamicReadAt {
1399 fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
1400 Self {
1401 inner: Mutex::new(inner),
1402 }
1403 }
1404}
1405
a6f87283
WB
1406impl ReadAt for BufferedDynamicReadAt {
1407 fn start_read_at<'a>(
1408 self: Pin<&'a Self>,
c443f58b 1409 _cx: &mut Context,
a6f87283 1410 buf: &'a mut [u8],
c443f58b 1411 offset: u64,
a6f87283 1412 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
a6f87283 1413 MaybeReady::Ready(tokio::task::block_in_place(move || {
c443f58b
WB
1414 let mut reader = self.inner.lock().unwrap();
1415 reader.seek(SeekFrom::Start(offset))?;
a6f87283
WB
1416 Ok(reader.read(buf)?)
1417 }))
1418 }
1419
1420 fn poll_complete<'a>(
1421 self: Pin<&'a Self>,
1422 _op: ReadAtOperation<'a>,
1423 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
bbc71e3b 1424 panic!("BufferedDynamicReadAt::start_read_at returned Pending");
c443f58b
WB
1425 }
1426}
1427
f2401311 1428fn main() {
33d64b81 1429
255f378a 1430 let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
49fddd98 1431 .arg_param(&["backupspec"])
d0a03d40 1432 .completion_cb("repository", complete_repository)
49811347 1433 .completion_cb("backupspec", complete_backup_source)
2b7f8dd5
WB
1434 .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
1435 .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
49811347 1436 .completion_cb("chunk-size", complete_chunk_size);
f8838fe9 1437
caea8d61
DM
1438 let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
1439 .completion_cb("repository", complete_repository)
2b7f8dd5 1440 .completion_cb("keyfile", pbs_tools::fs::complete_file_name);
caea8d61 1441
255f378a 1442 let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
d0a03d40 1443 .completion_cb("repository", complete_repository);
41c039e1 1444
255f378a 1445 let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
d0a03d40 1446 .completion_cb("repository", complete_repository);
8cc0d6af 1447
255f378a 1448 let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
49fddd98 1449 .arg_param(&["snapshot", "archive-name", "target"])
b2388518 1450 .completion_cb("repository", complete_repository)
08dc340a
DM
1451 .completion_cb("snapshot", complete_group_or_snapshot)
1452 .completion_cb("archive-name", complete_archive_name)
2b7f8dd5 1453 .completion_cb("target", pbs_tools::fs::complete_file_name);
9f912493 1454
255f378a 1455 let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
49fddd98 1456 .arg_param(&["group"])
9fdc3ef4 1457 .completion_cb("group", complete_backup_group)
d0a03d40 1458 .completion_cb("repository", complete_repository);
9f912493 1459
255f378a 1460 let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
34a816cc
DM
1461 .completion_cb("repository", complete_repository);
1462
255f378a 1463 let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
e240d8be
DM
1464 .completion_cb("repository", complete_repository);
1465
255f378a 1466 let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
e240d8be 1467 .completion_cb("repository", complete_repository);
32efac1c 1468
e39974af
TL
1469 let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
1470 .completion_cb("repository", complete_repository);
1471
344add38
DW
1472 let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
1473 .arg_param(&["group", "new-owner"])
1474 .completion_cb("group", complete_backup_group)
0224c3c2 1475 .completion_cb("new-owner", complete_auth_id)
344add38
DW
1476 .completion_cb("repository", complete_repository);
1477
41c039e1 1478 let cmd_def = CliCommandMap::new()
48ef3c33 1479 .insert("backup", backup_cmd_def)
48ef3c33
DM
1480 .insert("garbage-collect", garbage_collect_cmd_def)
1481 .insert("list", list_cmd_def)
1482 .insert("login", login_cmd_def)
1483 .insert("logout", logout_cmd_def)
1484 .insert("prune", prune_cmd_def)
1485 .insert("restore", restore_cmd_def)
a65e3e4b 1486 .insert("snapshot", snapshot_mgtm_cli())
48ef3c33 1487 .insert("status", status_cmd_def)
9696f519 1488 .insert("key", key::cli())
43abba4b 1489 .insert("mount", mount_cmd_def())
45f9b32e
SR
1490 .insert("map", map_cmd_def())
1491 .insert("unmap", unmap_cmd_def())
5830c205 1492 .insert("catalog", catalog_mgmt_cli())
caea8d61 1493 .insert("task", task_mgmt_cli())
e39974af 1494 .insert("version", version_cmd_def)
344add38 1495 .insert("benchmark", benchmark_cmd_def)
731eeef2
DM
1496 .insert("change-owner", change_owner_cmd_def)
1497
61205f00 1498 .alias(&["files"], &["snapshot", "files"])
edebd523 1499 .alias(&["forget"], &["snapshot", "forget"])
0c9209b0 1500 .alias(&["upload-log"], &["snapshot", "upload-log"])
731eeef2
DM
1501 .alias(&["snapshots"], &["snapshot", "list"])
1502 ;
48ef3c33 1503
7b22acd0
DM
1504 let rpcenv = CliEnvironment::new();
1505 run_cli_command(cmd_def, rpcenv, Some(|future| {
d420962f 1506 pbs_runtime::main(future)
d08bc483 1507 }));
ff5d3707 1508}