]> git.proxmox.com Git - proxmox-backup.git/blame - src/bin/proxmox-backup-client.rs
move 'wait_for_local_worker' from client to server
[proxmox-backup.git] / src / bin / proxmox-backup-client.rs
CommitLineData
f1a83e97 1use std::collections::HashSet;
0351f23b 2use std::io::{self, Read, Write, Seek, SeekFrom};
c443f58b
WB
3use std::path::{Path, PathBuf};
4use std::pin::Pin;
5use std::sync::{Arc, Mutex};
a6f87283 6use std::task::Context;
c443f58b
WB
7
8use anyhow::{bail, format_err, Error};
c443f58b 9use futures::stream::{StreamExt, TryStreamExt};
c443f58b 10use serde_json::{json, Value};
c443f58b 11use tokio::sync::mpsc;
7c667013 12use tokio_stream::wrappers::ReceiverStream;
c443f58b 13use xdg::BaseDirectories;
2761d6a4 14
c443f58b 15use pathpatterns::{MatchEntry, MatchType, PatternFlag};
6a7be83e
DM
16use proxmox::{
17 tools::{
18 time::{strftime_local, epoch_i64},
ff8945fd 19 fs::{file_get_json, replace_file, CreateOptions, image_size},
6a7be83e
DM
20 },
21 api::{
22 api,
6a7be83e
DM
23 ApiMethod,
24 RpcEnvironment,
6a7be83e
DM
25 cli::*,
26 },
27};
a6f87283 28use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
ff5d3707 29
f323e906 30use pbs_datastore::catalog::BackupCatalogWriter;
4805edc4
WB
31use pbs_tools::sync::StdChannelWriter;
32use pbs_tools::tokio::TokioWriterAdapter;
f323e906 33
bbf9e7e9 34use proxmox_backup::api2::types::*;
e39974af 35use proxmox_backup::api2::version;
151c6ce2 36use proxmox_backup::client::*;
4d16badf
WB
37use proxmox_backup::backup::{
38 archive_type,
0351f23b 39 decrypt_key,
8acfd15d 40 rsa_encrypt_key_config,
4d16badf
WB
41 verify_chunk_size,
42 ArchiveType,
8e6e18b7 43 AsyncReadChunk,
4d16badf
WB
44 BackupDir,
45 BackupGroup,
46 BackupManifest,
47 BufferedDynamicReader,
f28d9088 48 CATALOG_NAME,
4d16badf
WB
49 CatalogReader,
50 CatalogWriter,
4d16badf
WB
51 ChunkStream,
52 CryptConfig,
f28d9088 53 CryptMode,
4d16badf 54 DynamicIndexReader,
9990af30 55 ENCRYPTED_KEY_BLOB_NAME,
4d16badf
WB
56 FixedChunkStream,
57 FixedIndexReader,
8acfd15d 58 KeyConfig,
4d16badf 59 IndexFile,
4d16badf 60 MANIFEST_BLOB_NAME,
4d16badf 61 Shell,
e0665a64 62 PruneOptions,
4d16badf 63};
4805edc4 64use proxmox_backup::tools;
ae0be2dd 65
caea8d61
DM
66mod proxmox_backup_client;
67use proxmox_backup_client::*;
68
ff8945fd
SR
69pub mod proxmox_client_tools;
70use proxmox_client_tools::{
71 complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
72 complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
73 complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
74 extract_repository_from_value,
75 key_source::{
76 crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
77 KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
78 },
79 CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
80};
2665cef7 81
d0a03d40
DM
82fn record_repository(repo: &BackupRepository) {
83
84 let base = match BaseDirectories::with_prefix("proxmox-backup") {
85 Ok(v) => v,
86 _ => return,
87 };
88
89 // usually $HOME/.cache/proxmox-backup/repo-list
90 let path = match base.place_cache_file("repo-list") {
91 Ok(v) => v,
92 _ => return,
93 };
94
11377a47 95 let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
d0a03d40
DM
96
97 let repo = repo.to_string();
98
99 data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
100
101 let mut map = serde_json::map::Map::new();
102
103 loop {
104 let mut max_used = 0;
105 let mut max_repo = None;
106 for (repo, count) in data.as_object().unwrap() {
107 if map.contains_key(repo) { continue; }
108 if let Some(count) = count.as_i64() {
109 if count > max_used {
110 max_used = count;
111 max_repo = Some(repo);
112 }
113 }
114 }
115 if let Some(repo) = max_repo {
116 map.insert(repo.to_owned(), json!(max_used));
117 } else {
118 break;
119 }
120 if map.len() > 10 { // store max. 10 repos
121 break;
122 }
123 }
124
125 let new_data = json!(map);
126
feaa1ad3 127 let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
d0a03d40
DM
128}
129
42af4b8f
DM
130async fn api_datastore_list_snapshots(
131 client: &HttpClient,
132 store: &str,
133 group: Option<BackupGroup>,
f24fc116 134) -> Result<Value, Error> {
42af4b8f
DM
135
136 let path = format!("api2/json/admin/datastore/{}/snapshots", store);
137
138 let mut args = json!({});
139 if let Some(group) = group {
140 args["backup-type"] = group.backup_type().into();
141 args["backup-id"] = group.backup_id().into();
142 }
143
144 let mut result = client.get(&path, Some(args)).await?;
145
f24fc116 146 Ok(result["data"].take())
42af4b8f
DM
147}
148
43abba4b 149pub async fn api_datastore_latest_snapshot(
27c9affb
DM
150 client: &HttpClient,
151 store: &str,
152 group: BackupGroup,
6a7be83e 153) -> Result<(String, String, i64), Error> {
27c9affb 154
f24fc116
DM
155 let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
156 let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
27c9affb
DM
157
158 if list.is_empty() {
159 bail!("backup group {:?} does not contain any snapshots.", group.group_path());
160 }
161
162 list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
163
6a7be83e 164 let backup_time = list[0].backup_time;
27c9affb
DM
165
166 Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
167}
168
e9722f8b 169async fn backup_directory<P: AsRef<Path>>(
cf9271e2 170 client: &BackupWriter,
17d6979a 171 dir_path: P,
247cdbce 172 archive_name: &str,
36898ffc 173 chunk_size: Option<usize>,
f1d76ecf 174 catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
77486a60 175 pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
e43b9175 176 upload_options: UploadOptions,
2c3891d1 177) -> Result<BackupStats, Error> {
33d64b81 178
6fc053ed
CE
179 let pxar_stream = PxarBackupStream::open(
180 dir_path.as_ref(),
6fc053ed 181 catalog,
77486a60 182 pxar_create_options,
6fc053ed 183 )?;
e9722f8b 184 let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
ff3d3100 185
0bfcea6a 186 let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
5e7a09be 187
7c667013 188 let stream = ReceiverStream::new(rx)
e9722f8b 189 .map_err(Error::from);
17d6979a 190
c4ff3dce 191 // spawn chunker inside a separate task so that it can run parallel
e9722f8b 192 tokio::spawn(async move {
db0cb9ce
WB
193 while let Some(v) = chunk_stream.next().await {
194 let _ = tx.send(v).await;
195 }
e9722f8b 196 });
17d6979a 197
e43b9175
FG
198 if upload_options.fixed_size.is_some() {
199 bail!("cannot backup directory with fixed chunk size!");
200 }
201
e9722f8b 202 let stats = client
e43b9175 203 .upload_stream(archive_name, stream, upload_options)
e9722f8b 204 .await?;
bcd879cf 205
2c3891d1 206 Ok(stats)
bcd879cf
DM
207}
208
e9722f8b 209async fn backup_image<P: AsRef<Path>>(
cf9271e2 210 client: &BackupWriter,
6af905c1
DM
211 image_path: P,
212 archive_name: &str,
36898ffc 213 chunk_size: Option<usize>,
e43b9175 214 upload_options: UploadOptions,
2c3891d1 215) -> Result<BackupStats, Error> {
6af905c1 216
6af905c1
DM
217 let path = image_path.as_ref().to_owned();
218
e9722f8b 219 let file = tokio::fs::File::open(path).await?;
6af905c1 220
db0cb9ce 221 let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
6af905c1
DM
222 .map_err(Error::from);
223
36898ffc 224 let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
6af905c1 225
e43b9175
FG
226 if upload_options.fixed_size.is_none() {
227 bail!("cannot backup image with dynamic chunk size!");
228 }
229
e9722f8b 230 let stats = client
e43b9175 231 .upload_stream(archive_name, stream, upload_options)
e9722f8b 232 .await?;
6af905c1 233
2c3891d1 234 Ok(stats)
6af905c1
DM
235}
236
a47a02ae
DM
237#[api(
238 input: {
239 properties: {
240 repository: {
241 schema: REPO_URL_SCHEMA,
242 optional: true,
243 },
244 "output-format": {
245 schema: OUTPUT_FORMAT,
246 optional: true,
247 },
248 }
249 }
250)]
251/// List backup groups.
252async fn list_backup_groups(param: Value) -> Result<Value, Error> {
812c6f87 253
c81b2b7c
DM
254 let output_format = get_output_format(&param);
255
2665cef7 256 let repo = extract_repository_from_value(&param)?;
812c6f87 257
f3fde36b 258 let client = connect(&repo)?;
812c6f87 259
d0a03d40 260 let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
812c6f87 261
8a8a4703 262 let mut result = client.get(&path, None).await?;
812c6f87 263
d0a03d40
DM
264 record_repository(&repo);
265
c81b2b7c
DM
266 let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
267 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
268 let group = BackupGroup::new(item.backup_type, item.backup_id);
269 Ok(group.group_path().to_str().unwrap().to_owned())
270 };
812c6f87 271
18deda40
DM
272 let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
273 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 274 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
18deda40 275 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
c81b2b7c 276 };
812c6f87 277
c81b2b7c
DM
278 let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
279 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
770a36e5 280 Ok(pbs_tools::format::render_backup_file_list(&item.files))
c81b2b7c 281 };
812c6f87 282
c81b2b7c
DM
283 let options = default_table_format_options()
284 .sortby("backup-type", false)
285 .sortby("backup-id", false)
286 .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
18deda40
DM
287 .column(
288 ColumnConfig::new("last-backup")
289 .renderer(render_last_backup)
290 .header("last snapshot")
291 .right_align(false)
292 )
c81b2b7c
DM
293 .column(ColumnConfig::new("backup-count"))
294 .column(ColumnConfig::new("files").renderer(render_files));
ad20d198 295
c81b2b7c 296 let mut data: Value = result["data"].take();
ad20d198 297
b2362a12 298 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_LIST_GROUPS.returns;
812c6f87 299
b2362a12 300 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc 301
812c6f87
DM
302 Ok(Value::Null)
303}
304
344add38
DW
305#[api(
306 input: {
307 properties: {
308 repository: {
309 schema: REPO_URL_SCHEMA,
310 optional: true,
311 },
312 group: {
313 type: String,
314 description: "Backup group.",
315 },
316 "new-owner": {
e6dc35ac 317 type: Authid,
344add38
DW
318 },
319 }
320 }
321)]
322/// Change owner of a backup group
323async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
324
325 let repo = extract_repository_from_value(&param)?;
326
f3fde36b 327 let mut client = connect(&repo)?;
344add38
DW
328
329 param.as_object_mut().unwrap().remove("repository");
330
331 let group: BackupGroup = group.parse()?;
332
333 param["backup-type"] = group.backup_type().into();
334 param["backup-id"] = group.backup_id().into();
335
336 let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
337 client.post(&path, Some(param)).await?;
338
339 record_repository(&repo);
340
341 Ok(())
342}
343
a47a02ae
DM
344#[api(
345 input: {
346 properties: {
347 repository: {
348 schema: REPO_URL_SCHEMA,
349 optional: true,
350 },
351 }
352 }
353)]
354/// Try to login. If successful, store ticket.
355async fn api_login(param: Value) -> Result<Value, Error> {
e240d8be
DM
356
357 let repo = extract_repository_from_value(&param)?;
358
f3fde36b 359 let client = connect(&repo)?;
8a8a4703 360 client.login().await?;
e240d8be
DM
361
362 record_repository(&repo);
363
364 Ok(Value::Null)
365}
366
a47a02ae
DM
367#[api(
368 input: {
369 properties: {
370 repository: {
371 schema: REPO_URL_SCHEMA,
372 optional: true,
373 },
374 }
375 }
376)]
377/// Logout (delete stored ticket).
378fn api_logout(param: Value) -> Result<Value, Error> {
e240d8be
DM
379
380 let repo = extract_repository_from_value(&param)?;
381
5030b7ce 382 delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
e240d8be
DM
383
384 Ok(Value::Null)
385}
386
e39974af
TL
387#[api(
388 input: {
389 properties: {
390 repository: {
391 schema: REPO_URL_SCHEMA,
392 optional: true,
393 },
394 "output-format": {
395 schema: OUTPUT_FORMAT,
396 optional: true,
397 },
398 }
399 }
400)]
401/// Show client and optional server version
402async fn api_version(param: Value) -> Result<(), Error> {
403
404 let output_format = get_output_format(&param);
405
406 let mut version_info = json!({
407 "client": {
408 "version": version::PROXMOX_PKG_VERSION,
409 "release": version::PROXMOX_PKG_RELEASE,
410 "repoid": version::PROXMOX_PKG_REPOID,
411 }
412 });
413
414 let repo = extract_repository_from_value(&param);
415 if let Ok(repo) = repo {
f3fde36b 416 let client = connect(&repo)?;
e39974af
TL
417
418 match client.get("api2/json/version", None).await {
419 Ok(mut result) => version_info["server"] = result["data"].take(),
420 Err(e) => eprintln!("could not connect to server - {}", e),
421 }
422 }
423 if output_format == "text" {
424 println!("client version: {}.{}", version::PROXMOX_PKG_VERSION, version::PROXMOX_PKG_RELEASE);
425 if let Some(server) = version_info["server"].as_object() {
426 let server_version = server["version"].as_str().unwrap();
427 let server_release = server["release"].as_str().unwrap();
428 println!("server version: {}.{}", server_version, server_release);
429 }
430 } else {
431 format_and_print_result(&version_info, &output_format);
432 }
433
434 Ok(())
435}
436
a47a02ae 437#[api(
94913f35 438 input: {
a47a02ae
DM
439 properties: {
440 repository: {
441 schema: REPO_URL_SCHEMA,
442 optional: true,
443 },
94913f35
DM
444 "output-format": {
445 schema: OUTPUT_FORMAT,
446 optional: true,
447 },
448 },
449 },
a47a02ae
DM
450)]
451/// Start garbage collection for a specific repository.
452async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
8cc0d6af 453
2665cef7 454 let repo = extract_repository_from_value(&param)?;
c2043614
DM
455
456 let output_format = get_output_format(&param);
8cc0d6af 457
f3fde36b 458 let mut client = connect(&repo)?;
8cc0d6af 459
d0a03d40 460 let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
8cc0d6af 461
8a8a4703 462 let result = client.post(&path, None).await?;
8cc0d6af 463
8a8a4703 464 record_repository(&repo);
d0a03d40 465
e68269fc 466 view_task_result(&mut client, result, &output_format).await?;
e5f7def4 467
e5f7def4 468 Ok(Value::Null)
8cc0d6af 469}
33d64b81 470
6d233161 471struct CatalogUploadResult {
f1d76ecf 472 catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
6d233161
FG
473 result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
474}
475
bf6e3217 476fn spawn_catalog_upload(
3bad3e6e 477 client: Arc<BackupWriter>,
3638341a 478 encrypt: bool,
6d233161 479) -> Result<CatalogUploadResult, Error> {
f1d99e3f
DM
480 let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
481 let catalog_stream = crate::tools::StdChannelStream(catalog_rx);
bf6e3217
DM
482 let catalog_chunk_size = 512*1024;
483 let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
484
f1d76ecf 485 let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)))?));
bf6e3217
DM
486
487 let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
488
e43b9175
FG
489 let upload_options = UploadOptions {
490 encrypt,
491 compress: true,
492 ..UploadOptions::default()
493 };
494
bf6e3217
DM
495 tokio::spawn(async move {
496 let catalog_upload_result = client
e43b9175 497 .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
bf6e3217
DM
498 .await;
499
500 if let Err(ref err) = catalog_upload_result {
501 eprintln!("catalog upload error - {}", err);
502 client.cancel();
503 }
504
505 let _ = catalog_result_tx.send(catalog_upload_result);
506 });
507
6d233161 508 Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
bf6e3217
DM
509}
510
a47a02ae
DM
511#[api(
512 input: {
513 properties: {
514 backupspec: {
515 type: Array,
516 description: "List of backup source specifications ([<label.ext>:<path>] ...)",
517 items: {
518 schema: BACKUP_SOURCE_SCHEMA,
519 }
520 },
521 repository: {
522 schema: REPO_URL_SCHEMA,
523 optional: true,
524 },
525 "include-dev": {
526 description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
527 optional: true,
528 items: {
529 type: String,
530 description: "Path to file.",
531 }
532 },
58fcbf5a
FE
533 "all-file-systems": {
534 type: Boolean,
535 description: "Include all mounted subdirectories.",
536 optional: true,
537 },
a47a02ae
DM
538 keyfile: {
539 schema: KEYFILE_SCHEMA,
540 optional: true,
541 },
0351f23b
WB
542 "keyfd": {
543 schema: KEYFD_SCHEMA,
544 optional: true,
545 },
c0a87c12
FG
546 "master-pubkey-file": {
547 schema: MASTER_PUBKEY_FILE_SCHEMA,
548 optional: true,
549 },
550 "master-pubkey-fd": {
551 schema: MASTER_PUBKEY_FD_SCHEMA,
552 optional: true,
553 },
24be37e3
WB
554 "crypt-mode": {
555 type: CryptMode,
96ee8577
WB
556 optional: true,
557 },
a47a02ae
DM
558 "skip-lost-and-found": {
559 type: Boolean,
560 description: "Skip lost+found directory.",
561 optional: true,
562 },
563 "backup-type": {
564 schema: BACKUP_TYPE_SCHEMA,
565 optional: true,
566 },
567 "backup-id": {
568 schema: BACKUP_ID_SCHEMA,
569 optional: true,
570 },
571 "backup-time": {
572 schema: BACKUP_TIME_SCHEMA,
573 optional: true,
574 },
575 "chunk-size": {
576 schema: CHUNK_SIZE_SCHEMA,
577 optional: true,
578 },
189996cf
CE
579 "exclude": {
580 type: Array,
581 description: "List of paths or patterns for matching files to exclude.",
582 optional: true,
583 items: {
584 type: String,
585 description: "Path or match pattern.",
586 }
587 },
6fc053ed
CE
588 "entries-max": {
589 type: Integer,
590 description: "Max number of entries to hold in memory.",
591 optional: true,
c443f58b 592 default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
6fc053ed 593 },
e02c3d46
DM
594 "verbose": {
595 type: Boolean,
596 description: "Verbose output.",
597 optional: true,
598 },
a47a02ae
DM
599 }
600 }
601)]
602/// Create (host) backup.
603async fn create_backup(
6049b71f
DM
604 param: Value,
605 _info: &ApiMethod,
dd5495d6 606 _rpcenv: &mut dyn RpcEnvironment,
6049b71f 607) -> Result<Value, Error> {
ff5d3707 608
2665cef7 609 let repo = extract_repository_from_value(&param)?;
ae0be2dd
DM
610
611 let backupspec_list = tools::required_array_param(&param, "backupspec")?;
a914a774 612
eed6db39
DM
613 let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
614
5b72c9b4
DM
615 let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
616
219ef0e6
DM
617 let verbose = param["verbose"].as_bool().unwrap_or(false);
618
ca5d0b61
DM
619 let backup_time_opt = param["backup-time"].as_i64();
620
36898ffc 621 let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
2d9d143a 622
247cdbce
DM
623 if let Some(size) = chunk_size_opt {
624 verify_chunk_size(size)?;
2d9d143a
DM
625 }
626
c6a7ea0a 627 let crypto = crypto_parameters(&param)?;
6d0983db 628
f69adc81 629 let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
fba30411 630
bbf9e7e9 631 let backup_type = param["backup-type"].as_str().unwrap_or("host");
ca5d0b61 632
2eeaacb9
DM
633 let include_dev = param["include-dev"].as_array();
634
c443f58b
WB
635 let entries_max = param["entries-max"].as_u64()
636 .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
6fc053ed 637
189996cf 638 let empty = Vec::new();
c443f58b
WB
639 let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
640
239e49f9 641 let mut pattern_list = Vec::with_capacity(exclude_args.len());
c443f58b
WB
642 for entry in exclude_args {
643 let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
239e49f9 644 pattern_list.push(
c443f58b
WB
645 MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
646 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
647 );
189996cf
CE
648 }
649
2eeaacb9
DM
650 let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
651
652 if let Some(include_dev) = include_dev {
653 if all_file_systems {
654 bail!("option 'all-file-systems' conflicts with option 'include-dev'");
655 }
656
657 let mut set = HashSet::new();
658 for path in include_dev {
659 let path = path.as_str().unwrap();
660 let stat = nix::sys::stat::stat(path)
661 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
662 set.insert(stat.st_dev);
663 }
664 devices = Some(set);
665 }
666
ae0be2dd 667 let mut upload_list = vec![];
f2b4b4b9 668 let mut target_set = HashSet::new();
a914a774 669
ae0be2dd 670 for backupspec in backupspec_list {
7cc3473a
DM
671 let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
672 let filename = &spec.config_string;
673 let target = &spec.archive_name;
bcd879cf 674
f2b4b4b9
SI
675 if target_set.contains(target) {
676 bail!("got target twice: '{}'", target);
677 }
678 target_set.insert(target.to_string());
679
eb1804c5
DM
680 use std::os::unix::fs::FileTypeExt;
681
3fa71727
CE
682 let metadata = std::fs::metadata(filename)
683 .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
eb1804c5 684 let file_type = metadata.file_type();
23bb8780 685
7cc3473a
DM
686 match spec.spec_type {
687 BackupSpecificationType::PXAR => {
ec8a9bb9
DM
688 if !file_type.is_dir() {
689 bail!("got unexpected file type (expected directory)");
690 }
7cc3473a 691 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
ec8a9bb9 692 }
7cc3473a 693 BackupSpecificationType::IMAGE => {
ec8a9bb9
DM
694 if !(file_type.is_file() || file_type.is_block_device()) {
695 bail!("got unexpected file type (expected file or block device)");
696 }
eb1804c5 697
e18a6c9e 698 let size = image_size(&PathBuf::from(filename))?;
23bb8780 699
ec8a9bb9 700 if size == 0 { bail!("got zero-sized file '{}'", filename); }
ae0be2dd 701
7cc3473a 702 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
ec8a9bb9 703 }
7cc3473a 704 BackupSpecificationType::CONFIG => {
ec8a9bb9
DM
705 if !file_type.is_file() {
706 bail!("got unexpected file type (expected regular file)");
707 }
7cc3473a 708 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 709 }
7cc3473a 710 BackupSpecificationType::LOGFILE => {
79679c2d
DM
711 if !file_type.is_file() {
712 bail!("got unexpected file type (expected regular file)");
713 }
7cc3473a 714 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 715 }
ae0be2dd
DM
716 }
717 }
718
22a9189e 719 let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
ae0be2dd 720
f3fde36b 721 let client = connect(&repo)?;
d0a03d40
DM
722 record_repository(&repo);
723
6a7be83e 724 println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
ca5d0b61 725
f69adc81 726 println!("Client name: {}", proxmox::tools::nodename());
ca5d0b61 727
6a7be83e 728 let start_time = std::time::Instant::now();
ca5d0b61 729
6a7be83e 730 println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
51144821 731
c6a7ea0a 732 let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
bb823140 733 None => (None, None),
2f26b866
FG
734 Some(key_with_source) => {
735 println!(
736 "{}",
737 format_key_source(&key_with_source.source, "encryption")
738 );
739
740 let (key, created, fingerprint) =
ff8945fd 741 decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
6f2626ae 742 println!("Encryption key fingerprint: {}", fingerprint);
bb823140 743
44288184 744 let crypt_config = CryptConfig::new(key)?;
bb823140 745
c0a87c12 746 match crypto.master_pubkey {
2f26b866
FG
747 Some(pem_with_source) => {
748 println!("{}", format_key_source(&pem_with_source.source, "master"));
749
750 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
82a103c8 751
1c86893d 752 let mut key_config = KeyConfig::without_password(key)?;
82a103c8 753 key_config.created = created; // keep original value
82a103c8 754
8acfd15d 755 let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
6f2626ae 756
05389a01 757 (Some(Arc::new(crypt_config)), Some(enc_key))
2f26b866 758 },
05389a01 759 _ => (Some(Arc::new(crypt_config)), None),
bb823140 760 }
6d0983db
DM
761 }
762 };
f98ac774 763
8a8a4703
DM
764 let client = BackupWriter::start(
765 client,
b957aa81 766 crypt_config.clone(),
8a8a4703
DM
767 repo.store(),
768 backup_type,
769 &backup_id,
770 backup_time,
771 verbose,
61d7b501 772 false
8a8a4703
DM
773 ).await?;
774
8b7f8d3f
FG
775 let download_previous_manifest = match client.previous_backup_time().await {
776 Ok(Some(backup_time)) => {
777 println!(
778 "Downloading previous manifest ({})",
779 strftime_local("%c", backup_time)?
780 );
781 true
782 }
783 Ok(None) => {
784 println!("No previous manifest available.");
785 false
786 }
787 Err(_) => {
788 // Fallback for outdated server, TODO remove/bubble up with 2.0
789 true
790 }
791 };
792
793 let previous_manifest = if download_previous_manifest {
794 match client.download_previous_manifest().await {
795 Ok(previous_manifest) => {
796 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
797 Ok(()) => Some(Arc::new(previous_manifest)),
798 Err(err) => {
799 println!("Couldn't re-use previous manifest - {}", err);
800 None
801 }
802 }
23f9503a 803 }
8b7f8d3f
FG
804 Err(err) => {
805 println!("Couldn't download previous manifest - {}", err);
806 None
807 }
808 }
809 } else {
810 None
b957aa81
DM
811 };
812
6a7be83e 813 let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
8a8a4703
DM
814 let mut manifest = BackupManifest::new(snapshot);
815
5d85847f 816 let mut catalog = None;
6d233161 817 let mut catalog_result_rx = None;
8a8a4703
DM
818
819 for (backup_type, filename, target, size) in upload_list {
820 match backup_type {
7cc3473a 821 BackupSpecificationType::CONFIG => {
e43b9175
FG
822 let upload_options = UploadOptions {
823 compress: true,
c6a7ea0a 824 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
825 ..UploadOptions::default()
826 };
827
5b32820e 828 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 829 let stats = client
e43b9175 830 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 831 .await?;
c6a7ea0a 832 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 833 }
7cc3473a 834 BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
e43b9175
FG
835 let upload_options = UploadOptions {
836 compress: true,
c6a7ea0a 837 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
838 ..UploadOptions::default()
839 };
840
5b32820e 841 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 842 let stats = client
e43b9175 843 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 844 .await?;
c6a7ea0a 845 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 846 }
7cc3473a 847 BackupSpecificationType::PXAR => {
5d85847f
DC
848 // start catalog upload on first use
849 if catalog.is_none() {
c6a7ea0a 850 let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
6d233161
FG
851 catalog = Some(catalog_upload_res.catalog_writer);
852 catalog_result_rx = Some(catalog_upload_res.result);
5d85847f
DC
853 }
854 let catalog = catalog.as_ref().unwrap();
855
5b32820e 856 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
8a8a4703 857 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
77486a60
FG
858
859 let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
860 device_set: devices.clone(),
861 patterns: pattern_list.clone(),
862 entries_max: entries_max as usize,
863 skip_lost_and_found,
864 verbose,
865 };
866
e43b9175
FG
867 let upload_options = UploadOptions {
868 previous_manifest: previous_manifest.clone(),
869 compress: true,
c6a7ea0a 870 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
871 ..UploadOptions::default()
872 };
873
8a8a4703
DM
874 let stats = backup_directory(
875 &client,
876 &filename,
877 &target,
878 chunk_size_opt,
8a8a4703 879 catalog.clone(),
77486a60 880 pxar_options,
e43b9175 881 upload_options,
8a8a4703 882 ).await?;
c6a7ea0a 883 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703
DM
884 catalog.lock().unwrap().end_directory()?;
885 }
7cc3473a 886 BackupSpecificationType::IMAGE => {
8a8a4703 887 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
e43b9175
FG
888
889 let upload_options = UploadOptions {
890 previous_manifest: previous_manifest.clone(),
891 fixed_size: Some(size),
892 compress: true,
c6a7ea0a 893 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
894 };
895
8a8a4703
DM
896 let stats = backup_image(
897 &client,
e43b9175 898 &filename,
8a8a4703 899 &target,
8a8a4703 900 chunk_size_opt,
e43b9175 901 upload_options,
8a8a4703 902 ).await?;
c6a7ea0a 903 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
6af905c1
DM
904 }
905 }
8a8a4703 906 }
4818c8b6 907
8a8a4703 908 // finalize and upload catalog
5d85847f 909 if let Some(catalog) = catalog {
8a8a4703
DM
910 let mutex = Arc::try_unwrap(catalog)
911 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
912 let mut catalog = mutex.into_inner().unwrap();
bf6e3217 913
8a8a4703 914 catalog.finish()?;
2761d6a4 915
8a8a4703 916 drop(catalog); // close upload stream
2761d6a4 917
6d233161 918 if let Some(catalog_result_rx) = catalog_result_rx {
5d85847f 919 let stats = catalog_result_rx.await??;
c6a7ea0a 920 manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
5d85847f 921 }
8a8a4703 922 }
2761d6a4 923
8a8a4703 924 if let Some(rsa_encrypted_key) = rsa_encrypted_key {
9990af30 925 let target = ENCRYPTED_KEY_BLOB_NAME;
8a8a4703 926 println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
e43b9175 927 let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
8a8a4703 928 let stats = client
e43b9175 929 .upload_blob_from_data(rsa_encrypted_key, target, options)
8a8a4703 930 .await?;
c6a7ea0a 931 manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
8a8a4703 932
8a8a4703 933 }
8a8a4703 934 // create manifest (index.json)
3638341a 935 // manifests are never encrypted, but include a signature
dfa517ad 936 let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
b53f6379 937 .map_err(|err| format_err!("unable to format manifest - {}", err))?;
3638341a 938
b53f6379 939
9688f6de 940 if verbose { println!("Upload index.json to '{}'", repo) };
e43b9175 941 let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
8a8a4703 942 client
e43b9175 943 .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
8a8a4703 944 .await?;
2c3891d1 945
8a8a4703 946 client.finish().await?;
c4ff3dce 947
6a7be83e
DM
948 let end_time = std::time::Instant::now();
949 let elapsed = end_time.duration_since(start_time);
950 println!("Duration: {:.2}s", elapsed.as_secs_f64());
3ec3ec3f 951
6a7be83e 952 println!("End Time: {}", strftime_local("%c", epoch_i64())?);
3d5c11e5 953
8a8a4703 954 Ok(Value::Null)
f98ea63d
DM
955}
956
8e6e18b7 957async fn dump_image<W: Write>(
88892ea8
DM
958 client: Arc<BackupReader>,
959 crypt_config: Option<Arc<CryptConfig>>,
14f6c9cb 960 crypt_mode: CryptMode,
88892ea8
DM
961 index: FixedIndexReader,
962 mut writer: W,
fd04ca7a 963 verbose: bool,
88892ea8
DM
964) -> Result<(), Error> {
965
966 let most_used = index.find_most_used_chunks(8);
967
14f6c9cb 968 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
88892ea8
DM
969
970 // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
971 // and thus slows down reading. Instead, directly use RemoteChunkReader
fd04ca7a
DM
972 let mut per = 0;
973 let mut bytes = 0;
974 let start_time = std::time::Instant::now();
975
88892ea8
DM
976 for pos in 0..index.index_count() {
977 let digest = index.index_digest(pos).unwrap();
8e6e18b7 978 let raw_data = chunk_reader.read_chunk(&digest).await?;
88892ea8 979 writer.write_all(&raw_data)?;
fd04ca7a
DM
980 bytes += raw_data.len();
981 if verbose {
982 let next_per = ((pos+1)*100)/index.index_count();
983 if per != next_per {
984 eprintln!("progress {}% (read {} bytes, duration {} sec)",
985 next_per, bytes, start_time.elapsed().as_secs());
986 per = next_per;
987 }
988 }
88892ea8
DM
989 }
990
fd04ca7a
DM
991 let end_time = std::time::Instant::now();
992 let elapsed = end_time.duration_since(start_time);
993 eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
994 bytes,
995 elapsed.as_secs_f64(),
996 bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
997 );
998
999
88892ea8
DM
1000 Ok(())
1001}
1002
dc155e9b 1003fn parse_archive_type(name: &str) -> (String, ArchiveType) {
2d32fe2c
TL
1004 if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1005 (name.into(), archive_type(name).unwrap())
1006 } else if name.ends_with(".pxar") {
dc155e9b
TL
1007 (format!("{}.didx", name), ArchiveType::DynamicIndex)
1008 } else if name.ends_with(".img") {
1009 (format!("{}.fidx", name), ArchiveType::FixedIndex)
1010 } else {
1011 (format!("{}.blob", name), ArchiveType::Blob)
1012 }
1013}
1014
a47a02ae
DM
1015#[api(
1016 input: {
1017 properties: {
1018 repository: {
1019 schema: REPO_URL_SCHEMA,
1020 optional: true,
1021 },
1022 snapshot: {
1023 type: String,
1024 description: "Group/Snapshot path.",
1025 },
1026 "archive-name": {
1027 description: "Backup archive name.",
1028 type: String,
1029 },
1030 target: {
1031 type: String,
90c815bf 1032 description: r###"Target directory path. Use '-' to write to standard output.
8a8a4703 1033
d1d74c43 1034We do not extract '.pxar' archives when writing to standard output.
8a8a4703 1035
a47a02ae
DM
1036"###
1037 },
1038 "allow-existing-dirs": {
1039 type: Boolean,
1040 description: "Do not fail if directories already exists.",
1041 optional: true,
1042 },
1043 keyfile: {
1044 schema: KEYFILE_SCHEMA,
1045 optional: true,
1046 },
0351f23b
WB
1047 "keyfd": {
1048 schema: KEYFD_SCHEMA,
1049 optional: true,
1050 },
24be37e3
WB
1051 "crypt-mode": {
1052 type: CryptMode,
96ee8577
WB
1053 optional: true,
1054 },
a47a02ae
DM
1055 }
1056 }
1057)]
1058/// Restore backup repository.
1059async fn restore(param: Value) -> Result<Value, Error> {
2665cef7 1060 let repo = extract_repository_from_value(&param)?;
9f912493 1061
86eda3eb
DM
1062 let verbose = param["verbose"].as_bool().unwrap_or(false);
1063
46d5aa0a
DM
1064 let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1065
d5c34d98
DM
1066 let archive_name = tools::required_string_param(&param, "archive-name")?;
1067
f3fde36b 1068 let client = connect(&repo)?;
d0a03d40 1069
d0a03d40 1070 record_repository(&repo);
d5c34d98 1071
9f912493 1072 let path = tools::required_string_param(&param, "snapshot")?;
9f912493 1073
86eda3eb 1074 let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
d6d3b353 1075 let group: BackupGroup = path.parse()?;
27c9affb 1076 api_datastore_latest_snapshot(&client, repo.store(), group).await?
d5c34d98 1077 } else {
a67f7d0a 1078 let snapshot: BackupDir = path.parse()?;
86eda3eb
DM
1079 (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1080 };
9f912493 1081
d5c34d98 1082 let target = tools::required_string_param(&param, "target")?;
bf125261 1083 let target = if target == "-" { None } else { Some(target) };
2ae7d196 1084
c6a7ea0a 1085 let crypto = crypto_parameters(&param)?;
2ae7d196 1086
c6a7ea0a 1087 let crypt_config = match crypto.enc_key {
86eda3eb 1088 None => None,
2f26b866
FG
1089 Some(ref key) => {
1090 let (key, _, _) =
ff8945fd 1091 decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
2f26b866
FG
1092 eprintln!("{}", format_key_source(&key.source, "encryption"));
1093 err
1094 })?;
86eda3eb
DM
1095 Some(Arc::new(CryptConfig::new(key)?))
1096 }
1097 };
d5c34d98 1098
296c50ba
DM
1099 let client = BackupReader::start(
1100 client,
1101 crypt_config.clone(),
1102 repo.store(),
1103 &backup_type,
1104 &backup_id,
1105 backup_time,
1106 true,
1107 ).await?;
86eda3eb 1108
48fbbfeb
FG
1109 let (archive_name, archive_type) = parse_archive_type(archive_name);
1110
2107a5ae 1111 let (manifest, backup_index_data) = client.download_manifest().await?;
02fcf372 1112
48fbbfeb
FG
1113 if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1114 eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1115 } else {
2f26b866
FG
1116 if manifest.signature.is_some() {
1117 if let Some(key) = &crypto.enc_key {
1118 eprintln!("{}", format_key_source(&key.source, "encryption"));
1119 }
1120 if let Some(config) = &crypt_config {
1121 eprintln!("Fingerprint: {}", config.fingerprint());
1122 }
1123 }
48fbbfeb
FG
1124 manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1125 }
dc155e9b
TL
1126
1127 if archive_name == MANIFEST_BLOB_NAME {
02fcf372 1128 if let Some(target) = target {
2107a5ae 1129 replace_file(target, &backup_index_data, CreateOptions::new())?;
02fcf372
DM
1130 } else {
1131 let stdout = std::io::stdout();
1132 let mut writer = stdout.lock();
2107a5ae 1133 writer.write_all(&backup_index_data)
02fcf372
DM
1134 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1135 }
1136
14f6c9cb
FG
1137 return Ok(Value::Null);
1138 }
1139
1140 let file_info = manifest.lookup_file_info(&archive_name)?;
1141
1142 if archive_type == ArchiveType::Blob {
d2267b11 1143
dc155e9b 1144 let mut reader = client.download_blob(&manifest, &archive_name).await?;
f8100e96 1145
bf125261 1146 if let Some(target) = target {
0d986280
DM
1147 let mut writer = std::fs::OpenOptions::new()
1148 .write(true)
1149 .create(true)
1150 .create_new(true)
1151 .open(target)
1152 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1153 std::io::copy(&mut reader, &mut writer)?;
bf125261
DM
1154 } else {
1155 let stdout = std::io::stdout();
1156 let mut writer = stdout.lock();
0d986280 1157 std::io::copy(&mut reader, &mut writer)
bf125261
DM
1158 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1159 }
f8100e96 1160
dc155e9b 1161 } else if archive_type == ArchiveType::DynamicIndex {
86eda3eb 1162
dc155e9b 1163 let index = client.download_dynamic_index(&manifest, &archive_name).await?;
df65bd3d 1164
f4bf7dfc
DM
1165 let most_used = index.find_most_used_chunks(8);
1166
14f6c9cb 1167 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
f4bf7dfc 1168
afb4cd28 1169 let mut reader = BufferedDynamicReader::new(index, chunk_reader);
86eda3eb 1170
72064fd0
FG
1171 let options = proxmox_backup::pxar::PxarExtractOptions {
1172 match_list: &[],
1173 extract_match_default: true,
1174 allow_existing_dirs,
1175 on_error: None,
1176 };
1177
bf125261 1178 if let Some(target) = target {
c443f58b
WB
1179 proxmox_backup::pxar::extract_archive(
1180 pxar::decoder::Decoder::from_std(reader)?,
1181 Path::new(target),
5444fa94 1182 proxmox_backup::pxar::Flags::DEFAULT,
c443f58b
WB
1183 |path| {
1184 if verbose {
1185 println!("{:?}", path);
1186 }
1187 },
72064fd0 1188 options,
c443f58b
WB
1189 )
1190 .map_err(|err| format_err!("error extracting archive - {}", err))?;
bf125261 1191 } else {
88892ea8
DM
1192 let mut writer = std::fs::OpenOptions::new()
1193 .write(true)
1194 .open("/dev/stdout")
1195 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
afb4cd28 1196
bf125261
DM
1197 std::io::copy(&mut reader, &mut writer)
1198 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1199 }
dc155e9b 1200 } else if archive_type == ArchiveType::FixedIndex {
afb4cd28 1201
dc155e9b 1202 let index = client.download_fixed_index(&manifest, &archive_name).await?;
df65bd3d 1203
88892ea8
DM
1204 let mut writer = if let Some(target) = target {
1205 std::fs::OpenOptions::new()
bf125261
DM
1206 .write(true)
1207 .create(true)
1208 .create_new(true)
1209 .open(target)
88892ea8 1210 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
bf125261 1211 } else {
88892ea8
DM
1212 std::fs::OpenOptions::new()
1213 .write(true)
1214 .open("/dev/stdout")
1215 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1216 };
afb4cd28 1217
14f6c9cb 1218 dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
3031e44c 1219 }
fef44d4f
DM
1220
1221 Ok(Value::Null)
45db6f89
DM
1222}
1223
e0665a64
DC
1224#[api(
1225 input: {
1226 properties: {
1227 "dry-run": {
1228 type: bool,
1229 optional: true,
1230 description: "Just show what prune would do, but do not delete anything.",
1231 },
1232 group: {
1233 type: String,
1234 description: "Backup group",
1235 },
1236 "prune-options": {
1237 type: PruneOptions,
1238 flatten: true,
1239 },
1240 "output-format": {
1241 schema: OUTPUT_FORMAT,
1242 optional: true,
1243 },
1244 quiet: {
1245 type: bool,
1246 optional: true,
1247 default: false,
1248 description: "Minimal output - only show removals.",
1249 },
1250 repository: {
1251 schema: REPO_URL_SCHEMA,
1252 optional: true,
1253 },
1254 },
1255 },
1256)]
1257/// Prune a backup repository.
1258async fn prune(
1259 dry_run: Option<bool>,
1260 group: String,
1261 prune_options: PruneOptions,
1262 quiet: bool,
1263 mut param: Value
1264) -> Result<Value, Error> {
2665cef7 1265 let repo = extract_repository_from_value(&param)?;
83b7db02 1266
f3fde36b 1267 let mut client = connect(&repo)?;
83b7db02 1268
d0a03d40 1269 let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
83b7db02 1270
d6d3b353 1271 let group: BackupGroup = group.parse()?;
c2043614 1272
671c6a96 1273 let output_format = extract_output_format(&mut param);
9fdc3ef4 1274
e0665a64
DC
1275 let mut api_param = serde_json::to_value(prune_options)?;
1276 if let Some(dry_run) = dry_run {
1277 api_param["dry-run"] = dry_run.into();
1278 }
1279 api_param["backup-type"] = group.backup_type().into();
1280 api_param["backup-id"] = group.backup_id().into();
83b7db02 1281
e0665a64 1282 let mut result = client.post(&path, Some(api_param)).await?;
74fa81b8 1283
87c42375 1284 record_repository(&repo);
3b03abfe 1285
db1e061d
DM
1286 let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1287 let item: PruneListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 1288 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
db1e061d
DM
1289 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1290 };
1291
c48aa39f
DM
1292 let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1293 Ok(match v.as_bool() {
1294 Some(true) => "keep",
1295 Some(false) => "remove",
1296 None => "unknown",
1297 }.to_string())
1298 };
1299
db1e061d
DM
1300 let options = default_table_format_options()
1301 .sortby("backup-type", false)
1302 .sortby("backup-id", false)
1303 .sortby("backup-time", false)
1304 .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
770a36e5 1305 .column(ColumnConfig::new("backup-time").renderer(pbs_tools::format::render_epoch).header("date"))
c48aa39f 1306 .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
db1e061d
DM
1307 ;
1308
b2362a12 1309 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_PRUNE.returns;
db1e061d
DM
1310
1311 let mut data = result["data"].take();
1312
c48aa39f
DM
1313 if quiet {
1314 let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1315 item["keep"].as_bool() == Some(false)
a375df6f 1316 }).cloned().collect();
c48aa39f
DM
1317 data = list.into();
1318 }
1319
b2362a12 1320 format_and_print_result_full(&mut data, return_type, &output_format, &options);
d0a03d40 1321
43a406fd 1322 Ok(Value::Null)
83b7db02
DM
1323}
1324
a47a02ae
DM
1325#[api(
1326 input: {
1327 properties: {
1328 repository: {
1329 schema: REPO_URL_SCHEMA,
1330 optional: true,
1331 },
1332 "output-format": {
1333 schema: OUTPUT_FORMAT,
1334 optional: true,
1335 },
1336 }
f9beae9c
TL
1337 },
1338 returns: {
1339 type: StorageStatus,
1340 },
a47a02ae
DM
1341)]
1342/// Get repository status.
1343async fn status(param: Value) -> Result<Value, Error> {
34a816cc
DM
1344
1345 let repo = extract_repository_from_value(&param)?;
1346
c2043614 1347 let output_format = get_output_format(&param);
34a816cc 1348
f3fde36b 1349 let client = connect(&repo)?;
34a816cc
DM
1350
1351 let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1352
1dc117bb 1353 let mut result = client.get(&path, None).await?;
14e08625 1354 let mut data = result["data"].take();
34a816cc
DM
1355
1356 record_repository(&repo);
1357
390c5bdd
DM
1358 let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1359 let v = v.as_u64().unwrap();
1360 let total = record["total"].as_u64().unwrap();
1361 let roundup = total/200;
1362 let per = ((v+roundup)*100)/total;
e23f5863
DM
1363 let info = format!(" ({} %)", per);
1364 Ok(format!("{} {:>8}", v, info))
390c5bdd 1365 };
1dc117bb 1366
c2043614 1367 let options = default_table_format_options()
be2425ff 1368 .noheader(true)
e23f5863 1369 .column(ColumnConfig::new("total").renderer(render_total_percentage))
390c5bdd
DM
1370 .column(ColumnConfig::new("used").renderer(render_total_percentage))
1371 .column(ColumnConfig::new("avail").renderer(render_total_percentage));
34a816cc 1372
b2362a12 1373 let return_type = &API_METHOD_STATUS.returns;
390c5bdd 1374
b2362a12 1375 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc
DM
1376
1377 Ok(Value::Null)
1378}
1379
c443f58b
WB
1380use proxmox_backup::client::RemoteChunkReader;
1381/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1382/// async use!
1383///
1384/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1385/// so that we can properly access it from multiple threads simultaneously while not issuing
1386/// duplicate simultaneous reads over http.
43abba4b 1387pub struct BufferedDynamicReadAt {
c443f58b
WB
1388 inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1389}
1390
1391impl BufferedDynamicReadAt {
1392 fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
1393 Self {
1394 inner: Mutex::new(inner),
1395 }
1396 }
1397}
1398
a6f87283
WB
1399impl ReadAt for BufferedDynamicReadAt {
1400 fn start_read_at<'a>(
1401 self: Pin<&'a Self>,
c443f58b 1402 _cx: &mut Context,
a6f87283 1403 buf: &'a mut [u8],
c443f58b 1404 offset: u64,
a6f87283 1405 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
a6f87283 1406 MaybeReady::Ready(tokio::task::block_in_place(move || {
c443f58b
WB
1407 let mut reader = self.inner.lock().unwrap();
1408 reader.seek(SeekFrom::Start(offset))?;
a6f87283
WB
1409 Ok(reader.read(buf)?)
1410 }))
1411 }
1412
1413 fn poll_complete<'a>(
1414 self: Pin<&'a Self>,
1415 _op: ReadAtOperation<'a>,
1416 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
1417 panic!("LocalDynamicReadAt::start_read_at returned Pending");
c443f58b
WB
1418 }
1419}
1420
f2401311 1421fn main() {
33d64b81 1422
255f378a 1423 let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
49fddd98 1424 .arg_param(&["backupspec"])
d0a03d40 1425 .completion_cb("repository", complete_repository)
49811347 1426 .completion_cb("backupspec", complete_backup_source)
6d0983db 1427 .completion_cb("keyfile", tools::complete_file_name)
c0a87c12 1428 .completion_cb("master-pubkey-file", tools::complete_file_name)
49811347 1429 .completion_cb("chunk-size", complete_chunk_size);
f8838fe9 1430
caea8d61
DM
1431 let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
1432 .completion_cb("repository", complete_repository)
1433 .completion_cb("keyfile", tools::complete_file_name);
1434
255f378a 1435 let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
d0a03d40 1436 .completion_cb("repository", complete_repository);
41c039e1 1437
255f378a 1438 let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
d0a03d40 1439 .completion_cb("repository", complete_repository);
8cc0d6af 1440
255f378a 1441 let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
49fddd98 1442 .arg_param(&["snapshot", "archive-name", "target"])
b2388518 1443 .completion_cb("repository", complete_repository)
08dc340a
DM
1444 .completion_cb("snapshot", complete_group_or_snapshot)
1445 .completion_cb("archive-name", complete_archive_name)
1446 .completion_cb("target", tools::complete_file_name);
9f912493 1447
255f378a 1448 let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
49fddd98 1449 .arg_param(&["group"])
9fdc3ef4 1450 .completion_cb("group", complete_backup_group)
d0a03d40 1451 .completion_cb("repository", complete_repository);
9f912493 1452
255f378a 1453 let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
34a816cc
DM
1454 .completion_cb("repository", complete_repository);
1455
255f378a 1456 let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
e240d8be
DM
1457 .completion_cb("repository", complete_repository);
1458
255f378a 1459 let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
e240d8be 1460 .completion_cb("repository", complete_repository);
32efac1c 1461
e39974af
TL
1462 let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
1463 .completion_cb("repository", complete_repository);
1464
344add38
DW
1465 let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
1466 .arg_param(&["group", "new-owner"])
1467 .completion_cb("group", complete_backup_group)
0224c3c2 1468 .completion_cb("new-owner", complete_auth_id)
344add38
DW
1469 .completion_cb("repository", complete_repository);
1470
41c039e1 1471 let cmd_def = CliCommandMap::new()
48ef3c33 1472 .insert("backup", backup_cmd_def)
48ef3c33
DM
1473 .insert("garbage-collect", garbage_collect_cmd_def)
1474 .insert("list", list_cmd_def)
1475 .insert("login", login_cmd_def)
1476 .insert("logout", logout_cmd_def)
1477 .insert("prune", prune_cmd_def)
1478 .insert("restore", restore_cmd_def)
a65e3e4b 1479 .insert("snapshot", snapshot_mgtm_cli())
48ef3c33 1480 .insert("status", status_cmd_def)
9696f519 1481 .insert("key", key::cli())
43abba4b 1482 .insert("mount", mount_cmd_def())
45f9b32e
SR
1483 .insert("map", map_cmd_def())
1484 .insert("unmap", unmap_cmd_def())
5830c205 1485 .insert("catalog", catalog_mgmt_cli())
caea8d61 1486 .insert("task", task_mgmt_cli())
e39974af 1487 .insert("version", version_cmd_def)
344add38 1488 .insert("benchmark", benchmark_cmd_def)
731eeef2
DM
1489 .insert("change-owner", change_owner_cmd_def)
1490
61205f00 1491 .alias(&["files"], &["snapshot", "files"])
edebd523 1492 .alias(&["forget"], &["snapshot", "forget"])
0c9209b0 1493 .alias(&["upload-log"], &["snapshot", "upload-log"])
731eeef2
DM
1494 .alias(&["snapshots"], &["snapshot", "list"])
1495 ;
48ef3c33 1496
7b22acd0
DM
1497 let rpcenv = CliEnvironment::new();
1498 run_cli_command(cmd_def, rpcenv, Some(|future| {
d420962f 1499 pbs_runtime::main(future)
d08bc483 1500 }));
ff5d3707 1501}