]> git.proxmox.com Git - proxmox-backup.git/blame - src/bin/proxmox-backup-client.rs
move build.rs and friends to pbs-buildcfg
[proxmox-backup.git] / src / bin / proxmox-backup-client.rs
CommitLineData
f1a83e97 1use std::collections::HashSet;
0351f23b 2use std::io::{self, Read, Write, Seek, SeekFrom};
c443f58b
WB
3use std::path::{Path, PathBuf};
4use std::pin::Pin;
5use std::sync::{Arc, Mutex};
a6f87283 6use std::task::Context;
c443f58b
WB
7
8use anyhow::{bail, format_err, Error};
c443f58b 9use futures::stream::{StreamExt, TryStreamExt};
c443f58b 10use serde_json::{json, Value};
c443f58b 11use tokio::sync::mpsc;
7c667013 12use tokio_stream::wrappers::ReceiverStream;
c443f58b 13use xdg::BaseDirectories;
2761d6a4 14
c443f58b 15use pathpatterns::{MatchEntry, MatchType, PatternFlag};
6a7be83e
DM
16use proxmox::{
17 tools::{
18 time::{strftime_local, epoch_i64},
ff8945fd 19 fs::{file_get_json, replace_file, CreateOptions, image_size},
6a7be83e
DM
20 },
21 api::{
22 api,
6a7be83e
DM
23 ApiMethod,
24 RpcEnvironment,
6a7be83e
DM
25 cli::*,
26 },
27};
a6f87283 28use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
ff5d3707 29
2b7f8dd5
WB
30use pbs_api_types::CryptMode;
31use pbs_client::{
32 BACKUP_SOURCE_SCHEMA,
33 BackupReader,
34 BackupRepository,
35 BackupSpecificationType,
36 BackupStats,
37 BackupWriter,
38 HttpClient,
39 PxarBackupStream,
40 RemoteChunkReader,
41 UploadOptions,
42 delete_ticket_info,
43 parse_backup_specification,
44 view_task_result,
45};
46use pbs_client::catalog_shell::Shell;
47use pbs_client::tools::{
48 complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
49 complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
50 complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
51 extract_repository_from_value,
52 key_source::{
53 crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
54 KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
55 },
56 CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
57};
58use pbs_datastore::CryptConfig;
59use pbs_datastore::backup_info::{BackupDir, BackupGroup};
f323e906 60use pbs_datastore::catalog::BackupCatalogWriter;
2b7f8dd5
WB
61use pbs_datastore::dynamic_index::DynamicIndexReader;
62use pbs_datastore::fixed_index::FixedIndexReader;
63use pbs_datastore::index::IndexFile;
64use pbs_datastore::manifest::{MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type};
65use pbs_datastore::read_chunk::AsyncReadChunk;
4805edc4
WB
66use pbs_tools::sync::StdChannelWriter;
67use pbs_tools::tokio::TokioWriterAdapter;
f323e906 68
bbf9e7e9 69use proxmox_backup::api2::types::*;
e39974af 70use proxmox_backup::api2::version;
4d16badf 71use proxmox_backup::backup::{
0351f23b 72 decrypt_key,
8acfd15d 73 rsa_encrypt_key_config,
4d16badf 74 verify_chunk_size,
4d16badf 75 BufferedDynamicReader,
f28d9088 76 CATALOG_NAME,
4d16badf
WB
77 CatalogReader,
78 CatalogWriter,
4d16badf 79 ChunkStream,
9990af30 80 ENCRYPTED_KEY_BLOB_NAME,
4d16badf 81 FixedChunkStream,
8acfd15d 82 KeyConfig,
e0665a64 83 PruneOptions,
4d16badf 84};
4805edc4 85use proxmox_backup::tools;
ae0be2dd 86
caea8d61
DM
87mod proxmox_backup_client;
88use proxmox_backup_client::*;
89
d0a03d40
DM
90fn record_repository(repo: &BackupRepository) {
91
92 let base = match BaseDirectories::with_prefix("proxmox-backup") {
93 Ok(v) => v,
94 _ => return,
95 };
96
97 // usually $HOME/.cache/proxmox-backup/repo-list
98 let path = match base.place_cache_file("repo-list") {
99 Ok(v) => v,
100 _ => return,
101 };
102
11377a47 103 let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
d0a03d40
DM
104
105 let repo = repo.to_string();
106
107 data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
108
109 let mut map = serde_json::map::Map::new();
110
111 loop {
112 let mut max_used = 0;
113 let mut max_repo = None;
114 for (repo, count) in data.as_object().unwrap() {
115 if map.contains_key(repo) { continue; }
116 if let Some(count) = count.as_i64() {
117 if count > max_used {
118 max_used = count;
119 max_repo = Some(repo);
120 }
121 }
122 }
123 if let Some(repo) = max_repo {
124 map.insert(repo.to_owned(), json!(max_used));
125 } else {
126 break;
127 }
128 if map.len() > 10 { // store max. 10 repos
129 break;
130 }
131 }
132
133 let new_data = json!(map);
134
feaa1ad3 135 let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
d0a03d40
DM
136}
137
42af4b8f
DM
138async fn api_datastore_list_snapshots(
139 client: &HttpClient,
140 store: &str,
141 group: Option<BackupGroup>,
f24fc116 142) -> Result<Value, Error> {
42af4b8f
DM
143
144 let path = format!("api2/json/admin/datastore/{}/snapshots", store);
145
146 let mut args = json!({});
147 if let Some(group) = group {
148 args["backup-type"] = group.backup_type().into();
149 args["backup-id"] = group.backup_id().into();
150 }
151
152 let mut result = client.get(&path, Some(args)).await?;
153
f24fc116 154 Ok(result["data"].take())
42af4b8f
DM
155}
156
43abba4b 157pub async fn api_datastore_latest_snapshot(
27c9affb
DM
158 client: &HttpClient,
159 store: &str,
160 group: BackupGroup,
6a7be83e 161) -> Result<(String, String, i64), Error> {
27c9affb 162
f24fc116
DM
163 let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
164 let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
27c9affb
DM
165
166 if list.is_empty() {
167 bail!("backup group {:?} does not contain any snapshots.", group.group_path());
168 }
169
170 list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
171
6a7be83e 172 let backup_time = list[0].backup_time;
27c9affb
DM
173
174 Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
175}
176
e9722f8b 177async fn backup_directory<P: AsRef<Path>>(
cf9271e2 178 client: &BackupWriter,
17d6979a 179 dir_path: P,
247cdbce 180 archive_name: &str,
36898ffc 181 chunk_size: Option<usize>,
f1d76ecf 182 catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
2b7f8dd5 183 pxar_create_options: pbs_client::pxar::PxarCreateOptions,
e43b9175 184 upload_options: UploadOptions,
2c3891d1 185) -> Result<BackupStats, Error> {
33d64b81 186
6fc053ed
CE
187 let pxar_stream = PxarBackupStream::open(
188 dir_path.as_ref(),
6fc053ed 189 catalog,
77486a60 190 pxar_create_options,
6fc053ed 191 )?;
e9722f8b 192 let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
ff3d3100 193
0bfcea6a 194 let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
5e7a09be 195
7c667013 196 let stream = ReceiverStream::new(rx)
e9722f8b 197 .map_err(Error::from);
17d6979a 198
c4ff3dce 199 // spawn chunker inside a separate task so that it can run parallel
e9722f8b 200 tokio::spawn(async move {
db0cb9ce
WB
201 while let Some(v) = chunk_stream.next().await {
202 let _ = tx.send(v).await;
203 }
e9722f8b 204 });
17d6979a 205
e43b9175
FG
206 if upload_options.fixed_size.is_some() {
207 bail!("cannot backup directory with fixed chunk size!");
208 }
209
e9722f8b 210 let stats = client
e43b9175 211 .upload_stream(archive_name, stream, upload_options)
e9722f8b 212 .await?;
bcd879cf 213
2c3891d1 214 Ok(stats)
bcd879cf
DM
215}
216
e9722f8b 217async fn backup_image<P: AsRef<Path>>(
cf9271e2 218 client: &BackupWriter,
6af905c1
DM
219 image_path: P,
220 archive_name: &str,
36898ffc 221 chunk_size: Option<usize>,
e43b9175 222 upload_options: UploadOptions,
2c3891d1 223) -> Result<BackupStats, Error> {
6af905c1 224
6af905c1
DM
225 let path = image_path.as_ref().to_owned();
226
e9722f8b 227 let file = tokio::fs::File::open(path).await?;
6af905c1 228
db0cb9ce 229 let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
6af905c1
DM
230 .map_err(Error::from);
231
36898ffc 232 let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
6af905c1 233
e43b9175
FG
234 if upload_options.fixed_size.is_none() {
235 bail!("cannot backup image with dynamic chunk size!");
236 }
237
e9722f8b 238 let stats = client
e43b9175 239 .upload_stream(archive_name, stream, upload_options)
e9722f8b 240 .await?;
6af905c1 241
2c3891d1 242 Ok(stats)
6af905c1
DM
243}
244
a47a02ae
DM
245#[api(
246 input: {
247 properties: {
248 repository: {
249 schema: REPO_URL_SCHEMA,
250 optional: true,
251 },
252 "output-format": {
253 schema: OUTPUT_FORMAT,
254 optional: true,
255 },
256 }
257 }
258)]
259/// List backup groups.
260async fn list_backup_groups(param: Value) -> Result<Value, Error> {
812c6f87 261
c81b2b7c
DM
262 let output_format = get_output_format(&param);
263
2665cef7 264 let repo = extract_repository_from_value(&param)?;
812c6f87 265
f3fde36b 266 let client = connect(&repo)?;
812c6f87 267
d0a03d40 268 let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
812c6f87 269
8a8a4703 270 let mut result = client.get(&path, None).await?;
812c6f87 271
d0a03d40
DM
272 record_repository(&repo);
273
c81b2b7c
DM
274 let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
275 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
276 let group = BackupGroup::new(item.backup_type, item.backup_id);
277 Ok(group.group_path().to_str().unwrap().to_owned())
278 };
812c6f87 279
18deda40
DM
280 let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
281 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 282 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
18deda40 283 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
c81b2b7c 284 };
812c6f87 285
c81b2b7c
DM
286 let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
287 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
770a36e5 288 Ok(pbs_tools::format::render_backup_file_list(&item.files))
c81b2b7c 289 };
812c6f87 290
c81b2b7c
DM
291 let options = default_table_format_options()
292 .sortby("backup-type", false)
293 .sortby("backup-id", false)
294 .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
18deda40
DM
295 .column(
296 ColumnConfig::new("last-backup")
297 .renderer(render_last_backup)
298 .header("last snapshot")
299 .right_align(false)
300 )
c81b2b7c
DM
301 .column(ColumnConfig::new("backup-count"))
302 .column(ColumnConfig::new("files").renderer(render_files));
ad20d198 303
c81b2b7c 304 let mut data: Value = result["data"].take();
ad20d198 305
b2362a12 306 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_LIST_GROUPS.returns;
812c6f87 307
b2362a12 308 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc 309
812c6f87
DM
310 Ok(Value::Null)
311}
312
344add38
DW
313#[api(
314 input: {
315 properties: {
316 repository: {
317 schema: REPO_URL_SCHEMA,
318 optional: true,
319 },
320 group: {
321 type: String,
322 description: "Backup group.",
323 },
324 "new-owner": {
e6dc35ac 325 type: Authid,
344add38
DW
326 },
327 }
328 }
329)]
330/// Change owner of a backup group
331async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
332
333 let repo = extract_repository_from_value(&param)?;
334
f3fde36b 335 let mut client = connect(&repo)?;
344add38
DW
336
337 param.as_object_mut().unwrap().remove("repository");
338
339 let group: BackupGroup = group.parse()?;
340
341 param["backup-type"] = group.backup_type().into();
342 param["backup-id"] = group.backup_id().into();
343
344 let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
345 client.post(&path, Some(param)).await?;
346
347 record_repository(&repo);
348
349 Ok(())
350}
351
a47a02ae
DM
352#[api(
353 input: {
354 properties: {
355 repository: {
356 schema: REPO_URL_SCHEMA,
357 optional: true,
358 },
359 }
360 }
361)]
362/// Try to login. If successful, store ticket.
363async fn api_login(param: Value) -> Result<Value, Error> {
e240d8be
DM
364
365 let repo = extract_repository_from_value(&param)?;
366
f3fde36b 367 let client = connect(&repo)?;
8a8a4703 368 client.login().await?;
e240d8be
DM
369
370 record_repository(&repo);
371
372 Ok(Value::Null)
373}
374
a47a02ae
DM
375#[api(
376 input: {
377 properties: {
378 repository: {
379 schema: REPO_URL_SCHEMA,
380 optional: true,
381 },
382 }
383 }
384)]
385/// Logout (delete stored ticket).
386fn api_logout(param: Value) -> Result<Value, Error> {
e240d8be
DM
387
388 let repo = extract_repository_from_value(&param)?;
389
5030b7ce 390 delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
e240d8be
DM
391
392 Ok(Value::Null)
393}
394
e39974af
TL
395#[api(
396 input: {
397 properties: {
398 repository: {
399 schema: REPO_URL_SCHEMA,
400 optional: true,
401 },
402 "output-format": {
403 schema: OUTPUT_FORMAT,
404 optional: true,
405 },
406 }
407 }
408)]
409/// Show client and optional server version
410async fn api_version(param: Value) -> Result<(), Error> {
411
412 let output_format = get_output_format(&param);
413
414 let mut version_info = json!({
415 "client": {
a12b1be7
WB
416 "version": pbs_buildcfg::PROXMOX_PKG_VERSION,
417 "release": pbs_buildcfg::PROXMOX_PKG_RELEASE,
418 "repoid": pbs_buildcfg::PROXMOX_PKG_REPOID,
e39974af
TL
419 }
420 });
421
422 let repo = extract_repository_from_value(&param);
423 if let Ok(repo) = repo {
f3fde36b 424 let client = connect(&repo)?;
e39974af
TL
425
426 match client.get("api2/json/version", None).await {
427 Ok(mut result) => version_info["server"] = result["data"].take(),
428 Err(e) => eprintln!("could not connect to server - {}", e),
429 }
430 }
431 if output_format == "text" {
a12b1be7
WB
432 println!(
433 "client version: {}.{}",
434 pbs_buildcfg::PROXMOX_PKG_VERSION,
435 pbs_buildcfg::PROXMOX_PKG_RELEASE,
436 );
e39974af
TL
437 if let Some(server) = version_info["server"].as_object() {
438 let server_version = server["version"].as_str().unwrap();
439 let server_release = server["release"].as_str().unwrap();
440 println!("server version: {}.{}", server_version, server_release);
441 }
442 } else {
443 format_and_print_result(&version_info, &output_format);
444 }
445
446 Ok(())
447}
448
a47a02ae 449#[api(
94913f35 450 input: {
a47a02ae
DM
451 properties: {
452 repository: {
453 schema: REPO_URL_SCHEMA,
454 optional: true,
455 },
94913f35
DM
456 "output-format": {
457 schema: OUTPUT_FORMAT,
458 optional: true,
459 },
460 },
461 },
a47a02ae
DM
462)]
463/// Start garbage collection for a specific repository.
464async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
8cc0d6af 465
2665cef7 466 let repo = extract_repository_from_value(&param)?;
c2043614
DM
467
468 let output_format = get_output_format(&param);
8cc0d6af 469
f3fde36b 470 let mut client = connect(&repo)?;
8cc0d6af 471
d0a03d40 472 let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
8cc0d6af 473
8a8a4703 474 let result = client.post(&path, None).await?;
8cc0d6af 475
8a8a4703 476 record_repository(&repo);
d0a03d40 477
e68269fc 478 view_task_result(&mut client, result, &output_format).await?;
e5f7def4 479
e5f7def4 480 Ok(Value::Null)
8cc0d6af 481}
33d64b81 482
6d233161 483struct CatalogUploadResult {
f1d76ecf 484 catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
6d233161
FG
485 result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
486}
487
bf6e3217 488fn spawn_catalog_upload(
3bad3e6e 489 client: Arc<BackupWriter>,
3638341a 490 encrypt: bool,
6d233161 491) -> Result<CatalogUploadResult, Error> {
f1d99e3f
DM
492 let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
493 let catalog_stream = crate::tools::StdChannelStream(catalog_rx);
bf6e3217
DM
494 let catalog_chunk_size = 512*1024;
495 let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
496
f1d76ecf 497 let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)))?));
bf6e3217
DM
498
499 let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
500
e43b9175
FG
501 let upload_options = UploadOptions {
502 encrypt,
503 compress: true,
504 ..UploadOptions::default()
505 };
506
bf6e3217
DM
507 tokio::spawn(async move {
508 let catalog_upload_result = client
e43b9175 509 .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
bf6e3217
DM
510 .await;
511
512 if let Err(ref err) = catalog_upload_result {
513 eprintln!("catalog upload error - {}", err);
514 client.cancel();
515 }
516
517 let _ = catalog_result_tx.send(catalog_upload_result);
518 });
519
6d233161 520 Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
bf6e3217
DM
521}
522
a47a02ae
DM
523#[api(
524 input: {
525 properties: {
526 backupspec: {
527 type: Array,
528 description: "List of backup source specifications ([<label.ext>:<path>] ...)",
529 items: {
530 schema: BACKUP_SOURCE_SCHEMA,
531 }
532 },
533 repository: {
534 schema: REPO_URL_SCHEMA,
535 optional: true,
536 },
537 "include-dev": {
538 description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
539 optional: true,
540 items: {
541 type: String,
542 description: "Path to file.",
543 }
544 },
58fcbf5a
FE
545 "all-file-systems": {
546 type: Boolean,
547 description: "Include all mounted subdirectories.",
548 optional: true,
549 },
a47a02ae
DM
550 keyfile: {
551 schema: KEYFILE_SCHEMA,
552 optional: true,
553 },
0351f23b
WB
554 "keyfd": {
555 schema: KEYFD_SCHEMA,
556 optional: true,
557 },
c0a87c12
FG
558 "master-pubkey-file": {
559 schema: MASTER_PUBKEY_FILE_SCHEMA,
560 optional: true,
561 },
562 "master-pubkey-fd": {
563 schema: MASTER_PUBKEY_FD_SCHEMA,
564 optional: true,
565 },
24be37e3
WB
566 "crypt-mode": {
567 type: CryptMode,
96ee8577
WB
568 optional: true,
569 },
a47a02ae
DM
570 "skip-lost-and-found": {
571 type: Boolean,
572 description: "Skip lost+found directory.",
573 optional: true,
574 },
575 "backup-type": {
576 schema: BACKUP_TYPE_SCHEMA,
577 optional: true,
578 },
579 "backup-id": {
580 schema: BACKUP_ID_SCHEMA,
581 optional: true,
582 },
583 "backup-time": {
584 schema: BACKUP_TIME_SCHEMA,
585 optional: true,
586 },
587 "chunk-size": {
588 schema: CHUNK_SIZE_SCHEMA,
589 optional: true,
590 },
189996cf
CE
591 "exclude": {
592 type: Array,
593 description: "List of paths or patterns for matching files to exclude.",
594 optional: true,
595 items: {
596 type: String,
597 description: "Path or match pattern.",
598 }
599 },
6fc053ed
CE
600 "entries-max": {
601 type: Integer,
602 description: "Max number of entries to hold in memory.",
603 optional: true,
2b7f8dd5 604 default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
6fc053ed 605 },
e02c3d46
DM
606 "verbose": {
607 type: Boolean,
608 description: "Verbose output.",
609 optional: true,
610 },
a47a02ae
DM
611 }
612 }
613)]
614/// Create (host) backup.
615async fn create_backup(
6049b71f
DM
616 param: Value,
617 _info: &ApiMethod,
dd5495d6 618 _rpcenv: &mut dyn RpcEnvironment,
6049b71f 619) -> Result<Value, Error> {
ff5d3707 620
2665cef7 621 let repo = extract_repository_from_value(&param)?;
ae0be2dd
DM
622
623 let backupspec_list = tools::required_array_param(&param, "backupspec")?;
a914a774 624
eed6db39
DM
625 let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
626
5b72c9b4
DM
627 let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
628
219ef0e6
DM
629 let verbose = param["verbose"].as_bool().unwrap_or(false);
630
ca5d0b61
DM
631 let backup_time_opt = param["backup-time"].as_i64();
632
36898ffc 633 let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
2d9d143a 634
247cdbce
DM
635 if let Some(size) = chunk_size_opt {
636 verify_chunk_size(size)?;
2d9d143a
DM
637 }
638
c6a7ea0a 639 let crypto = crypto_parameters(&param)?;
6d0983db 640
f69adc81 641 let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
fba30411 642
bbf9e7e9 643 let backup_type = param["backup-type"].as_str().unwrap_or("host");
ca5d0b61 644
2eeaacb9
DM
645 let include_dev = param["include-dev"].as_array();
646
c443f58b 647 let entries_max = param["entries-max"].as_u64()
2b7f8dd5 648 .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
6fc053ed 649
189996cf 650 let empty = Vec::new();
c443f58b
WB
651 let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
652
239e49f9 653 let mut pattern_list = Vec::with_capacity(exclude_args.len());
c443f58b
WB
654 for entry in exclude_args {
655 let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
239e49f9 656 pattern_list.push(
c443f58b
WB
657 MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
658 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
659 );
189996cf
CE
660 }
661
2eeaacb9
DM
662 let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
663
664 if let Some(include_dev) = include_dev {
665 if all_file_systems {
666 bail!("option 'all-file-systems' conflicts with option 'include-dev'");
667 }
668
669 let mut set = HashSet::new();
670 for path in include_dev {
671 let path = path.as_str().unwrap();
672 let stat = nix::sys::stat::stat(path)
673 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
674 set.insert(stat.st_dev);
675 }
676 devices = Some(set);
677 }
678
ae0be2dd 679 let mut upload_list = vec![];
f2b4b4b9 680 let mut target_set = HashSet::new();
a914a774 681
ae0be2dd 682 for backupspec in backupspec_list {
7cc3473a
DM
683 let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
684 let filename = &spec.config_string;
685 let target = &spec.archive_name;
bcd879cf 686
f2b4b4b9
SI
687 if target_set.contains(target) {
688 bail!("got target twice: '{}'", target);
689 }
690 target_set.insert(target.to_string());
691
eb1804c5
DM
692 use std::os::unix::fs::FileTypeExt;
693
3fa71727
CE
694 let metadata = std::fs::metadata(filename)
695 .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
eb1804c5 696 let file_type = metadata.file_type();
23bb8780 697
7cc3473a
DM
698 match spec.spec_type {
699 BackupSpecificationType::PXAR => {
ec8a9bb9
DM
700 if !file_type.is_dir() {
701 bail!("got unexpected file type (expected directory)");
702 }
7cc3473a 703 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
ec8a9bb9 704 }
7cc3473a 705 BackupSpecificationType::IMAGE => {
ec8a9bb9
DM
706 if !(file_type.is_file() || file_type.is_block_device()) {
707 bail!("got unexpected file type (expected file or block device)");
708 }
eb1804c5 709
e18a6c9e 710 let size = image_size(&PathBuf::from(filename))?;
23bb8780 711
ec8a9bb9 712 if size == 0 { bail!("got zero-sized file '{}'", filename); }
ae0be2dd 713
7cc3473a 714 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
ec8a9bb9 715 }
7cc3473a 716 BackupSpecificationType::CONFIG => {
ec8a9bb9
DM
717 if !file_type.is_file() {
718 bail!("got unexpected file type (expected regular file)");
719 }
7cc3473a 720 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 721 }
7cc3473a 722 BackupSpecificationType::LOGFILE => {
79679c2d
DM
723 if !file_type.is_file() {
724 bail!("got unexpected file type (expected regular file)");
725 }
7cc3473a 726 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 727 }
ae0be2dd
DM
728 }
729 }
730
22a9189e 731 let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
ae0be2dd 732
f3fde36b 733 let client = connect(&repo)?;
d0a03d40
DM
734 record_repository(&repo);
735
6a7be83e 736 println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
ca5d0b61 737
f69adc81 738 println!("Client name: {}", proxmox::tools::nodename());
ca5d0b61 739
6a7be83e 740 let start_time = std::time::Instant::now();
ca5d0b61 741
6a7be83e 742 println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
51144821 743
c6a7ea0a 744 let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
bb823140 745 None => (None, None),
2f26b866
FG
746 Some(key_with_source) => {
747 println!(
748 "{}",
749 format_key_source(&key_with_source.source, "encryption")
750 );
751
752 let (key, created, fingerprint) =
ff8945fd 753 decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
6f2626ae 754 println!("Encryption key fingerprint: {}", fingerprint);
bb823140 755
44288184 756 let crypt_config = CryptConfig::new(key)?;
bb823140 757
c0a87c12 758 match crypto.master_pubkey {
2f26b866
FG
759 Some(pem_with_source) => {
760 println!("{}", format_key_source(&pem_with_source.source, "master"));
761
762 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
82a103c8 763
1c86893d 764 let mut key_config = KeyConfig::without_password(key)?;
82a103c8 765 key_config.created = created; // keep original value
82a103c8 766
8acfd15d 767 let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
6f2626ae 768
05389a01 769 (Some(Arc::new(crypt_config)), Some(enc_key))
2f26b866 770 },
05389a01 771 _ => (Some(Arc::new(crypt_config)), None),
bb823140 772 }
6d0983db
DM
773 }
774 };
f98ac774 775
8a8a4703
DM
776 let client = BackupWriter::start(
777 client,
b957aa81 778 crypt_config.clone(),
8a8a4703
DM
779 repo.store(),
780 backup_type,
781 &backup_id,
782 backup_time,
783 verbose,
61d7b501 784 false
8a8a4703
DM
785 ).await?;
786
8b7f8d3f
FG
787 let download_previous_manifest = match client.previous_backup_time().await {
788 Ok(Some(backup_time)) => {
789 println!(
790 "Downloading previous manifest ({})",
791 strftime_local("%c", backup_time)?
792 );
793 true
794 }
795 Ok(None) => {
796 println!("No previous manifest available.");
797 false
798 }
799 Err(_) => {
800 // Fallback for outdated server, TODO remove/bubble up with 2.0
801 true
802 }
803 };
804
805 let previous_manifest = if download_previous_manifest {
806 match client.download_previous_manifest().await {
807 Ok(previous_manifest) => {
808 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
809 Ok(()) => Some(Arc::new(previous_manifest)),
810 Err(err) => {
811 println!("Couldn't re-use previous manifest - {}", err);
812 None
813 }
814 }
23f9503a 815 }
8b7f8d3f
FG
816 Err(err) => {
817 println!("Couldn't download previous manifest - {}", err);
818 None
819 }
820 }
821 } else {
822 None
b957aa81
DM
823 };
824
6a7be83e 825 let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
8a8a4703
DM
826 let mut manifest = BackupManifest::new(snapshot);
827
5d85847f 828 let mut catalog = None;
6d233161 829 let mut catalog_result_rx = None;
8a8a4703
DM
830
831 for (backup_type, filename, target, size) in upload_list {
832 match backup_type {
7cc3473a 833 BackupSpecificationType::CONFIG => {
e43b9175
FG
834 let upload_options = UploadOptions {
835 compress: true,
c6a7ea0a 836 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
837 ..UploadOptions::default()
838 };
839
5b32820e 840 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 841 let stats = client
e43b9175 842 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 843 .await?;
c6a7ea0a 844 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 845 }
7cc3473a 846 BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
e43b9175
FG
847 let upload_options = UploadOptions {
848 compress: true,
c6a7ea0a 849 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
850 ..UploadOptions::default()
851 };
852
5b32820e 853 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 854 let stats = client
e43b9175 855 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 856 .await?;
c6a7ea0a 857 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 858 }
7cc3473a 859 BackupSpecificationType::PXAR => {
5d85847f
DC
860 // start catalog upload on first use
861 if catalog.is_none() {
c6a7ea0a 862 let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
6d233161
FG
863 catalog = Some(catalog_upload_res.catalog_writer);
864 catalog_result_rx = Some(catalog_upload_res.result);
5d85847f
DC
865 }
866 let catalog = catalog.as_ref().unwrap();
867
5b32820e 868 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
8a8a4703 869 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
77486a60 870
2b7f8dd5 871 let pxar_options = pbs_client::pxar::PxarCreateOptions {
77486a60
FG
872 device_set: devices.clone(),
873 patterns: pattern_list.clone(),
874 entries_max: entries_max as usize,
875 skip_lost_and_found,
876 verbose,
877 };
878
e43b9175
FG
879 let upload_options = UploadOptions {
880 previous_manifest: previous_manifest.clone(),
881 compress: true,
c6a7ea0a 882 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
883 ..UploadOptions::default()
884 };
885
8a8a4703
DM
886 let stats = backup_directory(
887 &client,
888 &filename,
889 &target,
890 chunk_size_opt,
8a8a4703 891 catalog.clone(),
77486a60 892 pxar_options,
e43b9175 893 upload_options,
8a8a4703 894 ).await?;
c6a7ea0a 895 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703
DM
896 catalog.lock().unwrap().end_directory()?;
897 }
7cc3473a 898 BackupSpecificationType::IMAGE => {
8a8a4703 899 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
e43b9175
FG
900
901 let upload_options = UploadOptions {
902 previous_manifest: previous_manifest.clone(),
903 fixed_size: Some(size),
904 compress: true,
c6a7ea0a 905 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
906 };
907
8a8a4703
DM
908 let stats = backup_image(
909 &client,
e43b9175 910 &filename,
8a8a4703 911 &target,
8a8a4703 912 chunk_size_opt,
e43b9175 913 upload_options,
8a8a4703 914 ).await?;
c6a7ea0a 915 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
6af905c1
DM
916 }
917 }
8a8a4703 918 }
4818c8b6 919
8a8a4703 920 // finalize and upload catalog
5d85847f 921 if let Some(catalog) = catalog {
8a8a4703
DM
922 let mutex = Arc::try_unwrap(catalog)
923 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
924 let mut catalog = mutex.into_inner().unwrap();
bf6e3217 925
8a8a4703 926 catalog.finish()?;
2761d6a4 927
8a8a4703 928 drop(catalog); // close upload stream
2761d6a4 929
6d233161 930 if let Some(catalog_result_rx) = catalog_result_rx {
5d85847f 931 let stats = catalog_result_rx.await??;
c6a7ea0a 932 manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
5d85847f 933 }
8a8a4703 934 }
2761d6a4 935
8a8a4703 936 if let Some(rsa_encrypted_key) = rsa_encrypted_key {
9990af30 937 let target = ENCRYPTED_KEY_BLOB_NAME;
8a8a4703 938 println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
e43b9175 939 let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
8a8a4703 940 let stats = client
e43b9175 941 .upload_blob_from_data(rsa_encrypted_key, target, options)
8a8a4703 942 .await?;
c6a7ea0a 943 manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
8a8a4703 944
8a8a4703 945 }
8a8a4703 946 // create manifest (index.json)
3638341a 947 // manifests are never encrypted, but include a signature
dfa517ad 948 let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
b53f6379 949 .map_err(|err| format_err!("unable to format manifest - {}", err))?;
3638341a 950
b53f6379 951
9688f6de 952 if verbose { println!("Upload index.json to '{}'", repo) };
e43b9175 953 let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
8a8a4703 954 client
e43b9175 955 .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
8a8a4703 956 .await?;
2c3891d1 957
8a8a4703 958 client.finish().await?;
c4ff3dce 959
6a7be83e
DM
960 let end_time = std::time::Instant::now();
961 let elapsed = end_time.duration_since(start_time);
962 println!("Duration: {:.2}s", elapsed.as_secs_f64());
3ec3ec3f 963
6a7be83e 964 println!("End Time: {}", strftime_local("%c", epoch_i64())?);
3d5c11e5 965
8a8a4703 966 Ok(Value::Null)
f98ea63d
DM
967}
968
8e6e18b7 969async fn dump_image<W: Write>(
88892ea8
DM
970 client: Arc<BackupReader>,
971 crypt_config: Option<Arc<CryptConfig>>,
14f6c9cb 972 crypt_mode: CryptMode,
88892ea8
DM
973 index: FixedIndexReader,
974 mut writer: W,
fd04ca7a 975 verbose: bool,
88892ea8
DM
976) -> Result<(), Error> {
977
978 let most_used = index.find_most_used_chunks(8);
979
14f6c9cb 980 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
88892ea8
DM
981
982 // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
983 // and thus slows down reading. Instead, directly use RemoteChunkReader
fd04ca7a
DM
984 let mut per = 0;
985 let mut bytes = 0;
986 let start_time = std::time::Instant::now();
987
88892ea8
DM
988 for pos in 0..index.index_count() {
989 let digest = index.index_digest(pos).unwrap();
8e6e18b7 990 let raw_data = chunk_reader.read_chunk(&digest).await?;
88892ea8 991 writer.write_all(&raw_data)?;
fd04ca7a
DM
992 bytes += raw_data.len();
993 if verbose {
994 let next_per = ((pos+1)*100)/index.index_count();
995 if per != next_per {
996 eprintln!("progress {}% (read {} bytes, duration {} sec)",
997 next_per, bytes, start_time.elapsed().as_secs());
998 per = next_per;
999 }
1000 }
88892ea8
DM
1001 }
1002
fd04ca7a
DM
1003 let end_time = std::time::Instant::now();
1004 let elapsed = end_time.duration_since(start_time);
1005 eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1006 bytes,
1007 elapsed.as_secs_f64(),
1008 bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1009 );
1010
1011
88892ea8
DM
1012 Ok(())
1013}
1014
dc155e9b 1015fn parse_archive_type(name: &str) -> (String, ArchiveType) {
2d32fe2c
TL
1016 if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1017 (name.into(), archive_type(name).unwrap())
1018 } else if name.ends_with(".pxar") {
dc155e9b
TL
1019 (format!("{}.didx", name), ArchiveType::DynamicIndex)
1020 } else if name.ends_with(".img") {
1021 (format!("{}.fidx", name), ArchiveType::FixedIndex)
1022 } else {
1023 (format!("{}.blob", name), ArchiveType::Blob)
1024 }
1025}
1026
a47a02ae
DM
1027#[api(
1028 input: {
1029 properties: {
1030 repository: {
1031 schema: REPO_URL_SCHEMA,
1032 optional: true,
1033 },
1034 snapshot: {
1035 type: String,
1036 description: "Group/Snapshot path.",
1037 },
1038 "archive-name": {
1039 description: "Backup archive name.",
1040 type: String,
1041 },
1042 target: {
1043 type: String,
90c815bf 1044 description: r###"Target directory path. Use '-' to write to standard output.
8a8a4703 1045
d1d74c43 1046We do not extract '.pxar' archives when writing to standard output.
8a8a4703 1047
a47a02ae
DM
1048"###
1049 },
1050 "allow-existing-dirs": {
1051 type: Boolean,
1052 description: "Do not fail if directories already exists.",
1053 optional: true,
1054 },
1055 keyfile: {
1056 schema: KEYFILE_SCHEMA,
1057 optional: true,
1058 },
0351f23b
WB
1059 "keyfd": {
1060 schema: KEYFD_SCHEMA,
1061 optional: true,
1062 },
24be37e3
WB
1063 "crypt-mode": {
1064 type: CryptMode,
96ee8577
WB
1065 optional: true,
1066 },
a47a02ae
DM
1067 }
1068 }
1069)]
1070/// Restore backup repository.
1071async fn restore(param: Value) -> Result<Value, Error> {
2665cef7 1072 let repo = extract_repository_from_value(&param)?;
9f912493 1073
86eda3eb
DM
1074 let verbose = param["verbose"].as_bool().unwrap_or(false);
1075
46d5aa0a
DM
1076 let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1077
d5c34d98
DM
1078 let archive_name = tools::required_string_param(&param, "archive-name")?;
1079
f3fde36b 1080 let client = connect(&repo)?;
d0a03d40 1081
d0a03d40 1082 record_repository(&repo);
d5c34d98 1083
9f912493 1084 let path = tools::required_string_param(&param, "snapshot")?;
9f912493 1085
86eda3eb 1086 let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
d6d3b353 1087 let group: BackupGroup = path.parse()?;
27c9affb 1088 api_datastore_latest_snapshot(&client, repo.store(), group).await?
d5c34d98 1089 } else {
a67f7d0a 1090 let snapshot: BackupDir = path.parse()?;
86eda3eb
DM
1091 (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1092 };
9f912493 1093
d5c34d98 1094 let target = tools::required_string_param(&param, "target")?;
bf125261 1095 let target = if target == "-" { None } else { Some(target) };
2ae7d196 1096
c6a7ea0a 1097 let crypto = crypto_parameters(&param)?;
2ae7d196 1098
c6a7ea0a 1099 let crypt_config = match crypto.enc_key {
86eda3eb 1100 None => None,
2f26b866
FG
1101 Some(ref key) => {
1102 let (key, _, _) =
ff8945fd 1103 decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
2f26b866
FG
1104 eprintln!("{}", format_key_source(&key.source, "encryption"));
1105 err
1106 })?;
86eda3eb
DM
1107 Some(Arc::new(CryptConfig::new(key)?))
1108 }
1109 };
d5c34d98 1110
296c50ba
DM
1111 let client = BackupReader::start(
1112 client,
1113 crypt_config.clone(),
1114 repo.store(),
1115 &backup_type,
1116 &backup_id,
1117 backup_time,
1118 true,
1119 ).await?;
86eda3eb 1120
48fbbfeb
FG
1121 let (archive_name, archive_type) = parse_archive_type(archive_name);
1122
2107a5ae 1123 let (manifest, backup_index_data) = client.download_manifest().await?;
02fcf372 1124
48fbbfeb
FG
1125 if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1126 eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1127 } else {
2f26b866
FG
1128 if manifest.signature.is_some() {
1129 if let Some(key) = &crypto.enc_key {
1130 eprintln!("{}", format_key_source(&key.source, "encryption"));
1131 }
1132 if let Some(config) = &crypt_config {
1133 eprintln!("Fingerprint: {}", config.fingerprint());
1134 }
1135 }
48fbbfeb
FG
1136 manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1137 }
dc155e9b
TL
1138
1139 if archive_name == MANIFEST_BLOB_NAME {
02fcf372 1140 if let Some(target) = target {
2107a5ae 1141 replace_file(target, &backup_index_data, CreateOptions::new())?;
02fcf372
DM
1142 } else {
1143 let stdout = std::io::stdout();
1144 let mut writer = stdout.lock();
2107a5ae 1145 writer.write_all(&backup_index_data)
02fcf372
DM
1146 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1147 }
1148
14f6c9cb
FG
1149 return Ok(Value::Null);
1150 }
1151
1152 let file_info = manifest.lookup_file_info(&archive_name)?;
1153
1154 if archive_type == ArchiveType::Blob {
d2267b11 1155
dc155e9b 1156 let mut reader = client.download_blob(&manifest, &archive_name).await?;
f8100e96 1157
bf125261 1158 if let Some(target) = target {
0d986280
DM
1159 let mut writer = std::fs::OpenOptions::new()
1160 .write(true)
1161 .create(true)
1162 .create_new(true)
1163 .open(target)
1164 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1165 std::io::copy(&mut reader, &mut writer)?;
bf125261
DM
1166 } else {
1167 let stdout = std::io::stdout();
1168 let mut writer = stdout.lock();
0d986280 1169 std::io::copy(&mut reader, &mut writer)
bf125261
DM
1170 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1171 }
f8100e96 1172
dc155e9b 1173 } else if archive_type == ArchiveType::DynamicIndex {
86eda3eb 1174
dc155e9b 1175 let index = client.download_dynamic_index(&manifest, &archive_name).await?;
df65bd3d 1176
f4bf7dfc
DM
1177 let most_used = index.find_most_used_chunks(8);
1178
14f6c9cb 1179 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
f4bf7dfc 1180
afb4cd28 1181 let mut reader = BufferedDynamicReader::new(index, chunk_reader);
86eda3eb 1182
2b7f8dd5 1183 let options = pbs_client::pxar::PxarExtractOptions {
72064fd0
FG
1184 match_list: &[],
1185 extract_match_default: true,
1186 allow_existing_dirs,
1187 on_error: None,
1188 };
1189
bf125261 1190 if let Some(target) = target {
2b7f8dd5 1191 pbs_client::pxar::extract_archive(
c443f58b
WB
1192 pxar::decoder::Decoder::from_std(reader)?,
1193 Path::new(target),
2b7f8dd5 1194 pbs_client::pxar::Flags::DEFAULT,
c443f58b
WB
1195 |path| {
1196 if verbose {
1197 println!("{:?}", path);
1198 }
1199 },
72064fd0 1200 options,
c443f58b
WB
1201 )
1202 .map_err(|err| format_err!("error extracting archive - {}", err))?;
bf125261 1203 } else {
88892ea8
DM
1204 let mut writer = std::fs::OpenOptions::new()
1205 .write(true)
1206 .open("/dev/stdout")
1207 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
afb4cd28 1208
bf125261
DM
1209 std::io::copy(&mut reader, &mut writer)
1210 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1211 }
dc155e9b 1212 } else if archive_type == ArchiveType::FixedIndex {
afb4cd28 1213
dc155e9b 1214 let index = client.download_fixed_index(&manifest, &archive_name).await?;
df65bd3d 1215
88892ea8
DM
1216 let mut writer = if let Some(target) = target {
1217 std::fs::OpenOptions::new()
bf125261
DM
1218 .write(true)
1219 .create(true)
1220 .create_new(true)
1221 .open(target)
88892ea8 1222 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
bf125261 1223 } else {
88892ea8
DM
1224 std::fs::OpenOptions::new()
1225 .write(true)
1226 .open("/dev/stdout")
1227 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1228 };
afb4cd28 1229
14f6c9cb 1230 dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
3031e44c 1231 }
fef44d4f
DM
1232
1233 Ok(Value::Null)
45db6f89
DM
1234}
1235
e0665a64
DC
1236#[api(
1237 input: {
1238 properties: {
1239 "dry-run": {
1240 type: bool,
1241 optional: true,
1242 description: "Just show what prune would do, but do not delete anything.",
1243 },
1244 group: {
1245 type: String,
1246 description: "Backup group",
1247 },
1248 "prune-options": {
1249 type: PruneOptions,
1250 flatten: true,
1251 },
1252 "output-format": {
1253 schema: OUTPUT_FORMAT,
1254 optional: true,
1255 },
1256 quiet: {
1257 type: bool,
1258 optional: true,
1259 default: false,
1260 description: "Minimal output - only show removals.",
1261 },
1262 repository: {
1263 schema: REPO_URL_SCHEMA,
1264 optional: true,
1265 },
1266 },
1267 },
1268)]
1269/// Prune a backup repository.
1270async fn prune(
1271 dry_run: Option<bool>,
1272 group: String,
1273 prune_options: PruneOptions,
1274 quiet: bool,
1275 mut param: Value
1276) -> Result<Value, Error> {
2665cef7 1277 let repo = extract_repository_from_value(&param)?;
83b7db02 1278
f3fde36b 1279 let mut client = connect(&repo)?;
83b7db02 1280
d0a03d40 1281 let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
83b7db02 1282
d6d3b353 1283 let group: BackupGroup = group.parse()?;
c2043614 1284
671c6a96 1285 let output_format = extract_output_format(&mut param);
9fdc3ef4 1286
e0665a64
DC
1287 let mut api_param = serde_json::to_value(prune_options)?;
1288 if let Some(dry_run) = dry_run {
1289 api_param["dry-run"] = dry_run.into();
1290 }
1291 api_param["backup-type"] = group.backup_type().into();
1292 api_param["backup-id"] = group.backup_id().into();
83b7db02 1293
e0665a64 1294 let mut result = client.post(&path, Some(api_param)).await?;
74fa81b8 1295
87c42375 1296 record_repository(&repo);
3b03abfe 1297
db1e061d
DM
1298 let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1299 let item: PruneListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 1300 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
db1e061d
DM
1301 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1302 };
1303
c48aa39f
DM
1304 let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1305 Ok(match v.as_bool() {
1306 Some(true) => "keep",
1307 Some(false) => "remove",
1308 None => "unknown",
1309 }.to_string())
1310 };
1311
db1e061d
DM
1312 let options = default_table_format_options()
1313 .sortby("backup-type", false)
1314 .sortby("backup-id", false)
1315 .sortby("backup-time", false)
1316 .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
770a36e5 1317 .column(ColumnConfig::new("backup-time").renderer(pbs_tools::format::render_epoch).header("date"))
c48aa39f 1318 .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
db1e061d
DM
1319 ;
1320
b2362a12 1321 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_PRUNE.returns;
db1e061d
DM
1322
1323 let mut data = result["data"].take();
1324
c48aa39f
DM
1325 if quiet {
1326 let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1327 item["keep"].as_bool() == Some(false)
a375df6f 1328 }).cloned().collect();
c48aa39f
DM
1329 data = list.into();
1330 }
1331
b2362a12 1332 format_and_print_result_full(&mut data, return_type, &output_format, &options);
d0a03d40 1333
43a406fd 1334 Ok(Value::Null)
83b7db02
DM
1335}
1336
a47a02ae
DM
1337#[api(
1338 input: {
1339 properties: {
1340 repository: {
1341 schema: REPO_URL_SCHEMA,
1342 optional: true,
1343 },
1344 "output-format": {
1345 schema: OUTPUT_FORMAT,
1346 optional: true,
1347 },
1348 }
f9beae9c
TL
1349 },
1350 returns: {
1351 type: StorageStatus,
1352 },
a47a02ae
DM
1353)]
1354/// Get repository status.
1355async fn status(param: Value) -> Result<Value, Error> {
34a816cc
DM
1356
1357 let repo = extract_repository_from_value(&param)?;
1358
c2043614 1359 let output_format = get_output_format(&param);
34a816cc 1360
f3fde36b 1361 let client = connect(&repo)?;
34a816cc
DM
1362
1363 let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1364
1dc117bb 1365 let mut result = client.get(&path, None).await?;
14e08625 1366 let mut data = result["data"].take();
34a816cc
DM
1367
1368 record_repository(&repo);
1369
390c5bdd
DM
1370 let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1371 let v = v.as_u64().unwrap();
1372 let total = record["total"].as_u64().unwrap();
1373 let roundup = total/200;
1374 let per = ((v+roundup)*100)/total;
e23f5863
DM
1375 let info = format!(" ({} %)", per);
1376 Ok(format!("{} {:>8}", v, info))
390c5bdd 1377 };
1dc117bb 1378
c2043614 1379 let options = default_table_format_options()
be2425ff 1380 .noheader(true)
e23f5863 1381 .column(ColumnConfig::new("total").renderer(render_total_percentage))
390c5bdd
DM
1382 .column(ColumnConfig::new("used").renderer(render_total_percentage))
1383 .column(ColumnConfig::new("avail").renderer(render_total_percentage));
34a816cc 1384
b2362a12 1385 let return_type = &API_METHOD_STATUS.returns;
390c5bdd 1386
b2362a12 1387 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc
DM
1388
1389 Ok(Value::Null)
1390}
1391
c443f58b
WB
1392/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1393/// async use!
1394///
1395/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1396/// so that we can properly access it from multiple threads simultaneously while not issuing
1397/// duplicate simultaneous reads over http.
43abba4b 1398pub struct BufferedDynamicReadAt {
c443f58b
WB
1399 inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1400}
1401
1402impl BufferedDynamicReadAt {
1403 fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
1404 Self {
1405 inner: Mutex::new(inner),
1406 }
1407 }
1408}
1409
a6f87283
WB
1410impl ReadAt for BufferedDynamicReadAt {
1411 fn start_read_at<'a>(
1412 self: Pin<&'a Self>,
c443f58b 1413 _cx: &mut Context,
a6f87283 1414 buf: &'a mut [u8],
c443f58b 1415 offset: u64,
a6f87283 1416 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
a6f87283 1417 MaybeReady::Ready(tokio::task::block_in_place(move || {
c443f58b
WB
1418 let mut reader = self.inner.lock().unwrap();
1419 reader.seek(SeekFrom::Start(offset))?;
a6f87283
WB
1420 Ok(reader.read(buf)?)
1421 }))
1422 }
1423
1424 fn poll_complete<'a>(
1425 self: Pin<&'a Self>,
1426 _op: ReadAtOperation<'a>,
1427 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
1428 panic!("LocalDynamicReadAt::start_read_at returned Pending");
c443f58b
WB
1429 }
1430}
1431
f2401311 1432fn main() {
33d64b81 1433
255f378a 1434 let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
49fddd98 1435 .arg_param(&["backupspec"])
d0a03d40 1436 .completion_cb("repository", complete_repository)
49811347 1437 .completion_cb("backupspec", complete_backup_source)
2b7f8dd5
WB
1438 .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
1439 .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
49811347 1440 .completion_cb("chunk-size", complete_chunk_size);
f8838fe9 1441
caea8d61
DM
1442 let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
1443 .completion_cb("repository", complete_repository)
2b7f8dd5 1444 .completion_cb("keyfile", pbs_tools::fs::complete_file_name);
caea8d61 1445
255f378a 1446 let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
d0a03d40 1447 .completion_cb("repository", complete_repository);
41c039e1 1448
255f378a 1449 let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
d0a03d40 1450 .completion_cb("repository", complete_repository);
8cc0d6af 1451
255f378a 1452 let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
49fddd98 1453 .arg_param(&["snapshot", "archive-name", "target"])
b2388518 1454 .completion_cb("repository", complete_repository)
08dc340a
DM
1455 .completion_cb("snapshot", complete_group_or_snapshot)
1456 .completion_cb("archive-name", complete_archive_name)
2b7f8dd5 1457 .completion_cb("target", pbs_tools::fs::complete_file_name);
9f912493 1458
255f378a 1459 let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
49fddd98 1460 .arg_param(&["group"])
9fdc3ef4 1461 .completion_cb("group", complete_backup_group)
d0a03d40 1462 .completion_cb("repository", complete_repository);
9f912493 1463
255f378a 1464 let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
34a816cc
DM
1465 .completion_cb("repository", complete_repository);
1466
255f378a 1467 let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
e240d8be
DM
1468 .completion_cb("repository", complete_repository);
1469
255f378a 1470 let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
e240d8be 1471 .completion_cb("repository", complete_repository);
32efac1c 1472
e39974af
TL
1473 let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
1474 .completion_cb("repository", complete_repository);
1475
344add38
DW
1476 let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
1477 .arg_param(&["group", "new-owner"])
1478 .completion_cb("group", complete_backup_group)
0224c3c2 1479 .completion_cb("new-owner", complete_auth_id)
344add38
DW
1480 .completion_cb("repository", complete_repository);
1481
41c039e1 1482 let cmd_def = CliCommandMap::new()
48ef3c33 1483 .insert("backup", backup_cmd_def)
48ef3c33
DM
1484 .insert("garbage-collect", garbage_collect_cmd_def)
1485 .insert("list", list_cmd_def)
1486 .insert("login", login_cmd_def)
1487 .insert("logout", logout_cmd_def)
1488 .insert("prune", prune_cmd_def)
1489 .insert("restore", restore_cmd_def)
a65e3e4b 1490 .insert("snapshot", snapshot_mgtm_cli())
48ef3c33 1491 .insert("status", status_cmd_def)
9696f519 1492 .insert("key", key::cli())
43abba4b 1493 .insert("mount", mount_cmd_def())
45f9b32e
SR
1494 .insert("map", map_cmd_def())
1495 .insert("unmap", unmap_cmd_def())
5830c205 1496 .insert("catalog", catalog_mgmt_cli())
caea8d61 1497 .insert("task", task_mgmt_cli())
e39974af 1498 .insert("version", version_cmd_def)
344add38 1499 .insert("benchmark", benchmark_cmd_def)
731eeef2
DM
1500 .insert("change-owner", change_owner_cmd_def)
1501
61205f00 1502 .alias(&["files"], &["snapshot", "files"])
edebd523 1503 .alias(&["forget"], &["snapshot", "forget"])
0c9209b0 1504 .alias(&["upload-log"], &["snapshot", "upload-log"])
731eeef2
DM
1505 .alias(&["snapshots"], &["snapshot", "list"])
1506 ;
48ef3c33 1507
7b22acd0
DM
1508 let rpcenv = CliEnvironment::new();
1509 run_cli_command(cmd_def, rpcenv, Some(|future| {
d420962f 1510 pbs_runtime::main(future)
d08bc483 1511 }));
ff5d3707 1512}