]> git.proxmox.com Git - proxmox-backup.git/blame - src/bin/proxmox-backup-client.rs
move required_X_param to pbs_tools::json
[proxmox-backup.git] / src / bin / proxmox-backup-client.rs
CommitLineData
f1a83e97 1use std::collections::HashSet;
0351f23b 2use std::io::{self, Read, Write, Seek, SeekFrom};
c443f58b
WB
3use std::path::{Path, PathBuf};
4use std::pin::Pin;
5use std::sync::{Arc, Mutex};
a6f87283 6use std::task::Context;
c443f58b
WB
7
8use anyhow::{bail, format_err, Error};
c443f58b 9use futures::stream::{StreamExt, TryStreamExt};
c443f58b 10use serde_json::{json, Value};
c443f58b 11use tokio::sync::mpsc;
7c667013 12use tokio_stream::wrappers::ReceiverStream;
c443f58b 13use xdg::BaseDirectories;
2761d6a4 14
c443f58b 15use pathpatterns::{MatchEntry, MatchType, PatternFlag};
6a7be83e
DM
16use proxmox::{
17 tools::{
18 time::{strftime_local, epoch_i64},
ff8945fd 19 fs::{file_get_json, replace_file, CreateOptions, image_size},
6a7be83e
DM
20 },
21 api::{
22 api,
6a7be83e
DM
23 ApiMethod,
24 RpcEnvironment,
6a7be83e
DM
25 cli::*,
26 },
27};
a6f87283 28use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
ff5d3707 29
51ec8a3c
WB
30use pbs_api_types::{
31 BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem,
32 PruneListItem, SnapshotListItem, StorageStatus,
33};
2b7f8dd5
WB
34use pbs_client::{
35 BACKUP_SOURCE_SCHEMA,
36 BackupReader,
37 BackupRepository,
38 BackupSpecificationType,
39 BackupStats,
40 BackupWriter,
38629c39
WB
41 ChunkStream,
42 FixedChunkStream,
2b7f8dd5
WB
43 HttpClient,
44 PxarBackupStream,
45 RemoteChunkReader,
46 UploadOptions,
47 delete_ticket_info,
48 parse_backup_specification,
49 view_task_result,
50};
51use pbs_client::catalog_shell::Shell;
52use pbs_client::tools::{
53 complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
54 complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
55 complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
56 extract_repository_from_value,
57 key_source::{
58 crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
59 KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
60 },
61 CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
62};
51ec8a3c 63use pbs_datastore::{CATALOG_NAME, CryptConfig, KeyConfig, decrypt_key, rsa_encrypt_key_config};
2b7f8dd5 64use pbs_datastore::backup_info::{BackupDir, BackupGroup};
51ec8a3c
WB
65use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter};
66use pbs_datastore::chunk_store::verify_chunk_size;
2b7f8dd5
WB
67use pbs_datastore::dynamic_index::DynamicIndexReader;
68use pbs_datastore::fixed_index::FixedIndexReader;
69use pbs_datastore::index::IndexFile;
51ec8a3c
WB
70use pbs_datastore::manifest::{
71 ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type,
72};
2b7f8dd5 73use pbs_datastore::read_chunk::AsyncReadChunk;
51ec8a3c 74use pbs_datastore::prune::PruneOptions;
4805edc4
WB
75use pbs_tools::sync::StdChannelWriter;
76use pbs_tools::tokio::TokioWriterAdapter;
3c8c2827 77use pbs_tools::json;
f323e906 78
4d16badf 79use proxmox_backup::backup::{
4d16badf 80 BufferedDynamicReader,
4d16badf 81};
4805edc4 82use proxmox_backup::tools;
ae0be2dd 83
caea8d61
DM
84mod proxmox_backup_client;
85use proxmox_backup_client::*;
86
d0a03d40
DM
87fn record_repository(repo: &BackupRepository) {
88
89 let base = match BaseDirectories::with_prefix("proxmox-backup") {
90 Ok(v) => v,
91 _ => return,
92 };
93
94 // usually $HOME/.cache/proxmox-backup/repo-list
95 let path = match base.place_cache_file("repo-list") {
96 Ok(v) => v,
97 _ => return,
98 };
99
11377a47 100 let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
d0a03d40
DM
101
102 let repo = repo.to_string();
103
104 data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
105
106 let mut map = serde_json::map::Map::new();
107
108 loop {
109 let mut max_used = 0;
110 let mut max_repo = None;
111 for (repo, count) in data.as_object().unwrap() {
112 if map.contains_key(repo) { continue; }
113 if let Some(count) = count.as_i64() {
114 if count > max_used {
115 max_used = count;
116 max_repo = Some(repo);
117 }
118 }
119 }
120 if let Some(repo) = max_repo {
121 map.insert(repo.to_owned(), json!(max_used));
122 } else {
123 break;
124 }
125 if map.len() > 10 { // store max. 10 repos
126 break;
127 }
128 }
129
130 let new_data = json!(map);
131
feaa1ad3 132 let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
d0a03d40
DM
133}
134
42af4b8f
DM
135async fn api_datastore_list_snapshots(
136 client: &HttpClient,
137 store: &str,
138 group: Option<BackupGroup>,
f24fc116 139) -> Result<Value, Error> {
42af4b8f
DM
140
141 let path = format!("api2/json/admin/datastore/{}/snapshots", store);
142
143 let mut args = json!({});
144 if let Some(group) = group {
145 args["backup-type"] = group.backup_type().into();
146 args["backup-id"] = group.backup_id().into();
147 }
148
149 let mut result = client.get(&path, Some(args)).await?;
150
f24fc116 151 Ok(result["data"].take())
42af4b8f
DM
152}
153
43abba4b 154pub async fn api_datastore_latest_snapshot(
27c9affb
DM
155 client: &HttpClient,
156 store: &str,
157 group: BackupGroup,
6a7be83e 158) -> Result<(String, String, i64), Error> {
27c9affb 159
f24fc116
DM
160 let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
161 let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
27c9affb
DM
162
163 if list.is_empty() {
164 bail!("backup group {:?} does not contain any snapshots.", group.group_path());
165 }
166
167 list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
168
6a7be83e 169 let backup_time = list[0].backup_time;
27c9affb
DM
170
171 Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
172}
173
e9722f8b 174async fn backup_directory<P: AsRef<Path>>(
cf9271e2 175 client: &BackupWriter,
17d6979a 176 dir_path: P,
247cdbce 177 archive_name: &str,
36898ffc 178 chunk_size: Option<usize>,
f1d76ecf 179 catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
2b7f8dd5 180 pxar_create_options: pbs_client::pxar::PxarCreateOptions,
e43b9175 181 upload_options: UploadOptions,
2c3891d1 182) -> Result<BackupStats, Error> {
33d64b81 183
6fc053ed
CE
184 let pxar_stream = PxarBackupStream::open(
185 dir_path.as_ref(),
6fc053ed 186 catalog,
77486a60 187 pxar_create_options,
6fc053ed 188 )?;
e9722f8b 189 let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
ff3d3100 190
0bfcea6a 191 let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
5e7a09be 192
7c667013 193 let stream = ReceiverStream::new(rx)
e9722f8b 194 .map_err(Error::from);
17d6979a 195
c4ff3dce 196 // spawn chunker inside a separate task so that it can run parallel
e9722f8b 197 tokio::spawn(async move {
db0cb9ce
WB
198 while let Some(v) = chunk_stream.next().await {
199 let _ = tx.send(v).await;
200 }
e9722f8b 201 });
17d6979a 202
e43b9175
FG
203 if upload_options.fixed_size.is_some() {
204 bail!("cannot backup directory with fixed chunk size!");
205 }
206
e9722f8b 207 let stats = client
e43b9175 208 .upload_stream(archive_name, stream, upload_options)
e9722f8b 209 .await?;
bcd879cf 210
2c3891d1 211 Ok(stats)
bcd879cf
DM
212}
213
e9722f8b 214async fn backup_image<P: AsRef<Path>>(
cf9271e2 215 client: &BackupWriter,
6af905c1
DM
216 image_path: P,
217 archive_name: &str,
36898ffc 218 chunk_size: Option<usize>,
e43b9175 219 upload_options: UploadOptions,
2c3891d1 220) -> Result<BackupStats, Error> {
6af905c1 221
6af905c1
DM
222 let path = image_path.as_ref().to_owned();
223
e9722f8b 224 let file = tokio::fs::File::open(path).await?;
6af905c1 225
db0cb9ce 226 let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
6af905c1
DM
227 .map_err(Error::from);
228
36898ffc 229 let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
6af905c1 230
e43b9175
FG
231 if upload_options.fixed_size.is_none() {
232 bail!("cannot backup image with dynamic chunk size!");
233 }
234
e9722f8b 235 let stats = client
e43b9175 236 .upload_stream(archive_name, stream, upload_options)
e9722f8b 237 .await?;
6af905c1 238
2c3891d1 239 Ok(stats)
6af905c1
DM
240}
241
a47a02ae
DM
242#[api(
243 input: {
244 properties: {
245 repository: {
246 schema: REPO_URL_SCHEMA,
247 optional: true,
248 },
249 "output-format": {
250 schema: OUTPUT_FORMAT,
251 optional: true,
252 },
253 }
254 }
255)]
256/// List backup groups.
257async fn list_backup_groups(param: Value) -> Result<Value, Error> {
812c6f87 258
c81b2b7c
DM
259 let output_format = get_output_format(&param);
260
2665cef7 261 let repo = extract_repository_from_value(&param)?;
812c6f87 262
f3fde36b 263 let client = connect(&repo)?;
812c6f87 264
d0a03d40 265 let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
812c6f87 266
8a8a4703 267 let mut result = client.get(&path, None).await?;
812c6f87 268
d0a03d40
DM
269 record_repository(&repo);
270
c81b2b7c
DM
271 let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
272 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
273 let group = BackupGroup::new(item.backup_type, item.backup_id);
274 Ok(group.group_path().to_str().unwrap().to_owned())
275 };
812c6f87 276
18deda40
DM
277 let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
278 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 279 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
18deda40 280 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
c81b2b7c 281 };
812c6f87 282
c81b2b7c
DM
283 let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
284 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
770a36e5 285 Ok(pbs_tools::format::render_backup_file_list(&item.files))
c81b2b7c 286 };
812c6f87 287
c81b2b7c
DM
288 let options = default_table_format_options()
289 .sortby("backup-type", false)
290 .sortby("backup-id", false)
291 .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
18deda40
DM
292 .column(
293 ColumnConfig::new("last-backup")
294 .renderer(render_last_backup)
295 .header("last snapshot")
296 .right_align(false)
297 )
c81b2b7c
DM
298 .column(ColumnConfig::new("backup-count"))
299 .column(ColumnConfig::new("files").renderer(render_files));
ad20d198 300
c81b2b7c 301 let mut data: Value = result["data"].take();
ad20d198 302
b2362a12 303 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_LIST_GROUPS.returns;
812c6f87 304
b2362a12 305 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc 306
812c6f87
DM
307 Ok(Value::Null)
308}
309
344add38
DW
310#[api(
311 input: {
312 properties: {
313 repository: {
314 schema: REPO_URL_SCHEMA,
315 optional: true,
316 },
317 group: {
318 type: String,
319 description: "Backup group.",
320 },
321 "new-owner": {
e6dc35ac 322 type: Authid,
344add38
DW
323 },
324 }
325 }
326)]
327/// Change owner of a backup group
328async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
329
330 let repo = extract_repository_from_value(&param)?;
331
f3fde36b 332 let mut client = connect(&repo)?;
344add38
DW
333
334 param.as_object_mut().unwrap().remove("repository");
335
336 let group: BackupGroup = group.parse()?;
337
338 param["backup-type"] = group.backup_type().into();
339 param["backup-id"] = group.backup_id().into();
340
341 let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
342 client.post(&path, Some(param)).await?;
343
344 record_repository(&repo);
345
346 Ok(())
347}
348
a47a02ae
DM
349#[api(
350 input: {
351 properties: {
352 repository: {
353 schema: REPO_URL_SCHEMA,
354 optional: true,
355 },
356 }
357 }
358)]
359/// Try to login. If successful, store ticket.
360async fn api_login(param: Value) -> Result<Value, Error> {
e240d8be
DM
361
362 let repo = extract_repository_from_value(&param)?;
363
f3fde36b 364 let client = connect(&repo)?;
8a8a4703 365 client.login().await?;
e240d8be
DM
366
367 record_repository(&repo);
368
369 Ok(Value::Null)
370}
371
a47a02ae
DM
372#[api(
373 input: {
374 properties: {
375 repository: {
376 schema: REPO_URL_SCHEMA,
377 optional: true,
378 },
379 }
380 }
381)]
382/// Logout (delete stored ticket).
383fn api_logout(param: Value) -> Result<Value, Error> {
e240d8be
DM
384
385 let repo = extract_repository_from_value(&param)?;
386
5030b7ce 387 delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
e240d8be
DM
388
389 Ok(Value::Null)
390}
391
e39974af
TL
392#[api(
393 input: {
394 properties: {
395 repository: {
396 schema: REPO_URL_SCHEMA,
397 optional: true,
398 },
399 "output-format": {
400 schema: OUTPUT_FORMAT,
401 optional: true,
402 },
403 }
404 }
405)]
406/// Show client and optional server version
407async fn api_version(param: Value) -> Result<(), Error> {
408
409 let output_format = get_output_format(&param);
410
411 let mut version_info = json!({
412 "client": {
a12b1be7
WB
413 "version": pbs_buildcfg::PROXMOX_PKG_VERSION,
414 "release": pbs_buildcfg::PROXMOX_PKG_RELEASE,
415 "repoid": pbs_buildcfg::PROXMOX_PKG_REPOID,
e39974af
TL
416 }
417 });
418
419 let repo = extract_repository_from_value(&param);
420 if let Ok(repo) = repo {
f3fde36b 421 let client = connect(&repo)?;
e39974af
TL
422
423 match client.get("api2/json/version", None).await {
424 Ok(mut result) => version_info["server"] = result["data"].take(),
425 Err(e) => eprintln!("could not connect to server - {}", e),
426 }
427 }
428 if output_format == "text" {
a12b1be7
WB
429 println!(
430 "client version: {}.{}",
431 pbs_buildcfg::PROXMOX_PKG_VERSION,
432 pbs_buildcfg::PROXMOX_PKG_RELEASE,
433 );
e39974af
TL
434 if let Some(server) = version_info["server"].as_object() {
435 let server_version = server["version"].as_str().unwrap();
436 let server_release = server["release"].as_str().unwrap();
437 println!("server version: {}.{}", server_version, server_release);
438 }
439 } else {
440 format_and_print_result(&version_info, &output_format);
441 }
442
443 Ok(())
444}
445
a47a02ae 446#[api(
94913f35 447 input: {
a47a02ae
DM
448 properties: {
449 repository: {
450 schema: REPO_URL_SCHEMA,
451 optional: true,
452 },
94913f35
DM
453 "output-format": {
454 schema: OUTPUT_FORMAT,
455 optional: true,
456 },
457 },
458 },
a47a02ae
DM
459)]
460/// Start garbage collection for a specific repository.
461async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
8cc0d6af 462
2665cef7 463 let repo = extract_repository_from_value(&param)?;
c2043614
DM
464
465 let output_format = get_output_format(&param);
8cc0d6af 466
f3fde36b 467 let mut client = connect(&repo)?;
8cc0d6af 468
d0a03d40 469 let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
8cc0d6af 470
8a8a4703 471 let result = client.post(&path, None).await?;
8cc0d6af 472
8a8a4703 473 record_repository(&repo);
d0a03d40 474
e68269fc 475 view_task_result(&mut client, result, &output_format).await?;
e5f7def4 476
e5f7def4 477 Ok(Value::Null)
8cc0d6af 478}
33d64b81 479
6d233161 480struct CatalogUploadResult {
f1d76ecf 481 catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
6d233161
FG
482 result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
483}
484
bf6e3217 485fn spawn_catalog_upload(
3bad3e6e 486 client: Arc<BackupWriter>,
3638341a 487 encrypt: bool,
6d233161 488) -> Result<CatalogUploadResult, Error> {
f1d99e3f 489 let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
3c8c2827 490 let catalog_stream = tools::StdChannelStream(catalog_rx);
bf6e3217
DM
491 let catalog_chunk_size = 512*1024;
492 let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
493
f1d76ecf 494 let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)))?));
bf6e3217
DM
495
496 let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
497
e43b9175
FG
498 let upload_options = UploadOptions {
499 encrypt,
500 compress: true,
501 ..UploadOptions::default()
502 };
503
bf6e3217
DM
504 tokio::spawn(async move {
505 let catalog_upload_result = client
e43b9175 506 .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
bf6e3217
DM
507 .await;
508
509 if let Err(ref err) = catalog_upload_result {
510 eprintln!("catalog upload error - {}", err);
511 client.cancel();
512 }
513
514 let _ = catalog_result_tx.send(catalog_upload_result);
515 });
516
6d233161 517 Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
bf6e3217
DM
518}
519
a47a02ae
DM
520#[api(
521 input: {
522 properties: {
523 backupspec: {
524 type: Array,
525 description: "List of backup source specifications ([<label.ext>:<path>] ...)",
526 items: {
527 schema: BACKUP_SOURCE_SCHEMA,
528 }
529 },
530 repository: {
531 schema: REPO_URL_SCHEMA,
532 optional: true,
533 },
534 "include-dev": {
535 description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
536 optional: true,
537 items: {
538 type: String,
539 description: "Path to file.",
540 }
541 },
58fcbf5a
FE
542 "all-file-systems": {
543 type: Boolean,
544 description: "Include all mounted subdirectories.",
545 optional: true,
546 },
a47a02ae
DM
547 keyfile: {
548 schema: KEYFILE_SCHEMA,
549 optional: true,
550 },
0351f23b
WB
551 "keyfd": {
552 schema: KEYFD_SCHEMA,
553 optional: true,
554 },
c0a87c12
FG
555 "master-pubkey-file": {
556 schema: MASTER_PUBKEY_FILE_SCHEMA,
557 optional: true,
558 },
559 "master-pubkey-fd": {
560 schema: MASTER_PUBKEY_FD_SCHEMA,
561 optional: true,
562 },
24be37e3
WB
563 "crypt-mode": {
564 type: CryptMode,
96ee8577
WB
565 optional: true,
566 },
a47a02ae
DM
567 "skip-lost-and-found": {
568 type: Boolean,
569 description: "Skip lost+found directory.",
570 optional: true,
571 },
572 "backup-type": {
573 schema: BACKUP_TYPE_SCHEMA,
574 optional: true,
575 },
576 "backup-id": {
577 schema: BACKUP_ID_SCHEMA,
578 optional: true,
579 },
580 "backup-time": {
581 schema: BACKUP_TIME_SCHEMA,
582 optional: true,
583 },
584 "chunk-size": {
585 schema: CHUNK_SIZE_SCHEMA,
586 optional: true,
587 },
189996cf
CE
588 "exclude": {
589 type: Array,
590 description: "List of paths or patterns for matching files to exclude.",
591 optional: true,
592 items: {
593 type: String,
594 description: "Path or match pattern.",
595 }
596 },
6fc053ed
CE
597 "entries-max": {
598 type: Integer,
599 description: "Max number of entries to hold in memory.",
600 optional: true,
2b7f8dd5 601 default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
6fc053ed 602 },
e02c3d46
DM
603 "verbose": {
604 type: Boolean,
605 description: "Verbose output.",
606 optional: true,
607 },
a47a02ae
DM
608 }
609 }
610)]
611/// Create (host) backup.
612async fn create_backup(
6049b71f
DM
613 param: Value,
614 _info: &ApiMethod,
dd5495d6 615 _rpcenv: &mut dyn RpcEnvironment,
6049b71f 616) -> Result<Value, Error> {
ff5d3707 617
2665cef7 618 let repo = extract_repository_from_value(&param)?;
ae0be2dd 619
3c8c2827 620 let backupspec_list = json::required_array_param(&param, "backupspec")?;
a914a774 621
eed6db39
DM
622 let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
623
5b72c9b4
DM
624 let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
625
219ef0e6
DM
626 let verbose = param["verbose"].as_bool().unwrap_or(false);
627
ca5d0b61
DM
628 let backup_time_opt = param["backup-time"].as_i64();
629
36898ffc 630 let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
2d9d143a 631
247cdbce
DM
632 if let Some(size) = chunk_size_opt {
633 verify_chunk_size(size)?;
2d9d143a
DM
634 }
635
c6a7ea0a 636 let crypto = crypto_parameters(&param)?;
6d0983db 637
f69adc81 638 let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
fba30411 639
bbf9e7e9 640 let backup_type = param["backup-type"].as_str().unwrap_or("host");
ca5d0b61 641
2eeaacb9
DM
642 let include_dev = param["include-dev"].as_array();
643
c443f58b 644 let entries_max = param["entries-max"].as_u64()
2b7f8dd5 645 .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
6fc053ed 646
189996cf 647 let empty = Vec::new();
c443f58b
WB
648 let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
649
239e49f9 650 let mut pattern_list = Vec::with_capacity(exclude_args.len());
c443f58b
WB
651 for entry in exclude_args {
652 let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
239e49f9 653 pattern_list.push(
c443f58b
WB
654 MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
655 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
656 );
189996cf
CE
657 }
658
2eeaacb9
DM
659 let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
660
661 if let Some(include_dev) = include_dev {
662 if all_file_systems {
663 bail!("option 'all-file-systems' conflicts with option 'include-dev'");
664 }
665
666 let mut set = HashSet::new();
667 for path in include_dev {
668 let path = path.as_str().unwrap();
669 let stat = nix::sys::stat::stat(path)
670 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
671 set.insert(stat.st_dev);
672 }
673 devices = Some(set);
674 }
675
ae0be2dd 676 let mut upload_list = vec![];
f2b4b4b9 677 let mut target_set = HashSet::new();
a914a774 678
ae0be2dd 679 for backupspec in backupspec_list {
7cc3473a
DM
680 let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
681 let filename = &spec.config_string;
682 let target = &spec.archive_name;
bcd879cf 683
f2b4b4b9
SI
684 if target_set.contains(target) {
685 bail!("got target twice: '{}'", target);
686 }
687 target_set.insert(target.to_string());
688
eb1804c5
DM
689 use std::os::unix::fs::FileTypeExt;
690
3fa71727
CE
691 let metadata = std::fs::metadata(filename)
692 .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
eb1804c5 693 let file_type = metadata.file_type();
23bb8780 694
7cc3473a
DM
695 match spec.spec_type {
696 BackupSpecificationType::PXAR => {
ec8a9bb9
DM
697 if !file_type.is_dir() {
698 bail!("got unexpected file type (expected directory)");
699 }
7cc3473a 700 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
ec8a9bb9 701 }
7cc3473a 702 BackupSpecificationType::IMAGE => {
ec8a9bb9
DM
703 if !(file_type.is_file() || file_type.is_block_device()) {
704 bail!("got unexpected file type (expected file or block device)");
705 }
eb1804c5 706
e18a6c9e 707 let size = image_size(&PathBuf::from(filename))?;
23bb8780 708
ec8a9bb9 709 if size == 0 { bail!("got zero-sized file '{}'", filename); }
ae0be2dd 710
7cc3473a 711 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
ec8a9bb9 712 }
7cc3473a 713 BackupSpecificationType::CONFIG => {
ec8a9bb9
DM
714 if !file_type.is_file() {
715 bail!("got unexpected file type (expected regular file)");
716 }
7cc3473a 717 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 718 }
7cc3473a 719 BackupSpecificationType::LOGFILE => {
79679c2d
DM
720 if !file_type.is_file() {
721 bail!("got unexpected file type (expected regular file)");
722 }
7cc3473a 723 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 724 }
ae0be2dd
DM
725 }
726 }
727
22a9189e 728 let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
ae0be2dd 729
f3fde36b 730 let client = connect(&repo)?;
d0a03d40
DM
731 record_repository(&repo);
732
6a7be83e 733 println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
ca5d0b61 734
f69adc81 735 println!("Client name: {}", proxmox::tools::nodename());
ca5d0b61 736
6a7be83e 737 let start_time = std::time::Instant::now();
ca5d0b61 738
6a7be83e 739 println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
51144821 740
c6a7ea0a 741 let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
bb823140 742 None => (None, None),
2f26b866
FG
743 Some(key_with_source) => {
744 println!(
745 "{}",
746 format_key_source(&key_with_source.source, "encryption")
747 );
748
749 let (key, created, fingerprint) =
ff8945fd 750 decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
6f2626ae 751 println!("Encryption key fingerprint: {}", fingerprint);
bb823140 752
44288184 753 let crypt_config = CryptConfig::new(key)?;
bb823140 754
c0a87c12 755 match crypto.master_pubkey {
2f26b866
FG
756 Some(pem_with_source) => {
757 println!("{}", format_key_source(&pem_with_source.source, "master"));
758
759 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
82a103c8 760
1c86893d 761 let mut key_config = KeyConfig::without_password(key)?;
82a103c8 762 key_config.created = created; // keep original value
82a103c8 763
8acfd15d 764 let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
6f2626ae 765
05389a01 766 (Some(Arc::new(crypt_config)), Some(enc_key))
2f26b866 767 },
05389a01 768 _ => (Some(Arc::new(crypt_config)), None),
bb823140 769 }
6d0983db
DM
770 }
771 };
f98ac774 772
8a8a4703
DM
773 let client = BackupWriter::start(
774 client,
b957aa81 775 crypt_config.clone(),
8a8a4703
DM
776 repo.store(),
777 backup_type,
778 &backup_id,
779 backup_time,
780 verbose,
61d7b501 781 false
8a8a4703
DM
782 ).await?;
783
8b7f8d3f
FG
784 let download_previous_manifest = match client.previous_backup_time().await {
785 Ok(Some(backup_time)) => {
786 println!(
787 "Downloading previous manifest ({})",
788 strftime_local("%c", backup_time)?
789 );
790 true
791 }
792 Ok(None) => {
793 println!("No previous manifest available.");
794 false
795 }
796 Err(_) => {
797 // Fallback for outdated server, TODO remove/bubble up with 2.0
798 true
799 }
800 };
801
802 let previous_manifest = if download_previous_manifest {
803 match client.download_previous_manifest().await {
804 Ok(previous_manifest) => {
805 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
806 Ok(()) => Some(Arc::new(previous_manifest)),
807 Err(err) => {
808 println!("Couldn't re-use previous manifest - {}", err);
809 None
810 }
811 }
23f9503a 812 }
8b7f8d3f
FG
813 Err(err) => {
814 println!("Couldn't download previous manifest - {}", err);
815 None
816 }
817 }
818 } else {
819 None
b957aa81
DM
820 };
821
6a7be83e 822 let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
8a8a4703
DM
823 let mut manifest = BackupManifest::new(snapshot);
824
5d85847f 825 let mut catalog = None;
6d233161 826 let mut catalog_result_rx = None;
8a8a4703
DM
827
828 for (backup_type, filename, target, size) in upload_list {
829 match backup_type {
7cc3473a 830 BackupSpecificationType::CONFIG => {
e43b9175
FG
831 let upload_options = UploadOptions {
832 compress: true,
c6a7ea0a 833 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
834 ..UploadOptions::default()
835 };
836
5b32820e 837 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 838 let stats = client
e43b9175 839 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 840 .await?;
c6a7ea0a 841 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 842 }
7cc3473a 843 BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
e43b9175
FG
844 let upload_options = UploadOptions {
845 compress: true,
c6a7ea0a 846 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
847 ..UploadOptions::default()
848 };
849
5b32820e 850 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 851 let stats = client
e43b9175 852 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 853 .await?;
c6a7ea0a 854 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 855 }
7cc3473a 856 BackupSpecificationType::PXAR => {
5d85847f
DC
857 // start catalog upload on first use
858 if catalog.is_none() {
c6a7ea0a 859 let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
6d233161
FG
860 catalog = Some(catalog_upload_res.catalog_writer);
861 catalog_result_rx = Some(catalog_upload_res.result);
5d85847f
DC
862 }
863 let catalog = catalog.as_ref().unwrap();
864
5b32820e 865 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
8a8a4703 866 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
77486a60 867
2b7f8dd5 868 let pxar_options = pbs_client::pxar::PxarCreateOptions {
77486a60
FG
869 device_set: devices.clone(),
870 patterns: pattern_list.clone(),
871 entries_max: entries_max as usize,
872 skip_lost_and_found,
873 verbose,
874 };
875
e43b9175
FG
876 let upload_options = UploadOptions {
877 previous_manifest: previous_manifest.clone(),
878 compress: true,
c6a7ea0a 879 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
880 ..UploadOptions::default()
881 };
882
8a8a4703
DM
883 let stats = backup_directory(
884 &client,
885 &filename,
886 &target,
887 chunk_size_opt,
8a8a4703 888 catalog.clone(),
77486a60 889 pxar_options,
e43b9175 890 upload_options,
8a8a4703 891 ).await?;
c6a7ea0a 892 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703
DM
893 catalog.lock().unwrap().end_directory()?;
894 }
7cc3473a 895 BackupSpecificationType::IMAGE => {
8a8a4703 896 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
e43b9175
FG
897
898 let upload_options = UploadOptions {
899 previous_manifest: previous_manifest.clone(),
900 fixed_size: Some(size),
901 compress: true,
c6a7ea0a 902 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
903 };
904
8a8a4703
DM
905 let stats = backup_image(
906 &client,
e43b9175 907 &filename,
8a8a4703 908 &target,
8a8a4703 909 chunk_size_opt,
e43b9175 910 upload_options,
8a8a4703 911 ).await?;
c6a7ea0a 912 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
6af905c1
DM
913 }
914 }
8a8a4703 915 }
4818c8b6 916
8a8a4703 917 // finalize and upload catalog
5d85847f 918 if let Some(catalog) = catalog {
8a8a4703
DM
919 let mutex = Arc::try_unwrap(catalog)
920 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
921 let mut catalog = mutex.into_inner().unwrap();
bf6e3217 922
8a8a4703 923 catalog.finish()?;
2761d6a4 924
8a8a4703 925 drop(catalog); // close upload stream
2761d6a4 926
6d233161 927 if let Some(catalog_result_rx) = catalog_result_rx {
5d85847f 928 let stats = catalog_result_rx.await??;
c6a7ea0a 929 manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
5d85847f 930 }
8a8a4703 931 }
2761d6a4 932
8a8a4703 933 if let Some(rsa_encrypted_key) = rsa_encrypted_key {
9990af30 934 let target = ENCRYPTED_KEY_BLOB_NAME;
8a8a4703 935 println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
e43b9175 936 let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
8a8a4703 937 let stats = client
e43b9175 938 .upload_blob_from_data(rsa_encrypted_key, target, options)
8a8a4703 939 .await?;
c6a7ea0a 940 manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
8a8a4703 941
8a8a4703 942 }
8a8a4703 943 // create manifest (index.json)
3638341a 944 // manifests are never encrypted, but include a signature
dfa517ad 945 let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
b53f6379 946 .map_err(|err| format_err!("unable to format manifest - {}", err))?;
3638341a 947
b53f6379 948
9688f6de 949 if verbose { println!("Upload index.json to '{}'", repo) };
e43b9175 950 let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
8a8a4703 951 client
e43b9175 952 .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
8a8a4703 953 .await?;
2c3891d1 954
8a8a4703 955 client.finish().await?;
c4ff3dce 956
6a7be83e
DM
957 let end_time = std::time::Instant::now();
958 let elapsed = end_time.duration_since(start_time);
959 println!("Duration: {:.2}s", elapsed.as_secs_f64());
3ec3ec3f 960
6a7be83e 961 println!("End Time: {}", strftime_local("%c", epoch_i64())?);
3d5c11e5 962
8a8a4703 963 Ok(Value::Null)
f98ea63d
DM
964}
965
8e6e18b7 966async fn dump_image<W: Write>(
88892ea8
DM
967 client: Arc<BackupReader>,
968 crypt_config: Option<Arc<CryptConfig>>,
14f6c9cb 969 crypt_mode: CryptMode,
88892ea8
DM
970 index: FixedIndexReader,
971 mut writer: W,
fd04ca7a 972 verbose: bool,
88892ea8
DM
973) -> Result<(), Error> {
974
975 let most_used = index.find_most_used_chunks(8);
976
14f6c9cb 977 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
88892ea8
DM
978
979 // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
980 // and thus slows down reading. Instead, directly use RemoteChunkReader
fd04ca7a
DM
981 let mut per = 0;
982 let mut bytes = 0;
983 let start_time = std::time::Instant::now();
984
88892ea8
DM
985 for pos in 0..index.index_count() {
986 let digest = index.index_digest(pos).unwrap();
8e6e18b7 987 let raw_data = chunk_reader.read_chunk(&digest).await?;
88892ea8 988 writer.write_all(&raw_data)?;
fd04ca7a
DM
989 bytes += raw_data.len();
990 if verbose {
991 let next_per = ((pos+1)*100)/index.index_count();
992 if per != next_per {
993 eprintln!("progress {}% (read {} bytes, duration {} sec)",
994 next_per, bytes, start_time.elapsed().as_secs());
995 per = next_per;
996 }
997 }
88892ea8
DM
998 }
999
fd04ca7a
DM
1000 let end_time = std::time::Instant::now();
1001 let elapsed = end_time.duration_since(start_time);
1002 eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1003 bytes,
1004 elapsed.as_secs_f64(),
1005 bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1006 );
1007
1008
88892ea8
DM
1009 Ok(())
1010}
1011
dc155e9b 1012fn parse_archive_type(name: &str) -> (String, ArchiveType) {
2d32fe2c
TL
1013 if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1014 (name.into(), archive_type(name).unwrap())
1015 } else if name.ends_with(".pxar") {
dc155e9b
TL
1016 (format!("{}.didx", name), ArchiveType::DynamicIndex)
1017 } else if name.ends_with(".img") {
1018 (format!("{}.fidx", name), ArchiveType::FixedIndex)
1019 } else {
1020 (format!("{}.blob", name), ArchiveType::Blob)
1021 }
1022}
1023
a47a02ae
DM
1024#[api(
1025 input: {
1026 properties: {
1027 repository: {
1028 schema: REPO_URL_SCHEMA,
1029 optional: true,
1030 },
1031 snapshot: {
1032 type: String,
1033 description: "Group/Snapshot path.",
1034 },
1035 "archive-name": {
1036 description: "Backup archive name.",
1037 type: String,
1038 },
1039 target: {
1040 type: String,
90c815bf 1041 description: r###"Target directory path. Use '-' to write to standard output.
8a8a4703 1042
d1d74c43 1043We do not extract '.pxar' archives when writing to standard output.
8a8a4703 1044
a47a02ae
DM
1045"###
1046 },
1047 "allow-existing-dirs": {
1048 type: Boolean,
1049 description: "Do not fail if directories already exists.",
1050 optional: true,
1051 },
1052 keyfile: {
1053 schema: KEYFILE_SCHEMA,
1054 optional: true,
1055 },
0351f23b
WB
1056 "keyfd": {
1057 schema: KEYFD_SCHEMA,
1058 optional: true,
1059 },
24be37e3
WB
1060 "crypt-mode": {
1061 type: CryptMode,
96ee8577
WB
1062 optional: true,
1063 },
a47a02ae
DM
1064 }
1065 }
1066)]
1067/// Restore backup repository.
1068async fn restore(param: Value) -> Result<Value, Error> {
2665cef7 1069 let repo = extract_repository_from_value(&param)?;
9f912493 1070
86eda3eb
DM
1071 let verbose = param["verbose"].as_bool().unwrap_or(false);
1072
46d5aa0a
DM
1073 let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1074
3c8c2827 1075 let archive_name = json::required_string_param(&param, "archive-name")?;
d5c34d98 1076
f3fde36b 1077 let client = connect(&repo)?;
d0a03d40 1078
d0a03d40 1079 record_repository(&repo);
d5c34d98 1080
3c8c2827 1081 let path = json::required_string_param(&param, "snapshot")?;
9f912493 1082
86eda3eb 1083 let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
d6d3b353 1084 let group: BackupGroup = path.parse()?;
27c9affb 1085 api_datastore_latest_snapshot(&client, repo.store(), group).await?
d5c34d98 1086 } else {
a67f7d0a 1087 let snapshot: BackupDir = path.parse()?;
86eda3eb
DM
1088 (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1089 };
9f912493 1090
3c8c2827 1091 let target = json::required_string_param(&param, "target")?;
bf125261 1092 let target = if target == "-" { None } else { Some(target) };
2ae7d196 1093
c6a7ea0a 1094 let crypto = crypto_parameters(&param)?;
2ae7d196 1095
c6a7ea0a 1096 let crypt_config = match crypto.enc_key {
86eda3eb 1097 None => None,
2f26b866
FG
1098 Some(ref key) => {
1099 let (key, _, _) =
ff8945fd 1100 decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
2f26b866
FG
1101 eprintln!("{}", format_key_source(&key.source, "encryption"));
1102 err
1103 })?;
86eda3eb
DM
1104 Some(Arc::new(CryptConfig::new(key)?))
1105 }
1106 };
d5c34d98 1107
296c50ba
DM
1108 let client = BackupReader::start(
1109 client,
1110 crypt_config.clone(),
1111 repo.store(),
1112 &backup_type,
1113 &backup_id,
1114 backup_time,
1115 true,
1116 ).await?;
86eda3eb 1117
48fbbfeb
FG
1118 let (archive_name, archive_type) = parse_archive_type(archive_name);
1119
2107a5ae 1120 let (manifest, backup_index_data) = client.download_manifest().await?;
02fcf372 1121
48fbbfeb
FG
1122 if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1123 eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1124 } else {
2f26b866
FG
1125 if manifest.signature.is_some() {
1126 if let Some(key) = &crypto.enc_key {
1127 eprintln!("{}", format_key_source(&key.source, "encryption"));
1128 }
1129 if let Some(config) = &crypt_config {
1130 eprintln!("Fingerprint: {}", config.fingerprint());
1131 }
1132 }
48fbbfeb
FG
1133 manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1134 }
dc155e9b
TL
1135
1136 if archive_name == MANIFEST_BLOB_NAME {
02fcf372 1137 if let Some(target) = target {
2107a5ae 1138 replace_file(target, &backup_index_data, CreateOptions::new())?;
02fcf372
DM
1139 } else {
1140 let stdout = std::io::stdout();
1141 let mut writer = stdout.lock();
2107a5ae 1142 writer.write_all(&backup_index_data)
02fcf372
DM
1143 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1144 }
1145
14f6c9cb
FG
1146 return Ok(Value::Null);
1147 }
1148
1149 let file_info = manifest.lookup_file_info(&archive_name)?;
1150
1151 if archive_type == ArchiveType::Blob {
d2267b11 1152
dc155e9b 1153 let mut reader = client.download_blob(&manifest, &archive_name).await?;
f8100e96 1154
bf125261 1155 if let Some(target) = target {
0d986280
DM
1156 let mut writer = std::fs::OpenOptions::new()
1157 .write(true)
1158 .create(true)
1159 .create_new(true)
1160 .open(target)
1161 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1162 std::io::copy(&mut reader, &mut writer)?;
bf125261
DM
1163 } else {
1164 let stdout = std::io::stdout();
1165 let mut writer = stdout.lock();
0d986280 1166 std::io::copy(&mut reader, &mut writer)
bf125261
DM
1167 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1168 }
f8100e96 1169
dc155e9b 1170 } else if archive_type == ArchiveType::DynamicIndex {
86eda3eb 1171
dc155e9b 1172 let index = client.download_dynamic_index(&manifest, &archive_name).await?;
df65bd3d 1173
f4bf7dfc
DM
1174 let most_used = index.find_most_used_chunks(8);
1175
14f6c9cb 1176 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
f4bf7dfc 1177
afb4cd28 1178 let mut reader = BufferedDynamicReader::new(index, chunk_reader);
86eda3eb 1179
2b7f8dd5 1180 let options = pbs_client::pxar::PxarExtractOptions {
72064fd0
FG
1181 match_list: &[],
1182 extract_match_default: true,
1183 allow_existing_dirs,
1184 on_error: None,
1185 };
1186
bf125261 1187 if let Some(target) = target {
2b7f8dd5 1188 pbs_client::pxar::extract_archive(
c443f58b
WB
1189 pxar::decoder::Decoder::from_std(reader)?,
1190 Path::new(target),
2b7f8dd5 1191 pbs_client::pxar::Flags::DEFAULT,
c443f58b
WB
1192 |path| {
1193 if verbose {
1194 println!("{:?}", path);
1195 }
1196 },
72064fd0 1197 options,
c443f58b
WB
1198 )
1199 .map_err(|err| format_err!("error extracting archive - {}", err))?;
bf125261 1200 } else {
88892ea8
DM
1201 let mut writer = std::fs::OpenOptions::new()
1202 .write(true)
1203 .open("/dev/stdout")
1204 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
afb4cd28 1205
bf125261
DM
1206 std::io::copy(&mut reader, &mut writer)
1207 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1208 }
dc155e9b 1209 } else if archive_type == ArchiveType::FixedIndex {
afb4cd28 1210
dc155e9b 1211 let index = client.download_fixed_index(&manifest, &archive_name).await?;
df65bd3d 1212
88892ea8
DM
1213 let mut writer = if let Some(target) = target {
1214 std::fs::OpenOptions::new()
bf125261
DM
1215 .write(true)
1216 .create(true)
1217 .create_new(true)
1218 .open(target)
88892ea8 1219 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
bf125261 1220 } else {
88892ea8
DM
1221 std::fs::OpenOptions::new()
1222 .write(true)
1223 .open("/dev/stdout")
1224 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1225 };
afb4cd28 1226
14f6c9cb 1227 dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
3031e44c 1228 }
fef44d4f
DM
1229
1230 Ok(Value::Null)
45db6f89
DM
1231}
1232
e0665a64
DC
1233#[api(
1234 input: {
1235 properties: {
1236 "dry-run": {
1237 type: bool,
1238 optional: true,
1239 description: "Just show what prune would do, but do not delete anything.",
1240 },
1241 group: {
1242 type: String,
1243 description: "Backup group",
1244 },
1245 "prune-options": {
1246 type: PruneOptions,
1247 flatten: true,
1248 },
1249 "output-format": {
1250 schema: OUTPUT_FORMAT,
1251 optional: true,
1252 },
1253 quiet: {
1254 type: bool,
1255 optional: true,
1256 default: false,
1257 description: "Minimal output - only show removals.",
1258 },
1259 repository: {
1260 schema: REPO_URL_SCHEMA,
1261 optional: true,
1262 },
1263 },
1264 },
1265)]
1266/// Prune a backup repository.
1267async fn prune(
1268 dry_run: Option<bool>,
1269 group: String,
1270 prune_options: PruneOptions,
1271 quiet: bool,
1272 mut param: Value
1273) -> Result<Value, Error> {
2665cef7 1274 let repo = extract_repository_from_value(&param)?;
83b7db02 1275
f3fde36b 1276 let mut client = connect(&repo)?;
83b7db02 1277
d0a03d40 1278 let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
83b7db02 1279
d6d3b353 1280 let group: BackupGroup = group.parse()?;
c2043614 1281
671c6a96 1282 let output_format = extract_output_format(&mut param);
9fdc3ef4 1283
e0665a64
DC
1284 let mut api_param = serde_json::to_value(prune_options)?;
1285 if let Some(dry_run) = dry_run {
1286 api_param["dry-run"] = dry_run.into();
1287 }
1288 api_param["backup-type"] = group.backup_type().into();
1289 api_param["backup-id"] = group.backup_id().into();
83b7db02 1290
e0665a64 1291 let mut result = client.post(&path, Some(api_param)).await?;
74fa81b8 1292
87c42375 1293 record_repository(&repo);
3b03abfe 1294
db1e061d
DM
1295 let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1296 let item: PruneListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 1297 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
db1e061d
DM
1298 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1299 };
1300
c48aa39f
DM
1301 let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1302 Ok(match v.as_bool() {
1303 Some(true) => "keep",
1304 Some(false) => "remove",
1305 None => "unknown",
1306 }.to_string())
1307 };
1308
db1e061d
DM
1309 let options = default_table_format_options()
1310 .sortby("backup-type", false)
1311 .sortby("backup-id", false)
1312 .sortby("backup-time", false)
1313 .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
770a36e5 1314 .column(ColumnConfig::new("backup-time").renderer(pbs_tools::format::render_epoch).header("date"))
c48aa39f 1315 .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
db1e061d
DM
1316 ;
1317
b2362a12 1318 let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_PRUNE.returns;
db1e061d
DM
1319
1320 let mut data = result["data"].take();
1321
c48aa39f
DM
1322 if quiet {
1323 let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1324 item["keep"].as_bool() == Some(false)
a375df6f 1325 }).cloned().collect();
c48aa39f
DM
1326 data = list.into();
1327 }
1328
b2362a12 1329 format_and_print_result_full(&mut data, return_type, &output_format, &options);
d0a03d40 1330
43a406fd 1331 Ok(Value::Null)
83b7db02
DM
1332}
1333
a47a02ae
DM
1334#[api(
1335 input: {
1336 properties: {
1337 repository: {
1338 schema: REPO_URL_SCHEMA,
1339 optional: true,
1340 },
1341 "output-format": {
1342 schema: OUTPUT_FORMAT,
1343 optional: true,
1344 },
1345 }
f9beae9c
TL
1346 },
1347 returns: {
1348 type: StorageStatus,
1349 },
a47a02ae
DM
1350)]
1351/// Get repository status.
1352async fn status(param: Value) -> Result<Value, Error> {
34a816cc
DM
1353
1354 let repo = extract_repository_from_value(&param)?;
1355
c2043614 1356 let output_format = get_output_format(&param);
34a816cc 1357
f3fde36b 1358 let client = connect(&repo)?;
34a816cc
DM
1359
1360 let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1361
1dc117bb 1362 let mut result = client.get(&path, None).await?;
14e08625 1363 let mut data = result["data"].take();
34a816cc
DM
1364
1365 record_repository(&repo);
1366
390c5bdd
DM
1367 let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1368 let v = v.as_u64().unwrap();
1369 let total = record["total"].as_u64().unwrap();
1370 let roundup = total/200;
1371 let per = ((v+roundup)*100)/total;
e23f5863
DM
1372 let info = format!(" ({} %)", per);
1373 Ok(format!("{} {:>8}", v, info))
390c5bdd 1374 };
1dc117bb 1375
c2043614 1376 let options = default_table_format_options()
be2425ff 1377 .noheader(true)
e23f5863 1378 .column(ColumnConfig::new("total").renderer(render_total_percentage))
390c5bdd
DM
1379 .column(ColumnConfig::new("used").renderer(render_total_percentage))
1380 .column(ColumnConfig::new("avail").renderer(render_total_percentage));
34a816cc 1381
b2362a12 1382 let return_type = &API_METHOD_STATUS.returns;
390c5bdd 1383
b2362a12 1384 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc
DM
1385
1386 Ok(Value::Null)
1387}
1388
c443f58b
WB
1389/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1390/// async use!
1391///
1392/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1393/// so that we can properly access it from multiple threads simultaneously while not issuing
1394/// duplicate simultaneous reads over http.
43abba4b 1395pub struct BufferedDynamicReadAt {
c443f58b
WB
1396 inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1397}
1398
1399impl BufferedDynamicReadAt {
1400 fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
1401 Self {
1402 inner: Mutex::new(inner),
1403 }
1404 }
1405}
1406
a6f87283
WB
1407impl ReadAt for BufferedDynamicReadAt {
1408 fn start_read_at<'a>(
1409 self: Pin<&'a Self>,
c443f58b 1410 _cx: &mut Context,
a6f87283 1411 buf: &'a mut [u8],
c443f58b 1412 offset: u64,
a6f87283 1413 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
a6f87283 1414 MaybeReady::Ready(tokio::task::block_in_place(move || {
c443f58b
WB
1415 let mut reader = self.inner.lock().unwrap();
1416 reader.seek(SeekFrom::Start(offset))?;
a6f87283
WB
1417 Ok(reader.read(buf)?)
1418 }))
1419 }
1420
1421 fn poll_complete<'a>(
1422 self: Pin<&'a Self>,
1423 _op: ReadAtOperation<'a>,
1424 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
1425 panic!("LocalDynamicReadAt::start_read_at returned Pending");
c443f58b
WB
1426 }
1427}
1428
f2401311 1429fn main() {
33d64b81 1430
255f378a 1431 let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
49fddd98 1432 .arg_param(&["backupspec"])
d0a03d40 1433 .completion_cb("repository", complete_repository)
49811347 1434 .completion_cb("backupspec", complete_backup_source)
2b7f8dd5
WB
1435 .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
1436 .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
49811347 1437 .completion_cb("chunk-size", complete_chunk_size);
f8838fe9 1438
caea8d61
DM
1439 let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
1440 .completion_cb("repository", complete_repository)
2b7f8dd5 1441 .completion_cb("keyfile", pbs_tools::fs::complete_file_name);
caea8d61 1442
255f378a 1443 let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
d0a03d40 1444 .completion_cb("repository", complete_repository);
41c039e1 1445
255f378a 1446 let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
d0a03d40 1447 .completion_cb("repository", complete_repository);
8cc0d6af 1448
255f378a 1449 let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
49fddd98 1450 .arg_param(&["snapshot", "archive-name", "target"])
b2388518 1451 .completion_cb("repository", complete_repository)
08dc340a
DM
1452 .completion_cb("snapshot", complete_group_or_snapshot)
1453 .completion_cb("archive-name", complete_archive_name)
2b7f8dd5 1454 .completion_cb("target", pbs_tools::fs::complete_file_name);
9f912493 1455
255f378a 1456 let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
49fddd98 1457 .arg_param(&["group"])
9fdc3ef4 1458 .completion_cb("group", complete_backup_group)
d0a03d40 1459 .completion_cb("repository", complete_repository);
9f912493 1460
255f378a 1461 let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
34a816cc
DM
1462 .completion_cb("repository", complete_repository);
1463
255f378a 1464 let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
e240d8be
DM
1465 .completion_cb("repository", complete_repository);
1466
255f378a 1467 let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
e240d8be 1468 .completion_cb("repository", complete_repository);
32efac1c 1469
e39974af
TL
1470 let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
1471 .completion_cb("repository", complete_repository);
1472
344add38
DW
1473 let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
1474 .arg_param(&["group", "new-owner"])
1475 .completion_cb("group", complete_backup_group)
0224c3c2 1476 .completion_cb("new-owner", complete_auth_id)
344add38
DW
1477 .completion_cb("repository", complete_repository);
1478
41c039e1 1479 let cmd_def = CliCommandMap::new()
48ef3c33 1480 .insert("backup", backup_cmd_def)
48ef3c33
DM
1481 .insert("garbage-collect", garbage_collect_cmd_def)
1482 .insert("list", list_cmd_def)
1483 .insert("login", login_cmd_def)
1484 .insert("logout", logout_cmd_def)
1485 .insert("prune", prune_cmd_def)
1486 .insert("restore", restore_cmd_def)
a65e3e4b 1487 .insert("snapshot", snapshot_mgtm_cli())
48ef3c33 1488 .insert("status", status_cmd_def)
9696f519 1489 .insert("key", key::cli())
43abba4b 1490 .insert("mount", mount_cmd_def())
45f9b32e
SR
1491 .insert("map", map_cmd_def())
1492 .insert("unmap", unmap_cmd_def())
5830c205 1493 .insert("catalog", catalog_mgmt_cli())
caea8d61 1494 .insert("task", task_mgmt_cli())
e39974af 1495 .insert("version", version_cmd_def)
344add38 1496 .insert("benchmark", benchmark_cmd_def)
731eeef2
DM
1497 .insert("change-owner", change_owner_cmd_def)
1498
61205f00 1499 .alias(&["files"], &["snapshot", "files"])
edebd523 1500 .alias(&["forget"], &["snapshot", "forget"])
0c9209b0 1501 .alias(&["upload-log"], &["snapshot", "upload-log"])
731eeef2
DM
1502 .alias(&["snapshots"], &["snapshot", "list"])
1503 ;
48ef3c33 1504
7b22acd0
DM
1505 let rpcenv = CliEnvironment::new();
1506 run_cli_command(cmd_def, rpcenv, Some(|future| {
d420962f 1507 pbs_runtime::main(future)
d08bc483 1508 }));
ff5d3707 1509}