]> git.proxmox.com Git - proxmox-backup.git/blame - proxmox-backup-client/src/main.rs
bump proxmox dependency to 0.14.0 and proxmox-http to 0.5.0
[proxmox-backup.git] / proxmox-backup-client / src / main.rs
CommitLineData
f1a83e97 1use std::collections::HashSet;
0351f23b 2use std::io::{self, Read, Write, Seek, SeekFrom};
c443f58b
WB
3use std::path::{Path, PathBuf};
4use std::pin::Pin;
5use std::sync::{Arc, Mutex};
a6f87283 6use std::task::Context;
c443f58b
WB
7
8use anyhow::{bail, format_err, Error};
c443f58b 9use futures::stream::{StreamExt, TryStreamExt};
c443f58b 10use serde_json::{json, Value};
c443f58b 11use tokio::sync::mpsc;
7c667013 12use tokio_stream::wrappers::ReceiverStream;
c443f58b 13use xdg::BaseDirectories;
2761d6a4 14
c443f58b 15use pathpatterns::{MatchEntry, MatchType, PatternFlag};
6a7be83e
DM
16use proxmox::{
17 tools::{
18 time::{strftime_local, epoch_i64},
ff8945fd 19 fs::{file_get_json, replace_file, CreateOptions, image_size},
6a7be83e
DM
20 },
21 api::{
22 api,
6a7be83e
DM
23 ApiMethod,
24 RpcEnvironment,
6a7be83e
DM
25 cli::*,
26 },
27};
a6f87283 28use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
ff5d3707 29
51ec8a3c
WB
30use pbs_api_types::{
31 BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem,
89725197 32 PruneListItem, SnapshotListItem, StorageStatus, Fingerprint, PruneOptions,
51ec8a3c 33};
2b7f8dd5
WB
34use pbs_client::{
35 BACKUP_SOURCE_SCHEMA,
36 BackupReader,
37 BackupRepository,
38 BackupSpecificationType,
39 BackupStats,
40 BackupWriter,
38629c39
WB
41 ChunkStream,
42 FixedChunkStream,
2b7f8dd5
WB
43 HttpClient,
44 PxarBackupStream,
45 RemoteChunkReader,
46 UploadOptions,
47 delete_ticket_info,
48 parse_backup_specification,
49 view_task_result,
50};
51use pbs_client::catalog_shell::Shell;
52use pbs_client::tools::{
53 complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
54 complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
55 complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
56 extract_repository_from_value,
57 key_source::{
58 crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
59 KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
60 },
61 CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
62};
bbdda58b
DM
63use pbs_config::key_config::{KeyConfig, decrypt_key, rsa_encrypt_key_config};
64use pbs_datastore::CATALOG_NAME;
2b7f8dd5 65use pbs_datastore::backup_info::{BackupDir, BackupGroup};
51ec8a3c
WB
66use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter};
67use pbs_datastore::chunk_store::verify_chunk_size;
eb5e0ae6 68use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader};
2b7f8dd5
WB
69use pbs_datastore::fixed_index::FixedIndexReader;
70use pbs_datastore::index::IndexFile;
51ec8a3c
WB
71use pbs_datastore::manifest::{
72 ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type,
73};
2b7f8dd5 74use pbs_datastore::read_chunk::AsyncReadChunk;
4805edc4
WB
75use pbs_tools::sync::StdChannelWriter;
76use pbs_tools::tokio::TokioWriterAdapter;
3c8c2827 77use pbs_tools::json;
bbdda58b 78use pbs_tools::crypt_config::CryptConfig;
f323e906 79
e351ac78
WB
80mod benchmark;
81pub use benchmark::*;
82mod mount;
83pub use mount::*;
84mod task;
85pub use task::*;
86mod catalog;
87pub use catalog::*;
88mod snapshot;
89pub use snapshot::*;
90pub mod key;
caea8d61 91
d0a03d40
DM
92fn record_repository(repo: &BackupRepository) {
93
94 let base = match BaseDirectories::with_prefix("proxmox-backup") {
95 Ok(v) => v,
96 _ => return,
97 };
98
99 // usually $HOME/.cache/proxmox-backup/repo-list
100 let path = match base.place_cache_file("repo-list") {
101 Ok(v) => v,
102 _ => return,
103 };
104
11377a47 105 let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
d0a03d40
DM
106
107 let repo = repo.to_string();
108
109 data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
110
111 let mut map = serde_json::map::Map::new();
112
113 loop {
114 let mut max_used = 0;
115 let mut max_repo = None;
116 for (repo, count) in data.as_object().unwrap() {
117 if map.contains_key(repo) { continue; }
118 if let Some(count) = count.as_i64() {
119 if count > max_used {
120 max_used = count;
121 max_repo = Some(repo);
122 }
123 }
124 }
125 if let Some(repo) = max_repo {
126 map.insert(repo.to_owned(), json!(max_used));
127 } else {
128 break;
129 }
130 if map.len() > 10 { // store max. 10 repos
131 break;
132 }
133 }
134
135 let new_data = json!(map);
136
feaa1ad3 137 let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
d0a03d40
DM
138}
139
42af4b8f
DM
140async fn api_datastore_list_snapshots(
141 client: &HttpClient,
142 store: &str,
143 group: Option<BackupGroup>,
f24fc116 144) -> Result<Value, Error> {
42af4b8f
DM
145
146 let path = format!("api2/json/admin/datastore/{}/snapshots", store);
147
148 let mut args = json!({});
149 if let Some(group) = group {
150 args["backup-type"] = group.backup_type().into();
151 args["backup-id"] = group.backup_id().into();
152 }
153
154 let mut result = client.get(&path, Some(args)).await?;
155
f24fc116 156 Ok(result["data"].take())
42af4b8f
DM
157}
158
43abba4b 159pub async fn api_datastore_latest_snapshot(
27c9affb
DM
160 client: &HttpClient,
161 store: &str,
162 group: BackupGroup,
6a7be83e 163) -> Result<(String, String, i64), Error> {
27c9affb 164
f24fc116
DM
165 let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
166 let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
27c9affb
DM
167
168 if list.is_empty() {
169 bail!("backup group {:?} does not contain any snapshots.", group.group_path());
170 }
171
172 list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
173
6a7be83e 174 let backup_time = list[0].backup_time;
27c9affb
DM
175
176 Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
177}
178
e9722f8b 179async fn backup_directory<P: AsRef<Path>>(
cf9271e2 180 client: &BackupWriter,
17d6979a 181 dir_path: P,
247cdbce 182 archive_name: &str,
36898ffc 183 chunk_size: Option<usize>,
f1d76ecf 184 catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
2b7f8dd5 185 pxar_create_options: pbs_client::pxar::PxarCreateOptions,
e43b9175 186 upload_options: UploadOptions,
2c3891d1 187) -> Result<BackupStats, Error> {
33d64b81 188
6fc053ed
CE
189 let pxar_stream = PxarBackupStream::open(
190 dir_path.as_ref(),
6fc053ed 191 catalog,
77486a60 192 pxar_create_options,
6fc053ed 193 )?;
e9722f8b 194 let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
ff3d3100 195
0bfcea6a 196 let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
5e7a09be 197
7c667013 198 let stream = ReceiverStream::new(rx)
e9722f8b 199 .map_err(Error::from);
17d6979a 200
c4ff3dce 201 // spawn chunker inside a separate task so that it can run parallel
e9722f8b 202 tokio::spawn(async move {
db0cb9ce
WB
203 while let Some(v) = chunk_stream.next().await {
204 let _ = tx.send(v).await;
205 }
e9722f8b 206 });
17d6979a 207
e43b9175
FG
208 if upload_options.fixed_size.is_some() {
209 bail!("cannot backup directory with fixed chunk size!");
210 }
211
e9722f8b 212 let stats = client
e43b9175 213 .upload_stream(archive_name, stream, upload_options)
e9722f8b 214 .await?;
bcd879cf 215
2c3891d1 216 Ok(stats)
bcd879cf
DM
217}
218
e9722f8b 219async fn backup_image<P: AsRef<Path>>(
cf9271e2 220 client: &BackupWriter,
6af905c1
DM
221 image_path: P,
222 archive_name: &str,
36898ffc 223 chunk_size: Option<usize>,
e43b9175 224 upload_options: UploadOptions,
2c3891d1 225) -> Result<BackupStats, Error> {
6af905c1 226
6af905c1
DM
227 let path = image_path.as_ref().to_owned();
228
e9722f8b 229 let file = tokio::fs::File::open(path).await?;
6af905c1 230
db0cb9ce 231 let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
6af905c1
DM
232 .map_err(Error::from);
233
36898ffc 234 let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
6af905c1 235
e43b9175
FG
236 if upload_options.fixed_size.is_none() {
237 bail!("cannot backup image with dynamic chunk size!");
238 }
239
e9722f8b 240 let stats = client
e43b9175 241 .upload_stream(archive_name, stream, upload_options)
e9722f8b 242 .await?;
6af905c1 243
2c3891d1 244 Ok(stats)
6af905c1
DM
245}
246
a47a02ae
DM
247#[api(
248 input: {
249 properties: {
250 repository: {
251 schema: REPO_URL_SCHEMA,
252 optional: true,
253 },
254 "output-format": {
255 schema: OUTPUT_FORMAT,
256 optional: true,
257 },
258 }
259 }
260)]
261/// List backup groups.
262async fn list_backup_groups(param: Value) -> Result<Value, Error> {
812c6f87 263
c81b2b7c
DM
264 let output_format = get_output_format(&param);
265
2665cef7 266 let repo = extract_repository_from_value(&param)?;
812c6f87 267
f3fde36b 268 let client = connect(&repo)?;
812c6f87 269
d0a03d40 270 let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
812c6f87 271
8a8a4703 272 let mut result = client.get(&path, None).await?;
812c6f87 273
d0a03d40
DM
274 record_repository(&repo);
275
c81b2b7c
DM
276 let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
277 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
278 let group = BackupGroup::new(item.backup_type, item.backup_id);
279 Ok(group.group_path().to_str().unwrap().to_owned())
280 };
812c6f87 281
18deda40
DM
282 let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
283 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 284 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
18deda40 285 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
c81b2b7c 286 };
812c6f87 287
c81b2b7c
DM
288 let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
289 let item: GroupListItem = serde_json::from_value(record.to_owned())?;
770a36e5 290 Ok(pbs_tools::format::render_backup_file_list(&item.files))
c81b2b7c 291 };
812c6f87 292
c81b2b7c
DM
293 let options = default_table_format_options()
294 .sortby("backup-type", false)
295 .sortby("backup-id", false)
296 .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
18deda40
DM
297 .column(
298 ColumnConfig::new("last-backup")
299 .renderer(render_last_backup)
300 .header("last snapshot")
301 .right_align(false)
302 )
c81b2b7c
DM
303 .column(ColumnConfig::new("backup-count"))
304 .column(ColumnConfig::new("files").renderer(render_files));
ad20d198 305
c81b2b7c 306 let mut data: Value = result["data"].take();
ad20d198 307
e351ac78 308 let return_type = &pbs_api_types::ADMIN_DATASTORE_LIST_GROUPS_RETURN_TYPE;
812c6f87 309
b2362a12 310 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc 311
812c6f87
DM
312 Ok(Value::Null)
313}
314
344add38
DW
315#[api(
316 input: {
317 properties: {
318 repository: {
319 schema: REPO_URL_SCHEMA,
320 optional: true,
321 },
322 group: {
323 type: String,
324 description: "Backup group.",
325 },
326 "new-owner": {
e6dc35ac 327 type: Authid,
344add38
DW
328 },
329 }
330 }
331)]
332/// Change owner of a backup group
333async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
334
335 let repo = extract_repository_from_value(&param)?;
336
f3fde36b 337 let mut client = connect(&repo)?;
344add38
DW
338
339 param.as_object_mut().unwrap().remove("repository");
340
341 let group: BackupGroup = group.parse()?;
342
343 param["backup-type"] = group.backup_type().into();
344 param["backup-id"] = group.backup_id().into();
345
346 let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
347 client.post(&path, Some(param)).await?;
348
349 record_repository(&repo);
350
351 Ok(())
352}
353
a47a02ae
DM
354#[api(
355 input: {
356 properties: {
357 repository: {
358 schema: REPO_URL_SCHEMA,
359 optional: true,
360 },
361 }
362 }
363)]
364/// Try to login. If successful, store ticket.
365async fn api_login(param: Value) -> Result<Value, Error> {
e240d8be
DM
366
367 let repo = extract_repository_from_value(&param)?;
368
f3fde36b 369 let client = connect(&repo)?;
8a8a4703 370 client.login().await?;
e240d8be
DM
371
372 record_repository(&repo);
373
374 Ok(Value::Null)
375}
376
a47a02ae
DM
377#[api(
378 input: {
379 properties: {
380 repository: {
381 schema: REPO_URL_SCHEMA,
382 optional: true,
383 },
384 }
385 }
386)]
387/// Logout (delete stored ticket).
388fn api_logout(param: Value) -> Result<Value, Error> {
e240d8be
DM
389
390 let repo = extract_repository_from_value(&param)?;
391
5030b7ce 392 delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
e240d8be
DM
393
394 Ok(Value::Null)
395}
396
e39974af
TL
397#[api(
398 input: {
399 properties: {
400 repository: {
401 schema: REPO_URL_SCHEMA,
402 optional: true,
403 },
404 "output-format": {
405 schema: OUTPUT_FORMAT,
406 optional: true,
407 },
408 }
409 }
410)]
411/// Show client and optional server version
412async fn api_version(param: Value) -> Result<(), Error> {
413
414 let output_format = get_output_format(&param);
415
416 let mut version_info = json!({
417 "client": {
a12b1be7
WB
418 "version": pbs_buildcfg::PROXMOX_PKG_VERSION,
419 "release": pbs_buildcfg::PROXMOX_PKG_RELEASE,
420 "repoid": pbs_buildcfg::PROXMOX_PKG_REPOID,
e39974af
TL
421 }
422 });
423
424 let repo = extract_repository_from_value(&param);
425 if let Ok(repo) = repo {
f3fde36b 426 let client = connect(&repo)?;
e39974af
TL
427
428 match client.get("api2/json/version", None).await {
429 Ok(mut result) => version_info["server"] = result["data"].take(),
430 Err(e) => eprintln!("could not connect to server - {}", e),
431 }
432 }
433 if output_format == "text" {
a12b1be7
WB
434 println!(
435 "client version: {}.{}",
436 pbs_buildcfg::PROXMOX_PKG_VERSION,
437 pbs_buildcfg::PROXMOX_PKG_RELEASE,
438 );
e39974af
TL
439 if let Some(server) = version_info["server"].as_object() {
440 let server_version = server["version"].as_str().unwrap();
441 let server_release = server["release"].as_str().unwrap();
442 println!("server version: {}.{}", server_version, server_release);
443 }
444 } else {
445 format_and_print_result(&version_info, &output_format);
446 }
447
448 Ok(())
449}
450
a47a02ae 451#[api(
94913f35 452 input: {
a47a02ae
DM
453 properties: {
454 repository: {
455 schema: REPO_URL_SCHEMA,
456 optional: true,
457 },
94913f35
DM
458 "output-format": {
459 schema: OUTPUT_FORMAT,
460 optional: true,
461 },
462 },
463 },
a47a02ae
DM
464)]
465/// Start garbage collection for a specific repository.
466async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
8cc0d6af 467
2665cef7 468 let repo = extract_repository_from_value(&param)?;
c2043614
DM
469
470 let output_format = get_output_format(&param);
8cc0d6af 471
f3fde36b 472 let mut client = connect(&repo)?;
8cc0d6af 473
d0a03d40 474 let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
8cc0d6af 475
8a8a4703 476 let result = client.post(&path, None).await?;
8cc0d6af 477
8a8a4703 478 record_repository(&repo);
d0a03d40 479
e68269fc 480 view_task_result(&mut client, result, &output_format).await?;
e5f7def4 481
e5f7def4 482 Ok(Value::Null)
8cc0d6af 483}
33d64b81 484
6d233161 485struct CatalogUploadResult {
f1d76ecf 486 catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
6d233161
FG
487 result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
488}
489
bf6e3217 490fn spawn_catalog_upload(
3bad3e6e 491 client: Arc<BackupWriter>,
3638341a 492 encrypt: bool,
6d233161 493) -> Result<CatalogUploadResult, Error> {
f1d99e3f 494 let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
fc5870be 495 let catalog_stream = pbs_tools::blocking::StdChannelStream(catalog_rx);
bf6e3217
DM
496 let catalog_chunk_size = 512*1024;
497 let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
498
f1d76ecf 499 let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)))?));
bf6e3217
DM
500
501 let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
502
e43b9175
FG
503 let upload_options = UploadOptions {
504 encrypt,
505 compress: true,
506 ..UploadOptions::default()
507 };
508
bf6e3217
DM
509 tokio::spawn(async move {
510 let catalog_upload_result = client
e43b9175 511 .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
bf6e3217
DM
512 .await;
513
514 if let Err(ref err) = catalog_upload_result {
515 eprintln!("catalog upload error - {}", err);
516 client.cancel();
517 }
518
519 let _ = catalog_result_tx.send(catalog_upload_result);
520 });
521
6d233161 522 Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
bf6e3217
DM
523}
524
a47a02ae
DM
525#[api(
526 input: {
527 properties: {
528 backupspec: {
529 type: Array,
530 description: "List of backup source specifications ([<label.ext>:<path>] ...)",
531 items: {
532 schema: BACKUP_SOURCE_SCHEMA,
533 }
534 },
535 repository: {
536 schema: REPO_URL_SCHEMA,
537 optional: true,
538 },
539 "include-dev": {
540 description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
541 optional: true,
542 items: {
543 type: String,
544 description: "Path to file.",
545 }
546 },
58fcbf5a
FE
547 "all-file-systems": {
548 type: Boolean,
549 description: "Include all mounted subdirectories.",
550 optional: true,
551 },
a47a02ae
DM
552 keyfile: {
553 schema: KEYFILE_SCHEMA,
554 optional: true,
555 },
0351f23b
WB
556 "keyfd": {
557 schema: KEYFD_SCHEMA,
558 optional: true,
559 },
c0a87c12
FG
560 "master-pubkey-file": {
561 schema: MASTER_PUBKEY_FILE_SCHEMA,
562 optional: true,
563 },
564 "master-pubkey-fd": {
565 schema: MASTER_PUBKEY_FD_SCHEMA,
566 optional: true,
567 },
24be37e3
WB
568 "crypt-mode": {
569 type: CryptMode,
96ee8577
WB
570 optional: true,
571 },
a47a02ae
DM
572 "skip-lost-and-found": {
573 type: Boolean,
574 description: "Skip lost+found directory.",
575 optional: true,
576 },
577 "backup-type": {
578 schema: BACKUP_TYPE_SCHEMA,
579 optional: true,
580 },
581 "backup-id": {
582 schema: BACKUP_ID_SCHEMA,
583 optional: true,
584 },
585 "backup-time": {
586 schema: BACKUP_TIME_SCHEMA,
587 optional: true,
588 },
589 "chunk-size": {
590 schema: CHUNK_SIZE_SCHEMA,
591 optional: true,
592 },
189996cf
CE
593 "exclude": {
594 type: Array,
595 description: "List of paths or patterns for matching files to exclude.",
596 optional: true,
597 items: {
598 type: String,
599 description: "Path or match pattern.",
600 }
601 },
6fc053ed
CE
602 "entries-max": {
603 type: Integer,
604 description: "Max number of entries to hold in memory.",
605 optional: true,
2b7f8dd5 606 default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
6fc053ed 607 },
e02c3d46
DM
608 "verbose": {
609 type: Boolean,
610 description: "Verbose output.",
611 optional: true,
612 },
a47a02ae
DM
613 }
614 }
615)]
616/// Create (host) backup.
617async fn create_backup(
6049b71f
DM
618 param: Value,
619 _info: &ApiMethod,
dd5495d6 620 _rpcenv: &mut dyn RpcEnvironment,
6049b71f 621) -> Result<Value, Error> {
ff5d3707 622
2665cef7 623 let repo = extract_repository_from_value(&param)?;
ae0be2dd 624
3c8c2827 625 let backupspec_list = json::required_array_param(&param, "backupspec")?;
a914a774 626
eed6db39
DM
627 let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
628
5b72c9b4
DM
629 let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
630
219ef0e6
DM
631 let verbose = param["verbose"].as_bool().unwrap_or(false);
632
ca5d0b61
DM
633 let backup_time_opt = param["backup-time"].as_i64();
634
36898ffc 635 let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
2d9d143a 636
247cdbce
DM
637 if let Some(size) = chunk_size_opt {
638 verify_chunk_size(size)?;
2d9d143a
DM
639 }
640
c6a7ea0a 641 let crypto = crypto_parameters(&param)?;
6d0983db 642
f69adc81 643 let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
fba30411 644
bbf9e7e9 645 let backup_type = param["backup-type"].as_str().unwrap_or("host");
ca5d0b61 646
2eeaacb9
DM
647 let include_dev = param["include-dev"].as_array();
648
c443f58b 649 let entries_max = param["entries-max"].as_u64()
2b7f8dd5 650 .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
6fc053ed 651
189996cf 652 let empty = Vec::new();
c443f58b
WB
653 let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
654
239e49f9 655 let mut pattern_list = Vec::with_capacity(exclude_args.len());
c443f58b
WB
656 for entry in exclude_args {
657 let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
239e49f9 658 pattern_list.push(
c443f58b
WB
659 MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
660 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
661 );
189996cf
CE
662 }
663
2eeaacb9
DM
664 let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
665
666 if let Some(include_dev) = include_dev {
667 if all_file_systems {
668 bail!("option 'all-file-systems' conflicts with option 'include-dev'");
669 }
670
671 let mut set = HashSet::new();
672 for path in include_dev {
673 let path = path.as_str().unwrap();
674 let stat = nix::sys::stat::stat(path)
675 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
676 set.insert(stat.st_dev);
677 }
678 devices = Some(set);
679 }
680
ae0be2dd 681 let mut upload_list = vec![];
f2b4b4b9 682 let mut target_set = HashSet::new();
a914a774 683
ae0be2dd 684 for backupspec in backupspec_list {
7cc3473a
DM
685 let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
686 let filename = &spec.config_string;
687 let target = &spec.archive_name;
bcd879cf 688
f2b4b4b9
SI
689 if target_set.contains(target) {
690 bail!("got target twice: '{}'", target);
691 }
692 target_set.insert(target.to_string());
693
eb1804c5
DM
694 use std::os::unix::fs::FileTypeExt;
695
3fa71727
CE
696 let metadata = std::fs::metadata(filename)
697 .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
eb1804c5 698 let file_type = metadata.file_type();
23bb8780 699
7cc3473a
DM
700 match spec.spec_type {
701 BackupSpecificationType::PXAR => {
ec8a9bb9
DM
702 if !file_type.is_dir() {
703 bail!("got unexpected file type (expected directory)");
704 }
7cc3473a 705 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
ec8a9bb9 706 }
7cc3473a 707 BackupSpecificationType::IMAGE => {
ec8a9bb9
DM
708 if !(file_type.is_file() || file_type.is_block_device()) {
709 bail!("got unexpected file type (expected file or block device)");
710 }
eb1804c5 711
e18a6c9e 712 let size = image_size(&PathBuf::from(filename))?;
23bb8780 713
ec8a9bb9 714 if size == 0 { bail!("got zero-sized file '{}'", filename); }
ae0be2dd 715
7cc3473a 716 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
ec8a9bb9 717 }
7cc3473a 718 BackupSpecificationType::CONFIG => {
ec8a9bb9
DM
719 if !file_type.is_file() {
720 bail!("got unexpected file type (expected regular file)");
721 }
7cc3473a 722 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 723 }
7cc3473a 724 BackupSpecificationType::LOGFILE => {
79679c2d
DM
725 if !file_type.is_file() {
726 bail!("got unexpected file type (expected regular file)");
727 }
7cc3473a 728 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
ec8a9bb9 729 }
ae0be2dd
DM
730 }
731 }
732
22a9189e 733 let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
ae0be2dd 734
f3fde36b 735 let client = connect(&repo)?;
d0a03d40
DM
736 record_repository(&repo);
737
6a7be83e 738 println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
ca5d0b61 739
f69adc81 740 println!("Client name: {}", proxmox::tools::nodename());
ca5d0b61 741
6a7be83e 742 let start_time = std::time::Instant::now();
ca5d0b61 743
6a7be83e 744 println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
51144821 745
c6a7ea0a 746 let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
bb823140 747 None => (None, None),
2f26b866
FG
748 Some(key_with_source) => {
749 println!(
750 "{}",
751 format_key_source(&key_with_source.source, "encryption")
752 );
753
754 let (key, created, fingerprint) =
ff8945fd 755 decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
6f2626ae 756 println!("Encryption key fingerprint: {}", fingerprint);
bb823140 757
44288184 758 let crypt_config = CryptConfig::new(key)?;
bb823140 759
c0a87c12 760 match crypto.master_pubkey {
2f26b866
FG
761 Some(pem_with_source) => {
762 println!("{}", format_key_source(&pem_with_source.source, "master"));
763
764 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
82a103c8 765
1c86893d 766 let mut key_config = KeyConfig::without_password(key)?;
82a103c8 767 key_config.created = created; // keep original value
82a103c8 768
8acfd15d 769 let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
6f2626ae 770
05389a01 771 (Some(Arc::new(crypt_config)), Some(enc_key))
2f26b866 772 },
05389a01 773 _ => (Some(Arc::new(crypt_config)), None),
bb823140 774 }
6d0983db
DM
775 }
776 };
f98ac774 777
8a8a4703
DM
778 let client = BackupWriter::start(
779 client,
b957aa81 780 crypt_config.clone(),
8a8a4703
DM
781 repo.store(),
782 backup_type,
783 &backup_id,
784 backup_time,
785 verbose,
61d7b501 786 false
8a8a4703
DM
787 ).await?;
788
8b7f8d3f
FG
789 let download_previous_manifest = match client.previous_backup_time().await {
790 Ok(Some(backup_time)) => {
791 println!(
792 "Downloading previous manifest ({})",
793 strftime_local("%c", backup_time)?
794 );
795 true
796 }
797 Ok(None) => {
798 println!("No previous manifest available.");
799 false
800 }
801 Err(_) => {
802 // Fallback for outdated server, TODO remove/bubble up with 2.0
803 true
804 }
805 };
806
807 let previous_manifest = if download_previous_manifest {
808 match client.download_previous_manifest().await {
809 Ok(previous_manifest) => {
810 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
811 Ok(()) => Some(Arc::new(previous_manifest)),
812 Err(err) => {
813 println!("Couldn't re-use previous manifest - {}", err);
814 None
815 }
816 }
23f9503a 817 }
8b7f8d3f
FG
818 Err(err) => {
819 println!("Couldn't download previous manifest - {}", err);
820 None
821 }
822 }
823 } else {
824 None
b957aa81
DM
825 };
826
6a7be83e 827 let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
8a8a4703
DM
828 let mut manifest = BackupManifest::new(snapshot);
829
5d85847f 830 let mut catalog = None;
6d233161 831 let mut catalog_result_rx = None;
8a8a4703
DM
832
833 for (backup_type, filename, target, size) in upload_list {
834 match backup_type {
7cc3473a 835 BackupSpecificationType::CONFIG => {
e43b9175
FG
836 let upload_options = UploadOptions {
837 compress: true,
c6a7ea0a 838 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
839 ..UploadOptions::default()
840 };
841
5b32820e 842 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 843 let stats = client
e43b9175 844 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 845 .await?;
c6a7ea0a 846 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 847 }
7cc3473a 848 BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
e43b9175
FG
849 let upload_options = UploadOptions {
850 compress: true,
c6a7ea0a 851 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
852 ..UploadOptions::default()
853 };
854
5b32820e 855 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
8a8a4703 856 let stats = client
e43b9175 857 .upload_blob_from_file(&filename, &target, upload_options)
8a8a4703 858 .await?;
c6a7ea0a 859 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703 860 }
7cc3473a 861 BackupSpecificationType::PXAR => {
5d85847f
DC
862 // start catalog upload on first use
863 if catalog.is_none() {
c6a7ea0a 864 let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
6d233161
FG
865 catalog = Some(catalog_upload_res.catalog_writer);
866 catalog_result_rx = Some(catalog_upload_res.result);
5d85847f
DC
867 }
868 let catalog = catalog.as_ref().unwrap();
869
5b32820e 870 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
8a8a4703 871 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
77486a60 872
2b7f8dd5 873 let pxar_options = pbs_client::pxar::PxarCreateOptions {
77486a60
FG
874 device_set: devices.clone(),
875 patterns: pattern_list.clone(),
876 entries_max: entries_max as usize,
877 skip_lost_and_found,
878 verbose,
879 };
880
e43b9175
FG
881 let upload_options = UploadOptions {
882 previous_manifest: previous_manifest.clone(),
883 compress: true,
c6a7ea0a 884 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
885 ..UploadOptions::default()
886 };
887
8a8a4703
DM
888 let stats = backup_directory(
889 &client,
890 &filename,
891 &target,
892 chunk_size_opt,
8a8a4703 893 catalog.clone(),
77486a60 894 pxar_options,
e43b9175 895 upload_options,
8a8a4703 896 ).await?;
c6a7ea0a 897 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
8a8a4703
DM
898 catalog.lock().unwrap().end_directory()?;
899 }
7cc3473a 900 BackupSpecificationType::IMAGE => {
8a8a4703 901 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
e43b9175
FG
902
903 let upload_options = UploadOptions {
904 previous_manifest: previous_manifest.clone(),
905 fixed_size: Some(size),
906 compress: true,
c6a7ea0a 907 encrypt: crypto.mode == CryptMode::Encrypt,
e43b9175
FG
908 };
909
8a8a4703
DM
910 let stats = backup_image(
911 &client,
e43b9175 912 &filename,
8a8a4703 913 &target,
8a8a4703 914 chunk_size_opt,
e43b9175 915 upload_options,
8a8a4703 916 ).await?;
c6a7ea0a 917 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
6af905c1
DM
918 }
919 }
8a8a4703 920 }
4818c8b6 921
8a8a4703 922 // finalize and upload catalog
5d85847f 923 if let Some(catalog) = catalog {
8a8a4703
DM
924 let mutex = Arc::try_unwrap(catalog)
925 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
926 let mut catalog = mutex.into_inner().unwrap();
bf6e3217 927
8a8a4703 928 catalog.finish()?;
2761d6a4 929
8a8a4703 930 drop(catalog); // close upload stream
2761d6a4 931
6d233161 932 if let Some(catalog_result_rx) = catalog_result_rx {
5d85847f 933 let stats = catalog_result_rx.await??;
c6a7ea0a 934 manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
5d85847f 935 }
8a8a4703 936 }
2761d6a4 937
8a8a4703 938 if let Some(rsa_encrypted_key) = rsa_encrypted_key {
9990af30 939 let target = ENCRYPTED_KEY_BLOB_NAME;
8a8a4703 940 println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
e43b9175 941 let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
8a8a4703 942 let stats = client
e43b9175 943 .upload_blob_from_data(rsa_encrypted_key, target, options)
8a8a4703 944 .await?;
c6a7ea0a 945 manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
8a8a4703 946
8a8a4703 947 }
8a8a4703 948 // create manifest (index.json)
3638341a 949 // manifests are never encrypted, but include a signature
dfa517ad 950 let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
b53f6379 951 .map_err(|err| format_err!("unable to format manifest - {}", err))?;
3638341a 952
b53f6379 953
9688f6de 954 if verbose { println!("Upload index.json to '{}'", repo) };
e43b9175 955 let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
8a8a4703 956 client
e43b9175 957 .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
8a8a4703 958 .await?;
2c3891d1 959
8a8a4703 960 client.finish().await?;
c4ff3dce 961
6a7be83e
DM
962 let end_time = std::time::Instant::now();
963 let elapsed = end_time.duration_since(start_time);
964 println!("Duration: {:.2}s", elapsed.as_secs_f64());
3ec3ec3f 965
6a7be83e 966 println!("End Time: {}", strftime_local("%c", epoch_i64())?);
3d5c11e5 967
8a8a4703 968 Ok(Value::Null)
f98ea63d
DM
969}
970
8e6e18b7 971async fn dump_image<W: Write>(
88892ea8
DM
972 client: Arc<BackupReader>,
973 crypt_config: Option<Arc<CryptConfig>>,
14f6c9cb 974 crypt_mode: CryptMode,
88892ea8
DM
975 index: FixedIndexReader,
976 mut writer: W,
fd04ca7a 977 verbose: bool,
88892ea8
DM
978) -> Result<(), Error> {
979
980 let most_used = index.find_most_used_chunks(8);
981
14f6c9cb 982 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
88892ea8
DM
983
984 // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
985 // and thus slows down reading. Instead, directly use RemoteChunkReader
fd04ca7a
DM
986 let mut per = 0;
987 let mut bytes = 0;
988 let start_time = std::time::Instant::now();
989
88892ea8
DM
990 for pos in 0..index.index_count() {
991 let digest = index.index_digest(pos).unwrap();
8e6e18b7 992 let raw_data = chunk_reader.read_chunk(&digest).await?;
88892ea8 993 writer.write_all(&raw_data)?;
fd04ca7a
DM
994 bytes += raw_data.len();
995 if verbose {
996 let next_per = ((pos+1)*100)/index.index_count();
997 if per != next_per {
998 eprintln!("progress {}% (read {} bytes, duration {} sec)",
999 next_per, bytes, start_time.elapsed().as_secs());
1000 per = next_per;
1001 }
1002 }
88892ea8
DM
1003 }
1004
fd04ca7a
DM
1005 let end_time = std::time::Instant::now();
1006 let elapsed = end_time.duration_since(start_time);
1007 eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1008 bytes,
1009 elapsed.as_secs_f64(),
1010 bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1011 );
1012
1013
88892ea8
DM
1014 Ok(())
1015}
1016
dc155e9b 1017fn parse_archive_type(name: &str) -> (String, ArchiveType) {
2d32fe2c
TL
1018 if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1019 (name.into(), archive_type(name).unwrap())
1020 } else if name.ends_with(".pxar") {
dc155e9b
TL
1021 (format!("{}.didx", name), ArchiveType::DynamicIndex)
1022 } else if name.ends_with(".img") {
1023 (format!("{}.fidx", name), ArchiveType::FixedIndex)
1024 } else {
1025 (format!("{}.blob", name), ArchiveType::Blob)
1026 }
1027}
1028
a47a02ae
DM
1029#[api(
1030 input: {
1031 properties: {
1032 repository: {
1033 schema: REPO_URL_SCHEMA,
1034 optional: true,
1035 },
1036 snapshot: {
1037 type: String,
1038 description: "Group/Snapshot path.",
1039 },
1040 "archive-name": {
1041 description: "Backup archive name.",
1042 type: String,
1043 },
1044 target: {
1045 type: String,
90c815bf 1046 description: r###"Target directory path. Use '-' to write to standard output.
8a8a4703 1047
d1d74c43 1048We do not extract '.pxar' archives when writing to standard output.
8a8a4703 1049
a47a02ae
DM
1050"###
1051 },
1052 "allow-existing-dirs": {
1053 type: Boolean,
1054 description: "Do not fail if directories already exists.",
1055 optional: true,
1056 },
1057 keyfile: {
1058 schema: KEYFILE_SCHEMA,
1059 optional: true,
1060 },
0351f23b
WB
1061 "keyfd": {
1062 schema: KEYFD_SCHEMA,
1063 optional: true,
1064 },
24be37e3
WB
1065 "crypt-mode": {
1066 type: CryptMode,
96ee8577
WB
1067 optional: true,
1068 },
a47a02ae
DM
1069 }
1070 }
1071)]
1072/// Restore backup repository.
1073async fn restore(param: Value) -> Result<Value, Error> {
2665cef7 1074 let repo = extract_repository_from_value(&param)?;
9f912493 1075
86eda3eb
DM
1076 let verbose = param["verbose"].as_bool().unwrap_or(false);
1077
46d5aa0a
DM
1078 let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1079
3c8c2827 1080 let archive_name = json::required_string_param(&param, "archive-name")?;
d5c34d98 1081
f3fde36b 1082 let client = connect(&repo)?;
d0a03d40 1083
d0a03d40 1084 record_repository(&repo);
d5c34d98 1085
3c8c2827 1086 let path = json::required_string_param(&param, "snapshot")?;
9f912493 1087
86eda3eb 1088 let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
d6d3b353 1089 let group: BackupGroup = path.parse()?;
27c9affb 1090 api_datastore_latest_snapshot(&client, repo.store(), group).await?
d5c34d98 1091 } else {
a67f7d0a 1092 let snapshot: BackupDir = path.parse()?;
86eda3eb
DM
1093 (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1094 };
9f912493 1095
3c8c2827 1096 let target = json::required_string_param(&param, "target")?;
bf125261 1097 let target = if target == "-" { None } else { Some(target) };
2ae7d196 1098
c6a7ea0a 1099 let crypto = crypto_parameters(&param)?;
2ae7d196 1100
c6a7ea0a 1101 let crypt_config = match crypto.enc_key {
86eda3eb 1102 None => None,
2f26b866
FG
1103 Some(ref key) => {
1104 let (key, _, _) =
ff8945fd 1105 decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
2f26b866
FG
1106 eprintln!("{}", format_key_source(&key.source, "encryption"));
1107 err
1108 })?;
86eda3eb
DM
1109 Some(Arc::new(CryptConfig::new(key)?))
1110 }
1111 };
d5c34d98 1112
296c50ba
DM
1113 let client = BackupReader::start(
1114 client,
1115 crypt_config.clone(),
1116 repo.store(),
1117 &backup_type,
1118 &backup_id,
1119 backup_time,
1120 true,
1121 ).await?;
86eda3eb 1122
48fbbfeb
FG
1123 let (archive_name, archive_type) = parse_archive_type(archive_name);
1124
2107a5ae 1125 let (manifest, backup_index_data) = client.download_manifest().await?;
02fcf372 1126
48fbbfeb
FG
1127 if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1128 eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1129 } else {
2f26b866
FG
1130 if manifest.signature.is_some() {
1131 if let Some(key) = &crypto.enc_key {
1132 eprintln!("{}", format_key_source(&key.source, "encryption"));
1133 }
1134 if let Some(config) = &crypt_config {
bbdda58b 1135 eprintln!("Fingerprint: {}", Fingerprint::new(config.fingerprint()));
2f26b866
FG
1136 }
1137 }
48fbbfeb
FG
1138 manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1139 }
dc155e9b
TL
1140
1141 if archive_name == MANIFEST_BLOB_NAME {
02fcf372 1142 if let Some(target) = target {
2107a5ae 1143 replace_file(target, &backup_index_data, CreateOptions::new())?;
02fcf372
DM
1144 } else {
1145 let stdout = std::io::stdout();
1146 let mut writer = stdout.lock();
2107a5ae 1147 writer.write_all(&backup_index_data)
02fcf372
DM
1148 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1149 }
1150
14f6c9cb
FG
1151 return Ok(Value::Null);
1152 }
1153
1154 let file_info = manifest.lookup_file_info(&archive_name)?;
1155
1156 if archive_type == ArchiveType::Blob {
d2267b11 1157
dc155e9b 1158 let mut reader = client.download_blob(&manifest, &archive_name).await?;
f8100e96 1159
bf125261 1160 if let Some(target) = target {
0d986280
DM
1161 let mut writer = std::fs::OpenOptions::new()
1162 .write(true)
1163 .create(true)
1164 .create_new(true)
1165 .open(target)
1166 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1167 std::io::copy(&mut reader, &mut writer)?;
bf125261
DM
1168 } else {
1169 let stdout = std::io::stdout();
1170 let mut writer = stdout.lock();
0d986280 1171 std::io::copy(&mut reader, &mut writer)
bf125261
DM
1172 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1173 }
f8100e96 1174
dc155e9b 1175 } else if archive_type == ArchiveType::DynamicIndex {
86eda3eb 1176
dc155e9b 1177 let index = client.download_dynamic_index(&manifest, &archive_name).await?;
df65bd3d 1178
f4bf7dfc
DM
1179 let most_used = index.find_most_used_chunks(8);
1180
14f6c9cb 1181 let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
f4bf7dfc 1182
afb4cd28 1183 let mut reader = BufferedDynamicReader::new(index, chunk_reader);
86eda3eb 1184
2b7f8dd5 1185 let options = pbs_client::pxar::PxarExtractOptions {
72064fd0
FG
1186 match_list: &[],
1187 extract_match_default: true,
1188 allow_existing_dirs,
1189 on_error: None,
1190 };
1191
bf125261 1192 if let Some(target) = target {
2b7f8dd5 1193 pbs_client::pxar::extract_archive(
c443f58b
WB
1194 pxar::decoder::Decoder::from_std(reader)?,
1195 Path::new(target),
2b7f8dd5 1196 pbs_client::pxar::Flags::DEFAULT,
c443f58b
WB
1197 |path| {
1198 if verbose {
1199 println!("{:?}", path);
1200 }
1201 },
72064fd0 1202 options,
c443f58b
WB
1203 )
1204 .map_err(|err| format_err!("error extracting archive - {}", err))?;
bf125261 1205 } else {
88892ea8
DM
1206 let mut writer = std::fs::OpenOptions::new()
1207 .write(true)
1208 .open("/dev/stdout")
1209 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
afb4cd28 1210
bf125261
DM
1211 std::io::copy(&mut reader, &mut writer)
1212 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1213 }
dc155e9b 1214 } else if archive_type == ArchiveType::FixedIndex {
afb4cd28 1215
dc155e9b 1216 let index = client.download_fixed_index(&manifest, &archive_name).await?;
df65bd3d 1217
88892ea8
DM
1218 let mut writer = if let Some(target) = target {
1219 std::fs::OpenOptions::new()
bf125261
DM
1220 .write(true)
1221 .create(true)
1222 .create_new(true)
1223 .open(target)
88892ea8 1224 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
bf125261 1225 } else {
88892ea8
DM
1226 std::fs::OpenOptions::new()
1227 .write(true)
1228 .open("/dev/stdout")
1229 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1230 };
afb4cd28 1231
14f6c9cb 1232 dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
3031e44c 1233 }
fef44d4f
DM
1234
1235 Ok(Value::Null)
45db6f89
DM
1236}
1237
e0665a64
DC
1238#[api(
1239 input: {
1240 properties: {
1241 "dry-run": {
1242 type: bool,
1243 optional: true,
1244 description: "Just show what prune would do, but do not delete anything.",
1245 },
1246 group: {
1247 type: String,
1248 description: "Backup group",
1249 },
1250 "prune-options": {
1251 type: PruneOptions,
1252 flatten: true,
1253 },
1254 "output-format": {
1255 schema: OUTPUT_FORMAT,
1256 optional: true,
1257 },
1258 quiet: {
1259 type: bool,
1260 optional: true,
1261 default: false,
1262 description: "Minimal output - only show removals.",
1263 },
1264 repository: {
1265 schema: REPO_URL_SCHEMA,
1266 optional: true,
1267 },
1268 },
1269 },
1270)]
1271/// Prune a backup repository.
1272async fn prune(
1273 dry_run: Option<bool>,
1274 group: String,
1275 prune_options: PruneOptions,
1276 quiet: bool,
1277 mut param: Value
1278) -> Result<Value, Error> {
2665cef7 1279 let repo = extract_repository_from_value(&param)?;
83b7db02 1280
f3fde36b 1281 let mut client = connect(&repo)?;
83b7db02 1282
d0a03d40 1283 let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
83b7db02 1284
d6d3b353 1285 let group: BackupGroup = group.parse()?;
c2043614 1286
671c6a96 1287 let output_format = extract_output_format(&mut param);
9fdc3ef4 1288
e0665a64
DC
1289 let mut api_param = serde_json::to_value(prune_options)?;
1290 if let Some(dry_run) = dry_run {
1291 api_param["dry-run"] = dry_run.into();
1292 }
1293 api_param["backup-type"] = group.backup_type().into();
1294 api_param["backup-id"] = group.backup_id().into();
83b7db02 1295
e0665a64 1296 let mut result = client.post(&path, Some(api_param)).await?;
74fa81b8 1297
87c42375 1298 record_repository(&repo);
3b03abfe 1299
db1e061d
DM
1300 let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1301 let item: PruneListItem = serde_json::from_value(record.to_owned())?;
e0e5b442 1302 let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
db1e061d
DM
1303 Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1304 };
1305
c48aa39f
DM
1306 let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1307 Ok(match v.as_bool() {
1308 Some(true) => "keep",
1309 Some(false) => "remove",
1310 None => "unknown",
1311 }.to_string())
1312 };
1313
db1e061d
DM
1314 let options = default_table_format_options()
1315 .sortby("backup-type", false)
1316 .sortby("backup-id", false)
1317 .sortby("backup-time", false)
1318 .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
770a36e5 1319 .column(ColumnConfig::new("backup-time").renderer(pbs_tools::format::render_epoch).header("date"))
c48aa39f 1320 .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
db1e061d
DM
1321 ;
1322
e351ac78 1323 let return_type = &pbs_api_types::ADMIN_DATASTORE_PRUNE_RETURN_TYPE;
db1e061d
DM
1324
1325 let mut data = result["data"].take();
1326
c48aa39f
DM
1327 if quiet {
1328 let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1329 item["keep"].as_bool() == Some(false)
a375df6f 1330 }).cloned().collect();
c48aa39f
DM
1331 data = list.into();
1332 }
1333
b2362a12 1334 format_and_print_result_full(&mut data, return_type, &output_format, &options);
d0a03d40 1335
43a406fd 1336 Ok(Value::Null)
83b7db02
DM
1337}
1338
a47a02ae
DM
1339#[api(
1340 input: {
1341 properties: {
1342 repository: {
1343 schema: REPO_URL_SCHEMA,
1344 optional: true,
1345 },
1346 "output-format": {
1347 schema: OUTPUT_FORMAT,
1348 optional: true,
1349 },
1350 }
f9beae9c
TL
1351 },
1352 returns: {
1353 type: StorageStatus,
1354 },
a47a02ae
DM
1355)]
1356/// Get repository status.
1357async fn status(param: Value) -> Result<Value, Error> {
34a816cc
DM
1358
1359 let repo = extract_repository_from_value(&param)?;
1360
c2043614 1361 let output_format = get_output_format(&param);
34a816cc 1362
f3fde36b 1363 let client = connect(&repo)?;
34a816cc
DM
1364
1365 let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1366
1dc117bb 1367 let mut result = client.get(&path, None).await?;
14e08625 1368 let mut data = result["data"].take();
34a816cc
DM
1369
1370 record_repository(&repo);
1371
390c5bdd
DM
1372 let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1373 let v = v.as_u64().unwrap();
1374 let total = record["total"].as_u64().unwrap();
1375 let roundup = total/200;
1376 let per = ((v+roundup)*100)/total;
e23f5863
DM
1377 let info = format!(" ({} %)", per);
1378 Ok(format!("{} {:>8}", v, info))
390c5bdd 1379 };
1dc117bb 1380
c2043614 1381 let options = default_table_format_options()
be2425ff 1382 .noheader(true)
e23f5863 1383 .column(ColumnConfig::new("total").renderer(render_total_percentage))
390c5bdd
DM
1384 .column(ColumnConfig::new("used").renderer(render_total_percentage))
1385 .column(ColumnConfig::new("avail").renderer(render_total_percentage));
34a816cc 1386
b2362a12 1387 let return_type = &API_METHOD_STATUS.returns;
390c5bdd 1388
b2362a12 1389 format_and_print_result_full(&mut data, return_type, &output_format, &options);
34a816cc
DM
1390
1391 Ok(Value::Null)
1392}
1393
c443f58b
WB
1394/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1395/// async use!
1396///
1397/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1398/// so that we can properly access it from multiple threads simultaneously while not issuing
1399/// duplicate simultaneous reads over http.
43abba4b 1400pub struct BufferedDynamicReadAt {
c443f58b
WB
1401 inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1402}
1403
1404impl BufferedDynamicReadAt {
1405 fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
1406 Self {
1407 inner: Mutex::new(inner),
1408 }
1409 }
1410}
1411
a6f87283
WB
1412impl ReadAt for BufferedDynamicReadAt {
1413 fn start_read_at<'a>(
1414 self: Pin<&'a Self>,
c443f58b 1415 _cx: &mut Context,
a6f87283 1416 buf: &'a mut [u8],
c443f58b 1417 offset: u64,
a6f87283 1418 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
a6f87283 1419 MaybeReady::Ready(tokio::task::block_in_place(move || {
c443f58b
WB
1420 let mut reader = self.inner.lock().unwrap();
1421 reader.seek(SeekFrom::Start(offset))?;
a6f87283
WB
1422 Ok(reader.read(buf)?)
1423 }))
1424 }
1425
1426 fn poll_complete<'a>(
1427 self: Pin<&'a Self>,
1428 _op: ReadAtOperation<'a>,
1429 ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
bbc71e3b 1430 panic!("BufferedDynamicReadAt::start_read_at returned Pending");
c443f58b
WB
1431 }
1432}
1433
f2401311 1434fn main() {
33d64b81 1435
255f378a 1436 let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
49fddd98 1437 .arg_param(&["backupspec"])
d0a03d40 1438 .completion_cb("repository", complete_repository)
49811347 1439 .completion_cb("backupspec", complete_backup_source)
2b7f8dd5
WB
1440 .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
1441 .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
49811347 1442 .completion_cb("chunk-size", complete_chunk_size);
f8838fe9 1443
caea8d61
DM
1444 let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
1445 .completion_cb("repository", complete_repository)
2b7f8dd5 1446 .completion_cb("keyfile", pbs_tools::fs::complete_file_name);
caea8d61 1447
255f378a 1448 let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
d0a03d40 1449 .completion_cb("repository", complete_repository);
41c039e1 1450
255f378a 1451 let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
d0a03d40 1452 .completion_cb("repository", complete_repository);
8cc0d6af 1453
255f378a 1454 let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
49fddd98 1455 .arg_param(&["snapshot", "archive-name", "target"])
b2388518 1456 .completion_cb("repository", complete_repository)
08dc340a
DM
1457 .completion_cb("snapshot", complete_group_or_snapshot)
1458 .completion_cb("archive-name", complete_archive_name)
2b7f8dd5 1459 .completion_cb("target", pbs_tools::fs::complete_file_name);
9f912493 1460
255f378a 1461 let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
49fddd98 1462 .arg_param(&["group"])
9fdc3ef4 1463 .completion_cb("group", complete_backup_group)
d0a03d40 1464 .completion_cb("repository", complete_repository);
9f912493 1465
255f378a 1466 let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
34a816cc
DM
1467 .completion_cb("repository", complete_repository);
1468
255f378a 1469 let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
e240d8be
DM
1470 .completion_cb("repository", complete_repository);
1471
255f378a 1472 let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
e240d8be 1473 .completion_cb("repository", complete_repository);
32efac1c 1474
e39974af
TL
1475 let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
1476 .completion_cb("repository", complete_repository);
1477
344add38
DW
1478 let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
1479 .arg_param(&["group", "new-owner"])
1480 .completion_cb("group", complete_backup_group)
0224c3c2 1481 .completion_cb("new-owner", complete_auth_id)
344add38
DW
1482 .completion_cb("repository", complete_repository);
1483
41c039e1 1484 let cmd_def = CliCommandMap::new()
48ef3c33 1485 .insert("backup", backup_cmd_def)
48ef3c33
DM
1486 .insert("garbage-collect", garbage_collect_cmd_def)
1487 .insert("list", list_cmd_def)
1488 .insert("login", login_cmd_def)
1489 .insert("logout", logout_cmd_def)
1490 .insert("prune", prune_cmd_def)
1491 .insert("restore", restore_cmd_def)
a65e3e4b 1492 .insert("snapshot", snapshot_mgtm_cli())
48ef3c33 1493 .insert("status", status_cmd_def)
9696f519 1494 .insert("key", key::cli())
43abba4b 1495 .insert("mount", mount_cmd_def())
45f9b32e
SR
1496 .insert("map", map_cmd_def())
1497 .insert("unmap", unmap_cmd_def())
5830c205 1498 .insert("catalog", catalog_mgmt_cli())
caea8d61 1499 .insert("task", task_mgmt_cli())
e39974af 1500 .insert("version", version_cmd_def)
344add38 1501 .insert("benchmark", benchmark_cmd_def)
731eeef2
DM
1502 .insert("change-owner", change_owner_cmd_def)
1503
61205f00 1504 .alias(&["files"], &["snapshot", "files"])
edebd523 1505 .alias(&["forget"], &["snapshot", "forget"])
0c9209b0 1506 .alias(&["upload-log"], &["snapshot", "upload-log"])
731eeef2
DM
1507 .alias(&["snapshots"], &["snapshot", "list"])
1508 ;
48ef3c33 1509
7b22acd0
DM
1510 let rpcenv = CliEnvironment::new();
1511 run_cli_command(cmd_def, rpcenv, Some(|future| {
d420962f 1512 pbs_runtime::main(future)
d08bc483 1513 }));
ff5d3707 1514}