src/bin/proxmox-backup-client.rs

   1 use std::collections::{HashSet, HashMap};
   2 use std::io::{self, Write, Seek, SeekFrom};
   3 use std::os::unix::fs::OpenOptionsExt;
   4 use std::path::{Path, PathBuf};
   5 use std::pin::Pin;
   6 use std::sync::{Arc, Mutex};
   7 use std::task::Context;
   8
   9 use anyhow::{bail, format_err, Error};
  10 use chrono::{Local, DateTime, Utc, TimeZone};
  11 use futures::future::FutureExt;
  12 use futures::stream::{StreamExt, TryStreamExt};
  13 use serde_json::{json, Value};
  14 use tokio::sync::mpsc;
  15 use xdg::BaseDirectories;
  16
  17 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
  18 use proxmox::{sortable, identity};
  19 use proxmox::tools::fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size};
  20 use proxmox::sys::linux::tty;
  21 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
  22 use proxmox::api::schema::*;
  23 use proxmox::api::cli::*;
  24 use proxmox::api::api;
  25 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
  26
  27 use proxmox_backup::tools;
  28 use proxmox_backup::api2::types::*;
  29 use proxmox_backup::client::*;
  30 use proxmox_backup::pxar::catalog::*;
  31 use proxmox_backup::backup::{
  32     archive_type,
  33     encrypt_key_with_passphrase,
  34     load_and_decrypt_key,
  35     store_key_config,
  36     verify_chunk_size,
  37     ArchiveType,
  38     AsyncReadChunk,
  39     BackupDir,
  40     BackupGroup,
  41     BackupManifest,
  42     BufferedDynamicReader,
  43     CatalogReader,
  44     CatalogWriter,
  45     CATALOG_NAME,
  46     ChunkStream,
  47     CryptConfig,
  48     DataBlob,
  49     DynamicIndexReader,
  50     FixedChunkStream,
  51     FixedIndexReader,
  52     IndexFile,
  53     KeyConfig,
  54     MANIFEST_BLOB_NAME,
  55     Shell,
  56 };
  57
  58 mod proxmox_backup_client;
  59 use proxmox_backup_client::*;
  60
  61 const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
  62 const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
  63
  64
  65 pub const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  66     .format(&BACKUP_REPO_URL)
  67     .max_length(256)
  68     .schema();
  69
  70 pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
  71     "Path to encryption key. All data will be encrypted using this key.")
  72     .schema();
  73
  74 const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new(
  75     "Chunk size in KB. Must be a power of 2.")
  76     .minimum(64)
  77     .maximum(4096)
  78     .default(4096)
  79     .schema();
  80
  81 fn get_default_repository() -> Option<String> {
  82     std::env::var("PBS_REPOSITORY").ok()
  83 }
  84
  85 pub fn extract_repository_from_value(
  86     param: &Value,
  87 ) -> Result<BackupRepository, Error> {
  88
  89     let repo_url = param["repository"]
  90         .as_str()
  91         .map(String::from)
  92         .or_else(get_default_repository)
  93         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
  94
  95     let repo: BackupRepository = repo_url.parse()?;
  96
  97     Ok(repo)
  98 }
  99
 100 fn extract_repository_from_map(
 101     param: &HashMap<String, String>,
 102 ) -> Option<BackupRepository> {
 103
 104     param.get("repository")
 105         .map(String::from)
 106         .or_else(get_default_repository)
 107         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
 108 }
 109
 110 fn record_repository(repo: &BackupRepository) {
 111
 112     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 113         Ok(v) => v,
 114         _ => return,
 115     };
 116
 117     // usually $HOME/.cache/proxmox-backup/repo-list
 118     let path = match base.place_cache_file("repo-list") {
 119         Ok(v) => v,
 120         _ => return,
 121     };
 122
 123     let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 124
 125     let repo = repo.to_string();
 126
 127     data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
 128
 129     let mut map = serde_json::map::Map::new();
 130
 131     loop {
 132         let mut max_used = 0;
 133         let mut max_repo = None;
 134         for (repo, count) in data.as_object().unwrap() {
 135             if map.contains_key(repo) { continue; }
 136             if let Some(count) = count.as_i64() {
 137                 if count > max_used {
 138                     max_used = count;
 139                     max_repo = Some(repo);
 140                 }
 141             }
 142         }
 143         if let Some(repo) = max_repo {
 144             map.insert(repo.to_owned(), json!(max_used));
 145         } else {
 146             break;
 147         }
 148         if map.len() > 10 { // store max. 10 repos
 149             break;
 150         }
 151     }
 152
 153     let new_data = json!(map);
 154
 155     let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
 156 }
 157
 158 pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 159
 160     let mut result = vec![];
 161
 162     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 163         Ok(v) => v,
 164         _ => return result,
 165     };
 166
 167     // usually $HOME/.cache/proxmox-backup/repo-list
 168     let path = match base.place_cache_file("repo-list") {
 169         Ok(v) => v,
 170         _ => return result,
 171     };
 172
 173     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 174
 175     if let Some(map) = data.as_object() {
 176         for (repo, _count) in map {
 177             result.push(repo.to_owned());
 178         }
 179     }
 180
 181     result
 182 }
 183
 184 fn connect(server: &str, userid: &str) -> Result<HttpClient, Error> {
 185
 186     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 187
 188     use std::env::VarError::*;
 189     let password = match std::env::var(ENV_VAR_PBS_PASSWORD) {
 190         Ok(p) => Some(p),
 191         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", ENV_VAR_PBS_PASSWORD)),
 192         Err(NotPresent) => None,
 193     };
 194
 195     let options = HttpClientOptions::new()
 196         .prefix(Some("proxmox-backup".to_string()))
 197         .password(password)
 198         .interactive(true)
 199         .fingerprint(fingerprint)
 200         .fingerprint_cache(true)
 201         .ticket_cache(true);
 202
 203     HttpClient::new(server, userid, options)
 204 }
 205
 206 async fn view_task_result(
 207     client: HttpClient,
 208     result: Value,
 209     output_format: &str,
 210 ) -> Result<(), Error> {
 211     let data = &result["data"];
 212     if output_format == "text" {
 213         if let Some(upid) = data.as_str() {
 214             display_task_log(client, upid, true).await?;
 215         }
 216     } else {
 217         format_and_print_result(&data, &output_format);
 218     }
 219
 220     Ok(())
 221 }
 222
 223 async fn api_datastore_list_snapshots(
 224     client: &HttpClient,
 225     store: &str,
 226     group: Option<BackupGroup>,
 227 ) -> Result<Value, Error> {
 228
 229     let path = format!("api2/json/admin/datastore/{}/snapshots", store);
 230
 231     let mut args = json!({});
 232     if let Some(group) = group {
 233         args["backup-type"] = group.backup_type().into();
 234         args["backup-id"] = group.backup_id().into();
 235     }
 236
 237     let mut result = client.get(&path, Some(args)).await?;
 238
 239     Ok(result["data"].take())
 240 }
 241
 242 pub async fn api_datastore_latest_snapshot(
 243     client: &HttpClient,
 244     store: &str,
 245     group: BackupGroup,
 246 ) -> Result<(String, String, DateTime<Utc>), Error> {
 247
 248     let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
 249     let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
 250
 251     if list.is_empty() {
 252         bail!("backup group {:?} does not contain any snapshots.", group.group_path());
 253     }
 254
 255     list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
 256
 257     let backup_time = Utc.timestamp(list[0].backup_time, 0);
 258
 259     Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
 260 }
 261
 262 async fn backup_directory<P: AsRef<Path>>(
 263     client: &BackupWriter,
 264     previous_manifest: Option<Arc<BackupManifest>>,
 265     dir_path: P,
 266     archive_name: &str,
 267     chunk_size: Option<usize>,
 268     device_set: Option<HashSet<u64>>,
 269     verbose: bool,
 270     skip_lost_and_found: bool,
 271     catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
 272     exclude_pattern: Vec<MatchEntry>,
 273     entries_max: usize,
 274 ) -> Result<BackupStats, Error> {
 275
 276     let pxar_stream = PxarBackupStream::open(
 277         dir_path.as_ref(),
 278         device_set,
 279         verbose,
 280         skip_lost_and_found,
 281         catalog,
 282         exclude_pattern,
 283         entries_max,
 284     )?;
 285     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 286
 287     let (mut tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
 288
 289     let stream = rx
 290         .map_err(Error::from);
 291
 292     // spawn chunker inside a separate task so that it can run parallel
 293     tokio::spawn(async move {
 294         while let Some(v) = chunk_stream.next().await {
 295             let _ = tx.send(v).await;
 296         }
 297     });
 298
 299     let stats = client
 300         .upload_stream(previous_manifest, archive_name, stream, "dynamic", None)
 301         .await?;
 302
 303     Ok(stats)
 304 }
 305
 306 async fn backup_image<P: AsRef<Path>>(
 307     client: &BackupWriter,
 308     previous_manifest: Option<Arc<BackupManifest>>,
 309     image_path: P,
 310     archive_name: &str,
 311     image_size: u64,
 312     chunk_size: Option<usize>,
 313     _verbose: bool,
 314 ) -> Result<BackupStats, Error> {
 315
 316     let path = image_path.as_ref().to_owned();
 317
 318     let file = tokio::fs::File::open(path).await?;
 319
 320     let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
 321         .map_err(Error::from);
 322
 323     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 324
 325     let stats = client
 326         .upload_stream(previous_manifest, archive_name, stream, "fixed", Some(image_size))
 327         .await?;
 328
 329     Ok(stats)
 330 }
 331
 332 #[api(
 333    input: {
 334         properties: {
 335             repository: {
 336                 schema: REPO_URL_SCHEMA,
 337                 optional: true,
 338             },
 339             "output-format": {
 340                 schema: OUTPUT_FORMAT,
 341                 optional: true,
 342             },
 343         }
 344    }
 345 )]
 346 /// List backup groups.
 347 async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 348
 349     let output_format = get_output_format(&param);
 350
 351     let repo = extract_repository_from_value(&param)?;
 352
 353     let client = connect(repo.host(), repo.user())?;
 354
 355     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 356
 357     let mut result = client.get(&path, None).await?;
 358
 359     record_repository(&repo);
 360
 361     let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
 362         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 363         let group = BackupGroup::new(item.backup_type, item.backup_id);
 364         Ok(group.group_path().to_str().unwrap().to_owned())
 365     };
 366
 367     let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
 368         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 369         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup);
 370         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
 371     };
 372
 373     let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
 374         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 375         Ok(tools::format::render_backup_file_list(&item.files))
 376     };
 377
 378     let options = default_table_format_options()
 379         .sortby("backup-type", false)
 380         .sortby("backup-id", false)
 381         .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
 382         .column(
 383             ColumnConfig::new("last-backup")
 384                 .renderer(render_last_backup)
 385                 .header("last snapshot")
 386                 .right_align(false)
 387         )
 388         .column(ColumnConfig::new("backup-count"))
 389         .column(ColumnConfig::new("files").renderer(render_files));
 390
 391     let mut data: Value = result["data"].take();
 392
 393     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_LIST_GROUPS;
 394
 395     format_and_print_result_full(&mut data, info, &output_format, &options);
 396
 397     Ok(Value::Null)
 398 }
 399
 400 #[api(
 401    input: {
 402         properties: {
 403             repository: {
 404                 schema: REPO_URL_SCHEMA,
 405                 optional: true,
 406             },
 407             group: {
 408                 type: String,
 409                 description: "Backup group.",
 410                 optional: true,
 411             },
 412             "output-format": {
 413                 schema: OUTPUT_FORMAT,
 414                 optional: true,
 415             },
 416         }
 417    }
 418 )]
 419 /// List backup snapshots.
 420 async fn list_snapshots(param: Value) -> Result<Value, Error> {
 421
 422     let repo = extract_repository_from_value(&param)?;
 423
 424     let output_format = get_output_format(&param);
 425
 426     let client = connect(repo.host(), repo.user())?;
 427
 428     let group: Option<BackupGroup> = if let Some(path) = param["group"].as_str() {
 429         Some(path.parse()?)
 430     } else {
 431         None
 432     };
 433
 434     let mut data = api_datastore_list_snapshots(&client, repo.store(), group).await?;
 435
 436     record_repository(&repo);
 437
 438     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
 439         let item: SnapshotListItem = serde_json::from_value(record.to_owned())?;
 440         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
 441         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
 442     };
 443
 444     let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
 445         let item: SnapshotListItem = serde_json::from_value(record.to_owned())?;
 446         let mut filenames = Vec::new();
 447         for file in &item.files {
 448             filenames.push(file.filename.to_string());
 449         }
 450         Ok(tools::format::render_backup_file_list(&filenames[..]))
 451     };
 452
 453     let options = default_table_format_options()
 454         .sortby("backup-type", false)
 455         .sortby("backup-id", false)
 456         .sortby("backup-time", false)
 457         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
 458         .column(ColumnConfig::new("size"))
 459         .column(ColumnConfig::new("files").renderer(render_files))
 460         ;
 461
 462     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_LIST_SNAPSHOTS;
 463
 464     format_and_print_result_full(&mut data, info, &output_format, &options);
 465
 466     Ok(Value::Null)
 467 }
 468
 469 #[api(
 470    input: {
 471         properties: {
 472             repository: {
 473                 schema: REPO_URL_SCHEMA,
 474                 optional: true,
 475             },
 476             snapshot: {
 477                 type: String,
 478                 description: "Snapshot path.",
 479              },
 480         }
 481    }
 482 )]
 483 /// Forget (remove) backup snapshots.
 484 async fn forget_snapshots(param: Value) -> Result<Value, Error> {
 485
 486     let repo = extract_repository_from_value(&param)?;
 487
 488     let path = tools::required_string_param(&param, "snapshot")?;
 489     let snapshot: BackupDir = path.parse()?;
 490
 491     let mut client = connect(repo.host(), repo.user())?;
 492
 493     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 494
 495     let result = client.delete(&path, Some(json!({
 496         "backup-type": snapshot.group().backup_type(),
 497         "backup-id": snapshot.group().backup_id(),
 498         "backup-time": snapshot.backup_time().timestamp(),
 499     }))).await?;
 500
 501     record_repository(&repo);
 502
 503     Ok(result)
 504 }
 505
 506 #[api(
 507    input: {
 508         properties: {
 509             repository: {
 510                 schema: REPO_URL_SCHEMA,
 511                 optional: true,
 512             },
 513         }
 514    }
 515 )]
 516 /// Try to login. If successful, store ticket.
 517 async fn api_login(param: Value) -> Result<Value, Error> {
 518
 519     let repo = extract_repository_from_value(&param)?;
 520
 521     let client = connect(repo.host(), repo.user())?;
 522     client.login().await?;
 523
 524     record_repository(&repo);
 525
 526     Ok(Value::Null)
 527 }
 528
 529 #[api(
 530    input: {
 531         properties: {
 532             repository: {
 533                 schema: REPO_URL_SCHEMA,
 534                 optional: true,
 535             },
 536         }
 537    }
 538 )]
 539 /// Logout (delete stored ticket).
 540 fn api_logout(param: Value) -> Result<Value, Error> {
 541
 542     let repo = extract_repository_from_value(&param)?;
 543
 544     delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
 545
 546     Ok(Value::Null)
 547 }
 548
 549 #[api(
 550    input: {
 551         properties: {
 552             repository: {
 553                 schema: REPO_URL_SCHEMA,
 554                 optional: true,
 555             },
 556             snapshot: {
 557                 type: String,
 558                 description: "Snapshot path.",
 559              },
 560         }
 561    }
 562 )]
 563 /// Dump catalog.
 564 async fn dump_catalog(param: Value) -> Result<Value, Error> {
 565
 566     let repo = extract_repository_from_value(&param)?;
 567
 568     let path = tools::required_string_param(&param, "snapshot")?;
 569     let snapshot: BackupDir = path.parse()?;
 570
 571     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 572
 573     let crypt_config = match keyfile {
 574         None => None,
 575         Some(path) => {
 576             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
 577             Some(Arc::new(CryptConfig::new(key)?))
 578         }
 579     };
 580
 581     let client = connect(repo.host(), repo.user())?;
 582
 583     let client = BackupReader::start(
 584         client,
 585         crypt_config.clone(),
 586         repo.store(),
 587         &snapshot.group().backup_type(),
 588         &snapshot.group().backup_id(),
 589         snapshot.backup_time(),
 590         true,
 591     ).await?;
 592
 593     let manifest = client.download_manifest().await?;
 594
 595     let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
 596
 597     let most_used = index.find_most_used_chunks(8);
 598
 599     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 600
 601     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 602
 603     let mut catalogfile = std::fs::OpenOptions::new()
 604         .write(true)
 605         .read(true)
 606         .custom_flags(libc::O_TMPFILE)
 607         .open("/tmp")?;
 608
 609     std::io::copy(&mut reader, &mut catalogfile)
 610         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
 611
 612     catalogfile.seek(SeekFrom::Start(0))?;
 613
 614     let mut catalog_reader = CatalogReader::new(catalogfile);
 615
 616     catalog_reader.dump()?;
 617
 618     record_repository(&repo);
 619
 620     Ok(Value::Null)
 621 }
 622
 623 #[api(
 624    input: {
 625         properties: {
 626             repository: {
 627                 schema: REPO_URL_SCHEMA,
 628                 optional: true,
 629             },
 630             snapshot: {
 631                 type: String,
 632                 description: "Snapshot path.",
 633              },
 634             "output-format": {
 635                 schema: OUTPUT_FORMAT,
 636                 optional: true,
 637             },
 638         }
 639    }
 640 )]
 641 /// List snapshot files.
 642 async fn list_snapshot_files(param: Value) -> Result<Value, Error> {
 643
 644     let repo = extract_repository_from_value(&param)?;
 645
 646     let path = tools::required_string_param(&param, "snapshot")?;
 647     let snapshot: BackupDir = path.parse()?;
 648
 649     let output_format = get_output_format(&param);
 650
 651     let client = connect(repo.host(), repo.user())?;
 652
 653     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 654
 655     let mut result = client.get(&path, Some(json!({
 656         "backup-type": snapshot.group().backup_type(),
 657         "backup-id": snapshot.group().backup_id(),
 658         "backup-time": snapshot.backup_time().timestamp(),
 659     }))).await?;
 660
 661     record_repository(&repo);
 662
 663     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_LIST_SNAPSHOT_FILES;
 664
 665     let mut data: Value = result["data"].take();
 666
 667     let options = default_table_format_options();
 668
 669     format_and_print_result_full(&mut data, info, &output_format, &options);
 670
 671     Ok(Value::Null)
 672 }
 673
 674 #[api(
 675     input: {
 676         properties: {
 677             repository: {
 678                 schema: REPO_URL_SCHEMA,
 679                 optional: true,
 680             },
 681             "output-format": {
 682                 schema: OUTPUT_FORMAT,
 683                 optional: true,
 684             },
 685         },
 686     },
 687 )]
 688 /// Start garbage collection for a specific repository.
 689 async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 690
 691     let repo = extract_repository_from_value(&param)?;
 692
 693     let output_format = get_output_format(&param);
 694
 695     let mut client = connect(repo.host(), repo.user())?;
 696
 697     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 698
 699     let result = client.post(&path, None).await?;
 700
 701     record_repository(&repo);
 702
 703     view_task_result(client, result, &output_format).await?;
 704
 705     Ok(Value::Null)
 706 }
 707
 708 fn spawn_catalog_upload(
 709     client: Arc<BackupWriter>
 710 ) -> Result<
 711         (
 712             Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
 713             tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>
 714         ), Error>
 715 {
 716     let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
 717     let catalog_stream = crate::tools::StdChannelStream(catalog_rx);
 718     let catalog_chunk_size = 512*1024;
 719     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 720
 721     let catalog = Arc::new(Mutex::new(CatalogWriter::new(crate::tools::StdChannelWriter::new(catalog_tx))?));
 722
 723     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 724
 725     tokio::spawn(async move {
 726         let catalog_upload_result = client
 727             .upload_stream(None, CATALOG_NAME, catalog_chunk_stream, "dynamic", None)
 728             .await;
 729
 730         if let Err(ref err) = catalog_upload_result {
 731             eprintln!("catalog upload error - {}", err);
 732             client.cancel();
 733         }
 734
 735         let _ = catalog_result_tx.send(catalog_upload_result);
 736     });
 737
 738     Ok((catalog, catalog_result_rx))
 739 }
 740
 741 #[api(
 742    input: {
 743        properties: {
 744            backupspec: {
 745                type: Array,
 746                description: "List of backup source specifications ([<label.ext>:<path>] ...)",
 747                items: {
 748                    schema: BACKUP_SOURCE_SCHEMA,
 749                }
 750            },
 751            repository: {
 752                schema: REPO_URL_SCHEMA,
 753                optional: true,
 754            },
 755            "include-dev": {
 756                description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
 757                optional: true,
 758                items: {
 759                    type: String,
 760                    description: "Path to file.",
 761                }
 762            },
 763            keyfile: {
 764                schema: KEYFILE_SCHEMA,
 765                optional: true,
 766            },
 767            "skip-lost-and-found": {
 768                type: Boolean,
 769                description: "Skip lost+found directory.",
 770                optional: true,
 771            },
 772            "backup-type": {
 773                schema: BACKUP_TYPE_SCHEMA,
 774                optional: true,
 775            },
 776            "backup-id": {
 777                schema: BACKUP_ID_SCHEMA,
 778                optional: true,
 779            },
 780            "backup-time": {
 781                schema: BACKUP_TIME_SCHEMA,
 782                optional: true,
 783            },
 784            "chunk-size": {
 785                schema: CHUNK_SIZE_SCHEMA,
 786                optional: true,
 787            },
 788            "exclude": {
 789                type: Array,
 790                description: "List of paths or patterns for matching files to exclude.",
 791                optional: true,
 792                items: {
 793                    type: String,
 794                    description: "Path or match pattern.",
 795                 }
 796            },
 797            "entries-max": {
 798                type: Integer,
 799                description: "Max number of entries to hold in memory.",
 800                optional: true,
 801                default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
 802            },
 803            "verbose": {
 804                type: Boolean,
 805                description: "Verbose output.",
 806                optional: true,
 807            },
 808        }
 809    }
 810 )]
 811 /// Create (host) backup.
 812 async fn create_backup(
 813     param: Value,
 814     _info: &ApiMethod,
 815     _rpcenv: &mut dyn RpcEnvironment,
 816 ) -> Result<Value, Error> {
 817
 818     let repo = extract_repository_from_value(&param)?;
 819
 820     let backupspec_list = tools::required_array_param(&param, "backupspec")?;
 821
 822     let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
 823
 824     let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
 825
 826     let verbose = param["verbose"].as_bool().unwrap_or(false);
 827
 828     let backup_time_opt = param["backup-time"].as_i64();
 829
 830     let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
 831
 832     if let Some(size) = chunk_size_opt {
 833         verify_chunk_size(size)?;
 834     }
 835
 836     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 837
 838     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 839
 840     let backup_type = param["backup-type"].as_str().unwrap_or("host");
 841
 842     let include_dev = param["include-dev"].as_array();
 843
 844     let entries_max = param["entries-max"].as_u64()
 845         .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
 846
 847     let empty = Vec::new();
 848     let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
 849
 850     let mut pattern_list = Vec::with_capacity(exclude_args.len());
 851     for entry in exclude_args {
 852         let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
 853         pattern_list.push(
 854             MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
 855                 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
 856         );
 857     }
 858
 859     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
 860
 861     if let Some(include_dev) = include_dev {
 862         if all_file_systems {
 863             bail!("option 'all-file-systems' conflicts with option 'include-dev'");
 864         }
 865
 866         let mut set = HashSet::new();
 867         for path in include_dev {
 868             let path = path.as_str().unwrap();
 869             let stat = nix::sys::stat::stat(path)
 870                 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
 871             set.insert(stat.st_dev);
 872         }
 873         devices = Some(set);
 874     }
 875
 876     let mut upload_list = vec![];
 877
 878     for backupspec in backupspec_list {
 879         let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
 880         let filename = &spec.config_string;
 881         let target = &spec.archive_name;
 882
 883         use std::os::unix::fs::FileTypeExt;
 884
 885         let metadata = std::fs::metadata(filename)
 886             .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
 887         let file_type = metadata.file_type();
 888
 889         match spec.spec_type {
 890             BackupSpecificationType::PXAR => {
 891                 if !file_type.is_dir() {
 892                     bail!("got unexpected file type (expected directory)");
 893                 }
 894                 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
 895             }
 896             BackupSpecificationType::IMAGE => {
 897                 if !(file_type.is_file() || file_type.is_block_device()) {
 898                     bail!("got unexpected file type (expected file or block device)");
 899                 }
 900
 901                 let size = image_size(&PathBuf::from(filename))?;
 902
 903                 if size == 0 { bail!("got zero-sized file '{}'", filename); }
 904
 905                 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
 906             }
 907             BackupSpecificationType::CONFIG => {
 908                 if !file_type.is_file() {
 909                     bail!("got unexpected file type (expected regular file)");
 910                 }
 911                 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 912             }
 913             BackupSpecificationType::LOGFILE => {
 914                 if !file_type.is_file() {
 915                     bail!("got unexpected file type (expected regular file)");
 916                 }
 917                 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 918             }
 919         }
 920     }
 921
 922     let backup_time = Utc.timestamp(backup_time_opt.unwrap_or_else(|| Utc::now().timestamp()), 0);
 923
 924     let client = connect(repo.host(), repo.user())?;
 925     record_repository(&repo);
 926
 927     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time));
 928
 929     println!("Client name: {}", proxmox::tools::nodename());
 930
 931     let start_time = Local::now();
 932
 933     println!("Starting protocol: {}", start_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 934
 935     let (crypt_config, rsa_encrypted_key) = match keyfile {
 936         None => (None, None),
 937         Some(path) => {
 938             let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
 939
 940             let crypt_config = CryptConfig::new(key)?;
 941
 942             let path = master_pubkey_path()?;
 943             if path.exists() {
 944                 let pem_data = file_get_contents(&path)?;
 945                 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
 946                 let enc_key = crypt_config.generate_rsa_encoded_key(rsa, created)?;
 947                 (Some(Arc::new(crypt_config)), Some(enc_key))
 948             } else {
 949                 (Some(Arc::new(crypt_config)), None)
 950             }
 951         }
 952     };
 953
 954     let is_encrypted = Some(crypt_config.is_some());
 955
 956     let client = BackupWriter::start(
 957         client,
 958         crypt_config.clone(),
 959         repo.store(),
 960         backup_type,
 961         &backup_id,
 962         backup_time,
 963         verbose,
 964     ).await?;
 965
 966     let previous_manifest = if let Ok(previous_manifest) = client.download_previous_manifest().await {
 967         Some(Arc::new(previous_manifest))
 968     } else {
 969         None
 970     };
 971
 972     let snapshot = BackupDir::new(backup_type, backup_id, backup_time.timestamp());
 973     let mut manifest = BackupManifest::new(snapshot);
 974
 975     let mut catalog = None;
 976     let mut catalog_result_tx = None;
 977
 978     for (backup_type, filename, target, size) in upload_list {
 979         match backup_type {
 980             BackupSpecificationType::CONFIG => {
 981                 println!("Upload config file '{}' to '{:?}' as {}", filename, repo, target);
 982                 let stats = client
 983                     .upload_blob_from_file(&filename, &target, true, Some(true))
 984                     .await?;
 985                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
 986             }
 987             BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
 988                 println!("Upload log file '{}' to '{:?}' as {}", filename, repo, target);
 989                 let stats = client
 990                     .upload_blob_from_file(&filename, &target, true, Some(true))
 991                     .await?;
 992                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
 993             }
 994             BackupSpecificationType::PXAR => {
 995                 // start catalog upload on first use
 996                 if catalog.is_none() {
 997                     let (cat, res) = spawn_catalog_upload(client.clone())?;
 998                     catalog = Some(cat);
 999                     catalog_result_tx = Some(res);
1000                 }
1001                 let catalog = catalog.as_ref().unwrap();
1002
1003                 println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
1004                 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
1005                 let stats = backup_directory(
1006                     &client,
1007                     previous_manifest.clone(),
1008                     &filename,
1009                     &target,
1010                     chunk_size_opt,
1011                     devices.clone(),
1012                     verbose,
1013                     skip_lost_and_found,
1014                     catalog.clone(),
1015                     pattern_list.clone(),
1016                     entries_max as usize,
1017                 ).await?;
1018                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
1019                 catalog.lock().unwrap().end_directory()?;
1020             }
1021             BackupSpecificationType::IMAGE => {
1022                 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
1023                 let stats = backup_image(
1024                     &client,
1025                     previous_manifest.clone(),
1026                      &filename,
1027                     &target,
1028                     size,
1029                     chunk_size_opt,
1030                     verbose,
1031                 ).await?;
1032                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
1033             }
1034         }
1035     }
1036
1037     // finalize and upload catalog
1038     if let Some(catalog) = catalog {
1039         let mutex = Arc::try_unwrap(catalog)
1040             .map_err(|_| format_err!("unable to get catalog (still used)"))?;
1041         let mut catalog = mutex.into_inner().unwrap();
1042
1043         catalog.finish()?;
1044
1045         drop(catalog); // close upload stream
1046
1047         if let Some(catalog_result_rx) = catalog_result_tx {
1048             let stats = catalog_result_rx.await??;
1049             manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, is_encrypted)?;
1050         }
1051     }
1052
1053     if let Some(rsa_encrypted_key) = rsa_encrypted_key {
1054         let target = "rsa-encrypted.key";
1055         println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
1056         let stats = client
1057             .upload_blob_from_data(rsa_encrypted_key, target, false, None)
1058             .await?;
1059         manifest.add_file(format!("{}.blob", target), stats.size, stats.csum, is_encrypted)?;
1060
1061         // openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out t
1062         /*
1063         let mut buffer2 = vec![0u8; rsa.size() as usize];
1064         let pem_data = file_get_contents("master-private.pem")?;
1065         let rsa = openssl::rsa::Rsa::private_key_from_pem(&pem_data)?;
1066         let len = rsa.private_decrypt(&buffer, &mut buffer2, openssl::rsa::Padding::PKCS1)?;
1067         println!("TEST {} {:?}", len, buffer2);
1068          */
1069     }
1070
1071     // create manifest (index.json)
1072     let manifest = manifest.into_json();
1073
1074     println!("Upload index.json to '{:?}'", repo);
1075     let manifest = serde_json::to_string_pretty(&manifest)?.into();
1076     client
1077         .upload_blob_from_data(manifest, MANIFEST_BLOB_NAME, true, Some(true))
1078         .await?;
1079
1080     client.finish().await?;
1081
1082     let end_time = Local::now();
1083     let elapsed = end_time.signed_duration_since(start_time);
1084     println!("Duration: {}", elapsed);
1085
1086     println!("End Time: {}", end_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
1087
1088     Ok(Value::Null)
1089 }
1090
1091 fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1092
1093     let mut result = vec![];
1094
1095     let data: Vec<&str> = arg.splitn(2, ':').collect();
1096
1097     if data.len() != 2 {
1098         result.push(String::from("root.pxar:/"));
1099         result.push(String::from("etc.pxar:/etc"));
1100         return result;
1101     }
1102
1103     let files = tools::complete_file_name(data[1], param);
1104
1105     for file in files {
1106         result.push(format!("{}:{}", data[0], file));
1107     }
1108
1109     result
1110 }
1111
1112 async fn dump_image<W: Write>(
1113     client: Arc<BackupReader>,
1114     crypt_config: Option<Arc<CryptConfig>>,
1115     index: FixedIndexReader,
1116     mut writer: W,
1117     verbose: bool,
1118 ) -> Result<(), Error> {
1119
1120     let most_used = index.find_most_used_chunks(8);
1121
1122     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1123
1124     // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
1125     // and thus slows down reading. Instead, directly use RemoteChunkReader
1126     let mut per = 0;
1127     let mut bytes = 0;
1128     let start_time = std::time::Instant::now();
1129
1130     for pos in 0..index.index_count() {
1131         let digest = index.index_digest(pos).unwrap();
1132         let raw_data = chunk_reader.read_chunk(&digest).await?;
1133         writer.write_all(&raw_data)?;
1134         bytes += raw_data.len();
1135         if verbose {
1136             let next_per = ((pos+1)*100)/index.index_count();
1137             if per != next_per {
1138                 eprintln!("progress {}% (read {} bytes, duration {} sec)",
1139                           next_per, bytes, start_time.elapsed().as_secs());
1140                 per = next_per;
1141             }
1142         }
1143     }
1144
1145     let end_time = std::time::Instant::now();
1146     let elapsed = end_time.duration_since(start_time);
1147     eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1148               bytes,
1149               elapsed.as_secs_f64(),
1150               bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1151     );
1152
1153
1154     Ok(())
1155 }
1156
1157 fn parse_archive_type(name: &str) -> (String, ArchiveType) {
1158     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1159         (name.into(), archive_type(name).unwrap())
1160     } else if name.ends_with(".pxar") {
1161         (format!("{}.didx", name), ArchiveType::DynamicIndex)
1162     } else if name.ends_with(".img") {
1163         (format!("{}.fidx", name), ArchiveType::FixedIndex)
1164     } else {
1165         (format!("{}.blob", name), ArchiveType::Blob)
1166     }
1167 }
1168
1169 #[api(
1170    input: {
1171        properties: {
1172            repository: {
1173                schema: REPO_URL_SCHEMA,
1174                optional: true,
1175            },
1176            snapshot: {
1177                type: String,
1178                description: "Group/Snapshot path.",
1179            },
1180            "archive-name": {
1181                description: "Backup archive name.",
1182                type: String,
1183            },
1184            target: {
1185                type: String,
1186                description: r###"Target directory path. Use '-' to write to standard output.
1187
1188 We do not extraxt '.pxar' archives when writing to standard output.
1189
1190 "###
1191            },
1192            "allow-existing-dirs": {
1193                type: Boolean,
1194                description: "Do not fail if directories already exists.",
1195                optional: true,
1196            },
1197            keyfile: {
1198                schema: KEYFILE_SCHEMA,
1199                optional: true,
1200            },
1201        }
1202    }
1203 )]
1204 /// Restore backup repository.
1205 async fn restore(param: Value) -> Result<Value, Error> {
1206     let repo = extract_repository_from_value(&param)?;
1207
1208     let verbose = param["verbose"].as_bool().unwrap_or(false);
1209
1210     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1211
1212     let archive_name = tools::required_string_param(&param, "archive-name")?;
1213
1214     let client = connect(repo.host(), repo.user())?;
1215
1216     record_repository(&repo);
1217
1218     let path = tools::required_string_param(&param, "snapshot")?;
1219
1220     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1221         let group: BackupGroup = path.parse()?;
1222         api_datastore_latest_snapshot(&client, repo.store(), group).await?
1223     } else {
1224         let snapshot: BackupDir = path.parse()?;
1225         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1226     };
1227
1228     let target = tools::required_string_param(&param, "target")?;
1229     let target = if target == "-" { None } else { Some(target) };
1230
1231     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1232
1233     let crypt_config = match keyfile {
1234         None => None,
1235         Some(path) => {
1236             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
1237             Some(Arc::new(CryptConfig::new(key)?))
1238         }
1239     };
1240
1241     let client = BackupReader::start(
1242         client,
1243         crypt_config.clone(),
1244         repo.store(),
1245         &backup_type,
1246         &backup_id,
1247         backup_time,
1248         true,
1249     ).await?;
1250
1251     let manifest = client.download_manifest().await?;
1252
1253     let (archive_name, archive_type) = parse_archive_type(archive_name);
1254
1255     if archive_name == MANIFEST_BLOB_NAME {
1256         let backup_index_data = manifest.into_json().to_string();
1257         if let Some(target) = target {
1258             replace_file(target, backup_index_data.as_bytes(), CreateOptions::new())?;
1259         } else {
1260             let stdout = std::io::stdout();
1261             let mut writer = stdout.lock();
1262             writer.write_all(backup_index_data.as_bytes())
1263                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1264         }
1265
1266     } else if archive_type == ArchiveType::Blob {
1267
1268         let mut reader = client.download_blob(&manifest, &archive_name).await?;
1269
1270         if let Some(target) = target {
1271            let mut writer = std::fs::OpenOptions::new()
1272                 .write(true)
1273                 .create(true)
1274                 .create_new(true)
1275                 .open(target)
1276                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1277             std::io::copy(&mut reader, &mut writer)?;
1278         } else {
1279             let stdout = std::io::stdout();
1280             let mut writer = stdout.lock();
1281             std::io::copy(&mut reader, &mut writer)
1282                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1283         }
1284
1285     } else if archive_type == ArchiveType::DynamicIndex {
1286
1287         let index = client.download_dynamic_index(&manifest, &archive_name).await?;
1288
1289         let most_used = index.find_most_used_chunks(8);
1290
1291         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1292
1293         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1294
1295         if let Some(target) = target {
1296             proxmox_backup::pxar::extract_archive(
1297                 pxar::decoder::Decoder::from_std(reader)?,
1298                 Path::new(target),
1299                 &[],
1300                 proxmox_backup::pxar::Flags::DEFAULT,
1301                 allow_existing_dirs,
1302                 |path| {
1303                     if verbose {
1304                         println!("{:?}", path);
1305                     }
1306                 },
1307             )
1308             .map_err(|err| format_err!("error extracting archive - {}", err))?;
1309         } else {
1310             let mut writer = std::fs::OpenOptions::new()
1311                 .write(true)
1312                 .open("/dev/stdout")
1313                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
1314
1315             std::io::copy(&mut reader, &mut writer)
1316                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1317         }
1318     } else if archive_type == ArchiveType::FixedIndex {
1319
1320         let index = client.download_fixed_index(&manifest, &archive_name).await?;
1321
1322         let mut writer = if let Some(target) = target {
1323             std::fs::OpenOptions::new()
1324                 .write(true)
1325                 .create(true)
1326                 .create_new(true)
1327                 .open(target)
1328                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
1329         } else {
1330             std::fs::OpenOptions::new()
1331                 .write(true)
1332                 .open("/dev/stdout")
1333                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1334         };
1335
1336         dump_image(client.clone(), crypt_config.clone(), index, &mut writer, verbose).await?;
1337     }
1338
1339     Ok(Value::Null)
1340 }
1341
1342 #[api(
1343    input: {
1344        properties: {
1345            repository: {
1346                schema: REPO_URL_SCHEMA,
1347                optional: true,
1348            },
1349            snapshot: {
1350                type: String,
1351                description: "Group/Snapshot path.",
1352            },
1353            logfile: {
1354                type: String,
1355                description: "The path to the log file you want to upload.",
1356            },
1357            keyfile: {
1358                schema: KEYFILE_SCHEMA,
1359                optional: true,
1360            },
1361        }
1362    }
1363 )]
1364 /// Upload backup log file.
1365 async fn upload_log(param: Value) -> Result<Value, Error> {
1366
1367     let logfile = tools::required_string_param(&param, "logfile")?;
1368     let repo = extract_repository_from_value(&param)?;
1369
1370     let snapshot = tools::required_string_param(&param, "snapshot")?;
1371     let snapshot: BackupDir = snapshot.parse()?;
1372
1373     let mut client = connect(repo.host(), repo.user())?;
1374
1375     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1376
1377     let crypt_config = match keyfile {
1378         None => None,
1379         Some(path) => {
1380             let (key, _created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
1381             let crypt_config = CryptConfig::new(key)?;
1382             Some(Arc::new(crypt_config))
1383         }
1384     };
1385
1386     let data = file_get_contents(logfile)?;
1387
1388     let blob = DataBlob::encode(&data, crypt_config.as_ref().map(Arc::as_ref), true)?;
1389
1390     let raw_data = blob.into_inner();
1391
1392     let path = format!("api2/json/admin/datastore/{}/upload-backup-log", repo.store());
1393
1394     let args = json!({
1395         "backup-type": snapshot.group().backup_type(),
1396         "backup-id":  snapshot.group().backup_id(),
1397         "backup-time": snapshot.backup_time().timestamp(),
1398     });
1399
1400     let body = hyper::Body::from(raw_data);
1401
1402     client.upload("application/octet-stream", body, &path, Some(args)).await
1403 }
1404
1405 const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
1406     &ApiHandler::Async(&prune),
1407     &ObjectSchema::new(
1408         "Prune a backup repository.",
1409         &proxmox_backup::add_common_prune_prameters!([
1410             ("dry-run", true, &BooleanSchema::new(
1411                 "Just show what prune would do, but do not delete anything.")
1412              .schema()),
1413             ("group", false, &StringSchema::new("Backup group.").schema()),
1414         ], [
1415             ("output-format", true, &OUTPUT_FORMAT),
1416             (
1417                 "quiet",
1418                 true,
1419                 &BooleanSchema::new("Minimal output - only show removals.")
1420                     .schema()
1421             ),
1422             ("repository", true, &REPO_URL_SCHEMA),
1423         ])
1424     )
1425 );
1426
1427 fn prune<'a>(
1428     param: Value,
1429     _info: &ApiMethod,
1430     _rpcenv: &'a mut dyn RpcEnvironment,
1431 ) -> proxmox::api::ApiFuture<'a> {
1432     async move {
1433         prune_async(param).await
1434     }.boxed()
1435 }
1436
1437 async fn prune_async(mut param: Value) -> Result<Value, Error> {
1438     let repo = extract_repository_from_value(&param)?;
1439
1440     let mut client = connect(repo.host(), repo.user())?;
1441
1442     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
1443
1444     let group = tools::required_string_param(&param, "group")?;
1445     let group: BackupGroup = group.parse()?;
1446
1447     let output_format = get_output_format(&param);
1448
1449     let quiet = param["quiet"].as_bool().unwrap_or(false);
1450
1451     param.as_object_mut().unwrap().remove("repository");
1452     param.as_object_mut().unwrap().remove("group");
1453     param.as_object_mut().unwrap().remove("output-format");
1454     param.as_object_mut().unwrap().remove("quiet");
1455
1456     param["backup-type"] = group.backup_type().into();
1457     param["backup-id"] = group.backup_id().into();
1458
1459     let mut result = client.post(&path, Some(param)).await?;
1460
1461     record_repository(&repo);
1462
1463     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1464         let item: PruneListItem = serde_json::from_value(record.to_owned())?;
1465         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
1466         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1467     };
1468
1469     let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1470         Ok(match v.as_bool() {
1471             Some(true) => "keep",
1472             Some(false) => "remove",
1473             None => "unknown",
1474         }.to_string())
1475     };
1476
1477     let options = default_table_format_options()
1478         .sortby("backup-type", false)
1479         .sortby("backup-id", false)
1480         .sortby("backup-time", false)
1481         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
1482         .column(ColumnConfig::new("backup-time").renderer(tools::format::render_epoch).header("date"))
1483         .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
1484         ;
1485
1486     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_PRUNE;
1487
1488     let mut data = result["data"].take();
1489
1490     if quiet {
1491         let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1492             item["keep"].as_bool() == Some(false)
1493         }).map(|v| v.clone()).collect();
1494         data = list.into();
1495     }
1496
1497     format_and_print_result_full(&mut data, info, &output_format, &options);
1498
1499     Ok(Value::Null)
1500 }
1501
1502 #[api(
1503    input: {
1504        properties: {
1505            repository: {
1506                schema: REPO_URL_SCHEMA,
1507                optional: true,
1508            },
1509            "output-format": {
1510                schema: OUTPUT_FORMAT,
1511                optional: true,
1512            },
1513        }
1514    }
1515 )]
1516 /// Get repository status.
1517 async fn status(param: Value) -> Result<Value, Error> {
1518
1519     let repo = extract_repository_from_value(&param)?;
1520
1521     let output_format = get_output_format(&param);
1522
1523     let client = connect(repo.host(), repo.user())?;
1524
1525     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1526
1527     let mut result = client.get(&path, None).await?;
1528     let mut data = result["data"].take();
1529
1530     record_repository(&repo);
1531
1532     let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1533         let v = v.as_u64().unwrap();
1534         let total = record["total"].as_u64().unwrap();
1535         let roundup = total/200;
1536         let per = ((v+roundup)*100)/total;
1537         let info = format!(" ({} %)", per);
1538         Ok(format!("{} {:>8}", v, info))
1539     };
1540
1541     let options = default_table_format_options()
1542         .noheader(true)
1543         .column(ColumnConfig::new("total").renderer(render_total_percentage))
1544         .column(ColumnConfig::new("used").renderer(render_total_percentage))
1545         .column(ColumnConfig::new("avail").renderer(render_total_percentage));
1546
1547     let schema = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_STATUS;
1548
1549     format_and_print_result_full(&mut data, schema, &output_format, &options);
1550
1551     Ok(Value::Null)
1552 }
1553
1554 // like get, but simply ignore errors and return Null instead
1555 async fn try_get(repo: &BackupRepository, url: &str) -> Value {
1556
1557     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
1558     let password = std::env::var(ENV_VAR_PBS_PASSWORD).ok();
1559
1560     let options = HttpClientOptions::new()
1561         .prefix(Some("proxmox-backup".to_string()))
1562         .password(password)
1563         .interactive(false)
1564         .fingerprint(fingerprint)
1565         .fingerprint_cache(true)
1566         .ticket_cache(true);
1567
1568     let client = match HttpClient::new(repo.host(), repo.user(), options) {
1569         Ok(v) => v,
1570         _ => return Value::Null,
1571     };
1572
1573     let mut resp = match client.get(url, None).await {
1574         Ok(v) => v,
1575         _ => return Value::Null,
1576     };
1577
1578     if let Some(map) = resp.as_object_mut() {
1579         if let Some(data) = map.remove("data") {
1580             return data;
1581         }
1582     }
1583     Value::Null
1584 }
1585
1586 fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1587     proxmox_backup::tools::runtime::main(async { complete_backup_group_do(param).await })
1588 }
1589
1590 async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
1591
1592     let mut result = vec![];
1593
1594     let repo = match extract_repository_from_map(param) {
1595         Some(v) => v,
1596         _ => return result,
1597     };
1598
1599     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
1600
1601     let data = try_get(&repo, &path).await;
1602
1603     if let Some(list) = data.as_array() {
1604         for item in list {
1605             if let (Some(backup_id), Some(backup_type)) =
1606                 (item["backup-id"].as_str(), item["backup-type"].as_str())
1607             {
1608                 result.push(format!("{}/{}", backup_type, backup_id));
1609             }
1610         }
1611     }
1612
1613     result
1614 }
1615
1616 pub fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1617     proxmox_backup::tools::runtime::main(async { complete_group_or_snapshot_do(arg, param).await })
1618 }
1619
1620 async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1621
1622     if arg.matches('/').count() < 2 {
1623         let groups = complete_backup_group_do(param).await;
1624         let mut result = vec![];
1625         for group in groups {
1626             result.push(group.to_string());
1627             result.push(format!("{}/", group));
1628         }
1629         return result;
1630     }
1631
1632     complete_backup_snapshot_do(param).await
1633 }
1634
1635 fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1636     proxmox_backup::tools::runtime::main(async { complete_backup_snapshot_do(param).await })
1637 }
1638
1639 async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
1640
1641     let mut result = vec![];
1642
1643     let repo = match extract_repository_from_map(param) {
1644         Some(v) => v,
1645         _ => return result,
1646     };
1647
1648     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1649
1650     let data = try_get(&repo, &path).await;
1651
1652     if let Some(list) = data.as_array() {
1653         for item in list {
1654             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
1655                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
1656             {
1657                 let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
1658                 result.push(snapshot.relative_path().to_str().unwrap().to_owned());
1659             }
1660         }
1661     }
1662
1663     result
1664 }
1665
1666 fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1667     proxmox_backup::tools::runtime::main(async { complete_server_file_name_do(param).await })
1668 }
1669
1670 async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
1671
1672     let mut result = vec![];
1673
1674     let repo = match extract_repository_from_map(param) {
1675         Some(v) => v,
1676         _ => return result,
1677     };
1678
1679     let snapshot: BackupDir = match param.get("snapshot") {
1680         Some(path) => {
1681             match path.parse() {
1682                 Ok(v) => v,
1683                 _ => return result,
1684             }
1685         }
1686         _ => return result,
1687     };
1688
1689     let query = tools::json_object_to_query(json!({
1690         "backup-type": snapshot.group().backup_type(),
1691         "backup-id": snapshot.group().backup_id(),
1692         "backup-time": snapshot.backup_time().timestamp(),
1693     })).unwrap();
1694
1695     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
1696
1697     let data = try_get(&repo, &path).await;
1698
1699     if let Some(list) = data.as_array() {
1700         for item in list {
1701             if let Some(filename) = item["filename"].as_str() {
1702                 result.push(filename.to_owned());
1703             }
1704         }
1705     }
1706
1707     result
1708 }
1709
1710 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1711     complete_server_file_name(arg, param)
1712         .iter()
1713         .map(|v| tools::format::strip_server_file_expenstion(&v))
1714         .collect()
1715 }
1716
1717 pub fn complete_pxar_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1718     complete_server_file_name(arg, param)
1719         .iter()
1720         .filter_map(|v| {
1721             let name = tools::format::strip_server_file_expenstion(&v);
1722             if name.ends_with(".pxar") {
1723                 Some(name)
1724             } else {
1725                 None
1726             }
1727         })
1728         .collect()
1729 }
1730
1731 fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
1732
1733     let mut result = vec![];
1734
1735     let mut size = 64;
1736     loop {
1737         result.push(size.to_string());
1738         size *= 2;
1739         if size > 4096 { break; }
1740     }
1741
1742     result
1743 }
1744
1745 fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
1746
1747     // fixme: implement other input methods
1748
1749     use std::env::VarError::*;
1750     match std::env::var("PBS_ENCRYPTION_PASSWORD") {
1751         Ok(p) => return Ok(p.as_bytes().to_vec()),
1752         Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
1753         Err(NotPresent) => {
1754             // Try another method
1755         }
1756     }
1757
1758     // If we're on a TTY, query the user for a password
1759     if tty::stdin_isatty() {
1760         return Ok(tty::read_password("Encryption Key Password: ")?);
1761     }
1762
1763     bail!("no password input mechanism available");
1764 }
1765
1766 fn key_create(
1767     param: Value,
1768     _info: &ApiMethod,
1769     _rpcenv: &mut dyn RpcEnvironment,
1770 ) -> Result<Value, Error> {
1771
1772     let path = tools::required_string_param(&param, "path")?;
1773     let path = PathBuf::from(path);
1774
1775     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1776
1777     let key = proxmox::sys::linux::random_data(32)?;
1778
1779     if kdf == "scrypt" {
1780         // always read passphrase from tty
1781         if !tty::stdin_isatty() {
1782             bail!("unable to read passphrase - no tty");
1783         }
1784
1785         let password = tty::read_and_verify_password("Encryption Key Password: ")?;
1786
1787         let key_config = encrypt_key_with_passphrase(&key, &password)?;
1788
1789         store_key_config(&path, false, key_config)?;
1790
1791         Ok(Value::Null)
1792     } else if kdf == "none" {
1793         let created =  Local.timestamp(Local::now().timestamp(), 0);
1794
1795         store_key_config(&path, false, KeyConfig {
1796             kdf: None,
1797             created,
1798             modified: created,
1799             data: key,
1800         })?;
1801
1802         Ok(Value::Null)
1803     } else {
1804         unreachable!();
1805     }
1806 }
1807
1808 fn master_pubkey_path() -> Result<PathBuf, Error> {
1809     let base = BaseDirectories::with_prefix("proxmox-backup")?;
1810
1811     // usually $HOME/.config/proxmox-backup/master-public.pem
1812     let path = base.place_config_file("master-public.pem")?;
1813
1814     Ok(path)
1815 }
1816
1817 fn key_import_master_pubkey(
1818     param: Value,
1819     _info: &ApiMethod,
1820     _rpcenv: &mut dyn RpcEnvironment,
1821 ) -> Result<Value, Error> {
1822
1823     let path = tools::required_string_param(&param, "path")?;
1824     let path = PathBuf::from(path);
1825
1826     let pem_data = file_get_contents(&path)?;
1827
1828     if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
1829         bail!("Unable to decode PEM data - {}", err);
1830     }
1831
1832     let target_path = master_pubkey_path()?;
1833
1834     replace_file(&target_path, &pem_data, CreateOptions::new())?;
1835
1836     println!("Imported public master key to {:?}", target_path);
1837
1838     Ok(Value::Null)
1839 }
1840
1841 fn key_create_master_key(
1842     _param: Value,
1843     _info: &ApiMethod,
1844     _rpcenv: &mut dyn RpcEnvironment,
1845 ) -> Result<Value, Error> {
1846
1847     // we need a TTY to query the new password
1848     if !tty::stdin_isatty() {
1849         bail!("unable to create master key - no tty");
1850     }
1851
1852     let rsa = openssl::rsa::Rsa::generate(4096)?;
1853     let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
1854
1855
1856     let password = String::from_utf8(tty::read_and_verify_password("Master Key Password: ")?)?;
1857
1858     let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
1859     let filename_pub = "master-public.pem";
1860     println!("Writing public master key to {}", filename_pub);
1861     replace_file(filename_pub, pub_key.as_slice(), CreateOptions::new())?;
1862
1863     let cipher = openssl::symm::Cipher::aes_256_cbc();
1864     let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, password.as_bytes())?;
1865
1866     let filename_priv = "master-private.pem";
1867     println!("Writing private master key to {}", filename_priv);
1868     replace_file(filename_priv, priv_key.as_slice(), CreateOptions::new())?;
1869
1870     Ok(Value::Null)
1871 }
1872
1873 fn key_change_passphrase(
1874     param: Value,
1875     _info: &ApiMethod,
1876     _rpcenv: &mut dyn RpcEnvironment,
1877 ) -> Result<Value, Error> {
1878
1879     let path = tools::required_string_param(&param, "path")?;
1880     let path = PathBuf::from(path);
1881
1882     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1883
1884     // we need a TTY to query the new password
1885     if !tty::stdin_isatty() {
1886         bail!("unable to change passphrase - no tty");
1887     }
1888
1889     let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
1890
1891     if kdf == "scrypt" {
1892
1893         let password = tty::read_and_verify_password("New Password: ")?;
1894
1895         let mut new_key_config = encrypt_key_with_passphrase(&key, &password)?;
1896         new_key_config.created = created; // keep original value
1897
1898         store_key_config(&path, true, new_key_config)?;
1899
1900         Ok(Value::Null)
1901     } else if kdf == "none" {
1902         let modified =  Local.timestamp(Local::now().timestamp(), 0);
1903
1904         store_key_config(&path, true, KeyConfig {
1905             kdf: None,
1906             created, // keep original value
1907             modified,
1908             data: key.to_vec(),
1909         })?;
1910
1911         Ok(Value::Null)
1912     } else {
1913         unreachable!();
1914     }
1915 }
1916
1917 fn key_mgmt_cli() -> CliCommandMap {
1918
1919     const KDF_SCHEMA: Schema =
1920         StringSchema::new("Key derivation function. Choose 'none' to store the key unecrypted.")
1921         .format(&ApiStringFormat::Enum(&[
1922             EnumEntry::new("scrypt", "SCrypt"),
1923             EnumEntry::new("none", "Do not encrypt the key")]))
1924         .default("scrypt")
1925         .schema();
1926
1927     #[sortable]
1928     const API_METHOD_KEY_CREATE: ApiMethod = ApiMethod::new(
1929         &ApiHandler::Sync(&key_create),
1930         &ObjectSchema::new(
1931             "Create a new encryption key.",
1932             &sorted!([
1933                 ("path", false, &StringSchema::new("File system path.").schema()),
1934                 ("kdf", true, &KDF_SCHEMA),
1935             ]),
1936         )
1937     );
1938
1939     let key_create_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE)
1940         .arg_param(&["path"])
1941         .completion_cb("path", tools::complete_file_name);
1942
1943     #[sortable]
1944     const API_METHOD_KEY_CHANGE_PASSPHRASE: ApiMethod = ApiMethod::new(
1945         &ApiHandler::Sync(&key_change_passphrase),
1946         &ObjectSchema::new(
1947             "Change the passphrase required to decrypt the key.",
1948             &sorted!([
1949                 ("path", false, &StringSchema::new("File system path.").schema()),
1950                 ("kdf", true, &KDF_SCHEMA),
1951             ]),
1952         )
1953     );
1954
1955     let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_KEY_CHANGE_PASSPHRASE)
1956         .arg_param(&["path"])
1957         .completion_cb("path", tools::complete_file_name);
1958
1959     const API_METHOD_KEY_CREATE_MASTER_KEY: ApiMethod = ApiMethod::new(
1960         &ApiHandler::Sync(&key_create_master_key),
1961         &ObjectSchema::new("Create a new 4096 bit RSA master pub/priv key pair.", &[])
1962     );
1963
1964     let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE_MASTER_KEY);
1965
1966     #[sortable]
1967     const API_METHOD_KEY_IMPORT_MASTER_PUBKEY: ApiMethod = ApiMethod::new(
1968         &ApiHandler::Sync(&key_import_master_pubkey),
1969         &ObjectSchema::new(
1970             "Import a new RSA public key and use it as master key. The key is expected to be in '.pem' format.",
1971             &sorted!([ ("path", false, &StringSchema::new("File system path.").schema()) ]),
1972         )
1973     );
1974
1975     let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_KEY_IMPORT_MASTER_PUBKEY)
1976         .arg_param(&["path"])
1977         .completion_cb("path", tools::complete_file_name);
1978
1979     CliCommandMap::new()
1980         .insert("create", key_create_cmd_def)
1981         .insert("create-master-key", key_create_master_key_cmd_def)
1982         .insert("import-master-pubkey", key_import_master_pubkey_cmd_def)
1983         .insert("change-passphrase", key_change_passphrase_cmd_def)
1984 }
1985
1986
1987 use proxmox_backup::client::RemoteChunkReader;
1988 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
1989 /// async use!
1990 ///
1991 /// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
1992 /// so that we can properly access it from multiple threads simultaneously while not issuing
1993 /// duplicate simultaneous reads over http.
1994 pub struct BufferedDynamicReadAt {
1995     inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
1996 }
1997
1998 impl BufferedDynamicReadAt {
1999     fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
2000         Self {
2001             inner: Mutex::new(inner),
2002         }
2003     }
2004 }
2005
2006 impl ReadAt for BufferedDynamicReadAt {
2007     fn start_read_at<'a>(
2008         self: Pin<&'a Self>,
2009         _cx: &mut Context,
2010         buf: &'a mut [u8],
2011         offset: u64,
2012     ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
2013         use std::io::Read;
2014         MaybeReady::Ready(tokio::task::block_in_place(move || {
2015             let mut reader = self.inner.lock().unwrap();
2016             reader.seek(SeekFrom::Start(offset))?;
2017             Ok(reader.read(buf)?)
2018         }))
2019     }
2020
2021     fn poll_complete<'a>(
2022         self: Pin<&'a Self>,
2023         _op: ReadAtOperation<'a>,
2024     ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
2025         panic!("LocalDynamicReadAt::start_read_at returned Pending");
2026     }
2027 }
2028
2029
2030 #[api(
2031     input: {
2032         properties: {
2033             "snapshot": {
2034                 type: String,
2035                 description: "Group/Snapshot path.",
2036             },
2037             "archive-name": {
2038                 type: String,
2039                 description: "Backup archive name.",
2040             },
2041             "repository": {
2042                 optional: true,
2043                 schema: REPO_URL_SCHEMA,
2044             },
2045             "keyfile": {
2046                 optional: true,
2047                 type: String,
2048                 description: "Path to encryption key.",
2049             },
2050         },
2051     },
2052 )]
2053 /// Shell to interactively inspect and restore snapshots.
2054 async fn catalog_shell(param: Value) -> Result<(), Error> {
2055     let repo = extract_repository_from_value(&param)?;
2056     let client = connect(repo.host(), repo.user())?;
2057     let path = tools::required_string_param(&param, "snapshot")?;
2058     let archive_name = tools::required_string_param(&param, "archive-name")?;
2059
2060     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
2061         let group: BackupGroup = path.parse()?;
2062         api_datastore_latest_snapshot(&client, repo.store(), group).await?
2063     } else {
2064         let snapshot: BackupDir = path.parse()?;
2065         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
2066     };
2067
2068     let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
2069     let crypt_config = match keyfile {
2070         None => None,
2071         Some(path) => {
2072             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
2073             Some(Arc::new(CryptConfig::new(key)?))
2074         }
2075     };
2076
2077     let server_archive_name = if archive_name.ends_with(".pxar") {
2078         format!("{}.didx", archive_name)
2079     } else {
2080         bail!("Can only mount pxar archives.");
2081     };
2082
2083     let client = BackupReader::start(
2084         client,
2085         crypt_config.clone(),
2086         repo.store(),
2087         &backup_type,
2088         &backup_id,
2089         backup_time,
2090         true,
2091     ).await?;
2092
2093     let mut tmpfile = std::fs::OpenOptions::new()
2094         .write(true)
2095         .read(true)
2096         .custom_flags(libc::O_TMPFILE)
2097         .open("/tmp")?;
2098
2099     let manifest = client.download_manifest().await?;
2100
2101     let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
2102     let most_used = index.find_most_used_chunks(8);
2103     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
2104     let reader = BufferedDynamicReader::new(index, chunk_reader);
2105     let archive_size = reader.archive_size();
2106     let reader: proxmox_backup::pxar::fuse::Reader =
2107         Arc::new(BufferedDynamicReadAt::new(reader));
2108     let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
2109
2110     client.download(CATALOG_NAME, &mut tmpfile).await?;
2111     let index = DynamicIndexReader::new(tmpfile)
2112         .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
2113
2114     // Note: do not use values stored in index (not trusted) - instead, computed them again
2115     let (csum, size) = index.compute_csum();
2116     manifest.verify_file(CATALOG_NAME, &csum, size)?;
2117
2118     let most_used = index.find_most_used_chunks(8);
2119     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
2120     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
2121     let mut catalogfile = std::fs::OpenOptions::new()
2122         .write(true)
2123         .read(true)
2124         .custom_flags(libc::O_TMPFILE)
2125         .open("/tmp")?;
2126
2127     std::io::copy(&mut reader, &mut catalogfile)
2128         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
2129
2130     catalogfile.seek(SeekFrom::Start(0))?;
2131     let catalog_reader = CatalogReader::new(catalogfile);
2132     let state = Shell::new(
2133         catalog_reader,
2134         &server_archive_name,
2135         decoder,
2136     ).await?;
2137
2138     println!("Starting interactive shell");
2139     state.shell().await?;
2140
2141     record_repository(&repo);
2142
2143     Ok(())
2144 }
2145
2146 fn catalog_mgmt_cli() -> CliCommandMap {
2147     let catalog_shell_cmd_def = CliCommand::new(&API_METHOD_CATALOG_SHELL)
2148         .arg_param(&["snapshot", "archive-name"])
2149         .completion_cb("repository", complete_repository)
2150         .completion_cb("archive-name", complete_pxar_archive_name)
2151         .completion_cb("snapshot", complete_group_or_snapshot);
2152
2153     let catalog_dump_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
2154         .arg_param(&["snapshot"])
2155         .completion_cb("repository", complete_repository)
2156         .completion_cb("snapshot", complete_backup_snapshot);
2157
2158     CliCommandMap::new()
2159         .insert("dump", catalog_dump_cmd_def)
2160         .insert("shell", catalog_shell_cmd_def)
2161 }
2162
2163 fn main() {
2164
2165     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
2166         .arg_param(&["backupspec"])
2167         .completion_cb("repository", complete_repository)
2168         .completion_cb("backupspec", complete_backup_source)
2169         .completion_cb("keyfile", tools::complete_file_name)
2170         .completion_cb("chunk-size", complete_chunk_size);
2171
2172     let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
2173         .completion_cb("repository", complete_repository)
2174         .completion_cb("keyfile", tools::complete_file_name);
2175
2176     let upload_log_cmd_def = CliCommand::new(&API_METHOD_UPLOAD_LOG)
2177         .arg_param(&["snapshot", "logfile"])
2178         .completion_cb("snapshot", complete_backup_snapshot)
2179         .completion_cb("logfile", tools::complete_file_name)
2180         .completion_cb("keyfile", tools::complete_file_name)
2181         .completion_cb("repository", complete_repository);
2182
2183     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
2184         .completion_cb("repository", complete_repository);
2185
2186     let snapshots_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOTS)
2187         .arg_param(&["group"])
2188         .completion_cb("group", complete_backup_group)
2189         .completion_cb("repository", complete_repository);
2190
2191     let forget_cmd_def = CliCommand::new(&API_METHOD_FORGET_SNAPSHOTS)
2192         .arg_param(&["snapshot"])
2193         .completion_cb("repository", complete_repository)
2194         .completion_cb("snapshot", complete_backup_snapshot);
2195
2196     let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
2197         .completion_cb("repository", complete_repository);
2198
2199     let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
2200         .arg_param(&["snapshot", "archive-name", "target"])
2201         .completion_cb("repository", complete_repository)
2202         .completion_cb("snapshot", complete_group_or_snapshot)
2203         .completion_cb("archive-name", complete_archive_name)
2204         .completion_cb("target", tools::complete_file_name);
2205
2206     let files_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOT_FILES)
2207         .arg_param(&["snapshot"])
2208         .completion_cb("repository", complete_repository)
2209         .completion_cb("snapshot", complete_backup_snapshot);
2210
2211     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
2212         .arg_param(&["group"])
2213         .completion_cb("group", complete_backup_group)
2214         .completion_cb("repository", complete_repository);
2215
2216     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
2217         .completion_cb("repository", complete_repository);
2218
2219     let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
2220         .completion_cb("repository", complete_repository);
2221
2222     let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
2223         .completion_cb("repository", complete_repository);
2224
2225     let cmd_def = CliCommandMap::new()
2226         .insert("backup", backup_cmd_def)
2227         .insert("upload-log", upload_log_cmd_def)
2228         .insert("forget", forget_cmd_def)
2229         .insert("garbage-collect", garbage_collect_cmd_def)
2230         .insert("list", list_cmd_def)
2231         .insert("login", login_cmd_def)
2232         .insert("logout", logout_cmd_def)
2233         .insert("prune", prune_cmd_def)
2234         .insert("restore", restore_cmd_def)
2235         .insert("snapshots", snapshots_cmd_def)
2236         .insert("files", files_cmd_def)
2237         .insert("status", status_cmd_def)
2238         .insert("key", key_mgmt_cli())
2239         .insert("mount", mount_cmd_def())
2240         .insert("catalog", catalog_mgmt_cli())
2241         .insert("task", task_mgmt_cli())
2242         .insert("benchmark", benchmark_cmd_def);
2243
2244     let rpcenv = CliEnvironment::new();
2245     run_cli_command(cmd_def, rpcenv, Some(|future| {
2246         proxmox_backup::tools::runtime::main(future)
2247     }));
2248 }