src/bin/proxmox-backup-client.rs

   1 use std::collections::{HashSet, HashMap};
   2 use std::ffi::OsStr;
   3 use std::io::{self, Write, Seek, SeekFrom};
   4 use std::os::unix::fs::OpenOptionsExt;
   5 use std::os::unix::io::RawFd;
   6 use std::path::{Path, PathBuf};
   7 use std::pin::Pin;
   8 use std::sync::{Arc, Mutex};
   9 use std::task::Context;
  10
  11 use anyhow::{bail, format_err, Error};
  12 use chrono::{Local, DateTime, Utc, TimeZone};
  13 use futures::future::FutureExt;
  14 use futures::select;
  15 use futures::stream::{StreamExt, TryStreamExt};
  16 use nix::unistd::{fork, ForkResult, pipe};
  17 use serde_json::{json, Value};
  18 use tokio::signal::unix::{signal, SignalKind};
  19 use tokio::sync::mpsc;
  20 use xdg::BaseDirectories;
  21
  22 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
  23 use proxmox::{sortable, identity};
  24 use proxmox::tools::fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size};
  25 use proxmox::sys::linux::tty;
  26 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
  27 use proxmox::api::schema::*;
  28 use proxmox::api::cli::*;
  29 use proxmox::api::api;
  30 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
  31
  32 use proxmox_backup::tools;
  33 use proxmox_backup::api2::types::*;
  34 use proxmox_backup::client::*;
  35 use proxmox_backup::pxar::catalog::*;
  36 use proxmox_backup::backup::{
  37     archive_type,
  38     encrypt_key_with_passphrase,
  39     load_and_decrypt_key,
  40     store_key_config,
  41     verify_chunk_size,
  42     ArchiveType,
  43     AsyncReadChunk,
  44     BackupDir,
  45     BackupGroup,
  46     BackupManifest,
  47     BufferedDynamicReader,
  48     CatalogReader,
  49     CatalogWriter,
  50     CATALOG_NAME,
  51     ChunkStream,
  52     CryptConfig,
  53     DataBlob,
  54     DynamicIndexReader,
  55     FixedChunkStream,
  56     FixedIndexReader,
  57     IndexFile,
  58     KeyConfig,
  59     MANIFEST_BLOB_NAME,
  60     Shell,
  61 };
  62
  63 mod proxmox_backup_client;
  64 use proxmox_backup_client::*;
  65
  66 const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
  67 const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
  68
  69
  70 pub const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  71     .format(&BACKUP_REPO_URL)
  72     .max_length(256)
  73     .schema();
  74
  75 pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
  76     "Path to encryption key. All data will be encrypted using this key.")
  77     .schema();
  78
  79 const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new(
  80     "Chunk size in KB. Must be a power of 2.")
  81     .minimum(64)
  82     .maximum(4096)
  83     .default(4096)
  84     .schema();
  85
  86 fn get_default_repository() -> Option<String> {
  87     std::env::var("PBS_REPOSITORY").ok()
  88 }
  89
  90 pub fn extract_repository_from_value(
  91     param: &Value,
  92 ) -> Result<BackupRepository, Error> {
  93
  94     let repo_url = param["repository"]
  95         .as_str()
  96         .map(String::from)
  97         .or_else(get_default_repository)
  98         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
  99
 100     let repo: BackupRepository = repo_url.parse()?;
 101
 102     Ok(repo)
 103 }
 104
 105 fn extract_repository_from_map(
 106     param: &HashMap<String, String>,
 107 ) -> Option<BackupRepository> {
 108
 109     param.get("repository")
 110         .map(String::from)
 111         .or_else(get_default_repository)
 112         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
 113 }
 114
 115 fn record_repository(repo: &BackupRepository) {
 116
 117     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 118         Ok(v) => v,
 119         _ => return,
 120     };
 121
 122     // usually $HOME/.cache/proxmox-backup/repo-list
 123     let path = match base.place_cache_file("repo-list") {
 124         Ok(v) => v,
 125         _ => return,
 126     };
 127
 128     let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 129
 130     let repo = repo.to_string();
 131
 132     data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
 133
 134     let mut map = serde_json::map::Map::new();
 135
 136     loop {
 137         let mut max_used = 0;
 138         let mut max_repo = None;
 139         for (repo, count) in data.as_object().unwrap() {
 140             if map.contains_key(repo) { continue; }
 141             if let Some(count) = count.as_i64() {
 142                 if count > max_used {
 143                     max_used = count;
 144                     max_repo = Some(repo);
 145                 }
 146             }
 147         }
 148         if let Some(repo) = max_repo {
 149             map.insert(repo.to_owned(), json!(max_used));
 150         } else {
 151             break;
 152         }
 153         if map.len() > 10 { // store max. 10 repos
 154             break;
 155         }
 156     }
 157
 158     let new_data = json!(map);
 159
 160     let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
 161 }
 162
 163 fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 164
 165     let mut result = vec![];
 166
 167     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 168         Ok(v) => v,
 169         _ => return result,
 170     };
 171
 172     // usually $HOME/.cache/proxmox-backup/repo-list
 173     let path = match base.place_cache_file("repo-list") {
 174         Ok(v) => v,
 175         _ => return result,
 176     };
 177
 178     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 179
 180     if let Some(map) = data.as_object() {
 181         for (repo, _count) in map {
 182             result.push(repo.to_owned());
 183         }
 184     }
 185
 186     result
 187 }
 188
 189 fn connect(server: &str, userid: &str) -> Result<HttpClient, Error> {
 190
 191     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 192
 193     use std::env::VarError::*;
 194     let password = match std::env::var(ENV_VAR_PBS_PASSWORD) {
 195         Ok(p) => Some(p),
 196         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", ENV_VAR_PBS_PASSWORD)),
 197         Err(NotPresent) => None,
 198     };
 199
 200     let options = HttpClientOptions::new()
 201         .prefix(Some("proxmox-backup".to_string()))
 202         .password(password)
 203         .interactive(true)
 204         .fingerprint(fingerprint)
 205         .fingerprint_cache(true)
 206         .ticket_cache(true);
 207
 208     HttpClient::new(server, userid, options)
 209 }
 210
 211 async fn view_task_result(
 212     client: HttpClient,
 213     result: Value,
 214     output_format: &str,
 215 ) -> Result<(), Error> {
 216     let data = &result["data"];
 217     if output_format == "text" {
 218         if let Some(upid) = data.as_str() {
 219             display_task_log(client, upid, true).await?;
 220         }
 221     } else {
 222         format_and_print_result(&data, &output_format);
 223     }
 224
 225     Ok(())
 226 }
 227
 228 async fn api_datastore_list_snapshots(
 229     client: &HttpClient,
 230     store: &str,
 231     group: Option<BackupGroup>,
 232 ) -> Result<Value, Error> {
 233
 234     let path = format!("api2/json/admin/datastore/{}/snapshots", store);
 235
 236     let mut args = json!({});
 237     if let Some(group) = group {
 238         args["backup-type"] = group.backup_type().into();
 239         args["backup-id"] = group.backup_id().into();
 240     }
 241
 242     let mut result = client.get(&path, Some(args)).await?;
 243
 244     Ok(result["data"].take())
 245 }
 246
 247 async fn api_datastore_latest_snapshot(
 248     client: &HttpClient,
 249     store: &str,
 250     group: BackupGroup,
 251 ) -> Result<(String, String, DateTime<Utc>), Error> {
 252
 253     let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
 254     let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
 255
 256     if list.is_empty() {
 257         bail!("backup group {:?} does not contain any snapshots.", group.group_path());
 258     }
 259
 260     list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
 261
 262     let backup_time = Utc.timestamp(list[0].backup_time, 0);
 263
 264     Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
 265 }
 266
 267 async fn backup_directory<P: AsRef<Path>>(
 268     client: &BackupWriter,
 269     previous_manifest: Option<Arc<BackupManifest>>,
 270     dir_path: P,
 271     archive_name: &str,
 272     chunk_size: Option<usize>,
 273     device_set: Option<HashSet<u64>>,
 274     verbose: bool,
 275     skip_lost_and_found: bool,
 276     catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
 277     exclude_pattern: Vec<MatchEntry>,
 278     entries_max: usize,
 279 ) -> Result<BackupStats, Error> {
 280
 281     let pxar_stream = PxarBackupStream::open(
 282         dir_path.as_ref(),
 283         device_set,
 284         verbose,
 285         skip_lost_and_found,
 286         catalog,
 287         exclude_pattern,
 288         entries_max,
 289     )?;
 290     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 291
 292     let (mut tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
 293
 294     let stream = rx
 295         .map_err(Error::from);
 296
 297     // spawn chunker inside a separate task so that it can run parallel
 298     tokio::spawn(async move {
 299         while let Some(v) = chunk_stream.next().await {
 300             let _ = tx.send(v).await;
 301         }
 302     });
 303
 304     let stats = client
 305         .upload_stream(previous_manifest, archive_name, stream, "dynamic", None)
 306         .await?;
 307
 308     Ok(stats)
 309 }
 310
 311 async fn backup_image<P: AsRef<Path>>(
 312     client: &BackupWriter,
 313     previous_manifest: Option<Arc<BackupManifest>>,
 314     image_path: P,
 315     archive_name: &str,
 316     image_size: u64,
 317     chunk_size: Option<usize>,
 318     _verbose: bool,
 319 ) -> Result<BackupStats, Error> {
 320
 321     let path = image_path.as_ref().to_owned();
 322
 323     let file = tokio::fs::File::open(path).await?;
 324
 325     let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
 326         .map_err(Error::from);
 327
 328     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 329
 330     let stats = client
 331         .upload_stream(previous_manifest, archive_name, stream, "fixed", Some(image_size))
 332         .await?;
 333
 334     Ok(stats)
 335 }
 336
 337 #[api(
 338    input: {
 339         properties: {
 340             repository: {
 341                 schema: REPO_URL_SCHEMA,
 342                 optional: true,
 343             },
 344             "output-format": {
 345                 schema: OUTPUT_FORMAT,
 346                 optional: true,
 347             },
 348         }
 349    }
 350 )]
 351 /// List backup groups.
 352 async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 353
 354     let output_format = get_output_format(&param);
 355
 356     let repo = extract_repository_from_value(&param)?;
 357
 358     let client = connect(repo.host(), repo.user())?;
 359
 360     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 361
 362     let mut result = client.get(&path, None).await?;
 363
 364     record_repository(&repo);
 365
 366     let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
 367         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 368         let group = BackupGroup::new(item.backup_type, item.backup_id);
 369         Ok(group.group_path().to_str().unwrap().to_owned())
 370     };
 371
 372     let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
 373         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 374         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup);
 375         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
 376     };
 377
 378     let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
 379         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 380         Ok(tools::format::render_backup_file_list(&item.files))
 381     };
 382
 383     let options = default_table_format_options()
 384         .sortby("backup-type", false)
 385         .sortby("backup-id", false)
 386         .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
 387         .column(
 388             ColumnConfig::new("last-backup")
 389                 .renderer(render_last_backup)
 390                 .header("last snapshot")
 391                 .right_align(false)
 392         )
 393         .column(ColumnConfig::new("backup-count"))
 394         .column(ColumnConfig::new("files").renderer(render_files));
 395
 396     let mut data: Value = result["data"].take();
 397
 398     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_LIST_GROUPS;
 399
 400     format_and_print_result_full(&mut data, info, &output_format, &options);
 401
 402     Ok(Value::Null)
 403 }
 404
 405 #[api(
 406    input: {
 407         properties: {
 408             repository: {
 409                 schema: REPO_URL_SCHEMA,
 410                 optional: true,
 411             },
 412             group: {
 413                 type: String,
 414                 description: "Backup group.",
 415                 optional: true,
 416             },
 417             "output-format": {
 418                 schema: OUTPUT_FORMAT,
 419                 optional: true,
 420             },
 421         }
 422    }
 423 )]
 424 /// List backup snapshots.
 425 async fn list_snapshots(param: Value) -> Result<Value, Error> {
 426
 427     let repo = extract_repository_from_value(&param)?;
 428
 429     let output_format = get_output_format(&param);
 430
 431     let client = connect(repo.host(), repo.user())?;
 432
 433     let group: Option<BackupGroup> = if let Some(path) = param["group"].as_str() {
 434         Some(path.parse()?)
 435     } else {
 436         None
 437     };
 438
 439     let mut data = api_datastore_list_snapshots(&client, repo.store(), group).await?;
 440
 441     record_repository(&repo);
 442
 443     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
 444         let item: SnapshotListItem = serde_json::from_value(record.to_owned())?;
 445         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
 446         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
 447     };
 448
 449     let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
 450         let item: SnapshotListItem = serde_json::from_value(record.to_owned())?;
 451         let mut filenames = Vec::new();
 452         for file in &item.files {
 453             filenames.push(file.filename.to_string());
 454         }
 455         Ok(tools::format::render_backup_file_list(&filenames[..]))
 456     };
 457
 458     let options = default_table_format_options()
 459         .sortby("backup-type", false)
 460         .sortby("backup-id", false)
 461         .sortby("backup-time", false)
 462         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
 463         .column(ColumnConfig::new("size"))
 464         .column(ColumnConfig::new("files").renderer(render_files))
 465         ;
 466
 467     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_LIST_SNAPSHOTS;
 468
 469     format_and_print_result_full(&mut data, info, &output_format, &options);
 470
 471     Ok(Value::Null)
 472 }
 473
 474 #[api(
 475    input: {
 476         properties: {
 477             repository: {
 478                 schema: REPO_URL_SCHEMA,
 479                 optional: true,
 480             },
 481             snapshot: {
 482                 type: String,
 483                 description: "Snapshot path.",
 484              },
 485         }
 486    }
 487 )]
 488 /// Forget (remove) backup snapshots.
 489 async fn forget_snapshots(param: Value) -> Result<Value, Error> {
 490
 491     let repo = extract_repository_from_value(&param)?;
 492
 493     let path = tools::required_string_param(&param, "snapshot")?;
 494     let snapshot: BackupDir = path.parse()?;
 495
 496     let mut client = connect(repo.host(), repo.user())?;
 497
 498     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 499
 500     let result = client.delete(&path, Some(json!({
 501         "backup-type": snapshot.group().backup_type(),
 502         "backup-id": snapshot.group().backup_id(),
 503         "backup-time": snapshot.backup_time().timestamp(),
 504     }))).await?;
 505
 506     record_repository(&repo);
 507
 508     Ok(result)
 509 }
 510
 511 #[api(
 512    input: {
 513         properties: {
 514             repository: {
 515                 schema: REPO_URL_SCHEMA,
 516                 optional: true,
 517             },
 518         }
 519    }
 520 )]
 521 /// Try to login. If successful, store ticket.
 522 async fn api_login(param: Value) -> Result<Value, Error> {
 523
 524     let repo = extract_repository_from_value(&param)?;
 525
 526     let client = connect(repo.host(), repo.user())?;
 527     client.login().await?;
 528
 529     record_repository(&repo);
 530
 531     Ok(Value::Null)
 532 }
 533
 534 #[api(
 535    input: {
 536         properties: {
 537             repository: {
 538                 schema: REPO_URL_SCHEMA,
 539                 optional: true,
 540             },
 541         }
 542    }
 543 )]
 544 /// Logout (delete stored ticket).
 545 fn api_logout(param: Value) -> Result<Value, Error> {
 546
 547     let repo = extract_repository_from_value(&param)?;
 548
 549     delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
 550
 551     Ok(Value::Null)
 552 }
 553
 554 #[api(
 555    input: {
 556         properties: {
 557             repository: {
 558                 schema: REPO_URL_SCHEMA,
 559                 optional: true,
 560             },
 561             snapshot: {
 562                 type: String,
 563                 description: "Snapshot path.",
 564              },
 565         }
 566    }
 567 )]
 568 /// Dump catalog.
 569 async fn dump_catalog(param: Value) -> Result<Value, Error> {
 570
 571     let repo = extract_repository_from_value(&param)?;
 572
 573     let path = tools::required_string_param(&param, "snapshot")?;
 574     let snapshot: BackupDir = path.parse()?;
 575
 576     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 577
 578     let crypt_config = match keyfile {
 579         None => None,
 580         Some(path) => {
 581             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
 582             Some(Arc::new(CryptConfig::new(key)?))
 583         }
 584     };
 585
 586     let client = connect(repo.host(), repo.user())?;
 587
 588     let client = BackupReader::start(
 589         client,
 590         crypt_config.clone(),
 591         repo.store(),
 592         &snapshot.group().backup_type(),
 593         &snapshot.group().backup_id(),
 594         snapshot.backup_time(),
 595         true,
 596     ).await?;
 597
 598     let manifest = client.download_manifest().await?;
 599
 600     let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
 601
 602     let most_used = index.find_most_used_chunks(8);
 603
 604     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 605
 606     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 607
 608     let mut catalogfile = std::fs::OpenOptions::new()
 609         .write(true)
 610         .read(true)
 611         .custom_flags(libc::O_TMPFILE)
 612         .open("/tmp")?;
 613
 614     std::io::copy(&mut reader, &mut catalogfile)
 615         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
 616
 617     catalogfile.seek(SeekFrom::Start(0))?;
 618
 619     let mut catalog_reader = CatalogReader::new(catalogfile);
 620
 621     catalog_reader.dump()?;
 622
 623     record_repository(&repo);
 624
 625     Ok(Value::Null)
 626 }
 627
 628 #[api(
 629    input: {
 630         properties: {
 631             repository: {
 632                 schema: REPO_URL_SCHEMA,
 633                 optional: true,
 634             },
 635             snapshot: {
 636                 type: String,
 637                 description: "Snapshot path.",
 638              },
 639             "output-format": {
 640                 schema: OUTPUT_FORMAT,
 641                 optional: true,
 642             },
 643         }
 644    }
 645 )]
 646 /// List snapshot files.
 647 async fn list_snapshot_files(param: Value) -> Result<Value, Error> {
 648
 649     let repo = extract_repository_from_value(&param)?;
 650
 651     let path = tools::required_string_param(&param, "snapshot")?;
 652     let snapshot: BackupDir = path.parse()?;
 653
 654     let output_format = get_output_format(&param);
 655
 656     let client = connect(repo.host(), repo.user())?;
 657
 658     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 659
 660     let mut result = client.get(&path, Some(json!({
 661         "backup-type": snapshot.group().backup_type(),
 662         "backup-id": snapshot.group().backup_id(),
 663         "backup-time": snapshot.backup_time().timestamp(),
 664     }))).await?;
 665
 666     record_repository(&repo);
 667
 668     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_LIST_SNAPSHOT_FILES;
 669
 670     let mut data: Value = result["data"].take();
 671
 672     let options = default_table_format_options();
 673
 674     format_and_print_result_full(&mut data, info, &output_format, &options);
 675
 676     Ok(Value::Null)
 677 }
 678
 679 #[api(
 680     input: {
 681         properties: {
 682             repository: {
 683                 schema: REPO_URL_SCHEMA,
 684                 optional: true,
 685             },
 686             "output-format": {
 687                 schema: OUTPUT_FORMAT,
 688                 optional: true,
 689             },
 690         },
 691     },
 692 )]
 693 /// Start garbage collection for a specific repository.
 694 async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 695
 696     let repo = extract_repository_from_value(&param)?;
 697
 698     let output_format = get_output_format(&param);
 699
 700     let mut client = connect(repo.host(), repo.user())?;
 701
 702     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 703
 704     let result = client.post(&path, None).await?;
 705
 706     record_repository(&repo);
 707
 708     view_task_result(client, result, &output_format).await?;
 709
 710     Ok(Value::Null)
 711 }
 712
 713 fn spawn_catalog_upload(
 714     client: Arc<BackupWriter>
 715 ) -> Result<
 716         (
 717             Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
 718             tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>
 719         ), Error>
 720 {
 721     let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
 722     let catalog_stream = crate::tools::StdChannelStream(catalog_rx);
 723     let catalog_chunk_size = 512*1024;
 724     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 725
 726     let catalog = Arc::new(Mutex::new(CatalogWriter::new(crate::tools::StdChannelWriter::new(catalog_tx))?));
 727
 728     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 729
 730     tokio::spawn(async move {
 731         let catalog_upload_result = client
 732             .upload_stream(None, CATALOG_NAME, catalog_chunk_stream, "dynamic", None)
 733             .await;
 734
 735         if let Err(ref err) = catalog_upload_result {
 736             eprintln!("catalog upload error - {}", err);
 737             client.cancel();
 738         }
 739
 740         let _ = catalog_result_tx.send(catalog_upload_result);
 741     });
 742
 743     Ok((catalog, catalog_result_rx))
 744 }
 745
 746 #[api(
 747    input: {
 748        properties: {
 749            backupspec: {
 750                type: Array,
 751                description: "List of backup source specifications ([<label.ext>:<path>] ...)",
 752                items: {
 753                    schema: BACKUP_SOURCE_SCHEMA,
 754                }
 755            },
 756            repository: {
 757                schema: REPO_URL_SCHEMA,
 758                optional: true,
 759            },
 760            "include-dev": {
 761                description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
 762                optional: true,
 763                items: {
 764                    type: String,
 765                    description: "Path to file.",
 766                }
 767            },
 768            keyfile: {
 769                schema: KEYFILE_SCHEMA,
 770                optional: true,
 771            },
 772            "skip-lost-and-found": {
 773                type: Boolean,
 774                description: "Skip lost+found directory.",
 775                optional: true,
 776            },
 777            "backup-type": {
 778                schema: BACKUP_TYPE_SCHEMA,
 779                optional: true,
 780            },
 781            "backup-id": {
 782                schema: BACKUP_ID_SCHEMA,
 783                optional: true,
 784            },
 785            "backup-time": {
 786                schema: BACKUP_TIME_SCHEMA,
 787                optional: true,
 788            },
 789            "chunk-size": {
 790                schema: CHUNK_SIZE_SCHEMA,
 791                optional: true,
 792            },
 793            "exclude": {
 794                type: Array,
 795                description: "List of paths or patterns for matching files to exclude.",
 796                optional: true,
 797                items: {
 798                    type: String,
 799                    description: "Path or match pattern.",
 800                 }
 801            },
 802            "entries-max": {
 803                type: Integer,
 804                description: "Max number of entries to hold in memory.",
 805                optional: true,
 806                default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
 807            },
 808            "verbose": {
 809                type: Boolean,
 810                description: "Verbose output.",
 811                optional: true,
 812            },
 813        }
 814    }
 815 )]
 816 /// Create (host) backup.
 817 async fn create_backup(
 818     param: Value,
 819     _info: &ApiMethod,
 820     _rpcenv: &mut dyn RpcEnvironment,
 821 ) -> Result<Value, Error> {
 822
 823     let repo = extract_repository_from_value(&param)?;
 824
 825     let backupspec_list = tools::required_array_param(&param, "backupspec")?;
 826
 827     let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
 828
 829     let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
 830
 831     let verbose = param["verbose"].as_bool().unwrap_or(false);
 832
 833     let backup_time_opt = param["backup-time"].as_i64();
 834
 835     let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
 836
 837     if let Some(size) = chunk_size_opt {
 838         verify_chunk_size(size)?;
 839     }
 840
 841     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 842
 843     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 844
 845     let backup_type = param["backup-type"].as_str().unwrap_or("host");
 846
 847     let include_dev = param["include-dev"].as_array();
 848
 849     let entries_max = param["entries-max"].as_u64()
 850         .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
 851
 852     let empty = Vec::new();
 853     let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
 854
 855     let mut pattern_list = Vec::with_capacity(exclude_args.len());
 856     for entry in exclude_args {
 857         let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
 858         pattern_list.push(
 859             MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
 860                 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
 861         );
 862     }
 863
 864     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
 865
 866     if let Some(include_dev) = include_dev {
 867         if all_file_systems {
 868             bail!("option 'all-file-systems' conflicts with option 'include-dev'");
 869         }
 870
 871         let mut set = HashSet::new();
 872         for path in include_dev {
 873             let path = path.as_str().unwrap();
 874             let stat = nix::sys::stat::stat(path)
 875                 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
 876             set.insert(stat.st_dev);
 877         }
 878         devices = Some(set);
 879     }
 880
 881     let mut upload_list = vec![];
 882
 883     for backupspec in backupspec_list {
 884         let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
 885         let filename = &spec.config_string;
 886         let target = &spec.archive_name;
 887
 888         use std::os::unix::fs::FileTypeExt;
 889
 890         let metadata = std::fs::metadata(filename)
 891             .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
 892         let file_type = metadata.file_type();
 893
 894         match spec.spec_type {
 895             BackupSpecificationType::PXAR => {
 896                 if !file_type.is_dir() {
 897                     bail!("got unexpected file type (expected directory)");
 898                 }
 899                 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
 900             }
 901             BackupSpecificationType::IMAGE => {
 902                 if !(file_type.is_file() || file_type.is_block_device()) {
 903                     bail!("got unexpected file type (expected file or block device)");
 904                 }
 905
 906                 let size = image_size(&PathBuf::from(filename))?;
 907
 908                 if size == 0 { bail!("got zero-sized file '{}'", filename); }
 909
 910                 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
 911             }
 912             BackupSpecificationType::CONFIG => {
 913                 if !file_type.is_file() {
 914                     bail!("got unexpected file type (expected regular file)");
 915                 }
 916                 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 917             }
 918             BackupSpecificationType::LOGFILE => {
 919                 if !file_type.is_file() {
 920                     bail!("got unexpected file type (expected regular file)");
 921                 }
 922                 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 923             }
 924         }
 925     }
 926
 927     let backup_time = Utc.timestamp(backup_time_opt.unwrap_or_else(|| Utc::now().timestamp()), 0);
 928
 929     let client = connect(repo.host(), repo.user())?;
 930     record_repository(&repo);
 931
 932     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time));
 933
 934     println!("Client name: {}", proxmox::tools::nodename());
 935
 936     let start_time = Local::now();
 937
 938     println!("Starting protocol: {}", start_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 939
 940     let (crypt_config, rsa_encrypted_key) = match keyfile {
 941         None => (None, None),
 942         Some(path) => {
 943             let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
 944
 945             let crypt_config = CryptConfig::new(key)?;
 946
 947             let path = master_pubkey_path()?;
 948             if path.exists() {
 949                 let pem_data = file_get_contents(&path)?;
 950                 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
 951                 let enc_key = crypt_config.generate_rsa_encoded_key(rsa, created)?;
 952                 (Some(Arc::new(crypt_config)), Some(enc_key))
 953             } else {
 954                 (Some(Arc::new(crypt_config)), None)
 955             }
 956         }
 957     };
 958
 959     let is_encrypted = Some(crypt_config.is_some());
 960
 961     let client = BackupWriter::start(
 962         client,
 963         crypt_config.clone(),
 964         repo.store(),
 965         backup_type,
 966         &backup_id,
 967         backup_time,
 968         verbose,
 969     ).await?;
 970
 971     let previous_manifest = if let Ok(previous_manifest) = client.download_previous_manifest().await {
 972         Some(Arc::new(previous_manifest))
 973     } else {
 974         None
 975     };
 976
 977     let snapshot = BackupDir::new(backup_type, backup_id, backup_time.timestamp());
 978     let mut manifest = BackupManifest::new(snapshot);
 979
 980     let mut catalog = None;
 981     let mut catalog_result_tx = None;
 982
 983     for (backup_type, filename, target, size) in upload_list {
 984         match backup_type {
 985             BackupSpecificationType::CONFIG => {
 986                 println!("Upload config file '{}' to '{:?}' as {}", filename, repo, target);
 987                 let stats = client
 988                     .upload_blob_from_file(&filename, &target, true, Some(true))
 989                     .await?;
 990                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
 991             }
 992             BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
 993                 println!("Upload log file '{}' to '{:?}' as {}", filename, repo, target);
 994                 let stats = client
 995                     .upload_blob_from_file(&filename, &target, true, Some(true))
 996                     .await?;
 997                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
 998             }
 999             BackupSpecificationType::PXAR => {
1000                 // start catalog upload on first use
1001                 if catalog.is_none() {
1002                     let (cat, res) = spawn_catalog_upload(client.clone())?;
1003                     catalog = Some(cat);
1004                     catalog_result_tx = Some(res);
1005                 }
1006                 let catalog = catalog.as_ref().unwrap();
1007
1008                 println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
1009                 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
1010                 let stats = backup_directory(
1011                     &client,
1012                     previous_manifest.clone(),
1013                     &filename,
1014                     &target,
1015                     chunk_size_opt,
1016                     devices.clone(),
1017                     verbose,
1018                     skip_lost_and_found,
1019                     catalog.clone(),
1020                     pattern_list.clone(),
1021                     entries_max as usize,
1022                 ).await?;
1023                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
1024                 catalog.lock().unwrap().end_directory()?;
1025             }
1026             BackupSpecificationType::IMAGE => {
1027                 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
1028                 let stats = backup_image(
1029                     &client,
1030                     previous_manifest.clone(),
1031                      &filename,
1032                     &target,
1033                     size,
1034                     chunk_size_opt,
1035                     verbose,
1036                 ).await?;
1037                 manifest.add_file(target, stats.size, stats.csum, is_encrypted)?;
1038             }
1039         }
1040     }
1041
1042     // finalize and upload catalog
1043     if let Some(catalog) = catalog {
1044         let mutex = Arc::try_unwrap(catalog)
1045             .map_err(|_| format_err!("unable to get catalog (still used)"))?;
1046         let mut catalog = mutex.into_inner().unwrap();
1047
1048         catalog.finish()?;
1049
1050         drop(catalog); // close upload stream
1051
1052         if let Some(catalog_result_rx) = catalog_result_tx {
1053             let stats = catalog_result_rx.await??;
1054             manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, is_encrypted)?;
1055         }
1056     }
1057
1058     if let Some(rsa_encrypted_key) = rsa_encrypted_key {
1059         let target = "rsa-encrypted.key";
1060         println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
1061         let stats = client
1062             .upload_blob_from_data(rsa_encrypted_key, target, false, None)
1063             .await?;
1064         manifest.add_file(format!("{}.blob", target), stats.size, stats.csum, is_encrypted)?;
1065
1066         // openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out t
1067         /*
1068         let mut buffer2 = vec![0u8; rsa.size() as usize];
1069         let pem_data = file_get_contents("master-private.pem")?;
1070         let rsa = openssl::rsa::Rsa::private_key_from_pem(&pem_data)?;
1071         let len = rsa.private_decrypt(&buffer, &mut buffer2, openssl::rsa::Padding::PKCS1)?;
1072         println!("TEST {} {:?}", len, buffer2);
1073          */
1074     }
1075
1076     // create manifest (index.json)
1077     let manifest = manifest.into_json();
1078
1079     println!("Upload index.json to '{:?}'", repo);
1080     let manifest = serde_json::to_string_pretty(&manifest)?.into();
1081     client
1082         .upload_blob_from_data(manifest, MANIFEST_BLOB_NAME, true, Some(true))
1083         .await?;
1084
1085     client.finish().await?;
1086
1087     let end_time = Local::now();
1088     let elapsed = end_time.signed_duration_since(start_time);
1089     println!("Duration: {}", elapsed);
1090
1091     println!("End Time: {}", end_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
1092
1093     Ok(Value::Null)
1094 }
1095
1096 fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1097
1098     let mut result = vec![];
1099
1100     let data: Vec<&str> = arg.splitn(2, ':').collect();
1101
1102     if data.len() != 2 {
1103         result.push(String::from("root.pxar:/"));
1104         result.push(String::from("etc.pxar:/etc"));
1105         return result;
1106     }
1107
1108     let files = tools::complete_file_name(data[1], param);
1109
1110     for file in files {
1111         result.push(format!("{}:{}", data[0], file));
1112     }
1113
1114     result
1115 }
1116
1117 async fn dump_image<W: Write>(
1118     client: Arc<BackupReader>,
1119     crypt_config: Option<Arc<CryptConfig>>,
1120     index: FixedIndexReader,
1121     mut writer: W,
1122     verbose: bool,
1123 ) -> Result<(), Error> {
1124
1125     let most_used = index.find_most_used_chunks(8);
1126
1127     let mut chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1128
1129     // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
1130     // and thus slows down reading. Instead, directly use RemoteChunkReader
1131     let mut per = 0;
1132     let mut bytes = 0;
1133     let start_time = std::time::Instant::now();
1134
1135     for pos in 0..index.index_count() {
1136         let digest = index.index_digest(pos).unwrap();
1137         let raw_data = chunk_reader.read_chunk(&digest).await?;
1138         writer.write_all(&raw_data)?;
1139         bytes += raw_data.len();
1140         if verbose {
1141             let next_per = ((pos+1)*100)/index.index_count();
1142             if per != next_per {
1143                 eprintln!("progress {}% (read {} bytes, duration {} sec)",
1144                           next_per, bytes, start_time.elapsed().as_secs());
1145                 per = next_per;
1146             }
1147         }
1148     }
1149
1150     let end_time = std::time::Instant::now();
1151     let elapsed = end_time.duration_since(start_time);
1152     eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1153               bytes,
1154               elapsed.as_secs_f64(),
1155               bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1156     );
1157
1158
1159     Ok(())
1160 }
1161
1162 fn parse_archive_type(name: &str) -> (String, ArchiveType) {
1163     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1164         (name.into(), archive_type(name).unwrap())
1165     } else if name.ends_with(".pxar") {
1166         (format!("{}.didx", name), ArchiveType::DynamicIndex)
1167     } else if name.ends_with(".img") {
1168         (format!("{}.fidx", name), ArchiveType::FixedIndex)
1169     } else {
1170         (format!("{}.blob", name), ArchiveType::Blob)
1171     }
1172 }
1173
1174 #[api(
1175    input: {
1176        properties: {
1177            repository: {
1178                schema: REPO_URL_SCHEMA,
1179                optional: true,
1180            },
1181            snapshot: {
1182                type: String,
1183                description: "Group/Snapshot path.",
1184            },
1185            "archive-name": {
1186                description: "Backup archive name.",
1187                type: String,
1188            },
1189            target: {
1190                type: String,
1191                description: r###"Target directory path. Use '-' to write to standard output.
1192
1193 We do not extraxt '.pxar' archives when writing to standard output.
1194
1195 "###
1196            },
1197            "allow-existing-dirs": {
1198                type: Boolean,
1199                description: "Do not fail if directories already exists.",
1200                optional: true,
1201            },
1202            keyfile: {
1203                schema: KEYFILE_SCHEMA,
1204                optional: true,
1205            },
1206        }
1207    }
1208 )]
1209 /// Restore backup repository.
1210 async fn restore(param: Value) -> Result<Value, Error> {
1211     let repo = extract_repository_from_value(&param)?;
1212
1213     let verbose = param["verbose"].as_bool().unwrap_or(false);
1214
1215     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1216
1217     let archive_name = tools::required_string_param(&param, "archive-name")?;
1218
1219     let client = connect(repo.host(), repo.user())?;
1220
1221     record_repository(&repo);
1222
1223     let path = tools::required_string_param(&param, "snapshot")?;
1224
1225     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1226         let group: BackupGroup = path.parse()?;
1227         api_datastore_latest_snapshot(&client, repo.store(), group).await?
1228     } else {
1229         let snapshot: BackupDir = path.parse()?;
1230         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1231     };
1232
1233     let target = tools::required_string_param(&param, "target")?;
1234     let target = if target == "-" { None } else { Some(target) };
1235
1236     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1237
1238     let crypt_config = match keyfile {
1239         None => None,
1240         Some(path) => {
1241             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
1242             Some(Arc::new(CryptConfig::new(key)?))
1243         }
1244     };
1245
1246     let client = BackupReader::start(
1247         client,
1248         crypt_config.clone(),
1249         repo.store(),
1250         &backup_type,
1251         &backup_id,
1252         backup_time,
1253         true,
1254     ).await?;
1255
1256     let manifest = client.download_manifest().await?;
1257
1258     let (archive_name, archive_type) = parse_archive_type(archive_name);
1259
1260     if archive_name == MANIFEST_BLOB_NAME {
1261         let backup_index_data = manifest.into_json().to_string();
1262         if let Some(target) = target {
1263             replace_file(target, backup_index_data.as_bytes(), CreateOptions::new())?;
1264         } else {
1265             let stdout = std::io::stdout();
1266             let mut writer = stdout.lock();
1267             writer.write_all(backup_index_data.as_bytes())
1268                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1269         }
1270
1271     } else if archive_type == ArchiveType::Blob {
1272
1273         let mut reader = client.download_blob(&manifest, &archive_name).await?;
1274
1275         if let Some(target) = target {
1276            let mut writer = std::fs::OpenOptions::new()
1277                 .write(true)
1278                 .create(true)
1279                 .create_new(true)
1280                 .open(target)
1281                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1282             std::io::copy(&mut reader, &mut writer)?;
1283         } else {
1284             let stdout = std::io::stdout();
1285             let mut writer = stdout.lock();
1286             std::io::copy(&mut reader, &mut writer)
1287                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1288         }
1289
1290     } else if archive_type == ArchiveType::DynamicIndex {
1291
1292         let index = client.download_dynamic_index(&manifest, &archive_name).await?;
1293
1294         let most_used = index.find_most_used_chunks(8);
1295
1296         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1297
1298         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1299
1300         if let Some(target) = target {
1301             proxmox_backup::pxar::extract_archive(
1302                 pxar::decoder::Decoder::from_std(reader)?,
1303                 Path::new(target),
1304                 &[],
1305                 proxmox_backup::pxar::Flags::DEFAULT,
1306                 allow_existing_dirs,
1307                 |path| {
1308                     if verbose {
1309                         println!("{:?}", path);
1310                     }
1311                 },
1312             )
1313             .map_err(|err| format_err!("error extracting archive - {}", err))?;
1314         } else {
1315             let mut writer = std::fs::OpenOptions::new()
1316                 .write(true)
1317                 .open("/dev/stdout")
1318                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
1319
1320             std::io::copy(&mut reader, &mut writer)
1321                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1322         }
1323     } else if archive_type == ArchiveType::FixedIndex {
1324
1325         let index = client.download_fixed_index(&manifest, &archive_name).await?;
1326
1327         let mut writer = if let Some(target) = target {
1328             std::fs::OpenOptions::new()
1329                 .write(true)
1330                 .create(true)
1331                 .create_new(true)
1332                 .open(target)
1333                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
1334         } else {
1335             std::fs::OpenOptions::new()
1336                 .write(true)
1337                 .open("/dev/stdout")
1338                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1339         };
1340
1341         dump_image(client.clone(), crypt_config.clone(), index, &mut writer, verbose).await?;
1342     }
1343
1344     Ok(Value::Null)
1345 }
1346
1347 #[api(
1348    input: {
1349        properties: {
1350            repository: {
1351                schema: REPO_URL_SCHEMA,
1352                optional: true,
1353            },
1354            snapshot: {
1355                type: String,
1356                description: "Group/Snapshot path.",
1357            },
1358            logfile: {
1359                type: String,
1360                description: "The path to the log file you want to upload.",
1361            },
1362            keyfile: {
1363                schema: KEYFILE_SCHEMA,
1364                optional: true,
1365            },
1366        }
1367    }
1368 )]
1369 /// Upload backup log file.
1370 async fn upload_log(param: Value) -> Result<Value, Error> {
1371
1372     let logfile = tools::required_string_param(&param, "logfile")?;
1373     let repo = extract_repository_from_value(&param)?;
1374
1375     let snapshot = tools::required_string_param(&param, "snapshot")?;
1376     let snapshot: BackupDir = snapshot.parse()?;
1377
1378     let mut client = connect(repo.host(), repo.user())?;
1379
1380     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1381
1382     let crypt_config = match keyfile {
1383         None => None,
1384         Some(path) => {
1385             let (key, _created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
1386             let crypt_config = CryptConfig::new(key)?;
1387             Some(Arc::new(crypt_config))
1388         }
1389     };
1390
1391     let data = file_get_contents(logfile)?;
1392
1393     let blob = DataBlob::encode(&data, crypt_config.as_ref().map(Arc::as_ref), true)?;
1394
1395     let raw_data = blob.into_inner();
1396
1397     let path = format!("api2/json/admin/datastore/{}/upload-backup-log", repo.store());
1398
1399     let args = json!({
1400         "backup-type": snapshot.group().backup_type(),
1401         "backup-id":  snapshot.group().backup_id(),
1402         "backup-time": snapshot.backup_time().timestamp(),
1403     });
1404
1405     let body = hyper::Body::from(raw_data);
1406
1407     client.upload("application/octet-stream", body, &path, Some(args)).await
1408 }
1409
1410 const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
1411     &ApiHandler::Async(&prune),
1412     &ObjectSchema::new(
1413         "Prune a backup repository.",
1414         &proxmox_backup::add_common_prune_prameters!([
1415             ("dry-run", true, &BooleanSchema::new(
1416                 "Just show what prune would do, but do not delete anything.")
1417              .schema()),
1418             ("group", false, &StringSchema::new("Backup group.").schema()),
1419         ], [
1420             ("output-format", true, &OUTPUT_FORMAT),
1421             (
1422                 "quiet",
1423                 true,
1424                 &BooleanSchema::new("Minimal output - only show removals.")
1425                     .schema()
1426             ),
1427             ("repository", true, &REPO_URL_SCHEMA),
1428         ])
1429     )
1430 );
1431
1432 fn prune<'a>(
1433     param: Value,
1434     _info: &ApiMethod,
1435     _rpcenv: &'a mut dyn RpcEnvironment,
1436 ) -> proxmox::api::ApiFuture<'a> {
1437     async move {
1438         prune_async(param).await
1439     }.boxed()
1440 }
1441
1442 async fn prune_async(mut param: Value) -> Result<Value, Error> {
1443     let repo = extract_repository_from_value(&param)?;
1444
1445     let mut client = connect(repo.host(), repo.user())?;
1446
1447     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
1448
1449     let group = tools::required_string_param(&param, "group")?;
1450     let group: BackupGroup = group.parse()?;
1451
1452     let output_format = get_output_format(&param);
1453
1454     let quiet = param["quiet"].as_bool().unwrap_or(false);
1455
1456     param.as_object_mut().unwrap().remove("repository");
1457     param.as_object_mut().unwrap().remove("group");
1458     param.as_object_mut().unwrap().remove("output-format");
1459     param.as_object_mut().unwrap().remove("quiet");
1460
1461     param["backup-type"] = group.backup_type().into();
1462     param["backup-id"] = group.backup_id().into();
1463
1464     let mut result = client.post(&path, Some(param)).await?;
1465
1466     record_repository(&repo);
1467
1468     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1469         let item: PruneListItem = serde_json::from_value(record.to_owned())?;
1470         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
1471         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1472     };
1473
1474     let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1475         Ok(match v.as_bool() {
1476             Some(true) => "keep",
1477             Some(false) => "remove",
1478             None => "unknown",
1479         }.to_string())
1480     };
1481
1482     let options = default_table_format_options()
1483         .sortby("backup-type", false)
1484         .sortby("backup-id", false)
1485         .sortby("backup-time", false)
1486         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
1487         .column(ColumnConfig::new("backup-time").renderer(tools::format::render_epoch).header("date"))
1488         .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
1489         ;
1490
1491     let info = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_PRUNE;
1492
1493     let mut data = result["data"].take();
1494
1495     if quiet {
1496         let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1497             item["keep"].as_bool() == Some(false)
1498         }).map(|v| v.clone()).collect();
1499         data = list.into();
1500     }
1501
1502     format_and_print_result_full(&mut data, info, &output_format, &options);
1503
1504     Ok(Value::Null)
1505 }
1506
1507 #[api(
1508    input: {
1509        properties: {
1510            repository: {
1511                schema: REPO_URL_SCHEMA,
1512                optional: true,
1513            },
1514            "output-format": {
1515                schema: OUTPUT_FORMAT,
1516                optional: true,
1517            },
1518        }
1519    }
1520 )]
1521 /// Get repository status.
1522 async fn status(param: Value) -> Result<Value, Error> {
1523
1524     let repo = extract_repository_from_value(&param)?;
1525
1526     let output_format = get_output_format(&param);
1527
1528     let client = connect(repo.host(), repo.user())?;
1529
1530     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1531
1532     let mut result = client.get(&path, None).await?;
1533     let mut data = result["data"].take();
1534
1535     record_repository(&repo);
1536
1537     let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1538         let v = v.as_u64().unwrap();
1539         let total = record["total"].as_u64().unwrap();
1540         let roundup = total/200;
1541         let per = ((v+roundup)*100)/total;
1542         let info = format!(" ({} %)", per);
1543         Ok(format!("{} {:>8}", v, info))
1544     };
1545
1546     let options = default_table_format_options()
1547         .noheader(true)
1548         .column(ColumnConfig::new("total").renderer(render_total_percentage))
1549         .column(ColumnConfig::new("used").renderer(render_total_percentage))
1550         .column(ColumnConfig::new("avail").renderer(render_total_percentage));
1551
1552     let schema = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_STATUS;
1553
1554     format_and_print_result_full(&mut data, schema, &output_format, &options);
1555
1556     Ok(Value::Null)
1557 }
1558
1559 // like get, but simply ignore errors and return Null instead
1560 async fn try_get(repo: &BackupRepository, url: &str) -> Value {
1561
1562     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
1563     let password = std::env::var(ENV_VAR_PBS_PASSWORD).ok();
1564
1565     let options = HttpClientOptions::new()
1566         .prefix(Some("proxmox-backup".to_string()))
1567         .password(password)
1568         .interactive(false)
1569         .fingerprint(fingerprint)
1570         .fingerprint_cache(true)
1571         .ticket_cache(true);
1572
1573     let client = match HttpClient::new(repo.host(), repo.user(), options) {
1574         Ok(v) => v,
1575         _ => return Value::Null,
1576     };
1577
1578     let mut resp = match client.get(url, None).await {
1579         Ok(v) => v,
1580         _ => return Value::Null,
1581     };
1582
1583     if let Some(map) = resp.as_object_mut() {
1584         if let Some(data) = map.remove("data") {
1585             return data;
1586         }
1587     }
1588     Value::Null
1589 }
1590
1591 fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1592     proxmox_backup::tools::runtime::main(async { complete_backup_group_do(param).await })
1593 }
1594
1595 async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
1596
1597     let mut result = vec![];
1598
1599     let repo = match extract_repository_from_map(param) {
1600         Some(v) => v,
1601         _ => return result,
1602     };
1603
1604     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
1605
1606     let data = try_get(&repo, &path).await;
1607
1608     if let Some(list) = data.as_array() {
1609         for item in list {
1610             if let (Some(backup_id), Some(backup_type)) =
1611                 (item["backup-id"].as_str(), item["backup-type"].as_str())
1612             {
1613                 result.push(format!("{}/{}", backup_type, backup_id));
1614             }
1615         }
1616     }
1617
1618     result
1619 }
1620
1621 fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1622     proxmox_backup::tools::runtime::main(async { complete_group_or_snapshot_do(arg, param).await })
1623 }
1624
1625 async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1626
1627     if arg.matches('/').count() < 2 {
1628         let groups = complete_backup_group_do(param).await;
1629         let mut result = vec![];
1630         for group in groups {
1631             result.push(group.to_string());
1632             result.push(format!("{}/", group));
1633         }
1634         return result;
1635     }
1636
1637     complete_backup_snapshot_do(param).await
1638 }
1639
1640 fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1641     proxmox_backup::tools::runtime::main(async { complete_backup_snapshot_do(param).await })
1642 }
1643
1644 async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
1645
1646     let mut result = vec![];
1647
1648     let repo = match extract_repository_from_map(param) {
1649         Some(v) => v,
1650         _ => return result,
1651     };
1652
1653     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1654
1655     let data = try_get(&repo, &path).await;
1656
1657     if let Some(list) = data.as_array() {
1658         for item in list {
1659             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
1660                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
1661             {
1662                 let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
1663                 result.push(snapshot.relative_path().to_str().unwrap().to_owned());
1664             }
1665         }
1666     }
1667
1668     result
1669 }
1670
1671 fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1672     proxmox_backup::tools::runtime::main(async { complete_server_file_name_do(param).await })
1673 }
1674
1675 async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
1676
1677     let mut result = vec![];
1678
1679     let repo = match extract_repository_from_map(param) {
1680         Some(v) => v,
1681         _ => return result,
1682     };
1683
1684     let snapshot: BackupDir = match param.get("snapshot") {
1685         Some(path) => {
1686             match path.parse() {
1687                 Ok(v) => v,
1688                 _ => return result,
1689             }
1690         }
1691         _ => return result,
1692     };
1693
1694     let query = tools::json_object_to_query(json!({
1695         "backup-type": snapshot.group().backup_type(),
1696         "backup-id": snapshot.group().backup_id(),
1697         "backup-time": snapshot.backup_time().timestamp(),
1698     })).unwrap();
1699
1700     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
1701
1702     let data = try_get(&repo, &path).await;
1703
1704     if let Some(list) = data.as_array() {
1705         for item in list {
1706             if let Some(filename) = item["filename"].as_str() {
1707                 result.push(filename.to_owned());
1708             }
1709         }
1710     }
1711
1712     result
1713 }
1714
1715 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1716     complete_server_file_name(arg, param)
1717         .iter()
1718         .map(|v| tools::format::strip_server_file_expenstion(&v))
1719         .collect()
1720 }
1721
1722 fn complete_pxar_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1723     complete_server_file_name(arg, param)
1724         .iter()
1725         .filter_map(|v| {
1726             let name = tools::format::strip_server_file_expenstion(&v);
1727             if name.ends_with(".pxar") {
1728                 Some(name)
1729             } else {
1730                 None
1731             }
1732         })
1733         .collect()
1734 }
1735
1736 fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
1737
1738     let mut result = vec![];
1739
1740     let mut size = 64;
1741     loop {
1742         result.push(size.to_string());
1743         size *= 2;
1744         if size > 4096 { break; }
1745     }
1746
1747     result
1748 }
1749
1750 fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
1751
1752     // fixme: implement other input methods
1753
1754     use std::env::VarError::*;
1755     match std::env::var("PBS_ENCRYPTION_PASSWORD") {
1756         Ok(p) => return Ok(p.as_bytes().to_vec()),
1757         Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
1758         Err(NotPresent) => {
1759             // Try another method
1760         }
1761     }
1762
1763     // If we're on a TTY, query the user for a password
1764     if tty::stdin_isatty() {
1765         return Ok(tty::read_password("Encryption Key Password: ")?);
1766     }
1767
1768     bail!("no password input mechanism available");
1769 }
1770
1771 fn key_create(
1772     param: Value,
1773     _info: &ApiMethod,
1774     _rpcenv: &mut dyn RpcEnvironment,
1775 ) -> Result<Value, Error> {
1776
1777     let path = tools::required_string_param(&param, "path")?;
1778     let path = PathBuf::from(path);
1779
1780     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1781
1782     let key = proxmox::sys::linux::random_data(32)?;
1783
1784     if kdf == "scrypt" {
1785         // always read passphrase from tty
1786         if !tty::stdin_isatty() {
1787             bail!("unable to read passphrase - no tty");
1788         }
1789
1790         let password = tty::read_and_verify_password("Encryption Key Password: ")?;
1791
1792         let key_config = encrypt_key_with_passphrase(&key, &password)?;
1793
1794         store_key_config(&path, false, key_config)?;
1795
1796         Ok(Value::Null)
1797     } else if kdf == "none" {
1798         let created =  Local.timestamp(Local::now().timestamp(), 0);
1799
1800         store_key_config(&path, false, KeyConfig {
1801             kdf: None,
1802             created,
1803             modified: created,
1804             data: key,
1805         })?;
1806
1807         Ok(Value::Null)
1808     } else {
1809         unreachable!();
1810     }
1811 }
1812
1813 fn master_pubkey_path() -> Result<PathBuf, Error> {
1814     let base = BaseDirectories::with_prefix("proxmox-backup")?;
1815
1816     // usually $HOME/.config/proxmox-backup/master-public.pem
1817     let path = base.place_config_file("master-public.pem")?;
1818
1819     Ok(path)
1820 }
1821
1822 fn key_import_master_pubkey(
1823     param: Value,
1824     _info: &ApiMethod,
1825     _rpcenv: &mut dyn RpcEnvironment,
1826 ) -> Result<Value, Error> {
1827
1828     let path = tools::required_string_param(&param, "path")?;
1829     let path = PathBuf::from(path);
1830
1831     let pem_data = file_get_contents(&path)?;
1832
1833     if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
1834         bail!("Unable to decode PEM data - {}", err);
1835     }
1836
1837     let target_path = master_pubkey_path()?;
1838
1839     replace_file(&target_path, &pem_data, CreateOptions::new())?;
1840
1841     println!("Imported public master key to {:?}", target_path);
1842
1843     Ok(Value::Null)
1844 }
1845
1846 fn key_create_master_key(
1847     _param: Value,
1848     _info: &ApiMethod,
1849     _rpcenv: &mut dyn RpcEnvironment,
1850 ) -> Result<Value, Error> {
1851
1852     // we need a TTY to query the new password
1853     if !tty::stdin_isatty() {
1854         bail!("unable to create master key - no tty");
1855     }
1856
1857     let rsa = openssl::rsa::Rsa::generate(4096)?;
1858     let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
1859
1860
1861     let password = String::from_utf8(tty::read_and_verify_password("Master Key Password: ")?)?;
1862
1863     let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
1864     let filename_pub = "master-public.pem";
1865     println!("Writing public master key to {}", filename_pub);
1866     replace_file(filename_pub, pub_key.as_slice(), CreateOptions::new())?;
1867
1868     let cipher = openssl::symm::Cipher::aes_256_cbc();
1869     let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, password.as_bytes())?;
1870
1871     let filename_priv = "master-private.pem";
1872     println!("Writing private master key to {}", filename_priv);
1873     replace_file(filename_priv, priv_key.as_slice(), CreateOptions::new())?;
1874
1875     Ok(Value::Null)
1876 }
1877
1878 fn key_change_passphrase(
1879     param: Value,
1880     _info: &ApiMethod,
1881     _rpcenv: &mut dyn RpcEnvironment,
1882 ) -> Result<Value, Error> {
1883
1884     let path = tools::required_string_param(&param, "path")?;
1885     let path = PathBuf::from(path);
1886
1887     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1888
1889     // we need a TTY to query the new password
1890     if !tty::stdin_isatty() {
1891         bail!("unable to change passphrase - no tty");
1892     }
1893
1894     let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
1895
1896     if kdf == "scrypt" {
1897
1898         let password = tty::read_and_verify_password("New Password: ")?;
1899
1900         let mut new_key_config = encrypt_key_with_passphrase(&key, &password)?;
1901         new_key_config.created = created; // keep original value
1902
1903         store_key_config(&path, true, new_key_config)?;
1904
1905         Ok(Value::Null)
1906     } else if kdf == "none" {
1907         let modified =  Local.timestamp(Local::now().timestamp(), 0);
1908
1909         store_key_config(&path, true, KeyConfig {
1910             kdf: None,
1911             created, // keep original value
1912             modified,
1913             data: key.to_vec(),
1914         })?;
1915
1916         Ok(Value::Null)
1917     } else {
1918         unreachable!();
1919     }
1920 }
1921
1922 fn key_mgmt_cli() -> CliCommandMap {
1923
1924     const KDF_SCHEMA: Schema =
1925         StringSchema::new("Key derivation function. Choose 'none' to store the key unecrypted.")
1926         .format(&ApiStringFormat::Enum(&[
1927             EnumEntry::new("scrypt", "SCrypt"),
1928             EnumEntry::new("none", "Do not encrypt the key")]))
1929         .default("scrypt")
1930         .schema();
1931
1932     #[sortable]
1933     const API_METHOD_KEY_CREATE: ApiMethod = ApiMethod::new(
1934         &ApiHandler::Sync(&key_create),
1935         &ObjectSchema::new(
1936             "Create a new encryption key.",
1937             &sorted!([
1938                 ("path", false, &StringSchema::new("File system path.").schema()),
1939                 ("kdf", true, &KDF_SCHEMA),
1940             ]),
1941         )
1942     );
1943
1944     let key_create_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE)
1945         .arg_param(&["path"])
1946         .completion_cb("path", tools::complete_file_name);
1947
1948     #[sortable]
1949     const API_METHOD_KEY_CHANGE_PASSPHRASE: ApiMethod = ApiMethod::new(
1950         &ApiHandler::Sync(&key_change_passphrase),
1951         &ObjectSchema::new(
1952             "Change the passphrase required to decrypt the key.",
1953             &sorted!([
1954                 ("path", false, &StringSchema::new("File system path.").schema()),
1955                 ("kdf", true, &KDF_SCHEMA),
1956             ]),
1957         )
1958     );
1959
1960     let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_KEY_CHANGE_PASSPHRASE)
1961         .arg_param(&["path"])
1962         .completion_cb("path", tools::complete_file_name);
1963
1964     const API_METHOD_KEY_CREATE_MASTER_KEY: ApiMethod = ApiMethod::new(
1965         &ApiHandler::Sync(&key_create_master_key),
1966         &ObjectSchema::new("Create a new 4096 bit RSA master pub/priv key pair.", &[])
1967     );
1968
1969     let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE_MASTER_KEY);
1970
1971     #[sortable]
1972     const API_METHOD_KEY_IMPORT_MASTER_PUBKEY: ApiMethod = ApiMethod::new(
1973         &ApiHandler::Sync(&key_import_master_pubkey),
1974         &ObjectSchema::new(
1975             "Import a new RSA public key and use it as master key. The key is expected to be in '.pem' format.",
1976             &sorted!([ ("path", false, &StringSchema::new("File system path.").schema()) ]),
1977         )
1978     );
1979
1980     let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_KEY_IMPORT_MASTER_PUBKEY)
1981         .arg_param(&["path"])
1982         .completion_cb("path", tools::complete_file_name);
1983
1984     CliCommandMap::new()
1985         .insert("create", key_create_cmd_def)
1986         .insert("create-master-key", key_create_master_key_cmd_def)
1987         .insert("import-master-pubkey", key_import_master_pubkey_cmd_def)
1988         .insert("change-passphrase", key_change_passphrase_cmd_def)
1989 }
1990
1991 fn mount(
1992     param: Value,
1993     _info: &ApiMethod,
1994     _rpcenv: &mut dyn RpcEnvironment,
1995 ) -> Result<Value, Error> {
1996     let verbose = param["verbose"].as_bool().unwrap_or(false);
1997     if verbose {
1998         // This will stay in foreground with debug output enabled as None is
1999         // passed for the RawFd.
2000         return proxmox_backup::tools::runtime::main(mount_do(param, None));
2001     }
2002
2003     // Process should be deamonized.
2004     // Make sure to fork before the async runtime is instantiated to avoid troubles.
2005     let pipe = pipe()?;
2006     match fork() {
2007         Ok(ForkResult::Parent { .. }) => {
2008             nix::unistd::close(pipe.1).unwrap();
2009             // Blocks the parent process until we are ready to go in the child
2010             let _res = nix::unistd::read(pipe.0, &mut [0]).unwrap();
2011             Ok(Value::Null)
2012         }
2013         Ok(ForkResult::Child) => {
2014             nix::unistd::close(pipe.0).unwrap();
2015             nix::unistd::setsid().unwrap();
2016             proxmox_backup::tools::runtime::main(mount_do(param, Some(pipe.1)))
2017         }
2018         Err(_) => bail!("failed to daemonize process"),
2019     }
2020 }
2021
2022 use proxmox_backup::client::RemoteChunkReader;
2023 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
2024 /// async use!
2025 ///
2026 /// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
2027 /// so that we can properly access it from multiple threads simultaneously while not issuing
2028 /// duplicate simultaneous reads over http.
2029 struct BufferedDynamicReadAt {
2030     inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
2031 }
2032
2033 impl BufferedDynamicReadAt {
2034     fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
2035         Self {
2036             inner: Mutex::new(inner),
2037         }
2038     }
2039 }
2040
2041 impl ReadAt for BufferedDynamicReadAt {
2042     fn start_read_at<'a>(
2043         self: Pin<&'a Self>,
2044         _cx: &mut Context,
2045         buf: &'a mut [u8],
2046         offset: u64,
2047     ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
2048         use std::io::Read;
2049         MaybeReady::Ready(tokio::task::block_in_place(move || {
2050             let mut reader = self.inner.lock().unwrap();
2051             reader.seek(SeekFrom::Start(offset))?;
2052             Ok(reader.read(buf)?)
2053         }))
2054     }
2055
2056     fn poll_complete<'a>(
2057         self: Pin<&'a Self>,
2058         _op: ReadAtOperation<'a>,
2059     ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
2060         panic!("LocalDynamicReadAt::start_read_at returned Pending");
2061     }
2062 }
2063
2064 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
2065     let repo = extract_repository_from_value(&param)?;
2066     let archive_name = tools::required_string_param(&param, "archive-name")?;
2067     let target = tools::required_string_param(&param, "target")?;
2068     let client = connect(repo.host(), repo.user())?;
2069
2070     record_repository(&repo);
2071
2072     let path = tools::required_string_param(&param, "snapshot")?;
2073     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
2074         let group: BackupGroup = path.parse()?;
2075         api_datastore_latest_snapshot(&client, repo.store(), group).await?
2076     } else {
2077         let snapshot: BackupDir = path.parse()?;
2078         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
2079     };
2080
2081     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
2082     let crypt_config = match keyfile {
2083         None => None,
2084         Some(path) => {
2085             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
2086             Some(Arc::new(CryptConfig::new(key)?))
2087         }
2088     };
2089
2090     let server_archive_name = if archive_name.ends_with(".pxar") {
2091         format!("{}.didx", archive_name)
2092     } else {
2093         bail!("Can only mount pxar archives.");
2094     };
2095
2096     let client = BackupReader::start(
2097         client,
2098         crypt_config.clone(),
2099         repo.store(),
2100         &backup_type,
2101         &backup_id,
2102         backup_time,
2103         true,
2104     ).await?;
2105
2106     let manifest = client.download_manifest().await?;
2107
2108     if server_archive_name.ends_with(".didx") {
2109         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
2110         let most_used = index.find_most_used_chunks(8);
2111         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
2112         let reader = BufferedDynamicReader::new(index, chunk_reader);
2113         let archive_size = reader.archive_size();
2114         let reader: proxmox_backup::pxar::fuse::Reader =
2115             Arc::new(BufferedDynamicReadAt::new(reader));
2116         let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
2117         let options = OsStr::new("ro,default_permissions");
2118
2119         let session = proxmox_backup::pxar::fuse::Session::mount(
2120             decoder,
2121             &options,
2122             false,
2123             Path::new(target),
2124         )
2125         .map_err(|err| format_err!("pxar mount failed: {}", err))?;
2126
2127         if let Some(pipe) = pipe {
2128             nix::unistd::chdir(Path::new("/")).unwrap();
2129             // Finish creation of daemon by redirecting filedescriptors.
2130             let nullfd = nix::fcntl::open(
2131                 "/dev/null",
2132                 nix::fcntl::OFlag::O_RDWR,
2133                 nix::sys::stat::Mode::empty(),
2134             ).unwrap();
2135             nix::unistd::dup2(nullfd, 0).unwrap();
2136             nix::unistd::dup2(nullfd, 1).unwrap();
2137             nix::unistd::dup2(nullfd, 2).unwrap();
2138             if nullfd > 2 {
2139                 nix::unistd::close(nullfd).unwrap();
2140             }
2141             // Signal the parent process that we are done with the setup and it can
2142             // terminate.
2143             nix::unistd::write(pipe, &[0u8])?;
2144             nix::unistd::close(pipe).unwrap();
2145         }
2146
2147         let mut interrupt = signal(SignalKind::interrupt())?;
2148         select! {
2149             res = session.fuse() => res?,
2150             _ = interrupt.recv().fuse() => {
2151                 // exit on interrupted
2152             }
2153         }
2154     } else {
2155         bail!("unknown archive file extension (expected .pxar)");
2156     }
2157
2158     Ok(Value::Null)
2159 }
2160
2161 #[api(
2162     input: {
2163         properties: {
2164             "snapshot": {
2165                 type: String,
2166                 description: "Group/Snapshot path.",
2167             },
2168             "archive-name": {
2169                 type: String,
2170                 description: "Backup archive name.",
2171             },
2172             "repository": {
2173                 optional: true,
2174                 schema: REPO_URL_SCHEMA,
2175             },
2176             "keyfile": {
2177                 optional: true,
2178                 type: String,
2179                 description: "Path to encryption key.",
2180             },
2181         },
2182     },
2183 )]
2184 /// Shell to interactively inspect and restore snapshots.
2185 async fn catalog_shell(param: Value) -> Result<(), Error> {
2186     let repo = extract_repository_from_value(&param)?;
2187     let client = connect(repo.host(), repo.user())?;
2188     let path = tools::required_string_param(&param, "snapshot")?;
2189     let archive_name = tools::required_string_param(&param, "archive-name")?;
2190
2191     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
2192         let group: BackupGroup = path.parse()?;
2193         api_datastore_latest_snapshot(&client, repo.store(), group).await?
2194     } else {
2195         let snapshot: BackupDir = path.parse()?;
2196         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
2197     };
2198
2199     let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
2200     let crypt_config = match keyfile {
2201         None => None,
2202         Some(path) => {
2203             let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
2204             Some(Arc::new(CryptConfig::new(key)?))
2205         }
2206     };
2207
2208     let server_archive_name = if archive_name.ends_with(".pxar") {
2209         format!("{}.didx", archive_name)
2210     } else {
2211         bail!("Can only mount pxar archives.");
2212     };
2213
2214     let client = BackupReader::start(
2215         client,
2216         crypt_config.clone(),
2217         repo.store(),
2218         &backup_type,
2219         &backup_id,
2220         backup_time,
2221         true,
2222     ).await?;
2223
2224     let mut tmpfile = std::fs::OpenOptions::new()
2225         .write(true)
2226         .read(true)
2227         .custom_flags(libc::O_TMPFILE)
2228         .open("/tmp")?;
2229
2230     let manifest = client.download_manifest().await?;
2231
2232     let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
2233     let most_used = index.find_most_used_chunks(8);
2234     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
2235     let reader = BufferedDynamicReader::new(index, chunk_reader);
2236     let archive_size = reader.archive_size();
2237     let reader: proxmox_backup::pxar::fuse::Reader =
2238         Arc::new(BufferedDynamicReadAt::new(reader));
2239     let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
2240
2241     client.download(CATALOG_NAME, &mut tmpfile).await?;
2242     let index = DynamicIndexReader::new(tmpfile)
2243         .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
2244
2245     // Note: do not use values stored in index (not trusted) - instead, computed them again
2246     let (csum, size) = index.compute_csum();
2247     manifest.verify_file(CATALOG_NAME, &csum, size)?;
2248
2249     let most_used = index.find_most_used_chunks(8);
2250     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
2251     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
2252     let mut catalogfile = std::fs::OpenOptions::new()
2253         .write(true)
2254         .read(true)
2255         .custom_flags(libc::O_TMPFILE)
2256         .open("/tmp")?;
2257
2258     std::io::copy(&mut reader, &mut catalogfile)
2259         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
2260
2261     catalogfile.seek(SeekFrom::Start(0))?;
2262     let catalog_reader = CatalogReader::new(catalogfile);
2263     let state = Shell::new(
2264         catalog_reader,
2265         &server_archive_name,
2266         decoder,
2267     ).await?;
2268
2269     println!("Starting interactive shell");
2270     state.shell().await?;
2271
2272     record_repository(&repo);
2273
2274     Ok(())
2275 }
2276
2277 fn catalog_mgmt_cli() -> CliCommandMap {
2278     let catalog_shell_cmd_def = CliCommand::new(&API_METHOD_CATALOG_SHELL)
2279         .arg_param(&["snapshot", "archive-name"])
2280         .completion_cb("repository", complete_repository)
2281         .completion_cb("archive-name", complete_pxar_archive_name)
2282         .completion_cb("snapshot", complete_group_or_snapshot);
2283
2284     let catalog_dump_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
2285         .arg_param(&["snapshot"])
2286         .completion_cb("repository", complete_repository)
2287         .completion_cb("snapshot", complete_backup_snapshot);
2288
2289     CliCommandMap::new()
2290         .insert("dump", catalog_dump_cmd_def)
2291         .insert("shell", catalog_shell_cmd_def)
2292 }
2293
2294 #[api(
2295     input: {
2296         properties: {
2297             repository: {
2298                 schema: REPO_URL_SCHEMA,
2299                 optional: true,
2300             },
2301             limit: {
2302                 description: "The maximal number of tasks to list.",
2303                 type: Integer,
2304                 optional: true,
2305                 minimum: 1,
2306                 maximum: 1000,
2307                 default: 50,
2308             },
2309             "output-format": {
2310                 schema: OUTPUT_FORMAT,
2311                 optional: true,
2312             },
2313             all: {
2314                 type: Boolean,
2315                 description: "Also list stopped tasks.",
2316                 optional: true,
2317             },
2318         }
2319     }
2320 )]
2321 /// List running server tasks for this repo user
2322 async fn task_list(param: Value) -> Result<Value, Error> {
2323
2324     let output_format = get_output_format(&param);
2325
2326     let repo = extract_repository_from_value(&param)?;
2327     let client = connect(repo.host(), repo.user())?;
2328
2329     let limit = param["limit"].as_u64().unwrap_or(50) as usize;
2330     let running = !param["all"].as_bool().unwrap_or(false);
2331
2332     let args = json!({
2333         "running": running,
2334         "start": 0,
2335         "limit": limit,
2336         "userfilter": repo.user(),
2337         "store": repo.store(),
2338     });
2339
2340     let mut result = client.get("api2/json/nodes/localhost/tasks", Some(args)).await?;
2341     let mut data = result["data"].take();
2342
2343     let schema = &proxmox_backup::api2::node::tasks::API_RETURN_SCHEMA_LIST_TASKS;
2344
2345     let options = default_table_format_options()
2346         .column(ColumnConfig::new("starttime").right_align(false).renderer(tools::format::render_epoch))
2347         .column(ColumnConfig::new("endtime").right_align(false).renderer(tools::format::render_epoch))
2348         .column(ColumnConfig::new("upid"))
2349         .column(ColumnConfig::new("status").renderer(tools::format::render_task_status));
2350
2351     format_and_print_result_full(&mut data, schema, &output_format, &options);
2352
2353     Ok(Value::Null)
2354 }
2355
2356 #[api(
2357     input: {
2358         properties: {
2359             repository: {
2360                 schema: REPO_URL_SCHEMA,
2361                 optional: true,
2362             },
2363             upid: {
2364                 schema: UPID_SCHEMA,
2365             },
2366         }
2367     }
2368 )]
2369 /// Display the task log.
2370 async fn task_log(param: Value) -> Result<Value, Error> {
2371
2372     let repo = extract_repository_from_value(&param)?;
2373     let upid =  tools::required_string_param(&param, "upid")?;
2374
2375     let client = connect(repo.host(), repo.user())?;
2376
2377     display_task_log(client, upid, true).await?;
2378
2379     Ok(Value::Null)
2380 }
2381
2382 #[api(
2383     input: {
2384         properties: {
2385             repository: {
2386                 schema: REPO_URL_SCHEMA,
2387                 optional: true,
2388             },
2389             upid: {
2390                 schema: UPID_SCHEMA,
2391             },
2392         }
2393     }
2394 )]
2395 /// Try to stop a specific task.
2396 async fn task_stop(param: Value) -> Result<Value, Error> {
2397
2398     let repo = extract_repository_from_value(&param)?;
2399     let upid_str =  tools::required_string_param(&param, "upid")?;
2400
2401     let mut client = connect(repo.host(), repo.user())?;
2402
2403     let path = format!("api2/json/nodes/localhost/tasks/{}", upid_str);
2404     let _ = client.delete(&path, None).await?;
2405
2406     Ok(Value::Null)
2407 }
2408
2409 fn task_mgmt_cli() -> CliCommandMap {
2410
2411     let task_list_cmd_def = CliCommand::new(&API_METHOD_TASK_LIST)
2412         .completion_cb("repository", complete_repository);
2413
2414     let task_log_cmd_def = CliCommand::new(&API_METHOD_TASK_LOG)
2415         .arg_param(&["upid"]);
2416
2417     let task_stop_cmd_def = CliCommand::new(&API_METHOD_TASK_STOP)
2418         .arg_param(&["upid"]);
2419
2420     CliCommandMap::new()
2421         .insert("log", task_log_cmd_def)
2422         .insert("list", task_list_cmd_def)
2423         .insert("stop", task_stop_cmd_def)
2424 }
2425
2426 fn main() {
2427
2428     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
2429         .arg_param(&["backupspec"])
2430         .completion_cb("repository", complete_repository)
2431         .completion_cb("backupspec", complete_backup_source)
2432         .completion_cb("keyfile", tools::complete_file_name)
2433         .completion_cb("chunk-size", complete_chunk_size);
2434
2435     let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
2436         .completion_cb("repository", complete_repository)
2437         .completion_cb("keyfile", tools::complete_file_name);
2438
2439     let upload_log_cmd_def = CliCommand::new(&API_METHOD_UPLOAD_LOG)
2440         .arg_param(&["snapshot", "logfile"])
2441         .completion_cb("snapshot", complete_backup_snapshot)
2442         .completion_cb("logfile", tools::complete_file_name)
2443         .completion_cb("keyfile", tools::complete_file_name)
2444         .completion_cb("repository", complete_repository);
2445
2446     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
2447         .completion_cb("repository", complete_repository);
2448
2449     let snapshots_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOTS)
2450         .arg_param(&["group"])
2451         .completion_cb("group", complete_backup_group)
2452         .completion_cb("repository", complete_repository);
2453
2454     let forget_cmd_def = CliCommand::new(&API_METHOD_FORGET_SNAPSHOTS)
2455         .arg_param(&["snapshot"])
2456         .completion_cb("repository", complete_repository)
2457         .completion_cb("snapshot", complete_backup_snapshot);
2458
2459     let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
2460         .completion_cb("repository", complete_repository);
2461
2462     let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
2463         .arg_param(&["snapshot", "archive-name", "target"])
2464         .completion_cb("repository", complete_repository)
2465         .completion_cb("snapshot", complete_group_or_snapshot)
2466         .completion_cb("archive-name", complete_archive_name)
2467         .completion_cb("target", tools::complete_file_name);
2468
2469     let files_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOT_FILES)
2470         .arg_param(&["snapshot"])
2471         .completion_cb("repository", complete_repository)
2472         .completion_cb("snapshot", complete_backup_snapshot);
2473
2474     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
2475         .arg_param(&["group"])
2476         .completion_cb("group", complete_backup_group)
2477         .completion_cb("repository", complete_repository);
2478
2479     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
2480         .completion_cb("repository", complete_repository);
2481
2482     let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
2483         .completion_cb("repository", complete_repository);
2484
2485     let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
2486         .completion_cb("repository", complete_repository);
2487
2488     #[sortable]
2489     const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
2490         &ApiHandler::Sync(&mount),
2491         &ObjectSchema::new(
2492             "Mount pxar archive.",
2493             &sorted!([
2494                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2495                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2496                 ("target", false, &StringSchema::new("Target directory path.").schema()),
2497                 ("repository", true, &REPO_URL_SCHEMA),
2498                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2499                 ("verbose", true, &BooleanSchema::new("Verbose output.").default(false).schema()),
2500             ]),
2501         )
2502     );
2503
2504     let mount_cmd_def = CliCommand::new(&API_METHOD_MOUNT)
2505         .arg_param(&["snapshot", "archive-name", "target"])
2506         .completion_cb("repository", complete_repository)
2507         .completion_cb("snapshot", complete_group_or_snapshot)
2508         .completion_cb("archive-name", complete_pxar_archive_name)
2509         .completion_cb("target", tools::complete_file_name);
2510
2511
2512     let cmd_def = CliCommandMap::new()
2513         .insert("backup", backup_cmd_def)
2514         .insert("upload-log", upload_log_cmd_def)
2515         .insert("forget", forget_cmd_def)
2516         .insert("garbage-collect", garbage_collect_cmd_def)
2517         .insert("list", list_cmd_def)
2518         .insert("login", login_cmd_def)
2519         .insert("logout", logout_cmd_def)
2520         .insert("prune", prune_cmd_def)
2521         .insert("restore", restore_cmd_def)
2522         .insert("snapshots", snapshots_cmd_def)
2523         .insert("files", files_cmd_def)
2524         .insert("status", status_cmd_def)
2525         .insert("key", key_mgmt_cli())
2526         .insert("mount", mount_cmd_def)
2527         .insert("catalog", catalog_mgmt_cli())
2528         .insert("task", task_mgmt_cli())
2529         .insert("benchmark", benchmark_cmd_def);
2530
2531     let rpcenv = CliEnvironment::new();
2532     run_cli_command(cmd_def, rpcenv, Some(|future| {
2533         proxmox_backup::tools::runtime::main(future)
2534     }));
2535 }