src/bin/proxmox-backup-client.rs

   1 use std::collections::{HashSet, HashMap};
   2 use std::convert::TryFrom;
   3 use std::io::{self, Read, Write, Seek, SeekFrom};
   4 use std::os::unix::io::{FromRawFd, RawFd};
   5 use std::path::{Path, PathBuf};
   6 use std::pin::Pin;
   7 use std::sync::{Arc, Mutex};
   8 use std::task::Context;
   9
  10 use anyhow::{bail, format_err, Error};
  11 use futures::future::FutureExt;
  12 use futures::stream::{StreamExt, TryStreamExt};
  13 use serde_json::{json, Value};
  14 use tokio::sync::mpsc;
  15 use tokio_stream::wrappers::ReceiverStream;
  16 use xdg::BaseDirectories;
  17
  18 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
  19 use proxmox::{
  20     tools::{
  21         time::{strftime_local, epoch_i64},
  22         fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size},
  23     },
  24     api::{
  25         api,
  26         ApiHandler,
  27         ApiMethod,
  28         RpcEnvironment,
  29         schema::*,
  30         cli::*,
  31     },
  32 };
  33 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
  34
  35 use proxmox_backup::tools;
  36 use proxmox_backup::api2::access::user::UserWithTokens;
  37 use proxmox_backup::api2::types::*;
  38 use proxmox_backup::api2::version;
  39 use proxmox_backup::client::*;
  40 use proxmox_backup::pxar::catalog::*;
  41 use proxmox_backup::backup::{
  42     archive_type,
  43     decrypt_key,
  44     rsa_encrypt_key_config,
  45     verify_chunk_size,
  46     ArchiveType,
  47     AsyncReadChunk,
  48     BackupDir,
  49     BackupGroup,
  50     BackupManifest,
  51     BufferedDynamicReader,
  52     CATALOG_NAME,
  53     CatalogReader,
  54     CatalogWriter,
  55     ChunkStream,
  56     CryptConfig,
  57     CryptMode,
  58     DynamicIndexReader,
  59     ENCRYPTED_KEY_BLOB_NAME,
  60     FixedChunkStream,
  61     FixedIndexReader,
  62     KeyConfig,
  63     IndexFile,
  64     MANIFEST_BLOB_NAME,
  65     Shell,
  66 };
  67
  68 mod proxmox_backup_client;
  69 use proxmox_backup_client::*;
  70
  71 const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
  72 const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
  73
  74
  75 pub const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  76     .format(&BACKUP_REPO_URL)
  77     .max_length(256)
  78     .schema();
  79
  80 pub const KEYFILE_SCHEMA: Schema =
  81     StringSchema::new("Path to encryption key. All data will be encrypted using this key.")
  82         .schema();
  83
  84 pub const KEYFD_SCHEMA: Schema =
  85     IntegerSchema::new("Pass an encryption key via an already opened file descriptor.")
  86         .minimum(0)
  87         .schema();
  88
  89 pub const MASTER_PUBKEY_FILE_SCHEMA: Schema = StringSchema::new(
  90     "Path to master public key. The encryption key used for a backup will be encrypted using this key and appended to the backup.")
  91     .schema();
  92
  93 pub const MASTER_PUBKEY_FD_SCHEMA: Schema =
  94     IntegerSchema::new("Pass a master public key via an already opened file descriptor.")
  95         .minimum(0)
  96         .schema();
  97
  98 const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new(
  99     "Chunk size in KB. Must be a power of 2.")
 100     .minimum(64)
 101     .maximum(4096)
 102     .default(4096)
 103     .schema();
 104
 105 fn get_default_repository() -> Option<String> {
 106     std::env::var("PBS_REPOSITORY").ok()
 107 }
 108
 109 pub fn extract_repository_from_value(
 110     param: &Value,
 111 ) -> Result<BackupRepository, Error> {
 112
 113     let repo_url = param["repository"]
 114         .as_str()
 115         .map(String::from)
 116         .or_else(get_default_repository)
 117         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
 118
 119     let repo: BackupRepository = repo_url.parse()?;
 120
 121     Ok(repo)
 122 }
 123
 124 fn extract_repository_from_map(
 125     param: &HashMap<String, String>,
 126 ) -> Option<BackupRepository> {
 127
 128     param.get("repository")
 129         .map(String::from)
 130         .or_else(get_default_repository)
 131         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
 132 }
 133
 134 fn record_repository(repo: &BackupRepository) {
 135
 136     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 137         Ok(v) => v,
 138         _ => return,
 139     };
 140
 141     // usually $HOME/.cache/proxmox-backup/repo-list
 142     let path = match base.place_cache_file("repo-list") {
 143         Ok(v) => v,
 144         _ => return,
 145     };
 146
 147     let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 148
 149     let repo = repo.to_string();
 150
 151     data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
 152
 153     let mut map = serde_json::map::Map::new();
 154
 155     loop {
 156         let mut max_used = 0;
 157         let mut max_repo = None;
 158         for (repo, count) in data.as_object().unwrap() {
 159             if map.contains_key(repo) { continue; }
 160             if let Some(count) = count.as_i64() {
 161                 if count > max_used {
 162                     max_used = count;
 163                     max_repo = Some(repo);
 164                 }
 165             }
 166         }
 167         if let Some(repo) = max_repo {
 168             map.insert(repo.to_owned(), json!(max_used));
 169         } else {
 170             break;
 171         }
 172         if map.len() > 10 { // store max. 10 repos
 173             break;
 174         }
 175     }
 176
 177     let new_data = json!(map);
 178
 179     let _ = replace_file(path, new_data.to_string().as_bytes(), CreateOptions::new());
 180 }
 181
 182 pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 183
 184     let mut result = vec![];
 185
 186     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 187         Ok(v) => v,
 188         _ => return result,
 189     };
 190
 191     // usually $HOME/.cache/proxmox-backup/repo-list
 192     let path = match base.place_cache_file("repo-list") {
 193         Ok(v) => v,
 194         _ => return result,
 195     };
 196
 197     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 198
 199     if let Some(map) = data.as_object() {
 200         for (repo, _count) in map {
 201             result.push(repo.to_owned());
 202         }
 203     }
 204
 205     result
 206 }
 207
 208 fn connect(repo: &BackupRepository) -> Result<HttpClient, Error> {
 209     connect_do(repo.host(), repo.port(), repo.auth_id())
 210         .map_err(|err| format_err!("error building client for repository {} - {}", repo, err))
 211 }
 212
 213 fn connect_do(server: &str, port: u16, auth_id: &Authid) -> Result<HttpClient, Error> {
 214     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 215
 216     use std::env::VarError::*;
 217     let password = match std::env::var(ENV_VAR_PBS_PASSWORD) {
 218         Ok(p) => Some(p),
 219         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", ENV_VAR_PBS_PASSWORD)),
 220         Err(NotPresent) => None,
 221     };
 222
 223     let options = HttpClientOptions::new_interactive(password, fingerprint);
 224
 225     HttpClient::new(server, port, auth_id, options)
 226 }
 227
 228 async fn api_datastore_list_snapshots(
 229     client: &HttpClient,
 230     store: &str,
 231     group: Option<BackupGroup>,
 232 ) -> Result<Value, Error> {
 233
 234     let path = format!("api2/json/admin/datastore/{}/snapshots", store);
 235
 236     let mut args = json!({});
 237     if let Some(group) = group {
 238         args["backup-type"] = group.backup_type().into();
 239         args["backup-id"] = group.backup_id().into();
 240     }
 241
 242     let mut result = client.get(&path, Some(args)).await?;
 243
 244     Ok(result["data"].take())
 245 }
 246
 247 pub async fn api_datastore_latest_snapshot(
 248     client: &HttpClient,
 249     store: &str,
 250     group: BackupGroup,
 251 ) -> Result<(String, String, i64), Error> {
 252
 253     let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
 254     let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
 255
 256     if list.is_empty() {
 257         bail!("backup group {:?} does not contain any snapshots.", group.group_path());
 258     }
 259
 260     list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
 261
 262     let backup_time = list[0].backup_time;
 263
 264     Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
 265 }
 266
 267 async fn backup_directory<P: AsRef<Path>>(
 268     client: &BackupWriter,
 269     dir_path: P,
 270     archive_name: &str,
 271     chunk_size: Option<usize>,
 272     catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
 273     pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
 274     upload_options: UploadOptions,
 275 ) -> Result<BackupStats, Error> {
 276
 277     let pxar_stream = PxarBackupStream::open(
 278         dir_path.as_ref(),
 279         catalog,
 280         pxar_create_options,
 281     )?;
 282     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 283
 284     let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
 285
 286     let stream = ReceiverStream::new(rx)
 287         .map_err(Error::from);
 288
 289     // spawn chunker inside a separate task so that it can run parallel
 290     tokio::spawn(async move {
 291         while let Some(v) = chunk_stream.next().await {
 292             let _ = tx.send(v).await;
 293         }
 294     });
 295
 296     if upload_options.fixed_size.is_some() {
 297         bail!("cannot backup directory with fixed chunk size!");
 298     }
 299
 300     let stats = client
 301         .upload_stream(archive_name, stream, upload_options)
 302         .await?;
 303
 304     Ok(stats)
 305 }
 306
 307 async fn backup_image<P: AsRef<Path>>(
 308     client: &BackupWriter,
 309     image_path: P,
 310     archive_name: &str,
 311     chunk_size: Option<usize>,
 312     upload_options: UploadOptions,
 313 ) -> Result<BackupStats, Error> {
 314
 315     let path = image_path.as_ref().to_owned();
 316
 317     let file = tokio::fs::File::open(path).await?;
 318
 319     let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
 320         .map_err(Error::from);
 321
 322     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 323
 324     if upload_options.fixed_size.is_none() {
 325         bail!("cannot backup image with dynamic chunk size!");
 326     }
 327
 328     let stats = client
 329         .upload_stream(archive_name, stream, upload_options)
 330         .await?;
 331
 332     Ok(stats)
 333 }
 334
 335 #[api(
 336    input: {
 337         properties: {
 338             repository: {
 339                 schema: REPO_URL_SCHEMA,
 340                 optional: true,
 341             },
 342             "output-format": {
 343                 schema: OUTPUT_FORMAT,
 344                 optional: true,
 345             },
 346         }
 347    }
 348 )]
 349 /// List backup groups.
 350 async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 351
 352     let output_format = get_output_format(&param);
 353
 354     let repo = extract_repository_from_value(&param)?;
 355
 356     let client = connect(&repo)?;
 357
 358     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 359
 360     let mut result = client.get(&path, None).await?;
 361
 362     record_repository(&repo);
 363
 364     let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
 365         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 366         let group = BackupGroup::new(item.backup_type, item.backup_id);
 367         Ok(group.group_path().to_str().unwrap().to_owned())
 368     };
 369
 370     let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
 371         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 372         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
 373         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
 374     };
 375
 376     let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
 377         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
 378         Ok(tools::format::render_backup_file_list(&item.files))
 379     };
 380
 381     let options = default_table_format_options()
 382         .sortby("backup-type", false)
 383         .sortby("backup-id", false)
 384         .column(ColumnConfig::new("backup-id").renderer(render_group_path).header("group"))
 385         .column(
 386             ColumnConfig::new("last-backup")
 387                 .renderer(render_last_backup)
 388                 .header("last snapshot")
 389                 .right_align(false)
 390         )
 391         .column(ColumnConfig::new("backup-count"))
 392         .column(ColumnConfig::new("files").renderer(render_files));
 393
 394     let mut data: Value = result["data"].take();
 395
 396     let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_LIST_GROUPS.returns;
 397
 398     format_and_print_result_full(&mut data, return_type, &output_format, &options);
 399
 400     Ok(Value::Null)
 401 }
 402
 403 #[api(
 404    input: {
 405         properties: {
 406             repository: {
 407                 schema: REPO_URL_SCHEMA,
 408                 optional: true,
 409             },
 410             group: {
 411                 type: String,
 412                 description: "Backup group.",
 413             },
 414             "new-owner": {
 415                 type: Authid,
 416             },
 417         }
 418    }
 419 )]
 420 /// Change owner of a backup group
 421 async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
 422
 423     let repo = extract_repository_from_value(&param)?;
 424
 425     let mut client = connect(&repo)?;
 426
 427     param.as_object_mut().unwrap().remove("repository");
 428
 429     let group: BackupGroup = group.parse()?;
 430
 431     param["backup-type"] = group.backup_type().into();
 432     param["backup-id"] = group.backup_id().into();
 433
 434     let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
 435     client.post(&path, Some(param)).await?;
 436
 437     record_repository(&repo);
 438
 439     Ok(())
 440 }
 441
 442 #[api(
 443    input: {
 444         properties: {
 445             repository: {
 446                 schema: REPO_URL_SCHEMA,
 447                 optional: true,
 448             },
 449         }
 450    }
 451 )]
 452 /// Try to login. If successful, store ticket.
 453 async fn api_login(param: Value) -> Result<Value, Error> {
 454
 455     let repo = extract_repository_from_value(&param)?;
 456
 457     let client = connect(&repo)?;
 458     client.login().await?;
 459
 460     record_repository(&repo);
 461
 462     Ok(Value::Null)
 463 }
 464
 465 #[api(
 466    input: {
 467         properties: {
 468             repository: {
 469                 schema: REPO_URL_SCHEMA,
 470                 optional: true,
 471             },
 472         }
 473    }
 474 )]
 475 /// Logout (delete stored ticket).
 476 fn api_logout(param: Value) -> Result<Value, Error> {
 477
 478     let repo = extract_repository_from_value(&param)?;
 479
 480     delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
 481
 482     Ok(Value::Null)
 483 }
 484
 485 #[api(
 486    input: {
 487         properties: {
 488             repository: {
 489                 schema: REPO_URL_SCHEMA,
 490                 optional: true,
 491             },
 492             "output-format": {
 493                 schema: OUTPUT_FORMAT,
 494                 optional: true,
 495             },
 496         }
 497    }
 498 )]
 499 /// Show client and optional server version
 500 async fn api_version(param: Value) -> Result<(), Error> {
 501
 502     let output_format = get_output_format(&param);
 503
 504     let mut version_info = json!({
 505         "client": {
 506             "version": version::PROXMOX_PKG_VERSION,
 507             "release": version::PROXMOX_PKG_RELEASE,
 508             "repoid": version::PROXMOX_PKG_REPOID,
 509         }
 510     });
 511
 512     let repo = extract_repository_from_value(&param);
 513     if let Ok(repo) = repo {
 514         let client = connect(&repo)?;
 515
 516         match client.get("api2/json/version", None).await {
 517             Ok(mut result) => version_info["server"] = result["data"].take(),
 518             Err(e) => eprintln!("could not connect to server - {}", e),
 519         }
 520     }
 521     if output_format == "text" {
 522         println!("client version: {}.{}", version::PROXMOX_PKG_VERSION, version::PROXMOX_PKG_RELEASE);
 523         if let Some(server) = version_info["server"].as_object() {
 524             let server_version = server["version"].as_str().unwrap();
 525             let server_release = server["release"].as_str().unwrap();
 526             println!("server version: {}.{}", server_version, server_release);
 527         }
 528     } else {
 529         format_and_print_result(&version_info, &output_format);
 530     }
 531
 532     Ok(())
 533 }
 534
 535 #[api(
 536     input: {
 537         properties: {
 538             repository: {
 539                 schema: REPO_URL_SCHEMA,
 540                 optional: true,
 541             },
 542             "output-format": {
 543                 schema: OUTPUT_FORMAT,
 544                 optional: true,
 545             },
 546         },
 547     },
 548 )]
 549 /// Start garbage collection for a specific repository.
 550 async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 551
 552     let repo = extract_repository_from_value(&param)?;
 553
 554     let output_format = get_output_format(&param);
 555
 556     let mut client = connect(&repo)?;
 557
 558     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 559
 560     let result = client.post(&path, None).await?;
 561
 562     record_repository(&repo);
 563
 564     view_task_result(&mut client, result, &output_format).await?;
 565
 566     Ok(Value::Null)
 567 }
 568
 569 struct CatalogUploadResult {
 570     catalog_writer: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
 571     result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
 572 }
 573
 574 fn spawn_catalog_upload(
 575     client: Arc<BackupWriter>,
 576     encrypt: bool,
 577 ) -> Result<CatalogUploadResult, Error> {
 578     let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
 579     let catalog_stream = crate::tools::StdChannelStream(catalog_rx);
 580     let catalog_chunk_size = 512*1024;
 581     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 582
 583     let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(crate::tools::StdChannelWriter::new(catalog_tx))?));
 584
 585     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 586
 587     let upload_options = UploadOptions {
 588         encrypt,
 589         compress: true,
 590         ..UploadOptions::default()
 591     };
 592
 593     tokio::spawn(async move {
 594         let catalog_upload_result = client
 595             .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
 596             .await;
 597
 598         if let Err(ref err) = catalog_upload_result {
 599             eprintln!("catalog upload error - {}", err);
 600             client.cancel();
 601         }
 602
 603         let _ = catalog_result_tx.send(catalog_upload_result);
 604     });
 605
 606     Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
 607 }
 608
 609 #[derive(Debug, Eq, PartialEq)]
 610 struct CryptoParams {
 611     mode: CryptMode,
 612     enc_key: Option<Vec<u8>>,
 613     // FIXME switch to openssl::rsa::rsa<openssl::pkey::Public> once that is Eq?
 614     master_pubkey: Option<Vec<u8>>,
 615 }
 616
 617 fn crypto_parameters(param: &Value) -> Result<CryptoParams, Error> {
 618     let keyfile = match param.get("keyfile") {
 619         Some(Value::String(keyfile)) => Some(keyfile),
 620         Some(_) => bail!("bad --keyfile parameter type"),
 621         None => None,
 622     };
 623
 624     let key_fd = match param.get("keyfd") {
 625         Some(Value::Number(key_fd)) => Some(
 626             RawFd::try_from(key_fd
 627                 .as_i64()
 628                 .ok_or_else(|| format_err!("bad key fd: {:?}", key_fd))?
 629             )
 630             .map_err(|err| format_err!("bad key fd: {:?}: {}", key_fd, err))?
 631         ),
 632         Some(_) => bail!("bad --keyfd parameter type"),
 633         None => None,
 634     };
 635
 636     let master_pubkey_file = match param.get("master-pubkey-file") {
 637         Some(Value::String(keyfile)) => Some(keyfile),
 638         Some(_) => bail!("bad --master-pubkey-file parameter type"),
 639         None => None,
 640     };
 641
 642     let master_pubkey_fd = match param.get("master-pubkey-fd") {
 643         Some(Value::Number(key_fd)) => Some(
 644             RawFd::try_from(key_fd
 645                 .as_i64()
 646                 .ok_or_else(|| format_err!("bad master public key fd: {:?}", key_fd))?
 647             )
 648             .map_err(|err| format_err!("bad public master key fd: {:?}: {}", key_fd, err))?
 649         ),
 650         Some(_) => bail!("bad --master-pubkey-fd parameter type"),
 651         None => None,
 652     };
 653
 654     let mode: Option<CryptMode> = match param.get("crypt-mode") {
 655         Some(mode) => Some(serde_json::from_value(mode.clone())?),
 656         None => None,
 657     };
 658
 659     let keydata = match (keyfile, key_fd) {
 660         (None, None) => None,
 661         (Some(_), Some(_)) => bail!("--keyfile and --keyfd are mutually exclusive"),
 662         (Some(keyfile), None) => {
 663             eprintln!("Using encryption key file: {}", keyfile);
 664             Some(file_get_contents(keyfile)?)
 665         },
 666         (None, Some(fd)) => {
 667             let input = unsafe { std::fs::File::from_raw_fd(fd) };
 668             let mut data = Vec::new();
 669             let _len: usize = { input }.read_to_end(&mut data)
 670                 .map_err(|err| {
 671                     format_err!("error reading encryption key from fd {}: {}", fd, err)
 672                 })?;
 673             eprintln!("Using encryption key from file descriptor");
 674             Some(data)
 675         }
 676     };
 677
 678     let master_pubkey_data = match (master_pubkey_file, master_pubkey_fd) {
 679         (None, None) => None,
 680         (Some(_), Some(_)) => bail!("--keyfile and --keyfd are mutually exclusive"),
 681         (Some(keyfile), None) => {
 682             eprintln!("Using master key from file: {}", keyfile);
 683             Some(file_get_contents(keyfile)?)
 684         },
 685         (None, Some(fd)) => {
 686             let input = unsafe { std::fs::File::from_raw_fd(fd) };
 687             let mut data = Vec::new();
 688             let _len: usize = { input }.read_to_end(&mut data)
 689                 .map_err(|err| {
 690                     format_err!("error reading master key from fd {}: {}", fd, err)
 691                 })?;
 692             eprintln!("Using master key from file descriptor");
 693             Some(data)
 694         }
 695     };
 696
 697     let res = match mode {
 698         // no crypt mode, enable encryption if keys are available
 699         None => match (keydata, master_pubkey_data) {
 700             // only default keys if available
 701             (None, None) => match key::read_optional_default_encryption_key()? {
 702                 None => CryptoParams { mode: CryptMode::None, enc_key: None, master_pubkey: None },
 703                 enc_key => {
 704                     eprintln!("Encrypting with default encryption key!");
 705                     let master_pubkey = key::read_optional_default_master_pubkey()?;
 706                     CryptoParams {
 707                         mode: CryptMode::Encrypt,
 708                         enc_key,
 709                         master_pubkey,
 710                     }
 711                 },
 712             },
 713
 714             // explicit master key, default enc key needed
 715             (None, master_pubkey) => match key::read_optional_default_encryption_key()? {
 716                 None => bail!("--master-pubkey-file/--master-pubkey-fd specified, but no key available"),
 717                 enc_key => {
 718                     eprintln!("Encrypting with default encryption key!");
 719                     CryptoParams {
 720                         mode: CryptMode::Encrypt,
 721                         enc_key,
 722                         master_pubkey,
 723                     }
 724                 },
 725             },
 726
 727             // explicit keyfile, maybe default master key
 728             (enc_key, None) => CryptoParams { mode: CryptMode::Encrypt, enc_key, master_pubkey: key::read_optional_default_master_pubkey()? },
 729
 730             // explicit keyfile and master key
 731             (enc_key, master_pubkey) => CryptoParams { mode: CryptMode::Encrypt, enc_key, master_pubkey },
 732         },
 733
 734         // explicitly disabled encryption
 735         Some(CryptMode::None) => match (keydata, master_pubkey_data) {
 736             // no keys => OK, no encryption
 737             (None, None) => CryptoParams { mode: CryptMode::None, enc_key: None, master_pubkey: None },
 738
 739             // --keyfile and --crypt-mode=none
 740             (Some(_), _) => bail!("--keyfile/--keyfd and --crypt-mode=none are mutually exclusive"),
 741
 742             // --master-pubkey-file and --crypt-mode=none
 743             (_, Some(_)) => bail!("--master-pubkey-file/--master-pubkey-fd and --crypt-mode=none are mutually exclusive"),
 744         },
 745
 746         // explicitly enabled encryption
 747         Some(mode) => match (keydata, master_pubkey_data) {
 748             // no key, maybe master key
 749             (None, master_pubkey) => match key::read_optional_default_encryption_key()? {
 750                 None => bail!("--crypt-mode without --keyfile and no default key file available"),
 751                 enc_key => {
 752                     eprintln!("Encrypting with default encryption key!");
 753                     let master_pubkey = match master_pubkey {
 754                         None => key::read_optional_default_master_pubkey()?,
 755                         master_pubkey => master_pubkey,
 756                     };
 757
 758                     CryptoParams {
 759                         mode,
 760                         enc_key,
 761                         master_pubkey,
 762                     }
 763                 },
 764             },
 765
 766             // --keyfile and --crypt-mode other than none
 767             (enc_key, master_pubkey) => {
 768                 let master_pubkey = match master_pubkey {
 769                     None => key::read_optional_default_master_pubkey()?,
 770                     master_pubkey => master_pubkey,
 771                 };
 772
 773                 CryptoParams { mode, enc_key, master_pubkey }
 774             },
 775         },
 776     };
 777
 778     Ok(res)
 779 }
 780
 781 #[test]
 782 // WARNING: there must only be one test for crypto_parameters as the default key handling is not
 783 // safe w.r.t. concurrency
 784 fn test_crypto_parameters_handling() -> Result<(), Error> {
 785     let some_key = Some(vec![1;1]);
 786     let default_key = Some(vec![2;1]);
 787
 788     let some_master_key = Some(vec![3;1]);
 789     let default_master_key = Some(vec![4;1]);
 790
 791     let no_key_res = CryptoParams {
 792         enc_key: None,
 793         master_pubkey: None,
 794         mode: CryptMode::None,
 795     };
 796     let some_key_res = CryptoParams {
 797         enc_key: some_key.clone(),
 798         master_pubkey: None,
 799         mode: CryptMode::Encrypt,
 800     };
 801     let some_key_some_master_res = CryptoParams {
 802         enc_key: some_key.clone(),
 803         master_pubkey: some_master_key.clone(),
 804         mode: CryptMode::Encrypt,
 805     };
 806     let some_key_default_master_res = CryptoParams {
 807         enc_key: some_key.clone(),
 808         master_pubkey: default_master_key.clone(),
 809         mode: CryptMode::Encrypt,
 810     };
 811
 812     let some_key_sign_res = CryptoParams {
 813         enc_key: some_key.clone(),
 814         master_pubkey: None,
 815         mode: CryptMode::SignOnly,
 816     };
 817     let default_key_res = CryptoParams {
 818         enc_key: default_key.clone(),
 819         master_pubkey: None,
 820         mode: CryptMode::Encrypt,
 821     };
 822     let default_key_sign_res = CryptoParams {
 823         enc_key: default_key.clone(),
 824         master_pubkey: None,
 825         mode: CryptMode::SignOnly,
 826     };
 827
 828     let keypath = "./tests/keyfile.test";
 829     replace_file(&keypath, some_key.as_ref().unwrap(), CreateOptions::default())?;
 830     let master_keypath = "./tests/masterkeyfile.test";
 831     replace_file(&master_keypath, some_master_key.as_ref().unwrap(), CreateOptions::default())?;
 832     let invalid_keypath = "./tests/invalid_keyfile.test";
 833
 834     // no params, no default key == no key
 835     let res = crypto_parameters(&json!({}));
 836     assert_eq!(res.unwrap(), no_key_res);
 837
 838     // keyfile param == key from keyfile
 839     let res = crypto_parameters(&json!({"keyfile": keypath}));
 840     assert_eq!(res.unwrap(), some_key_res);
 841
 842     // crypt mode none == no key
 843     let res = crypto_parameters(&json!({"crypt-mode": "none"}));
 844     assert_eq!(res.unwrap(), no_key_res);
 845
 846     // crypt mode encrypt/sign-only, no keyfile, no default key == Error
 847     assert!(crypto_parameters(&json!({"crypt-mode": "sign-only"})).is_err());
 848     assert!(crypto_parameters(&json!({"crypt-mode": "encrypt"})).is_err());
 849
 850     // crypt mode none with explicit key == Error
 851     assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
 852
 853     // crypt mode sign-only/encrypt with keyfile == key from keyfile with correct mode
 854     let res = crypto_parameters(&json!({"crypt-mode": "sign-only", "keyfile": keypath}));
 855     assert_eq!(res.unwrap(), some_key_sign_res);
 856     let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
 857     assert_eq!(res.unwrap(), some_key_res);
 858
 859     // invalid keyfile parameter always errors
 860     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath})).is_err());
 861     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "none"})).is_err());
 862     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "sign-only"})).is_err());
 863     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "encrypt"})).is_err());
 864
 865     // now set a default key
 866     unsafe { key::set_test_encryption_key(Ok(default_key.clone())); }
 867
 868     // and repeat
 869
 870     // no params but default key == default key
 871     let res = crypto_parameters(&json!({}));
 872     assert_eq!(res.unwrap(), default_key_res);
 873
 874     // keyfile param == key from keyfile
 875     let res = crypto_parameters(&json!({"keyfile": keypath}));
 876     assert_eq!(res.unwrap(), some_key_res);
 877
 878     // crypt mode none == no key
 879     let res = crypto_parameters(&json!({"crypt-mode": "none"}));
 880     assert_eq!(res.unwrap(), no_key_res);
 881
 882     // crypt mode encrypt/sign-only, no keyfile, default key == default key with correct mode
 883     let res = crypto_parameters(&json!({"crypt-mode": "sign-only"}));
 884     assert_eq!(res.unwrap(), default_key_sign_res);
 885     let res = crypto_parameters(&json!({"crypt-mode": "encrypt"}));
 886     assert_eq!(res.unwrap(), default_key_res);
 887
 888     // crypt mode none with explicit key == Error
 889     assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
 890
 891     // crypt mode sign-only/encrypt with keyfile == key from keyfile with correct mode
 892     let res = crypto_parameters(&json!({"crypt-mode": "sign-only", "keyfile": keypath}));
 893     assert_eq!(res.unwrap(), some_key_sign_res);
 894     let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
 895     assert_eq!(res.unwrap(), some_key_res);
 896
 897     // invalid keyfile parameter always errors
 898     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath})).is_err());
 899     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "none"})).is_err());
 900     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "sign-only"})).is_err());
 901     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "encrypt"})).is_err());
 902
 903     // now make default key retrieval error
 904     unsafe { key::set_test_encryption_key(Err(format_err!("test error"))); }
 905
 906     // and repeat
 907
 908     // no params, default key retrieval errors == Error
 909     assert!(crypto_parameters(&json!({})).is_err());
 910
 911     // keyfile param == key from keyfile
 912     let res = crypto_parameters(&json!({"keyfile": keypath}));
 913     assert_eq!(res.unwrap(), some_key_res);
 914
 915     // crypt mode none == no key
 916     let res = crypto_parameters(&json!({"crypt-mode": "none"}));
 917     assert_eq!(res.unwrap(), no_key_res);
 918
 919     // crypt mode encrypt/sign-only, no keyfile, default key error == Error
 920     assert!(crypto_parameters(&json!({"crypt-mode": "sign-only"})).is_err());
 921     assert!(crypto_parameters(&json!({"crypt-mode": "encrypt"})).is_err());
 922
 923     // crypt mode none with explicit key == Error
 924     assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
 925
 926     // crypt mode sign-only/encrypt with keyfile == key from keyfile with correct mode
 927     let res = crypto_parameters(&json!({"crypt-mode": "sign-only", "keyfile": keypath}));
 928     assert_eq!(res.unwrap(), some_key_sign_res);
 929     let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
 930     assert_eq!(res.unwrap(), some_key_res);
 931
 932     // invalid keyfile parameter always errors
 933     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath})).is_err());
 934     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "none"})).is_err());
 935     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "sign-only"})).is_err());
 936     assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "encrypt"})).is_err());
 937
 938     // now remove default key again
 939     unsafe { key::set_test_encryption_key(Ok(None)); }
 940     // set a default master key
 941     unsafe { key::set_test_default_master_pubkey(Ok(default_master_key.clone())); }
 942
 943     // and use an explicit master key
 944     assert!(crypto_parameters(&json!({"master-pubkey-file": master_keypath})).is_err());
 945     // just a default == no key
 946     let res = crypto_parameters(&json!({}));
 947     assert_eq!(res.unwrap(), no_key_res);
 948
 949     // keyfile param == key from keyfile
 950     let res = crypto_parameters(&json!({"keyfile": keypath, "master-pubkey-file": master_keypath}));
 951     assert_eq!(res.unwrap(), some_key_some_master_res);
 952     // same with fallback to default master key
 953     let res = crypto_parameters(&json!({"keyfile": keypath}));
 954     assert_eq!(res.unwrap(), some_key_default_master_res);
 955
 956     // crypt mode none == error
 957     assert!(crypto_parameters(&json!({"crypt-mode": "none", "master-pubkey-file": master_keypath})).is_err());
 958     // with just default master key == no key
 959     let res = crypto_parameters(&json!({"crypt-mode": "none"}));
 960     assert_eq!(res.unwrap(), no_key_res);
 961
 962     // crypt mode encrypt without enc key == error
 963     assert!(crypto_parameters(&json!({"crypt-mode": "encrypt", "master-pubkey-file": master_keypath})).is_err());
 964     assert!(crypto_parameters(&json!({"crypt-mode": "encrypt"})).is_err());
 965
 966     // crypt mode none with explicit key == Error
 967     assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath, "master-pubkey-file": master_keypath})).is_err());
 968     assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
 969
 970     // crypt mode encrypt with keyfile == key from keyfile with correct mode
 971     let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath, "master-pubkey-file": master_keypath}));
 972     assert_eq!(res.unwrap(), some_key_some_master_res);
 973     let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
 974     assert_eq!(res.unwrap(), some_key_default_master_res);
 975
 976     // invalid master keyfile parameter always errors when a key is passed, even with a valid
 977     // default master key
 978     assert!(crypto_parameters(&json!({"keyfile": keypath, "master-pubkey-file": invalid_keypath})).is_err());
 979     assert!(crypto_parameters(&json!({"keyfile": keypath, "master-pubkey-file": invalid_keypath,"crypt-mode": "none"})).is_err());
 980     assert!(crypto_parameters(&json!({"keyfile": keypath, "master-pubkey-file": invalid_keypath,"crypt-mode": "sign-only"})).is_err());
 981     assert!(crypto_parameters(&json!({"keyfile": keypath, "master-pubkey-file": invalid_keypath,"crypt-mode": "encrypt"})).is_err());
 982
 983     Ok(())
 984 }
 985
 986 #[api(
 987    input: {
 988        properties: {
 989            backupspec: {
 990                type: Array,
 991                description: "List of backup source specifications ([<label.ext>:<path>] ...)",
 992                items: {
 993                    schema: BACKUP_SOURCE_SCHEMA,
 994                }
 995            },
 996            repository: {
 997                schema: REPO_URL_SCHEMA,
 998                optional: true,
 999            },
1000            "include-dev": {
1001                description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
1002                optional: true,
1003                items: {
1004                    type: String,
1005                    description: "Path to file.",
1006                }
1007            },
1008            "all-file-systems": {
1009                type: Boolean,
1010                description: "Include all mounted subdirectories.",
1011                optional: true,
1012            },
1013            keyfile: {
1014                schema: KEYFILE_SCHEMA,
1015                optional: true,
1016            },
1017            "keyfd": {
1018                schema: KEYFD_SCHEMA,
1019                optional: true,
1020            },
1021            "master-pubkey-file": {
1022                schema: MASTER_PUBKEY_FILE_SCHEMA,
1023                optional: true,
1024            },
1025            "master-pubkey-fd": {
1026                schema: MASTER_PUBKEY_FD_SCHEMA,
1027                optional: true,
1028            },
1029            "crypt-mode": {
1030                type: CryptMode,
1031                optional: true,
1032            },
1033            "skip-lost-and-found": {
1034                type: Boolean,
1035                description: "Skip lost+found directory.",
1036                optional: true,
1037            },
1038            "backup-type": {
1039                schema: BACKUP_TYPE_SCHEMA,
1040                optional: true,
1041            },
1042            "backup-id": {
1043                schema: BACKUP_ID_SCHEMA,
1044                optional: true,
1045            },
1046            "backup-time": {
1047                schema: BACKUP_TIME_SCHEMA,
1048                optional: true,
1049            },
1050            "chunk-size": {
1051                schema: CHUNK_SIZE_SCHEMA,
1052                optional: true,
1053            },
1054            "exclude": {
1055                type: Array,
1056                description: "List of paths or patterns for matching files to exclude.",
1057                optional: true,
1058                items: {
1059                    type: String,
1060                    description: "Path or match pattern.",
1061                 }
1062            },
1063            "entries-max": {
1064                type: Integer,
1065                description: "Max number of entries to hold in memory.",
1066                optional: true,
1067                default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
1068            },
1069            "verbose": {
1070                type: Boolean,
1071                description: "Verbose output.",
1072                optional: true,
1073            },
1074        }
1075    }
1076 )]
1077 /// Create (host) backup.
1078 async fn create_backup(
1079     param: Value,
1080     _info: &ApiMethod,
1081     _rpcenv: &mut dyn RpcEnvironment,
1082 ) -> Result<Value, Error> {
1083
1084     let repo = extract_repository_from_value(&param)?;
1085
1086     let backupspec_list = tools::required_array_param(&param, "backupspec")?;
1087
1088     let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
1089
1090     let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
1091
1092     let verbose = param["verbose"].as_bool().unwrap_or(false);
1093
1094     let backup_time_opt = param["backup-time"].as_i64();
1095
1096     let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
1097
1098     if let Some(size) = chunk_size_opt {
1099         verify_chunk_size(size)?;
1100     }
1101
1102     let crypto = crypto_parameters(&param)?;
1103
1104     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
1105
1106     let backup_type = param["backup-type"].as_str().unwrap_or("host");
1107
1108     let include_dev = param["include-dev"].as_array();
1109
1110     let entries_max = param["entries-max"].as_u64()
1111         .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
1112
1113     let empty = Vec::new();
1114     let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
1115
1116     let mut pattern_list = Vec::with_capacity(exclude_args.len());
1117     for entry in exclude_args {
1118         let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
1119         pattern_list.push(
1120             MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
1121                 .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
1122         );
1123     }
1124
1125     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
1126
1127     if let Some(include_dev) = include_dev {
1128         if all_file_systems {
1129             bail!("option 'all-file-systems' conflicts with option 'include-dev'");
1130         }
1131
1132         let mut set = HashSet::new();
1133         for path in include_dev {
1134             let path = path.as_str().unwrap();
1135             let stat = nix::sys::stat::stat(path)
1136                 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
1137             set.insert(stat.st_dev);
1138         }
1139         devices = Some(set);
1140     }
1141
1142     let mut upload_list = vec![];
1143     let mut target_set = HashSet::new();
1144
1145     for backupspec in backupspec_list {
1146         let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
1147         let filename = &spec.config_string;
1148         let target = &spec.archive_name;
1149
1150         if target_set.contains(target) {
1151             bail!("got target twice: '{}'", target);
1152         }
1153         target_set.insert(target.to_string());
1154
1155         use std::os::unix::fs::FileTypeExt;
1156
1157         let metadata = std::fs::metadata(filename)
1158             .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
1159         let file_type = metadata.file_type();
1160
1161         match spec.spec_type {
1162             BackupSpecificationType::PXAR => {
1163                 if !file_type.is_dir() {
1164                     bail!("got unexpected file type (expected directory)");
1165                 }
1166                 upload_list.push((BackupSpecificationType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
1167             }
1168             BackupSpecificationType::IMAGE => {
1169                 if !(file_type.is_file() || file_type.is_block_device()) {
1170                     bail!("got unexpected file type (expected file or block device)");
1171                 }
1172
1173                 let size = image_size(&PathBuf::from(filename))?;
1174
1175                 if size == 0 { bail!("got zero-sized file '{}'", filename); }
1176
1177                 upload_list.push((BackupSpecificationType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
1178             }
1179             BackupSpecificationType::CONFIG => {
1180                 if !file_type.is_file() {
1181                     bail!("got unexpected file type (expected regular file)");
1182                 }
1183                 upload_list.push((BackupSpecificationType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
1184             }
1185             BackupSpecificationType::LOGFILE => {
1186                 if !file_type.is_file() {
1187                     bail!("got unexpected file type (expected regular file)");
1188                 }
1189                 upload_list.push((BackupSpecificationType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
1190             }
1191         }
1192     }
1193
1194     let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
1195
1196     let client = connect(&repo)?;
1197     record_repository(&repo);
1198
1199     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
1200
1201     println!("Client name: {}", proxmox::tools::nodename());
1202
1203     let start_time = std::time::Instant::now();
1204
1205     println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
1206
1207     let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
1208         None => (None, None),
1209         Some(key) => {
1210             let (key, created, fingerprint) = decrypt_key(&key, &key::get_encryption_key_password)?;
1211             println!("Encryption key fingerprint: {}", fingerprint);
1212
1213             let crypt_config = CryptConfig::new(key)?;
1214
1215             match crypto.master_pubkey {
1216                 Some(pem_data) => {
1217                     let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
1218
1219                     let mut key_config = KeyConfig::without_password(key)?;
1220                     key_config.created = created; // keep original value
1221
1222                     let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
1223
1224                     (Some(Arc::new(crypt_config)), Some(enc_key))
1225                 }
1226                 _ => (Some(Arc::new(crypt_config)), None),
1227             }
1228         }
1229     };
1230
1231     let client = BackupWriter::start(
1232         client,
1233         crypt_config.clone(),
1234         repo.store(),
1235         backup_type,
1236         &backup_id,
1237         backup_time,
1238         verbose,
1239         false
1240     ).await?;
1241
1242     let download_previous_manifest = match client.previous_backup_time().await {
1243         Ok(Some(backup_time)) => {
1244             println!(
1245                 "Downloading previous manifest ({})",
1246                 strftime_local("%c", backup_time)?
1247             );
1248             true
1249         }
1250         Ok(None) => {
1251             println!("No previous manifest available.");
1252             false
1253         }
1254         Err(_) => {
1255             // Fallback for outdated server, TODO remove/bubble up with 2.0
1256             true
1257         }
1258     };
1259
1260     let previous_manifest = if download_previous_manifest {
1261         match client.download_previous_manifest().await {
1262             Ok(previous_manifest) => {
1263                 match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
1264                     Ok(()) => Some(Arc::new(previous_manifest)),
1265                     Err(err) => {
1266                         println!("Couldn't re-use previous manifest - {}", err);
1267                         None
1268                     }
1269                 }
1270             }
1271             Err(err) => {
1272                 println!("Couldn't download previous manifest - {}", err);
1273                 None
1274             }
1275         }
1276     } else {
1277         None
1278     };
1279
1280     let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
1281     let mut manifest = BackupManifest::new(snapshot);
1282
1283     let mut catalog = None;
1284     let mut catalog_result_rx = None;
1285
1286     for (backup_type, filename, target, size) in upload_list {
1287         match backup_type {
1288             BackupSpecificationType::CONFIG => {
1289                 let upload_options = UploadOptions {
1290                     compress: true,
1291                     encrypt: crypto.mode == CryptMode::Encrypt,
1292                     ..UploadOptions::default()
1293                 };
1294
1295                 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
1296                 let stats = client
1297                     .upload_blob_from_file(&filename, &target, upload_options)
1298                     .await?;
1299                 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
1300             }
1301             BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
1302                 let upload_options = UploadOptions {
1303                     compress: true,
1304                     encrypt: crypto.mode == CryptMode::Encrypt,
1305                     ..UploadOptions::default()
1306                 };
1307
1308                 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
1309                 let stats = client
1310                     .upload_blob_from_file(&filename, &target, upload_options)
1311                     .await?;
1312                 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
1313             }
1314             BackupSpecificationType::PXAR => {
1315                 // start catalog upload on first use
1316                 if catalog.is_none() {
1317                     let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
1318                     catalog = Some(catalog_upload_res.catalog_writer);
1319                     catalog_result_rx = Some(catalog_upload_res.result);
1320                 }
1321                 let catalog = catalog.as_ref().unwrap();
1322
1323                 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
1324                 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
1325
1326                 let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
1327                     device_set: devices.clone(),
1328                     patterns: pattern_list.clone(),
1329                     entries_max: entries_max as usize,
1330                     skip_lost_and_found,
1331                     verbose,
1332                 };
1333
1334                 let upload_options = UploadOptions {
1335                     previous_manifest: previous_manifest.clone(),
1336                     compress: true,
1337                     encrypt: crypto.mode == CryptMode::Encrypt,
1338                     ..UploadOptions::default()
1339                 };
1340
1341                 let stats = backup_directory(
1342                     &client,
1343                     &filename,
1344                     &target,
1345                     chunk_size_opt,
1346                     catalog.clone(),
1347                     pxar_options,
1348                     upload_options,
1349                 ).await?;
1350                 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
1351                 catalog.lock().unwrap().end_directory()?;
1352             }
1353             BackupSpecificationType::IMAGE => {
1354                 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
1355
1356                 let upload_options = UploadOptions {
1357                     previous_manifest: previous_manifest.clone(),
1358                     fixed_size: Some(size),
1359                     compress: true,
1360                     encrypt: crypto.mode == CryptMode::Encrypt,
1361                 };
1362
1363                 let stats = backup_image(
1364                     &client,
1365                     &filename,
1366                     &target,
1367                     chunk_size_opt,
1368                     upload_options,
1369                 ).await?;
1370                 manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
1371             }
1372         }
1373     }
1374
1375     // finalize and upload catalog
1376     if let Some(catalog) = catalog {
1377         let mutex = Arc::try_unwrap(catalog)
1378             .map_err(|_| format_err!("unable to get catalog (still used)"))?;
1379         let mut catalog = mutex.into_inner().unwrap();
1380
1381         catalog.finish()?;
1382
1383         drop(catalog); // close upload stream
1384
1385         if let Some(catalog_result_rx) = catalog_result_rx {
1386             let stats = catalog_result_rx.await??;
1387             manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
1388         }
1389     }
1390
1391     if let Some(rsa_encrypted_key) = rsa_encrypted_key {
1392         let target = ENCRYPTED_KEY_BLOB_NAME;
1393         println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
1394         let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
1395         let stats = client
1396             .upload_blob_from_data(rsa_encrypted_key, target, options)
1397             .await?;
1398         manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
1399
1400     }
1401     // create manifest (index.json)
1402     // manifests are never encrypted, but include a signature
1403     let manifest = manifest.to_string(crypt_config.as_ref().map(Arc::as_ref))
1404         .map_err(|err| format_err!("unable to format manifest - {}", err))?;
1405
1406
1407     if verbose { println!("Upload index.json to '{}'", repo) };
1408     let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
1409     client
1410         .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
1411         .await?;
1412
1413     client.finish().await?;
1414
1415     let end_time = std::time::Instant::now();
1416     let elapsed = end_time.duration_since(start_time);
1417     println!("Duration: {:.2}s", elapsed.as_secs_f64());
1418
1419     println!("End Time: {}", strftime_local("%c", epoch_i64())?);
1420
1421     Ok(Value::Null)
1422 }
1423
1424 fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1425
1426     let mut result = vec![];
1427
1428     let data: Vec<&str> = arg.splitn(2, ':').collect();
1429
1430     if data.len() != 2 {
1431         result.push(String::from("root.pxar:/"));
1432         result.push(String::from("etc.pxar:/etc"));
1433         return result;
1434     }
1435
1436     let files = tools::complete_file_name(data[1], param);
1437
1438     for file in files {
1439         result.push(format!("{}:{}", data[0], file));
1440     }
1441
1442     result
1443 }
1444
1445 async fn dump_image<W: Write>(
1446     client: Arc<BackupReader>,
1447     crypt_config: Option<Arc<CryptConfig>>,
1448     crypt_mode: CryptMode,
1449     index: FixedIndexReader,
1450     mut writer: W,
1451     verbose: bool,
1452 ) -> Result<(), Error> {
1453
1454     let most_used = index.find_most_used_chunks(8);
1455
1456     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
1457
1458     // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
1459     // and thus slows down reading. Instead, directly use RemoteChunkReader
1460     let mut per = 0;
1461     let mut bytes = 0;
1462     let start_time = std::time::Instant::now();
1463
1464     for pos in 0..index.index_count() {
1465         let digest = index.index_digest(pos).unwrap();
1466         let raw_data = chunk_reader.read_chunk(&digest).await?;
1467         writer.write_all(&raw_data)?;
1468         bytes += raw_data.len();
1469         if verbose {
1470             let next_per = ((pos+1)*100)/index.index_count();
1471             if per != next_per {
1472                 eprintln!("progress {}% (read {} bytes, duration {} sec)",
1473                           next_per, bytes, start_time.elapsed().as_secs());
1474                 per = next_per;
1475             }
1476         }
1477     }
1478
1479     let end_time = std::time::Instant::now();
1480     let elapsed = end_time.duration_since(start_time);
1481     eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
1482               bytes,
1483               elapsed.as_secs_f64(),
1484               bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
1485     );
1486
1487
1488     Ok(())
1489 }
1490
1491 fn parse_archive_type(name: &str) -> (String, ArchiveType) {
1492     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
1493         (name.into(), archive_type(name).unwrap())
1494     } else if name.ends_with(".pxar") {
1495         (format!("{}.didx", name), ArchiveType::DynamicIndex)
1496     } else if name.ends_with(".img") {
1497         (format!("{}.fidx", name), ArchiveType::FixedIndex)
1498     } else {
1499         (format!("{}.blob", name), ArchiveType::Blob)
1500     }
1501 }
1502
1503 #[api(
1504    input: {
1505        properties: {
1506            repository: {
1507                schema: REPO_URL_SCHEMA,
1508                optional: true,
1509            },
1510            snapshot: {
1511                type: String,
1512                description: "Group/Snapshot path.",
1513            },
1514            "archive-name": {
1515                description: "Backup archive name.",
1516                type: String,
1517            },
1518            target: {
1519                type: String,
1520                description: r###"Target directory path. Use '-' to write to standard output.
1521
1522 We do not extraxt '.pxar' archives when writing to standard output.
1523
1524 "###
1525            },
1526            "allow-existing-dirs": {
1527                type: Boolean,
1528                description: "Do not fail if directories already exists.",
1529                optional: true,
1530            },
1531            keyfile: {
1532                schema: KEYFILE_SCHEMA,
1533                optional: true,
1534            },
1535            "keyfd": {
1536                schema: KEYFD_SCHEMA,
1537                optional: true,
1538            },
1539            "crypt-mode": {
1540                type: CryptMode,
1541                optional: true,
1542            },
1543        }
1544    }
1545 )]
1546 /// Restore backup repository.
1547 async fn restore(param: Value) -> Result<Value, Error> {
1548     let repo = extract_repository_from_value(&param)?;
1549
1550     let verbose = param["verbose"].as_bool().unwrap_or(false);
1551
1552     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
1553
1554     let archive_name = tools::required_string_param(&param, "archive-name")?;
1555
1556     let client = connect(&repo)?;
1557
1558     record_repository(&repo);
1559
1560     let path = tools::required_string_param(&param, "snapshot")?;
1561
1562     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1563         let group: BackupGroup = path.parse()?;
1564         api_datastore_latest_snapshot(&client, repo.store(), group).await?
1565     } else {
1566         let snapshot: BackupDir = path.parse()?;
1567         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1568     };
1569
1570     let target = tools::required_string_param(&param, "target")?;
1571     let target = if target == "-" { None } else { Some(target) };
1572
1573     let crypto = crypto_parameters(&param)?;
1574
1575     let crypt_config = match crypto.enc_key {
1576         None => None,
1577         Some(key) => {
1578             let (key, _, fingerprint) = decrypt_key(&key, &key::get_encryption_key_password)?;
1579             eprintln!("Encryption key fingerprint: '{}'", fingerprint);
1580             Some(Arc::new(CryptConfig::new(key)?))
1581         }
1582     };
1583
1584     let client = BackupReader::start(
1585         client,
1586         crypt_config.clone(),
1587         repo.store(),
1588         &backup_type,
1589         &backup_id,
1590         backup_time,
1591         true,
1592     ).await?;
1593
1594     let (archive_name, archive_type) = parse_archive_type(archive_name);
1595
1596     let (manifest, backup_index_data) = client.download_manifest().await?;
1597
1598     if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
1599         eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
1600     } else {
1601         manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
1602     }
1603
1604     if archive_name == MANIFEST_BLOB_NAME {
1605         if let Some(target) = target {
1606             replace_file(target, &backup_index_data, CreateOptions::new())?;
1607         } else {
1608             let stdout = std::io::stdout();
1609             let mut writer = stdout.lock();
1610             writer.write_all(&backup_index_data)
1611                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1612         }
1613
1614         return Ok(Value::Null);
1615     }
1616
1617     let file_info = manifest.lookup_file_info(&archive_name)?;
1618
1619     if archive_type == ArchiveType::Blob {
1620
1621         let mut reader = client.download_blob(&manifest, &archive_name).await?;
1622
1623         if let Some(target) = target {
1624            let mut writer = std::fs::OpenOptions::new()
1625                 .write(true)
1626                 .create(true)
1627                 .create_new(true)
1628                 .open(target)
1629                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1630             std::io::copy(&mut reader, &mut writer)?;
1631         } else {
1632             let stdout = std::io::stdout();
1633             let mut writer = stdout.lock();
1634             std::io::copy(&mut reader, &mut writer)
1635                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1636         }
1637
1638     } else if archive_type == ArchiveType::DynamicIndex {
1639
1640         let index = client.download_dynamic_index(&manifest, &archive_name).await?;
1641
1642         let most_used = index.find_most_used_chunks(8);
1643
1644         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
1645
1646         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1647
1648         let options = proxmox_backup::pxar::PxarExtractOptions {
1649             match_list: &[],
1650             extract_match_default: true,
1651             allow_existing_dirs,
1652             on_error: None,
1653         };
1654
1655         if let Some(target) = target {
1656             proxmox_backup::pxar::extract_archive(
1657                 pxar::decoder::Decoder::from_std(reader)?,
1658                 Path::new(target),
1659                 proxmox_backup::pxar::Flags::DEFAULT,
1660                 |path| {
1661                     if verbose {
1662                         println!("{:?}", path);
1663                     }
1664                 },
1665                 options,
1666             )
1667             .map_err(|err| format_err!("error extracting archive - {}", err))?;
1668         } else {
1669             let mut writer = std::fs::OpenOptions::new()
1670                 .write(true)
1671                 .open("/dev/stdout")
1672                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
1673
1674             std::io::copy(&mut reader, &mut writer)
1675                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1676         }
1677     } else if archive_type == ArchiveType::FixedIndex {
1678
1679         let index = client.download_fixed_index(&manifest, &archive_name).await?;
1680
1681         let mut writer = if let Some(target) = target {
1682             std::fs::OpenOptions::new()
1683                 .write(true)
1684                 .create(true)
1685                 .create_new(true)
1686                 .open(target)
1687                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
1688         } else {
1689             std::fs::OpenOptions::new()
1690                 .write(true)
1691                 .open("/dev/stdout")
1692                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1693         };
1694
1695         dump_image(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), index, &mut writer, verbose).await?;
1696     }
1697
1698     Ok(Value::Null)
1699 }
1700
1701 const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
1702     &ApiHandler::Async(&prune),
1703     &ObjectSchema::new(
1704         "Prune a backup repository.",
1705         &proxmox_backup::add_common_prune_prameters!([
1706             ("dry-run", true, &BooleanSchema::new(
1707                 "Just show what prune would do, but do not delete anything.")
1708              .schema()),
1709             ("group", false, &StringSchema::new("Backup group.").schema()),
1710         ], [
1711             ("output-format", true, &OUTPUT_FORMAT),
1712             (
1713                 "quiet",
1714                 true,
1715                 &BooleanSchema::new("Minimal output - only show removals.")
1716                     .schema()
1717             ),
1718             ("repository", true, &REPO_URL_SCHEMA),
1719         ])
1720     )
1721 );
1722
1723 fn prune<'a>(
1724     param: Value,
1725     _info: &ApiMethod,
1726     _rpcenv: &'a mut dyn RpcEnvironment,
1727 ) -> proxmox::api::ApiFuture<'a> {
1728     async move {
1729         prune_async(param).await
1730     }.boxed()
1731 }
1732
1733 async fn prune_async(mut param: Value) -> Result<Value, Error> {
1734     let repo = extract_repository_from_value(&param)?;
1735
1736     let mut client = connect(&repo)?;
1737
1738     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
1739
1740     let group = tools::required_string_param(&param, "group")?;
1741     let group: BackupGroup = group.parse()?;
1742
1743     let output_format = get_output_format(&param);
1744
1745     let quiet = param["quiet"].as_bool().unwrap_or(false);
1746
1747     param.as_object_mut().unwrap().remove("repository");
1748     param.as_object_mut().unwrap().remove("group");
1749     param.as_object_mut().unwrap().remove("output-format");
1750     param.as_object_mut().unwrap().remove("quiet");
1751
1752     param["backup-type"] = group.backup_type().into();
1753     param["backup-id"] = group.backup_id().into();
1754
1755     let mut result = client.post(&path, Some(param)).await?;
1756
1757     record_repository(&repo);
1758
1759     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
1760         let item: PruneListItem = serde_json::from_value(record.to_owned())?;
1761         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
1762         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
1763     };
1764
1765     let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
1766         Ok(match v.as_bool() {
1767             Some(true) => "keep",
1768             Some(false) => "remove",
1769             None => "unknown",
1770         }.to_string())
1771     };
1772
1773     let options = default_table_format_options()
1774         .sortby("backup-type", false)
1775         .sortby("backup-id", false)
1776         .sortby("backup-time", false)
1777         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
1778         .column(ColumnConfig::new("backup-time").renderer(tools::format::render_epoch).header("date"))
1779         .column(ColumnConfig::new("keep").renderer(render_prune_action).header("action"))
1780         ;
1781
1782     let return_type = &proxmox_backup::api2::admin::datastore::API_METHOD_PRUNE.returns;
1783
1784     let mut data = result["data"].take();
1785
1786     if quiet {
1787         let list: Vec<Value> = data.as_array().unwrap().iter().filter(|item| {
1788             item["keep"].as_bool() == Some(false)
1789         }).cloned().collect();
1790         data = list.into();
1791     }
1792
1793     format_and_print_result_full(&mut data, return_type, &output_format, &options);
1794
1795     Ok(Value::Null)
1796 }
1797
1798 #[api(
1799    input: {
1800        properties: {
1801            repository: {
1802                schema: REPO_URL_SCHEMA,
1803                optional: true,
1804            },
1805            "output-format": {
1806                schema: OUTPUT_FORMAT,
1807                optional: true,
1808            },
1809        }
1810    },
1811     returns: {
1812         type: StorageStatus,
1813     },
1814 )]
1815 /// Get repository status.
1816 async fn status(param: Value) -> Result<Value, Error> {
1817
1818     let repo = extract_repository_from_value(&param)?;
1819
1820     let output_format = get_output_format(&param);
1821
1822     let client = connect(&repo)?;
1823
1824     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1825
1826     let mut result = client.get(&path, None).await?;
1827     let mut data = result["data"].take();
1828
1829     record_repository(&repo);
1830
1831     let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
1832         let v = v.as_u64().unwrap();
1833         let total = record["total"].as_u64().unwrap();
1834         let roundup = total/200;
1835         let per = ((v+roundup)*100)/total;
1836         let info = format!(" ({} %)", per);
1837         Ok(format!("{} {:>8}", v, info))
1838     };
1839
1840     let options = default_table_format_options()
1841         .noheader(true)
1842         .column(ColumnConfig::new("total").renderer(render_total_percentage))
1843         .column(ColumnConfig::new("used").renderer(render_total_percentage))
1844         .column(ColumnConfig::new("avail").renderer(render_total_percentage));
1845
1846     let return_type = &API_METHOD_STATUS.returns;
1847
1848     format_and_print_result_full(&mut data, return_type, &output_format, &options);
1849
1850     Ok(Value::Null)
1851 }
1852
1853 // like get, but simply ignore errors and return Null instead
1854 async fn try_get(repo: &BackupRepository, url: &str) -> Value {
1855
1856     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
1857     let password = std::env::var(ENV_VAR_PBS_PASSWORD).ok();
1858
1859     // ticket cache, but no questions asked
1860     let options = HttpClientOptions::new_interactive(password, fingerprint)
1861         .interactive(false);
1862
1863     let client = match HttpClient::new(repo.host(), repo.port(), repo.auth_id(), options) {
1864         Ok(v) => v,
1865         _ => return Value::Null,
1866     };
1867
1868     let mut resp = match client.get(url, None).await {
1869         Ok(v) => v,
1870         _ => return Value::Null,
1871     };
1872
1873     if let Some(map) = resp.as_object_mut() {
1874         if let Some(data) = map.remove("data") {
1875             return data;
1876         }
1877     }
1878     Value::Null
1879 }
1880
1881 fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1882     proxmox_backup::tools::runtime::main(async { complete_backup_group_do(param).await })
1883 }
1884
1885 async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
1886
1887     let mut result = vec![];
1888
1889     let repo = match extract_repository_from_map(param) {
1890         Some(v) => v,
1891         _ => return result,
1892     };
1893
1894     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
1895
1896     let data = try_get(&repo, &path).await;
1897
1898     if let Some(list) = data.as_array() {
1899         for item in list {
1900             if let (Some(backup_id), Some(backup_type)) =
1901                 (item["backup-id"].as_str(), item["backup-type"].as_str())
1902             {
1903                 result.push(format!("{}/{}", backup_type, backup_id));
1904             }
1905         }
1906     }
1907
1908     result
1909 }
1910
1911 pub fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1912     proxmox_backup::tools::runtime::main(async { complete_group_or_snapshot_do(arg, param).await })
1913 }
1914
1915 async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1916
1917     if arg.matches('/').count() < 2 {
1918         let groups = complete_backup_group_do(param).await;
1919         let mut result = vec![];
1920         for group in groups {
1921             result.push(group.to_string());
1922             result.push(format!("{}/", group));
1923         }
1924         return result;
1925     }
1926
1927     complete_backup_snapshot_do(param).await
1928 }
1929
1930 fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1931     proxmox_backup::tools::runtime::main(async { complete_backup_snapshot_do(param).await })
1932 }
1933
1934 async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
1935
1936     let mut result = vec![];
1937
1938     let repo = match extract_repository_from_map(param) {
1939         Some(v) => v,
1940         _ => return result,
1941     };
1942
1943     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1944
1945     let data = try_get(&repo, &path).await;
1946
1947     if let Some(list) = data.as_array() {
1948         for item in list {
1949             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
1950                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
1951             {
1952                 if let Ok(snapshot) = BackupDir::new(backup_type, backup_id, backup_time) {
1953                     result.push(snapshot.relative_path().to_str().unwrap().to_owned());
1954                 }
1955             }
1956         }
1957     }
1958
1959     result
1960 }
1961
1962 fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1963     proxmox_backup::tools::runtime::main(async { complete_server_file_name_do(param).await })
1964 }
1965
1966 async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
1967
1968     let mut result = vec![];
1969
1970     let repo = match extract_repository_from_map(param) {
1971         Some(v) => v,
1972         _ => return result,
1973     };
1974
1975     let snapshot: BackupDir = match param.get("snapshot") {
1976         Some(path) => {
1977             match path.parse() {
1978                 Ok(v) => v,
1979                 _ => return result,
1980             }
1981         }
1982         _ => return result,
1983     };
1984
1985     let query = tools::json_object_to_query(json!({
1986         "backup-type": snapshot.group().backup_type(),
1987         "backup-id": snapshot.group().backup_id(),
1988         "backup-time": snapshot.backup_time(),
1989     })).unwrap();
1990
1991     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
1992
1993     let data = try_get(&repo, &path).await;
1994
1995     if let Some(list) = data.as_array() {
1996         for item in list {
1997             if let Some(filename) = item["filename"].as_str() {
1998                 result.push(filename.to_owned());
1999             }
2000         }
2001     }
2002
2003     result
2004 }
2005
2006 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
2007     complete_server_file_name(arg, param)
2008         .iter()
2009         .map(|v| tools::format::strip_server_file_extension(&v))
2010         .collect()
2011 }
2012
2013 pub fn complete_pxar_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
2014     complete_server_file_name(arg, param)
2015         .iter()
2016         .filter_map(|name| {
2017             if name.ends_with(".pxar.didx") {
2018                 Some(tools::format::strip_server_file_extension(name))
2019             } else {
2020                 None
2021             }
2022         })
2023         .collect()
2024 }
2025
2026 pub fn complete_img_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
2027     complete_server_file_name(arg, param)
2028         .iter()
2029         .filter_map(|name| {
2030             if name.ends_with(".img.fidx") {
2031                 Some(tools::format::strip_server_file_extension(name))
2032             } else {
2033                 None
2034             }
2035         })
2036         .collect()
2037 }
2038
2039 fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
2040
2041     let mut result = vec![];
2042
2043     let mut size = 64;
2044     loop {
2045         result.push(size.to_string());
2046         size *= 2;
2047         if size > 4096 { break; }
2048     }
2049
2050     result
2051 }
2052
2053 fn complete_auth_id(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
2054     proxmox_backup::tools::runtime::main(async { complete_auth_id_do(param).await })
2055 }
2056
2057 async fn complete_auth_id_do(param: &HashMap<String, String>) -> Vec<String> {
2058
2059     let mut result = vec![];
2060
2061     let repo = match extract_repository_from_map(param) {
2062         Some(v) => v,
2063         _ => return result,
2064     };
2065
2066     let data = try_get(&repo, "api2/json/access/users?include_tokens=true").await;
2067
2068     if let Ok(parsed) = serde_json::from_value::<Vec<UserWithTokens>>(data) {
2069         for user in parsed {
2070             result.push(user.userid.to_string());
2071             for token in user.tokens {
2072                 result.push(token.tokenid.to_string());
2073             }
2074         }
2075     };
2076
2077     result
2078 }
2079
2080 use proxmox_backup::client::RemoteChunkReader;
2081 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
2082 /// async use!
2083 ///
2084 /// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
2085 /// so that we can properly access it from multiple threads simultaneously while not issuing
2086 /// duplicate simultaneous reads over http.
2087 pub struct BufferedDynamicReadAt {
2088     inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
2089 }
2090
2091 impl BufferedDynamicReadAt {
2092     fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
2093         Self {
2094             inner: Mutex::new(inner),
2095         }
2096     }
2097 }
2098
2099 impl ReadAt for BufferedDynamicReadAt {
2100     fn start_read_at<'a>(
2101         self: Pin<&'a Self>,
2102         _cx: &mut Context,
2103         buf: &'a mut [u8],
2104         offset: u64,
2105     ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
2106         MaybeReady::Ready(tokio::task::block_in_place(move || {
2107             let mut reader = self.inner.lock().unwrap();
2108             reader.seek(SeekFrom::Start(offset))?;
2109             Ok(reader.read(buf)?)
2110         }))
2111     }
2112
2113     fn poll_complete<'a>(
2114         self: Pin<&'a Self>,
2115         _op: ReadAtOperation<'a>,
2116     ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
2117         panic!("LocalDynamicReadAt::start_read_at returned Pending");
2118     }
2119 }
2120
2121 fn main() {
2122
2123     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
2124         .arg_param(&["backupspec"])
2125         .completion_cb("repository", complete_repository)
2126         .completion_cb("backupspec", complete_backup_source)
2127         .completion_cb("keyfile", tools::complete_file_name)
2128         .completion_cb("master-pubkey-file", tools::complete_file_name)
2129         .completion_cb("chunk-size", complete_chunk_size);
2130
2131     let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
2132         .completion_cb("repository", complete_repository)
2133         .completion_cb("keyfile", tools::complete_file_name);
2134
2135     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
2136         .completion_cb("repository", complete_repository);
2137
2138     let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
2139         .completion_cb("repository", complete_repository);
2140
2141     let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
2142         .arg_param(&["snapshot", "archive-name", "target"])
2143         .completion_cb("repository", complete_repository)
2144         .completion_cb("snapshot", complete_group_or_snapshot)
2145         .completion_cb("archive-name", complete_archive_name)
2146         .completion_cb("target", tools::complete_file_name);
2147
2148     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
2149         .arg_param(&["group"])
2150         .completion_cb("group", complete_backup_group)
2151         .completion_cb("repository", complete_repository);
2152
2153     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
2154         .completion_cb("repository", complete_repository);
2155
2156     let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
2157         .completion_cb("repository", complete_repository);
2158
2159     let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
2160         .completion_cb("repository", complete_repository);
2161
2162     let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
2163         .completion_cb("repository", complete_repository);
2164
2165     let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
2166         .arg_param(&["group", "new-owner"])
2167         .completion_cb("group", complete_backup_group)
2168         .completion_cb("new-owner",  complete_auth_id)
2169         .completion_cb("repository", complete_repository);
2170
2171     let cmd_def = CliCommandMap::new()
2172         .insert("backup", backup_cmd_def)
2173         .insert("garbage-collect", garbage_collect_cmd_def)
2174         .insert("list", list_cmd_def)
2175         .insert("login", login_cmd_def)
2176         .insert("logout", logout_cmd_def)
2177         .insert("prune", prune_cmd_def)
2178         .insert("restore", restore_cmd_def)
2179         .insert("snapshot", snapshot_mgtm_cli())
2180         .insert("status", status_cmd_def)
2181         .insert("key", key::cli())
2182         .insert("mount", mount_cmd_def())
2183         .insert("map", map_cmd_def())
2184         .insert("unmap", unmap_cmd_def())
2185         .insert("catalog", catalog_mgmt_cli())
2186         .insert("task", task_mgmt_cli())
2187         .insert("version", version_cmd_def)
2188         .insert("benchmark", benchmark_cmd_def)
2189         .insert("change-owner", change_owner_cmd_def)
2190
2191         .alias(&["files"], &["snapshot", "files"])
2192         .alias(&["forget"], &["snapshot", "forget"])
2193         .alias(&["upload-log"], &["snapshot", "upload-log"])
2194         .alias(&["snapshots"], &["snapshot", "list"])
2195         ;
2196
2197     let rpcenv = CliEnvironment::new();
2198     run_cli_command(cmd_def, rpcenv, Some(|future| {
2199         proxmox_backup::tools::runtime::main(future)
2200     }));
2201 }