src/bin/proxmox-backup-client.rs

   1 use failure::*;
   2 use nix::unistd::{fork, ForkResult, pipe};
   3 use std::os::unix::io::RawFd;
   4 use chrono::{Local, Utc, TimeZone};
   5 use std::path::{Path, PathBuf};
   6 use std::collections::{HashSet, HashMap};
   7 use std::ffi::OsStr;
   8 use std::io::{Write, Seek, SeekFrom};
   9 use std::os::unix::fs::OpenOptionsExt;
  10
  11 use proxmox::{sortable, identity};
  12 use proxmox::tools::fs::{file_get_contents, file_get_json, file_set_contents, image_size};
  13 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
  14 use proxmox::api::schema::*;
  15
  16 use proxmox_backup::tools;
  17 use proxmox_backup::cli::*;
  18 use proxmox_backup::api2::types::*;
  19 use proxmox_backup::client::*;
  20 use proxmox_backup::backup::*;
  21 use proxmox_backup::pxar::{ self, catalog::* };
  22
  23 //use proxmox_backup::backup::image_index::*;
  24 //use proxmox_backup::config::datastore;
  25 //use proxmox_backup::pxar::encoder::*;
  26 //use proxmox_backup::backup::datastore::*;
  27
  28 use serde_json::{json, Value};
  29 //use hyper::Body;
  30 use std::sync::{Arc, Mutex};
  31 //use regex::Regex;
  32 use xdg::BaseDirectories;
  33
  34 use futures::*;
  35 use tokio::sync::mpsc;
  36
  37 proxmox::api::const_regex! {
  38     BACKUPSPEC_REGEX = r"^([a-zA-Z0-9_-]+\.(?:pxar|img|conf|log)):(.+)$";
  39 }
  40
  41 const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  42     .format(&BACKUP_REPO_URL)
  43     .max_length(256)
  44     .schema();
  45
  46 fn get_default_repository() -> Option<String> {
  47     std::env::var("PBS_REPOSITORY").ok()
  48 }
  49
  50 fn extract_repository_from_value(
  51     param: &Value,
  52 ) -> Result<BackupRepository, Error> {
  53
  54     let repo_url = param["repository"]
  55         .as_str()
  56         .map(String::from)
  57         .or_else(get_default_repository)
  58         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
  59
  60     let repo: BackupRepository = repo_url.parse()?;
  61
  62     Ok(repo)
  63 }
  64
  65 fn extract_repository_from_map(
  66     param: &HashMap<String, String>,
  67 ) -> Option<BackupRepository> {
  68
  69     param.get("repository")
  70         .map(String::from)
  71         .or_else(get_default_repository)
  72         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
  73 }
  74
  75 fn record_repository(repo: &BackupRepository) {
  76
  77     let base = match BaseDirectories::with_prefix("proxmox-backup") {
  78         Ok(v) => v,
  79         _ => return,
  80     };
  81
  82     // usually $HOME/.cache/proxmox-backup/repo-list
  83     let path = match base.place_cache_file("repo-list") {
  84         Ok(v) => v,
  85         _ => return,
  86     };
  87
  88     let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
  89
  90     let repo = repo.to_string();
  91
  92     data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
  93
  94     let mut map = serde_json::map::Map::new();
  95
  96     loop {
  97         let mut max_used = 0;
  98         let mut max_repo = None;
  99         for (repo, count) in data.as_object().unwrap() {
 100             if map.contains_key(repo) { continue; }
 101             if let Some(count) = count.as_i64() {
 102                 if count > max_used {
 103                     max_used = count;
 104                     max_repo = Some(repo);
 105                 }
 106             }
 107         }
 108         if let Some(repo) = max_repo {
 109             map.insert(repo.to_owned(), json!(max_used));
 110         } else {
 111             break;
 112         }
 113         if map.len() > 10 { // store max. 10 repos
 114             break;
 115         }
 116     }
 117
 118     let new_data = json!(map);
 119
 120     let _ = file_set_contents(path, new_data.to_string().as_bytes(), None);
 121 }
 122
 123 fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 124
 125     let mut result = vec![];
 126
 127     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 128         Ok(v) => v,
 129         _ => return result,
 130     };
 131
 132     // usually $HOME/.cache/proxmox-backup/repo-list
 133     let path = match base.place_cache_file("repo-list") {
 134         Ok(v) => v,
 135         _ => return result,
 136     };
 137
 138     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 139
 140     if let Some(map) = data.as_object() {
 141         for (repo, _count) in map {
 142             result.push(repo.to_owned());
 143         }
 144     }
 145
 146     result
 147 }
 148
 149 async fn backup_directory<P: AsRef<Path>>(
 150     client: &BackupWriter,
 151     dir_path: P,
 152     archive_name: &str,
 153     chunk_size: Option<usize>,
 154     device_set: Option<HashSet<u64>>,
 155     verbose: bool,
 156     skip_lost_and_found: bool,
 157     crypt_config: Option<Arc<CryptConfig>>,
 158     catalog: Arc<Mutex<CatalogWriter<SenderWriter>>>,
 159 ) -> Result<BackupStats, Error> {
 160
 161     let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found, catalog)?;
 162     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 163
 164     let (mut tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
 165
 166     let stream = rx
 167         .map_err(Error::from);
 168
 169     // spawn chunker inside a separate task so that it can run parallel
 170     tokio::spawn(async move {
 171         let _ = tx.send_all(&mut chunk_stream).await;
 172     });
 173
 174     let stats = client
 175         .upload_stream(archive_name, stream, "dynamic", None, crypt_config)
 176         .await?;
 177
 178     Ok(stats)
 179 }
 180
 181 async fn backup_image<P: AsRef<Path>>(
 182     client: &BackupWriter,
 183     image_path: P,
 184     archive_name: &str,
 185     image_size: u64,
 186     chunk_size: Option<usize>,
 187     _verbose: bool,
 188     crypt_config: Option<Arc<CryptConfig>>,
 189 ) -> Result<BackupStats, Error> {
 190
 191     let path = image_path.as_ref().to_owned();
 192
 193     let file = tokio::fs::File::open(path).await?;
 194
 195     let stream = tokio::codec::FramedRead::new(file, tokio::codec::BytesCodec::new())
 196         .map_err(Error::from);
 197
 198     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 199
 200     let stats = client
 201         .upload_stream(archive_name, stream, "fixed", Some(image_size), crypt_config)
 202         .await?;
 203
 204     Ok(stats)
 205 }
 206
 207 fn strip_server_file_expenstion(name: &str) -> String {
 208
 209     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
 210         name[..name.len()-5].to_owned()
 211     } else {
 212         name.to_owned() // should not happen
 213     }
 214 }
 215
 216 fn list_backup_groups(
 217     param: Value,
 218     _info: &ApiMethod,
 219     _rpcenv: &mut dyn RpcEnvironment,
 220 ) -> Result<Value, Error> {
 221
 222     let repo = extract_repository_from_value(&param)?;
 223
 224     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 225
 226     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 227
 228     let mut result = async_main(async move {
 229         client.get(&path, None).await
 230     })?;
 231
 232     record_repository(&repo);
 233
 234     // fixme: implement and use output formatter instead ..
 235     let list = result["data"].as_array_mut().unwrap();
 236
 237     list.sort_unstable_by(|a, b| {
 238         let a_id = a["backup-id"].as_str().unwrap();
 239         let a_backup_type = a["backup-type"].as_str().unwrap();
 240         let b_id = b["backup-id"].as_str().unwrap();
 241         let b_backup_type = b["backup-type"].as_str().unwrap();
 242
 243         let type_order = a_backup_type.cmp(b_backup_type);
 244         if type_order == std::cmp::Ordering::Equal {
 245             a_id.cmp(b_id)
 246         } else {
 247             type_order
 248         }
 249     });
 250
 251     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 252
 253     let mut result = vec![];
 254
 255     for item in list {
 256
 257         let id = item["backup-id"].as_str().unwrap();
 258         let btype = item["backup-type"].as_str().unwrap();
 259         let epoch = item["last-backup"].as_i64().unwrap();
 260         let last_backup = Utc.timestamp(epoch, 0);
 261         let backup_count = item["backup-count"].as_u64().unwrap();
 262
 263         let group = BackupGroup::new(btype, id);
 264
 265         let path = group.group_path().to_str().unwrap().to_owned();
 266
 267         let files = item["files"].as_array().unwrap().iter()
 268             .map(|v| strip_server_file_expenstion(v.as_str().unwrap())).collect();
 269
 270         if output_format == "text" {
 271             println!(
 272                 "{:20} | {} | {:5} | {}",
 273                 path,
 274                 BackupDir::backup_time_to_string(last_backup),
 275                 backup_count,
 276                 tools::join(&files, ' '),
 277             );
 278         } else {
 279             result.push(json!({
 280                 "backup-type": btype,
 281                 "backup-id": id,
 282                 "last-backup": epoch,
 283                 "backup-count": backup_count,
 284                 "files": files,
 285             }));
 286         }
 287     }
 288
 289     if output_format != "text" { format_and_print_result(&result.into(), &output_format); }
 290
 291     Ok(Value::Null)
 292 }
 293
 294 fn list_snapshots(
 295     param: Value,
 296     _info: &ApiMethod,
 297     _rpcenv: &mut dyn RpcEnvironment,
 298 ) -> Result<Value, Error> {
 299
 300     let repo = extract_repository_from_value(&param)?;
 301
 302     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 303
 304     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 305
 306     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 307
 308     let mut args = json!({});
 309     if let Some(path) = param["group"].as_str() {
 310         let group = BackupGroup::parse(path)?;
 311         args["backup-type"] = group.backup_type().into();
 312         args["backup-id"] = group.backup_id().into();
 313     }
 314
 315     let result = async_main(async move {
 316         client.get(&path, Some(args)).await
 317     })?;
 318
 319     record_repository(&repo);
 320
 321     let list = result["data"].as_array().unwrap();
 322
 323     let mut result = vec![];
 324
 325     for item in list {
 326
 327         let id = item["backup-id"].as_str().unwrap();
 328         let btype = item["backup-type"].as_str().unwrap();
 329         let epoch = item["backup-time"].as_i64().unwrap();
 330
 331         let snapshot = BackupDir::new(btype, id, epoch);
 332
 333         let path = snapshot.relative_path().to_str().unwrap().to_owned();
 334
 335         let files = item["files"].as_array().unwrap().iter()
 336             .map(|v|  strip_server_file_expenstion(v.as_str().unwrap())).collect();
 337
 338         if output_format == "text" {
 339             let size_str = if let Some(size) = item["size"].as_u64() {
 340                 size.to_string()
 341             } else {
 342                 String::from("-")
 343             };
 344             println!("{} | {} | {}", path, size_str, tools::join(&files, ' '));
 345         } else {
 346             let mut data = json!({
 347                 "backup-type": btype,
 348                 "backup-id": id,
 349                 "backup-time": epoch,
 350                 "files": files,
 351             });
 352             if let Some(size) = item["size"].as_u64() {
 353                 data["size"] = size.into();
 354             }
 355             result.push(data);
 356         }
 357     }
 358
 359     if output_format != "text" { format_and_print_result(&result.into(), &output_format); }
 360
 361     Ok(Value::Null)
 362 }
 363
 364 fn forget_snapshots(
 365     param: Value,
 366     _info: &ApiMethod,
 367     _rpcenv: &mut dyn RpcEnvironment,
 368 ) -> Result<Value, Error> {
 369
 370     let repo = extract_repository_from_value(&param)?;
 371
 372     let path = tools::required_string_param(&param, "snapshot")?;
 373     let snapshot = BackupDir::parse(path)?;
 374
 375     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
 376
 377     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 378
 379     let result = async_main(async move {
 380         client.delete(&path, Some(json!({
 381             "backup-type": snapshot.group().backup_type(),
 382             "backup-id": snapshot.group().backup_id(),
 383             "backup-time": snapshot.backup_time().timestamp(),
 384         }))).await
 385     })?;
 386
 387     record_repository(&repo);
 388
 389     Ok(result)
 390 }
 391
 392 fn api_login(
 393     param: Value,
 394     _info: &ApiMethod,
 395     _rpcenv: &mut dyn RpcEnvironment,
 396 ) -> Result<Value, Error> {
 397
 398     let repo = extract_repository_from_value(&param)?;
 399
 400     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 401     async_main(async move { client.login().await })?;
 402
 403     record_repository(&repo);
 404
 405     Ok(Value::Null)
 406 }
 407
 408 fn api_logout(
 409     param: Value,
 410     _info: &ApiMethod,
 411     _rpcenv: &mut dyn RpcEnvironment,
 412 ) -> Result<Value, Error> {
 413
 414     let repo = extract_repository_from_value(&param)?;
 415
 416     delete_ticket_info(repo.host(), repo.user())?;
 417
 418     Ok(Value::Null)
 419 }
 420
 421 fn dump_catalog(
 422     param: Value,
 423     _info: &ApiMethod,
 424     _rpcenv: &mut dyn RpcEnvironment,
 425 ) -> Result<Value, Error> {
 426
 427     let repo = extract_repository_from_value(&param)?;
 428
 429     let path = tools::required_string_param(&param, "snapshot")?;
 430     let snapshot = BackupDir::parse(path)?;
 431
 432     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 433
 434     let crypt_config = match keyfile {
 435         None => None,
 436         Some(path) => {
 437             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 438             Some(Arc::new(CryptConfig::new(key)?))
 439         }
 440     };
 441
 442     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 443
 444     async_main(async move {
 445         let client = BackupReader::start(
 446             client,
 447             crypt_config.clone(),
 448             repo.store(),
 449             &snapshot.group().backup_type(),
 450             &snapshot.group().backup_id(),
 451             snapshot.backup_time(),
 452             true,
 453         ).await?;
 454
 455         let manifest = client.download_manifest().await?;
 456
 457         let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
 458
 459         let most_used = index.find_most_used_chunks(8);
 460
 461         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 462
 463         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 464
 465         let mut catalogfile = std::fs::OpenOptions::new()
 466             .write(true)
 467             .read(true)
 468             .custom_flags(libc::O_TMPFILE)
 469             .open("/tmp")?;
 470
 471         std::io::copy(&mut reader, &mut catalogfile)
 472             .map_err(|err| format_err!("unable to download catalog - {}", err))?;
 473
 474         catalogfile.seek(SeekFrom::Start(0))?;
 475
 476         let mut catalog_reader = CatalogReader::new(catalogfile);
 477
 478         catalog_reader.dump()?;
 479
 480         record_repository(&repo);
 481
 482         Ok::<(), Error>(())
 483     })?;
 484
 485     Ok(Value::Null)
 486 }
 487
 488 fn list_snapshot_files(
 489     param: Value,
 490     _info: &ApiMethod,
 491     _rpcenv: &mut dyn RpcEnvironment,
 492 ) -> Result<Value, Error> {
 493
 494     let repo = extract_repository_from_value(&param)?;
 495
 496     let path = tools::required_string_param(&param, "snapshot")?;
 497     let snapshot = BackupDir::parse(path)?;
 498
 499     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 500
 501     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 502
 503     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 504
 505     let mut result = async_main(async move {
 506         client.get(&path, Some(json!({
 507             "backup-type": snapshot.group().backup_type(),
 508             "backup-id": snapshot.group().backup_id(),
 509             "backup-time": snapshot.backup_time().timestamp(),
 510         }))).await
 511     })?;
 512
 513     record_repository(&repo);
 514
 515     let list: Value = result["data"].take();
 516
 517     if output_format == "text" {
 518         for item in list.as_array().unwrap().iter() {
 519             println!(
 520                 "{} {}",
 521                 strip_server_file_expenstion(item["filename"].as_str().unwrap()),
 522                 item["size"].as_u64().unwrap_or(0),
 523             );
 524         }
 525     } else {
 526         format_and_print_result(&list, &output_format);
 527     }
 528
 529     Ok(Value::Null)
 530 }
 531
 532 fn start_garbage_collection(
 533     param: Value,
 534     _info: &ApiMethod,
 535     _rpcenv: &mut dyn RpcEnvironment,
 536 ) -> Result<Value, Error> {
 537
 538     let repo = extract_repository_from_value(&param)?;
 539
 540     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
 541
 542     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 543
 544     let result = async_main(async move { client.post(&path, None).await })?;
 545
 546     record_repository(&repo);
 547
 548     Ok(result)
 549 }
 550
 551 fn parse_backupspec(value: &str) -> Result<(&str, &str), Error> {
 552
 553     if let Some(caps) = (BACKUPSPEC_REGEX.regex_obj)().captures(value) {
 554         return Ok((caps.get(1).unwrap().as_str(), caps.get(2).unwrap().as_str()));
 555     }
 556     bail!("unable to parse directory specification '{}'", value);
 557 }
 558
 559 fn spawn_catalog_upload(
 560     client: Arc<BackupWriter>,
 561     crypt_config: Option<Arc<CryptConfig>>,
 562 ) -> Result<
 563         (
 564             Arc<Mutex<CatalogWriter<SenderWriter>>>,
 565             tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>
 566         ), Error>
 567 {
 568     let (catalog_tx, catalog_rx) = mpsc::channel(10); // allow to buffer 10 writes
 569     let catalog_stream = catalog_rx.map_err(Error::from);
 570     let catalog_chunk_size = 512*1024;
 571     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 572
 573     let catalog = Arc::new(Mutex::new(CatalogWriter::new(SenderWriter::new(catalog_tx))?));
 574
 575     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 576
 577     tokio::spawn(async move {
 578         let catalog_upload_result = client
 579             .upload_stream(CATALOG_NAME, catalog_chunk_stream, "dynamic", None, crypt_config)
 580             .await;
 581
 582         if let Err(ref err) = catalog_upload_result {
 583             eprintln!("catalog upload error - {}", err);
 584             client.cancel();
 585         }
 586
 587         let _ = catalog_result_tx.send(catalog_upload_result);
 588     });
 589
 590     Ok((catalog, catalog_result_rx))
 591 }
 592
 593 fn create_backup(
 594     param: Value,
 595     _info: &ApiMethod,
 596     _rpcenv: &mut dyn RpcEnvironment,
 597 ) -> Result<Value, Error> {
 598
 599     let repo = extract_repository_from_value(&param)?;
 600
 601     let backupspec_list = tools::required_array_param(&param, "backupspec")?;
 602
 603     let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
 604
 605     let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
 606
 607     let verbose = param["verbose"].as_bool().unwrap_or(false);
 608
 609     let backup_time_opt = param["backup-time"].as_i64();
 610
 611     let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
 612
 613     if let Some(size) = chunk_size_opt {
 614         verify_chunk_size(size)?;
 615     }
 616
 617     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 618
 619     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 620
 621     let backup_type = param["backup-type"].as_str().unwrap_or("host");
 622
 623     let include_dev = param["include-dev"].as_array();
 624
 625     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
 626
 627     if let Some(include_dev) = include_dev {
 628         if all_file_systems {
 629             bail!("option 'all-file-systems' conflicts with option 'include-dev'");
 630         }
 631
 632         let mut set = HashSet::new();
 633         for path in include_dev {
 634             let path = path.as_str().unwrap();
 635             let stat = nix::sys::stat::stat(path)
 636                 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
 637             set.insert(stat.st_dev);
 638         }
 639         devices = Some(set);
 640     }
 641
 642     let mut upload_list = vec![];
 643
 644     enum BackupType { PXAR, IMAGE, CONFIG, LOGFILE };
 645
 646     let mut upload_catalog = false;
 647
 648     for backupspec in backupspec_list {
 649         let (target, filename) = parse_backupspec(backupspec.as_str().unwrap())?;
 650
 651         use std::os::unix::fs::FileTypeExt;
 652
 653         let metadata = std::fs::metadata(filename)
 654             .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
 655         let file_type = metadata.file_type();
 656
 657         let extension = target.rsplit('.').next()
 658             .ok_or_else(|| format_err!("missing target file extenion '{}'", target))?;
 659
 660         match extension {
 661             "pxar" => {
 662                 if !file_type.is_dir() {
 663                     bail!("got unexpected file type (expected directory)");
 664                 }
 665                 upload_list.push((BackupType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
 666                 upload_catalog = true;
 667             }
 668             "img" => {
 669
 670                 if !(file_type.is_file() || file_type.is_block_device()) {
 671                     bail!("got unexpected file type (expected file or block device)");
 672                 }
 673
 674                 let size = image_size(&PathBuf::from(filename))?;
 675
 676                 if size == 0 { bail!("got zero-sized file '{}'", filename); }
 677
 678                 upload_list.push((BackupType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
 679             }
 680             "conf" => {
 681                 if !file_type.is_file() {
 682                     bail!("got unexpected file type (expected regular file)");
 683                 }
 684                 upload_list.push((BackupType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 685             }
 686             "log" => {
 687                 if !file_type.is_file() {
 688                     bail!("got unexpected file type (expected regular file)");
 689                 }
 690                 upload_list.push((BackupType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 691             }
 692             _ => {
 693                 bail!("got unknown archive extension '{}'", extension);
 694             }
 695         }
 696     }
 697
 698     let backup_time = Utc.timestamp(backup_time_opt.unwrap_or_else(|| Utc::now().timestamp()), 0);
 699
 700     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 701     record_repository(&repo);
 702
 703     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time));
 704
 705     println!("Client name: {}", proxmox::tools::nodename());
 706
 707     let start_time = Local::now();
 708
 709     println!("Starting protocol: {}", start_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 710
 711     let (crypt_config, rsa_encrypted_key) = match keyfile {
 712         None => (None, None),
 713         Some(path) => {
 714             let (key, created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 715
 716             let crypt_config = CryptConfig::new(key)?;
 717
 718             let path = master_pubkey_path()?;
 719             if path.exists() {
 720                 let pem_data = file_get_contents(&path)?;
 721                 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
 722                 let enc_key = crypt_config.generate_rsa_encoded_key(rsa, created)?;
 723                 (Some(Arc::new(crypt_config)), Some(enc_key))
 724             } else {
 725                 (Some(Arc::new(crypt_config)), None)
 726             }
 727         }
 728     };
 729
 730     async_main(async move {
 731         let client = BackupWriter::start(
 732             client,
 733             repo.store(),
 734             backup_type,
 735             &backup_id,
 736             backup_time,
 737             verbose,
 738         ).await?;
 739
 740         let snapshot = BackupDir::new(backup_type, backup_id, backup_time.timestamp());
 741         let mut manifest = BackupManifest::new(snapshot);
 742
 743         let (catalog, catalog_result_rx) = spawn_catalog_upload(client.clone(), crypt_config.clone())?;
 744
 745         for (backup_type, filename, target, size) in upload_list {
 746             match backup_type {
 747                 BackupType::CONFIG => {
 748                     println!("Upload config file '{}' to '{:?}' as {}", filename, repo, target);
 749                     let stats = client
 750                         .upload_blob_from_file(&filename, &target, crypt_config.clone(), true)
 751                         .await?;
 752                     manifest.add_file(target, stats.size, stats.csum);
 753                 }
 754                 BackupType::LOGFILE => { // fixme: remove - not needed anymore ?
 755                     println!("Upload log file '{}' to '{:?}' as {}", filename, repo, target);
 756                     let stats = client
 757                         .upload_blob_from_file(&filename, &target, crypt_config.clone(), true)
 758                         .await?;
 759                     manifest.add_file(target, stats.size, stats.csum);
 760                 }
 761                 BackupType::PXAR => {
 762                     println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
 763                     catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
 764                     let stats = backup_directory(
 765                         &client,
 766                         &filename,
 767                         &target,
 768                         chunk_size_opt,
 769                         devices.clone(),
 770                         verbose,
 771                         skip_lost_and_found,
 772                         crypt_config.clone(),
 773                         catalog.clone(),
 774                     ).await?;
 775                     manifest.add_file(target, stats.size, stats.csum);
 776                     catalog.lock().unwrap().end_directory()?;
 777                 }
 778                 BackupType::IMAGE => {
 779                     println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
 780                     let stats = backup_image(
 781                         &client,
 782                         &filename,
 783                         &target,
 784                         size,
 785                         chunk_size_opt,
 786                         verbose,
 787                         crypt_config.clone(),
 788                     ).await?;
 789                     manifest.add_file(target, stats.size, stats.csum);
 790                 }
 791             }
 792         }
 793
 794         // finalize and upload catalog
 795         if upload_catalog {
 796             let mutex = Arc::try_unwrap(catalog)
 797                 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
 798             let mut catalog = mutex.into_inner().unwrap();
 799
 800             catalog.finish()?;
 801
 802             drop(catalog); // close upload stream
 803
 804             let stats = catalog_result_rx.await??;
 805
 806             manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum);
 807         }
 808
 809         if let Some(rsa_encrypted_key) = rsa_encrypted_key {
 810             let target = "rsa-encrypted.key";
 811             println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
 812             let stats = client
 813                 .upload_blob_from_data(rsa_encrypted_key, target, None, false, false)
 814                 .await?;
 815             manifest.add_file(format!("{}.blob", target), stats.size, stats.csum);
 816
 817             // openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out t
 818             /*
 819             let mut buffer2 = vec![0u8; rsa.size() as usize];
 820             let pem_data = file_get_contents("master-private.pem")?;
 821             let rsa = openssl::rsa::Rsa::private_key_from_pem(&pem_data)?;
 822             let len = rsa.private_decrypt(&buffer, &mut buffer2, openssl::rsa::Padding::PKCS1)?;
 823             println!("TEST {} {:?}", len, buffer2);
 824              */
 825         }
 826
 827         // create manifest (index.json)
 828         let manifest = manifest.into_json();
 829
 830         println!("Upload index.json to '{:?}'", repo);
 831         let manifest = serde_json::to_string_pretty(&manifest)?.into();
 832         client
 833             .upload_blob_from_data(manifest, MANIFEST_BLOB_NAME, crypt_config.clone(), true, true)
 834             .await?;
 835
 836         client.finish().await?;
 837
 838         let end_time = Local::now();
 839         let elapsed = end_time.signed_duration_since(start_time);
 840         println!("Duration: {}", elapsed);
 841
 842         println!("End Time: {}", end_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 843
 844         Ok(Value::Null)
 845     })
 846 }
 847
 848 fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 849
 850     let mut result = vec![];
 851
 852     let data: Vec<&str> = arg.splitn(2, ':').collect();
 853
 854     if data.len() != 2 {
 855         result.push(String::from("root.pxar:/"));
 856         result.push(String::from("etc.pxar:/etc"));
 857         return result;
 858     }
 859
 860     let files = tools::complete_file_name(data[1], param);
 861
 862     for file in files {
 863         result.push(format!("{}:{}", data[0], file));
 864     }
 865
 866     result
 867 }
 868
 869 fn restore(
 870     param: Value,
 871     _info: &ApiMethod,
 872     _rpcenv: &mut dyn RpcEnvironment,
 873 ) -> Result<Value, Error> {
 874     async_main(restore_do(param))
 875 }
 876
 877 fn dump_image<W: Write>(
 878     client: Arc<BackupReader>,
 879     crypt_config: Option<Arc<CryptConfig>>,
 880     index: FixedIndexReader,
 881     mut writer: W,
 882     verbose: bool,
 883 ) -> Result<(), Error> {
 884
 885     let most_used = index.find_most_used_chunks(8);
 886
 887     let mut chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 888
 889     // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
 890     // and thus slows down reading. Instead, directly use RemoteChunkReader
 891     let mut per = 0;
 892     let mut bytes = 0;
 893     let start_time = std::time::Instant::now();
 894
 895     for pos in 0..index.index_count() {
 896         let digest = index.index_digest(pos).unwrap();
 897         let raw_data = chunk_reader.read_chunk(&digest)?;
 898         writer.write_all(&raw_data)?;
 899         bytes += raw_data.len();
 900         if verbose {
 901             let next_per = ((pos+1)*100)/index.index_count();
 902             if per != next_per {
 903                 eprintln!("progress {}% (read {} bytes, duration {} sec)",
 904                           next_per, bytes, start_time.elapsed().as_secs());
 905                 per = next_per;
 906             }
 907         }
 908     }
 909
 910     let end_time = std::time::Instant::now();
 911     let elapsed = end_time.duration_since(start_time);
 912     eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
 913               bytes,
 914               elapsed.as_secs_f64(),
 915               bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
 916     );
 917
 918
 919     Ok(())
 920 }
 921
 922 async fn restore_do(param: Value) -> Result<Value, Error> {
 923     let repo = extract_repository_from_value(&param)?;
 924
 925     let verbose = param["verbose"].as_bool().unwrap_or(false);
 926
 927     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
 928
 929     let archive_name = tools::required_string_param(&param, "archive-name")?;
 930
 931     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 932
 933     record_repository(&repo);
 934
 935     let path = tools::required_string_param(&param, "snapshot")?;
 936
 937     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
 938         let group = BackupGroup::parse(path)?;
 939
 940         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 941         let result = client.get(&path, Some(json!({
 942             "backup-type": group.backup_type(),
 943             "backup-id": group.backup_id(),
 944         }))).await?;
 945
 946         let list = result["data"].as_array().unwrap();
 947         if list.is_empty() {
 948             bail!("backup group '{}' does not contain any snapshots:", path);
 949         }
 950
 951         let epoch = list[0]["backup-time"].as_i64().unwrap();
 952         let backup_time = Utc.timestamp(epoch, 0);
 953         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
 954     } else {
 955         let snapshot = BackupDir::parse(path)?;
 956         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
 957     };
 958
 959     let target = tools::required_string_param(&param, "target")?;
 960     let target = if target == "-" { None } else { Some(target) };
 961
 962     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 963
 964     let crypt_config = match keyfile {
 965         None => None,
 966         Some(path) => {
 967             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 968             Some(Arc::new(CryptConfig::new(key)?))
 969         }
 970     };
 971
 972     let server_archive_name = if archive_name.ends_with(".pxar") {
 973         format!("{}.didx", archive_name)
 974     } else if archive_name.ends_with(".img") {
 975         format!("{}.fidx", archive_name)
 976     } else {
 977         format!("{}.blob", archive_name)
 978     };
 979
 980     let client = BackupReader::start(
 981         client,
 982         crypt_config.clone(),
 983         repo.store(),
 984         &backup_type,
 985         &backup_id,
 986         backup_time,
 987         true,
 988     ).await?;
 989
 990     let manifest = client.download_manifest().await?;
 991
 992     if server_archive_name == MANIFEST_BLOB_NAME {
 993         let backup_index_data = manifest.into_json().to_string();
 994         if let Some(target) = target {
 995             file_set_contents(target, backup_index_data.as_bytes(), None)?;
 996         } else {
 997             let stdout = std::io::stdout();
 998             let mut writer = stdout.lock();
 999             writer.write_all(backup_index_data.as_bytes())
1000                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1001         }
1002
1003     } else if server_archive_name.ends_with(".blob") {
1004
1005         let mut reader = client.download_blob(&manifest, &server_archive_name).await?;
1006
1007         if let Some(target) = target {
1008            let mut writer = std::fs::OpenOptions::new()
1009                 .write(true)
1010                 .create(true)
1011                 .create_new(true)
1012                 .open(target)
1013                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1014             std::io::copy(&mut reader, &mut writer)?;
1015         } else {
1016             let stdout = std::io::stdout();
1017             let mut writer = stdout.lock();
1018             std::io::copy(&mut reader, &mut writer)
1019                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1020         }
1021
1022     } else if server_archive_name.ends_with(".didx") {
1023
1024         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1025
1026         let most_used = index.find_most_used_chunks(8);
1027
1028         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1029
1030         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1031
1032         if let Some(target) = target {
1033
1034             let feature_flags = pxar::flags::DEFAULT;
1035             let mut decoder = pxar::SequentialDecoder::new(&mut reader, feature_flags, |path| {
1036                 if verbose {
1037                     eprintln!("{:?}", path);
1038                 }
1039                 Ok(())
1040             });
1041             decoder.set_allow_existing_dirs(allow_existing_dirs);
1042
1043             decoder.restore(Path::new(target), &Vec::new())?;
1044         } else {
1045             let mut writer = std::fs::OpenOptions::new()
1046                 .write(true)
1047                 .open("/dev/stdout")
1048                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
1049
1050             std::io::copy(&mut reader, &mut writer)
1051                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1052         }
1053     } else if server_archive_name.ends_with(".fidx") {
1054
1055         let index = client.download_fixed_index(&manifest, &server_archive_name).await?;
1056
1057         let mut writer = if let Some(target) = target {
1058             std::fs::OpenOptions::new()
1059                 .write(true)
1060                 .create(true)
1061                 .create_new(true)
1062                 .open(target)
1063                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
1064         } else {
1065             std::fs::OpenOptions::new()
1066                 .write(true)
1067                 .open("/dev/stdout")
1068                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1069         };
1070
1071         dump_image(client.clone(), crypt_config.clone(), index, &mut writer, verbose)?;
1072
1073      } else {
1074         bail!("unknown archive file extension (expected .pxar of .img)");
1075     }
1076
1077     Ok(Value::Null)
1078 }
1079
1080 fn upload_log(
1081     param: Value,
1082     _info: &ApiMethod,
1083     _rpcenv: &mut dyn RpcEnvironment,
1084 ) -> Result<Value, Error> {
1085
1086     let logfile = tools::required_string_param(&param, "logfile")?;
1087     let repo = extract_repository_from_value(&param)?;
1088
1089     let snapshot = tools::required_string_param(&param, "snapshot")?;
1090     let snapshot = BackupDir::parse(snapshot)?;
1091
1092     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
1093
1094     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1095
1096     let crypt_config = match keyfile {
1097         None => None,
1098         Some(path) => {
1099             let (key, _created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1100             let crypt_config = CryptConfig::new(key)?;
1101             Some(Arc::new(crypt_config))
1102         }
1103     };
1104
1105     let data = file_get_contents(logfile)?;
1106
1107     let blob = DataBlob::encode(&data, crypt_config.as_ref().map(Arc::as_ref), true)?;
1108
1109     let raw_data = blob.into_inner();
1110
1111     let path = format!("api2/json/admin/datastore/{}/upload-backup-log", repo.store());
1112
1113     let args = json!({
1114         "backup-type": snapshot.group().backup_type(),
1115         "backup-id":  snapshot.group().backup_id(),
1116         "backup-time": snapshot.backup_time().timestamp(),
1117     });
1118
1119     let body = hyper::Body::from(raw_data);
1120
1121     async_main(async move {
1122         client.upload("application/octet-stream", body, &path, Some(args)).await
1123     })
1124 }
1125
1126 fn prune(
1127     mut param: Value,
1128     _info: &ApiMethod,
1129     _rpcenv: &mut dyn RpcEnvironment,
1130 ) -> Result<Value, Error> {
1131
1132     let repo = extract_repository_from_value(&param)?;
1133
1134     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
1135
1136     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
1137
1138     let group = tools::required_string_param(&param, "group")?;
1139     let group = BackupGroup::parse(group)?;
1140
1141     param.as_object_mut().unwrap().remove("repository");
1142     param.as_object_mut().unwrap().remove("group");
1143
1144     param["backup-type"] = group.backup_type().into();
1145     param["backup-id"] = group.backup_id().into();
1146
1147     let _result = async_main(async move { client.post(&path, Some(param)).await })?;
1148
1149     record_repository(&repo);
1150
1151     Ok(Value::Null)
1152 }
1153
1154 fn status(
1155     param: Value,
1156     _info: &ApiMethod,
1157     _rpcenv: &mut dyn RpcEnvironment,
1158 ) -> Result<Value, Error> {
1159
1160     let repo = extract_repository_from_value(&param)?;
1161
1162     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
1163
1164     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1165
1166     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1167
1168     let result = async_main(async move { client.get(&path, None).await })?;
1169     let data = &result["data"];
1170
1171     record_repository(&repo);
1172
1173     if output_format == "text" {
1174         let total = data["total"].as_u64().unwrap();
1175         let used = data["used"].as_u64().unwrap();
1176         let avail = data["avail"].as_u64().unwrap();
1177         let roundup = total/200;
1178
1179         println!(
1180             "total: {} used: {} ({} %) available: {}",
1181             total,
1182             used,
1183             ((used+roundup)*100)/total,
1184             avail,
1185         );
1186     } else {
1187         format_and_print_result(data, &output_format);
1188     }
1189
1190     Ok(Value::Null)
1191 }
1192
1193 // like get, but simply ignore errors and return Null instead
1194 async fn try_get(repo: &BackupRepository, url: &str) -> Value {
1195
1196     let client = match HttpClient::new(repo.host(), repo.user(), None) {
1197         Ok(v) => v,
1198         _ => return Value::Null,
1199     };
1200
1201     let mut resp = match client.get(url, None).await {
1202         Ok(v) => v,
1203         _ => return Value::Null,
1204     };
1205
1206     if let Some(map) = resp.as_object_mut() {
1207         if let Some(data) = map.remove("data") {
1208             return data;
1209         }
1210     }
1211     Value::Null
1212 }
1213
1214 fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1215     async_main(async { complete_backup_group_do(param).await })
1216 }
1217
1218 async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
1219
1220     let mut result = vec![];
1221
1222     let repo = match extract_repository_from_map(param) {
1223         Some(v) => v,
1224         _ => return result,
1225     };
1226
1227     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
1228
1229     let data = try_get(&repo, &path).await;
1230
1231     if let Some(list) = data.as_array() {
1232         for item in list {
1233             if let (Some(backup_id), Some(backup_type)) =
1234                 (item["backup-id"].as_str(), item["backup-type"].as_str())
1235             {
1236                 result.push(format!("{}/{}", backup_type, backup_id));
1237             }
1238         }
1239     }
1240
1241     result
1242 }
1243
1244 fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1245     async_main(async { complete_group_or_snapshot_do(arg, param).await })
1246 }
1247
1248 async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1249
1250     if arg.matches('/').count() < 2 {
1251         let groups = complete_backup_group_do(param).await;
1252         let mut result = vec![];
1253         for group in groups {
1254             result.push(group.to_string());
1255             result.push(format!("{}/", group));
1256         }
1257         return result;
1258     }
1259
1260     complete_backup_snapshot_do(param).await
1261 }
1262
1263 fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1264     async_main(async { complete_backup_snapshot_do(param).await })
1265 }
1266
1267 async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
1268
1269     let mut result = vec![];
1270
1271     let repo = match extract_repository_from_map(param) {
1272         Some(v) => v,
1273         _ => return result,
1274     };
1275
1276     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1277
1278     let data = try_get(&repo, &path).await;
1279
1280     if let Some(list) = data.as_array() {
1281         for item in list {
1282             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
1283                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
1284             {
1285                 let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
1286                 result.push(snapshot.relative_path().to_str().unwrap().to_owned());
1287             }
1288         }
1289     }
1290
1291     result
1292 }
1293
1294 fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1295     async_main(async { complete_server_file_name_do(param).await })
1296 }
1297
1298 async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
1299
1300     let mut result = vec![];
1301
1302     let repo = match extract_repository_from_map(param) {
1303         Some(v) => v,
1304         _ => return result,
1305     };
1306
1307     let snapshot = match param.get("snapshot") {
1308         Some(path) => {
1309             match BackupDir::parse(path) {
1310                 Ok(v) => v,
1311                 _ => return result,
1312             }
1313         }
1314         _ => return result,
1315     };
1316
1317     let query = tools::json_object_to_query(json!({
1318         "backup-type": snapshot.group().backup_type(),
1319         "backup-id": snapshot.group().backup_id(),
1320         "backup-time": snapshot.backup_time().timestamp(),
1321     })).unwrap();
1322
1323     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
1324
1325     let data = try_get(&repo, &path).await;
1326
1327     if let Some(list) = data.as_array() {
1328         for item in list {
1329             if let Some(filename) = item["filename"].as_str() {
1330                 result.push(filename.to_owned());
1331             }
1332         }
1333     }
1334
1335     result
1336 }
1337
1338 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1339     complete_server_file_name(arg, param)
1340         .iter()
1341         .map(|v| strip_server_file_expenstion(&v))
1342         .collect()
1343 }
1344
1345 fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
1346
1347     let mut result = vec![];
1348
1349     let mut size = 64;
1350     loop {
1351         result.push(size.to_string());
1352         size *= 2;
1353         if size > 4096 { break; }
1354     }
1355
1356     result
1357 }
1358
1359 fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
1360
1361     // fixme: implement other input methods
1362
1363     use std::env::VarError::*;
1364     match std::env::var("PBS_ENCRYPTION_PASSWORD") {
1365         Ok(p) => return Ok(p.as_bytes().to_vec()),
1366         Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
1367         Err(NotPresent) => {
1368             // Try another method
1369         }
1370     }
1371
1372     // If we're on a TTY, query the user for a password
1373     if crate::tools::tty::stdin_isatty() {
1374         return Ok(crate::tools::tty::read_password("Encryption Key Password: ")?);
1375     }
1376
1377     bail!("no password input mechanism available");
1378 }
1379
1380 fn key_create(
1381     param: Value,
1382     _info: &ApiMethod,
1383     _rpcenv: &mut dyn RpcEnvironment,
1384 ) -> Result<Value, Error> {
1385
1386     let path = tools::required_string_param(&param, "path")?;
1387     let path = PathBuf::from(path);
1388
1389     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1390
1391     let key = proxmox::sys::linux::random_data(32)?;
1392
1393     if kdf == "scrypt" {
1394         // always read passphrase from tty
1395         if !crate::tools::tty::stdin_isatty() {
1396             bail!("unable to read passphrase - no tty");
1397         }
1398
1399         let password = crate::tools::tty::read_password("Encryption Key Password: ")?;
1400
1401         let key_config = encrypt_key_with_passphrase(&key, &password)?;
1402
1403         store_key_config(&path, false, key_config)?;
1404
1405         Ok(Value::Null)
1406     } else if kdf == "none" {
1407         let created =  Local.timestamp(Local::now().timestamp(), 0);
1408
1409         store_key_config(&path, false, KeyConfig {
1410             kdf: None,
1411             created,
1412             modified: created,
1413             data: key,
1414         })?;
1415
1416         Ok(Value::Null)
1417     } else {
1418         unreachable!();
1419     }
1420 }
1421
1422 fn master_pubkey_path() -> Result<PathBuf, Error> {
1423     let base = BaseDirectories::with_prefix("proxmox-backup")?;
1424
1425     // usually $HOME/.config/proxmox-backup/master-public.pem
1426     let path = base.place_config_file("master-public.pem")?;
1427
1428     Ok(path)
1429 }
1430
1431 fn key_import_master_pubkey(
1432     param: Value,
1433     _info: &ApiMethod,
1434     _rpcenv: &mut dyn RpcEnvironment,
1435 ) -> Result<Value, Error> {
1436
1437     let path = tools::required_string_param(&param, "path")?;
1438     let path = PathBuf::from(path);
1439
1440     let pem_data = file_get_contents(&path)?;
1441
1442     if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
1443         bail!("Unable to decode PEM data - {}", err);
1444     }
1445
1446     let target_path = master_pubkey_path()?;
1447
1448     file_set_contents(&target_path, &pem_data, None)?;
1449
1450     println!("Imported public master key to {:?}", target_path);
1451
1452     Ok(Value::Null)
1453 }
1454
1455 fn key_create_master_key(
1456     _param: Value,
1457     _info: &ApiMethod,
1458     _rpcenv: &mut dyn RpcEnvironment,
1459 ) -> Result<Value, Error> {
1460
1461     // we need a TTY to query the new password
1462     if !crate::tools::tty::stdin_isatty() {
1463         bail!("unable to create master key - no tty");
1464     }
1465
1466     let rsa = openssl::rsa::Rsa::generate(4096)?;
1467     let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
1468
1469     let new_pw = String::from_utf8(crate::tools::tty::read_password("Master Key Password: ")?)?;
1470     let verify_pw = String::from_utf8(crate::tools::tty::read_password("Verify Password: ")?)?;
1471
1472     if new_pw != verify_pw {
1473         bail!("Password verification fail!");
1474     }
1475
1476     if new_pw.len() < 5 {
1477         bail!("Password is too short!");
1478     }
1479
1480     let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
1481     let filename_pub = "master-public.pem";
1482     println!("Writing public master key to {}", filename_pub);
1483     file_set_contents(filename_pub, pub_key.as_slice(), None)?;
1484
1485     let cipher = openssl::symm::Cipher::aes_256_cbc();
1486     let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, new_pw.as_bytes())?;
1487
1488     let filename_priv = "master-private.pem";
1489     println!("Writing private master key to {}", filename_priv);
1490     file_set_contents(filename_priv, priv_key.as_slice(), None)?;
1491
1492     Ok(Value::Null)
1493 }
1494
1495 fn key_change_passphrase(
1496     param: Value,
1497     _info: &ApiMethod,
1498     _rpcenv: &mut dyn RpcEnvironment,
1499 ) -> Result<Value, Error> {
1500
1501     let path = tools::required_string_param(&param, "path")?;
1502     let path = PathBuf::from(path);
1503
1504     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1505
1506     // we need a TTY to query the new password
1507     if !crate::tools::tty::stdin_isatty() {
1508         bail!("unable to change passphrase - no tty");
1509     }
1510
1511     let (key, created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1512
1513     if kdf == "scrypt" {
1514
1515         let new_pw = String::from_utf8(crate::tools::tty::read_password("New Password: ")?)?;
1516         let verify_pw = String::from_utf8(crate::tools::tty::read_password("Verify Password: ")?)?;
1517
1518         if new_pw != verify_pw {
1519             bail!("Password verification fail!");
1520         }
1521
1522         if new_pw.len() < 5 {
1523             bail!("Password is too short!");
1524         }
1525
1526         let mut new_key_config = encrypt_key_with_passphrase(&key, new_pw.as_bytes())?;
1527         new_key_config.created = created; // keep original value
1528
1529         store_key_config(&path, true, new_key_config)?;
1530
1531         Ok(Value::Null)
1532     } else if kdf == "none" {
1533         let modified =  Local.timestamp(Local::now().timestamp(), 0);
1534
1535         store_key_config(&path, true, KeyConfig {
1536             kdf: None,
1537             created, // keep original value
1538             modified,
1539             data: key.to_vec(),
1540         })?;
1541
1542         Ok(Value::Null)
1543     } else {
1544         unreachable!();
1545     }
1546 }
1547
1548 fn key_mgmt_cli() -> CliCommandMap {
1549
1550     const KDF_SCHEMA: Schema =
1551         StringSchema::new("Key derivation function. Choose 'none' to store the key unecrypted.")
1552         .format(&ApiStringFormat::Enum(&["scrypt", "none"]))
1553         .default("scrypt")
1554         .schema();
1555
1556     #[sortable]
1557     const API_METHOD_KEY_CREATE: ApiMethod = ApiMethod::new(
1558         &ApiHandler::Sync(&key_create),
1559         &ObjectSchema::new(
1560             "Create a new encryption key.",
1561             &sorted!([
1562                 ("path", false, &StringSchema::new("File system path.").schema()),
1563                 ("kdf", true, &KDF_SCHEMA),
1564             ]),
1565         )
1566     );
1567
1568     let key_create_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE)
1569         .arg_param(vec!["path"])
1570         .completion_cb("path", tools::complete_file_name);
1571
1572     #[sortable]
1573     const API_METHOD_KEY_CHANGE_PASSPHRASE: ApiMethod = ApiMethod::new(
1574         &ApiHandler::Sync(&key_change_passphrase),
1575         &ObjectSchema::new(
1576             "Change the passphrase required to decrypt the key.",
1577             &sorted!([
1578                 ("path", false, &StringSchema::new("File system path.").schema()),
1579                 ("kdf", true, &KDF_SCHEMA),
1580             ]),
1581         )
1582     );
1583
1584     let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_KEY_CHANGE_PASSPHRASE)
1585         .arg_param(vec!["path"])
1586         .completion_cb("path", tools::complete_file_name);
1587
1588     const API_METHOD_KEY_CREATE_MASTER_KEY: ApiMethod = ApiMethod::new(
1589         &ApiHandler::Sync(&key_create_master_key),
1590         &ObjectSchema::new("Create a new 4096 bit RSA master pub/priv key pair.", &[])
1591     );
1592
1593     let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE_MASTER_KEY);
1594
1595     #[sortable]
1596     const API_METHOD_KEY_IMPORT_MASTER_PUBKEY: ApiMethod = ApiMethod::new(
1597         &ApiHandler::Sync(&key_import_master_pubkey),
1598         &ObjectSchema::new(
1599             "Import a new RSA public key and use it as master key. The key is expected to be in '.pem' format.",
1600             &sorted!([ ("path", false, &StringSchema::new("File system path.").schema()) ]),
1601         )
1602     );
1603
1604     let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_KEY_IMPORT_MASTER_PUBKEY)
1605         .arg_param(vec!["path"])
1606         .completion_cb("path", tools::complete_file_name);
1607
1608     CliCommandMap::new()
1609         .insert("create".to_owned(), key_create_cmd_def.into())
1610         .insert("create-master-key".to_owned(), key_create_master_key_cmd_def.into())
1611         .insert("import-master-pubkey".to_owned(), key_import_master_pubkey_cmd_def.into())
1612         .insert("change-passphrase".to_owned(), key_change_passphrase_cmd_def.into())
1613 }
1614
1615 fn mount(
1616     param: Value,
1617     _info: &ApiMethod,
1618     _rpcenv: &mut dyn RpcEnvironment,
1619 ) -> Result<Value, Error> {
1620     let verbose = param["verbose"].as_bool().unwrap_or(false);
1621     if verbose {
1622         // This will stay in foreground with debug output enabled as None is
1623         // passed for the RawFd.
1624         return async_main(mount_do(param, None));
1625     }
1626
1627     // Process should be deamonized.
1628     // Make sure to fork before the async runtime is instantiated to avoid troubles.
1629     let pipe = pipe()?;
1630     match fork() {
1631         Ok(ForkResult::Parent { .. }) => {
1632             nix::unistd::close(pipe.1).unwrap();
1633             // Blocks the parent process until we are ready to go in the child
1634             let _res = nix::unistd::read(pipe.0, &mut [0]).unwrap();
1635             Ok(Value::Null)
1636         }
1637         Ok(ForkResult::Child) => {
1638             nix::unistd::close(pipe.0).unwrap();
1639             nix::unistd::setsid().unwrap();
1640             async_main(mount_do(param, Some(pipe.1)))
1641         }
1642         Err(_) => bail!("failed to daemonize process"),
1643     }
1644 }
1645
1646 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
1647     let repo = extract_repository_from_value(&param)?;
1648     let archive_name = tools::required_string_param(&param, "archive-name")?;
1649     let target = tools::required_string_param(&param, "target")?;
1650     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1651
1652     record_repository(&repo);
1653
1654     let path = tools::required_string_param(&param, "snapshot")?;
1655     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1656         let group = BackupGroup::parse(path)?;
1657
1658         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1659         let result = client.get(&path, Some(json!({
1660             "backup-type": group.backup_type(),
1661             "backup-id": group.backup_id(),
1662         }))).await?;
1663
1664         let list = result["data"].as_array().unwrap();
1665         if list.is_empty() {
1666             bail!("backup group '{}' does not contain any snapshots:", path);
1667         }
1668
1669         let epoch = list[0]["backup-time"].as_i64().unwrap();
1670         let backup_time = Utc.timestamp(epoch, 0);
1671         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
1672     } else {
1673         let snapshot = BackupDir::parse(path)?;
1674         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1675     };
1676
1677     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1678     let crypt_config = match keyfile {
1679         None => None,
1680         Some(path) => {
1681             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1682             Some(Arc::new(CryptConfig::new(key)?))
1683         }
1684     };
1685
1686     let server_archive_name = if archive_name.ends_with(".pxar") {
1687         format!("{}.didx", archive_name)
1688     } else {
1689         bail!("Can only mount pxar archives.");
1690     };
1691
1692     let client = BackupReader::start(
1693         client,
1694         crypt_config.clone(),
1695         repo.store(),
1696         &backup_type,
1697         &backup_id,
1698         backup_time,
1699         true,
1700     ).await?;
1701
1702     let manifest = client.download_manifest().await?;
1703
1704     if server_archive_name.ends_with(".didx") {
1705         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1706         let most_used = index.find_most_used_chunks(8);
1707         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1708         let reader = BufferedDynamicReader::new(index, chunk_reader);
1709         let decoder =
1710             pxar::Decoder::<Box<dyn pxar::fuse::ReadSeek>, fn(&Path) -> Result<(), Error>>::new(
1711                 Box::new(reader),
1712                 |_| Ok(()),
1713             )?;
1714         let options = OsStr::new("ro,default_permissions");
1715         let mut session = pxar::fuse::Session::from_decoder(decoder, &options, pipe.is_none())
1716             .map_err(|err| format_err!("pxar mount failed: {}", err))?;
1717
1718         // Mount the session but not call fuse deamonize as this will cause
1719         // issues with the runtime after the fork
1720         let deamonize = false;
1721         session.mount(&Path::new(target), deamonize)?;
1722
1723         if let Some(pipe) = pipe {
1724             nix::unistd::chdir(Path::new("/")).unwrap();
1725             // Finish creation of deamon by redirecting filedescriptors.
1726             let nullfd = nix::fcntl::open(
1727                 "/dev/null",
1728                 nix::fcntl::OFlag::O_RDWR,
1729                 nix::sys::stat::Mode::empty(),
1730             ).unwrap();
1731             nix::unistd::dup2(nullfd, 0).unwrap();
1732             nix::unistd::dup2(nullfd, 1).unwrap();
1733             nix::unistd::dup2(nullfd, 2).unwrap();
1734             if nullfd > 2 {
1735                 nix::unistd::close(nullfd).unwrap();
1736             }
1737             // Signal the parent process that we are done with the setup and it can
1738             // terminate.
1739             nix::unistd::write(pipe, &[0u8])?;
1740             nix::unistd::close(pipe).unwrap();
1741         }
1742
1743         let multithreaded = true;
1744         session.run_loop(multithreaded)?;
1745     } else {
1746         bail!("unknown archive file extension (expected .pxar)");
1747     }
1748
1749     Ok(Value::Null)
1750 }
1751
1752 fn shell(
1753     param: Value,
1754     _info: &ApiMethod,
1755     _rpcenv: &mut dyn RpcEnvironment,
1756 ) -> Result<Value, Error> {
1757     async_main(catalog_shell(param))
1758 }
1759
1760 async fn catalog_shell(param: Value) -> Result<Value, Error> {
1761     let repo = extract_repository_from_value(&param)?;
1762     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1763     let path = tools::required_string_param(&param, "snapshot")?;
1764     let archive_name = tools::required_string_param(&param, "archive-name")?;
1765
1766     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1767         let group = BackupGroup::parse(path)?;
1768
1769         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1770         let result = client.get(&path, Some(json!({
1771             "backup-type": group.backup_type(),
1772             "backup-id": group.backup_id(),
1773         }))).await?;
1774
1775         let list = result["data"].as_array().unwrap();
1776         if list.len() == 0 {
1777             bail!("backup group '{}' does not contain any snapshots:", path);
1778         }
1779
1780         let epoch = list[0]["backup-time"].as_i64().unwrap();
1781         let backup_time = Utc.timestamp(epoch, 0);
1782         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
1783     } else {
1784         let snapshot = BackupDir::parse(path)?;
1785         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1786     };
1787
1788     let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
1789     let crypt_config = match keyfile {
1790         None => None,
1791         Some(path) => {
1792             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1793             Some(Arc::new(CryptConfig::new(key)?))
1794         }
1795     };
1796
1797     let server_archive_name = if archive_name.ends_with(".pxar") {
1798         format!("{}.didx", archive_name)
1799     } else {
1800         bail!("Can only mount pxar archives.");
1801     };
1802
1803     let client = BackupReader::start(
1804         client,
1805         crypt_config.clone(),
1806         repo.store(),
1807         &backup_type,
1808         &backup_id,
1809         backup_time,
1810         true,
1811     ).await?;
1812
1813     let tmpfile = std::fs::OpenOptions::new()
1814         .write(true)
1815         .read(true)
1816         .custom_flags(libc::O_TMPFILE)
1817         .open("/tmp")?;
1818
1819     let manifest = client.download_manifest().await?;
1820
1821     let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1822     let most_used = index.find_most_used_chunks(8);
1823     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
1824     let reader = BufferedDynamicReader::new(index, chunk_reader);
1825     let decoder =
1826         pxar::Decoder::<BufferedDynamicReader<RemoteChunkReader>, fn(&Path) -> Result<(), Error>>::new(
1827             reader,
1828             |path| {
1829                 println!("{:?}", path);
1830                 Ok(())
1831             }
1832         )?;
1833
1834     let tmpfile = client.download(CATALOG_NAME, tmpfile).await?;
1835     let index = DynamicIndexReader::new(tmpfile)
1836         .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
1837
1838     // Note: do not use values stored in index (not trusted) - instead, computed them again
1839     let (csum, size) = index.compute_csum();
1840     manifest.verify_file(CATALOG_NAME, &csum, size)?;
1841
1842     let most_used = index.find_most_used_chunks(8);
1843     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1844     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1845     let mut catalogfile = std::fs::OpenOptions::new()
1846         .write(true)
1847         .read(true)
1848         .custom_flags(libc::O_TMPFILE)
1849         .open("/tmp")?;
1850
1851     std::io::copy(&mut reader, &mut catalogfile)
1852         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
1853
1854     catalogfile.seek(SeekFrom::Start(0))?;
1855     let catalog_reader = CatalogReader::new(catalogfile);
1856     let state = Shell::new(
1857         catalog_reader,
1858         &server_archive_name,
1859         decoder,
1860     )?;
1861
1862     println!("Starting interactive shell");
1863     state.shell()?;
1864
1865     record_repository(&repo);
1866
1867     Ok(Value::Null)
1868 }
1869
1870 fn main() {
1871
1872     const BACKUP_SOURCE_SCHEMA: Schema = StringSchema::new("Backup source specification ([<label>:<path>]).")
1873         .format(&ApiStringFormat::Pattern(&BACKUPSPEC_REGEX))
1874         .schema();
1875
1876     #[sortable]
1877     const API_METHOD_CREATE_BACKUP: ApiMethod = ApiMethod::new(
1878         &ApiHandler::Sync(&create_backup),
1879         &ObjectSchema::new(
1880             "Create (host) backup.",
1881             &sorted!([
1882                 (
1883                     "backupspec",
1884                     false,
1885                     &ArraySchema::new(
1886                         "List of backup source specifications ([<label.ext>:<path>] ...)",
1887                         &BACKUP_SOURCE_SCHEMA,
1888                     ).min_length(1).schema()
1889                 ),
1890                 (
1891                     "repository",
1892                     true,
1893                     &REPO_URL_SCHEMA
1894                 ),
1895                 (
1896                     "include-dev",
1897                     true,
1898                     &ArraySchema::new(
1899                         "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
1900                         &StringSchema::new("Path to file.").schema()
1901                     ).schema()
1902                 ),
1903                 (
1904                     "keyfile",
1905                     true,
1906                     &StringSchema::new("Path to encryption key. All data will be encrypted using this key.").schema()
1907                 ),
1908                 (
1909                     "verbose",
1910                     true,
1911                     &BooleanSchema::new("Verbose output.")
1912                         .default(false)
1913                         .schema()
1914                 ),
1915                 (
1916                     "skip-lost-and-found",
1917                     true,
1918                     &BooleanSchema::new("Skip lost+found directory")
1919                         .default(false)
1920                         .schema()
1921                 ),
1922                 (
1923                     "backup-type",
1924                     true,
1925                     &BACKUP_TYPE_SCHEMA,
1926                 ),
1927                 (
1928                     "backup-id",
1929                     true,
1930                     &BACKUP_ID_SCHEMA
1931                 ),
1932                 (
1933                     "backup-time",
1934                     true,
1935                     &BACKUP_TIME_SCHEMA
1936                 ),
1937                 (
1938                     "chunk-size",
1939                     true,
1940                     &IntegerSchema::new("Chunk size in KB. Must be a power of 2.")
1941                         .minimum(64)
1942                         .maximum(4096)
1943                         .default(4096)
1944                         .schema()
1945                 ),
1946             ]),
1947         )
1948     );
1949
1950     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
1951         .arg_param(vec!["backupspec"])
1952         .completion_cb("repository", complete_repository)
1953         .completion_cb("backupspec", complete_backup_source)
1954         .completion_cb("keyfile", tools::complete_file_name)
1955         .completion_cb("chunk-size", complete_chunk_size);
1956
1957     #[sortable]
1958     const API_METHOD_UPLOAD_LOG: ApiMethod = ApiMethod::new(
1959         &ApiHandler::Sync(&upload_log),
1960         &ObjectSchema::new(
1961             "Upload backup log file.",
1962             &sorted!([
1963                 (
1964                     "snapshot",
1965                     false,
1966                     &StringSchema::new("Snapshot path.").schema()
1967                 ),
1968                 (
1969                     "logfile",
1970                     false,
1971                     &StringSchema::new("The path to the log file you want to upload.").schema()
1972                 ),
1973                 (
1974                     "repository",
1975                     true,
1976                     &REPO_URL_SCHEMA
1977                 ),
1978                 (
1979                     "keyfile",
1980                     true,
1981                     &StringSchema::new("Path to encryption key. All data will be encrypted using this key.").schema()
1982                 ),
1983             ]),
1984         )
1985     );
1986
1987     let upload_log_cmd_def = CliCommand::new(&API_METHOD_UPLOAD_LOG)
1988         .arg_param(vec!["snapshot", "logfile"])
1989         .completion_cb("snapshot", complete_backup_snapshot)
1990         .completion_cb("logfile", tools::complete_file_name)
1991         .completion_cb("keyfile", tools::complete_file_name)
1992         .completion_cb("repository", complete_repository);
1993
1994     #[sortable]
1995     const API_METHOD_LIST_BACKUP_GROUPS: ApiMethod = ApiMethod::new(
1996         &ApiHandler::Sync(&list_backup_groups),
1997         &ObjectSchema::new(
1998             "List backup groups.",
1999             &sorted!([
2000                 ("repository", true, &REPO_URL_SCHEMA),
2001                 ("output-format", true, &OUTPUT_FORMAT),
2002             ]),
2003         )
2004     );
2005
2006     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
2007         .completion_cb("repository", complete_repository);
2008
2009     #[sortable]
2010     const API_METHOD_LIST_SNAPSHOTS: ApiMethod = ApiMethod::new(
2011         &ApiHandler::Sync(&list_snapshots),
2012         &ObjectSchema::new(
2013             "List backup snapshots.",
2014             &sorted!([
2015                 ("group", true, &StringSchema::new("Backup group.").schema()),
2016                 ("repository", true, &REPO_URL_SCHEMA),
2017                 ("output-format", true, &OUTPUT_FORMAT),
2018             ]),
2019         )
2020     );
2021
2022     let snapshots_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOTS)
2023         .arg_param(vec!["group"])
2024         .completion_cb("group", complete_backup_group)
2025         .completion_cb("repository", complete_repository);
2026
2027     #[sortable]
2028     const API_METHOD_FORGET_SNAPSHOTS: ApiMethod = ApiMethod::new(
2029         &ApiHandler::Sync(&forget_snapshots),
2030         &ObjectSchema::new(
2031             "Forget (remove) backup snapshots.",
2032             &sorted!([
2033                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2034                 ("repository", true, &REPO_URL_SCHEMA),
2035             ]),
2036         )
2037     );
2038
2039     let forget_cmd_def = CliCommand::new(&API_METHOD_FORGET_SNAPSHOTS)
2040         .arg_param(vec!["snapshot"])
2041         .completion_cb("repository", complete_repository)
2042         .completion_cb("snapshot", complete_backup_snapshot);
2043
2044     #[sortable]
2045     const API_METHOD_START_GARBAGE_COLLECTION: ApiMethod = ApiMethod::new(
2046         &ApiHandler::Sync(&start_garbage_collection),
2047         &ObjectSchema::new(
2048             "Start garbage collection for a specific repository.",
2049             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2050         )
2051     );
2052
2053     let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
2054         .completion_cb("repository", complete_repository);
2055
2056     #[sortable]
2057     const API_METHOD_RESTORE: ApiMethod = ApiMethod::new(
2058         &ApiHandler::Sync(&restore),
2059         &ObjectSchema::new(
2060             "Restore backup repository.",
2061             &sorted!([
2062                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2063                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2064                 (
2065                     "target",
2066                     false,
2067                     &StringSchema::new(
2068                         r###"Target directory path. Use '-' to write to stdandard output.
2069
2070 We do not extraxt '.pxar' archives when writing to stdandard output.
2071
2072 "###
2073                     ).schema()
2074                 ),
2075                 (
2076                     "allow-existing-dirs",
2077                     true,
2078                     &BooleanSchema::new("Do not fail if directories already exists.")
2079                         .default(false)
2080                         .schema()
2081                 ),
2082                 ("repository", true, &REPO_URL_SCHEMA),
2083                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2084                 (
2085                     "verbose",
2086                     true,
2087                     &BooleanSchema::new("Verbose output.")
2088                         .default(false)
2089                         .schema()
2090                 ),
2091             ]),
2092         )
2093     );
2094
2095     let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
2096         .arg_param(vec!["snapshot", "archive-name", "target"])
2097         .completion_cb("repository", complete_repository)
2098         .completion_cb("snapshot", complete_group_or_snapshot)
2099         .completion_cb("archive-name", complete_archive_name)
2100         .completion_cb("target", tools::complete_file_name);
2101
2102     #[sortable]
2103     const API_METHOD_LIST_SNAPSHOT_FILES: ApiMethod = ApiMethod::new(
2104         &ApiHandler::Sync(&list_snapshot_files),
2105         &ObjectSchema::new(
2106             "List snapshot files.",
2107             &sorted!([
2108                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2109                 ("repository", true, &REPO_URL_SCHEMA),
2110                 ("output-format", true, &OUTPUT_FORMAT),
2111             ]),
2112         )
2113     );
2114
2115     let files_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOT_FILES)
2116         .arg_param(vec!["snapshot"])
2117         .completion_cb("repository", complete_repository)
2118         .completion_cb("snapshot", complete_backup_snapshot);
2119
2120     #[sortable]
2121     const API_METHOD_DUMP_CATALOG: ApiMethod = ApiMethod::new(
2122         &ApiHandler::Sync(&dump_catalog),
2123         &ObjectSchema::new(
2124             "Dump catalog.",
2125             &sorted!([
2126                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2127                 ("repository", true, &REPO_URL_SCHEMA),
2128             ]),
2129         )
2130     );
2131
2132     let catalog_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
2133         .arg_param(vec!["snapshot"])
2134         .completion_cb("repository", complete_repository)
2135         .completion_cb("snapshot", complete_backup_snapshot);
2136
2137     const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
2138         &ApiHandler::Sync(&prune),
2139         &ObjectSchema::new(
2140             "Prune backup repository.",
2141             &proxmox_backup::add_common_prune_prameters!([
2142                 ("group", false, &StringSchema::new("Backup group.").schema()),
2143             ], [
2144                 ("repository", true, &REPO_URL_SCHEMA),
2145             ])
2146         )
2147     );
2148
2149     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
2150         .arg_param(vec!["group"])
2151         .completion_cb("group", complete_backup_group)
2152         .completion_cb("repository", complete_repository);
2153
2154     #[sortable]
2155     const API_METHOD_STATUS: ApiMethod = ApiMethod::new(
2156         &ApiHandler::Sync(&status),
2157         &ObjectSchema::new(
2158             "Get repository status.",
2159             &sorted!([
2160                 ("repository", true, &REPO_URL_SCHEMA),
2161                 ("output-format", true, &OUTPUT_FORMAT),
2162             ]),
2163         )
2164     );
2165
2166     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
2167         .completion_cb("repository", complete_repository);
2168
2169     #[sortable]
2170     const API_METHOD_API_LOGIN: ApiMethod = ApiMethod::new(
2171         &ApiHandler::Sync(&api_login),
2172         &ObjectSchema::new(
2173             "Try to login. If successful, store ticket.",
2174             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2175         )
2176     );
2177
2178     let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
2179         .completion_cb("repository", complete_repository);
2180
2181     #[sortable]
2182     const API_METHOD_API_LOGOUT: ApiMethod = ApiMethod::new(
2183         &ApiHandler::Sync(&api_logout),
2184         &ObjectSchema::new(
2185             "Logout (delete stored ticket).",
2186             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2187         )
2188     );
2189
2190     let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
2191         .completion_cb("repository", complete_repository);
2192
2193     #[sortable]
2194     const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
2195         &ApiHandler::Sync(&mount),
2196         &ObjectSchema::new(
2197             "Mount pxar archive.",
2198             &sorted!([
2199                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2200                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2201                 ("target", false, &StringSchema::new("Target directory path.").schema()),
2202                 ("repository", true, &REPO_URL_SCHEMA),
2203                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2204                 ("verbose", true, &BooleanSchema::new("Verbose output.").default(false).schema()),
2205             ]),
2206         )
2207     );
2208
2209     let mount_cmd_def = CliCommand::new(&API_METHOD_MOUNT)
2210         .arg_param(vec!["snapshot", "archive-name", "target"])
2211         .completion_cb("repository", complete_repository)
2212         .completion_cb("snapshot", complete_group_or_snapshot)
2213         .completion_cb("archive-name", complete_archive_name)
2214         .completion_cb("target", tools::complete_file_name);
2215
2216     #[sortable]
2217     const API_METHOD_SHELL: ApiMethod = ApiMethod::new(
2218         &ApiHandler::Sync(&shell),
2219         &ObjectSchema::new(
2220             "Shell to interactively inspect and restore snapshots.",
2221             &sorted!([
2222                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2223                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2224                 ("repository", true, &REPO_URL_SCHEMA),
2225                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2226             ]),
2227         )
2228     );
2229
2230     let shell_cmd_def = CliCommand::new(&API_METHOD_SHELL)
2231         .arg_param(vec!["snapshot", "archive-name"])
2232         .completion_cb("repository", complete_repository)
2233         .completion_cb("archive-name", complete_archive_name)
2234         .completion_cb("snapshot", complete_group_or_snapshot);
2235
2236     let cmd_def = CliCommandMap::new()
2237         .insert("backup".to_owned(), backup_cmd_def.into())
2238         .insert("upload-log".to_owned(), upload_log_cmd_def.into())
2239         .insert("forget".to_owned(), forget_cmd_def.into())
2240         .insert("catalog".to_owned(), catalog_cmd_def.into())
2241         .insert("garbage-collect".to_owned(), garbage_collect_cmd_def.into())
2242         .insert("list".to_owned(), list_cmd_def.into())
2243         .insert("login".to_owned(), login_cmd_def.into())
2244         .insert("logout".to_owned(), logout_cmd_def.into())
2245         .insert("prune".to_owned(), prune_cmd_def.into())
2246         .insert("restore".to_owned(), restore_cmd_def.into())
2247         .insert("snapshots".to_owned(), snapshots_cmd_def.into())
2248         .insert("files".to_owned(), files_cmd_def.into())
2249         .insert("status".to_owned(), status_cmd_def.into())
2250         .insert("key".to_owned(), key_mgmt_cli().into())
2251         .insert("mount".to_owned(), mount_cmd_def.into())
2252         .insert("shell".to_owned(), shell_cmd_def.into());
2253
2254     run_cli_command(cmd_def.into());
2255 }
2256
2257 fn async_main<F: Future>(fut: F) -> <F as Future>::Output {
2258     let rt = tokio::runtime::Runtime::new().unwrap();
2259     let ret = rt.block_on(fut);
2260     rt.shutdown_now();
2261     ret
2262 }