src/bin/proxmox-backup-client.rs

   1 use failure::*;
   2 use nix::unistd::{fork, ForkResult, pipe};
   3 use std::os::unix::io::RawFd;
   4 use chrono::{Local, Utc, TimeZone};
   5 use std::path::{Path, PathBuf};
   6 use std::collections::{HashSet, HashMap};
   7 use std::ffi::OsStr;
   8 use std::io::{Write, Seek, SeekFrom};
   9 use std::os::unix::fs::OpenOptionsExt;
  10
  11 use proxmox::{sortable, identity};
  12 use proxmox::tools::fs::{file_get_contents, file_get_json, file_set_contents, image_size};
  13 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
  14 use proxmox::api::schema::*;
  15 use proxmox::api::cli::*;
  16
  17 use proxmox_backup::tools;
  18 use proxmox_backup::api2::types::*;
  19 use proxmox_backup::client::*;
  20 use proxmox_backup::backup::*;
  21 use proxmox_backup::pxar::{ self, catalog::* };
  22
  23 //use proxmox_backup::backup::image_index::*;
  24 //use proxmox_backup::config::datastore;
  25 //use proxmox_backup::pxar::encoder::*;
  26 //use proxmox_backup::backup::datastore::*;
  27
  28 use serde_json::{json, Value};
  29 //use hyper::Body;
  30 use std::sync::{Arc, Mutex};
  31 //use regex::Regex;
  32 use xdg::BaseDirectories;
  33
  34 use futures::*;
  35 use tokio::sync::mpsc;
  36
  37 proxmox::api::const_regex! {
  38     BACKUPSPEC_REGEX = r"^([a-zA-Z0-9_-]+\.(?:pxar|img|conf|log)):(.+)$";
  39 }
  40
  41 const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  42     .format(&BACKUP_REPO_URL)
  43     .max_length(256)
  44     .schema();
  45
  46 fn get_default_repository() -> Option<String> {
  47     std::env::var("PBS_REPOSITORY").ok()
  48 }
  49
  50 fn extract_repository_from_value(
  51     param: &Value,
  52 ) -> Result<BackupRepository, Error> {
  53
  54     let repo_url = param["repository"]
  55         .as_str()
  56         .map(String::from)
  57         .or_else(get_default_repository)
  58         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
  59
  60     let repo: BackupRepository = repo_url.parse()?;
  61
  62     Ok(repo)
  63 }
  64
  65 fn extract_repository_from_map(
  66     param: &HashMap<String, String>,
  67 ) -> Option<BackupRepository> {
  68
  69     param.get("repository")
  70         .map(String::from)
  71         .or_else(get_default_repository)
  72         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
  73 }
  74
  75 fn record_repository(repo: &BackupRepository) {
  76
  77     let base = match BaseDirectories::with_prefix("proxmox-backup") {
  78         Ok(v) => v,
  79         _ => return,
  80     };
  81
  82     // usually $HOME/.cache/proxmox-backup/repo-list
  83     let path = match base.place_cache_file("repo-list") {
  84         Ok(v) => v,
  85         _ => return,
  86     };
  87
  88     let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
  89
  90     let repo = repo.to_string();
  91
  92     data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
  93
  94     let mut map = serde_json::map::Map::new();
  95
  96     loop {
  97         let mut max_used = 0;
  98         let mut max_repo = None;
  99         for (repo, count) in data.as_object().unwrap() {
 100             if map.contains_key(repo) { continue; }
 101             if let Some(count) = count.as_i64() {
 102                 if count > max_used {
 103                     max_used = count;
 104                     max_repo = Some(repo);
 105                 }
 106             }
 107         }
 108         if let Some(repo) = max_repo {
 109             map.insert(repo.to_owned(), json!(max_used));
 110         } else {
 111             break;
 112         }
 113         if map.len() > 10 { // store max. 10 repos
 114             break;
 115         }
 116     }
 117
 118     let new_data = json!(map);
 119
 120     let _ = file_set_contents(path, new_data.to_string().as_bytes(), None);
 121 }
 122
 123 fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 124
 125     let mut result = vec![];
 126
 127     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 128         Ok(v) => v,
 129         _ => return result,
 130     };
 131
 132     // usually $HOME/.cache/proxmox-backup/repo-list
 133     let path = match base.place_cache_file("repo-list") {
 134         Ok(v) => v,
 135         _ => return result,
 136     };
 137
 138     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 139
 140     if let Some(map) = data.as_object() {
 141         for (repo, _count) in map {
 142             result.push(repo.to_owned());
 143         }
 144     }
 145
 146     result
 147 }
 148
 149 async fn backup_directory<P: AsRef<Path>>(
 150     client: &BackupWriter,
 151     dir_path: P,
 152     archive_name: &str,
 153     chunk_size: Option<usize>,
 154     device_set: Option<HashSet<u64>>,
 155     verbose: bool,
 156     skip_lost_and_found: bool,
 157     crypt_config: Option<Arc<CryptConfig>>,
 158     catalog: Arc<Mutex<CatalogWriter<SenderWriter>>>,
 159 ) -> Result<BackupStats, Error> {
 160
 161     let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found, catalog)?;
 162     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 163
 164     let (mut tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
 165
 166     let stream = rx
 167         .map_err(Error::from);
 168
 169     // spawn chunker inside a separate task so that it can run parallel
 170     tokio::spawn(async move {
 171         let _ = tx.send_all(&mut chunk_stream).await;
 172     });
 173
 174     let stats = client
 175         .upload_stream(archive_name, stream, "dynamic", None, crypt_config)
 176         .await?;
 177
 178     Ok(stats)
 179 }
 180
 181 async fn backup_image<P: AsRef<Path>>(
 182     client: &BackupWriter,
 183     image_path: P,
 184     archive_name: &str,
 185     image_size: u64,
 186     chunk_size: Option<usize>,
 187     _verbose: bool,
 188     crypt_config: Option<Arc<CryptConfig>>,
 189 ) -> Result<BackupStats, Error> {
 190
 191     let path = image_path.as_ref().to_owned();
 192
 193     let file = tokio::fs::File::open(path).await?;
 194
 195     let stream = tokio::codec::FramedRead::new(file, tokio::codec::BytesCodec::new())
 196         .map_err(Error::from);
 197
 198     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 199
 200     let stats = client
 201         .upload_stream(archive_name, stream, "fixed", Some(image_size), crypt_config)
 202         .await?;
 203
 204     Ok(stats)
 205 }
 206
 207 fn strip_server_file_expenstion(name: &str) -> String {
 208
 209     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
 210         name[..name.len()-5].to_owned()
 211     } else {
 212         name.to_owned() // should not happen
 213     }
 214 }
 215
 216 fn list_backup_groups(
 217     param: Value,
 218     _info: &ApiMethod,
 219     _rpcenv: &mut dyn RpcEnvironment,
 220 ) -> Result<Value, Error> {
 221
 222     let repo = extract_repository_from_value(&param)?;
 223
 224     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 225
 226     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 227
 228     let mut result = async_main(async move {
 229         client.get(&path, None).await
 230     })?;
 231
 232     record_repository(&repo);
 233
 234     // fixme: implement and use output formatter instead ..
 235     let list = result["data"].as_array_mut().unwrap();
 236
 237     list.sort_unstable_by(|a, b| {
 238         let a_id = a["backup-id"].as_str().unwrap();
 239         let a_backup_type = a["backup-type"].as_str().unwrap();
 240         let b_id = b["backup-id"].as_str().unwrap();
 241         let b_backup_type = b["backup-type"].as_str().unwrap();
 242
 243         let type_order = a_backup_type.cmp(b_backup_type);
 244         if type_order == std::cmp::Ordering::Equal {
 245             a_id.cmp(b_id)
 246         } else {
 247             type_order
 248         }
 249     });
 250
 251     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 252
 253     let mut result = vec![];
 254
 255     for item in list {
 256
 257         let id = item["backup-id"].as_str().unwrap();
 258         let btype = item["backup-type"].as_str().unwrap();
 259         let epoch = item["last-backup"].as_i64().unwrap();
 260         let last_backup = Utc.timestamp(epoch, 0);
 261         let backup_count = item["backup-count"].as_u64().unwrap();
 262
 263         let group = BackupGroup::new(btype, id);
 264
 265         let path = group.group_path().to_str().unwrap().to_owned();
 266
 267         let files = item["files"].as_array().unwrap().iter()
 268             .map(|v| strip_server_file_expenstion(v.as_str().unwrap())).collect();
 269
 270         if output_format == "text" {
 271             println!(
 272                 "{:20} | {} | {:5} | {}",
 273                 path,
 274                 BackupDir::backup_time_to_string(last_backup),
 275                 backup_count,
 276                 tools::join(&files, ' '),
 277             );
 278         } else {
 279             result.push(json!({
 280                 "backup-type": btype,
 281                 "backup-id": id,
 282                 "last-backup": epoch,
 283                 "backup-count": backup_count,
 284                 "files": files,
 285             }));
 286         }
 287     }
 288
 289     if output_format != "text" { format_and_print_result(&result.into(), &output_format); }
 290
 291     Ok(Value::Null)
 292 }
 293
 294 fn list_snapshots(
 295     param: Value,
 296     _info: &ApiMethod,
 297     _rpcenv: &mut dyn RpcEnvironment,
 298 ) -> Result<Value, Error> {
 299
 300     let repo = extract_repository_from_value(&param)?;
 301
 302     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 303
 304     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 305
 306     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 307
 308     let mut args = json!({});
 309     if let Some(path) = param["group"].as_str() {
 310         let group = BackupGroup::parse(path)?;
 311         args["backup-type"] = group.backup_type().into();
 312         args["backup-id"] = group.backup_id().into();
 313     }
 314
 315     let result = async_main(async move {
 316         client.get(&path, Some(args)).await
 317     })?;
 318
 319     record_repository(&repo);
 320
 321     let list = result["data"].as_array().unwrap();
 322
 323     let mut result = vec![];
 324
 325     for item in list {
 326
 327         let id = item["backup-id"].as_str().unwrap();
 328         let btype = item["backup-type"].as_str().unwrap();
 329         let epoch = item["backup-time"].as_i64().unwrap();
 330
 331         let snapshot = BackupDir::new(btype, id, epoch);
 332
 333         let path = snapshot.relative_path().to_str().unwrap().to_owned();
 334
 335         let files = item["files"].as_array().unwrap().iter()
 336             .map(|v|  strip_server_file_expenstion(v.as_str().unwrap())).collect();
 337
 338         if output_format == "text" {
 339             let size_str = if let Some(size) = item["size"].as_u64() {
 340                 size.to_string()
 341             } else {
 342                 String::from("-")
 343             };
 344             println!("{} | {} | {}", path, size_str, tools::join(&files, ' '));
 345         } else {
 346             let mut data = json!({
 347                 "backup-type": btype,
 348                 "backup-id": id,
 349                 "backup-time": epoch,
 350                 "files": files,
 351             });
 352             if let Some(size) = item["size"].as_u64() {
 353                 data["size"] = size.into();
 354             }
 355             result.push(data);
 356         }
 357     }
 358
 359     if output_format != "text" { format_and_print_result(&result.into(), &output_format); }
 360
 361     Ok(Value::Null)
 362 }
 363
 364 fn forget_snapshots(
 365     param: Value,
 366     _info: &ApiMethod,
 367     _rpcenv: &mut dyn RpcEnvironment,
 368 ) -> Result<Value, Error> {
 369
 370     let repo = extract_repository_from_value(&param)?;
 371
 372     let path = tools::required_string_param(&param, "snapshot")?;
 373     let snapshot = BackupDir::parse(path)?;
 374
 375     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
 376
 377     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 378
 379     let result = async_main(async move {
 380         client.delete(&path, Some(json!({
 381             "backup-type": snapshot.group().backup_type(),
 382             "backup-id": snapshot.group().backup_id(),
 383             "backup-time": snapshot.backup_time().timestamp(),
 384         }))).await
 385     })?;
 386
 387     record_repository(&repo);
 388
 389     Ok(result)
 390 }
 391
 392 fn api_login(
 393     param: Value,
 394     _info: &ApiMethod,
 395     _rpcenv: &mut dyn RpcEnvironment,
 396 ) -> Result<Value, Error> {
 397
 398     let repo = extract_repository_from_value(&param)?;
 399
 400     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 401     async_main(async move { client.login().await })?;
 402
 403     record_repository(&repo);
 404
 405     Ok(Value::Null)
 406 }
 407
 408 fn api_logout(
 409     param: Value,
 410     _info: &ApiMethod,
 411     _rpcenv: &mut dyn RpcEnvironment,
 412 ) -> Result<Value, Error> {
 413
 414     let repo = extract_repository_from_value(&param)?;
 415
 416     delete_ticket_info(repo.host(), repo.user())?;
 417
 418     Ok(Value::Null)
 419 }
 420
 421 fn dump_catalog(
 422     param: Value,
 423     _info: &ApiMethod,
 424     _rpcenv: &mut dyn RpcEnvironment,
 425 ) -> Result<Value, Error> {
 426
 427     let repo = extract_repository_from_value(&param)?;
 428
 429     let path = tools::required_string_param(&param, "snapshot")?;
 430     let snapshot = BackupDir::parse(path)?;
 431
 432     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 433
 434     let crypt_config = match keyfile {
 435         None => None,
 436         Some(path) => {
 437             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 438             Some(Arc::new(CryptConfig::new(key)?))
 439         }
 440     };
 441
 442     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 443
 444     async_main(async move {
 445         let client = BackupReader::start(
 446             client,
 447             crypt_config.clone(),
 448             repo.store(),
 449             &snapshot.group().backup_type(),
 450             &snapshot.group().backup_id(),
 451             snapshot.backup_time(),
 452             true,
 453         ).await?;
 454
 455         let manifest = client.download_manifest().await?;
 456
 457         let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
 458
 459         let most_used = index.find_most_used_chunks(8);
 460
 461         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 462
 463         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 464
 465         let mut catalogfile = std::fs::OpenOptions::new()
 466             .write(true)
 467             .read(true)
 468             .custom_flags(libc::O_TMPFILE)
 469             .open("/tmp")?;
 470
 471         std::io::copy(&mut reader, &mut catalogfile)
 472             .map_err(|err| format_err!("unable to download catalog - {}", err))?;
 473
 474         catalogfile.seek(SeekFrom::Start(0))?;
 475
 476         let mut catalog_reader = CatalogReader::new(catalogfile);
 477
 478         catalog_reader.dump()?;
 479
 480         record_repository(&repo);
 481
 482         Ok::<(), Error>(())
 483     })?;
 484
 485     Ok(Value::Null)
 486 }
 487
 488 fn list_snapshot_files(
 489     param: Value,
 490     _info: &ApiMethod,
 491     _rpcenv: &mut dyn RpcEnvironment,
 492 ) -> Result<Value, Error> {
 493
 494     let repo = extract_repository_from_value(&param)?;
 495
 496     let path = tools::required_string_param(&param, "snapshot")?;
 497     let snapshot = BackupDir::parse(path)?;
 498
 499     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 500
 501     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 502
 503     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 504
 505     let mut result = async_main(async move {
 506         client.get(&path, Some(json!({
 507             "backup-type": snapshot.group().backup_type(),
 508             "backup-id": snapshot.group().backup_id(),
 509             "backup-time": snapshot.backup_time().timestamp(),
 510         }))).await
 511     })?;
 512
 513     record_repository(&repo);
 514
 515     let list: Value = result["data"].take();
 516
 517     if output_format == "text" {
 518         for item in list.as_array().unwrap().iter() {
 519             println!(
 520                 "{} {}",
 521                 strip_server_file_expenstion(item["filename"].as_str().unwrap()),
 522                 item["size"].as_u64().unwrap_or(0),
 523             );
 524         }
 525     } else {
 526         format_and_print_result(&list, &output_format);
 527     }
 528
 529     Ok(Value::Null)
 530 }
 531
 532 fn start_garbage_collection(
 533     param: Value,
 534     _info: &ApiMethod,
 535     _rpcenv: &mut dyn RpcEnvironment,
 536 ) -> Result<Value, Error> {
 537
 538     let repo = extract_repository_from_value(&param)?;
 539
 540     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
 541
 542     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 543
 544     let result = async_main(async move { client.post(&path, None).await })?;
 545
 546     record_repository(&repo);
 547
 548     Ok(result)
 549 }
 550
 551 fn parse_backupspec(value: &str) -> Result<(&str, &str), Error> {
 552
 553     if let Some(caps) = (BACKUPSPEC_REGEX.regex_obj)().captures(value) {
 554         return Ok((caps.get(1).unwrap().as_str(), caps.get(2).unwrap().as_str()));
 555     }
 556     bail!("unable to parse directory specification '{}'", value);
 557 }
 558
 559 fn spawn_catalog_upload(
 560     client: Arc<BackupWriter>,
 561     crypt_config: Option<Arc<CryptConfig>>,
 562 ) -> Result<
 563         (
 564             Arc<Mutex<CatalogWriter<SenderWriter>>>,
 565             tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>
 566         ), Error>
 567 {
 568     let (catalog_tx, catalog_rx) = mpsc::channel(10); // allow to buffer 10 writes
 569     let catalog_stream = catalog_rx.map_err(Error::from);
 570     let catalog_chunk_size = 512*1024;
 571     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 572
 573     let catalog = Arc::new(Mutex::new(CatalogWriter::new(SenderWriter::new(catalog_tx))?));
 574
 575     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 576
 577     tokio::spawn(async move {
 578         let catalog_upload_result = client
 579             .upload_stream(CATALOG_NAME, catalog_chunk_stream, "dynamic", None, crypt_config)
 580             .await;
 581
 582         if let Err(ref err) = catalog_upload_result {
 583             eprintln!("catalog upload error - {}", err);
 584             client.cancel();
 585         }
 586
 587         let _ = catalog_result_tx.send(catalog_upload_result);
 588     });
 589
 590     Ok((catalog, catalog_result_rx))
 591 }
 592
 593 fn create_backup(
 594     param: Value,
 595     _info: &ApiMethod,
 596     _rpcenv: &mut dyn RpcEnvironment,
 597 ) -> Result<Value, Error> {
 598
 599     let repo = extract_repository_from_value(&param)?;
 600
 601     let backupspec_list = tools::required_array_param(&param, "backupspec")?;
 602
 603     let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
 604
 605     let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
 606
 607     let verbose = param["verbose"].as_bool().unwrap_or(false);
 608
 609     let backup_time_opt = param["backup-time"].as_i64();
 610
 611     let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
 612
 613     if let Some(size) = chunk_size_opt {
 614         verify_chunk_size(size)?;
 615     }
 616
 617     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 618
 619     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 620
 621     let backup_type = param["backup-type"].as_str().unwrap_or("host");
 622
 623     let include_dev = param["include-dev"].as_array();
 624
 625     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
 626
 627     if let Some(include_dev) = include_dev {
 628         if all_file_systems {
 629             bail!("option 'all-file-systems' conflicts with option 'include-dev'");
 630         }
 631
 632         let mut set = HashSet::new();
 633         for path in include_dev {
 634             let path = path.as_str().unwrap();
 635             let stat = nix::sys::stat::stat(path)
 636                 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
 637             set.insert(stat.st_dev);
 638         }
 639         devices = Some(set);
 640     }
 641
 642     let mut upload_list = vec![];
 643
 644     enum BackupType { PXAR, IMAGE, CONFIG, LOGFILE };
 645
 646     let mut upload_catalog = false;
 647
 648     for backupspec in backupspec_list {
 649         let (target, filename) = parse_backupspec(backupspec.as_str().unwrap())?;
 650
 651         use std::os::unix::fs::FileTypeExt;
 652
 653         let metadata = std::fs::metadata(filename)
 654             .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
 655         let file_type = metadata.file_type();
 656
 657         let extension = target.rsplit('.').next()
 658             .ok_or_else(|| format_err!("missing target file extenion '{}'", target))?;
 659
 660         match extension {
 661             "pxar" => {
 662                 if !file_type.is_dir() {
 663                     bail!("got unexpected file type (expected directory)");
 664                 }
 665                 upload_list.push((BackupType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
 666                 upload_catalog = true;
 667             }
 668             "img" => {
 669
 670                 if !(file_type.is_file() || file_type.is_block_device()) {
 671                     bail!("got unexpected file type (expected file or block device)");
 672                 }
 673
 674                 let size = image_size(&PathBuf::from(filename))?;
 675
 676                 if size == 0 { bail!("got zero-sized file '{}'", filename); }
 677
 678                 upload_list.push((BackupType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
 679             }
 680             "conf" => {
 681                 if !file_type.is_file() {
 682                     bail!("got unexpected file type (expected regular file)");
 683                 }
 684                 upload_list.push((BackupType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 685             }
 686             "log" => {
 687                 if !file_type.is_file() {
 688                     bail!("got unexpected file type (expected regular file)");
 689                 }
 690                 upload_list.push((BackupType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 691             }
 692             _ => {
 693                 bail!("got unknown archive extension '{}'", extension);
 694             }
 695         }
 696     }
 697
 698     let backup_time = Utc.timestamp(backup_time_opt.unwrap_or_else(|| Utc::now().timestamp()), 0);
 699
 700     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 701     record_repository(&repo);
 702
 703     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time));
 704
 705     println!("Client name: {}", proxmox::tools::nodename());
 706
 707     let start_time = Local::now();
 708
 709     println!("Starting protocol: {}", start_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 710
 711     let (crypt_config, rsa_encrypted_key) = match keyfile {
 712         None => (None, None),
 713         Some(path) => {
 714             let (key, created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 715
 716             let crypt_config = CryptConfig::new(key)?;
 717
 718             let path = master_pubkey_path()?;
 719             if path.exists() {
 720                 let pem_data = file_get_contents(&path)?;
 721                 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
 722                 let enc_key = crypt_config.generate_rsa_encoded_key(rsa, created)?;
 723                 (Some(Arc::new(crypt_config)), Some(enc_key))
 724             } else {
 725                 (Some(Arc::new(crypt_config)), None)
 726             }
 727         }
 728     };
 729
 730     async_main(async move {
 731         let client = BackupWriter::start(
 732             client,
 733             repo.store(),
 734             backup_type,
 735             &backup_id,
 736             backup_time,
 737             verbose,
 738         ).await?;
 739
 740         let snapshot = BackupDir::new(backup_type, backup_id, backup_time.timestamp());
 741         let mut manifest = BackupManifest::new(snapshot);
 742
 743         let (catalog, catalog_result_rx) = spawn_catalog_upload(client.clone(), crypt_config.clone())?;
 744
 745         for (backup_type, filename, target, size) in upload_list {
 746             match backup_type {
 747                 BackupType::CONFIG => {
 748                     println!("Upload config file '{}' to '{:?}' as {}", filename, repo, target);
 749                     let stats = client
 750                         .upload_blob_from_file(&filename, &target, crypt_config.clone(), true)
 751                         .await?;
 752                     manifest.add_file(target, stats.size, stats.csum);
 753                 }
 754                 BackupType::LOGFILE => { // fixme: remove - not needed anymore ?
 755                     println!("Upload log file '{}' to '{:?}' as {}", filename, repo, target);
 756                     let stats = client
 757                         .upload_blob_from_file(&filename, &target, crypt_config.clone(), true)
 758                         .await?;
 759                     manifest.add_file(target, stats.size, stats.csum);
 760                 }
 761                 BackupType::PXAR => {
 762                     println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
 763                     catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
 764                     let stats = backup_directory(
 765                         &client,
 766                         &filename,
 767                         &target,
 768                         chunk_size_opt,
 769                         devices.clone(),
 770                         verbose,
 771                         skip_lost_and_found,
 772                         crypt_config.clone(),
 773                         catalog.clone(),
 774                     ).await?;
 775                     manifest.add_file(target, stats.size, stats.csum);
 776                     catalog.lock().unwrap().end_directory()?;
 777                 }
 778                 BackupType::IMAGE => {
 779                     println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
 780                     let stats = backup_image(
 781                         &client,
 782                         &filename,
 783                         &target,
 784                         size,
 785                         chunk_size_opt,
 786                         verbose,
 787                         crypt_config.clone(),
 788                     ).await?;
 789                     manifest.add_file(target, stats.size, stats.csum);
 790                 }
 791             }
 792         }
 793
 794         // finalize and upload catalog
 795         if upload_catalog {
 796             let mutex = Arc::try_unwrap(catalog)
 797                 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
 798             let mut catalog = mutex.into_inner().unwrap();
 799
 800             catalog.finish()?;
 801
 802             drop(catalog); // close upload stream
 803
 804             let stats = catalog_result_rx.await??;
 805
 806             manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum);
 807         }
 808
 809         if let Some(rsa_encrypted_key) = rsa_encrypted_key {
 810             let target = "rsa-encrypted.key";
 811             println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
 812             let stats = client
 813                 .upload_blob_from_data(rsa_encrypted_key, target, None, false, false)
 814                 .await?;
 815             manifest.add_file(format!("{}.blob", target), stats.size, stats.csum);
 816
 817             // openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out t
 818             /*
 819             let mut buffer2 = vec![0u8; rsa.size() as usize];
 820             let pem_data = file_get_contents("master-private.pem")?;
 821             let rsa = openssl::rsa::Rsa::private_key_from_pem(&pem_data)?;
 822             let len = rsa.private_decrypt(&buffer, &mut buffer2, openssl::rsa::Padding::PKCS1)?;
 823             println!("TEST {} {:?}", len, buffer2);
 824              */
 825         }
 826
 827         // create manifest (index.json)
 828         let manifest = manifest.into_json();
 829
 830         println!("Upload index.json to '{:?}'", repo);
 831         let manifest = serde_json::to_string_pretty(&manifest)?.into();
 832         client
 833             .upload_blob_from_data(manifest, MANIFEST_BLOB_NAME, crypt_config.clone(), true, true)
 834             .await?;
 835
 836         client.finish().await?;
 837
 838         let end_time = Local::now();
 839         let elapsed = end_time.signed_duration_since(start_time);
 840         println!("Duration: {}", elapsed);
 841
 842         println!("End Time: {}", end_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 843
 844         Ok(Value::Null)
 845     })
 846 }
 847
 848 fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 849
 850     let mut result = vec![];
 851
 852     let data: Vec<&str> = arg.splitn(2, ':').collect();
 853
 854     if data.len() != 2 {
 855         result.push(String::from("root.pxar:/"));
 856         result.push(String::from("etc.pxar:/etc"));
 857         return result;
 858     }
 859
 860     let files = tools::complete_file_name(data[1], param);
 861
 862     for file in files {
 863         result.push(format!("{}:{}", data[0], file));
 864     }
 865
 866     result
 867 }
 868
 869 fn restore(
 870     param: Value,
 871     _info: &ApiMethod,
 872     _rpcenv: &mut dyn RpcEnvironment,
 873 ) -> Result<Value, Error> {
 874     async_main(restore_do(param))
 875 }
 876
 877 fn dump_image<W: Write>(
 878     client: Arc<BackupReader>,
 879     crypt_config: Option<Arc<CryptConfig>>,
 880     index: FixedIndexReader,
 881     mut writer: W,
 882     verbose: bool,
 883 ) -> Result<(), Error> {
 884
 885     let most_used = index.find_most_used_chunks(8);
 886
 887     let mut chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 888
 889     // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
 890     // and thus slows down reading. Instead, directly use RemoteChunkReader
 891     let mut per = 0;
 892     let mut bytes = 0;
 893     let start_time = std::time::Instant::now();
 894
 895     for pos in 0..index.index_count() {
 896         let digest = index.index_digest(pos).unwrap();
 897         let raw_data = chunk_reader.read_chunk(&digest)?;
 898         writer.write_all(&raw_data)?;
 899         bytes += raw_data.len();
 900         if verbose {
 901             let next_per = ((pos+1)*100)/index.index_count();
 902             if per != next_per {
 903                 eprintln!("progress {}% (read {} bytes, duration {} sec)",
 904                           next_per, bytes, start_time.elapsed().as_secs());
 905                 per = next_per;
 906             }
 907         }
 908     }
 909
 910     let end_time = std::time::Instant::now();
 911     let elapsed = end_time.duration_since(start_time);
 912     eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
 913               bytes,
 914               elapsed.as_secs_f64(),
 915               bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
 916     );
 917
 918
 919     Ok(())
 920 }
 921
 922 async fn restore_do(param: Value) -> Result<Value, Error> {
 923     let repo = extract_repository_from_value(&param)?;
 924
 925     let verbose = param["verbose"].as_bool().unwrap_or(false);
 926
 927     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
 928
 929     let archive_name = tools::required_string_param(&param, "archive-name")?;
 930
 931     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 932
 933     record_repository(&repo);
 934
 935     let path = tools::required_string_param(&param, "snapshot")?;
 936
 937     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
 938         let group = BackupGroup::parse(path)?;
 939
 940         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 941         let result = client.get(&path, Some(json!({
 942             "backup-type": group.backup_type(),
 943             "backup-id": group.backup_id(),
 944         }))).await?;
 945
 946         let list = result["data"].as_array().unwrap();
 947         if list.is_empty() {
 948             bail!("backup group '{}' does not contain any snapshots:", path);
 949         }
 950
 951         let epoch = list[0]["backup-time"].as_i64().unwrap();
 952         let backup_time = Utc.timestamp(epoch, 0);
 953         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
 954     } else {
 955         let snapshot = BackupDir::parse(path)?;
 956         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
 957     };
 958
 959     let target = tools::required_string_param(&param, "target")?;
 960     let target = if target == "-" { None } else { Some(target) };
 961
 962     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 963
 964     let crypt_config = match keyfile {
 965         None => None,
 966         Some(path) => {
 967             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 968             Some(Arc::new(CryptConfig::new(key)?))
 969         }
 970     };
 971
 972     let server_archive_name = if archive_name.ends_with(".pxar") {
 973         format!("{}.didx", archive_name)
 974     } else if archive_name.ends_with(".img") {
 975         format!("{}.fidx", archive_name)
 976     } else {
 977         format!("{}.blob", archive_name)
 978     };
 979
 980     let client = BackupReader::start(
 981         client,
 982         crypt_config.clone(),
 983         repo.store(),
 984         &backup_type,
 985         &backup_id,
 986         backup_time,
 987         true,
 988     ).await?;
 989
 990     let manifest = client.download_manifest().await?;
 991
 992     if server_archive_name == MANIFEST_BLOB_NAME {
 993         let backup_index_data = manifest.into_json().to_string();
 994         if let Some(target) = target {
 995             file_set_contents(target, backup_index_data.as_bytes(), None)?;
 996         } else {
 997             let stdout = std::io::stdout();
 998             let mut writer = stdout.lock();
 999             writer.write_all(backup_index_data.as_bytes())
1000                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1001         }
1002
1003     } else if server_archive_name.ends_with(".blob") {
1004
1005         let mut reader = client.download_blob(&manifest, &server_archive_name).await?;
1006
1007         if let Some(target) = target {
1008            let mut writer = std::fs::OpenOptions::new()
1009                 .write(true)
1010                 .create(true)
1011                 .create_new(true)
1012                 .open(target)
1013                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1014             std::io::copy(&mut reader, &mut writer)?;
1015         } else {
1016             let stdout = std::io::stdout();
1017             let mut writer = stdout.lock();
1018             std::io::copy(&mut reader, &mut writer)
1019                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1020         }
1021
1022     } else if server_archive_name.ends_with(".didx") {
1023
1024         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1025
1026         let most_used = index.find_most_used_chunks(8);
1027
1028         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1029
1030         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1031
1032         if let Some(target) = target {
1033
1034             let feature_flags = pxar::flags::DEFAULT;
1035             let mut decoder = pxar::SequentialDecoder::new(&mut reader, feature_flags);
1036             decoder.set_callback(move |path| {
1037                 if verbose {
1038                     eprintln!("{:?}", path);
1039                 }
1040                 Ok(())
1041             });
1042             decoder.set_allow_existing_dirs(allow_existing_dirs);
1043
1044             decoder.restore(Path::new(target), &Vec::new())?;
1045         } else {
1046             let mut writer = std::fs::OpenOptions::new()
1047                 .write(true)
1048                 .open("/dev/stdout")
1049                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
1050
1051             std::io::copy(&mut reader, &mut writer)
1052                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1053         }
1054     } else if server_archive_name.ends_with(".fidx") {
1055
1056         let index = client.download_fixed_index(&manifest, &server_archive_name).await?;
1057
1058         let mut writer = if let Some(target) = target {
1059             std::fs::OpenOptions::new()
1060                 .write(true)
1061                 .create(true)
1062                 .create_new(true)
1063                 .open(target)
1064                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
1065         } else {
1066             std::fs::OpenOptions::new()
1067                 .write(true)
1068                 .open("/dev/stdout")
1069                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1070         };
1071
1072         dump_image(client.clone(), crypt_config.clone(), index, &mut writer, verbose)?;
1073
1074      } else {
1075         bail!("unknown archive file extension (expected .pxar of .img)");
1076     }
1077
1078     Ok(Value::Null)
1079 }
1080
1081 fn upload_log(
1082     param: Value,
1083     _info: &ApiMethod,
1084     _rpcenv: &mut dyn RpcEnvironment,
1085 ) -> Result<Value, Error> {
1086
1087     let logfile = tools::required_string_param(&param, "logfile")?;
1088     let repo = extract_repository_from_value(&param)?;
1089
1090     let snapshot = tools::required_string_param(&param, "snapshot")?;
1091     let snapshot = BackupDir::parse(snapshot)?;
1092
1093     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
1094
1095     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1096
1097     let crypt_config = match keyfile {
1098         None => None,
1099         Some(path) => {
1100             let (key, _created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1101             let crypt_config = CryptConfig::new(key)?;
1102             Some(Arc::new(crypt_config))
1103         }
1104     };
1105
1106     let data = file_get_contents(logfile)?;
1107
1108     let blob = DataBlob::encode(&data, crypt_config.as_ref().map(Arc::as_ref), true)?;
1109
1110     let raw_data = blob.into_inner();
1111
1112     let path = format!("api2/json/admin/datastore/{}/upload-backup-log", repo.store());
1113
1114     let args = json!({
1115         "backup-type": snapshot.group().backup_type(),
1116         "backup-id":  snapshot.group().backup_id(),
1117         "backup-time": snapshot.backup_time().timestamp(),
1118     });
1119
1120     let body = hyper::Body::from(raw_data);
1121
1122     async_main(async move {
1123         client.upload("application/octet-stream", body, &path, Some(args)).await
1124     })
1125 }
1126
1127 fn display_task_log(
1128     client: HttpClient,
1129     upid_str: &str,
1130     strip_date: bool,
1131 ) -> Result<(), Error> {
1132
1133     let path = format!("api2/json/nodes/localhost/tasks/{}/log", upid_str);
1134
1135     let mut start = 1;
1136     let limit = 500;
1137
1138     loop {
1139         let param = json!({ "start": start, "limit": limit, "test-status": true });
1140         let result = async_main(async { client.get(&path, Some(param)).await })?;
1141
1142         let active = result["active"].as_bool().unwrap();
1143         let total = result["total"].as_u64().unwrap();
1144         let data = result["data"].as_array().unwrap();
1145
1146         let lines = data.len();
1147
1148         for item in data {
1149             let n = item["n"].as_u64().unwrap();
1150             let t = item["t"].as_str().unwrap();
1151             if n != start { bail!("got wrong line number in response data ({} != {}", n, start); }
1152             let b = t.as_bytes();
1153             if strip_date && b.len() > 27 && b[25] == b':' && b[26] == b' '{
1154                 let line = &t[27..];
1155                 println!("{}", line);
1156             } else {
1157                 println!("{}", t);
1158             }
1159             start += 1;
1160        }
1161
1162         if start > total {
1163             if active {
1164                 std::thread::sleep(std::time::Duration::from_millis(1000));
1165             } else {
1166                 break;
1167             }
1168         } else {
1169             if lines != limit { bail!("got wrong number of lines from server ({} != {})", lines, limit); }
1170         }
1171     }
1172
1173     Ok(())
1174 }
1175
1176 fn prune(
1177     mut param: Value,
1178     _info: &ApiMethod,
1179     _rpcenv: &mut dyn RpcEnvironment,
1180 ) -> Result<Value, Error> {
1181
1182     let repo = extract_repository_from_value(&param)?;
1183
1184     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
1185
1186     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
1187
1188     let group = tools::required_string_param(&param, "group")?;
1189     let group = BackupGroup::parse(group)?;
1190     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
1191
1192     let dry_run = param["dry-run"].as_bool().unwrap_or(false);
1193
1194     param.as_object_mut().unwrap().remove("repository");
1195     param.as_object_mut().unwrap().remove("group");
1196     param.as_object_mut().unwrap().remove("dry-run");
1197     param.as_object_mut().unwrap().remove("output-format");
1198
1199     param["backup-type"] = group.backup_type().into();
1200     param["backup-id"] = group.backup_id().into();
1201
1202     if dry_run {
1203         let result = async_main(async { client.get(&path, Some(param)).await })?;
1204         let data = &result["data"];
1205
1206         if output_format == "text" {
1207             for item in data.as_array().unwrap() {
1208                 let timestamp = item["backup-time"].as_i64().unwrap();
1209                 let timestamp = BackupDir::backup_time_to_string(Utc.timestamp(timestamp, 0));
1210                 let keep =  item["keep"].as_bool().unwrap();
1211                 println!("{}/{}/{} {}",
1212                          group.backup_type(),
1213                          group.backup_id(),
1214                          timestamp,
1215                          if keep { "keep" } else { "remove" },
1216                 );
1217             }
1218         } else {
1219             format_and_print_result(&data, &output_format);
1220         }
1221     } else {
1222         let result = async_main(async { client.post(&path, Some(param)).await })?;
1223         let data = &result["data"];
1224         if output_format == "text" {
1225             if let Some(upid) = data.as_str() {
1226                 display_task_log(client, upid, true)?;
1227              }
1228         } else {
1229             format_and_print_result(&data, &output_format);
1230         }
1231     }
1232     record_repository(&repo);
1233
1234     Ok(Value::Null)
1235 }
1236
1237 fn status(
1238     param: Value,
1239     _info: &ApiMethod,
1240     _rpcenv: &mut dyn RpcEnvironment,
1241 ) -> Result<Value, Error> {
1242
1243     let repo = extract_repository_from_value(&param)?;
1244
1245     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
1246
1247     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1248
1249     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1250
1251     let result = async_main(async move { client.get(&path, None).await })?;
1252     let data = &result["data"];
1253
1254     record_repository(&repo);
1255
1256     if output_format == "text" {
1257         let total = data["total"].as_u64().unwrap();
1258         let used = data["used"].as_u64().unwrap();
1259         let avail = data["avail"].as_u64().unwrap();
1260         let roundup = total/200;
1261
1262         println!(
1263             "total: {} used: {} ({} %) available: {}",
1264             total,
1265             used,
1266             ((used+roundup)*100)/total,
1267             avail,
1268         );
1269     } else {
1270         format_and_print_result(data, &output_format);
1271     }
1272
1273     Ok(Value::Null)
1274 }
1275
1276 // like get, but simply ignore errors and return Null instead
1277 async fn try_get(repo: &BackupRepository, url: &str) -> Value {
1278
1279     let client = match HttpClient::new(repo.host(), repo.user(), None) {
1280         Ok(v) => v,
1281         _ => return Value::Null,
1282     };
1283
1284     let mut resp = match client.get(url, None).await {
1285         Ok(v) => v,
1286         _ => return Value::Null,
1287     };
1288
1289     if let Some(map) = resp.as_object_mut() {
1290         if let Some(data) = map.remove("data") {
1291             return data;
1292         }
1293     }
1294     Value::Null
1295 }
1296
1297 fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1298     async_main(async { complete_backup_group_do(param).await })
1299 }
1300
1301 async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
1302
1303     let mut result = vec![];
1304
1305     let repo = match extract_repository_from_map(param) {
1306         Some(v) => v,
1307         _ => return result,
1308     };
1309
1310     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
1311
1312     let data = try_get(&repo, &path).await;
1313
1314     if let Some(list) = data.as_array() {
1315         for item in list {
1316             if let (Some(backup_id), Some(backup_type)) =
1317                 (item["backup-id"].as_str(), item["backup-type"].as_str())
1318             {
1319                 result.push(format!("{}/{}", backup_type, backup_id));
1320             }
1321         }
1322     }
1323
1324     result
1325 }
1326
1327 fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1328     async_main(async { complete_group_or_snapshot_do(arg, param).await })
1329 }
1330
1331 async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1332
1333     if arg.matches('/').count() < 2 {
1334         let groups = complete_backup_group_do(param).await;
1335         let mut result = vec![];
1336         for group in groups {
1337             result.push(group.to_string());
1338             result.push(format!("{}/", group));
1339         }
1340         return result;
1341     }
1342
1343     complete_backup_snapshot_do(param).await
1344 }
1345
1346 fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1347     async_main(async { complete_backup_snapshot_do(param).await })
1348 }
1349
1350 async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
1351
1352     let mut result = vec![];
1353
1354     let repo = match extract_repository_from_map(param) {
1355         Some(v) => v,
1356         _ => return result,
1357     };
1358
1359     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1360
1361     let data = try_get(&repo, &path).await;
1362
1363     if let Some(list) = data.as_array() {
1364         for item in list {
1365             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
1366                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
1367             {
1368                 let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
1369                 result.push(snapshot.relative_path().to_str().unwrap().to_owned());
1370             }
1371         }
1372     }
1373
1374     result
1375 }
1376
1377 fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1378     async_main(async { complete_server_file_name_do(param).await })
1379 }
1380
1381 async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
1382
1383     let mut result = vec![];
1384
1385     let repo = match extract_repository_from_map(param) {
1386         Some(v) => v,
1387         _ => return result,
1388     };
1389
1390     let snapshot = match param.get("snapshot") {
1391         Some(path) => {
1392             match BackupDir::parse(path) {
1393                 Ok(v) => v,
1394                 _ => return result,
1395             }
1396         }
1397         _ => return result,
1398     };
1399
1400     let query = tools::json_object_to_query(json!({
1401         "backup-type": snapshot.group().backup_type(),
1402         "backup-id": snapshot.group().backup_id(),
1403         "backup-time": snapshot.backup_time().timestamp(),
1404     })).unwrap();
1405
1406     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
1407
1408     let data = try_get(&repo, &path).await;
1409
1410     if let Some(list) = data.as_array() {
1411         for item in list {
1412             if let Some(filename) = item["filename"].as_str() {
1413                 result.push(filename.to_owned());
1414             }
1415         }
1416     }
1417
1418     result
1419 }
1420
1421 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1422     complete_server_file_name(arg, param)
1423         .iter()
1424         .map(|v| strip_server_file_expenstion(&v))
1425         .collect()
1426 }
1427
1428 fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
1429
1430     let mut result = vec![];
1431
1432     let mut size = 64;
1433     loop {
1434         result.push(size.to_string());
1435         size *= 2;
1436         if size > 4096 { break; }
1437     }
1438
1439     result
1440 }
1441
1442 fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
1443
1444     // fixme: implement other input methods
1445
1446     use std::env::VarError::*;
1447     match std::env::var("PBS_ENCRYPTION_PASSWORD") {
1448         Ok(p) => return Ok(p.as_bytes().to_vec()),
1449         Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
1450         Err(NotPresent) => {
1451             // Try another method
1452         }
1453     }
1454
1455     // If we're on a TTY, query the user for a password
1456     if crate::tools::tty::stdin_isatty() {
1457         return Ok(crate::tools::tty::read_password("Encryption Key Password: ")?);
1458     }
1459
1460     bail!("no password input mechanism available");
1461 }
1462
1463 fn key_create(
1464     param: Value,
1465     _info: &ApiMethod,
1466     _rpcenv: &mut dyn RpcEnvironment,
1467 ) -> Result<Value, Error> {
1468
1469     let path = tools::required_string_param(&param, "path")?;
1470     let path = PathBuf::from(path);
1471
1472     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1473
1474     let key = proxmox::sys::linux::random_data(32)?;
1475
1476     if kdf == "scrypt" {
1477         // always read passphrase from tty
1478         if !crate::tools::tty::stdin_isatty() {
1479             bail!("unable to read passphrase - no tty");
1480         }
1481
1482         let password = crate::tools::tty::read_password("Encryption Key Password: ")?;
1483
1484         let key_config = encrypt_key_with_passphrase(&key, &password)?;
1485
1486         store_key_config(&path, false, key_config)?;
1487
1488         Ok(Value::Null)
1489     } else if kdf == "none" {
1490         let created =  Local.timestamp(Local::now().timestamp(), 0);
1491
1492         store_key_config(&path, false, KeyConfig {
1493             kdf: None,
1494             created,
1495             modified: created,
1496             data: key,
1497         })?;
1498
1499         Ok(Value::Null)
1500     } else {
1501         unreachable!();
1502     }
1503 }
1504
1505 fn master_pubkey_path() -> Result<PathBuf, Error> {
1506     let base = BaseDirectories::with_prefix("proxmox-backup")?;
1507
1508     // usually $HOME/.config/proxmox-backup/master-public.pem
1509     let path = base.place_config_file("master-public.pem")?;
1510
1511     Ok(path)
1512 }
1513
1514 fn key_import_master_pubkey(
1515     param: Value,
1516     _info: &ApiMethod,
1517     _rpcenv: &mut dyn RpcEnvironment,
1518 ) -> Result<Value, Error> {
1519
1520     let path = tools::required_string_param(&param, "path")?;
1521     let path = PathBuf::from(path);
1522
1523     let pem_data = file_get_contents(&path)?;
1524
1525     if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
1526         bail!("Unable to decode PEM data - {}", err);
1527     }
1528
1529     let target_path = master_pubkey_path()?;
1530
1531     file_set_contents(&target_path, &pem_data, None)?;
1532
1533     println!("Imported public master key to {:?}", target_path);
1534
1535     Ok(Value::Null)
1536 }
1537
1538 fn key_create_master_key(
1539     _param: Value,
1540     _info: &ApiMethod,
1541     _rpcenv: &mut dyn RpcEnvironment,
1542 ) -> Result<Value, Error> {
1543
1544     // we need a TTY to query the new password
1545     if !crate::tools::tty::stdin_isatty() {
1546         bail!("unable to create master key - no tty");
1547     }
1548
1549     let rsa = openssl::rsa::Rsa::generate(4096)?;
1550     let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
1551
1552     let new_pw = String::from_utf8(crate::tools::tty::read_password("Master Key Password: ")?)?;
1553     let verify_pw = String::from_utf8(crate::tools::tty::read_password("Verify Password: ")?)?;
1554
1555     if new_pw != verify_pw {
1556         bail!("Password verification fail!");
1557     }
1558
1559     if new_pw.len() < 5 {
1560         bail!("Password is too short!");
1561     }
1562
1563     let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
1564     let filename_pub = "master-public.pem";
1565     println!("Writing public master key to {}", filename_pub);
1566     file_set_contents(filename_pub, pub_key.as_slice(), None)?;
1567
1568     let cipher = openssl::symm::Cipher::aes_256_cbc();
1569     let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, new_pw.as_bytes())?;
1570
1571     let filename_priv = "master-private.pem";
1572     println!("Writing private master key to {}", filename_priv);
1573     file_set_contents(filename_priv, priv_key.as_slice(), None)?;
1574
1575     Ok(Value::Null)
1576 }
1577
1578 fn key_change_passphrase(
1579     param: Value,
1580     _info: &ApiMethod,
1581     _rpcenv: &mut dyn RpcEnvironment,
1582 ) -> Result<Value, Error> {
1583
1584     let path = tools::required_string_param(&param, "path")?;
1585     let path = PathBuf::from(path);
1586
1587     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1588
1589     // we need a TTY to query the new password
1590     if !crate::tools::tty::stdin_isatty() {
1591         bail!("unable to change passphrase - no tty");
1592     }
1593
1594     let (key, created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1595
1596     if kdf == "scrypt" {
1597
1598         let new_pw = String::from_utf8(crate::tools::tty::read_password("New Password: ")?)?;
1599         let verify_pw = String::from_utf8(crate::tools::tty::read_password("Verify Password: ")?)?;
1600
1601         if new_pw != verify_pw {
1602             bail!("Password verification fail!");
1603         }
1604
1605         if new_pw.len() < 5 {
1606             bail!("Password is too short!");
1607         }
1608
1609         let mut new_key_config = encrypt_key_with_passphrase(&key, new_pw.as_bytes())?;
1610         new_key_config.created = created; // keep original value
1611
1612         store_key_config(&path, true, new_key_config)?;
1613
1614         Ok(Value::Null)
1615     } else if kdf == "none" {
1616         let modified =  Local.timestamp(Local::now().timestamp(), 0);
1617
1618         store_key_config(&path, true, KeyConfig {
1619             kdf: None,
1620             created, // keep original value
1621             modified,
1622             data: key.to_vec(),
1623         })?;
1624
1625         Ok(Value::Null)
1626     } else {
1627         unreachable!();
1628     }
1629 }
1630
1631 fn key_mgmt_cli() -> CliCommandMap {
1632
1633     const KDF_SCHEMA: Schema =
1634         StringSchema::new("Key derivation function. Choose 'none' to store the key unecrypted.")
1635         .format(&ApiStringFormat::Enum(&["scrypt", "none"]))
1636         .default("scrypt")
1637         .schema();
1638
1639     #[sortable]
1640     const API_METHOD_KEY_CREATE: ApiMethod = ApiMethod::new(
1641         &ApiHandler::Sync(&key_create),
1642         &ObjectSchema::new(
1643             "Create a new encryption key.",
1644             &sorted!([
1645                 ("path", false, &StringSchema::new("File system path.").schema()),
1646                 ("kdf", true, &KDF_SCHEMA),
1647             ]),
1648         )
1649     );
1650
1651     let key_create_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE)
1652         .arg_param(&["path"])
1653         .completion_cb("path", tools::complete_file_name);
1654
1655     #[sortable]
1656     const API_METHOD_KEY_CHANGE_PASSPHRASE: ApiMethod = ApiMethod::new(
1657         &ApiHandler::Sync(&key_change_passphrase),
1658         &ObjectSchema::new(
1659             "Change the passphrase required to decrypt the key.",
1660             &sorted!([
1661                 ("path", false, &StringSchema::new("File system path.").schema()),
1662                 ("kdf", true, &KDF_SCHEMA),
1663             ]),
1664         )
1665     );
1666
1667     let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_KEY_CHANGE_PASSPHRASE)
1668         .arg_param(&["path"])
1669         .completion_cb("path", tools::complete_file_name);
1670
1671     const API_METHOD_KEY_CREATE_MASTER_KEY: ApiMethod = ApiMethod::new(
1672         &ApiHandler::Sync(&key_create_master_key),
1673         &ObjectSchema::new("Create a new 4096 bit RSA master pub/priv key pair.", &[])
1674     );
1675
1676     let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE_MASTER_KEY);
1677
1678     #[sortable]
1679     const API_METHOD_KEY_IMPORT_MASTER_PUBKEY: ApiMethod = ApiMethod::new(
1680         &ApiHandler::Sync(&key_import_master_pubkey),
1681         &ObjectSchema::new(
1682             "Import a new RSA public key and use it as master key. The key is expected to be in '.pem' format.",
1683             &sorted!([ ("path", false, &StringSchema::new("File system path.").schema()) ]),
1684         )
1685     );
1686
1687     let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_KEY_IMPORT_MASTER_PUBKEY)
1688         .arg_param(&["path"])
1689         .completion_cb("path", tools::complete_file_name);
1690
1691     CliCommandMap::new()
1692         .insert("create".to_owned(), key_create_cmd_def.into())
1693         .insert("create-master-key".to_owned(), key_create_master_key_cmd_def.into())
1694         .insert("import-master-pubkey".to_owned(), key_import_master_pubkey_cmd_def.into())
1695         .insert("change-passphrase".to_owned(), key_change_passphrase_cmd_def.into())
1696 }
1697
1698 fn mount(
1699     param: Value,
1700     _info: &ApiMethod,
1701     _rpcenv: &mut dyn RpcEnvironment,
1702 ) -> Result<Value, Error> {
1703     let verbose = param["verbose"].as_bool().unwrap_or(false);
1704     if verbose {
1705         // This will stay in foreground with debug output enabled as None is
1706         // passed for the RawFd.
1707         return async_main(mount_do(param, None));
1708     }
1709
1710     // Process should be deamonized.
1711     // Make sure to fork before the async runtime is instantiated to avoid troubles.
1712     let pipe = pipe()?;
1713     match fork() {
1714         Ok(ForkResult::Parent { .. }) => {
1715             nix::unistd::close(pipe.1).unwrap();
1716             // Blocks the parent process until we are ready to go in the child
1717             let _res = nix::unistd::read(pipe.0, &mut [0]).unwrap();
1718             Ok(Value::Null)
1719         }
1720         Ok(ForkResult::Child) => {
1721             nix::unistd::close(pipe.0).unwrap();
1722             nix::unistd::setsid().unwrap();
1723             async_main(mount_do(param, Some(pipe.1)))
1724         }
1725         Err(_) => bail!("failed to daemonize process"),
1726     }
1727 }
1728
1729 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
1730     let repo = extract_repository_from_value(&param)?;
1731     let archive_name = tools::required_string_param(&param, "archive-name")?;
1732     let target = tools::required_string_param(&param, "target")?;
1733     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1734
1735     record_repository(&repo);
1736
1737     let path = tools::required_string_param(&param, "snapshot")?;
1738     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1739         let group = BackupGroup::parse(path)?;
1740
1741         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1742         let result = client.get(&path, Some(json!({
1743             "backup-type": group.backup_type(),
1744             "backup-id": group.backup_id(),
1745         }))).await?;
1746
1747         let list = result["data"].as_array().unwrap();
1748         if list.is_empty() {
1749             bail!("backup group '{}' does not contain any snapshots:", path);
1750         }
1751
1752         let epoch = list[0]["backup-time"].as_i64().unwrap();
1753         let backup_time = Utc.timestamp(epoch, 0);
1754         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
1755     } else {
1756         let snapshot = BackupDir::parse(path)?;
1757         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1758     };
1759
1760     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1761     let crypt_config = match keyfile {
1762         None => None,
1763         Some(path) => {
1764             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1765             Some(Arc::new(CryptConfig::new(key)?))
1766         }
1767     };
1768
1769     let server_archive_name = if archive_name.ends_with(".pxar") {
1770         format!("{}.didx", archive_name)
1771     } else {
1772         bail!("Can only mount pxar archives.");
1773     };
1774
1775     let client = BackupReader::start(
1776         client,
1777         crypt_config.clone(),
1778         repo.store(),
1779         &backup_type,
1780         &backup_id,
1781         backup_time,
1782         true,
1783     ).await?;
1784
1785     let manifest = client.download_manifest().await?;
1786
1787     if server_archive_name.ends_with(".didx") {
1788         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1789         let most_used = index.find_most_used_chunks(8);
1790         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1791         let reader = BufferedDynamicReader::new(index, chunk_reader);
1792         let decoder = pxar::Decoder::new(reader)?;
1793         let options = OsStr::new("ro,default_permissions");
1794         let mut session = pxar::fuse::Session::new(decoder, &options, pipe.is_none())
1795             .map_err(|err| format_err!("pxar mount failed: {}", err))?;
1796
1797         // Mount the session but not call fuse deamonize as this will cause
1798         // issues with the runtime after the fork
1799         let deamonize = false;
1800         session.mount(&Path::new(target), deamonize)?;
1801
1802         if let Some(pipe) = pipe {
1803             nix::unistd::chdir(Path::new("/")).unwrap();
1804             // Finish creation of deamon by redirecting filedescriptors.
1805             let nullfd = nix::fcntl::open(
1806                 "/dev/null",
1807                 nix::fcntl::OFlag::O_RDWR,
1808                 nix::sys::stat::Mode::empty(),
1809             ).unwrap();
1810             nix::unistd::dup2(nullfd, 0).unwrap();
1811             nix::unistd::dup2(nullfd, 1).unwrap();
1812             nix::unistd::dup2(nullfd, 2).unwrap();
1813             if nullfd > 2 {
1814                 nix::unistd::close(nullfd).unwrap();
1815             }
1816             // Signal the parent process that we are done with the setup and it can
1817             // terminate.
1818             nix::unistd::write(pipe, &[0u8])?;
1819             nix::unistd::close(pipe).unwrap();
1820         }
1821
1822         let multithreaded = true;
1823         session.run_loop(multithreaded)?;
1824     } else {
1825         bail!("unknown archive file extension (expected .pxar)");
1826     }
1827
1828     Ok(Value::Null)
1829 }
1830
1831 fn shell(
1832     param: Value,
1833     _info: &ApiMethod,
1834     _rpcenv: &mut dyn RpcEnvironment,
1835 ) -> Result<Value, Error> {
1836     async_main(catalog_shell(param))
1837 }
1838
1839 async fn catalog_shell(param: Value) -> Result<Value, Error> {
1840     let repo = extract_repository_from_value(&param)?;
1841     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1842     let path = tools::required_string_param(&param, "snapshot")?;
1843     let archive_name = tools::required_string_param(&param, "archive-name")?;
1844
1845     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1846         let group = BackupGroup::parse(path)?;
1847
1848         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1849         let result = client.get(&path, Some(json!({
1850             "backup-type": group.backup_type(),
1851             "backup-id": group.backup_id(),
1852         }))).await?;
1853
1854         let list = result["data"].as_array().unwrap();
1855         if list.is_empty() {
1856             bail!("backup group '{}' does not contain any snapshots:", path);
1857         }
1858
1859         let epoch = list[0]["backup-time"].as_i64().unwrap();
1860         let backup_time = Utc.timestamp(epoch, 0);
1861         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
1862     } else {
1863         let snapshot = BackupDir::parse(path)?;
1864         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1865     };
1866
1867     let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
1868     let crypt_config = match keyfile {
1869         None => None,
1870         Some(path) => {
1871             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1872             Some(Arc::new(CryptConfig::new(key)?))
1873         }
1874     };
1875
1876     let server_archive_name = if archive_name.ends_with(".pxar") {
1877         format!("{}.didx", archive_name)
1878     } else {
1879         bail!("Can only mount pxar archives.");
1880     };
1881
1882     let client = BackupReader::start(
1883         client,
1884         crypt_config.clone(),
1885         repo.store(),
1886         &backup_type,
1887         &backup_id,
1888         backup_time,
1889         true,
1890     ).await?;
1891
1892     let tmpfile = std::fs::OpenOptions::new()
1893         .write(true)
1894         .read(true)
1895         .custom_flags(libc::O_TMPFILE)
1896         .open("/tmp")?;
1897
1898     let manifest = client.download_manifest().await?;
1899
1900     let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1901     let most_used = index.find_most_used_chunks(8);
1902     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
1903     let reader = BufferedDynamicReader::new(index, chunk_reader);
1904     let mut decoder = pxar::Decoder::new(reader)?;
1905     decoder.set_callback(|path| {
1906         println!("{:?}", path);
1907         Ok(())
1908     });
1909
1910     let tmpfile = client.download(CATALOG_NAME, tmpfile).await?;
1911     let index = DynamicIndexReader::new(tmpfile)
1912         .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
1913
1914     // Note: do not use values stored in index (not trusted) - instead, computed them again
1915     let (csum, size) = index.compute_csum();
1916     manifest.verify_file(CATALOG_NAME, &csum, size)?;
1917
1918     let most_used = index.find_most_used_chunks(8);
1919     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1920     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1921     let mut catalogfile = std::fs::OpenOptions::new()
1922         .write(true)
1923         .read(true)
1924         .custom_flags(libc::O_TMPFILE)
1925         .open("/tmp")?;
1926
1927     std::io::copy(&mut reader, &mut catalogfile)
1928         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
1929
1930     catalogfile.seek(SeekFrom::Start(0))?;
1931     let catalog_reader = CatalogReader::new(catalogfile);
1932     let state = Shell::new(
1933         catalog_reader,
1934         &server_archive_name,
1935         decoder,
1936     )?;
1937
1938     println!("Starting interactive shell");
1939     state.shell()?;
1940
1941     record_repository(&repo);
1942
1943     Ok(Value::Null)
1944 }
1945
1946 fn main() {
1947
1948     const BACKUP_SOURCE_SCHEMA: Schema = StringSchema::new("Backup source specification ([<label>:<path>]).")
1949         .format(&ApiStringFormat::Pattern(&BACKUPSPEC_REGEX))
1950         .schema();
1951
1952     #[sortable]
1953     const API_METHOD_CREATE_BACKUP: ApiMethod = ApiMethod::new(
1954         &ApiHandler::Sync(&create_backup),
1955         &ObjectSchema::new(
1956             "Create (host) backup.",
1957             &sorted!([
1958                 (
1959                     "backupspec",
1960                     false,
1961                     &ArraySchema::new(
1962                         "List of backup source specifications ([<label.ext>:<path>] ...)",
1963                         &BACKUP_SOURCE_SCHEMA,
1964                     ).min_length(1).schema()
1965                 ),
1966                 (
1967                     "repository",
1968                     true,
1969                     &REPO_URL_SCHEMA
1970                 ),
1971                 (
1972                     "include-dev",
1973                     true,
1974                     &ArraySchema::new(
1975                         "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
1976                         &StringSchema::new("Path to file.").schema()
1977                     ).schema()
1978                 ),
1979                 (
1980                     "keyfile",
1981                     true,
1982                     &StringSchema::new("Path to encryption key. All data will be encrypted using this key.").schema()
1983                 ),
1984                 (
1985                     "verbose",
1986                     true,
1987                     &BooleanSchema::new("Verbose output.")
1988                         .default(false)
1989                         .schema()
1990                 ),
1991                 (
1992                     "skip-lost-and-found",
1993                     true,
1994                     &BooleanSchema::new("Skip lost+found directory")
1995                         .default(false)
1996                         .schema()
1997                 ),
1998                 (
1999                     "backup-type",
2000                     true,
2001                     &BACKUP_TYPE_SCHEMA,
2002                 ),
2003                 (
2004                     "backup-id",
2005                     true,
2006                     &BACKUP_ID_SCHEMA
2007                 ),
2008                 (
2009                     "backup-time",
2010                     true,
2011                     &BACKUP_TIME_SCHEMA
2012                 ),
2013                 (
2014                     "chunk-size",
2015                     true,
2016                     &IntegerSchema::new("Chunk size in KB. Must be a power of 2.")
2017                         .minimum(64)
2018                         .maximum(4096)
2019                         .default(4096)
2020                         .schema()
2021                 ),
2022             ]),
2023         )
2024     );
2025
2026     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
2027         .arg_param(&["backupspec"])
2028         .completion_cb("repository", complete_repository)
2029         .completion_cb("backupspec", complete_backup_source)
2030         .completion_cb("keyfile", tools::complete_file_name)
2031         .completion_cb("chunk-size", complete_chunk_size);
2032
2033     #[sortable]
2034     const API_METHOD_UPLOAD_LOG: ApiMethod = ApiMethod::new(
2035         &ApiHandler::Sync(&upload_log),
2036         &ObjectSchema::new(
2037             "Upload backup log file.",
2038             &sorted!([
2039                 (
2040                     "snapshot",
2041                     false,
2042                     &StringSchema::new("Snapshot path.").schema()
2043                 ),
2044                 (
2045                     "logfile",
2046                     false,
2047                     &StringSchema::new("The path to the log file you want to upload.").schema()
2048                 ),
2049                 (
2050                     "repository",
2051                     true,
2052                     &REPO_URL_SCHEMA
2053                 ),
2054                 (
2055                     "keyfile",
2056                     true,
2057                     &StringSchema::new("Path to encryption key. All data will be encrypted using this key.").schema()
2058                 ),
2059             ]),
2060         )
2061     );
2062
2063     let upload_log_cmd_def = CliCommand::new(&API_METHOD_UPLOAD_LOG)
2064         .arg_param(&["snapshot", "logfile"])
2065         .completion_cb("snapshot", complete_backup_snapshot)
2066         .completion_cb("logfile", tools::complete_file_name)
2067         .completion_cb("keyfile", tools::complete_file_name)
2068         .completion_cb("repository", complete_repository);
2069
2070     #[sortable]
2071     const API_METHOD_LIST_BACKUP_GROUPS: ApiMethod = ApiMethod::new(
2072         &ApiHandler::Sync(&list_backup_groups),
2073         &ObjectSchema::new(
2074             "List backup groups.",
2075             &sorted!([
2076                 ("repository", true, &REPO_URL_SCHEMA),
2077                 ("output-format", true, &OUTPUT_FORMAT),
2078             ]),
2079         )
2080     );
2081
2082     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
2083         .completion_cb("repository", complete_repository);
2084
2085     #[sortable]
2086     const API_METHOD_LIST_SNAPSHOTS: ApiMethod = ApiMethod::new(
2087         &ApiHandler::Sync(&list_snapshots),
2088         &ObjectSchema::new(
2089             "List backup snapshots.",
2090             &sorted!([
2091                 ("group", true, &StringSchema::new("Backup group.").schema()),
2092                 ("repository", true, &REPO_URL_SCHEMA),
2093                 ("output-format", true, &OUTPUT_FORMAT),
2094             ]),
2095         )
2096     );
2097
2098     let snapshots_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOTS)
2099         .arg_param(&["group"])
2100         .completion_cb("group", complete_backup_group)
2101         .completion_cb("repository", complete_repository);
2102
2103     #[sortable]
2104     const API_METHOD_FORGET_SNAPSHOTS: ApiMethod = ApiMethod::new(
2105         &ApiHandler::Sync(&forget_snapshots),
2106         &ObjectSchema::new(
2107             "Forget (remove) backup snapshots.",
2108             &sorted!([
2109                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2110                 ("repository", true, &REPO_URL_SCHEMA),
2111             ]),
2112         )
2113     );
2114
2115     let forget_cmd_def = CliCommand::new(&API_METHOD_FORGET_SNAPSHOTS)
2116         .arg_param(&["snapshot"])
2117         .completion_cb("repository", complete_repository)
2118         .completion_cb("snapshot", complete_backup_snapshot);
2119
2120     #[sortable]
2121     const API_METHOD_START_GARBAGE_COLLECTION: ApiMethod = ApiMethod::new(
2122         &ApiHandler::Sync(&start_garbage_collection),
2123         &ObjectSchema::new(
2124             "Start garbage collection for a specific repository.",
2125             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2126         )
2127     );
2128
2129     let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
2130         .completion_cb("repository", complete_repository);
2131
2132     #[sortable]
2133     const API_METHOD_RESTORE: ApiMethod = ApiMethod::new(
2134         &ApiHandler::Sync(&restore),
2135         &ObjectSchema::new(
2136             "Restore backup repository.",
2137             &sorted!([
2138                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2139                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2140                 (
2141                     "target",
2142                     false,
2143                     &StringSchema::new(
2144                         r###"Target directory path. Use '-' to write to stdandard output.
2145
2146 We do not extraxt '.pxar' archives when writing to stdandard output.
2147
2148 "###
2149                     ).schema()
2150                 ),
2151                 (
2152                     "allow-existing-dirs",
2153                     true,
2154                     &BooleanSchema::new("Do not fail if directories already exists.")
2155                         .default(false)
2156                         .schema()
2157                 ),
2158                 ("repository", true, &REPO_URL_SCHEMA),
2159                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2160                 (
2161                     "verbose",
2162                     true,
2163                     &BooleanSchema::new("Verbose output.")
2164                         .default(false)
2165                         .schema()
2166                 ),
2167             ]),
2168         )
2169     );
2170
2171     let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
2172         .arg_param(&["snapshot", "archive-name", "target"])
2173         .completion_cb("repository", complete_repository)
2174         .completion_cb("snapshot", complete_group_or_snapshot)
2175         .completion_cb("archive-name", complete_archive_name)
2176         .completion_cb("target", tools::complete_file_name);
2177
2178     #[sortable]
2179     const API_METHOD_LIST_SNAPSHOT_FILES: ApiMethod = ApiMethod::new(
2180         &ApiHandler::Sync(&list_snapshot_files),
2181         &ObjectSchema::new(
2182             "List snapshot files.",
2183             &sorted!([
2184                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2185                 ("repository", true, &REPO_URL_SCHEMA),
2186                 ("output-format", true, &OUTPUT_FORMAT),
2187             ]),
2188         )
2189     );
2190
2191     let files_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOT_FILES)
2192         .arg_param(&["snapshot"])
2193         .completion_cb("repository", complete_repository)
2194         .completion_cb("snapshot", complete_backup_snapshot);
2195
2196     #[sortable]
2197     const API_METHOD_DUMP_CATALOG: ApiMethod = ApiMethod::new(
2198         &ApiHandler::Sync(&dump_catalog),
2199         &ObjectSchema::new(
2200             "Dump catalog.",
2201             &sorted!([
2202                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2203                 ("repository", true, &REPO_URL_SCHEMA),
2204             ]),
2205         )
2206     );
2207
2208     let catalog_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
2209         .arg_param(&["snapshot"])
2210         .completion_cb("repository", complete_repository)
2211         .completion_cb("snapshot", complete_backup_snapshot);
2212
2213     const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
2214         &ApiHandler::Sync(&prune),
2215         &ObjectSchema::new(
2216             "Prune backup repository.",
2217             &proxmox_backup::add_common_prune_prameters!([
2218                 ("dry-run", true, &BooleanSchema::new(
2219                     "Just show what prune would do, but do not delete anything.")
2220                  .schema()),
2221                 ("group", false, &StringSchema::new("Backup group.").schema()),
2222             ], [
2223                 ("output-format", true, &OUTPUT_FORMAT),
2224                 ("repository", true, &REPO_URL_SCHEMA),
2225             ])
2226         )
2227     );
2228
2229     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
2230         .arg_param(&["group"])
2231         .completion_cb("group", complete_backup_group)
2232         .completion_cb("repository", complete_repository);
2233
2234     #[sortable]
2235     const API_METHOD_STATUS: ApiMethod = ApiMethod::new(
2236         &ApiHandler::Sync(&status),
2237         &ObjectSchema::new(
2238             "Get repository status.",
2239             &sorted!([
2240                 ("repository", true, &REPO_URL_SCHEMA),
2241                 ("output-format", true, &OUTPUT_FORMAT),
2242             ]),
2243         )
2244     );
2245
2246     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
2247         .completion_cb("repository", complete_repository);
2248
2249     #[sortable]
2250     const API_METHOD_API_LOGIN: ApiMethod = ApiMethod::new(
2251         &ApiHandler::Sync(&api_login),
2252         &ObjectSchema::new(
2253             "Try to login. If successful, store ticket.",
2254             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2255         )
2256     );
2257
2258     let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
2259         .completion_cb("repository", complete_repository);
2260
2261     #[sortable]
2262     const API_METHOD_API_LOGOUT: ApiMethod = ApiMethod::new(
2263         &ApiHandler::Sync(&api_logout),
2264         &ObjectSchema::new(
2265             "Logout (delete stored ticket).",
2266             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2267         )
2268     );
2269
2270     let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
2271         .completion_cb("repository", complete_repository);
2272
2273     #[sortable]
2274     const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
2275         &ApiHandler::Sync(&mount),
2276         &ObjectSchema::new(
2277             "Mount pxar archive.",
2278             &sorted!([
2279                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2280                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2281                 ("target", false, &StringSchema::new("Target directory path.").schema()),
2282                 ("repository", true, &REPO_URL_SCHEMA),
2283                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2284                 ("verbose", true, &BooleanSchema::new("Verbose output.").default(false).schema()),
2285             ]),
2286         )
2287     );
2288
2289     let mount_cmd_def = CliCommand::new(&API_METHOD_MOUNT)
2290         .arg_param(&["snapshot", "archive-name", "target"])
2291         .completion_cb("repository", complete_repository)
2292         .completion_cb("snapshot", complete_group_or_snapshot)
2293         .completion_cb("archive-name", complete_archive_name)
2294         .completion_cb("target", tools::complete_file_name);
2295
2296     #[sortable]
2297     const API_METHOD_SHELL: ApiMethod = ApiMethod::new(
2298         &ApiHandler::Sync(&shell),
2299         &ObjectSchema::new(
2300             "Shell to interactively inspect and restore snapshots.",
2301             &sorted!([
2302                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2303                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2304                 ("repository", true, &REPO_URL_SCHEMA),
2305                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2306             ]),
2307         )
2308     );
2309
2310     let shell_cmd_def = CliCommand::new(&API_METHOD_SHELL)
2311         .arg_param(&["snapshot", "archive-name"])
2312         .completion_cb("repository", complete_repository)
2313         .completion_cb("archive-name", complete_archive_name)
2314         .completion_cb("snapshot", complete_group_or_snapshot);
2315
2316     let cmd_def = CliCommandMap::new()
2317         .insert("backup".to_owned(), backup_cmd_def.into())
2318         .insert("upload-log".to_owned(), upload_log_cmd_def.into())
2319         .insert("forget".to_owned(), forget_cmd_def.into())
2320         .insert("catalog".to_owned(), catalog_cmd_def.into())
2321         .insert("garbage-collect".to_owned(), garbage_collect_cmd_def.into())
2322         .insert("list".to_owned(), list_cmd_def.into())
2323         .insert("login".to_owned(), login_cmd_def.into())
2324         .insert("logout".to_owned(), logout_cmd_def.into())
2325         .insert("prune".to_owned(), prune_cmd_def.into())
2326         .insert("restore".to_owned(), restore_cmd_def.into())
2327         .insert("snapshots".to_owned(), snapshots_cmd_def.into())
2328         .insert("files".to_owned(), files_cmd_def.into())
2329         .insert("status".to_owned(), status_cmd_def.into())
2330         .insert("key".to_owned(), key_mgmt_cli().into())
2331         .insert("mount".to_owned(), mount_cmd_def.into())
2332         .insert("shell".to_owned(), shell_cmd_def.into());
2333
2334     run_cli_command(cmd_def.into());
2335 }
2336
2337 fn async_main<F: Future>(fut: F) -> <F as Future>::Output {
2338     let rt = tokio::runtime::Runtime::new().unwrap();
2339     let ret = rt.block_on(fut);
2340     rt.shutdown_now();
2341     ret
2342 }