src/bin/proxmox-backup-client.rs

   1 use failure::*;
   2 use nix::unistd::{fork, ForkResult, pipe};
   3 use std::os::unix::io::RawFd;
   4 use chrono::{Local, Utc, TimeZone};
   5 use std::path::{Path, PathBuf};
   6 use std::collections::{HashSet, HashMap};
   7 use std::ffi::OsStr;
   8 use std::io::{Write, Seek, SeekFrom};
   9 use std::os::unix::fs::OpenOptionsExt;
  10
  11 use proxmox::{sortable, identity};
  12 use proxmox::tools::fs::{file_get_contents, file_get_json, file_set_contents, image_size};
  13 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
  14 use proxmox::api::schema::*;
  15 use proxmox::api::cli::*;
  16
  17 use proxmox_backup::tools;
  18 use proxmox_backup::api2::types::*;
  19 use proxmox_backup::client::*;
  20 use proxmox_backup::backup::*;
  21 use proxmox_backup::pxar::{ self, catalog::* };
  22
  23 //use proxmox_backup::backup::image_index::*;
  24 //use proxmox_backup::config::datastore;
  25 //use proxmox_backup::pxar::encoder::*;
  26 //use proxmox_backup::backup::datastore::*;
  27
  28 use serde_json::{json, Value};
  29 //use hyper::Body;
  30 use std::sync::{Arc, Mutex};
  31 //use regex::Regex;
  32 use xdg::BaseDirectories;
  33
  34 use futures::*;
  35 use tokio::sync::mpsc;
  36
  37 proxmox::api::const_regex! {
  38     BACKUPSPEC_REGEX = r"^([a-zA-Z0-9_-]+\.(?:pxar|img|conf|log)):(.+)$";
  39 }
  40
  41 const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  42     .format(&BACKUP_REPO_URL)
  43     .max_length(256)
  44     .schema();
  45
  46 fn get_default_repository() -> Option<String> {
  47     std::env::var("PBS_REPOSITORY").ok()
  48 }
  49
  50 fn extract_repository_from_value(
  51     param: &Value,
  52 ) -> Result<BackupRepository, Error> {
  53
  54     let repo_url = param["repository"]
  55         .as_str()
  56         .map(String::from)
  57         .or_else(get_default_repository)
  58         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
  59
  60     let repo: BackupRepository = repo_url.parse()?;
  61
  62     Ok(repo)
  63 }
  64
  65 fn extract_repository_from_map(
  66     param: &HashMap<String, String>,
  67 ) -> Option<BackupRepository> {
  68
  69     param.get("repository")
  70         .map(String::from)
  71         .or_else(get_default_repository)
  72         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
  73 }
  74
  75 fn record_repository(repo: &BackupRepository) {
  76
  77     let base = match BaseDirectories::with_prefix("proxmox-backup") {
  78         Ok(v) => v,
  79         _ => return,
  80     };
  81
  82     // usually $HOME/.cache/proxmox-backup/repo-list
  83     let path = match base.place_cache_file("repo-list") {
  84         Ok(v) => v,
  85         _ => return,
  86     };
  87
  88     let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
  89
  90     let repo = repo.to_string();
  91
  92     data[&repo] = json!{ data[&repo].as_i64().unwrap_or(0) + 1 };
  93
  94     let mut map = serde_json::map::Map::new();
  95
  96     loop {
  97         let mut max_used = 0;
  98         let mut max_repo = None;
  99         for (repo, count) in data.as_object().unwrap() {
 100             if map.contains_key(repo) { continue; }
 101             if let Some(count) = count.as_i64() {
 102                 if count > max_used {
 103                     max_used = count;
 104                     max_repo = Some(repo);
 105                 }
 106             }
 107         }
 108         if let Some(repo) = max_repo {
 109             map.insert(repo.to_owned(), json!(max_used));
 110         } else {
 111             break;
 112         }
 113         if map.len() > 10 { // store max. 10 repos
 114             break;
 115         }
 116     }
 117
 118     let new_data = json!(map);
 119
 120     let _ = file_set_contents(path, new_data.to_string().as_bytes(), None);
 121 }
 122
 123 fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 124
 125     let mut result = vec![];
 126
 127     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 128         Ok(v) => v,
 129         _ => return result,
 130     };
 131
 132     // usually $HOME/.cache/proxmox-backup/repo-list
 133     let path = match base.place_cache_file("repo-list") {
 134         Ok(v) => v,
 135         _ => return result,
 136     };
 137
 138     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 139
 140     if let Some(map) = data.as_object() {
 141         for (repo, _count) in map {
 142             result.push(repo.to_owned());
 143         }
 144     }
 145
 146     result
 147 }
 148
 149 async fn backup_directory<P: AsRef<Path>>(
 150     client: &BackupWriter,
 151     dir_path: P,
 152     archive_name: &str,
 153     chunk_size: Option<usize>,
 154     device_set: Option<HashSet<u64>>,
 155     verbose: bool,
 156     skip_lost_and_found: bool,
 157     crypt_config: Option<Arc<CryptConfig>>,
 158     catalog: Arc<Mutex<CatalogWriter<SenderWriter>>>,
 159 ) -> Result<BackupStats, Error> {
 160
 161     let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found, catalog)?;
 162     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 163
 164     let (mut tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
 165
 166     let stream = rx
 167         .map_err(Error::from);
 168
 169     // spawn chunker inside a separate task so that it can run parallel
 170     tokio::spawn(async move {
 171         let _ = tx.send_all(&mut chunk_stream).await;
 172     });
 173
 174     let stats = client
 175         .upload_stream(archive_name, stream, "dynamic", None, crypt_config)
 176         .await?;
 177
 178     Ok(stats)
 179 }
 180
 181 async fn backup_image<P: AsRef<Path>>(
 182     client: &BackupWriter,
 183     image_path: P,
 184     archive_name: &str,
 185     image_size: u64,
 186     chunk_size: Option<usize>,
 187     _verbose: bool,
 188     crypt_config: Option<Arc<CryptConfig>>,
 189 ) -> Result<BackupStats, Error> {
 190
 191     let path = image_path.as_ref().to_owned();
 192
 193     let file = tokio::fs::File::open(path).await?;
 194
 195     let stream = tokio::codec::FramedRead::new(file, tokio::codec::BytesCodec::new())
 196         .map_err(Error::from);
 197
 198     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 199
 200     let stats = client
 201         .upload_stream(archive_name, stream, "fixed", Some(image_size), crypt_config)
 202         .await?;
 203
 204     Ok(stats)
 205 }
 206
 207 fn strip_server_file_expenstion(name: &str) -> String {
 208
 209     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
 210         name[..name.len()-5].to_owned()
 211     } else {
 212         name.to_owned() // should not happen
 213     }
 214 }
 215
 216 fn list_backup_groups(
 217     param: Value,
 218     _info: &ApiMethod,
 219     _rpcenv: &mut dyn RpcEnvironment,
 220 ) -> Result<Value, Error> {
 221
 222     let repo = extract_repository_from_value(&param)?;
 223
 224     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 225
 226     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 227
 228     let mut result = async_main(async move {
 229         client.get(&path, None).await
 230     })?;
 231
 232     record_repository(&repo);
 233
 234     // fixme: implement and use output formatter instead ..
 235     let list = result["data"].as_array_mut().unwrap();
 236
 237     list.sort_unstable_by(|a, b| {
 238         let a_id = a["backup-id"].as_str().unwrap();
 239         let a_backup_type = a["backup-type"].as_str().unwrap();
 240         let b_id = b["backup-id"].as_str().unwrap();
 241         let b_backup_type = b["backup-type"].as_str().unwrap();
 242
 243         let type_order = a_backup_type.cmp(b_backup_type);
 244         if type_order == std::cmp::Ordering::Equal {
 245             a_id.cmp(b_id)
 246         } else {
 247             type_order
 248         }
 249     });
 250
 251     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 252
 253     let mut result = vec![];
 254
 255     for item in list {
 256
 257         let id = item["backup-id"].as_str().unwrap();
 258         let btype = item["backup-type"].as_str().unwrap();
 259         let epoch = item["last-backup"].as_i64().unwrap();
 260         let last_backup = Utc.timestamp(epoch, 0);
 261         let backup_count = item["backup-count"].as_u64().unwrap();
 262
 263         let group = BackupGroup::new(btype, id);
 264
 265         let path = group.group_path().to_str().unwrap().to_owned();
 266
 267         let files = item["files"].as_array().unwrap().iter()
 268             .map(|v| strip_server_file_expenstion(v.as_str().unwrap())).collect();
 269
 270         if output_format == "text" {
 271             println!(
 272                 "{:20} | {} | {:5} | {}",
 273                 path,
 274                 BackupDir::backup_time_to_string(last_backup),
 275                 backup_count,
 276                 tools::join(&files, ' '),
 277             );
 278         } else {
 279             result.push(json!({
 280                 "backup-type": btype,
 281                 "backup-id": id,
 282                 "last-backup": epoch,
 283                 "backup-count": backup_count,
 284                 "files": files,
 285             }));
 286         }
 287     }
 288
 289     if output_format != "text" { format_and_print_result(&result.into(), &output_format); }
 290
 291     Ok(Value::Null)
 292 }
 293
 294 fn list_snapshots(
 295     param: Value,
 296     _info: &ApiMethod,
 297     _rpcenv: &mut dyn RpcEnvironment,
 298 ) -> Result<Value, Error> {
 299
 300     let repo = extract_repository_from_value(&param)?;
 301
 302     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 303
 304     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 305
 306     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 307
 308     let mut args = json!({});
 309     if let Some(path) = param["group"].as_str() {
 310         let group = BackupGroup::parse(path)?;
 311         args["backup-type"] = group.backup_type().into();
 312         args["backup-id"] = group.backup_id().into();
 313     }
 314
 315     let result = async_main(async move {
 316         client.get(&path, Some(args)).await
 317     })?;
 318
 319     record_repository(&repo);
 320
 321     let list = result["data"].as_array().unwrap();
 322
 323     let mut result = vec![];
 324
 325     for item in list {
 326
 327         let id = item["backup-id"].as_str().unwrap();
 328         let btype = item["backup-type"].as_str().unwrap();
 329         let epoch = item["backup-time"].as_i64().unwrap();
 330
 331         let snapshot = BackupDir::new(btype, id, epoch);
 332
 333         let path = snapshot.relative_path().to_str().unwrap().to_owned();
 334
 335         let files = item["files"].as_array().unwrap().iter()
 336             .map(|v|  strip_server_file_expenstion(v.as_str().unwrap())).collect();
 337
 338         if output_format == "text" {
 339             let size_str = if let Some(size) = item["size"].as_u64() {
 340                 size.to_string()
 341             } else {
 342                 String::from("-")
 343             };
 344             println!("{} | {} | {}", path, size_str, tools::join(&files, ' '));
 345         } else {
 346             let mut data = json!({
 347                 "backup-type": btype,
 348                 "backup-id": id,
 349                 "backup-time": epoch,
 350                 "files": files,
 351             });
 352             if let Some(size) = item["size"].as_u64() {
 353                 data["size"] = size.into();
 354             }
 355             result.push(data);
 356         }
 357     }
 358
 359     if output_format != "text" { format_and_print_result(&result.into(), &output_format); }
 360
 361     Ok(Value::Null)
 362 }
 363
 364 fn forget_snapshots(
 365     param: Value,
 366     _info: &ApiMethod,
 367     _rpcenv: &mut dyn RpcEnvironment,
 368 ) -> Result<Value, Error> {
 369
 370     let repo = extract_repository_from_value(&param)?;
 371
 372     let path = tools::required_string_param(&param, "snapshot")?;
 373     let snapshot = BackupDir::parse(path)?;
 374
 375     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
 376
 377     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 378
 379     let result = async_main(async move {
 380         client.delete(&path, Some(json!({
 381             "backup-type": snapshot.group().backup_type(),
 382             "backup-id": snapshot.group().backup_id(),
 383             "backup-time": snapshot.backup_time().timestamp(),
 384         }))).await
 385     })?;
 386
 387     record_repository(&repo);
 388
 389     Ok(result)
 390 }
 391
 392 fn api_login(
 393     param: Value,
 394     _info: &ApiMethod,
 395     _rpcenv: &mut dyn RpcEnvironment,
 396 ) -> Result<Value, Error> {
 397
 398     let repo = extract_repository_from_value(&param)?;
 399
 400     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 401     async_main(async move { client.login().await })?;
 402
 403     record_repository(&repo);
 404
 405     Ok(Value::Null)
 406 }
 407
 408 fn api_logout(
 409     param: Value,
 410     _info: &ApiMethod,
 411     _rpcenv: &mut dyn RpcEnvironment,
 412 ) -> Result<Value, Error> {
 413
 414     let repo = extract_repository_from_value(&param)?;
 415
 416     delete_ticket_info(repo.host(), repo.user())?;
 417
 418     Ok(Value::Null)
 419 }
 420
 421 fn dump_catalog(
 422     param: Value,
 423     _info: &ApiMethod,
 424     _rpcenv: &mut dyn RpcEnvironment,
 425 ) -> Result<Value, Error> {
 426
 427     let repo = extract_repository_from_value(&param)?;
 428
 429     let path = tools::required_string_param(&param, "snapshot")?;
 430     let snapshot = BackupDir::parse(path)?;
 431
 432     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 433
 434     let crypt_config = match keyfile {
 435         None => None,
 436         Some(path) => {
 437             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 438             Some(Arc::new(CryptConfig::new(key)?))
 439         }
 440     };
 441
 442     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 443
 444     async_main(async move {
 445         let client = BackupReader::start(
 446             client,
 447             crypt_config.clone(),
 448             repo.store(),
 449             &snapshot.group().backup_type(),
 450             &snapshot.group().backup_id(),
 451             snapshot.backup_time(),
 452             true,
 453         ).await?;
 454
 455         let manifest = client.download_manifest().await?;
 456
 457         let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
 458
 459         let most_used = index.find_most_used_chunks(8);
 460
 461         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 462
 463         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 464
 465         let mut catalogfile = std::fs::OpenOptions::new()
 466             .write(true)
 467             .read(true)
 468             .custom_flags(libc::O_TMPFILE)
 469             .open("/tmp")?;
 470
 471         std::io::copy(&mut reader, &mut catalogfile)
 472             .map_err(|err| format_err!("unable to download catalog - {}", err))?;
 473
 474         catalogfile.seek(SeekFrom::Start(0))?;
 475
 476         let mut catalog_reader = CatalogReader::new(catalogfile);
 477
 478         catalog_reader.dump()?;
 479
 480         record_repository(&repo);
 481
 482         Ok::<(), Error>(())
 483     })?;
 484
 485     Ok(Value::Null)
 486 }
 487
 488 fn list_snapshot_files(
 489     param: Value,
 490     _info: &ApiMethod,
 491     _rpcenv: &mut dyn RpcEnvironment,
 492 ) -> Result<Value, Error> {
 493
 494     let repo = extract_repository_from_value(&param)?;
 495
 496     let path = tools::required_string_param(&param, "snapshot")?;
 497     let snapshot = BackupDir::parse(path)?;
 498
 499     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
 500
 501     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 502
 503     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 504
 505     let mut result = async_main(async move {
 506         client.get(&path, Some(json!({
 507             "backup-type": snapshot.group().backup_type(),
 508             "backup-id": snapshot.group().backup_id(),
 509             "backup-time": snapshot.backup_time().timestamp(),
 510         }))).await
 511     })?;
 512
 513     record_repository(&repo);
 514
 515     let list: Value = result["data"].take();
 516
 517     if output_format == "text" {
 518         for item in list.as_array().unwrap().iter() {
 519             println!(
 520                 "{} {}",
 521                 strip_server_file_expenstion(item["filename"].as_str().unwrap()),
 522                 item["size"].as_u64().unwrap_or(0),
 523             );
 524         }
 525     } else {
 526         format_and_print_result(&list, &output_format);
 527     }
 528
 529     Ok(Value::Null)
 530 }
 531
 532 fn start_garbage_collection(
 533     param: Value,
 534     _info: &ApiMethod,
 535     _rpcenv: &mut dyn RpcEnvironment,
 536 ) -> Result<Value, Error> {
 537
 538     let repo = extract_repository_from_value(&param)?;
 539
 540     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
 541
 542     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 543
 544     let result = async_main(async move { client.post(&path, None).await })?;
 545
 546     record_repository(&repo);
 547
 548     Ok(result)
 549 }
 550
 551 fn parse_backupspec(value: &str) -> Result<(&str, &str), Error> {
 552
 553     if let Some(caps) = (BACKUPSPEC_REGEX.regex_obj)().captures(value) {
 554         return Ok((caps.get(1).unwrap().as_str(), caps.get(2).unwrap().as_str()));
 555     }
 556     bail!("unable to parse directory specification '{}'", value);
 557 }
 558
 559 fn spawn_catalog_upload(
 560     client: Arc<BackupWriter>,
 561     crypt_config: Option<Arc<CryptConfig>>,
 562 ) -> Result<
 563         (
 564             Arc<Mutex<CatalogWriter<SenderWriter>>>,
 565             tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>
 566         ), Error>
 567 {
 568     let (catalog_tx, catalog_rx) = mpsc::channel(10); // allow to buffer 10 writes
 569     let catalog_stream = catalog_rx.map_err(Error::from);
 570     let catalog_chunk_size = 512*1024;
 571     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 572
 573     let catalog = Arc::new(Mutex::new(CatalogWriter::new(SenderWriter::new(catalog_tx))?));
 574
 575     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 576
 577     tokio::spawn(async move {
 578         let catalog_upload_result = client
 579             .upload_stream(CATALOG_NAME, catalog_chunk_stream, "dynamic", None, crypt_config)
 580             .await;
 581
 582         if let Err(ref err) = catalog_upload_result {
 583             eprintln!("catalog upload error - {}", err);
 584             client.cancel();
 585         }
 586
 587         let _ = catalog_result_tx.send(catalog_upload_result);
 588     });
 589
 590     Ok((catalog, catalog_result_rx))
 591 }
 592
 593 fn create_backup(
 594     param: Value,
 595     _info: &ApiMethod,
 596     _rpcenv: &mut dyn RpcEnvironment,
 597 ) -> Result<Value, Error> {
 598
 599     let repo = extract_repository_from_value(&param)?;
 600
 601     let backupspec_list = tools::required_array_param(&param, "backupspec")?;
 602
 603     let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
 604
 605     let skip_lost_and_found = param["skip-lost-and-found"].as_bool().unwrap_or(false);
 606
 607     let verbose = param["verbose"].as_bool().unwrap_or(false);
 608
 609     let backup_time_opt = param["backup-time"].as_i64();
 610
 611     let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v*1024) as usize);
 612
 613     if let Some(size) = chunk_size_opt {
 614         verify_chunk_size(size)?;
 615     }
 616
 617     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 618
 619     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 620
 621     let backup_type = param["backup-type"].as_str().unwrap_or("host");
 622
 623     let include_dev = param["include-dev"].as_array();
 624
 625     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
 626
 627     if let Some(include_dev) = include_dev {
 628         if all_file_systems {
 629             bail!("option 'all-file-systems' conflicts with option 'include-dev'");
 630         }
 631
 632         let mut set = HashSet::new();
 633         for path in include_dev {
 634             let path = path.as_str().unwrap();
 635             let stat = nix::sys::stat::stat(path)
 636                 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
 637             set.insert(stat.st_dev);
 638         }
 639         devices = Some(set);
 640     }
 641
 642     let mut upload_list = vec![];
 643
 644     enum BackupType { PXAR, IMAGE, CONFIG, LOGFILE };
 645
 646     let mut upload_catalog = false;
 647
 648     for backupspec in backupspec_list {
 649         let (target, filename) = parse_backupspec(backupspec.as_str().unwrap())?;
 650
 651         use std::os::unix::fs::FileTypeExt;
 652
 653         let metadata = std::fs::metadata(filename)
 654             .map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
 655         let file_type = metadata.file_type();
 656
 657         let extension = target.rsplit('.').next()
 658             .ok_or_else(|| format_err!("missing target file extenion '{}'", target))?;
 659
 660         match extension {
 661             "pxar" => {
 662                 if !file_type.is_dir() {
 663                     bail!("got unexpected file type (expected directory)");
 664                 }
 665                 upload_list.push((BackupType::PXAR, filename.to_owned(), format!("{}.didx", target), 0));
 666                 upload_catalog = true;
 667             }
 668             "img" => {
 669
 670                 if !(file_type.is_file() || file_type.is_block_device()) {
 671                     bail!("got unexpected file type (expected file or block device)");
 672                 }
 673
 674                 let size = image_size(&PathBuf::from(filename))?;
 675
 676                 if size == 0 { bail!("got zero-sized file '{}'", filename); }
 677
 678                 upload_list.push((BackupType::IMAGE, filename.to_owned(), format!("{}.fidx", target), size));
 679             }
 680             "conf" => {
 681                 if !file_type.is_file() {
 682                     bail!("got unexpected file type (expected regular file)");
 683                 }
 684                 upload_list.push((BackupType::CONFIG, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 685             }
 686             "log" => {
 687                 if !file_type.is_file() {
 688                     bail!("got unexpected file type (expected regular file)");
 689                 }
 690                 upload_list.push((BackupType::LOGFILE, filename.to_owned(), format!("{}.blob", target), metadata.len()));
 691             }
 692             _ => {
 693                 bail!("got unknown archive extension '{}'", extension);
 694             }
 695         }
 696     }
 697
 698     let backup_time = Utc.timestamp(backup_time_opt.unwrap_or_else(|| Utc::now().timestamp()), 0);
 699
 700     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 701     record_repository(&repo);
 702
 703     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time));
 704
 705     println!("Client name: {}", proxmox::tools::nodename());
 706
 707     let start_time = Local::now();
 708
 709     println!("Starting protocol: {}", start_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 710
 711     let (crypt_config, rsa_encrypted_key) = match keyfile {
 712         None => (None, None),
 713         Some(path) => {
 714             let (key, created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 715
 716             let crypt_config = CryptConfig::new(key)?;
 717
 718             let path = master_pubkey_path()?;
 719             if path.exists() {
 720                 let pem_data = file_get_contents(&path)?;
 721                 let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
 722                 let enc_key = crypt_config.generate_rsa_encoded_key(rsa, created)?;
 723                 (Some(Arc::new(crypt_config)), Some(enc_key))
 724             } else {
 725                 (Some(Arc::new(crypt_config)), None)
 726             }
 727         }
 728     };
 729
 730     async_main(async move {
 731         let client = BackupWriter::start(
 732             client,
 733             repo.store(),
 734             backup_type,
 735             &backup_id,
 736             backup_time,
 737             verbose,
 738         ).await?;
 739
 740         let snapshot = BackupDir::new(backup_type, backup_id, backup_time.timestamp());
 741         let mut manifest = BackupManifest::new(snapshot);
 742
 743         let (catalog, catalog_result_rx) = spawn_catalog_upload(client.clone(), crypt_config.clone())?;
 744
 745         for (backup_type, filename, target, size) in upload_list {
 746             match backup_type {
 747                 BackupType::CONFIG => {
 748                     println!("Upload config file '{}' to '{:?}' as {}", filename, repo, target);
 749                     let stats = client
 750                         .upload_blob_from_file(&filename, &target, crypt_config.clone(), true)
 751                         .await?;
 752                     manifest.add_file(target, stats.size, stats.csum);
 753                 }
 754                 BackupType::LOGFILE => { // fixme: remove - not needed anymore ?
 755                     println!("Upload log file '{}' to '{:?}' as {}", filename, repo, target);
 756                     let stats = client
 757                         .upload_blob_from_file(&filename, &target, crypt_config.clone(), true)
 758                         .await?;
 759                     manifest.add_file(target, stats.size, stats.csum);
 760                 }
 761                 BackupType::PXAR => {
 762                     println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
 763                     catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
 764                     let stats = backup_directory(
 765                         &client,
 766                         &filename,
 767                         &target,
 768                         chunk_size_opt,
 769                         devices.clone(),
 770                         verbose,
 771                         skip_lost_and_found,
 772                         crypt_config.clone(),
 773                         catalog.clone(),
 774                     ).await?;
 775                     manifest.add_file(target, stats.size, stats.csum);
 776                     catalog.lock().unwrap().end_directory()?;
 777                 }
 778                 BackupType::IMAGE => {
 779                     println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
 780                     let stats = backup_image(
 781                         &client,
 782                         &filename,
 783                         &target,
 784                         size,
 785                         chunk_size_opt,
 786                         verbose,
 787                         crypt_config.clone(),
 788                     ).await?;
 789                     manifest.add_file(target, stats.size, stats.csum);
 790                 }
 791             }
 792         }
 793
 794         // finalize and upload catalog
 795         if upload_catalog {
 796             let mutex = Arc::try_unwrap(catalog)
 797                 .map_err(|_| format_err!("unable to get catalog (still used)"))?;
 798             let mut catalog = mutex.into_inner().unwrap();
 799
 800             catalog.finish()?;
 801
 802             drop(catalog); // close upload stream
 803
 804             let stats = catalog_result_rx.await??;
 805
 806             manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum);
 807         }
 808
 809         if let Some(rsa_encrypted_key) = rsa_encrypted_key {
 810             let target = "rsa-encrypted.key";
 811             println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
 812             let stats = client
 813                 .upload_blob_from_data(rsa_encrypted_key, target, None, false, false)
 814                 .await?;
 815             manifest.add_file(format!("{}.blob", target), stats.size, stats.csum);
 816
 817             // openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out t
 818             /*
 819             let mut buffer2 = vec![0u8; rsa.size() as usize];
 820             let pem_data = file_get_contents("master-private.pem")?;
 821             let rsa = openssl::rsa::Rsa::private_key_from_pem(&pem_data)?;
 822             let len = rsa.private_decrypt(&buffer, &mut buffer2, openssl::rsa::Padding::PKCS1)?;
 823             println!("TEST {} {:?}", len, buffer2);
 824              */
 825         }
 826
 827         // create manifest (index.json)
 828         let manifest = manifest.into_json();
 829
 830         println!("Upload index.json to '{:?}'", repo);
 831         let manifest = serde_json::to_string_pretty(&manifest)?.into();
 832         client
 833             .upload_blob_from_data(manifest, MANIFEST_BLOB_NAME, crypt_config.clone(), true, true)
 834             .await?;
 835
 836         client.finish().await?;
 837
 838         let end_time = Local::now();
 839         let elapsed = end_time.signed_duration_since(start_time);
 840         println!("Duration: {}", elapsed);
 841
 842         println!("End Time: {}", end_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
 843
 844         Ok(Value::Null)
 845     })
 846 }
 847
 848 fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 849
 850     let mut result = vec![];
 851
 852     let data: Vec<&str> = arg.splitn(2, ':').collect();
 853
 854     if data.len() != 2 {
 855         result.push(String::from("root.pxar:/"));
 856         result.push(String::from("etc.pxar:/etc"));
 857         return result;
 858     }
 859
 860     let files = tools::complete_file_name(data[1], param);
 861
 862     for file in files {
 863         result.push(format!("{}:{}", data[0], file));
 864     }
 865
 866     result
 867 }
 868
 869 fn restore(
 870     param: Value,
 871     _info: &ApiMethod,
 872     _rpcenv: &mut dyn RpcEnvironment,
 873 ) -> Result<Value, Error> {
 874     async_main(restore_do(param))
 875 }
 876
 877 fn dump_image<W: Write>(
 878     client: Arc<BackupReader>,
 879     crypt_config: Option<Arc<CryptConfig>>,
 880     index: FixedIndexReader,
 881     mut writer: W,
 882     verbose: bool,
 883 ) -> Result<(), Error> {
 884
 885     let most_used = index.find_most_used_chunks(8);
 886
 887     let mut chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
 888
 889     // Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
 890     // and thus slows down reading. Instead, directly use RemoteChunkReader
 891     let mut per = 0;
 892     let mut bytes = 0;
 893     let start_time = std::time::Instant::now();
 894
 895     for pos in 0..index.index_count() {
 896         let digest = index.index_digest(pos).unwrap();
 897         let raw_data = chunk_reader.read_chunk(&digest)?;
 898         writer.write_all(&raw_data)?;
 899         bytes += raw_data.len();
 900         if verbose {
 901             let next_per = ((pos+1)*100)/index.index_count();
 902             if per != next_per {
 903                 eprintln!("progress {}% (read {} bytes, duration {} sec)",
 904                           next_per, bytes, start_time.elapsed().as_secs());
 905                 per = next_per;
 906             }
 907         }
 908     }
 909
 910     let end_time = std::time::Instant::now();
 911     let elapsed = end_time.duration_since(start_time);
 912     eprintln!("restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
 913               bytes,
 914               elapsed.as_secs_f64(),
 915               bytes as f64/(1024.0*1024.0*elapsed.as_secs_f64())
 916     );
 917
 918
 919     Ok(())
 920 }
 921
 922 async fn restore_do(param: Value) -> Result<Value, Error> {
 923     let repo = extract_repository_from_value(&param)?;
 924
 925     let verbose = param["verbose"].as_bool().unwrap_or(false);
 926
 927     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
 928
 929     let archive_name = tools::required_string_param(&param, "archive-name")?;
 930
 931     let client = HttpClient::new(repo.host(), repo.user(), None)?;
 932
 933     record_repository(&repo);
 934
 935     let path = tools::required_string_param(&param, "snapshot")?;
 936
 937     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
 938         let group = BackupGroup::parse(path)?;
 939
 940         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 941         let result = client.get(&path, Some(json!({
 942             "backup-type": group.backup_type(),
 943             "backup-id": group.backup_id(),
 944         }))).await?;
 945
 946         let list = result["data"].as_array().unwrap();
 947         if list.is_empty() {
 948             bail!("backup group '{}' does not contain any snapshots:", path);
 949         }
 950
 951         let epoch = list[0]["backup-time"].as_i64().unwrap();
 952         let backup_time = Utc.timestamp(epoch, 0);
 953         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
 954     } else {
 955         let snapshot = BackupDir::parse(path)?;
 956         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
 957     };
 958
 959     let target = tools::required_string_param(&param, "target")?;
 960     let target = if target == "-" { None } else { Some(target) };
 961
 962     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
 963
 964     let crypt_config = match keyfile {
 965         None => None,
 966         Some(path) => {
 967             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
 968             Some(Arc::new(CryptConfig::new(key)?))
 969         }
 970     };
 971
 972     let server_archive_name = if archive_name.ends_with(".pxar") {
 973         format!("{}.didx", archive_name)
 974     } else if archive_name.ends_with(".img") {
 975         format!("{}.fidx", archive_name)
 976     } else {
 977         format!("{}.blob", archive_name)
 978     };
 979
 980     let client = BackupReader::start(
 981         client,
 982         crypt_config.clone(),
 983         repo.store(),
 984         &backup_type,
 985         &backup_id,
 986         backup_time,
 987         true,
 988     ).await?;
 989
 990     let manifest = client.download_manifest().await?;
 991
 992     if server_archive_name == MANIFEST_BLOB_NAME {
 993         let backup_index_data = manifest.into_json().to_string();
 994         if let Some(target) = target {
 995             file_set_contents(target, backup_index_data.as_bytes(), None)?;
 996         } else {
 997             let stdout = std::io::stdout();
 998             let mut writer = stdout.lock();
 999             writer.write_all(backup_index_data.as_bytes())
1000                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1001         }
1002
1003     } else if server_archive_name.ends_with(".blob") {
1004
1005         let mut reader = client.download_blob(&manifest, &server_archive_name).await?;
1006
1007         if let Some(target) = target {
1008            let mut writer = std::fs::OpenOptions::new()
1009                 .write(true)
1010                 .create(true)
1011                 .create_new(true)
1012                 .open(target)
1013                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?;
1014             std::io::copy(&mut reader, &mut writer)?;
1015         } else {
1016             let stdout = std::io::stdout();
1017             let mut writer = stdout.lock();
1018             std::io::copy(&mut reader, &mut writer)
1019                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1020         }
1021
1022     } else if server_archive_name.ends_with(".didx") {
1023
1024         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1025
1026         let most_used = index.find_most_used_chunks(8);
1027
1028         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1029
1030         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1031
1032         if let Some(target) = target {
1033
1034             let feature_flags = pxar::flags::DEFAULT;
1035             let mut decoder = pxar::SequentialDecoder::new(&mut reader, feature_flags);
1036             decoder.set_callback(move |path| {
1037                 if verbose {
1038                     eprintln!("{:?}", path);
1039                 }
1040                 Ok(())
1041             });
1042             decoder.set_allow_existing_dirs(allow_existing_dirs);
1043
1044             decoder.restore(Path::new(target), &Vec::new())?;
1045         } else {
1046             let mut writer = std::fs::OpenOptions::new()
1047                 .write(true)
1048                 .open("/dev/stdout")
1049                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
1050
1051             std::io::copy(&mut reader, &mut writer)
1052                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
1053         }
1054     } else if server_archive_name.ends_with(".fidx") {
1055
1056         let index = client.download_fixed_index(&manifest, &server_archive_name).await?;
1057
1058         let mut writer = if let Some(target) = target {
1059             std::fs::OpenOptions::new()
1060                 .write(true)
1061                 .create(true)
1062                 .create_new(true)
1063                 .open(target)
1064                 .map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
1065         } else {
1066             std::fs::OpenOptions::new()
1067                 .write(true)
1068                 .open("/dev/stdout")
1069                 .map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
1070         };
1071
1072         dump_image(client.clone(), crypt_config.clone(), index, &mut writer, verbose)?;
1073
1074      } else {
1075         bail!("unknown archive file extension (expected .pxar of .img)");
1076     }
1077
1078     Ok(Value::Null)
1079 }
1080
1081 fn upload_log(
1082     param: Value,
1083     _info: &ApiMethod,
1084     _rpcenv: &mut dyn RpcEnvironment,
1085 ) -> Result<Value, Error> {
1086
1087     let logfile = tools::required_string_param(&param, "logfile")?;
1088     let repo = extract_repository_from_value(&param)?;
1089
1090     let snapshot = tools::required_string_param(&param, "snapshot")?;
1091     let snapshot = BackupDir::parse(snapshot)?;
1092
1093     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
1094
1095     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1096
1097     let crypt_config = match keyfile {
1098         None => None,
1099         Some(path) => {
1100             let (key, _created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1101             let crypt_config = CryptConfig::new(key)?;
1102             Some(Arc::new(crypt_config))
1103         }
1104     };
1105
1106     let data = file_get_contents(logfile)?;
1107
1108     let blob = DataBlob::encode(&data, crypt_config.as_ref().map(Arc::as_ref), true)?;
1109
1110     let raw_data = blob.into_inner();
1111
1112     let path = format!("api2/json/admin/datastore/{}/upload-backup-log", repo.store());
1113
1114     let args = json!({
1115         "backup-type": snapshot.group().backup_type(),
1116         "backup-id":  snapshot.group().backup_id(),
1117         "backup-time": snapshot.backup_time().timestamp(),
1118     });
1119
1120     let body = hyper::Body::from(raw_data);
1121
1122     async_main(async move {
1123         client.upload("application/octet-stream", body, &path, Some(args)).await
1124     })
1125 }
1126
1127 fn prune(
1128     mut param: Value,
1129     _info: &ApiMethod,
1130     _rpcenv: &mut dyn RpcEnvironment,
1131 ) -> Result<Value, Error> {
1132
1133     let repo = extract_repository_from_value(&param)?;
1134
1135     let mut client = HttpClient::new(repo.host(), repo.user(), None)?;
1136
1137     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
1138
1139     let group = tools::required_string_param(&param, "group")?;
1140     let group = BackupGroup::parse(group)?;
1141
1142     let dry_run = param["dry-run"].as_bool().unwrap_or(false);
1143
1144     param.as_object_mut().unwrap().remove("repository");
1145     param.as_object_mut().unwrap().remove("group");
1146     param.as_object_mut().unwrap().remove("dry-run");
1147
1148     param["backup-type"] = group.backup_type().into();
1149     param["backup-id"] = group.backup_id().into();
1150
1151     if dry_run {
1152         let result = async_main(async move { client.get(&path, Some(param)).await })?;
1153         let data = &result["data"];
1154
1155         for item in data.as_array().unwrap() {
1156             let timestamp = item["backup-time"].as_i64().unwrap();
1157             let timestamp = BackupDir::backup_time_to_string(Utc.timestamp(timestamp, 0));
1158             let keep =  item["keep"].as_bool().unwrap();
1159             println!("{}/{}/{} {}",
1160                      group.backup_type(),
1161                      group.backup_id(),
1162                      timestamp,
1163                      if keep { "keep" } else { "remove" },
1164              );
1165         }
1166     } else {
1167         let _result = async_main(async move { client.post(&path, Some(param)).await })?;
1168     }
1169     record_repository(&repo);
1170
1171     Ok(Value::Null)
1172 }
1173
1174 fn status(
1175     param: Value,
1176     _info: &ApiMethod,
1177     _rpcenv: &mut dyn RpcEnvironment,
1178 ) -> Result<Value, Error> {
1179
1180     let repo = extract_repository_from_value(&param)?;
1181
1182     let output_format = param["output-format"].as_str().unwrap_or("text").to_owned();
1183
1184     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1185
1186     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
1187
1188     let result = async_main(async move { client.get(&path, None).await })?;
1189     let data = &result["data"];
1190
1191     record_repository(&repo);
1192
1193     if output_format == "text" {
1194         let total = data["total"].as_u64().unwrap();
1195         let used = data["used"].as_u64().unwrap();
1196         let avail = data["avail"].as_u64().unwrap();
1197         let roundup = total/200;
1198
1199         println!(
1200             "total: {} used: {} ({} %) available: {}",
1201             total,
1202             used,
1203             ((used+roundup)*100)/total,
1204             avail,
1205         );
1206     } else {
1207         format_and_print_result(data, &output_format);
1208     }
1209
1210     Ok(Value::Null)
1211 }
1212
1213 // like get, but simply ignore errors and return Null instead
1214 async fn try_get(repo: &BackupRepository, url: &str) -> Value {
1215
1216     let client = match HttpClient::new(repo.host(), repo.user(), None) {
1217         Ok(v) => v,
1218         _ => return Value::Null,
1219     };
1220
1221     let mut resp = match client.get(url, None).await {
1222         Ok(v) => v,
1223         _ => return Value::Null,
1224     };
1225
1226     if let Some(map) = resp.as_object_mut() {
1227         if let Some(data) = map.remove("data") {
1228             return data;
1229         }
1230     }
1231     Value::Null
1232 }
1233
1234 fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1235     async_main(async { complete_backup_group_do(param).await })
1236 }
1237
1238 async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
1239
1240     let mut result = vec![];
1241
1242     let repo = match extract_repository_from_map(param) {
1243         Some(v) => v,
1244         _ => return result,
1245     };
1246
1247     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
1248
1249     let data = try_get(&repo, &path).await;
1250
1251     if let Some(list) = data.as_array() {
1252         for item in list {
1253             if let (Some(backup_id), Some(backup_type)) =
1254                 (item["backup-id"].as_str(), item["backup-type"].as_str())
1255             {
1256                 result.push(format!("{}/{}", backup_type, backup_id));
1257             }
1258         }
1259     }
1260
1261     result
1262 }
1263
1264 fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1265     async_main(async { complete_group_or_snapshot_do(arg, param).await })
1266 }
1267
1268 async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1269
1270     if arg.matches('/').count() < 2 {
1271         let groups = complete_backup_group_do(param).await;
1272         let mut result = vec![];
1273         for group in groups {
1274             result.push(group.to_string());
1275             result.push(format!("{}/", group));
1276         }
1277         return result;
1278     }
1279
1280     complete_backup_snapshot_do(param).await
1281 }
1282
1283 fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1284     async_main(async { complete_backup_snapshot_do(param).await })
1285 }
1286
1287 async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
1288
1289     let mut result = vec![];
1290
1291     let repo = match extract_repository_from_map(param) {
1292         Some(v) => v,
1293         _ => return result,
1294     };
1295
1296     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1297
1298     let data = try_get(&repo, &path).await;
1299
1300     if let Some(list) = data.as_array() {
1301         for item in list {
1302             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
1303                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
1304             {
1305                 let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
1306                 result.push(snapshot.relative_path().to_str().unwrap().to_owned());
1307             }
1308         }
1309     }
1310
1311     result
1312 }
1313
1314 fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1315     async_main(async { complete_server_file_name_do(param).await })
1316 }
1317
1318 async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
1319
1320     let mut result = vec![];
1321
1322     let repo = match extract_repository_from_map(param) {
1323         Some(v) => v,
1324         _ => return result,
1325     };
1326
1327     let snapshot = match param.get("snapshot") {
1328         Some(path) => {
1329             match BackupDir::parse(path) {
1330                 Ok(v) => v,
1331                 _ => return result,
1332             }
1333         }
1334         _ => return result,
1335     };
1336
1337     let query = tools::json_object_to_query(json!({
1338         "backup-type": snapshot.group().backup_type(),
1339         "backup-id": snapshot.group().backup_id(),
1340         "backup-time": snapshot.backup_time().timestamp(),
1341     })).unwrap();
1342
1343     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
1344
1345     let data = try_get(&repo, &path).await;
1346
1347     if let Some(list) = data.as_array() {
1348         for item in list {
1349             if let Some(filename) = item["filename"].as_str() {
1350                 result.push(filename.to_owned());
1351             }
1352         }
1353     }
1354
1355     result
1356 }
1357
1358 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
1359     complete_server_file_name(arg, param)
1360         .iter()
1361         .map(|v| strip_server_file_expenstion(&v))
1362         .collect()
1363 }
1364
1365 fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
1366
1367     let mut result = vec![];
1368
1369     let mut size = 64;
1370     loop {
1371         result.push(size.to_string());
1372         size *= 2;
1373         if size > 4096 { break; }
1374     }
1375
1376     result
1377 }
1378
1379 fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
1380
1381     // fixme: implement other input methods
1382
1383     use std::env::VarError::*;
1384     match std::env::var("PBS_ENCRYPTION_PASSWORD") {
1385         Ok(p) => return Ok(p.as_bytes().to_vec()),
1386         Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
1387         Err(NotPresent) => {
1388             // Try another method
1389         }
1390     }
1391
1392     // If we're on a TTY, query the user for a password
1393     if crate::tools::tty::stdin_isatty() {
1394         return Ok(crate::tools::tty::read_password("Encryption Key Password: ")?);
1395     }
1396
1397     bail!("no password input mechanism available");
1398 }
1399
1400 fn key_create(
1401     param: Value,
1402     _info: &ApiMethod,
1403     _rpcenv: &mut dyn RpcEnvironment,
1404 ) -> Result<Value, Error> {
1405
1406     let path = tools::required_string_param(&param, "path")?;
1407     let path = PathBuf::from(path);
1408
1409     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1410
1411     let key = proxmox::sys::linux::random_data(32)?;
1412
1413     if kdf == "scrypt" {
1414         // always read passphrase from tty
1415         if !crate::tools::tty::stdin_isatty() {
1416             bail!("unable to read passphrase - no tty");
1417         }
1418
1419         let password = crate::tools::tty::read_password("Encryption Key Password: ")?;
1420
1421         let key_config = encrypt_key_with_passphrase(&key, &password)?;
1422
1423         store_key_config(&path, false, key_config)?;
1424
1425         Ok(Value::Null)
1426     } else if kdf == "none" {
1427         let created =  Local.timestamp(Local::now().timestamp(), 0);
1428
1429         store_key_config(&path, false, KeyConfig {
1430             kdf: None,
1431             created,
1432             modified: created,
1433             data: key,
1434         })?;
1435
1436         Ok(Value::Null)
1437     } else {
1438         unreachable!();
1439     }
1440 }
1441
1442 fn master_pubkey_path() -> Result<PathBuf, Error> {
1443     let base = BaseDirectories::with_prefix("proxmox-backup")?;
1444
1445     // usually $HOME/.config/proxmox-backup/master-public.pem
1446     let path = base.place_config_file("master-public.pem")?;
1447
1448     Ok(path)
1449 }
1450
1451 fn key_import_master_pubkey(
1452     param: Value,
1453     _info: &ApiMethod,
1454     _rpcenv: &mut dyn RpcEnvironment,
1455 ) -> Result<Value, Error> {
1456
1457     let path = tools::required_string_param(&param, "path")?;
1458     let path = PathBuf::from(path);
1459
1460     let pem_data = file_get_contents(&path)?;
1461
1462     if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
1463         bail!("Unable to decode PEM data - {}", err);
1464     }
1465
1466     let target_path = master_pubkey_path()?;
1467
1468     file_set_contents(&target_path, &pem_data, None)?;
1469
1470     println!("Imported public master key to {:?}", target_path);
1471
1472     Ok(Value::Null)
1473 }
1474
1475 fn key_create_master_key(
1476     _param: Value,
1477     _info: &ApiMethod,
1478     _rpcenv: &mut dyn RpcEnvironment,
1479 ) -> Result<Value, Error> {
1480
1481     // we need a TTY to query the new password
1482     if !crate::tools::tty::stdin_isatty() {
1483         bail!("unable to create master key - no tty");
1484     }
1485
1486     let rsa = openssl::rsa::Rsa::generate(4096)?;
1487     let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
1488
1489     let new_pw = String::from_utf8(crate::tools::tty::read_password("Master Key Password: ")?)?;
1490     let verify_pw = String::from_utf8(crate::tools::tty::read_password("Verify Password: ")?)?;
1491
1492     if new_pw != verify_pw {
1493         bail!("Password verification fail!");
1494     }
1495
1496     if new_pw.len() < 5 {
1497         bail!("Password is too short!");
1498     }
1499
1500     let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
1501     let filename_pub = "master-public.pem";
1502     println!("Writing public master key to {}", filename_pub);
1503     file_set_contents(filename_pub, pub_key.as_slice(), None)?;
1504
1505     let cipher = openssl::symm::Cipher::aes_256_cbc();
1506     let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, new_pw.as_bytes())?;
1507
1508     let filename_priv = "master-private.pem";
1509     println!("Writing private master key to {}", filename_priv);
1510     file_set_contents(filename_priv, priv_key.as_slice(), None)?;
1511
1512     Ok(Value::Null)
1513 }
1514
1515 fn key_change_passphrase(
1516     param: Value,
1517     _info: &ApiMethod,
1518     _rpcenv: &mut dyn RpcEnvironment,
1519 ) -> Result<Value, Error> {
1520
1521     let path = tools::required_string_param(&param, "path")?;
1522     let path = PathBuf::from(path);
1523
1524     let kdf = param["kdf"].as_str().unwrap_or("scrypt");
1525
1526     // we need a TTY to query the new password
1527     if !crate::tools::tty::stdin_isatty() {
1528         bail!("unable to change passphrase - no tty");
1529     }
1530
1531     let (key, created) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1532
1533     if kdf == "scrypt" {
1534
1535         let new_pw = String::from_utf8(crate::tools::tty::read_password("New Password: ")?)?;
1536         let verify_pw = String::from_utf8(crate::tools::tty::read_password("Verify Password: ")?)?;
1537
1538         if new_pw != verify_pw {
1539             bail!("Password verification fail!");
1540         }
1541
1542         if new_pw.len() < 5 {
1543             bail!("Password is too short!");
1544         }
1545
1546         let mut new_key_config = encrypt_key_with_passphrase(&key, new_pw.as_bytes())?;
1547         new_key_config.created = created; // keep original value
1548
1549         store_key_config(&path, true, new_key_config)?;
1550
1551         Ok(Value::Null)
1552     } else if kdf == "none" {
1553         let modified =  Local.timestamp(Local::now().timestamp(), 0);
1554
1555         store_key_config(&path, true, KeyConfig {
1556             kdf: None,
1557             created, // keep original value
1558             modified,
1559             data: key.to_vec(),
1560         })?;
1561
1562         Ok(Value::Null)
1563     } else {
1564         unreachable!();
1565     }
1566 }
1567
1568 fn key_mgmt_cli() -> CliCommandMap {
1569
1570     const KDF_SCHEMA: Schema =
1571         StringSchema::new("Key derivation function. Choose 'none' to store the key unecrypted.")
1572         .format(&ApiStringFormat::Enum(&["scrypt", "none"]))
1573         .default("scrypt")
1574         .schema();
1575
1576     #[sortable]
1577     const API_METHOD_KEY_CREATE: ApiMethod = ApiMethod::new(
1578         &ApiHandler::Sync(&key_create),
1579         &ObjectSchema::new(
1580             "Create a new encryption key.",
1581             &sorted!([
1582                 ("path", false, &StringSchema::new("File system path.").schema()),
1583                 ("kdf", true, &KDF_SCHEMA),
1584             ]),
1585         )
1586     );
1587
1588     let key_create_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE)
1589         .arg_param(&["path"])
1590         .completion_cb("path", tools::complete_file_name);
1591
1592     #[sortable]
1593     const API_METHOD_KEY_CHANGE_PASSPHRASE: ApiMethod = ApiMethod::new(
1594         &ApiHandler::Sync(&key_change_passphrase),
1595         &ObjectSchema::new(
1596             "Change the passphrase required to decrypt the key.",
1597             &sorted!([
1598                 ("path", false, &StringSchema::new("File system path.").schema()),
1599                 ("kdf", true, &KDF_SCHEMA),
1600             ]),
1601         )
1602     );
1603
1604     let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_KEY_CHANGE_PASSPHRASE)
1605         .arg_param(&["path"])
1606         .completion_cb("path", tools::complete_file_name);
1607
1608     const API_METHOD_KEY_CREATE_MASTER_KEY: ApiMethod = ApiMethod::new(
1609         &ApiHandler::Sync(&key_create_master_key),
1610         &ObjectSchema::new("Create a new 4096 bit RSA master pub/priv key pair.", &[])
1611     );
1612
1613     let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE_MASTER_KEY);
1614
1615     #[sortable]
1616     const API_METHOD_KEY_IMPORT_MASTER_PUBKEY: ApiMethod = ApiMethod::new(
1617         &ApiHandler::Sync(&key_import_master_pubkey),
1618         &ObjectSchema::new(
1619             "Import a new RSA public key and use it as master key. The key is expected to be in '.pem' format.",
1620             &sorted!([ ("path", false, &StringSchema::new("File system path.").schema()) ]),
1621         )
1622     );
1623
1624     let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_KEY_IMPORT_MASTER_PUBKEY)
1625         .arg_param(&["path"])
1626         .completion_cb("path", tools::complete_file_name);
1627
1628     CliCommandMap::new()
1629         .insert("create".to_owned(), key_create_cmd_def.into())
1630         .insert("create-master-key".to_owned(), key_create_master_key_cmd_def.into())
1631         .insert("import-master-pubkey".to_owned(), key_import_master_pubkey_cmd_def.into())
1632         .insert("change-passphrase".to_owned(), key_change_passphrase_cmd_def.into())
1633 }
1634
1635 fn mount(
1636     param: Value,
1637     _info: &ApiMethod,
1638     _rpcenv: &mut dyn RpcEnvironment,
1639 ) -> Result<Value, Error> {
1640     let verbose = param["verbose"].as_bool().unwrap_or(false);
1641     if verbose {
1642         // This will stay in foreground with debug output enabled as None is
1643         // passed for the RawFd.
1644         return async_main(mount_do(param, None));
1645     }
1646
1647     // Process should be deamonized.
1648     // Make sure to fork before the async runtime is instantiated to avoid troubles.
1649     let pipe = pipe()?;
1650     match fork() {
1651         Ok(ForkResult::Parent { .. }) => {
1652             nix::unistd::close(pipe.1).unwrap();
1653             // Blocks the parent process until we are ready to go in the child
1654             let _res = nix::unistd::read(pipe.0, &mut [0]).unwrap();
1655             Ok(Value::Null)
1656         }
1657         Ok(ForkResult::Child) => {
1658             nix::unistd::close(pipe.0).unwrap();
1659             nix::unistd::setsid().unwrap();
1660             async_main(mount_do(param, Some(pipe.1)))
1661         }
1662         Err(_) => bail!("failed to daemonize process"),
1663     }
1664 }
1665
1666 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
1667     let repo = extract_repository_from_value(&param)?;
1668     let archive_name = tools::required_string_param(&param, "archive-name")?;
1669     let target = tools::required_string_param(&param, "target")?;
1670     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1671
1672     record_repository(&repo);
1673
1674     let path = tools::required_string_param(&param, "snapshot")?;
1675     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1676         let group = BackupGroup::parse(path)?;
1677
1678         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1679         let result = client.get(&path, Some(json!({
1680             "backup-type": group.backup_type(),
1681             "backup-id": group.backup_id(),
1682         }))).await?;
1683
1684         let list = result["data"].as_array().unwrap();
1685         if list.is_empty() {
1686             bail!("backup group '{}' does not contain any snapshots:", path);
1687         }
1688
1689         let epoch = list[0]["backup-time"].as_i64().unwrap();
1690         let backup_time = Utc.timestamp(epoch, 0);
1691         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
1692     } else {
1693         let snapshot = BackupDir::parse(path)?;
1694         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1695     };
1696
1697     let keyfile = param["keyfile"].as_str().map(PathBuf::from);
1698     let crypt_config = match keyfile {
1699         None => None,
1700         Some(path) => {
1701             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1702             Some(Arc::new(CryptConfig::new(key)?))
1703         }
1704     };
1705
1706     let server_archive_name = if archive_name.ends_with(".pxar") {
1707         format!("{}.didx", archive_name)
1708     } else {
1709         bail!("Can only mount pxar archives.");
1710     };
1711
1712     let client = BackupReader::start(
1713         client,
1714         crypt_config.clone(),
1715         repo.store(),
1716         &backup_type,
1717         &backup_id,
1718         backup_time,
1719         true,
1720     ).await?;
1721
1722     let manifest = client.download_manifest().await?;
1723
1724     if server_archive_name.ends_with(".didx") {
1725         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1726         let most_used = index.find_most_used_chunks(8);
1727         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1728         let reader = BufferedDynamicReader::new(index, chunk_reader);
1729         let decoder = pxar::Decoder::new(reader)?;
1730         let options = OsStr::new("ro,default_permissions");
1731         let mut session = pxar::fuse::Session::new(decoder, &options, pipe.is_none())
1732             .map_err(|err| format_err!("pxar mount failed: {}", err))?;
1733
1734         // Mount the session but not call fuse deamonize as this will cause
1735         // issues with the runtime after the fork
1736         let deamonize = false;
1737         session.mount(&Path::new(target), deamonize)?;
1738
1739         if let Some(pipe) = pipe {
1740             nix::unistd::chdir(Path::new("/")).unwrap();
1741             // Finish creation of deamon by redirecting filedescriptors.
1742             let nullfd = nix::fcntl::open(
1743                 "/dev/null",
1744                 nix::fcntl::OFlag::O_RDWR,
1745                 nix::sys::stat::Mode::empty(),
1746             ).unwrap();
1747             nix::unistd::dup2(nullfd, 0).unwrap();
1748             nix::unistd::dup2(nullfd, 1).unwrap();
1749             nix::unistd::dup2(nullfd, 2).unwrap();
1750             if nullfd > 2 {
1751                 nix::unistd::close(nullfd).unwrap();
1752             }
1753             // Signal the parent process that we are done with the setup and it can
1754             // terminate.
1755             nix::unistd::write(pipe, &[0u8])?;
1756             nix::unistd::close(pipe).unwrap();
1757         }
1758
1759         let multithreaded = true;
1760         session.run_loop(multithreaded)?;
1761     } else {
1762         bail!("unknown archive file extension (expected .pxar)");
1763     }
1764
1765     Ok(Value::Null)
1766 }
1767
1768 fn shell(
1769     param: Value,
1770     _info: &ApiMethod,
1771     _rpcenv: &mut dyn RpcEnvironment,
1772 ) -> Result<Value, Error> {
1773     async_main(catalog_shell(param))
1774 }
1775
1776 async fn catalog_shell(param: Value) -> Result<Value, Error> {
1777     let repo = extract_repository_from_value(&param)?;
1778     let client = HttpClient::new(repo.host(), repo.user(), None)?;
1779     let path = tools::required_string_param(&param, "snapshot")?;
1780     let archive_name = tools::required_string_param(&param, "archive-name")?;
1781
1782     let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
1783         let group = BackupGroup::parse(path)?;
1784
1785         let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
1786         let result = client.get(&path, Some(json!({
1787             "backup-type": group.backup_type(),
1788             "backup-id": group.backup_id(),
1789         }))).await?;
1790
1791         let list = result["data"].as_array().unwrap();
1792         if list.is_empty() {
1793             bail!("backup group '{}' does not contain any snapshots:", path);
1794         }
1795
1796         let epoch = list[0]["backup-time"].as_i64().unwrap();
1797         let backup_time = Utc.timestamp(epoch, 0);
1798         (group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time)
1799     } else {
1800         let snapshot = BackupDir::parse(path)?;
1801         (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
1802     };
1803
1804     let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
1805     let crypt_config = match keyfile {
1806         None => None,
1807         Some(path) => {
1808             let (key, _) = load_and_decrtypt_key(&path, &get_encryption_key_password)?;
1809             Some(Arc::new(CryptConfig::new(key)?))
1810         }
1811     };
1812
1813     let server_archive_name = if archive_name.ends_with(".pxar") {
1814         format!("{}.didx", archive_name)
1815     } else {
1816         bail!("Can only mount pxar archives.");
1817     };
1818
1819     let client = BackupReader::start(
1820         client,
1821         crypt_config.clone(),
1822         repo.store(),
1823         &backup_type,
1824         &backup_id,
1825         backup_time,
1826         true,
1827     ).await?;
1828
1829     let tmpfile = std::fs::OpenOptions::new()
1830         .write(true)
1831         .read(true)
1832         .custom_flags(libc::O_TMPFILE)
1833         .open("/tmp")?;
1834
1835     let manifest = client.download_manifest().await?;
1836
1837     let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
1838     let most_used = index.find_most_used_chunks(8);
1839     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
1840     let reader = BufferedDynamicReader::new(index, chunk_reader);
1841     let mut decoder = pxar::Decoder::new(reader)?;
1842     decoder.set_callback(|path| {
1843         println!("{:?}", path);
1844         Ok(())
1845     });
1846
1847     let tmpfile = client.download(CATALOG_NAME, tmpfile).await?;
1848     let index = DynamicIndexReader::new(tmpfile)
1849         .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
1850
1851     // Note: do not use values stored in index (not trusted) - instead, computed them again
1852     let (csum, size) = index.compute_csum();
1853     manifest.verify_file(CATALOG_NAME, &csum, size)?;
1854
1855     let most_used = index.find_most_used_chunks(8);
1856     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
1857     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
1858     let mut catalogfile = std::fs::OpenOptions::new()
1859         .write(true)
1860         .read(true)
1861         .custom_flags(libc::O_TMPFILE)
1862         .open("/tmp")?;
1863
1864     std::io::copy(&mut reader, &mut catalogfile)
1865         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
1866
1867     catalogfile.seek(SeekFrom::Start(0))?;
1868     let catalog_reader = CatalogReader::new(catalogfile);
1869     let state = Shell::new(
1870         catalog_reader,
1871         &server_archive_name,
1872         decoder,
1873     )?;
1874
1875     println!("Starting interactive shell");
1876     state.shell()?;
1877
1878     record_repository(&repo);
1879
1880     Ok(Value::Null)
1881 }
1882
1883 fn main() {
1884
1885     const BACKUP_SOURCE_SCHEMA: Schema = StringSchema::new("Backup source specification ([<label>:<path>]).")
1886         .format(&ApiStringFormat::Pattern(&BACKUPSPEC_REGEX))
1887         .schema();
1888
1889     #[sortable]
1890     const API_METHOD_CREATE_BACKUP: ApiMethod = ApiMethod::new(
1891         &ApiHandler::Sync(&create_backup),
1892         &ObjectSchema::new(
1893             "Create (host) backup.",
1894             &sorted!([
1895                 (
1896                     "backupspec",
1897                     false,
1898                     &ArraySchema::new(
1899                         "List of backup source specifications ([<label.ext>:<path>] ...)",
1900                         &BACKUP_SOURCE_SCHEMA,
1901                     ).min_length(1).schema()
1902                 ),
1903                 (
1904                     "repository",
1905                     true,
1906                     &REPO_URL_SCHEMA
1907                 ),
1908                 (
1909                     "include-dev",
1910                     true,
1911                     &ArraySchema::new(
1912                         "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
1913                         &StringSchema::new("Path to file.").schema()
1914                     ).schema()
1915                 ),
1916                 (
1917                     "keyfile",
1918                     true,
1919                     &StringSchema::new("Path to encryption key. All data will be encrypted using this key.").schema()
1920                 ),
1921                 (
1922                     "verbose",
1923                     true,
1924                     &BooleanSchema::new("Verbose output.")
1925                         .default(false)
1926                         .schema()
1927                 ),
1928                 (
1929                     "skip-lost-and-found",
1930                     true,
1931                     &BooleanSchema::new("Skip lost+found directory")
1932                         .default(false)
1933                         .schema()
1934                 ),
1935                 (
1936                     "backup-type",
1937                     true,
1938                     &BACKUP_TYPE_SCHEMA,
1939                 ),
1940                 (
1941                     "backup-id",
1942                     true,
1943                     &BACKUP_ID_SCHEMA
1944                 ),
1945                 (
1946                     "backup-time",
1947                     true,
1948                     &BACKUP_TIME_SCHEMA
1949                 ),
1950                 (
1951                     "chunk-size",
1952                     true,
1953                     &IntegerSchema::new("Chunk size in KB. Must be a power of 2.")
1954                         .minimum(64)
1955                         .maximum(4096)
1956                         .default(4096)
1957                         .schema()
1958                 ),
1959             ]),
1960         )
1961     );
1962
1963     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
1964         .arg_param(&["backupspec"])
1965         .completion_cb("repository", complete_repository)
1966         .completion_cb("backupspec", complete_backup_source)
1967         .completion_cb("keyfile", tools::complete_file_name)
1968         .completion_cb("chunk-size", complete_chunk_size);
1969
1970     #[sortable]
1971     const API_METHOD_UPLOAD_LOG: ApiMethod = ApiMethod::new(
1972         &ApiHandler::Sync(&upload_log),
1973         &ObjectSchema::new(
1974             "Upload backup log file.",
1975             &sorted!([
1976                 (
1977                     "snapshot",
1978                     false,
1979                     &StringSchema::new("Snapshot path.").schema()
1980                 ),
1981                 (
1982                     "logfile",
1983                     false,
1984                     &StringSchema::new("The path to the log file you want to upload.").schema()
1985                 ),
1986                 (
1987                     "repository",
1988                     true,
1989                     &REPO_URL_SCHEMA
1990                 ),
1991                 (
1992                     "keyfile",
1993                     true,
1994                     &StringSchema::new("Path to encryption key. All data will be encrypted using this key.").schema()
1995                 ),
1996             ]),
1997         )
1998     );
1999
2000     let upload_log_cmd_def = CliCommand::new(&API_METHOD_UPLOAD_LOG)
2001         .arg_param(&["snapshot", "logfile"])
2002         .completion_cb("snapshot", complete_backup_snapshot)
2003         .completion_cb("logfile", tools::complete_file_name)
2004         .completion_cb("keyfile", tools::complete_file_name)
2005         .completion_cb("repository", complete_repository);
2006
2007     #[sortable]
2008     const API_METHOD_LIST_BACKUP_GROUPS: ApiMethod = ApiMethod::new(
2009         &ApiHandler::Sync(&list_backup_groups),
2010         &ObjectSchema::new(
2011             "List backup groups.",
2012             &sorted!([
2013                 ("repository", true, &REPO_URL_SCHEMA),
2014                 ("output-format", true, &OUTPUT_FORMAT),
2015             ]),
2016         )
2017     );
2018
2019     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
2020         .completion_cb("repository", complete_repository);
2021
2022     #[sortable]
2023     const API_METHOD_LIST_SNAPSHOTS: ApiMethod = ApiMethod::new(
2024         &ApiHandler::Sync(&list_snapshots),
2025         &ObjectSchema::new(
2026             "List backup snapshots.",
2027             &sorted!([
2028                 ("group", true, &StringSchema::new("Backup group.").schema()),
2029                 ("repository", true, &REPO_URL_SCHEMA),
2030                 ("output-format", true, &OUTPUT_FORMAT),
2031             ]),
2032         )
2033     );
2034
2035     let snapshots_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOTS)
2036         .arg_param(&["group"])
2037         .completion_cb("group", complete_backup_group)
2038         .completion_cb("repository", complete_repository);
2039
2040     #[sortable]
2041     const API_METHOD_FORGET_SNAPSHOTS: ApiMethod = ApiMethod::new(
2042         &ApiHandler::Sync(&forget_snapshots),
2043         &ObjectSchema::new(
2044             "Forget (remove) backup snapshots.",
2045             &sorted!([
2046                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2047                 ("repository", true, &REPO_URL_SCHEMA),
2048             ]),
2049         )
2050     );
2051
2052     let forget_cmd_def = CliCommand::new(&API_METHOD_FORGET_SNAPSHOTS)
2053         .arg_param(&["snapshot"])
2054         .completion_cb("repository", complete_repository)
2055         .completion_cb("snapshot", complete_backup_snapshot);
2056
2057     #[sortable]
2058     const API_METHOD_START_GARBAGE_COLLECTION: ApiMethod = ApiMethod::new(
2059         &ApiHandler::Sync(&start_garbage_collection),
2060         &ObjectSchema::new(
2061             "Start garbage collection for a specific repository.",
2062             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2063         )
2064     );
2065
2066     let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
2067         .completion_cb("repository", complete_repository);
2068
2069     #[sortable]
2070     const API_METHOD_RESTORE: ApiMethod = ApiMethod::new(
2071         &ApiHandler::Sync(&restore),
2072         &ObjectSchema::new(
2073             "Restore backup repository.",
2074             &sorted!([
2075                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2076                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2077                 (
2078                     "target",
2079                     false,
2080                     &StringSchema::new(
2081                         r###"Target directory path. Use '-' to write to stdandard output.
2082
2083 We do not extraxt '.pxar' archives when writing to stdandard output.
2084
2085 "###
2086                     ).schema()
2087                 ),
2088                 (
2089                     "allow-existing-dirs",
2090                     true,
2091                     &BooleanSchema::new("Do not fail if directories already exists.")
2092                         .default(false)
2093                         .schema()
2094                 ),
2095                 ("repository", true, &REPO_URL_SCHEMA),
2096                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2097                 (
2098                     "verbose",
2099                     true,
2100                     &BooleanSchema::new("Verbose output.")
2101                         .default(false)
2102                         .schema()
2103                 ),
2104             ]),
2105         )
2106     );
2107
2108     let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
2109         .arg_param(&["snapshot", "archive-name", "target"])
2110         .completion_cb("repository", complete_repository)
2111         .completion_cb("snapshot", complete_group_or_snapshot)
2112         .completion_cb("archive-name", complete_archive_name)
2113         .completion_cb("target", tools::complete_file_name);
2114
2115     #[sortable]
2116     const API_METHOD_LIST_SNAPSHOT_FILES: ApiMethod = ApiMethod::new(
2117         &ApiHandler::Sync(&list_snapshot_files),
2118         &ObjectSchema::new(
2119             "List snapshot files.",
2120             &sorted!([
2121                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2122                 ("repository", true, &REPO_URL_SCHEMA),
2123                 ("output-format", true, &OUTPUT_FORMAT),
2124             ]),
2125         )
2126     );
2127
2128     let files_cmd_def = CliCommand::new(&API_METHOD_LIST_SNAPSHOT_FILES)
2129         .arg_param(&["snapshot"])
2130         .completion_cb("repository", complete_repository)
2131         .completion_cb("snapshot", complete_backup_snapshot);
2132
2133     #[sortable]
2134     const API_METHOD_DUMP_CATALOG: ApiMethod = ApiMethod::new(
2135         &ApiHandler::Sync(&dump_catalog),
2136         &ObjectSchema::new(
2137             "Dump catalog.",
2138             &sorted!([
2139                 ("snapshot", false, &StringSchema::new("Snapshot path.").schema()),
2140                 ("repository", true, &REPO_URL_SCHEMA),
2141             ]),
2142         )
2143     );
2144
2145     let catalog_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
2146         .arg_param(&["snapshot"])
2147         .completion_cb("repository", complete_repository)
2148         .completion_cb("snapshot", complete_backup_snapshot);
2149
2150     const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
2151         &ApiHandler::Sync(&prune),
2152         &ObjectSchema::new(
2153             "Prune backup repository.",
2154             &proxmox_backup::add_common_prune_prameters!([
2155                 ("dry-run", true, &BooleanSchema::new(
2156                     "Just show what prune would do, but do not delete anything.")
2157                  .schema()),
2158                 ("group", false, &StringSchema::new("Backup group.").schema()),
2159             ], [
2160                 ("repository", true, &REPO_URL_SCHEMA),
2161             ])
2162         )
2163     );
2164
2165     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
2166         .arg_param(&["group"])
2167         .completion_cb("group", complete_backup_group)
2168         .completion_cb("repository", complete_repository);
2169
2170     #[sortable]
2171     const API_METHOD_STATUS: ApiMethod = ApiMethod::new(
2172         &ApiHandler::Sync(&status),
2173         &ObjectSchema::new(
2174             "Get repository status.",
2175             &sorted!([
2176                 ("repository", true, &REPO_URL_SCHEMA),
2177                 ("output-format", true, &OUTPUT_FORMAT),
2178             ]),
2179         )
2180     );
2181
2182     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS)
2183         .completion_cb("repository", complete_repository);
2184
2185     #[sortable]
2186     const API_METHOD_API_LOGIN: ApiMethod = ApiMethod::new(
2187         &ApiHandler::Sync(&api_login),
2188         &ObjectSchema::new(
2189             "Try to login. If successful, store ticket.",
2190             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2191         )
2192     );
2193
2194     let login_cmd_def = CliCommand::new(&API_METHOD_API_LOGIN)
2195         .completion_cb("repository", complete_repository);
2196
2197     #[sortable]
2198     const API_METHOD_API_LOGOUT: ApiMethod = ApiMethod::new(
2199         &ApiHandler::Sync(&api_logout),
2200         &ObjectSchema::new(
2201             "Logout (delete stored ticket).",
2202             &sorted!([ ("repository", true, &REPO_URL_SCHEMA) ]),
2203         )
2204     );
2205
2206     let logout_cmd_def = CliCommand::new(&API_METHOD_API_LOGOUT)
2207         .completion_cb("repository", complete_repository);
2208
2209     #[sortable]
2210     const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
2211         &ApiHandler::Sync(&mount),
2212         &ObjectSchema::new(
2213             "Mount pxar archive.",
2214             &sorted!([
2215                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2216                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2217                 ("target", false, &StringSchema::new("Target directory path.").schema()),
2218                 ("repository", true, &REPO_URL_SCHEMA),
2219                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2220                 ("verbose", true, &BooleanSchema::new("Verbose output.").default(false).schema()),
2221             ]),
2222         )
2223     );
2224
2225     let mount_cmd_def = CliCommand::new(&API_METHOD_MOUNT)
2226         .arg_param(&["snapshot", "archive-name", "target"])
2227         .completion_cb("repository", complete_repository)
2228         .completion_cb("snapshot", complete_group_or_snapshot)
2229         .completion_cb("archive-name", complete_archive_name)
2230         .completion_cb("target", tools::complete_file_name);
2231
2232     #[sortable]
2233     const API_METHOD_SHELL: ApiMethod = ApiMethod::new(
2234         &ApiHandler::Sync(&shell),
2235         &ObjectSchema::new(
2236             "Shell to interactively inspect and restore snapshots.",
2237             &sorted!([
2238                 ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
2239                 ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
2240                 ("repository", true, &REPO_URL_SCHEMA),
2241                 ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
2242             ]),
2243         )
2244     );
2245
2246     let shell_cmd_def = CliCommand::new(&API_METHOD_SHELL)
2247         .arg_param(&["snapshot", "archive-name"])
2248         .completion_cb("repository", complete_repository)
2249         .completion_cb("archive-name", complete_archive_name)
2250         .completion_cb("snapshot", complete_group_or_snapshot);
2251
2252     let cmd_def = CliCommandMap::new()
2253         .insert("backup".to_owned(), backup_cmd_def.into())
2254         .insert("upload-log".to_owned(), upload_log_cmd_def.into())
2255         .insert("forget".to_owned(), forget_cmd_def.into())
2256         .insert("catalog".to_owned(), catalog_cmd_def.into())
2257         .insert("garbage-collect".to_owned(), garbage_collect_cmd_def.into())
2258         .insert("list".to_owned(), list_cmd_def.into())
2259         .insert("login".to_owned(), login_cmd_def.into())
2260         .insert("logout".to_owned(), logout_cmd_def.into())
2261         .insert("prune".to_owned(), prune_cmd_def.into())
2262         .insert("restore".to_owned(), restore_cmd_def.into())
2263         .insert("snapshots".to_owned(), snapshots_cmd_def.into())
2264         .insert("files".to_owned(), files_cmd_def.into())
2265         .insert("status".to_owned(), status_cmd_def.into())
2266         .insert("key".to_owned(), key_mgmt_cli().into())
2267         .insert("mount".to_owned(), mount_cmd_def.into())
2268         .insert("shell".to_owned(), shell_cmd_def.into());
2269
2270     run_cli_command(cmd_def.into());
2271 }
2272
2273 fn async_main<F: Future>(fut: F) -> <F as Future>::Output {
2274     let rt = tokio::runtime::Runtime::new().unwrap();
2275     let ret = rt.block_on(fut);
2276     rt.shutdown_now();
2277     ret
2278 }