]> git.proxmox.com Git - proxmox-backup.git/blobdiff - src/bin/proxmox-backup-client.rs
client: allow passing specific master key
[proxmox-backup.git] / src / bin / proxmox-backup-client.rs
index 2297d302be3be7ff0431948f22a313c864b0c596..b737d9f048a2c05d914d653655520416025ce20d 100644 (file)
@@ -77,15 +77,24 @@ pub const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
     .max_length(256)
     .schema();
 
-pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
-    "Path to encryption key. All data will be encrypted using this key.")
-    .schema();
+pub const KEYFILE_SCHEMA: Schema =
+    StringSchema::new("Path to encryption key. All data will be encrypted using this key.")
+        .schema();
+
+pub const KEYFD_SCHEMA: Schema =
+    IntegerSchema::new("Pass an encryption key via an already opened file descriptor.")
+        .minimum(0)
+        .schema();
 
-pub const KEYFD_SCHEMA: Schema = IntegerSchema::new(
-    "Pass an encryption key via an already opened file descriptor.")
-    .minimum(0)
+pub const MASTER_PUBKEY_FILE_SCHEMA: Schema = StringSchema::new(
+    "Path to master public key. The encryption key used for a backup will be encrypted using this key and appended to the backup.")
     .schema();
 
+pub const MASTER_PUBKEY_FD_SCHEMA: Schema =
+    IntegerSchema::new("Pass a master public key via an already opened file descriptor.")
+        .minimum(0)
+        .schema();
+
 const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new(
     "Chunk size in KB. Must be a power of 2.")
     .minimum(64)
@@ -211,34 +220,11 @@ fn connect_do(server: &str, port: u16, auth_id: &Authid) -> Result<HttpClient, E
         Err(NotPresent) => None,
     };
 
-    let options = HttpClientOptions::new()
-        .prefix(Some("proxmox-backup".to_string()))
-        .password(password)
-        .interactive(true)
-        .fingerprint(fingerprint)
-        .fingerprint_cache(true)
-        .ticket_cache(true);
+    let options = HttpClientOptions::new_interactive(password, fingerprint);
 
     HttpClient::new(server, port, auth_id, options)
 }
 
-async fn view_task_result(
-    client: HttpClient,
-    result: Value,
-    output_format: &str,
-) -> Result<(), Error> {
-    let data = &result["data"];
-    if output_format == "text" {
-        if let Some(upid) = data.as_str() {
-            display_task_log(client, upid, true).await?;
-        }
-    } else {
-        format_and_print_result(&data, &output_format);
-    }
-
-    Ok(())
-}
-
 async fn api_datastore_list_snapshots(
     client: &HttpClient,
     store: &str,
@@ -280,28 +266,18 @@ pub async fn api_datastore_latest_snapshot(
 
 async fn backup_directory<P: AsRef<Path>>(
     client: &BackupWriter,
-    previous_manifest: Option<Arc<BackupManifest>>,
     dir_path: P,
     archive_name: &str,
     chunk_size: Option<usize>,
-    device_set: Option<HashSet<u64>>,
-    verbose: bool,
-    skip_lost_and_found: bool,
     catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
-    exclude_pattern: Vec<MatchEntry>,
-    entries_max: usize,
-    compress: bool,
-    encrypt: bool,
+    pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
+    upload_options: UploadOptions,
 ) -> Result<BackupStats, Error> {
 
     let pxar_stream = PxarBackupStream::open(
         dir_path.as_ref(),
-        device_set,
-        verbose,
-        skip_lost_and_found,
         catalog,
-        exclude_pattern,
-        entries_max,
+        pxar_create_options,
     )?;
     let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 
@@ -317,8 +293,12 @@ async fn backup_directory<P: AsRef<Path>>(
         }
     });
 
+    if upload_options.fixed_size.is_some() {
+        bail!("cannot backup directory with fixed chunk size!");
+    }
+
     let stats = client
-        .upload_stream(previous_manifest, archive_name, stream, "dynamic", None, compress, encrypt)
+        .upload_stream(archive_name, stream, upload_options)
         .await?;
 
     Ok(stats)
@@ -326,14 +306,10 @@ async fn backup_directory<P: AsRef<Path>>(
 
 async fn backup_image<P: AsRef<Path>>(
     client: &BackupWriter,
-    previous_manifest: Option<Arc<BackupManifest>>,
     image_path: P,
     archive_name: &str,
-    image_size: u64,
     chunk_size: Option<usize>,
-    compress: bool,
-    encrypt: bool,
-    _verbose: bool,
+    upload_options: UploadOptions,
 ) -> Result<BackupStats, Error> {
 
     let path = image_path.as_ref().to_owned();
@@ -345,8 +321,12 @@ async fn backup_image<P: AsRef<Path>>(
 
     let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024));
 
+    if upload_options.fixed_size.is_none() {
+        bail!("cannot backup image with dynamic chunk size!");
+    }
+
     let stats = client
-        .upload_stream(previous_manifest, archive_name, stream, "fixed", Some(image_size), compress, encrypt)
+        .upload_stream(archive_name, stream, upload_options)
         .await?;
 
     Ok(stats)
@@ -581,32 +561,38 @@ async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 
     record_repository(&repo);
 
-    view_task_result(client, result, &output_format).await?;
+    view_task_result(&mut client, result, &output_format).await?;
 
     Ok(Value::Null)
 }
 
+struct CatalogUploadResult {
+    catalog_writer: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
+    result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
+}
+
 fn spawn_catalog_upload(
     client: Arc<BackupWriter>,
     encrypt: bool,
-) -> Result<
-        (
-            Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
-            tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>
-        ), Error>
-{
+) -> Result<CatalogUploadResult, Error> {
     let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
     let catalog_stream = crate::tools::StdChannelStream(catalog_rx);
     let catalog_chunk_size = 512*1024;
     let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
 
-    let catalog = Arc::new(Mutex::new(CatalogWriter::new(crate::tools::StdChannelWriter::new(catalog_tx))?));
+    let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(crate::tools::StdChannelWriter::new(catalog_tx))?));
 
     let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
 
+    let upload_options = UploadOptions {
+        encrypt,
+        compress: true,
+        ..UploadOptions::default()
+    };
+
     tokio::spawn(async move {
         let catalog_upload_result = client
-            .upload_stream(None, CATALOG_NAME, catalog_chunk_stream, "dynamic", None, true, encrypt)
+            .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
             .await;
 
         if let Err(ref err) = catalog_upload_result {
@@ -617,10 +603,18 @@ fn spawn_catalog_upload(
         let _ = catalog_result_tx.send(catalog_upload_result);
     });
 
-    Ok((catalog, catalog_result_rx))
+    Ok(CatalogUploadResult { catalog_writer, result: catalog_result_rx })
+}
+
+#[derive(Debug, Eq, PartialEq)]
+struct CryptoParams {
+    mode: CryptMode,
+    enc_key: Option<Vec<u8>>,
+    // FIXME switch to openssl::rsa::rsa<openssl::pkey::Public> once that is Eq?
+    master_pubkey: Option<Vec<u8>>,
 }
 
-fn keyfile_parameters(param: &Value) -> Result<(Option<Vec<u8>>, CryptMode), Error> {
+fn crypto_parameters(param: &Value) -> Result<CryptoParams, Error> {
     let keyfile = match param.get("keyfile") {
         Some(Value::String(keyfile)) => Some(keyfile),
         Some(_) => bail!("bad --keyfile parameter type"),
@@ -639,7 +633,25 @@ fn keyfile_parameters(param: &Value) -> Result<(Option<Vec<u8>>, CryptMode), Err
         None => None,
     };
 
-    let crypt_mode: Option<CryptMode> = match param.get("crypt-mode") {
+    let master_pubkey_file = match param.get("master-pubkey-file") {
+        Some(Value::String(keyfile)) => Some(keyfile),
+        Some(_) => bail!("bad --master-pubkey-file parameter type"),
+        None => None,
+    };
+
+    let master_pubkey_fd = match param.get("master-pubkey-fd") {
+        Some(Value::Number(key_fd)) => Some(
+            RawFd::try_from(key_fd
+                .as_i64()
+                .ok_or_else(|| format_err!("bad master public key fd: {:?}", key_fd))?
+            )
+            .map_err(|err| format_err!("bad public master key fd: {:?}: {}", key_fd, err))?
+        ),
+        Some(_) => bail!("bad --master-pubkey-fd parameter type"),
+        None => None,
+    };
+
+    let mode: Option<CryptMode> = match param.get("crypt-mode") {
         Some(mode) => Some(serde_json::from_value(mode.clone())?),
         None => None,
     };
@@ -663,41 +675,251 @@ fn keyfile_parameters(param: &Value) -> Result<(Option<Vec<u8>>, CryptMode), Err
         }
     };
 
-    Ok(match (keydata, crypt_mode) {
+    let master_pubkey_data = match (master_pubkey_file, master_pubkey_fd) {
+        (None, None) => None,
+        (Some(_), Some(_)) => bail!("--keyfile and --keyfd are mutually exclusive"),
+        (Some(keyfile), None) => {
+            eprintln!("Using master key from file: {}", keyfile);
+            Some(file_get_contents(keyfile)?)
+        },
+        (None, Some(fd)) => {
+            let input = unsafe { std::fs::File::from_raw_fd(fd) };
+            let mut data = Vec::new();
+            let _len: usize = { input }.read_to_end(&mut data)
+                .map_err(|err| {
+                    format_err!("error reading master key from fd {}: {}", fd, err)
+                })?;
+            eprintln!("Using master key from file descriptor");
+            Some(data)
+        }
+    };
+
+    Ok(match (keydata, master_pubkey_data, mode) {
         // no parameters:
-        (None, None) => match key::read_optional_default_encryption_key()? {
-            Some(key) => {
+        (None, None, None) => match key::read_optional_default_encryption_key()? {
+            None => CryptoParams { mode: CryptMode::None, enc_key: None, master_pubkey: None },
+            enc_key => {
                 eprintln!("Encrypting with default encryption key!");
-                (Some(key), CryptMode::Encrypt)
+                let master_pubkey = key::read_optional_default_master_pubkey()?;
+                CryptoParams {
+                    mode: CryptMode::Encrypt,
+                    enc_key,
+                    master_pubkey,
+                }
             },
-            None => (None, CryptMode::None),
         },
 
         // just --crypt-mode=none
-        (None, Some(CryptMode::None)) => (None, CryptMode::None),
+        (None, None, Some(CryptMode::None)) => CryptoParams { mode: CryptMode::None, enc_key: None, master_pubkey: None },
+
+        // --keyfile and --crypt-mode=none
+        (Some(_), _, Some(CryptMode::None)) => {
+            bail!("--keyfile/--keyfd and --crypt-mode=none are mutually exclusive");
+        },
+
+        // --master-pubkey-file and --crypt-mode=none
+        (_, Some(_), Some(CryptMode::None)) => {
+            bail!("--master-pubkey-file/--master-pubkey-fd and --crypt-mode=none are mutually exclusive");
+        },
+
+        // --master-pubkey-file and nothing else
+        (None, master_pubkey, None) => {
+            match key::read_optional_default_encryption_key()? {
+                None => bail!("--master-pubkey-file/--master-pubkey-fd specified, but no key available"),
+                enc_key => {
+                    eprintln!("Encrypting with default encryption key!");
+                    CryptoParams {
+                        mode: CryptMode::Encrypt,
+                        enc_key,
+                        master_pubkey,
+                    }
+                },
+            }
+        },
 
-        // just --crypt-mode other than none
-        (None, Some(crypt_mode)) => match key::read_optional_default_encryption_key()? {
+        // --crypt-mode other than none, without keyfile, with or without master key
+        (None, master_pubkey, Some(mode)) => match key::read_optional_default_encryption_key()? {
             None => bail!("--crypt-mode without --keyfile and no default key file available"),
-            Some(key) => {
+            enc_key => {
                 eprintln!("Encrypting with default encryption key!");
-                (Some(key), crypt_mode)
+                let master_pubkey = match master_pubkey {
+                    None => key::read_optional_default_master_pubkey()?,
+                    master_pubkey => master_pubkey,
+                };
+
+                CryptoParams {
+                    mode,
+                    enc_key,
+                    master_pubkey,
+                }
             },
         }
 
         // just --keyfile
-        (Some(key), None) => (Some(key), CryptMode::Encrypt),
+        (enc_key, master_pubkey, None) => {
+            let master_pubkey = match master_pubkey {
+                None => key::read_optional_default_master_pubkey()?,
+                master_pubkey => master_pubkey,
+            };
 
-        // --keyfile and --crypt-mode=none
-        (Some(_), Some(CryptMode::None)) => {
-            bail!("--keyfile/--keyfd and --crypt-mode=none are mutually exclusive");
-        }
+            CryptoParams { mode: CryptMode::Encrypt, enc_key, master_pubkey }
+        },
 
         // --keyfile and --crypt-mode other than none
-        (Some(key), Some(crypt_mode)) => (Some(key), crypt_mode),
+        (enc_key, master_pubkey, Some(mode)) => {
+            let master_pubkey = match master_pubkey {
+                None => key::read_optional_default_master_pubkey()?,
+                master_pubkey => master_pubkey,
+            };
+
+            CryptoParams { mode, enc_key, master_pubkey }
+        },
     })
 }
 
+#[test]
+// WARNING: there must only be one test for crypto_parameters as the default key handling is not
+// safe w.r.t. concurrency
+fn test_crypto_parameters_handling() -> Result<(), Error> {
+    let some_key = Some(vec![1;1]);
+    let default_key = Some(vec![2;1]);
+
+    let no_key_res = CryptoParams {
+        enc_key: None,
+        master_pubkey: None,
+        mode: CryptMode::None,
+    };
+    let some_key_res = CryptoParams {
+        enc_key: some_key.clone(),
+        master_pubkey: None,
+        mode: CryptMode::Encrypt,
+    };
+    let some_key_sign_res = CryptoParams {
+        enc_key: some_key.clone(),
+        master_pubkey: None,
+        mode: CryptMode::SignOnly,
+    };
+    let default_key_res = CryptoParams {
+        enc_key: default_key.clone(),
+        master_pubkey: None,
+        mode: CryptMode::Encrypt,
+    };
+    let default_key_sign_res = CryptoParams {
+        enc_key: default_key.clone(),
+        master_pubkey: None,
+        mode: CryptMode::SignOnly,
+    };
+
+    let keypath = "./tests/keyfile.test";
+    replace_file(&keypath, some_key.as_ref().unwrap(), CreateOptions::default())?;
+    let invalid_keypath = "./tests/invalid_keyfile.test";
+
+    // no params, no default key == no key
+    let res = crypto_parameters(&json!({}));
+    assert_eq!(res.unwrap(), no_key_res);
+
+    // keyfile param == key from keyfile
+    let res = crypto_parameters(&json!({"keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_res);
+
+    // crypt mode none == no key
+    let res = crypto_parameters(&json!({"crypt-mode": "none"}));
+    assert_eq!(res.unwrap(), no_key_res);
+
+    // crypt mode encrypt/sign-only, no keyfile, no default key == Error
+    assert!(crypto_parameters(&json!({"crypt-mode": "sign-only"})).is_err());
+    assert!(crypto_parameters(&json!({"crypt-mode": "encrypt"})).is_err());
+
+    // crypt mode none with explicit key == Error
+    assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
+
+    // crypt mode sign-only/encrypt with keyfile == key from keyfile with correct mode
+    let res = crypto_parameters(&json!({"crypt-mode": "sign-only", "keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_sign_res);
+    let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_res);
+
+    // invalid keyfile parameter always errors
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "none"})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "sign-only"})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "encrypt"})).is_err());
+
+    // now set a default key
+    unsafe { key::set_test_encryption_key(Ok(default_key.clone())); }
+
+    // and repeat
+
+    // no params but default key == default key
+    let res = crypto_parameters(&json!({}));
+    assert_eq!(res.unwrap(), default_key_res);
+
+    // keyfile param == key from keyfile
+    let res = crypto_parameters(&json!({"keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_res);
+
+    // crypt mode none == no key
+    let res = crypto_parameters(&json!({"crypt-mode": "none"}));
+    assert_eq!(res.unwrap(), no_key_res);
+
+    // crypt mode encrypt/sign-only, no keyfile, default key == default key with correct mode
+    let res = crypto_parameters(&json!({"crypt-mode": "sign-only"}));
+    assert_eq!(res.unwrap(), default_key_sign_res);
+    let res = crypto_parameters(&json!({"crypt-mode": "encrypt"}));
+    assert_eq!(res.unwrap(), default_key_res);
+
+    // crypt mode none with explicit key == Error
+    assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
+
+    // crypt mode sign-only/encrypt with keyfile == key from keyfile with correct mode
+    let res = crypto_parameters(&json!({"crypt-mode": "sign-only", "keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_sign_res);
+    let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_res);
+
+    // invalid keyfile parameter always errors
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "none"})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "sign-only"})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "encrypt"})).is_err());
+
+    // now make default key retrieval error
+    unsafe { key::set_test_encryption_key(Err(format_err!("test error"))); }
+
+    // and repeat
+
+    // no params, default key retrieval errors == Error
+    assert!(crypto_parameters(&json!({})).is_err());
+
+    // keyfile param == key from keyfile
+    let res = crypto_parameters(&json!({"keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_res);
+
+    // crypt mode none == no key
+    let res = crypto_parameters(&json!({"crypt-mode": "none"}));
+    assert_eq!(res.unwrap(), no_key_res);
+
+    // crypt mode encrypt/sign-only, no keyfile, default key error == Error
+    assert!(crypto_parameters(&json!({"crypt-mode": "sign-only"})).is_err());
+    assert!(crypto_parameters(&json!({"crypt-mode": "encrypt"})).is_err());
+
+    // crypt mode none with explicit key == Error
+    assert!(crypto_parameters(&json!({"crypt-mode": "none", "keyfile": keypath})).is_err());
+
+    // crypt mode sign-only/encrypt with keyfile == key from keyfile with correct mode
+    let res = crypto_parameters(&json!({"crypt-mode": "sign-only", "keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_sign_res);
+    let res = crypto_parameters(&json!({"crypt-mode": "encrypt", "keyfile": keypath}));
+    assert_eq!(res.unwrap(), some_key_res);
+
+    // invalid keyfile parameter always errors
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "none"})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "sign-only"})).is_err());
+    assert!(crypto_parameters(&json!({"keyfile": invalid_keypath, "crypt-mode": "encrypt"})).is_err());
+    Ok(())
+}
+
 #[api(
    input: {
        properties: {
@@ -733,6 +955,14 @@ fn keyfile_parameters(param: &Value) -> Result<(Option<Vec<u8>>, CryptMode), Err
                schema: KEYFD_SCHEMA,
                optional: true,
            },
+           "master-pubkey-file": {
+               schema: MASTER_PUBKEY_FILE_SCHEMA,
+               optional: true,
+           },
+           "master-pubkey-fd": {
+               schema: MASTER_PUBKEY_FD_SCHEMA,
+               optional: true,
+           },
            "crypt-mode": {
                type: CryptMode,
                optional: true,
@@ -806,7 +1036,7 @@ async fn create_backup(
         verify_chunk_size(size)?;
     }
 
-    let (keydata, crypt_mode) = keyfile_parameters(&param)?;
+    let crypto = crypto_parameters(&param)?;
 
     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 
@@ -911,7 +1141,7 @@ async fn create_backup(
 
     println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
 
-    let (crypt_config, rsa_encrypted_key) = match keydata {
+    let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
         None => (None, None),
         Some(key) => {
             let (key, created, fingerprint) = decrypt_key(&key, &key::get_encryption_key_password)?;
@@ -919,19 +1149,14 @@ async fn create_backup(
 
             let crypt_config = CryptConfig::new(key)?;
 
-            match key::find_master_pubkey()? {
-                Some(ref path) if path.exists() => {
-                    let pem_data = file_get_contents(path)?;
+            match crypto.master_pubkey {
+                Some(pem_data) => {
                     let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_data)?;
-                    let key_config = KeyConfig {
-                        kdf: None,
-                        created,
-                        modified: proxmox::tools::time::epoch_i64(),
-                        data: key.to_vec(),
-                        fingerprint: Some(fingerprint),
-                    };
+
+                    let mut key_config = KeyConfig::without_password(key)?;
+                    key_config.created = created; // keep original value
+
                     let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
-                    println!("Master key '{:?}'", path);
 
                     (Some(Arc::new(crypt_config)), Some(enc_key))
                 }
@@ -993,67 +1218,93 @@ async fn create_backup(
     let mut manifest = BackupManifest::new(snapshot);
 
     let mut catalog = None;
-    let mut catalog_result_tx = None;
+    let mut catalog_result_rx = None;
 
     for (backup_type, filename, target, size) in upload_list {
         match backup_type {
             BackupSpecificationType::CONFIG => {
+                let upload_options = UploadOptions {
+                    compress: true,
+                    encrypt: crypto.mode == CryptMode::Encrypt,
+                    ..UploadOptions::default()
+                };
+
                 println!("Upload config file '{}' to '{}' as {}", filename, repo, target);
                 let stats = client
-                    .upload_blob_from_file(&filename, &target, true, crypt_mode == CryptMode::Encrypt)
+                    .upload_blob_from_file(&filename, &target, upload_options)
                     .await?;
-                manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;
+                manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
             }
             BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ?
+                let upload_options = UploadOptions {
+                    compress: true,
+                    encrypt: crypto.mode == CryptMode::Encrypt,
+                    ..UploadOptions::default()
+                };
+
                 println!("Upload log file '{}' to '{}' as {}", filename, repo, target);
                 let stats = client
-                    .upload_blob_from_file(&filename, &target, true, crypt_mode == CryptMode::Encrypt)
+                    .upload_blob_from_file(&filename, &target, upload_options)
                     .await?;
-                manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;
+                manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
             }
             BackupSpecificationType::PXAR => {
                 // start catalog upload on first use
                 if catalog.is_none() {
-                    let (cat, res) = spawn_catalog_upload(client.clone(), crypt_mode == CryptMode::Encrypt)?;
-                    catalog = Some(cat);
-                    catalog_result_tx = Some(res);
+                    let catalog_upload_res = spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
+                    catalog = Some(catalog_upload_res.catalog_writer);
+                    catalog_result_rx = Some(catalog_upload_res.result);
                 }
                 let catalog = catalog.as_ref().unwrap();
 
                 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
                 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
+
+                let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
+                    device_set: devices.clone(),
+                    patterns: pattern_list.clone(),
+                    entries_max: entries_max as usize,
+                    skip_lost_and_found,
+                    verbose,
+                };
+
+                let upload_options = UploadOptions {
+                    previous_manifest: previous_manifest.clone(),
+                    compress: true,
+                    encrypt: crypto.mode == CryptMode::Encrypt,
+                    ..UploadOptions::default()
+                };
+
                 let stats = backup_directory(
                     &client,
-                    previous_manifest.clone(),
                     &filename,
                     &target,
                     chunk_size_opt,
-                    devices.clone(),
-                    verbose,
-                    skip_lost_and_found,
                     catalog.clone(),
-                    pattern_list.clone(),
-                    entries_max as usize,
-                    true,
-                    crypt_mode == CryptMode::Encrypt,
+                    pxar_options,
+                    upload_options,
                 ).await?;
-                manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;
+                manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
                 catalog.lock().unwrap().end_directory()?;
             }
             BackupSpecificationType::IMAGE => {
                 println!("Upload image '{}' to '{:?}' as {}", filename, repo, target);
+
+                let upload_options = UploadOptions {
+                    previous_manifest: previous_manifest.clone(),
+                    fixed_size: Some(size),
+                    compress: true,
+                    encrypt: crypto.mode == CryptMode::Encrypt,
+                };
+
                 let stats = backup_image(
                     &client,
-                    previous_manifest.clone(),
-                     &filename,
+                    &filename,
                     &target,
-                    size,
                     chunk_size_opt,
-                    true,
-                    crypt_mode == CryptMode::Encrypt,
-                    verbose,
+                    upload_options,
                 ).await?;
-                manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;
+                manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
             }
         }
     }
@@ -1068,19 +1319,20 @@ async fn create_backup(
 
         drop(catalog); // close upload stream
 
-        if let Some(catalog_result_rx) = catalog_result_tx {
+        if let Some(catalog_result_rx) = catalog_result_rx {
             let stats = catalog_result_rx.await??;
-            manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypt_mode)?;
+            manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
         }
     }
 
     if let Some(rsa_encrypted_key) = rsa_encrypted_key {
         let target = ENCRYPTED_KEY_BLOB_NAME;
         println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
+        let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() };
         let stats = client
-            .upload_blob_from_data(rsa_encrypted_key, target, false, false)
+            .upload_blob_from_data(rsa_encrypted_key, target, options)
             .await?;
-        manifest.add_file(target.to_string(), stats.size, stats.csum, crypt_mode)?;
+        manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
 
     }
     // create manifest (index.json)
@@ -1090,8 +1342,9 @@ async fn create_backup(
 
 
     if verbose { println!("Upload index.json to '{}'", repo) };
+    let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() };
     client
-        .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, true, false)
+        .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
         .await?;
 
     client.finish().await?;
@@ -1254,9 +1507,9 @@ async fn restore(param: Value) -> Result<Value, Error> {
     let target = tools::required_string_param(&param, "target")?;
     let target = if target == "-" { None } else { Some(target) };
 
-    let (keydata, _crypt_mode) = keyfile_parameters(&param)?;
+    let crypto = crypto_parameters(&param)?;
 
-    let crypt_config = match keydata {
+    let crypt_config = match crypto.enc_key {
         None => None,
         Some(key) => {
             let (key, _, fingerprint) = decrypt_key(&key, &key::get_encryption_key_password)?;
@@ -1329,20 +1582,24 @@ async fn restore(param: Value) -> Result<Value, Error> {
 
         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 
+        let options = proxmox_backup::pxar::PxarExtractOptions {
+            match_list: &[],
+            extract_match_default: true,
+            allow_existing_dirs,
+            on_error: None,
+        };
+
         if let Some(target) = target {
             proxmox_backup::pxar::extract_archive(
                 pxar::decoder::Decoder::from_std(reader)?,
                 Path::new(target),
-                &[],
-                true,
                 proxmox_backup::pxar::Flags::DEFAULT,
-                allow_existing_dirs,
                 |path| {
                     if verbose {
                         println!("{:?}", path);
                     }
                 },
-                None,
+                options,
             )
             .map_err(|err| format_err!("error extracting archive - {}", err))?;
         } else {
@@ -1536,13 +1793,9 @@ async fn try_get(repo: &BackupRepository, url: &str) -> Value {
     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
     let password = std::env::var(ENV_VAR_PBS_PASSWORD).ok();
 
-    let options = HttpClientOptions::new()
-        .prefix(Some("proxmox-backup".to_string()))
-        .password(password)
-        .interactive(false)
-        .fingerprint(fingerprint)
-        .fingerprint_cache(true)
-        .ticket_cache(true);
+    // ticket cache, but no questions asked
+    let options = HttpClientOptions::new_interactive(password, fingerprint)
+        .interactive(false);
 
     let client = match HttpClient::new(repo.host(), repo.port(), repo.auth_id(), options) {
         Ok(v) => v,
@@ -1809,6 +2062,7 @@ fn main() {
         .completion_cb("repository", complete_repository)
         .completion_cb("backupspec", complete_backup_source)
         .completion_cb("keyfile", tools::complete_file_name)
+        .completion_cb("master-pubkey-file", tools::complete_file_name)
         .completion_cb("chunk-size", complete_chunk_size);
 
     let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)