src/client/http_client.rs

   1 use std::collections::HashSet;
   2 use std::io::Write;
   3 use std::sync::atomic::{AtomicUsize, Ordering};
   4 use std::sync::{Arc, Mutex};
   5
   6 use chrono::{DateTime, Utc};
   7 use failure::*;
   8 use futures::*;
   9 use futures::stream::Stream;
  10 use http::Uri;
  11 use http::header::HeaderValue;
  12 use http::{Request, Response};
  13 use hyper::Body;
  14 use hyper::client::{Client, HttpConnector};
  15 use openssl::ssl::{SslConnector, SslMethod};
  16 use serde_json::{json, Value};
  17 use tokio::io::AsyncReadExt;
  18 use tokio::sync::{mpsc, oneshot};
  19 use url::percent_encoding::{percent_encode,  DEFAULT_ENCODE_SET};
  20 use xdg::BaseDirectories;
  21
  22 use proxmox::tools::{
  23     digest_to_hex,
  24     fs::{file_get_json, file_set_contents},
  25 };
  26
  27 use super::merge_known_chunks::{MergedChunkInfo, MergeKnownChunks};
  28 use super::pipe_to_stream::PipeToSendStream;
  29 use crate::backup::*;
  30 use crate::tools::async_io::EitherStream;
  31 use crate::tools::futures::{cancellable, Canceller};
  32 use crate::tools::{self, tty, BroadcastFuture};
  33
  34 #[derive(Clone)]
  35 pub struct AuthInfo {
  36     username: String,
  37     ticket: String,
  38     token: String,
  39 }
  40
  41 /// HTTP(S) API client
  42 pub struct HttpClient {
  43     client: Client<HttpsConnector>,
  44     server: String,
  45     auth: BroadcastFuture<AuthInfo>,
  46 }
  47
  48 /// Delete stored ticket data (logout)
  49 pub fn delete_ticket_info(server: &str, username: &str) -> Result<(), Error> {
  50
  51     let base = BaseDirectories::with_prefix("proxmox-backup")?;
  52
  53     // usually /run/user/<uid>/...
  54     let path = base.place_runtime_file("tickets")?;
  55
  56     let mode = nix::sys::stat::Mode::from_bits_truncate(0o0600);
  57
  58     let mut data = file_get_json(&path, Some(json!({})))?;
  59
  60     if let Some(map) = data[server].as_object_mut() {
  61         map.remove(username);
  62     }
  63
  64     file_set_contents(path, data.to_string().as_bytes(), Some(mode))?;
  65
  66     Ok(())
  67 }
  68
  69 fn store_ticket_info(server: &str, username: &str, ticket: &str, token: &str) -> Result<(), Error> {
  70
  71     let base = BaseDirectories::with_prefix("proxmox-backup")?;
  72
  73     // usually /run/user/<uid>/...
  74     let path = base.place_runtime_file("tickets")?;
  75
  76     let mode = nix::sys::stat::Mode::from_bits_truncate(0o0600);
  77
  78     let mut data = file_get_json(&path, Some(json!({})))?;
  79
  80     let now = Utc::now().timestamp();
  81
  82     data[server][username] = json!({ "timestamp": now, "ticket": ticket, "token": token});
  83
  84     let mut new_data = json!({});
  85
  86     let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60;
  87
  88     let empty = serde_json::map::Map::new();
  89     for (server, info) in data.as_object().unwrap_or(&empty) {
  90         for (_user, uinfo) in info.as_object().unwrap_or(&empty) {
  91             if let Some(timestamp) = uinfo["timestamp"].as_i64() {
  92                 let age = now - timestamp;
  93                 if age < ticket_lifetime {
  94                     new_data[server][username] = uinfo.clone();
  95                 }
  96             }
  97         }
  98     }
  99
 100     file_set_contents(path, new_data.to_string().as_bytes(), Some(mode))?;
 101
 102     Ok(())
 103 }
 104
 105 fn load_ticket_info(server: &str, username: &str) -> Option<(String, String)> {
 106     let base = BaseDirectories::with_prefix("proxmox-backup").ok()?;
 107
 108     // usually /run/user/<uid>/...
 109     let path = base.place_runtime_file("tickets").ok()?;
 110     let data = file_get_json(&path, None).ok()?;
 111     let now = Utc::now().timestamp();
 112     let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60;
 113     let uinfo = data[server][username].as_object()?;
 114     let timestamp = uinfo["timestamp"].as_i64()?;
 115     let age = now - timestamp;
 116
 117     if age < ticket_lifetime {
 118         let ticket = uinfo["ticket"].as_str()?;
 119         let token = uinfo["token"].as_str()?;
 120         Some((ticket.to_owned(), token.to_owned()))
 121     } else {
 122         None
 123     }
 124 }
 125
 126 impl HttpClient {
 127
 128     pub fn new(server: &str, username: &str, password: Option<String>) -> Result<Self, Error> {
 129         let client = Self::build_client();
 130
 131         let password = if let Some(password) = password {
 132             password
 133         } else if let Some((ticket, _token)) = load_ticket_info(server, username) {
 134             ticket
 135         } else {
 136             Self::get_password(&username)?
 137         };
 138
 139         let login_future = Self::credentials(client.clone(), server.to_owned(), username.to_owned(), password);
 140
 141         Ok(Self {
 142             client,
 143             server: String::from(server),
 144             auth: BroadcastFuture::new(Box::new(login_future)),
 145         })
 146     }
 147
 148     /// Login
 149     ///
 150     /// Login is done on demand, so this is onyl required if you need
 151     /// access to authentication data in 'AuthInfo'.
 152     pub async fn login(&self) -> Result<AuthInfo, Error> {
 153         self.auth.listen().await
 154     }
 155
 156     fn get_password(_username: &str) -> Result<String, Error> {
 157         use std::env::VarError::*;
 158         match std::env::var("PBS_PASSWORD") {
 159             Ok(p) => return Ok(p),
 160             Err(NotUnicode(_)) => bail!("PBS_PASSWORD contains bad characters"),
 161             Err(NotPresent) => {
 162                 // Try another method
 163             }
 164         }
 165
 166         // If we're on a TTY, query the user for a password
 167         if tty::stdin_isatty() {
 168             return Ok(String::from_utf8(tty::read_password("Password: ")?)?);
 169         }
 170
 171         bail!("no password input mechanism available");
 172     }
 173
 174     fn build_client() -> Client<HttpsConnector> {
 175
 176         let mut ssl_connector_builder = SslConnector::builder(SslMethod::tls()).unwrap();
 177
 178         ssl_connector_builder.set_verify(openssl::ssl::SslVerifyMode::NONE); // fixme!
 179
 180         let mut httpc = hyper::client::HttpConnector::new();
 181         httpc.set_nodelay(true); // important for h2 download performance!
 182         httpc.set_recv_buffer_size(Some(1024*1024)); //important for h2 download performance!
 183         httpc.enforce_http(false); // we want https...
 184
 185         let https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build());
 186
 187         Client::builder()
 188         //.http2_initial_stream_window_size( (1 << 31) - 2)
 189         //.http2_initial_connection_window_size( (1 << 31) - 2)
 190             .build::<_, Body>(https)
 191     }
 192
 193     pub async fn request(&self, mut req: Request<Body>) -> Result<Value, Error> {
 194
 195         let client = self.client.clone();
 196
 197         let auth =  self.login().await?;
 198
 199         let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
 200         req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap());
 201         req.headers_mut().insert("CSRFPreventionToken", HeaderValue::from_str(&auth.token).unwrap());
 202
 203         Self::api_request(client, req).await
 204     }
 205
 206     pub async fn get(
 207         &self,
 208         path: &str,
 209         data: Option<Value>,
 210     ) -> Result<Value, Error> {
 211         let req = Self::request_builder(&self.server, "GET", path, data).unwrap();
 212         self.request(req).await
 213     }
 214
 215     pub async fn delete(
 216         &mut self,
 217         path: &str,
 218         data: Option<Value>,
 219     ) -> Result<Value, Error> {
 220         let req = Self::request_builder(&self.server, "DELETE", path, data).unwrap();
 221         self.request(req).await
 222     }
 223
 224     pub async fn post(
 225         &mut self,
 226         path: &str,
 227         data: Option<Value>,
 228     ) -> Result<Value, Error> {
 229         let req = Self::request_builder(&self.server, "POST", path, data).unwrap();
 230         self.request(req).await
 231     }
 232
 233     pub async fn download(
 234         &mut self,
 235         path: &str,
 236         output: &mut (dyn Write + Send),
 237     ) ->  Result<(), Error> {
 238         let mut req = Self::request_builder(&self.server, "GET", path, None).unwrap();
 239
 240         let client = self.client.clone();
 241
 242         let auth = self.login().await?;
 243
 244         let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
 245         req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap());
 246
 247         let resp = client.request(req).await?;
 248         let status = resp.status();
 249         if !status.is_success() {
 250             HttpClient::api_response(resp)
 251                 .map(|_| Err(format_err!("unknown error")))
 252                 .await?
 253         } else {
 254             resp.into_body()
 255                 .map_err(Error::from)
 256                 .try_fold(output, move |acc, chunk| async move {
 257                     acc.write_all(&chunk)?;
 258                     Ok::<_, Error>(acc)
 259                 })
 260                 .await?;
 261         }
 262         Ok(())
 263     }
 264
 265     pub async fn upload(
 266         &mut self,
 267         content_type: &str,
 268         body: Body,
 269         path: &str,
 270         data: Option<Value>,
 271     ) -> Result<Value, Error> {
 272
 273         let path = path.trim_matches('/');
 274         let mut url = format!("https://{}:8007/{}", &self.server, path);
 275
 276         if let Some(data) = data {
 277             let query = tools::json_object_to_query(data).unwrap();
 278             url.push('?');
 279             url.push_str(&query);
 280         }
 281
 282         let url: Uri = url.parse().unwrap();
 283
 284         let req = Request::builder()
 285             .method("POST")
 286             .uri(url)
 287             .header("User-Agent", "proxmox-backup-client/1.0")
 288             .header("Content-Type", content_type)
 289             .body(body).unwrap();
 290
 291         self.request(req).await
 292     }
 293
 294     pub async fn start_backup(
 295         &self,
 296         datastore: &str,
 297         backup_type: &str,
 298         backup_id: &str,
 299         backup_time: DateTime<Utc>,
 300         debug: bool,
 301     ) -> Result<Arc<BackupClient>, Error> {
 302
 303         let param = json!({
 304             "backup-type": backup_type,
 305             "backup-id": backup_id,
 306             "backup-time": backup_time.timestamp(),
 307             "store": datastore,
 308             "debug": debug
 309         });
 310
 311         let req = Self::request_builder(&self.server, "GET", "/api2/json/backup", Some(param)).unwrap();
 312
 313         let (h2, canceller) = self.start_h2_connection(req, String::from(PROXMOX_BACKUP_PROTOCOL_ID_V1!())).await?;
 314
 315         Ok(BackupClient::new(h2, canceller))
 316     }
 317
 318     pub async fn start_backup_reader(
 319         &self,
 320         datastore: &str,
 321         backup_type: &str,
 322         backup_id: &str,
 323         backup_time: DateTime<Utc>,
 324         debug: bool,
 325     ) -> Result<Arc<BackupReader>, Error> {
 326
 327         let param = json!({
 328             "backup-type": backup_type,
 329             "backup-id": backup_id,
 330             "backup-time": backup_time.timestamp(),
 331             "store": datastore,
 332             "debug": debug,
 333         });
 334         let req = Self::request_builder(&self.server, "GET", "/api2/json/reader", Some(param)).unwrap();
 335
 336         let (h2, canceller) = self.start_h2_connection(req, String::from(PROXMOX_BACKUP_READER_PROTOCOL_ID_V1!())).await?;
 337
 338         Ok(BackupReader::new(h2, canceller))
 339     }
 340
 341     pub async fn start_h2_connection(
 342         &self,
 343         mut req: Request<Body>,
 344         protocol_name: String,
 345     ) -> Result<(H2Client, Canceller), Error> {
 346
 347         let auth = self.login().await?;
 348         let client = self.client.clone();
 349
 350         let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
 351         req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap());
 352         req.headers_mut().insert("UPGRADE", HeaderValue::from_str(&protocol_name).unwrap());
 353
 354         let resp = client.request(req).await?;
 355         let status = resp.status();
 356
 357         if status != http::StatusCode::SWITCHING_PROTOCOLS {
 358             Self::api_response(resp)
 359                 .map(|_| Err(format_err!("unknown error")))
 360                 .await?;
 361             unreachable!();
 362         }
 363
 364         let upgraded = resp
 365             .into_body()
 366             .on_upgrade()
 367             .await?;
 368
 369         let max_window_size = (1 << 31) - 2;
 370
 371         let (h2, connection) = h2::client::Builder::new()
 372             .initial_connection_window_size(max_window_size)
 373             .initial_window_size(max_window_size)
 374             .max_frame_size(4*1024*1024)
 375             .handshake(upgraded)
 376             .await?;
 377
 378         let connection = connection
 379             .map_err(|_| panic!("HTTP/2.0 connection failed"));
 380
 381         let (connection, canceller) = cancellable(connection)?;
 382         // A cancellable future returns an Option which is None when cancelled and
 383         // Some when it finished instead, since we don't care about the return type we
 384         // need to map it away:
 385         let connection = connection.map(|_| ());
 386
 387         // Spawn a new task to drive the connection state
 388         hyper::rt::spawn(connection);
 389
 390         // Wait until the `SendRequest` handle has available capacity.
 391         let c = h2.ready().await?;
 392         Ok((H2Client::new(c), canceller))
 393     }
 394
 395     async fn credentials(
 396         client: Client<HttpsConnector>,
 397         server: String,
 398         username: String,
 399         password: String,
 400     ) -> Result<AuthInfo, Error> {
 401         let data = json!({ "username": username, "password": password });
 402         let req = Self::request_builder(&server, "POST", "/api2/json/access/ticket", Some(data)).unwrap();
 403         let cred = Self::api_request(client, req).await?;
 404         let auth = AuthInfo {
 405             username: cred["data"]["username"].as_str().unwrap().to_owned(),
 406             ticket: cred["data"]["ticket"].as_str().unwrap().to_owned(),
 407             token: cred["data"]["CSRFPreventionToken"].as_str().unwrap().to_owned(),
 408         };
 409
 410         let _ = store_ticket_info(&server, &auth.username, &auth.ticket, &auth.token);
 411
 412         Ok(auth)
 413     }
 414
 415     async fn api_response(response: Response<Body>) -> Result<Value, Error> {
 416         let status = response.status();
 417         let data = response
 418             .into_body()
 419             .try_concat()
 420             .await?;
 421
 422         let text = String::from_utf8(data.to_vec()).unwrap();
 423         if status.is_success() {
 424             if text.len() > 0 {
 425                 let value: Value = serde_json::from_str(&text)?;
 426                 Ok(value)
 427             } else {
 428                 Ok(Value::Null)
 429             }
 430         } else {
 431             bail!("HTTP Error {}: {}", status, text);
 432         }
 433     }
 434
 435     async fn api_request(
 436         client: Client<HttpsConnector>,
 437         req: Request<Body>
 438     ) -> Result<Value, Error> {
 439
 440         client.request(req)
 441             .map_err(Error::from)
 442             .and_then(Self::api_response)
 443             .await
 444     }
 445
 446     pub fn request_builder(server: &str, method: &str, path: &str, data: Option<Value>) -> Result<Request<Body>, Error> {
 447         let path = path.trim_matches('/');
 448         let url: Uri = format!("https://{}:8007/{}", server, path).parse()?;
 449
 450         if let Some(data) = data {
 451             if method == "POST" {
 452                 let request = Request::builder()
 453                     .method(method)
 454                     .uri(url)
 455                     .header("User-Agent", "proxmox-backup-client/1.0")
 456                     .header(hyper::header::CONTENT_TYPE, "application/json")
 457                     .body(Body::from(data.to_string()))?;
 458                 return Ok(request);
 459             } else {
 460                 let query = tools::json_object_to_query(data)?;
 461                 let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?;
 462                 let request = Request::builder()
 463                     .method(method)
 464                     .uri(url)
 465                     .header("User-Agent", "proxmox-backup-client/1.0")
 466                     .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
 467                     .body(Body::empty())?;
 468                 return Ok(request);
 469             }
 470         }
 471
 472         let request = Request::builder()
 473             .method(method)
 474             .uri(url)
 475             .header("User-Agent", "proxmox-backup-client/1.0")
 476             .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
 477             .body(Body::empty())?;
 478
 479         Ok(request)
 480     }
 481 }
 482
 483
 484 pub struct BackupReader {
 485     h2: H2Client,
 486     canceller: Canceller,
 487 }
 488
 489 impl Drop for BackupReader {
 490
 491     fn drop(&mut self) {
 492         self.canceller.cancel();
 493     }
 494 }
 495
 496 impl BackupReader {
 497
 498     pub fn new(h2: H2Client, canceller: Canceller) -> Arc<Self> {
 499         Arc::new(Self { h2, canceller })
 500     }
 501
 502     pub async fn get(
 503         &self,
 504         path: &str,
 505         param: Option<Value>,
 506     ) -> Result<Value, Error> {
 507         self.h2.get(path, param).await
 508     }
 509
 510     pub async fn put(
 511         &self,
 512         path: &str,
 513         param: Option<Value>,
 514     ) -> Result<Value, Error> {
 515         self.h2.put(path, param).await
 516     }
 517
 518     pub async fn post(
 519         &self,
 520         path: &str,
 521         param: Option<Value>,
 522     ) -> Result<Value, Error> {
 523         self.h2.post(path, param).await
 524     }
 525
 526     pub async fn download<W: Write + Send>(
 527         &self,
 528         file_name: &str,
 529         output: W,
 530     ) -> Result<W, Error> {
 531         let path = "download";
 532         let param = json!({ "file-name": file_name });
 533         self.h2.download(path, Some(param), output).await
 534     }
 535
 536     pub async fn speedtest<W: Write + Send>(
 537         &self,
 538         output: W,
 539     ) -> Result<W, Error> {
 540         self.h2.download("speedtest", None, output).await
 541     }
 542
 543     pub async fn download_chunk<W: Write + Send>(
 544         &self,
 545         digest: &[u8; 32],
 546         output: W,
 547     ) -> Result<W, Error> {
 548         let path = "chunk";
 549         let param = json!({ "digest": digest_to_hex(digest) });
 550         self.h2.download(path, Some(param), output).await
 551     }
 552
 553     pub fn force_close(self) {
 554         self.canceller.cancel();
 555     }
 556 }
 557
 558 pub struct BackupClient {
 559     h2: H2Client,
 560     canceller: Canceller,
 561 }
 562
 563 impl Drop for BackupClient {
 564
 565     fn drop(&mut self) {
 566         self.canceller.cancel();
 567     }
 568 }
 569
 570 pub struct BackupStats {
 571     pub size: u64,
 572     pub csum: [u8; 32],
 573 }
 574
 575 impl BackupClient {
 576     pub fn new(h2: H2Client, canceller: Canceller) -> Arc<Self> {
 577         Arc::new(Self { h2, canceller })
 578     }
 579
 580     pub async fn get(
 581         &self,
 582         path: &str,
 583         param: Option<Value>,
 584     ) -> Result<Value, Error> {
 585         self.h2.get(path, param).await
 586     }
 587
 588     pub async fn put(
 589         &self,
 590         path: &str,
 591         param: Option<Value>,
 592     ) -> Result<Value, Error> {
 593         self.h2.put(path, param).await
 594     }
 595
 596     pub async fn post(
 597         &self,
 598         path: &str,
 599         param: Option<Value>,
 600     ) -> Result<Value, Error> {
 601         self.h2.post(path, param).await
 602     }
 603
 604     pub async fn upload_post(
 605         &self,
 606         path: &str,
 607         param: Option<Value>,
 608         content_type: &str,
 609         data: Vec<u8>,
 610     ) -> Result<Value, Error> {
 611         self.h2.upload(path, "POST", param, content_type, data).await
 612     }
 613
 614     pub async fn upload_put(
 615         &self,
 616         path: &str,
 617         param: Option<Value>,
 618         content_type: &str,
 619         data: Vec<u8>,
 620     ) -> Result<Value, Error> {
 621         self.h2.upload(path, "PUT", param, content_type, data).await
 622     }
 623
 624     pub async fn finish(self: Arc<Self>) -> Result<(), Error> {
 625         let h2 = self.h2.clone();
 626
 627         h2.post("finish", None)
 628             .map_ok(move |_| {
 629                 self.canceller.cancel();
 630             })
 631             .await
 632     }
 633
 634     pub fn force_close(self) {
 635         self.canceller.cancel();
 636     }
 637
 638     pub async fn upload_blob<R: std::io::Read>(
 639         &self,
 640         mut reader: R,
 641         file_name: &str,
 642      ) -> Result<BackupStats, Error> {
 643         let mut raw_data = Vec::new();
 644         // fixme: avoid loading into memory
 645         reader.read_to_end(&mut raw_data)?;
 646
 647         let csum = openssl::sha::sha256(&raw_data);
 648         let param = json!({"encoded-size": raw_data.len(), "file-name": file_name });
 649         let size = raw_data.len() as u64; // fixme: should be decoded size instead??
 650         let _value = self.h2.upload("blob", "POST", Some(param), "application/octet-stream", raw_data).await?;
 651         Ok(BackupStats { size, csum })
 652     }
 653
 654     pub async fn upload_blob_from_data(
 655         &self,
 656         data: Vec<u8>,
 657         file_name: &str,
 658         crypt_config: Option<Arc<CryptConfig>>,
 659         compress: bool,
 660         sign_only: bool,
 661      ) -> Result<BackupStats, Error> {
 662
 663         let size = data.len() as u64;
 664
 665         let blob = if let Some(crypt_config) = crypt_config {
 666             if sign_only {
 667                 DataBlob::create_signed(&data, crypt_config, compress)?
 668             } else {
 669                 DataBlob::encode(&data, Some(crypt_config.clone()), compress)?
 670             }
 671         } else {
 672             DataBlob::encode(&data, None, compress)?
 673         };
 674
 675         let raw_data = blob.into_inner();
 676
 677         let csum = openssl::sha::sha256(&raw_data);
 678         let param = json!({"encoded-size": raw_data.len(), "file-name": file_name });
 679         let _value = self.h2.upload("blob", "POST", Some(param), "application/octet-stream", raw_data).await?;
 680         Ok(BackupStats { size, csum })
 681     }
 682
 683     pub async fn upload_blob_from_file<P: AsRef<std::path::Path>>(
 684         &self,
 685         src_path: P,
 686         file_name: &str,
 687         crypt_config: Option<Arc<CryptConfig>>,
 688         compress: bool,
 689      ) -> Result<BackupStats, Error> {
 690
 691         let src_path = src_path.as_ref();
 692
 693         let mut file = tokio::fs::File::open(src_path)
 694             .await
 695             .map_err(|err| format_err!("unable to open file {:?} - {}", src_path, err))?;
 696
 697         let mut contents = Vec::new();
 698
 699         file.read_to_end(&mut contents)
 700             .await
 701             .map_err(|err| format_err!("unable to read file {:?} - {}", src_path, err))?;
 702
 703         let size: u64 = contents.len() as u64;
 704         let blob = DataBlob::encode(&contents, crypt_config, compress)?;
 705         let raw_data = blob.into_inner();
 706         let csum = openssl::sha::sha256(&raw_data);
 707         let param = json!({
 708             "encoded-size": raw_data.len(),
 709             "file-name": file_name,
 710         });
 711         self.h2.upload("blob", "POST", Some(param), "application/octet-stream", raw_data).await?;
 712         Ok(BackupStats { size, csum })
 713     }
 714
 715     pub async fn upload_stream(
 716         &self,
 717         archive_name: &str,
 718         stream: impl Stream<Item = Result<bytes::BytesMut, Error>>,
 719         prefix: &str,
 720         fixed_size: Option<u64>,
 721         crypt_config: Option<Arc<CryptConfig>>,
 722     ) -> Result<BackupStats, Error> {
 723         let known_chunks = Arc::new(Mutex::new(HashSet::new()));
 724
 725         let mut param = json!({ "archive-name": archive_name });
 726         if let Some(size) = fixed_size {
 727             param["size"] = size.into();
 728         }
 729
 730         let index_path = format!("{}_index", prefix);
 731         let close_path = format!("{}_close", prefix);
 732
 733         self.download_chunk_list(&index_path, archive_name, known_chunks.clone()).await?;
 734
 735         let wid = self.h2.post(&index_path, Some(param)).await?.as_u64().unwrap();
 736
 737         let (chunk_count, size, _speed, csum) =
 738             Self::upload_chunk_info_stream(
 739                 self.h2.clone(),
 740                 wid,
 741                 stream,
 742                 &prefix,
 743                 known_chunks.clone(),
 744                 crypt_config,
 745             )
 746             .await?;
 747
 748         let param = json!({
 749             "wid": wid ,
 750             "chunk-count": chunk_count,
 751             "size": size,
 752         });
 753         let _value = self.h2.post(&close_path, Some(param)).await?;
 754         Ok(BackupStats {
 755             size: size as u64,
 756             csum,
 757         })
 758     }
 759
 760     fn response_queue() -> (
 761         mpsc::Sender<h2::client::ResponseFuture>,
 762         oneshot::Receiver<Result<(), Error>>
 763     ) {
 764         let (verify_queue_tx, verify_queue_rx) = mpsc::channel(100);
 765         let (verify_result_tx, verify_result_rx) = oneshot::channel();
 766
 767         hyper::rt::spawn(
 768             verify_queue_rx
 769                 .map(Ok::<_, Error>)
 770                 .try_for_each(|response: h2::client::ResponseFuture| {
 771                     response
 772                         .map_err(Error::from)
 773                         .and_then(H2Client::h2api_response)
 774                         .map_ok(|result| println!("RESPONSE: {:?}", result))
 775                         .map_err(|err| format_err!("pipelined request failed: {}", err))
 776                 })
 777                 .map(|result| {
 778                       let _ignore_closed_channel = verify_result_tx.send(result);
 779                 })
 780         );
 781
 782         (verify_queue_tx, verify_result_rx)
 783     }
 784
 785     fn append_chunk_queue(h2: H2Client, wid: u64, path: String) -> (
 786         mpsc::Sender<(MergedChunkInfo, Option<h2::client::ResponseFuture>)>,
 787         oneshot::Receiver<Result<(), Error>>
 788     ) {
 789         let (verify_queue_tx, verify_queue_rx) = mpsc::channel(64);
 790         let (verify_result_tx, verify_result_rx) = oneshot::channel();
 791
 792         let h2_2 = h2.clone();
 793
 794         hyper::rt::spawn(
 795             verify_queue_rx
 796                 .map(Ok::<_, Error>)
 797                 .and_then(move |(merged_chunk_info, response): (MergedChunkInfo, Option<h2::client::ResponseFuture>)| {
 798                     match (response, merged_chunk_info) {
 799                         (Some(response), MergedChunkInfo::Known(list)) => {
 800                             future::Either::Left(
 801                                 response
 802                                     .map_err(Error::from)
 803                                     .and_then(H2Client::h2api_response)
 804                                     .and_then(move |_result| {
 805                                         future::ok(MergedChunkInfo::Known(list))
 806                                     })
 807                             )
 808                         }
 809                         (None, MergedChunkInfo::Known(list)) => {
 810                             future::Either::Right(future::ok(MergedChunkInfo::Known(list)))
 811                         }
 812                         _ => unreachable!(),
 813                     }
 814                 })
 815                 .merge_known_chunks()
 816                 .and_then(move |merged_chunk_info| {
 817                     match merged_chunk_info {
 818                         MergedChunkInfo::Known(chunk_list) => {
 819                             let mut digest_list = vec![];
 820                             let mut offset_list = vec![];
 821                             for (offset, digest) in chunk_list {
 822                                 //println!("append chunk {} (offset {})", proxmox::tools::digest_to_hex(&digest), offset);
 823                                 digest_list.push(digest_to_hex(&digest));
 824                                 offset_list.push(offset);
 825                             }
 826                             println!("append chunks list len ({})", digest_list.len());
 827                             let param = json!({ "wid": wid, "digest-list": digest_list, "offset-list": offset_list });
 828                             let request = H2Client::request_builder("localhost", "PUT", &path, None, Some("application/json")).unwrap();
 829                             let param_data = bytes::Bytes::from(param.to_string().as_bytes());
 830                             let upload_data = Some(param_data);
 831                             h2_2.send_request(request, upload_data)
 832                                 .and_then(move |response| {
 833                                     response
 834                                         .map_err(Error::from)
 835                                         .and_then(H2Client::h2api_response)
 836                                         .map_ok(|_| ())
 837                                 })
 838                                 .map_err(|err| format_err!("pipelined request failed: {}", err))
 839                         }
 840                         _ => unreachable!(),
 841                     }
 842                 })
 843                 .try_for_each(|_| future::ok(()))
 844                 .map(|result| {
 845                       let _ignore_closed_channel = verify_result_tx.send(result);
 846                 })
 847         );
 848
 849         (verify_queue_tx, verify_result_rx)
 850     }
 851
 852     pub async fn download_chunk_list(
 853         &self,
 854         path: &str,
 855         archive_name: &str,
 856         known_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
 857     ) -> Result<(), Error> {
 858
 859         let param = json!({ "archive-name": archive_name });
 860         let request = H2Client::request_builder("localhost", "GET", path, Some(param), None).unwrap();
 861
 862         let h2request = self.h2.send_request(request, None).await?;
 863         let resp = h2request.await?;
 864
 865         let status = resp.status();
 866
 867         if !status.is_success() {
 868             H2Client::h2api_response(resp).await?; // raise error
 869             unreachable!();
 870         }
 871
 872         let mut body = resp.into_body();
 873         let mut release_capacity = body.release_capacity().clone();
 874
 875         let mut stream = DigestListDecoder::new(body.map_err(Error::from));
 876
 877         while let Some(chunk) = stream.try_next().await? {
 878             let _ = release_capacity.release_capacity(chunk.len());
 879             println!("GOT DOWNLOAD {}", digest_to_hex(&chunk));
 880             known_chunks.lock().unwrap().insert(chunk);
 881         }
 882
 883         Ok(())
 884     }
 885
 886     fn upload_chunk_info_stream(
 887         h2: H2Client,
 888         wid: u64,
 889         stream: impl Stream<Item = Result<bytes::BytesMut, Error>>,
 890         prefix: &str,
 891         known_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
 892         crypt_config: Option<Arc<CryptConfig>>,
 893     ) -> impl Future<Output = Result<(usize, usize, usize, [u8; 32]), Error>> {
 894
 895         let repeat = Arc::new(AtomicUsize::new(0));
 896         let repeat2 = repeat.clone();
 897
 898         let stream_len = Arc::new(AtomicUsize::new(0));
 899         let stream_len2 = stream_len.clone();
 900
 901         let append_chunk_path = format!("{}_index", prefix);
 902         let upload_chunk_path = format!("{}_chunk", prefix);
 903
 904         let (upload_queue, upload_result) =
 905             Self::append_chunk_queue(h2.clone(), wid, append_chunk_path.to_owned());
 906
 907         let start_time = std::time::Instant::now();
 908
 909         let index_csum = Arc::new(Mutex::new(Some(openssl::sha::Sha256::new())));
 910         let index_csum_2 = index_csum.clone();
 911
 912         stream
 913             .and_then(move |data| {
 914
 915                 let chunk_len = data.len();
 916
 917                 repeat.fetch_add(1, Ordering::SeqCst);
 918                 let offset = stream_len.fetch_add(chunk_len, Ordering::SeqCst) as u64;
 919
 920                 let mut chunk_builder = DataChunkBuilder::new(data.as_ref())
 921                     .compress(true);
 922
 923                 if let Some(ref crypt_config) = crypt_config {
 924                     chunk_builder = chunk_builder.crypt_config(crypt_config);
 925                 }
 926
 927                 let mut known_chunks = known_chunks.lock().unwrap();
 928                 let digest = chunk_builder.digest();
 929
 930                 let mut guard = index_csum.lock().unwrap();
 931                 let csum = guard.as_mut().unwrap();
 932
 933                 let chunk_end = offset + chunk_len as u64;
 934
 935                 csum.update(&chunk_end.to_le_bytes());
 936                 csum.update(digest);
 937
 938                 let chunk_is_known = known_chunks.contains(digest);
 939                 if chunk_is_known {
 940                     future::ok(MergedChunkInfo::Known(vec![(offset, *digest)]))
 941                 } else {
 942                     known_chunks.insert(*digest);
 943                     future::ready(chunk_builder
 944                         .build()
 945                         .map(move |chunk| MergedChunkInfo::New(ChunkInfo {
 946                             chunk,
 947                             chunk_len: chunk_len as u64,
 948                             offset,
 949                         }))
 950                     )
 951                 }
 952             })
 953             .merge_known_chunks()
 954             .try_for_each(move |merged_chunk_info| {
 955
 956                 if let MergedChunkInfo::New(chunk_info) = merged_chunk_info {
 957                     let offset = chunk_info.offset;
 958                     let digest = *chunk_info.chunk.digest();
 959                     let digest_str = digest_to_hex(&digest);
 960
 961                     println!("upload new chunk {} ({} bytes, offset {})", digest_str,
 962                              chunk_info.chunk_len, offset);
 963
 964                     let chunk_data = chunk_info.chunk.raw_data();
 965                     let param = json!({
 966                         "wid": wid,
 967                         "digest": digest_str,
 968                         "size": chunk_info.chunk_len,
 969                         "encoded-size": chunk_data.len(),
 970                     });
 971
 972                     let ct = "application/octet-stream";
 973                     let request = H2Client::request_builder("localhost", "POST", &upload_chunk_path, Some(param), Some(ct)).unwrap();
 974                     let upload_data = Some(bytes::Bytes::from(chunk_data));
 975
 976                     let new_info = MergedChunkInfo::Known(vec![(offset, digest)]);
 977
 978                     let mut upload_queue = upload_queue.clone();
 979                     future::Either::Left(h2
 980                         .send_request(request, upload_data)
 981                         .and_then(move |response| async move {
 982                             upload_queue
 983                                 .send((new_info, Some(response)))
 984                                 .await
 985                                 .map_err(Error::from)
 986                         })
 987                     )
 988                 } else {
 989                     let mut upload_queue = upload_queue.clone();
 990                     future::Either::Right(async move {
 991                         upload_queue
 992                             .send((merged_chunk_info, None))
 993                             .await
 994                             .map_err(Error::from)
 995                     })
 996                 }
 997             })
 998             .then(move |result| async move {
 999                 upload_result.await?.and(result)
1000             }.boxed())
1001             .and_then(move |_| {
1002                 let repeat = repeat2.load(Ordering::SeqCst);
1003                 let stream_len = stream_len2.load(Ordering::SeqCst);
1004                 let speed = ((stream_len*1000000)/(1024*1024))/(start_time.elapsed().as_micros() as usize);
1005                 println!("Uploaded {} chunks in {} seconds ({} MB/s).", repeat, start_time.elapsed().as_secs(), speed);
1006                 if repeat > 0 {
1007                     println!("Average chunk size was {} bytes.", stream_len/repeat);
1008                     println!("Time per request: {} microseconds.", (start_time.elapsed().as_micros())/(repeat as u128));
1009                 }
1010
1011                 let mut guard = index_csum_2.lock().unwrap();
1012                 let csum = guard.take().unwrap().finish();
1013
1014                 futures::future::ok((repeat, stream_len, speed, csum))
1015             })
1016     }
1017
1018     pub async fn upload_speedtest(&self) -> Result<usize, Error> {
1019
1020         let mut data = vec![];
1021         // generate pseudo random byte sequence
1022         for i in 0..1024*1024 {
1023             for j in 0..4 {
1024                 let byte = ((i >> (j<<3))&0xff) as u8;
1025                 data.push(byte);
1026             }
1027         }
1028
1029         let item_len = data.len();
1030
1031         let mut repeat = 0;
1032
1033         let (upload_queue, upload_result) = Self::response_queue();
1034
1035         let start_time = std::time::Instant::now();
1036
1037         loop {
1038             repeat += 1;
1039             if start_time.elapsed().as_secs() >= 5 {
1040                 break;
1041             }
1042
1043             let mut upload_queue = upload_queue.clone();
1044
1045             println!("send test data ({} bytes)", data.len());
1046             let request = H2Client::request_builder("localhost", "POST", "speedtest", None, None).unwrap();
1047             let request_future = self.h2.send_request(request, Some(bytes::Bytes::from(data.clone()))).await?;
1048
1049             upload_queue.send(request_future).await?;
1050         }
1051
1052         drop(upload_queue); // close queue
1053
1054         let _ = upload_result.await?;
1055
1056         println!("Uploaded {} chunks in {} seconds.", repeat, start_time.elapsed().as_secs());
1057         let speed = ((item_len*1000000*(repeat as usize))/(1024*1024))/(start_time.elapsed().as_micros() as usize);
1058         println!("Time per request: {} microseconds.", (start_time.elapsed().as_micros())/(repeat as u128));
1059
1060         Ok(speed)
1061     }
1062 }
1063
1064 #[derive(Clone)]
1065 pub struct H2Client {
1066     h2: h2::client::SendRequest<bytes::Bytes>,
1067 }
1068
1069 impl H2Client {
1070
1071     pub fn new(h2: h2::client::SendRequest<bytes::Bytes>) -> Self {
1072         Self { h2 }
1073     }
1074
1075     pub async fn get(
1076         &self,
1077         path: &str,
1078         param: Option<Value>
1079     ) -> Result<Value, Error> {
1080         let req = Self::request_builder("localhost", "GET", path, param, None).unwrap();
1081         self.request(req).await
1082     }
1083
1084     pub async fn put(
1085         &self,
1086         path: &str,
1087         param: Option<Value>
1088     ) -> Result<Value, Error> {
1089         let req = Self::request_builder("localhost", "PUT", path, param, None).unwrap();
1090         self.request(req).await
1091     }
1092
1093     pub async fn post(
1094         &self,
1095         path: &str,
1096         param: Option<Value>
1097     ) -> Result<Value, Error> {
1098         let req = Self::request_builder("localhost", "POST", path, param, None).unwrap();
1099         self.request(req).await
1100     }
1101
1102     pub async fn download<W: Write + Send>(
1103         &self,
1104         path: &str,
1105         param: Option<Value>,
1106         mut output: W,
1107     ) -> Result<W, Error> {
1108         let request = Self::request_builder("localhost", "GET", path, param, None).unwrap();
1109
1110         let response_future = self.send_request(request, None).await?;
1111
1112         let resp = response_future.await?;
1113
1114         let status = resp.status();
1115         if !status.is_success() {
1116             H2Client::h2api_response(resp).await?; // raise error
1117             unreachable!();
1118         }
1119
1120         let mut body = resp.into_body();
1121         let mut release_capacity = body.release_capacity().clone();
1122
1123         while let Some(chunk) = body.try_next().await? {
1124             let _ = release_capacity.release_capacity(chunk.len());
1125             output.write_all(&chunk)?;
1126         }
1127
1128         Ok(output)
1129     }
1130
1131     pub async fn upload(
1132         &self,
1133         method: &str, // POST or PUT
1134         path: &str,
1135         param: Option<Value>,
1136         content_type: &str,
1137         data: Vec<u8>,
1138     ) -> Result<Value, Error> {
1139         let request = Self::request_builder("localhost", method, path, param, Some(content_type)).unwrap();
1140
1141         let mut send_request = self.h2.clone().ready().await?;
1142
1143         let (response, stream) = send_request.send_request(request, false).unwrap();
1144
1145         PipeToSendStream::new(bytes::Bytes::from(data), stream).await?;
1146
1147         response
1148             .map_err(Error::from)
1149             .and_then(Self::h2api_response)
1150             .await
1151     }
1152
1153     async fn request(
1154         &self,
1155         request: Request<()>,
1156     ) -> Result<Value, Error> {
1157
1158         self.send_request(request, None)
1159             .and_then(move |response| {
1160                 response
1161                     .map_err(Error::from)
1162                     .and_then(Self::h2api_response)
1163             })
1164             .await
1165     }
1166
1167     fn send_request(
1168         &self,
1169         request: Request<()>,
1170         data: Option<bytes::Bytes>,
1171     ) -> impl Future<Output = Result<h2::client::ResponseFuture, Error>> {
1172
1173         self.h2.clone()
1174             .ready()
1175             .map_err(Error::from)
1176             .and_then(move |mut send_request| async move {
1177                 if let Some(data) = data {
1178                     let (response, stream) = send_request.send_request(request, false).unwrap();
1179                     PipeToSendStream::new(data, stream).await?;
1180                     Ok(response)
1181                 } else {
1182                     let (response, _stream) = send_request.send_request(request, true).unwrap();
1183                     Ok(response)
1184                 }
1185             })
1186     }
1187
1188     async fn h2api_response(
1189         response: Response<h2::RecvStream>,
1190     ) -> Result<Value, Error> {
1191         let status = response.status();
1192
1193         let (_head, mut body) = response.into_parts();
1194
1195         // The `release_capacity` handle allows the caller to manage
1196         // flow control.
1197         //
1198         // Whenever data is received, the caller is responsible for
1199         // releasing capacity back to the server once it has freed
1200         // the data from memory.
1201         let mut release_capacity = body.release_capacity().clone();
1202
1203         let mut data = Vec::new();
1204         while let Some(chunk) = body.try_next().await? {
1205             // Let the server send more data.
1206             let _ = release_capacity.release_capacity(chunk.len());
1207             data.extend(chunk);
1208         }
1209
1210         let text = String::from_utf8(data.to_vec()).unwrap();
1211         if status.is_success() {
1212             if text.len() > 0 {
1213                 let mut value: Value = serde_json::from_str(&text)?;
1214                 if let Some(map) = value.as_object_mut() {
1215                     if let Some(data) = map.remove("data") {
1216                         return Ok(data);
1217                     }
1218                 }
1219                 bail!("got result without data property");
1220             } else {
1221                 Ok(Value::Null)
1222             }
1223         } else {
1224             bail!("HTTP Error {}: {}", status, text);
1225         }
1226     }
1227
1228     // Note: We always encode parameters with the url
1229     pub fn request_builder(
1230         server: &str,
1231         method: &str,
1232         path: &str,
1233         param: Option<Value>,
1234         content_type: Option<&str>,
1235     ) -> Result<Request<()>, Error> {
1236         let path = path.trim_matches('/');
1237
1238         let content_type = content_type.unwrap_or("application/x-www-form-urlencoded");
1239
1240         if let Some(param) = param {
1241             let query = tools::json_object_to_query(param)?;
1242             // We detected problem with hyper around 6000 characters - seo we try to keep on the safe side
1243             if query.len() > 4096 { bail!("h2 query data too large ({} bytes) - please encode data inside body", query.len()); }
1244             let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?;
1245              let request = Request::builder()
1246                 .method(method)
1247                 .uri(url)
1248                 .header("User-Agent", "proxmox-backup-client/1.0")
1249                 .header(hyper::header::CONTENT_TYPE, content_type)
1250                 .body(())?;
1251             return Ok(request);
1252         } else {
1253             let url: Uri = format!("https://{}:8007/{}", server, path).parse()?;
1254             let request = Request::builder()
1255                 .method(method)
1256                 .uri(url)
1257                 .header("User-Agent", "proxmox-backup-client/1.0")
1258                 .header(hyper::header::CONTENT_TYPE, content_type)
1259                 .body(())?;
1260
1261             Ok(request)
1262         }
1263     }
1264 }
1265
1266 pub struct HttpsConnector {
1267     http: HttpConnector,
1268     ssl_connector: SslConnector,
1269 }
1270
1271 impl HttpsConnector {
1272     pub fn with_connector(mut http: HttpConnector, ssl_connector: SslConnector) -> Self {
1273         http.enforce_http(false);
1274
1275         Self {
1276             http,
1277             ssl_connector,
1278         }
1279     }
1280 }
1281
1282 type MaybeTlsStream = EitherStream<
1283     tokio::net::TcpStream,
1284     tokio_openssl::SslStream<tokio::net::TcpStream>,
1285 >;
1286
1287 impl hyper::client::connect::Connect for HttpsConnector {
1288     type Transport = MaybeTlsStream;
1289     type Error = Error;
1290     type Future = Box<dyn Future<Output = Result<(
1291         Self::Transport,
1292         hyper::client::connect::Connected,
1293     ), Error>> + Send + Unpin + 'static>;
1294
1295     fn connect(&self, dst: hyper::client::connect::Destination) -> Self::Future {
1296         let is_https = dst.scheme() == "https";
1297         let host = dst.host().to_string();
1298
1299         let config = self.ssl_connector.configure();
1300         let conn = self.http.connect(dst);
1301
1302         Box::new(Box::pin(async move {
1303             let (conn, connected) = conn.await?;
1304             if is_https {
1305                 let conn = tokio_openssl::connect(config?, &host, conn).await?;
1306                 Ok((MaybeTlsStream::Right(conn), connected))
1307             } else {
1308                 Ok((MaybeTlsStream::Left(conn), connected))
1309             }
1310         }))
1311     }
1312 }