]>
Commit | Line | Data |
---|---|---|
597641fd DM |
1 | use failure::*; |
2 | ||
3 | use http::Uri; | |
4 | use hyper::Body; | |
5 | use hyper::client::Client; | |
ba3a60b2 DM |
6 | use xdg::BaseDirectories; |
7 | use chrono::Utc; | |
553610b4 DM |
8 | use std::collections::HashSet; |
9 | use std::sync::{Arc, Mutex}; | |
597641fd | 10 | |
b57cb264 | 11 | use http::{Request, Response}; |
5a2df000 DM |
12 | use http::header::HeaderValue; |
13 | ||
82ab7230 | 14 | use futures::*; |
1fdb4c6f | 15 | use futures::stream::Stream; |
82ab7230 DM |
16 | use std::sync::atomic::{AtomicUsize, Ordering}; |
17 | use tokio::sync::mpsc; | |
1fdb4c6f | 18 | |
ba3a60b2 | 19 | use serde_json::{json, Value}; |
0dffe3f9 | 20 | use url::percent_encoding::{percent_encode, DEFAULT_ENCODE_SET}; |
1fdb4c6f | 21 | |
5a2df000 | 22 | use crate::tools::{self, BroadcastFuture, tty}; |
e3dbd41b | 23 | use super::pipe_to_stream::*; |
aa1b2e04 DM |
24 | use super::merge_known_chunks::*; |
25 | ||
5a2df000 DM |
26 | |
27 | #[derive(Clone)] | |
28 | struct AuthInfo { | |
29 | username: String, | |
30 | ticket: String, | |
31 | token: String, | |
32 | } | |
56458d97 | 33 | |
151c6ce2 | 34 | /// HTTP(S) API client |
597641fd | 35 | pub struct HttpClient { |
5a2df000 | 36 | client: Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>, |
597641fd | 37 | server: String, |
5a2df000 | 38 | auth: BroadcastFuture<AuthInfo>, |
597641fd DM |
39 | } |
40 | ||
ba3a60b2 DM |
41 | fn store_ticket_info(server: &str, username: &str, ticket: &str, token: &str) -> Result<(), Error> { |
42 | ||
43 | let base = BaseDirectories::with_prefix("proxmox-backup")?; | |
44 | ||
45 | // usually /run/user/<uid>/... | |
46 | let path = base.place_runtime_file("tickets")?; | |
47 | ||
48 | let mode = nix::sys::stat::Mode::from_bits_truncate(0o0600); | |
49 | ||
49cf9f3d | 50 | let mut data = tools::file_get_json(&path, Some(json!({})))?; |
ba3a60b2 DM |
51 | |
52 | let now = Utc::now().timestamp(); | |
53 | ||
54 | data[server][username] = json!({ "timestamp": now, "ticket": ticket, "token": token}); | |
55 | ||
56 | let mut new_data = json!({}); | |
57 | ||
58 | let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60; | |
59 | ||
60 | let empty = serde_json::map::Map::new(); | |
61 | for (server, info) in data.as_object().unwrap_or(&empty) { | |
62 | for (_user, uinfo) in info.as_object().unwrap_or(&empty) { | |
63 | if let Some(timestamp) = uinfo["timestamp"].as_i64() { | |
64 | let age = now - timestamp; | |
65 | if age < ticket_lifetime { | |
66 | new_data[server][username] = uinfo.clone(); | |
67 | } | |
68 | } | |
69 | } | |
70 | } | |
71 | ||
72 | tools::file_set_contents(path, new_data.to_string().as_bytes(), Some(mode))?; | |
73 | ||
74 | Ok(()) | |
75 | } | |
76 | ||
77 | fn load_ticket_info(server: &str, username: &str) -> Option<(String, String)> { | |
78 | let base = match BaseDirectories::with_prefix("proxmox-backup") { | |
79 | Ok(b) => b, | |
80 | _ => return None, | |
81 | }; | |
82 | ||
83 | // usually /run/user/<uid>/... | |
84 | let path = match base.place_runtime_file("tickets") { | |
85 | Ok(p) => p, | |
86 | _ => return None, | |
87 | }; | |
88 | ||
49cf9f3d DM |
89 | let data = match tools::file_get_json(&path, None) { |
90 | Ok(v) => v, | |
91 | _ => return None, | |
92 | }; | |
ba3a60b2 DM |
93 | |
94 | let now = Utc::now().timestamp(); | |
95 | ||
96 | let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60; | |
97 | ||
98 | if let Some(uinfo) = data[server][username].as_object() { | |
99 | if let Some(timestamp) = uinfo["timestamp"].as_i64() { | |
100 | let age = now - timestamp; | |
101 | if age < ticket_lifetime { | |
102 | let ticket = match uinfo["ticket"].as_str() { | |
103 | Some(t) => t, | |
104 | None => return None, | |
105 | }; | |
106 | let token = match uinfo["token"].as_str() { | |
107 | Some(t) => t, | |
108 | None => return None, | |
21ea0158 | 109 | }; |
ba3a60b2 DM |
110 | return Some((ticket.to_owned(), token.to_owned())); |
111 | } | |
112 | } | |
113 | } | |
114 | ||
115 | None | |
116 | } | |
117 | ||
597641fd DM |
118 | impl HttpClient { |
119 | ||
45cdce06 | 120 | pub fn new(server: &str, username: &str) -> Result<Self, Error> { |
5a2df000 | 121 | let client = Self::build_client(); |
5a2df000 | 122 | |
45cdce06 DM |
123 | let password = if let Some((ticket, _token)) = load_ticket_info(server, username) { |
124 | ticket | |
125 | } else { | |
126 | Self::get_password(&username)? | |
127 | }; | |
128 | ||
129 | let login = Self::credentials(client.clone(), server.to_owned(), username.to_owned(), password); | |
130 | ||
131 | Ok(Self { | |
5a2df000 | 132 | client, |
597641fd | 133 | server: String::from(server), |
5a2df000 | 134 | auth: BroadcastFuture::new(login), |
45cdce06 | 135 | }) |
597641fd DM |
136 | } |
137 | ||
5a2df000 | 138 | fn get_password(_username: &str) -> Result<String, Error> { |
56458d97 WB |
139 | use std::env::VarError::*; |
140 | match std::env::var("PBS_PASSWORD") { | |
141 | Ok(p) => return Ok(p), | |
142 | Err(NotUnicode(_)) => bail!("PBS_PASSWORD contains bad characters"), | |
143 | Err(NotPresent) => { | |
144 | // Try another method | |
145 | } | |
146 | } | |
147 | ||
148 | // If we're on a TTY, query the user for a password | |
149 | if tty::stdin_isatty() { | |
150 | return Ok(String::from_utf8(tty::read_password("Password: ")?)?); | |
151 | } | |
152 | ||
153 | bail!("no password input mechanism available"); | |
154 | } | |
155 | ||
5a2df000 | 156 | fn build_client() -> Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>> { |
4a3f6517 WB |
157 | let mut builder = native_tls::TlsConnector::builder(); |
158 | // FIXME: We need a CLI option for this! | |
159 | builder.danger_accept_invalid_certs(true); | |
5a2df000 | 160 | let tlsconnector = builder.build().unwrap(); |
4a3f6517 | 161 | let mut httpc = hyper::client::HttpConnector::new(1); |
9c9ad941 | 162 | //httpc.set_nodelay(true); // not sure if this help? |
4a3f6517 WB |
163 | httpc.enforce_http(false); // we want https... |
164 | let mut https = hyper_tls::HttpsConnector::from((httpc, tlsconnector)); | |
165 | https.https_only(true); // force it! | |
adec8ea2 DM |
166 | Client::builder() |
167 | //.http2_initial_stream_window_size( (1 << 31) - 2) | |
168 | //.http2_initial_connection_window_size( (1 << 31) - 2) | |
169 | .build::<_, Body>(https) | |
a6b75513 DM |
170 | } |
171 | ||
5a2df000 | 172 | pub fn request(&self, mut req: Request<Body>) -> impl Future<Item=Value, Error=Error> { |
597641fd | 173 | |
5a2df000 | 174 | let login = self.auth.listen(); |
597641fd | 175 | |
5a2df000 | 176 | let client = self.client.clone(); |
597641fd | 177 | |
5a2df000 | 178 | login.and_then(move |auth| { |
597641fd | 179 | |
5a2df000 DM |
180 | let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET)); |
181 | req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap()); | |
182 | req.headers_mut().insert("CSRFPreventionToken", HeaderValue::from_str(&auth.token).unwrap()); | |
597641fd | 183 | |
5a2df000 | 184 | let request = Self::api_request(client, req); |
597641fd | 185 | |
5a2df000 DM |
186 | request |
187 | }) | |
1fdb4c6f DM |
188 | } |
189 | ||
9e391bb7 | 190 | pub fn get(&self, path: &str, data: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
a6b75513 | 191 | |
9e391bb7 | 192 | let req = Self::request_builder(&self.server, "GET", path, data).unwrap(); |
5a2df000 | 193 | self.request(req) |
a6b75513 DM |
194 | } |
195 | ||
9e391bb7 | 196 | pub fn delete(&mut self, path: &str, data: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
a6b75513 | 197 | |
9e391bb7 | 198 | let req = Self::request_builder(&self.server, "DELETE", path, data).unwrap(); |
5a2df000 | 199 | self.request(req) |
a6b75513 DM |
200 | } |
201 | ||
5a2df000 | 202 | pub fn post(&mut self, path: &str, data: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
024f11bb | 203 | |
5a2df000 DM |
204 | let req = Self::request_builder(&self.server, "POST", path, data).unwrap(); |
205 | self.request(req) | |
024f11bb DM |
206 | } |
207 | ||
5a2df000 | 208 | pub fn download(&mut self, path: &str, mut output: Box<dyn std::io::Write + Send>) -> impl Future<Item=(), Error=Error> { |
024f11bb | 209 | |
5a2df000 | 210 | let mut req = Self::request_builder(&self.server, "GET", path, None).unwrap(); |
024f11bb | 211 | |
5a2df000 | 212 | let login = self.auth.listen(); |
024f11bb | 213 | |
5a2df000 | 214 | let client = self.client.clone(); |
1fdb4c6f | 215 | |
5a2df000 | 216 | login.and_then(move |auth| { |
81da38c1 | 217 | |
5a2df000 DM |
218 | let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET)); |
219 | req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap()); | |
6f62c924 | 220 | |
5a2df000 DM |
221 | client.request(req) |
222 | .map_err(Error::from) | |
223 | .and_then(|resp| { | |
6f62c924 | 224 | |
5a2df000 | 225 | let _status = resp.status(); // fixme: ?? |
6f62c924 | 226 | |
5a2df000 DM |
227 | resp.into_body() |
228 | .map_err(Error::from) | |
229 | .for_each(move |chunk| { | |
230 | output.write_all(&chunk)?; | |
231 | Ok(()) | |
232 | }) | |
6f62c924 | 233 | |
5a2df000 DM |
234 | }) |
235 | }) | |
6f62c924 DM |
236 | } |
237 | ||
5a2df000 | 238 | pub fn upload(&mut self, content_type: &str, body: Body, path: &str) -> impl Future<Item=Value, Error=Error> { |
81da38c1 DM |
239 | |
240 | let path = path.trim_matches('/'); | |
5a2df000 | 241 | let url: Uri = format!("https://{}:8007/{}", &self.server, path).parse().unwrap(); |
81da38c1 | 242 | |
5a2df000 | 243 | let req = Request::builder() |
81da38c1 DM |
244 | .method("POST") |
245 | .uri(url) | |
246 | .header("User-Agent", "proxmox-backup-client/1.0") | |
5a2df000 DM |
247 | .header("Content-Type", content_type) |
248 | .body(body).unwrap(); | |
81da38c1 | 249 | |
5a2df000 | 250 | self.request(req) |
1fdb4c6f DM |
251 | } |
252 | ||
6ab34afa DM |
253 | pub fn start_backup( |
254 | &self, | |
255 | datastore: &str, | |
256 | backup_type: &str, | |
257 | backup_id: &str, | |
258 | ) -> impl Future<Item=BackupClient, Error=Error> { | |
cf639a47 | 259 | |
6ab34afa DM |
260 | let path = format!("/api2/json/admin/datastore/{}/backup", datastore); |
261 | let param = json!({"backup-type": backup_type, "backup-id": backup_id}); | |
262 | let mut req = Self::request_builder(&self.server, "GET", &path, Some(param)).unwrap(); | |
cf639a47 DM |
263 | |
264 | let login = self.auth.listen(); | |
265 | ||
266 | let client = self.client.clone(); | |
267 | ||
268 | login.and_then(move |auth| { | |
269 | ||
270 | let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET)); | |
271 | req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap()); | |
272 | req.headers_mut().insert("UPGRADE", HeaderValue::from_str("proxmox-backup-protocol-h2").unwrap()); | |
273 | ||
274 | client.request(req) | |
275 | .map_err(Error::from) | |
276 | .and_then(|resp| { | |
277 | ||
278 | let status = resp.status(); | |
279 | if status != http::StatusCode::SWITCHING_PROTOCOLS { | |
9af37c8f DM |
280 | future::Either::A(Self::api_response(resp).and_then(|_| { bail!("unknown error"); })) |
281 | } else { | |
282 | future::Either::B(resp.into_body().on_upgrade().map_err(Error::from)) | |
cf639a47 | 283 | } |
cf639a47 | 284 | }) |
cf639a47 | 285 | .and_then(|upgraded| { |
cf639a47 DM |
286 | h2::client::handshake(upgraded).map_err(Error::from) |
287 | }) | |
288 | .and_then(|(h2, connection)| { | |
289 | let connection = connection | |
290 | .map_err(|_| panic!("HTTP/2.0 connection failed")); | |
291 | ||
292 | // Spawn a new task to drive the connection state | |
293 | hyper::rt::spawn(connection); | |
294 | ||
295 | // Wait until the `SendRequest` handle has available capacity. | |
850ac6d0 | 296 | h2.ready() |
6ab34afa | 297 | .map(BackupClient::new) |
850ac6d0 | 298 | .map_err(Error::from) |
cf639a47 DM |
299 | }) |
300 | }) | |
301 | } | |
302 | ||
5a2df000 DM |
303 | fn credentials( |
304 | client: Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>, | |
45cdce06 DM |
305 | server: String, |
306 | username: String, | |
307 | password: String, | |
5a2df000 | 308 | ) -> Box<Future<Item=AuthInfo, Error=Error> + Send> { |
0ffbccce | 309 | |
45cdce06 | 310 | let server2 = server.clone(); |
0ffbccce | 311 | |
5a2df000 | 312 | let create_request = futures::future::lazy(move || { |
45cdce06 | 313 | let data = json!({ "username": username, "password": password }); |
5a2df000 | 314 | let req = Self::request_builder(&server, "POST", "/api2/json/access/ticket", Some(data)).unwrap(); |
45cdce06 | 315 | Self::api_request(client, req) |
5a2df000 | 316 | }); |
0dffe3f9 | 317 | |
5a2df000 DM |
318 | let login_future = create_request |
319 | .and_then(move |cred| { | |
320 | let auth = AuthInfo { | |
321 | username: cred["data"]["username"].as_str().unwrap().to_owned(), | |
322 | ticket: cred["data"]["ticket"].as_str().unwrap().to_owned(), | |
323 | token: cred["data"]["CSRFPreventionToken"].as_str().unwrap().to_owned(), | |
324 | }; | |
0dffe3f9 | 325 | |
5a2df000 | 326 | let _ = store_ticket_info(&server2, &auth.username, &auth.ticket, &auth.token); |
0dffe3f9 | 327 | |
5a2df000 DM |
328 | Ok(auth) |
329 | }); | |
0dffe3f9 | 330 | |
5a2df000 | 331 | Box::new(login_future) |
ba3a60b2 DM |
332 | } |
333 | ||
d2c48afc DM |
334 | fn api_response(response: Response<Body>) -> impl Future<Item=Value, Error=Error> { |
335 | ||
336 | let status = response.status(); | |
337 | ||
338 | response | |
339 | .into_body() | |
340 | .concat2() | |
341 | .map_err(Error::from) | |
342 | .and_then(move |data| { | |
343 | ||
344 | let text = String::from_utf8(data.to_vec()).unwrap(); | |
345 | if status.is_success() { | |
346 | if text.len() > 0 { | |
347 | let value: Value = serde_json::from_str(&text)?; | |
348 | Ok(value) | |
349 | } else { | |
350 | Ok(Value::Null) | |
351 | } | |
352 | } else { | |
353 | bail!("HTTP Error {}: {}", status, text); | |
354 | } | |
355 | }) | |
356 | } | |
357 | ||
5a2df000 DM |
358 | fn api_request( |
359 | client: Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>, | |
360 | req: Request<Body> | |
361 | ) -> impl Future<Item=Value, Error=Error> { | |
ba3a60b2 | 362 | |
5a2df000 DM |
363 | client.request(req) |
364 | .map_err(Error::from) | |
d2c48afc | 365 | .and_then(Self::api_response) |
0dffe3f9 DM |
366 | } |
367 | ||
5a2df000 | 368 | pub fn request_builder(server: &str, method: &str, path: &str, data: Option<Value>) -> Result<Request<Body>, Error> { |
591f570b | 369 | let path = path.trim_matches('/'); |
5a2df000 DM |
370 | let url: Uri = format!("https://{}:8007/{}", server, path).parse()?; |
371 | ||
372 | if let Some(data) = data { | |
373 | if method == "POST" { | |
374 | let request = Request::builder() | |
375 | .method(method) | |
376 | .uri(url) | |
377 | .header("User-Agent", "proxmox-backup-client/1.0") | |
378 | .header(hyper::header::CONTENT_TYPE, "application/json") | |
379 | .body(Body::from(data.to_string()))?; | |
380 | return Ok(request); | |
381 | } else { | |
9e391bb7 DM |
382 | let query = tools::json_object_to_query(data)?; |
383 | let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?; | |
384 | let request = Request::builder() | |
385 | .method(method) | |
386 | .uri(url) | |
387 | .header("User-Agent", "proxmox-backup-client/1.0") | |
388 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") | |
389 | .body(Body::empty())?; | |
390 | return Ok(request); | |
5a2df000 | 391 | } |
5a2df000 | 392 | } |
0dffe3f9 | 393 | |
1fdb4c6f | 394 | let request = Request::builder() |
5a2df000 | 395 | .method(method) |
1fdb4c6f DM |
396 | .uri(url) |
397 | .header("User-Agent", "proxmox-backup-client/1.0") | |
5a2df000 DM |
398 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") |
399 | .body(Body::empty())?; | |
1fdb4c6f | 400 | |
5a2df000 | 401 | Ok(request) |
597641fd DM |
402 | } |
403 | } | |
b57cb264 | 404 | |
6ab34afa DM |
405 | //#[derive(Clone)] |
406 | pub struct BackupClient { | |
9af37c8f | 407 | h2: H2Client, |
b57cb264 DM |
408 | } |
409 | ||
91320f08 | 410 | |
6ab34afa | 411 | impl BackupClient { |
b57cb264 DM |
412 | |
413 | pub fn new(h2: h2::client::SendRequest<bytes::Bytes>) -> Self { | |
9af37c8f | 414 | Self { h2: H2Client::new(h2) } |
b57cb264 DM |
415 | } |
416 | ||
417 | pub fn get(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
9af37c8f | 418 | self.h2.get(path, param) |
b57cb264 DM |
419 | } |
420 | ||
82ab7230 | 421 | pub fn put(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
9af37c8f | 422 | self.h2.put(path, param) |
82ab7230 DM |
423 | } |
424 | ||
b57cb264 | 425 | pub fn post(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
9af37c8f | 426 | self.h2.post(path, param) |
97f22ce5 DM |
427 | } |
428 | ||
d6f204ed DM |
429 | pub fn finish(&self) -> impl Future<Item=(), Error=Error> { |
430 | self.h2.clone().post("finish", None).map(|_| ()) | |
431 | } | |
432 | ||
433 | pub fn upload_dynamic_stream( | |
434 | &self, | |
435 | archive_name: &str, | |
436 | stream: impl Stream<Item=bytes::BytesMut, Error=Error>, | |
437 | ) -> impl Future<Item=(), Error=Error> { | |
438 | ||
439 | let known_chunks = Arc::new(Mutex::new(HashSet::new())); | |
440 | ||
91320f08 DM |
441 | let mut stream_len = 0u64; |
442 | ||
443 | let stream = stream. | |
444 | map(move |data| { | |
445 | let digest = openssl::sha::sha256(&data); | |
446 | stream_len += data.len() as u64; | |
447 | ChunkInfo { data, digest, offset: stream_len } | |
448 | }); | |
449 | ||
d6f204ed DM |
450 | let h2 = self.h2.clone(); |
451 | let h2_2 = self.h2.clone(); | |
452 | let h2_3 = self.h2.clone(); | |
453 | let h2_4 = self.h2.clone(); | |
454 | ||
455 | let param = json!({ "archive-name": archive_name }); | |
456 | ||
457 | Self::download_chunk_list(h2, "dynamic_index", archive_name, known_chunks.clone()) | |
458 | .and_then(move |_| { | |
459 | h2_2.post("dynamic_index", Some(param)) | |
460 | }) | |
461 | .and_then(move |res| { | |
462 | let wid = res.as_u64().unwrap(); | |
463 | Self::upload_stream(h2_3, wid, stream, known_chunks.clone()) | |
8bea85b4 DM |
464 | .and_then(move |(chunk_count, size, _speed)| { |
465 | let param = json!({ | |
466 | "wid": wid , | |
467 | "chunk-count": chunk_count, | |
468 | "size": size, | |
469 | }); | |
470 | h2_4.post("dynamic_close", Some(param)) | |
d6f204ed DM |
471 | }) |
472 | .map(|_| ()) | |
473 | }) | |
474 | } | |
475 | ||
6ab34afa | 476 | fn response_queue() -> ( |
82ab7230 DM |
477 | mpsc::Sender<h2::client::ResponseFuture>, |
478 | sync::oneshot::Receiver<Result<(), Error>> | |
479 | ) { | |
480 | let (verify_queue_tx, verify_queue_rx) = mpsc::channel(100); | |
481 | let (verify_result_tx, verify_result_rx) = sync::oneshot::channel(); | |
adec8ea2 | 482 | |
82ab7230 DM |
483 | hyper::rt::spawn( |
484 | verify_queue_rx | |
485 | .map_err(Error::from) | |
486 | .for_each(|response: h2::client::ResponseFuture| { | |
487 | response | |
488 | .map_err(Error::from) | |
9af37c8f | 489 | .and_then(H2Client::h2api_response) |
82ab7230 DM |
490 | .and_then(|result| { |
491 | println!("RESPONSE: {:?}", result); | |
492 | Ok(()) | |
493 | }) | |
494 | .map_err(|err| format_err!("pipelined request failed: {}", err)) | |
495 | }) | |
496 | .then(|result| | |
497 | verify_result_tx.send(result) | |
498 | ) | |
499 | .map_err(|_| { /* ignore closed channel */ }) | |
500 | ); | |
adec8ea2 | 501 | |
82ab7230 DM |
502 | (verify_queue_tx, verify_result_rx) |
503 | } | |
504 | ||
6ab34afa | 505 | fn download_chunk_list( |
9af37c8f | 506 | h2: H2Client, |
553610b4 DM |
507 | path: &str, |
508 | archive_name: &str, | |
509 | known_chunks: Arc<Mutex<HashSet<[u8;32]>>>, | |
510 | ) -> impl Future<Item=(), Error=Error> { | |
511 | ||
512 | let param = json!({ "archive-name": archive_name }); | |
9af37c8f | 513 | let request = H2Client::request_builder("localhost", "GET", path, Some(param)).unwrap(); |
553610b4 | 514 | |
9af37c8f | 515 | h2.send_request(request, None) |
553610b4 DM |
516 | .and_then(move |response| { |
517 | response | |
518 | .map_err(Error::from) | |
519 | .and_then(move |resp| { | |
520 | let status = resp.status(); | |
521 | if !status.is_success() { | |
522 | bail!("download chunk list failed with status {}", status); | |
523 | } | |
524 | ||
525 | let (_head, body) = resp.into_parts(); | |
526 | ||
527 | Ok(body) | |
528 | }) | |
529 | .and_then(move |mut body| { | |
530 | ||
531 | let mut release_capacity = body.release_capacity().clone(); | |
532 | ||
533 | crate::backup::DigestListDecoder::new(body.map_err(Error::from)) | |
534 | .for_each(move |chunk| { | |
535 | let _ = release_capacity.release_capacity(chunk.len()); | |
536 | println!("GOT DOWNLOAD {}", tools::digest_to_hex(&chunk)); | |
537 | known_chunks.lock().unwrap().insert(chunk); | |
538 | Ok(()) | |
539 | }) | |
540 | }) | |
541 | }) | |
542 | } | |
543 | ||
6ab34afa | 544 | fn upload_stream( |
9af37c8f | 545 | h2: H2Client, |
82ab7230 | 546 | wid: u64, |
91320f08 | 547 | stream: impl Stream<Item=ChunkInfo, Error=Error>, |
553610b4 | 548 | known_chunks: Arc<Mutex<HashSet<[u8;32]>>>, |
8bea85b4 | 549 | ) -> impl Future<Item=(usize, usize, usize), Error=Error> { |
adec8ea2 | 550 | |
82ab7230 DM |
551 | let repeat = std::sync::Arc::new(AtomicUsize::new(0)); |
552 | let repeat2 = repeat.clone(); | |
adec8ea2 | 553 | |
82ab7230 DM |
554 | let stream_len = std::sync::Arc::new(AtomicUsize::new(0)); |
555 | let stream_len2 = stream_len.clone(); | |
9c9ad941 | 556 | |
6ab34afa | 557 | let (upload_queue, upload_result) = Self::response_queue(); |
9c9ad941 | 558 | |
82ab7230 | 559 | let start_time = std::time::Instant::now(); |
9c9ad941 | 560 | |
82ab7230 | 561 | stream |
aa1b2e04 | 562 | .map(move |chunk_info| { |
82ab7230 | 563 | repeat.fetch_add(1, Ordering::SeqCst); |
91320f08 | 564 | stream_len.fetch_add(chunk_info.data.len(), Ordering::SeqCst); |
aa1b2e04 DM |
565 | chunk_info |
566 | }) | |
567 | .merge_known_chunks(known_chunks.clone()) | |
568 | .for_each(move |merged_chunk_info| { | |
569 | let h2 = h2.clone(); | |
9c9ad941 | 570 | |
82ab7230 | 571 | let upload_queue = upload_queue.clone(); |
adec8ea2 | 572 | |
82ab7230 DM |
573 | let upload_data; |
574 | let request; | |
575 | ||
aa1b2e04 DM |
576 | match merged_chunk_info { |
577 | MergedChunkInfo::New(chunk_info) => { | |
578 | println!("upload new chunk {} ({} bytes)", tools::digest_to_hex(&chunk_info.digest), chunk_info.data.len()); | |
579 | let param = json!({ "wid": wid, "size" : chunk_info.data.len() }); | |
580 | request = H2Client::request_builder("localhost", "POST", "dynamic_chunk", Some(param)).unwrap(); | |
581 | upload_data = Some(chunk_info.data.freeze()); | |
582 | } | |
583 | MergedChunkInfo::Known(chunk_list) => { | |
584 | let mut digest_list = vec![]; | |
585 | for chunk_info in chunk_list { | |
586 | //println!("append existing chunk ({} bytes)", chunk_info.data.len()); | |
587 | digest_list.push(tools::digest_to_hex(&chunk_info.digest)); | |
588 | } | |
589 | println!("append existing chunks ({})", digest_list.len()); | |
590 | let param = json!({ "wid": wid, "digest-list": digest_list }); | |
591 | request = H2Client::request_builder("localhost", "PUT", "dynamic_index", Some(param)).unwrap(); | |
592 | upload_data = None; | |
593 | } | |
82ab7230 DM |
594 | } |
595 | ||
9af37c8f | 596 | h2.send_request(request, upload_data) |
82ab7230 DM |
597 | .and_then(move |response| { |
598 | upload_queue.send(response) | |
599 | .map(|_| ()).map_err(Error::from) | |
9c9ad941 | 600 | }) |
82ab7230 DM |
601 | }) |
602 | .then(move |result| { | |
603 | println!("RESULT {:?}", result); | |
604 | upload_result.map_err(Error::from).and_then(|upload1_result| { | |
605 | Ok(upload1_result.and(result)) | |
606 | }) | |
607 | }) | |
608 | .flatten() | |
609 | .and_then(move |_| { | |
610 | let repeat = repeat2.load(Ordering::SeqCst); | |
611 | let stream_len = stream_len2.load(Ordering::SeqCst); | |
612 | let speed = ((stream_len*1000000)/(1024*1024))/(start_time.elapsed().as_micros() as usize); | |
613 | println!("Uploaded {} chunks in {} seconds ({} MB/s).", repeat, start_time.elapsed().as_secs(), speed); | |
614 | if repeat > 0 { | |
615 | println!("Average chunk size was {} bytes.", stream_len/repeat); | |
616 | println!("Time per request: {} microseconds.", (start_time.elapsed().as_micros())/(repeat as u128)); | |
617 | } | |
8bea85b4 | 618 | Ok((repeat, stream_len, speed)) |
82ab7230 DM |
619 | }) |
620 | } | |
621 | ||
622 | pub fn upload_speedtest(&self) -> impl Future<Item=usize, Error=Error> { | |
623 | ||
624 | let mut data = vec![]; | |
625 | // generate pseudo random byte sequence | |
626 | for i in 0..1024*1024 { | |
627 | for j in 0..4 { | |
628 | let byte = ((i >> (j<<3))&0xff) as u8; | |
629 | data.push(byte); | |
630 | } | |
631 | } | |
632 | ||
633 | let item_len = data.len(); | |
634 | ||
635 | let repeat = std::sync::Arc::new(AtomicUsize::new(0)); | |
636 | let repeat2 = repeat.clone(); | |
637 | ||
6ab34afa | 638 | let (upload_queue, upload_result) = Self::response_queue(); |
82ab7230 DM |
639 | |
640 | let start_time = std::time::Instant::now(); | |
641 | ||
6ab34afa | 642 | let h2 = self.h2.clone(); |
82ab7230 DM |
643 | |
644 | futures::stream::repeat(data) | |
645 | .take_while(move |_| { | |
646 | repeat.fetch_add(1, Ordering::SeqCst); | |
647 | Ok(start_time.elapsed().as_secs() < 5) | |
648 | }) | |
649 | .for_each(move |data| { | |
6ab34afa | 650 | let h2 = h2.clone(); |
82ab7230 DM |
651 | |
652 | let upload_queue = upload_queue.clone(); | |
653 | ||
654 | println!("send test data ({} bytes)", data.len()); | |
9af37c8f DM |
655 | let request = H2Client::request_builder("localhost", "POST", "speedtest", None).unwrap(); |
656 | h2.send_request(request, Some(bytes::Bytes::from(data))) | |
82ab7230 DM |
657 | .and_then(move |response| { |
658 | upload_queue.send(response) | |
659 | .map(|_| ()).map_err(Error::from) | |
adec8ea2 DM |
660 | }) |
661 | }) | |
82ab7230 DM |
662 | .then(move |result| { |
663 | println!("RESULT {:?}", result); | |
664 | upload_result.map_err(Error::from).and_then(|upload1_result| { | |
665 | Ok(upload1_result.and(result)) | |
666 | }) | |
667 | }) | |
668 | .flatten() | |
669 | .and_then(move |_| { | |
670 | let repeat = repeat2.load(Ordering::SeqCst); | |
671 | println!("Uploaded {} chunks in {} seconds.", repeat, start_time.elapsed().as_secs()); | |
672 | let speed = ((item_len*1000000*(repeat as usize))/(1024*1024))/(start_time.elapsed().as_micros() as usize); | |
673 | if repeat > 0 { | |
674 | println!("Time per request: {} microseconds.", (start_time.elapsed().as_micros())/(repeat as u128)); | |
675 | } | |
676 | Ok(speed) | |
677 | }) | |
adec8ea2 | 678 | } |
9af37c8f DM |
679 | } |
680 | ||
681 | #[derive(Clone)] | |
682 | pub struct H2Client { | |
683 | h2: h2::client::SendRequest<bytes::Bytes>, | |
684 | } | |
685 | ||
686 | impl H2Client { | |
687 | ||
688 | pub fn new(h2: h2::client::SendRequest<bytes::Bytes>) -> Self { | |
689 | Self { h2 } | |
690 | } | |
691 | ||
692 | pub fn get(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
693 | let req = Self::request_builder("localhost", "GET", path, param).unwrap(); | |
694 | self.request(req) | |
695 | } | |
696 | ||
697 | pub fn put(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
698 | let req = Self::request_builder("localhost", "PUT", path, param).unwrap(); | |
699 | self.request(req) | |
700 | } | |
701 | ||
702 | pub fn post(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
703 | let req = Self::request_builder("localhost", "POST", path, param).unwrap(); | |
704 | self.request(req) | |
705 | } | |
706 | ||
707 | pub fn upload(&self, path: &str, param: Option<Value>, data: Vec<u8>) -> impl Future<Item=Value, Error=Error> { | |
708 | let request = Self::request_builder("localhost", "POST", path, param).unwrap(); | |
709 | ||
710 | ||
711 | self.h2.clone() | |
712 | .ready() | |
713 | .map_err(Error::from) | |
714 | .and_then(move |mut send_request| { | |
715 | let (response, stream) = send_request.send_request(request, false).unwrap(); | |
716 | PipeToSendStream::new(bytes::Bytes::from(data), stream) | |
717 | .and_then(|_| { | |
718 | response | |
719 | .map_err(Error::from) | |
720 | .and_then(Self::h2api_response) | |
721 | }) | |
722 | }) | |
723 | } | |
adec8ea2 | 724 | |
b57cb264 | 725 | fn request( |
9af37c8f | 726 | &self, |
b57cb264 DM |
727 | request: Request<()>, |
728 | ) -> impl Future<Item=Value, Error=Error> { | |
729 | ||
9af37c8f | 730 | self.send_request(request, None) |
82ab7230 DM |
731 | .and_then(move |response| { |
732 | response | |
733 | .map_err(Error::from) | |
734 | .and_then(Self::h2api_response) | |
735 | }) | |
736 | } | |
737 | ||
738 | fn send_request( | |
9af37c8f | 739 | &self, |
82ab7230 DM |
740 | request: Request<()>, |
741 | data: Option<bytes::Bytes>, | |
742 | ) -> impl Future<Item=h2::client::ResponseFuture, Error=Error> { | |
743 | ||
9af37c8f | 744 | self.h2.clone() |
10130cf4 DM |
745 | .ready() |
746 | .map_err(Error::from) | |
747 | .and_then(move |mut send_request| { | |
82ab7230 DM |
748 | if let Some(data) = data { |
749 | let (response, stream) = send_request.send_request(request, false).unwrap(); | |
750 | future::Either::A(PipeToSendStream::new(bytes::Bytes::from(data), stream) | |
751 | .and_then(move |_| { | |
752 | future::ok(response) | |
753 | })) | |
754 | } else { | |
755 | let (response, _stream) = send_request.send_request(request, true).unwrap(); | |
756 | future::Either::B(future::ok(response)) | |
757 | } | |
b57cb264 DM |
758 | }) |
759 | } | |
760 | ||
761 | fn h2api_response(response: Response<h2::RecvStream>) -> impl Future<Item=Value, Error=Error> { | |
762 | ||
763 | let status = response.status(); | |
764 | ||
765 | let (_head, mut body) = response.into_parts(); | |
766 | ||
767 | // The `release_capacity` handle allows the caller to manage | |
768 | // flow control. | |
769 | // | |
770 | // Whenever data is received, the caller is responsible for | |
771 | // releasing capacity back to the server once it has freed | |
772 | // the data from memory. | |
773 | let mut release_capacity = body.release_capacity().clone(); | |
774 | ||
775 | body | |
776 | .map(move |chunk| { | |
b57cb264 DM |
777 | // Let the server send more data. |
778 | let _ = release_capacity.release_capacity(chunk.len()); | |
779 | chunk | |
780 | }) | |
781 | .concat2() | |
782 | .map_err(Error::from) | |
783 | .and_then(move |data| { | |
b57cb264 DM |
784 | let text = String::from_utf8(data.to_vec()).unwrap(); |
785 | if status.is_success() { | |
786 | if text.len() > 0 { | |
787 | let mut value: Value = serde_json::from_str(&text)?; | |
788 | if let Some(map) = value.as_object_mut() { | |
789 | if let Some(data) = map.remove("data") { | |
790 | return Ok(data); | |
791 | } | |
792 | } | |
793 | bail!("got result without data property"); | |
794 | } else { | |
795 | Ok(Value::Null) | |
796 | } | |
797 | } else { | |
798 | bail!("HTTP Error {}: {}", status, text); | |
799 | } | |
800 | }) | |
801 | } | |
802 | ||
803 | pub fn request_builder(server: &str, method: &str, path: &str, data: Option<Value>) -> Result<Request<()>, Error> { | |
804 | let path = path.trim_matches('/'); | |
805 | let url: Uri = format!("https://{}:8007/{}", server, path).parse()?; | |
806 | ||
807 | if let Some(data) = data { | |
808 | let query = tools::json_object_to_query(data)?; | |
809 | let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?; | |
810 | let request = Request::builder() | |
811 | .method(method) | |
812 | .uri(url) | |
813 | .header("User-Agent", "proxmox-backup-client/1.0") | |
814 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") | |
815 | .body(())?; | |
816 | return Ok(request); | |
817 | } | |
818 | ||
819 | let request = Request::builder() | |
820 | .method(method) | |
821 | .uri(url) | |
822 | .header("User-Agent", "proxmox-backup-client/1.0") | |
823 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") | |
824 | .body(())?; | |
825 | ||
826 | Ok(request) | |
827 | } | |
828 | } |