]>
Commit | Line | Data |
---|---|---|
597641fd DM |
1 | use failure::*; |
2 | ||
3 | use http::Uri; | |
4 | use hyper::Body; | |
5 | use hyper::client::Client; | |
ba3a60b2 DM |
6 | use xdg::BaseDirectories; |
7 | use chrono::Utc; | |
553610b4 DM |
8 | use std::collections::HashSet; |
9 | use std::sync::{Arc, Mutex}; | |
597641fd | 10 | |
b57cb264 | 11 | use http::{Request, Response}; |
5a2df000 DM |
12 | use http::header::HeaderValue; |
13 | ||
82ab7230 | 14 | use futures::*; |
1fdb4c6f | 15 | use futures::stream::Stream; |
82ab7230 DM |
16 | use std::sync::atomic::{AtomicUsize, Ordering}; |
17 | use tokio::sync::mpsc; | |
1fdb4c6f | 18 | |
ba3a60b2 | 19 | use serde_json::{json, Value}; |
0dffe3f9 | 20 | use url::percent_encoding::{percent_encode, DEFAULT_ENCODE_SET}; |
1fdb4c6f | 21 | |
5a2df000 | 22 | use crate::tools::{self, BroadcastFuture, tty}; |
e3dbd41b | 23 | use super::pipe_to_stream::*; |
aa1b2e04 DM |
24 | use super::merge_known_chunks::*; |
25 | ||
5a2df000 DM |
26 | |
27 | #[derive(Clone)] | |
28 | struct AuthInfo { | |
29 | username: String, | |
30 | ticket: String, | |
31 | token: String, | |
32 | } | |
56458d97 | 33 | |
151c6ce2 | 34 | /// HTTP(S) API client |
597641fd | 35 | pub struct HttpClient { |
5a2df000 | 36 | client: Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>, |
597641fd | 37 | server: String, |
5a2df000 | 38 | auth: BroadcastFuture<AuthInfo>, |
597641fd DM |
39 | } |
40 | ||
ba3a60b2 DM |
41 | fn store_ticket_info(server: &str, username: &str, ticket: &str, token: &str) -> Result<(), Error> { |
42 | ||
43 | let base = BaseDirectories::with_prefix("proxmox-backup")?; | |
44 | ||
45 | // usually /run/user/<uid>/... | |
46 | let path = base.place_runtime_file("tickets")?; | |
47 | ||
48 | let mode = nix::sys::stat::Mode::from_bits_truncate(0o0600); | |
49 | ||
49cf9f3d | 50 | let mut data = tools::file_get_json(&path, Some(json!({})))?; |
ba3a60b2 DM |
51 | |
52 | let now = Utc::now().timestamp(); | |
53 | ||
54 | data[server][username] = json!({ "timestamp": now, "ticket": ticket, "token": token}); | |
55 | ||
56 | let mut new_data = json!({}); | |
57 | ||
58 | let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60; | |
59 | ||
60 | let empty = serde_json::map::Map::new(); | |
61 | for (server, info) in data.as_object().unwrap_or(&empty) { | |
62 | for (_user, uinfo) in info.as_object().unwrap_or(&empty) { | |
63 | if let Some(timestamp) = uinfo["timestamp"].as_i64() { | |
64 | let age = now - timestamp; | |
65 | if age < ticket_lifetime { | |
66 | new_data[server][username] = uinfo.clone(); | |
67 | } | |
68 | } | |
69 | } | |
70 | } | |
71 | ||
72 | tools::file_set_contents(path, new_data.to_string().as_bytes(), Some(mode))?; | |
73 | ||
74 | Ok(()) | |
75 | } | |
76 | ||
77 | fn load_ticket_info(server: &str, username: &str) -> Option<(String, String)> { | |
78 | let base = match BaseDirectories::with_prefix("proxmox-backup") { | |
79 | Ok(b) => b, | |
80 | _ => return None, | |
81 | }; | |
82 | ||
83 | // usually /run/user/<uid>/... | |
84 | let path = match base.place_runtime_file("tickets") { | |
85 | Ok(p) => p, | |
86 | _ => return None, | |
87 | }; | |
88 | ||
49cf9f3d DM |
89 | let data = match tools::file_get_json(&path, None) { |
90 | Ok(v) => v, | |
91 | _ => return None, | |
92 | }; | |
ba3a60b2 DM |
93 | |
94 | let now = Utc::now().timestamp(); | |
95 | ||
96 | let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60; | |
97 | ||
98 | if let Some(uinfo) = data[server][username].as_object() { | |
99 | if let Some(timestamp) = uinfo["timestamp"].as_i64() { | |
100 | let age = now - timestamp; | |
101 | if age < ticket_lifetime { | |
102 | let ticket = match uinfo["ticket"].as_str() { | |
103 | Some(t) => t, | |
104 | None => return None, | |
105 | }; | |
106 | let token = match uinfo["token"].as_str() { | |
107 | Some(t) => t, | |
108 | None => return None, | |
21ea0158 | 109 | }; |
ba3a60b2 DM |
110 | return Some((ticket.to_owned(), token.to_owned())); |
111 | } | |
112 | } | |
113 | } | |
114 | ||
115 | None | |
116 | } | |
117 | ||
597641fd DM |
118 | impl HttpClient { |
119 | ||
45cdce06 | 120 | pub fn new(server: &str, username: &str) -> Result<Self, Error> { |
5a2df000 | 121 | let client = Self::build_client(); |
5a2df000 | 122 | |
45cdce06 DM |
123 | let password = if let Some((ticket, _token)) = load_ticket_info(server, username) { |
124 | ticket | |
125 | } else { | |
126 | Self::get_password(&username)? | |
127 | }; | |
128 | ||
129 | let login = Self::credentials(client.clone(), server.to_owned(), username.to_owned(), password); | |
130 | ||
131 | Ok(Self { | |
5a2df000 | 132 | client, |
597641fd | 133 | server: String::from(server), |
5a2df000 | 134 | auth: BroadcastFuture::new(login), |
45cdce06 | 135 | }) |
597641fd DM |
136 | } |
137 | ||
5a2df000 | 138 | fn get_password(_username: &str) -> Result<String, Error> { |
56458d97 WB |
139 | use std::env::VarError::*; |
140 | match std::env::var("PBS_PASSWORD") { | |
141 | Ok(p) => return Ok(p), | |
142 | Err(NotUnicode(_)) => bail!("PBS_PASSWORD contains bad characters"), | |
143 | Err(NotPresent) => { | |
144 | // Try another method | |
145 | } | |
146 | } | |
147 | ||
148 | // If we're on a TTY, query the user for a password | |
149 | if tty::stdin_isatty() { | |
150 | return Ok(String::from_utf8(tty::read_password("Password: ")?)?); | |
151 | } | |
152 | ||
153 | bail!("no password input mechanism available"); | |
154 | } | |
155 | ||
5a2df000 | 156 | fn build_client() -> Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>> { |
4a3f6517 WB |
157 | let mut builder = native_tls::TlsConnector::builder(); |
158 | // FIXME: We need a CLI option for this! | |
159 | builder.danger_accept_invalid_certs(true); | |
5a2df000 | 160 | let tlsconnector = builder.build().unwrap(); |
4a3f6517 | 161 | let mut httpc = hyper::client::HttpConnector::new(1); |
9c9ad941 | 162 | //httpc.set_nodelay(true); // not sure if this help? |
4a3f6517 WB |
163 | httpc.enforce_http(false); // we want https... |
164 | let mut https = hyper_tls::HttpsConnector::from((httpc, tlsconnector)); | |
165 | https.https_only(true); // force it! | |
adec8ea2 DM |
166 | Client::builder() |
167 | //.http2_initial_stream_window_size( (1 << 31) - 2) | |
168 | //.http2_initial_connection_window_size( (1 << 31) - 2) | |
169 | .build::<_, Body>(https) | |
a6b75513 DM |
170 | } |
171 | ||
5a2df000 | 172 | pub fn request(&self, mut req: Request<Body>) -> impl Future<Item=Value, Error=Error> { |
597641fd | 173 | |
5a2df000 | 174 | let login = self.auth.listen(); |
597641fd | 175 | |
5a2df000 | 176 | let client = self.client.clone(); |
597641fd | 177 | |
5a2df000 | 178 | login.and_then(move |auth| { |
597641fd | 179 | |
5a2df000 DM |
180 | let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET)); |
181 | req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap()); | |
182 | req.headers_mut().insert("CSRFPreventionToken", HeaderValue::from_str(&auth.token).unwrap()); | |
597641fd | 183 | |
5a2df000 | 184 | let request = Self::api_request(client, req); |
597641fd | 185 | |
5a2df000 DM |
186 | request |
187 | }) | |
1fdb4c6f DM |
188 | } |
189 | ||
9e391bb7 | 190 | pub fn get(&self, path: &str, data: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
a6b75513 | 191 | |
9e391bb7 | 192 | let req = Self::request_builder(&self.server, "GET", path, data).unwrap(); |
5a2df000 | 193 | self.request(req) |
a6b75513 DM |
194 | } |
195 | ||
9e391bb7 | 196 | pub fn delete(&mut self, path: &str, data: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
a6b75513 | 197 | |
9e391bb7 | 198 | let req = Self::request_builder(&self.server, "DELETE", path, data).unwrap(); |
5a2df000 | 199 | self.request(req) |
a6b75513 DM |
200 | } |
201 | ||
5a2df000 | 202 | pub fn post(&mut self, path: &str, data: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
024f11bb | 203 | |
5a2df000 DM |
204 | let req = Self::request_builder(&self.server, "POST", path, data).unwrap(); |
205 | self.request(req) | |
024f11bb DM |
206 | } |
207 | ||
5a2df000 | 208 | pub fn download(&mut self, path: &str, mut output: Box<dyn std::io::Write + Send>) -> impl Future<Item=(), Error=Error> { |
024f11bb | 209 | |
5a2df000 | 210 | let mut req = Self::request_builder(&self.server, "GET", path, None).unwrap(); |
024f11bb | 211 | |
5a2df000 | 212 | let login = self.auth.listen(); |
024f11bb | 213 | |
5a2df000 | 214 | let client = self.client.clone(); |
1fdb4c6f | 215 | |
5a2df000 | 216 | login.and_then(move |auth| { |
81da38c1 | 217 | |
5a2df000 DM |
218 | let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET)); |
219 | req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap()); | |
6f62c924 | 220 | |
5a2df000 DM |
221 | client.request(req) |
222 | .map_err(Error::from) | |
223 | .and_then(|resp| { | |
6f62c924 | 224 | |
5a2df000 | 225 | let _status = resp.status(); // fixme: ?? |
6f62c924 | 226 | |
5a2df000 DM |
227 | resp.into_body() |
228 | .map_err(Error::from) | |
229 | .for_each(move |chunk| { | |
230 | output.write_all(&chunk)?; | |
231 | Ok(()) | |
232 | }) | |
6f62c924 | 233 | |
5a2df000 DM |
234 | }) |
235 | }) | |
6f62c924 DM |
236 | } |
237 | ||
5a2df000 | 238 | pub fn upload(&mut self, content_type: &str, body: Body, path: &str) -> impl Future<Item=Value, Error=Error> { |
81da38c1 DM |
239 | |
240 | let path = path.trim_matches('/'); | |
5a2df000 | 241 | let url: Uri = format!("https://{}:8007/{}", &self.server, path).parse().unwrap(); |
81da38c1 | 242 | |
5a2df000 | 243 | let req = Request::builder() |
81da38c1 DM |
244 | .method("POST") |
245 | .uri(url) | |
246 | .header("User-Agent", "proxmox-backup-client/1.0") | |
5a2df000 DM |
247 | .header("Content-Type", content_type) |
248 | .body(body).unwrap(); | |
81da38c1 | 249 | |
5a2df000 | 250 | self.request(req) |
1fdb4c6f DM |
251 | } |
252 | ||
6ab34afa DM |
253 | pub fn start_backup( |
254 | &self, | |
255 | datastore: &str, | |
256 | backup_type: &str, | |
257 | backup_id: &str, | |
258 | ) -> impl Future<Item=BackupClient, Error=Error> { | |
cf639a47 | 259 | |
6ab34afa DM |
260 | let path = format!("/api2/json/admin/datastore/{}/backup", datastore); |
261 | let param = json!({"backup-type": backup_type, "backup-id": backup_id}); | |
262 | let mut req = Self::request_builder(&self.server, "GET", &path, Some(param)).unwrap(); | |
cf639a47 DM |
263 | |
264 | let login = self.auth.listen(); | |
265 | ||
266 | let client = self.client.clone(); | |
267 | ||
268 | login.and_then(move |auth| { | |
269 | ||
270 | let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET)); | |
271 | req.headers_mut().insert("Cookie", HeaderValue::from_str(&enc_ticket).unwrap()); | |
272 | req.headers_mut().insert("UPGRADE", HeaderValue::from_str("proxmox-backup-protocol-h2").unwrap()); | |
273 | ||
274 | client.request(req) | |
275 | .map_err(Error::from) | |
276 | .and_then(|resp| { | |
277 | ||
278 | let status = resp.status(); | |
279 | if status != http::StatusCode::SWITCHING_PROTOCOLS { | |
9af37c8f DM |
280 | future::Either::A(Self::api_response(resp).and_then(|_| { bail!("unknown error"); })) |
281 | } else { | |
282 | future::Either::B(resp.into_body().on_upgrade().map_err(Error::from)) | |
cf639a47 | 283 | } |
cf639a47 | 284 | }) |
cf639a47 | 285 | .and_then(|upgraded| { |
cf639a47 DM |
286 | h2::client::handshake(upgraded).map_err(Error::from) |
287 | }) | |
288 | .and_then(|(h2, connection)| { | |
289 | let connection = connection | |
290 | .map_err(|_| panic!("HTTP/2.0 connection failed")); | |
291 | ||
292 | // Spawn a new task to drive the connection state | |
293 | hyper::rt::spawn(connection); | |
294 | ||
295 | // Wait until the `SendRequest` handle has available capacity. | |
850ac6d0 | 296 | h2.ready() |
6ab34afa | 297 | .map(BackupClient::new) |
850ac6d0 | 298 | .map_err(Error::from) |
cf639a47 DM |
299 | }) |
300 | }) | |
301 | } | |
302 | ||
5a2df000 DM |
303 | fn credentials( |
304 | client: Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>, | |
45cdce06 DM |
305 | server: String, |
306 | username: String, | |
307 | password: String, | |
5a2df000 | 308 | ) -> Box<Future<Item=AuthInfo, Error=Error> + Send> { |
0ffbccce | 309 | |
45cdce06 | 310 | let server2 = server.clone(); |
0ffbccce | 311 | |
5a2df000 | 312 | let create_request = futures::future::lazy(move || { |
45cdce06 | 313 | let data = json!({ "username": username, "password": password }); |
5a2df000 | 314 | let req = Self::request_builder(&server, "POST", "/api2/json/access/ticket", Some(data)).unwrap(); |
45cdce06 | 315 | Self::api_request(client, req) |
5a2df000 | 316 | }); |
0dffe3f9 | 317 | |
5a2df000 DM |
318 | let login_future = create_request |
319 | .and_then(move |cred| { | |
320 | let auth = AuthInfo { | |
321 | username: cred["data"]["username"].as_str().unwrap().to_owned(), | |
322 | ticket: cred["data"]["ticket"].as_str().unwrap().to_owned(), | |
323 | token: cred["data"]["CSRFPreventionToken"].as_str().unwrap().to_owned(), | |
324 | }; | |
0dffe3f9 | 325 | |
5a2df000 | 326 | let _ = store_ticket_info(&server2, &auth.username, &auth.ticket, &auth.token); |
0dffe3f9 | 327 | |
5a2df000 DM |
328 | Ok(auth) |
329 | }); | |
0dffe3f9 | 330 | |
5a2df000 | 331 | Box::new(login_future) |
ba3a60b2 DM |
332 | } |
333 | ||
d2c48afc DM |
334 | fn api_response(response: Response<Body>) -> impl Future<Item=Value, Error=Error> { |
335 | ||
336 | let status = response.status(); | |
337 | ||
338 | response | |
339 | .into_body() | |
340 | .concat2() | |
341 | .map_err(Error::from) | |
342 | .and_then(move |data| { | |
343 | ||
344 | let text = String::from_utf8(data.to_vec()).unwrap(); | |
345 | if status.is_success() { | |
346 | if text.len() > 0 { | |
347 | let value: Value = serde_json::from_str(&text)?; | |
348 | Ok(value) | |
349 | } else { | |
350 | Ok(Value::Null) | |
351 | } | |
352 | } else { | |
353 | bail!("HTTP Error {}: {}", status, text); | |
354 | } | |
355 | }) | |
356 | } | |
357 | ||
5a2df000 DM |
358 | fn api_request( |
359 | client: Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>, | |
360 | req: Request<Body> | |
361 | ) -> impl Future<Item=Value, Error=Error> { | |
ba3a60b2 | 362 | |
5a2df000 DM |
363 | client.request(req) |
364 | .map_err(Error::from) | |
d2c48afc | 365 | .and_then(Self::api_response) |
0dffe3f9 DM |
366 | } |
367 | ||
5a2df000 | 368 | pub fn request_builder(server: &str, method: &str, path: &str, data: Option<Value>) -> Result<Request<Body>, Error> { |
591f570b | 369 | let path = path.trim_matches('/'); |
5a2df000 DM |
370 | let url: Uri = format!("https://{}:8007/{}", server, path).parse()?; |
371 | ||
372 | if let Some(data) = data { | |
373 | if method == "POST" { | |
374 | let request = Request::builder() | |
375 | .method(method) | |
376 | .uri(url) | |
377 | .header("User-Agent", "proxmox-backup-client/1.0") | |
378 | .header(hyper::header::CONTENT_TYPE, "application/json") | |
379 | .body(Body::from(data.to_string()))?; | |
380 | return Ok(request); | |
381 | } else { | |
9e391bb7 DM |
382 | let query = tools::json_object_to_query(data)?; |
383 | let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?; | |
384 | let request = Request::builder() | |
385 | .method(method) | |
386 | .uri(url) | |
387 | .header("User-Agent", "proxmox-backup-client/1.0") | |
388 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") | |
389 | .body(Body::empty())?; | |
390 | return Ok(request); | |
5a2df000 | 391 | } |
5a2df000 | 392 | } |
0dffe3f9 | 393 | |
1fdb4c6f | 394 | let request = Request::builder() |
5a2df000 | 395 | .method(method) |
1fdb4c6f DM |
396 | .uri(url) |
397 | .header("User-Agent", "proxmox-backup-client/1.0") | |
5a2df000 DM |
398 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") |
399 | .body(Body::empty())?; | |
1fdb4c6f | 400 | |
5a2df000 | 401 | Ok(request) |
597641fd DM |
402 | } |
403 | } | |
b57cb264 | 404 | |
6ab34afa DM |
405 | //#[derive(Clone)] |
406 | pub struct BackupClient { | |
9af37c8f | 407 | h2: H2Client, |
b57cb264 DM |
408 | } |
409 | ||
91320f08 | 410 | |
6ab34afa | 411 | impl BackupClient { |
b57cb264 DM |
412 | |
413 | pub fn new(h2: h2::client::SendRequest<bytes::Bytes>) -> Self { | |
9af37c8f | 414 | Self { h2: H2Client::new(h2) } |
b57cb264 DM |
415 | } |
416 | ||
417 | pub fn get(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
9af37c8f | 418 | self.h2.get(path, param) |
b57cb264 DM |
419 | } |
420 | ||
82ab7230 | 421 | pub fn put(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
9af37c8f | 422 | self.h2.put(path, param) |
82ab7230 DM |
423 | } |
424 | ||
b57cb264 | 425 | pub fn post(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { |
9af37c8f | 426 | self.h2.post(path, param) |
97f22ce5 DM |
427 | } |
428 | ||
d6f204ed DM |
429 | pub fn finish(&self) -> impl Future<Item=(), Error=Error> { |
430 | self.h2.clone().post("finish", None).map(|_| ()) | |
431 | } | |
432 | ||
433 | pub fn upload_dynamic_stream( | |
434 | &self, | |
435 | archive_name: &str, | |
436 | stream: impl Stream<Item=bytes::BytesMut, Error=Error>, | |
437 | ) -> impl Future<Item=(), Error=Error> { | |
438 | ||
439 | let known_chunks = Arc::new(Mutex::new(HashSet::new())); | |
440 | ||
91320f08 DM |
441 | let mut stream_len = 0u64; |
442 | ||
443 | let stream = stream. | |
444 | map(move |data| { | |
445 | let digest = openssl::sha::sha256(&data); | |
446 | stream_len += data.len() as u64; | |
447 | ChunkInfo { data, digest, offset: stream_len } | |
448 | }); | |
449 | ||
d6f204ed DM |
450 | let h2 = self.h2.clone(); |
451 | let h2_2 = self.h2.clone(); | |
452 | let h2_3 = self.h2.clone(); | |
453 | let h2_4 = self.h2.clone(); | |
454 | ||
455 | let param = json!({ "archive-name": archive_name }); | |
456 | ||
457 | Self::download_chunk_list(h2, "dynamic_index", archive_name, known_chunks.clone()) | |
458 | .and_then(move |_| { | |
459 | h2_2.post("dynamic_index", Some(param)) | |
460 | }) | |
461 | .and_then(move |res| { | |
462 | let wid = res.as_u64().unwrap(); | |
463 | Self::upload_stream(h2_3, wid, stream, known_chunks.clone()) | |
417cb073 | 464 | .and_then(move |(chunk_count, size, _speed)| { |
8bea85b4 DM |
465 | let param = json!({ |
466 | "wid": wid , | |
467 | "chunk-count": chunk_count, | |
468 | "size": size, | |
469 | }); | |
470 | h2_4.post("dynamic_close", Some(param)) | |
d6f204ed DM |
471 | }) |
472 | .map(|_| ()) | |
473 | }) | |
474 | } | |
475 | ||
6ab34afa | 476 | fn response_queue() -> ( |
82ab7230 DM |
477 | mpsc::Sender<h2::client::ResponseFuture>, |
478 | sync::oneshot::Receiver<Result<(), Error>> | |
479 | ) { | |
480 | let (verify_queue_tx, verify_queue_rx) = mpsc::channel(100); | |
481 | let (verify_result_tx, verify_result_rx) = sync::oneshot::channel(); | |
adec8ea2 | 482 | |
82ab7230 DM |
483 | hyper::rt::spawn( |
484 | verify_queue_rx | |
485 | .map_err(Error::from) | |
486 | .for_each(|response: h2::client::ResponseFuture| { | |
487 | response | |
488 | .map_err(Error::from) | |
9af37c8f | 489 | .and_then(H2Client::h2api_response) |
82ab7230 DM |
490 | .and_then(|result| { |
491 | println!("RESPONSE: {:?}", result); | |
492 | Ok(()) | |
493 | }) | |
494 | .map_err(|err| format_err!("pipelined request failed: {}", err)) | |
495 | }) | |
496 | .then(|result| | |
497 | verify_result_tx.send(result) | |
498 | ) | |
499 | .map_err(|_| { /* ignore closed channel */ }) | |
500 | ); | |
adec8ea2 | 501 | |
82ab7230 DM |
502 | (verify_queue_tx, verify_result_rx) |
503 | } | |
504 | ||
6ab34afa | 505 | fn download_chunk_list( |
9af37c8f | 506 | h2: H2Client, |
553610b4 DM |
507 | path: &str, |
508 | archive_name: &str, | |
509 | known_chunks: Arc<Mutex<HashSet<[u8;32]>>>, | |
510 | ) -> impl Future<Item=(), Error=Error> { | |
511 | ||
512 | let param = json!({ "archive-name": archive_name }); | |
9af37c8f | 513 | let request = H2Client::request_builder("localhost", "GET", path, Some(param)).unwrap(); |
553610b4 | 514 | |
9af37c8f | 515 | h2.send_request(request, None) |
553610b4 DM |
516 | .and_then(move |response| { |
517 | response | |
518 | .map_err(Error::from) | |
519 | .and_then(move |resp| { | |
520 | let status = resp.status(); | |
521 | if !status.is_success() { | |
522 | bail!("download chunk list failed with status {}", status); | |
523 | } | |
524 | ||
525 | let (_head, body) = resp.into_parts(); | |
526 | ||
527 | Ok(body) | |
528 | }) | |
529 | .and_then(move |mut body| { | |
530 | ||
531 | let mut release_capacity = body.release_capacity().clone(); | |
532 | ||
533 | crate::backup::DigestListDecoder::new(body.map_err(Error::from)) | |
534 | .for_each(move |chunk| { | |
535 | let _ = release_capacity.release_capacity(chunk.len()); | |
536 | println!("GOT DOWNLOAD {}", tools::digest_to_hex(&chunk)); | |
537 | known_chunks.lock().unwrap().insert(chunk); | |
538 | Ok(()) | |
539 | }) | |
540 | }) | |
541 | }) | |
542 | } | |
543 | ||
6ab34afa | 544 | fn upload_stream( |
9af37c8f | 545 | h2: H2Client, |
82ab7230 | 546 | wid: u64, |
91320f08 | 547 | stream: impl Stream<Item=ChunkInfo, Error=Error>, |
553610b4 | 548 | known_chunks: Arc<Mutex<HashSet<[u8;32]>>>, |
8bea85b4 | 549 | ) -> impl Future<Item=(usize, usize, usize), Error=Error> { |
adec8ea2 | 550 | |
82ab7230 DM |
551 | let repeat = std::sync::Arc::new(AtomicUsize::new(0)); |
552 | let repeat2 = repeat.clone(); | |
adec8ea2 | 553 | |
82ab7230 DM |
554 | let stream_len = std::sync::Arc::new(AtomicUsize::new(0)); |
555 | let stream_len2 = stream_len.clone(); | |
9c9ad941 | 556 | |
6ab34afa | 557 | let (upload_queue, upload_result) = Self::response_queue(); |
9c9ad941 | 558 | |
82ab7230 | 559 | let start_time = std::time::Instant::now(); |
9c9ad941 | 560 | |
82ab7230 | 561 | stream |
aa1b2e04 | 562 | .map(move |chunk_info| { |
82ab7230 | 563 | repeat.fetch_add(1, Ordering::SeqCst); |
91320f08 | 564 | stream_len.fetch_add(chunk_info.data.len(), Ordering::SeqCst); |
aa1b2e04 DM |
565 | chunk_info |
566 | }) | |
567 | .merge_known_chunks(known_chunks.clone()) | |
568 | .for_each(move |merged_chunk_info| { | |
569 | let h2 = h2.clone(); | |
9c9ad941 | 570 | |
82ab7230 | 571 | let upload_queue = upload_queue.clone(); |
adec8ea2 | 572 | |
82ab7230 | 573 | let upload_data; |
eb2bdd1b | 574 | let mut request; |
82ab7230 | 575 | |
aa1b2e04 DM |
576 | match merged_chunk_info { |
577 | MergedChunkInfo::New(chunk_info) => { | |
417cb073 DM |
578 | println!("upload new chunk {} ({} bytes, offset {})", tools::digest_to_hex(&chunk_info.digest), |
579 | chunk_info.data.len(), chunk_info.offset); | |
580 | let param = json!({ "wid": wid, "offset": chunk_info.offset, "size" : chunk_info.data.len() }); | |
aa1b2e04 DM |
581 | request = H2Client::request_builder("localhost", "POST", "dynamic_chunk", Some(param)).unwrap(); |
582 | upload_data = Some(chunk_info.data.freeze()); | |
583 | } | |
584 | MergedChunkInfo::Known(chunk_list) => { | |
585 | let mut digest_list = vec![]; | |
417cb073 DM |
586 | let mut offset_list = vec![]; |
587 | for (offset, digest) in chunk_list { | |
588 | println!("append chunk {} (offset {})", tools::digest_to_hex(&digest), offset); | |
9bb675ec | 589 | digest_list.push(tools::digest_to_hex(&digest)); |
417cb073 | 590 | offset_list.push(offset); |
aa1b2e04 | 591 | } |
417cb073 DM |
592 | println!("append existing chunks list len ({})", digest_list.len()); |
593 | let param = json!({ "wid": wid, "digest-list": digest_list, "offset-list": offset_list }); | |
eb2bdd1b DM |
594 | request = H2Client::request_builder("localhost", "PUT", "dynamic_index", None).unwrap(); |
595 | request.headers_mut().insert(hyper::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); | |
596 | let param_data = bytes::Bytes::from(param.to_string().as_bytes()); | |
417cb073 | 597 | println!("DATALEN {}", param_data.len()); |
eb2bdd1b | 598 | upload_data = Some(param_data); |
aa1b2e04 | 599 | } |
82ab7230 DM |
600 | } |
601 | ||
9af37c8f | 602 | h2.send_request(request, upload_data) |
82ab7230 DM |
603 | .and_then(move |response| { |
604 | upload_queue.send(response) | |
605 | .map(|_| ()).map_err(Error::from) | |
9c9ad941 | 606 | }) |
82ab7230 DM |
607 | }) |
608 | .then(move |result| { | |
609 | println!("RESULT {:?}", result); | |
610 | upload_result.map_err(Error::from).and_then(|upload1_result| { | |
611 | Ok(upload1_result.and(result)) | |
612 | }) | |
613 | }) | |
614 | .flatten() | |
615 | .and_then(move |_| { | |
616 | let repeat = repeat2.load(Ordering::SeqCst); | |
617 | let stream_len = stream_len2.load(Ordering::SeqCst); | |
618 | let speed = ((stream_len*1000000)/(1024*1024))/(start_time.elapsed().as_micros() as usize); | |
619 | println!("Uploaded {} chunks in {} seconds ({} MB/s).", repeat, start_time.elapsed().as_secs(), speed); | |
620 | if repeat > 0 { | |
621 | println!("Average chunk size was {} bytes.", stream_len/repeat); | |
622 | println!("Time per request: {} microseconds.", (start_time.elapsed().as_micros())/(repeat as u128)); | |
623 | } | |
8bea85b4 | 624 | Ok((repeat, stream_len, speed)) |
82ab7230 DM |
625 | }) |
626 | } | |
627 | ||
628 | pub fn upload_speedtest(&self) -> impl Future<Item=usize, Error=Error> { | |
629 | ||
630 | let mut data = vec![]; | |
631 | // generate pseudo random byte sequence | |
632 | for i in 0..1024*1024 { | |
633 | for j in 0..4 { | |
634 | let byte = ((i >> (j<<3))&0xff) as u8; | |
635 | data.push(byte); | |
636 | } | |
637 | } | |
638 | ||
639 | let item_len = data.len(); | |
640 | ||
641 | let repeat = std::sync::Arc::new(AtomicUsize::new(0)); | |
642 | let repeat2 = repeat.clone(); | |
643 | ||
6ab34afa | 644 | let (upload_queue, upload_result) = Self::response_queue(); |
82ab7230 DM |
645 | |
646 | let start_time = std::time::Instant::now(); | |
647 | ||
6ab34afa | 648 | let h2 = self.h2.clone(); |
82ab7230 DM |
649 | |
650 | futures::stream::repeat(data) | |
651 | .take_while(move |_| { | |
652 | repeat.fetch_add(1, Ordering::SeqCst); | |
653 | Ok(start_time.elapsed().as_secs() < 5) | |
654 | }) | |
655 | .for_each(move |data| { | |
6ab34afa | 656 | let h2 = h2.clone(); |
82ab7230 DM |
657 | |
658 | let upload_queue = upload_queue.clone(); | |
659 | ||
660 | println!("send test data ({} bytes)", data.len()); | |
9af37c8f DM |
661 | let request = H2Client::request_builder("localhost", "POST", "speedtest", None).unwrap(); |
662 | h2.send_request(request, Some(bytes::Bytes::from(data))) | |
82ab7230 DM |
663 | .and_then(move |response| { |
664 | upload_queue.send(response) | |
665 | .map(|_| ()).map_err(Error::from) | |
adec8ea2 DM |
666 | }) |
667 | }) | |
82ab7230 DM |
668 | .then(move |result| { |
669 | println!("RESULT {:?}", result); | |
670 | upload_result.map_err(Error::from).and_then(|upload1_result| { | |
671 | Ok(upload1_result.and(result)) | |
672 | }) | |
673 | }) | |
674 | .flatten() | |
675 | .and_then(move |_| { | |
676 | let repeat = repeat2.load(Ordering::SeqCst); | |
677 | println!("Uploaded {} chunks in {} seconds.", repeat, start_time.elapsed().as_secs()); | |
678 | let speed = ((item_len*1000000*(repeat as usize))/(1024*1024))/(start_time.elapsed().as_micros() as usize); | |
679 | if repeat > 0 { | |
680 | println!("Time per request: {} microseconds.", (start_time.elapsed().as_micros())/(repeat as u128)); | |
681 | } | |
682 | Ok(speed) | |
683 | }) | |
adec8ea2 | 684 | } |
9af37c8f DM |
685 | } |
686 | ||
687 | #[derive(Clone)] | |
688 | pub struct H2Client { | |
689 | h2: h2::client::SendRequest<bytes::Bytes>, | |
690 | } | |
691 | ||
692 | impl H2Client { | |
693 | ||
694 | pub fn new(h2: h2::client::SendRequest<bytes::Bytes>) -> Self { | |
695 | Self { h2 } | |
696 | } | |
697 | ||
698 | pub fn get(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
699 | let req = Self::request_builder("localhost", "GET", path, param).unwrap(); | |
700 | self.request(req) | |
701 | } | |
702 | ||
703 | pub fn put(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
704 | let req = Self::request_builder("localhost", "PUT", path, param).unwrap(); | |
705 | self.request(req) | |
706 | } | |
707 | ||
708 | pub fn post(&self, path: &str, param: Option<Value>) -> impl Future<Item=Value, Error=Error> { | |
709 | let req = Self::request_builder("localhost", "POST", path, param).unwrap(); | |
710 | self.request(req) | |
711 | } | |
712 | ||
713 | pub fn upload(&self, path: &str, param: Option<Value>, data: Vec<u8>) -> impl Future<Item=Value, Error=Error> { | |
714 | let request = Self::request_builder("localhost", "POST", path, param).unwrap(); | |
715 | ||
716 | ||
717 | self.h2.clone() | |
718 | .ready() | |
719 | .map_err(Error::from) | |
720 | .and_then(move |mut send_request| { | |
721 | let (response, stream) = send_request.send_request(request, false).unwrap(); | |
722 | PipeToSendStream::new(bytes::Bytes::from(data), stream) | |
723 | .and_then(|_| { | |
724 | response | |
725 | .map_err(Error::from) | |
726 | .and_then(Self::h2api_response) | |
727 | }) | |
728 | }) | |
729 | } | |
adec8ea2 | 730 | |
b57cb264 | 731 | fn request( |
9af37c8f | 732 | &self, |
b57cb264 DM |
733 | request: Request<()>, |
734 | ) -> impl Future<Item=Value, Error=Error> { | |
735 | ||
9af37c8f | 736 | self.send_request(request, None) |
82ab7230 DM |
737 | .and_then(move |response| { |
738 | response | |
739 | .map_err(Error::from) | |
740 | .and_then(Self::h2api_response) | |
741 | }) | |
742 | } | |
743 | ||
744 | fn send_request( | |
9af37c8f | 745 | &self, |
82ab7230 DM |
746 | request: Request<()>, |
747 | data: Option<bytes::Bytes>, | |
748 | ) -> impl Future<Item=h2::client::ResponseFuture, Error=Error> { | |
749 | ||
9af37c8f | 750 | self.h2.clone() |
10130cf4 DM |
751 | .ready() |
752 | .map_err(Error::from) | |
753 | .and_then(move |mut send_request| { | |
82ab7230 DM |
754 | if let Some(data) = data { |
755 | let (response, stream) = send_request.send_request(request, false).unwrap(); | |
756 | future::Either::A(PipeToSendStream::new(bytes::Bytes::from(data), stream) | |
757 | .and_then(move |_| { | |
758 | future::ok(response) | |
759 | })) | |
760 | } else { | |
761 | let (response, _stream) = send_request.send_request(request, true).unwrap(); | |
762 | future::Either::B(future::ok(response)) | |
763 | } | |
b57cb264 DM |
764 | }) |
765 | } | |
766 | ||
767 | fn h2api_response(response: Response<h2::RecvStream>) -> impl Future<Item=Value, Error=Error> { | |
768 | ||
769 | let status = response.status(); | |
770 | ||
771 | let (_head, mut body) = response.into_parts(); | |
772 | ||
773 | // The `release_capacity` handle allows the caller to manage | |
774 | // flow control. | |
775 | // | |
776 | // Whenever data is received, the caller is responsible for | |
777 | // releasing capacity back to the server once it has freed | |
778 | // the data from memory. | |
779 | let mut release_capacity = body.release_capacity().clone(); | |
780 | ||
781 | body | |
782 | .map(move |chunk| { | |
b57cb264 DM |
783 | // Let the server send more data. |
784 | let _ = release_capacity.release_capacity(chunk.len()); | |
785 | chunk | |
786 | }) | |
787 | .concat2() | |
788 | .map_err(Error::from) | |
789 | .and_then(move |data| { | |
b57cb264 DM |
790 | let text = String::from_utf8(data.to_vec()).unwrap(); |
791 | if status.is_success() { | |
792 | if text.len() > 0 { | |
793 | let mut value: Value = serde_json::from_str(&text)?; | |
794 | if let Some(map) = value.as_object_mut() { | |
795 | if let Some(data) = map.remove("data") { | |
796 | return Ok(data); | |
797 | } | |
798 | } | |
799 | bail!("got result without data property"); | |
800 | } else { | |
801 | Ok(Value::Null) | |
802 | } | |
803 | } else { | |
804 | bail!("HTTP Error {}: {}", status, text); | |
805 | } | |
806 | }) | |
807 | } | |
808 | ||
eb2bdd1b | 809 | // Note: We always encode parameters with the url |
b57cb264 DM |
810 | pub fn request_builder(server: &str, method: &str, path: &str, data: Option<Value>) -> Result<Request<()>, Error> { |
811 | let path = path.trim_matches('/'); | |
b57cb264 DM |
812 | |
813 | if let Some(data) = data { | |
814 | let query = tools::json_object_to_query(data)?; | |
eb2bdd1b DM |
815 | // We detected problem with hyper around 6000 characters - seo we try to keep on the safe side |
816 | if query.len() > 4096 { bail!("h2 query data too large ({} bytes) - please encode data inside body", query.len()); } | |
b57cb264 | 817 | let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?; |
eb2bdd1b | 818 | let request = Request::builder() |
b57cb264 DM |
819 | .method(method) |
820 | .uri(url) | |
821 | .header("User-Agent", "proxmox-backup-client/1.0") | |
822 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") | |
823 | .body(())?; | |
824 | return Ok(request); | |
eb2bdd1b DM |
825 | } else { |
826 | let url: Uri = format!("https://{}:8007/{}", server, path).parse()?; | |
827 | let request = Request::builder() | |
828 | .method(method) | |
829 | .uri(url) | |
830 | .header("User-Agent", "proxmox-backup-client/1.0") | |
831 | .header(hyper::header::CONTENT_TYPE, "application/x-www-form-urlencoded") | |
832 | .body(())?; | |
b57cb264 | 833 | |
eb2bdd1b DM |
834 | Ok(request) |
835 | } | |
b57cb264 DM |
836 | } |
837 | } |