]>
Commit | Line | Data |
---|---|---|
152764ec | 1 | use failure::*; |
92ac375a | 2 | use futures::*; |
152764ec | 3 | use hyper::header::{HeaderValue, UPGRADE}; |
152764ec | 4 | use hyper::http::request::Parts; |
cad540e9 | 5 | use hyper::{Body, Response, StatusCode}; |
f9578f3c | 6 | use serde_json::{json, Value}; |
152764ec | 7 | |
552c2259 | 8 | use proxmox::{sortable, identity}; |
3d229a4a | 9 | use proxmox::api::list_subdirs_api_method; |
cad540e9 WB |
10 | use proxmox::api::{ApiFuture, ApiHandler, ApiMethod, Router, RpcEnvironment}; |
11 | use proxmox::api::router::SubdirMap; | |
12 | use proxmox::api::schema::*; | |
552c2259 | 13 | |
92ac375a | 14 | use crate::tools; |
d3611366 | 15 | use crate::tools::wrapped_reader_stream::*; |
42a87f7b | 16 | use crate::server::{WorkerTask, H2Service}; |
21ee7912 | 17 | use crate::backup::*; |
6762db70 | 18 | use crate::api2::types::*; |
152764ec | 19 | |
d95ced64 DM |
20 | mod environment; |
21 | use environment::*; | |
22 | ||
21ee7912 DM |
23 | mod upload_chunk; |
24 | use upload_chunk::*; | |
25 | ||
255f378a DM |
26 | pub const ROUTER: Router = Router::new() |
27 | .upgrade(&API_METHOD_UPGRADE_BACKUP); | |
28 | ||
552c2259 | 29 | #[sortable] |
255f378a | 30 | pub const API_METHOD_UPGRADE_BACKUP: ApiMethod = ApiMethod::new( |
329d40b5 | 31 | &ApiHandler::AsyncHttp(&upgrade_to_backup_protocol), |
255f378a DM |
32 | &ObjectSchema::new( |
33 | concat!("Upgraded to backup protocol ('", PROXMOX_BACKUP_PROTOCOL_ID_V1!(), "')."), | |
552c2259 | 34 | &sorted!([ |
66c49c21 | 35 | ("store", false, &DATASTORE_SCHEMA), |
255f378a DM |
36 | ("backup-type", false, &BACKUP_TYPE_SCHEMA), |
37 | ("backup-id", false, &BACKUP_ID_SCHEMA), | |
38 | ("backup-time", false, &BACKUP_TIME_SCHEMA), | |
39 | ("debug", true, &BooleanSchema::new("Enable verbose debug logging.").schema()), | |
552c2259 | 40 | ]), |
152764ec | 41 | ) |
255f378a | 42 | ); |
152764ec | 43 | |
0aadd40b | 44 | fn upgrade_to_backup_protocol( |
152764ec DM |
45 | parts: Parts, |
46 | req_body: Body, | |
0aadd40b | 47 | param: Value, |
255f378a | 48 | _info: &ApiMethod, |
dd5495d6 | 49 | rpcenv: Box<dyn RpcEnvironment>, |
ad51d02a | 50 | ) -> ApiFuture { |
0aadd40b | 51 | |
ad51d02a | 52 | async move { |
a42d1f55 DM |
53 | let debug = param["debug"].as_bool().unwrap_or(false); |
54 | ||
bb105f9d DM |
55 | let store = tools::required_string_param(¶m, "store")?.to_owned(); |
56 | let datastore = DataStore::lookup_datastore(&store)?; | |
21ee7912 | 57 | |
0aadd40b DM |
58 | let backup_type = tools::required_string_param(¶m, "backup-type")?; |
59 | let backup_id = tools::required_string_param(¶m, "backup-id")?; | |
ca5d0b61 | 60 | let backup_time = tools::required_integer_param(¶m, "backup-time")?; |
152764ec DM |
61 | |
62 | let protocols = parts | |
63 | .headers | |
64 | .get("UPGRADE") | |
65 | .ok_or_else(|| format_err!("missing Upgrade header"))? | |
66 | .to_str()?; | |
67 | ||
986bef16 | 68 | if protocols != PROXMOX_BACKUP_PROTOCOL_ID_V1!() { |
152764ec DM |
69 | bail!("invalid protocol name"); |
70 | } | |
71 | ||
96e95fc1 DM |
72 | if parts.version >= http::version::Version::HTTP_2 { |
73 | bail!("unexpected http version '{:?}' (expected version < 2)", parts.version); | |
74 | } | |
75 | ||
0aadd40b | 76 | let worker_id = format!("{}_{}_{}", store, backup_type, backup_id); |
d9bd06ea | 77 | |
58c8d7d9 DM |
78 | let username = rpcenv.get_user().unwrap(); |
79 | let env_type = rpcenv.env_type(); | |
92ac375a | 80 | |
51a4f63f | 81 | let backup_group = BackupGroup::new(backup_type, backup_id); |
fbb798f6 | 82 | let last_backup = BackupInfo::last_backup(&datastore.base_path(), &backup_group).unwrap_or(None); |
ca5d0b61 DM |
83 | let backup_dir = BackupDir::new_with_group(backup_group, backup_time); |
84 | ||
85 | if let Some(last) = &last_backup { | |
86 | if backup_dir.backup_time() <= last.backup_dir.backup_time() { | |
87 | bail!("backup timestamp is older than last backup."); | |
88 | } | |
9ce42759 DM |
89 | // fixme: abort if last backup is still running - howto test? |
90 | // Idea: write upid into a file inside snapshot dir. then test if | |
91 | // it is still running here. | |
ca5d0b61 | 92 | } |
f9578f3c | 93 | |
bb105f9d | 94 | let (path, is_new) = datastore.create_backup_dir(&backup_dir)?; |
f9578f3c DM |
95 | if !is_new { bail!("backup directorty already exists."); } |
96 | ||
0aadd40b | 97 | WorkerTask::spawn("backup", Some(worker_id), &username.clone(), true, move |worker| { |
bb105f9d | 98 | let mut env = BackupEnvironment::new( |
6b95c7df | 99 | env_type, username.clone(), worker.clone(), datastore, backup_dir); |
b02a52e3 | 100 | |
a42d1f55 | 101 | env.debug = debug; |
bb105f9d | 102 | env.last_backup = last_backup; |
b02a52e3 | 103 | |
bb105f9d DM |
104 | env.log(format!("starting new backup on datastore '{}': {:?}", store, path)); |
105 | ||
255f378a | 106 | let service = H2Service::new(env.clone(), worker.clone(), &BACKUP_API_ROUTER, debug); |
72375ce6 | 107 | |
a66ab8ae DM |
108 | let abort_future = worker.abort_future(); |
109 | ||
372724af | 110 | let env2 = env.clone(); |
a42d1f55 | 111 | let env3 = env.clone(); |
bb105f9d | 112 | |
59b2baa0 | 113 | let req_fut = req_body |
152764ec | 114 | .on_upgrade() |
92ac375a | 115 | .map_err(Error::from) |
152764ec | 116 | .and_then(move |conn| { |
a42d1f55 | 117 | env3.debug("protocol upgrade done"); |
92ac375a DM |
118 | |
119 | let mut http = hyper::server::conn::Http::new(); | |
120 | http.http2_only(true); | |
adec8ea2 | 121 | // increase window size: todo - find optiomal size |
771953f9 DM |
122 | let window_size = 32*1024*1024; // max = (1 << 31) - 2 |
123 | http.http2_initial_stream_window_size(window_size); | |
124 | http.http2_initial_connection_window_size(window_size); | |
92ac375a | 125 | |
d9bd06ea DM |
126 | http.serve_connection(conn, service) |
127 | .map_err(Error::from) | |
59b2baa0 WB |
128 | }); |
129 | let abort_future = abort_future | |
130 | .map(|_| Err(format_err!("task aborted"))); | |
131 | ||
132 | use futures::future::Either; | |
133 | future::select(req_fut, abort_future) | |
134 | .map(|res| match res { | |
135 | Either::Left((Ok(res), _)) => Ok(res), | |
136 | Either::Left((Err(err), _)) => Err(err), | |
137 | Either::Right((Ok(res), _)) => Ok(res), | |
138 | Either::Right((Err(err), _)) => Err(err), | |
139 | }) | |
140 | .and_then(move |_result| async move { | |
372724af DM |
141 | env.ensure_finished()?; |
142 | env.log("backup finished sucessfully"); | |
bb105f9d DM |
143 | Ok(()) |
144 | }) | |
59b2baa0 | 145 | .then(move |result| async move { |
372724af | 146 | if let Err(err) = result { |
a9584932 | 147 | match env2.ensure_finished() { |
e3d525fe | 148 | Ok(()) => {}, // ignore error after finish |
a9584932 DM |
149 | _ => { |
150 | env2.log(format!("backup failed: {}", err)); | |
151 | env2.log("removing failed backup"); | |
152 | env2.remove_backup()?; | |
153 | return Err(err); | |
154 | } | |
155 | } | |
372724af DM |
156 | } |
157 | Ok(()) | |
bb105f9d | 158 | }) |
090ac9f7 DM |
159 | })?; |
160 | ||
161 | let response = Response::builder() | |
162 | .status(StatusCode::SWITCHING_PROTOCOLS) | |
986bef16 | 163 | .header(UPGRADE, HeaderValue::from_static(PROXMOX_BACKUP_PROTOCOL_ID_V1!())) |
090ac9f7 DM |
164 | .body(Body::empty())?; |
165 | ||
ad51d02a DM |
166 | Ok(response) |
167 | }.boxed() | |
152764ec | 168 | } |
92ac375a | 169 | |
255f378a DM |
170 | pub const BACKUP_API_SUBDIRS: SubdirMap = &[ |
171 | ( | |
172 | "blob", &Router::new() | |
173 | .upload(&API_METHOD_UPLOAD_BLOB) | |
174 | ), | |
175 | ( | |
176 | "dynamic_chunk", &Router::new() | |
177 | .upload(&API_METHOD_UPLOAD_DYNAMIC_CHUNK) | |
178 | ), | |
179 | ( | |
180 | "dynamic_close", &Router::new() | |
181 | .post(&API_METHOD_CLOSE_DYNAMIC_INDEX) | |
182 | ), | |
183 | ( | |
184 | "dynamic_index", &Router::new() | |
185 | .download(&API_METHOD_DYNAMIC_CHUNK_INDEX) | |
186 | .post(&API_METHOD_CREATE_DYNAMIC_INDEX) | |
187 | .put(&API_METHOD_DYNAMIC_APPEND) | |
188 | ), | |
189 | ( | |
190 | "finish", &Router::new() | |
191 | .post( | |
192 | &ApiMethod::new( | |
193 | &ApiHandler::Sync(&finish_backup), | |
194 | &ObjectSchema::new("Mark backup as finished.", &[]) | |
372724af | 195 | ) |
255f378a DM |
196 | ) |
197 | ), | |
198 | ( | |
199 | "fixed_chunk", &Router::new() | |
200 | .upload(&API_METHOD_UPLOAD_FIXED_CHUNK) | |
201 | ), | |
202 | ( | |
203 | "fixed_close", &Router::new() | |
204 | .post(&API_METHOD_CLOSE_FIXED_INDEX) | |
205 | ), | |
206 | ( | |
207 | "fixed_index", &Router::new() | |
208 | .download(&API_METHOD_FIXED_CHUNK_INDEX) | |
209 | .post(&API_METHOD_CREATE_FIXED_INDEX) | |
210 | .put(&API_METHOD_FIXED_APPEND) | |
211 | ), | |
212 | ( | |
213 | "speedtest", &Router::new() | |
214 | .upload(&API_METHOD_UPLOAD_SPEEDTEST) | |
215 | ), | |
216 | ]; | |
217 | ||
218 | pub const BACKUP_API_ROUTER: Router = Router::new() | |
219 | .get(&list_subdirs_api_method!(BACKUP_API_SUBDIRS)) | |
220 | .subdirs(BACKUP_API_SUBDIRS); | |
221 | ||
3d229a4a DM |
222 | #[sortable] |
223 | pub const API_METHOD_CREATE_DYNAMIC_INDEX: ApiMethod = ApiMethod::new( | |
224 | &ApiHandler::Sync(&create_dynamic_index), | |
225 | &ObjectSchema::new( | |
226 | "Create dynamic chunk index file.", | |
227 | &sorted!([ | |
228 | ("archive-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA), | |
229 | ]), | |
230 | ) | |
231 | ); | |
232 | ||
f9578f3c DM |
233 | fn create_dynamic_index( |
234 | param: Value, | |
3d229a4a | 235 | _info: &ApiMethod, |
dd5495d6 | 236 | rpcenv: &mut dyn RpcEnvironment, |
f9578f3c DM |
237 | ) -> Result<Value, Error> { |
238 | ||
239 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
f9578f3c | 240 | |
8bea85b4 | 241 | let name = tools::required_string_param(¶m, "archive-name")?.to_owned(); |
f9578f3c | 242 | |
4af0ee05 | 243 | let archive_name = name.clone(); |
0997967d | 244 | if !archive_name.ends_with(".didx") { |
a42fa400 | 245 | bail!("wrong archive extension: '{}'", archive_name); |
f9578f3c DM |
246 | } |
247 | ||
6b95c7df | 248 | let mut path = env.backup_dir.relative_path(); |
f9578f3c DM |
249 | path.push(archive_name); |
250 | ||
976595e1 | 251 | let index = env.datastore.create_dynamic_writer(&path)?; |
8bea85b4 | 252 | let wid = env.register_dynamic_writer(index, name)?; |
f9578f3c | 253 | |
bb105f9d | 254 | env.log(format!("created new dynamic index {} ({:?})", wid, path)); |
f9578f3c | 255 | |
bb105f9d | 256 | Ok(json!(wid)) |
f9578f3c DM |
257 | } |
258 | ||
552c2259 | 259 | #[sortable] |
255f378a DM |
260 | pub const API_METHOD_CREATE_FIXED_INDEX: ApiMethod = ApiMethod::new( |
261 | &ApiHandler::Sync(&create_fixed_index), | |
262 | &ObjectSchema::new( | |
263 | "Create fixed chunk index file.", | |
552c2259 | 264 | &sorted!([ |
255f378a DM |
265 | ("archive-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA), |
266 | ("size", false, &IntegerSchema::new("File size.") | |
267 | .minimum(1) | |
268 | .schema() | |
269 | ), | |
552c2259 | 270 | ]), |
a42fa400 | 271 | ) |
255f378a | 272 | ); |
a42fa400 DM |
273 | |
274 | fn create_fixed_index( | |
275 | param: Value, | |
276 | _info: &ApiMethod, | |
dd5495d6 | 277 | rpcenv: &mut dyn RpcEnvironment, |
a42fa400 DM |
278 | ) -> Result<Value, Error> { |
279 | ||
280 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
281 | ||
282 | println!("PARAM: {:?}", param); | |
283 | ||
284 | let name = tools::required_string_param(¶m, "archive-name")?.to_owned(); | |
285 | let size = tools::required_integer_param(¶m, "size")? as usize; | |
286 | ||
4af0ee05 | 287 | let archive_name = name.clone(); |
0997967d | 288 | if !archive_name.ends_with(".fidx") { |
a42fa400 | 289 | bail!("wrong archive extension: '{}'", archive_name); |
a42fa400 DM |
290 | } |
291 | ||
292 | let mut path = env.backup_dir.relative_path(); | |
293 | path.push(archive_name); | |
294 | ||
295 | let chunk_size = 4096*1024; // todo: ?? | |
296 | ||
297 | let index = env.datastore.create_fixed_writer(&path, size, chunk_size)?; | |
298 | let wid = env.register_fixed_writer(index, name, size, chunk_size as u32)?; | |
299 | ||
300 | env.log(format!("created new fixed index {} ({:?})", wid, path)); | |
301 | ||
302 | Ok(json!(wid)) | |
303 | } | |
304 | ||
552c2259 | 305 | #[sortable] |
255f378a DM |
306 | pub const API_METHOD_DYNAMIC_APPEND: ApiMethod = ApiMethod::new( |
307 | &ApiHandler::Sync(&dynamic_append), | |
308 | &ObjectSchema::new( | |
309 | "Append chunk to dynamic index writer.", | |
552c2259 | 310 | &sorted!([ |
255f378a DM |
311 | ( |
312 | "wid", | |
313 | false, | |
314 | &IntegerSchema::new("Dynamic writer ID.") | |
315 | .minimum(1) | |
316 | .maximum(256) | |
317 | .schema() | |
318 | ), | |
319 | ( | |
320 | "digest-list", | |
321 | false, | |
322 | &ArraySchema::new("Chunk digest list.", &CHUNK_DIGEST_SCHEMA).schema() | |
323 | ), | |
324 | ( | |
325 | "offset-list", | |
326 | false, | |
327 | &ArraySchema::new( | |
328 | "Chunk offset list.", | |
329 | &IntegerSchema::new("Corresponding chunk offsets.") | |
330 | .minimum(0) | |
331 | .schema() | |
332 | ).schema() | |
333 | ), | |
552c2259 | 334 | ]), |
82ab7230 | 335 | ) |
255f378a | 336 | ); |
82ab7230 DM |
337 | |
338 | fn dynamic_append ( | |
339 | param: Value, | |
340 | _info: &ApiMethod, | |
dd5495d6 | 341 | rpcenv: &mut dyn RpcEnvironment, |
82ab7230 DM |
342 | ) -> Result<Value, Error> { |
343 | ||
344 | let wid = tools::required_integer_param(¶m, "wid")? as usize; | |
aa1b2e04 | 345 | let digest_list = tools::required_array_param(¶m, "digest-list")?; |
417cb073 | 346 | let offset_list = tools::required_array_param(¶m, "offset-list")?; |
aa1b2e04 | 347 | |
417cb073 DM |
348 | if offset_list.len() != digest_list.len() { |
349 | bail!("offset list has wrong length ({} != {})", offset_list.len(), digest_list.len()); | |
350 | } | |
351 | ||
82ab7230 DM |
352 | let env: &BackupEnvironment = rpcenv.as_ref(); |
353 | ||
39e60bd6 DM |
354 | env.debug(format!("dynamic_append {} chunks", digest_list.len())); |
355 | ||
417cb073 | 356 | for (i, item) in digest_list.iter().enumerate() { |
aa1b2e04 | 357 | let digest_str = item.as_str().unwrap(); |
bffd40d6 | 358 | let digest = proxmox::tools::hex_to_digest(digest_str)?; |
417cb073 | 359 | let offset = offset_list[i].as_u64().unwrap(); |
aa1b2e04 | 360 | let size = env.lookup_chunk(&digest).ok_or_else(|| format_err!("no such chunk {}", digest_str))?; |
39e60bd6 | 361 | |
417cb073 | 362 | env.dynamic_writer_append_chunk(wid, offset, size, &digest)?; |
82ab7230 | 363 | |
39e60bd6 | 364 | env.debug(format!("sucessfully added chunk {} to dynamic index {} (offset {}, size {})", digest_str, wid, offset, size)); |
aa1b2e04 | 365 | } |
82ab7230 DM |
366 | |
367 | Ok(Value::Null) | |
368 | } | |
369 | ||
552c2259 | 370 | #[sortable] |
255f378a DM |
371 | pub const API_METHOD_FIXED_APPEND: ApiMethod = ApiMethod::new( |
372 | &ApiHandler::Sync(&fixed_append), | |
373 | &ObjectSchema::new( | |
374 | "Append chunk to fixed index writer.", | |
552c2259 | 375 | &sorted!([ |
255f378a DM |
376 | ( |
377 | "wid", | |
378 | false, | |
379 | &IntegerSchema::new("Fixed writer ID.") | |
380 | .minimum(1) | |
381 | .maximum(256) | |
382 | .schema() | |
383 | ), | |
384 | ( | |
385 | "digest-list", | |
386 | false, | |
387 | &ArraySchema::new("Chunk digest list.", &CHUNK_DIGEST_SCHEMA).schema() | |
388 | ), | |
389 | ( | |
390 | "offset-list", | |
391 | false, | |
392 | &ArraySchema::new( | |
393 | "Chunk offset list.", | |
394 | &IntegerSchema::new("Corresponding chunk offsets.") | |
395 | .minimum(0) | |
396 | .schema() | |
397 | ).schema() | |
a42fa400 | 398 | ) |
552c2259 | 399 | ]), |
a42fa400 | 400 | ) |
255f378a | 401 | ); |
a42fa400 DM |
402 | |
403 | fn fixed_append ( | |
404 | param: Value, | |
405 | _info: &ApiMethod, | |
dd5495d6 | 406 | rpcenv: &mut dyn RpcEnvironment, |
a42fa400 DM |
407 | ) -> Result<Value, Error> { |
408 | ||
409 | let wid = tools::required_integer_param(¶m, "wid")? as usize; | |
410 | let digest_list = tools::required_array_param(¶m, "digest-list")?; | |
411 | let offset_list = tools::required_array_param(¶m, "offset-list")?; | |
412 | ||
a42fa400 DM |
413 | if offset_list.len() != digest_list.len() { |
414 | bail!("offset list has wrong length ({} != {})", offset_list.len(), digest_list.len()); | |
415 | } | |
416 | ||
417 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
418 | ||
39e60bd6 DM |
419 | env.debug(format!("fixed_append {} chunks", digest_list.len())); |
420 | ||
a42fa400 DM |
421 | for (i, item) in digest_list.iter().enumerate() { |
422 | let digest_str = item.as_str().unwrap(); | |
bffd40d6 | 423 | let digest = proxmox::tools::hex_to_digest(digest_str)?; |
a42fa400 DM |
424 | let offset = offset_list[i].as_u64().unwrap(); |
425 | let size = env.lookup_chunk(&digest).ok_or_else(|| format_err!("no such chunk {}", digest_str))?; | |
39e60bd6 | 426 | |
a42fa400 DM |
427 | env.fixed_writer_append_chunk(wid, offset, size, &digest)?; |
428 | ||
39e60bd6 | 429 | env.debug(format!("sucessfully added chunk {} to fixed index {} (offset {}, size {})", digest_str, wid, offset, size)); |
a42fa400 DM |
430 | } |
431 | ||
432 | Ok(Value::Null) | |
433 | } | |
434 | ||
552c2259 | 435 | #[sortable] |
255f378a DM |
436 | pub const API_METHOD_CLOSE_DYNAMIC_INDEX: ApiMethod = ApiMethod::new( |
437 | &ApiHandler::Sync(&close_dynamic_index), | |
438 | &ObjectSchema::new( | |
439 | "Close dynamic index writer.", | |
552c2259 | 440 | &sorted!([ |
255f378a DM |
441 | ( |
442 | "wid", | |
443 | false, | |
444 | &IntegerSchema::new("Dynamic writer ID.") | |
445 | .minimum(1) | |
446 | .maximum(256) | |
447 | .schema() | |
448 | ), | |
449 | ( | |
450 | "chunk-count", | |
451 | false, | |
452 | &IntegerSchema::new("Chunk count. This is used to verify that the server got all chunks.") | |
453 | .minimum(1) | |
454 | .schema() | |
455 | ), | |
456 | ( | |
457 | "size", | |
458 | false, | |
459 | &IntegerSchema::new("File size. This is used to verify that the server got all data.") | |
460 | .minimum(1) | |
461 | .schema() | |
462 | ), | |
463 | ("csum", false, &StringSchema::new("Digest list checksum.").schema()), | |
552c2259 | 464 | ]), |
a2077252 | 465 | ) |
255f378a | 466 | ); |
a2077252 DM |
467 | |
468 | fn close_dynamic_index ( | |
469 | param: Value, | |
470 | _info: &ApiMethod, | |
dd5495d6 | 471 | rpcenv: &mut dyn RpcEnvironment, |
a2077252 DM |
472 | ) -> Result<Value, Error> { |
473 | ||
474 | let wid = tools::required_integer_param(¶m, "wid")? as usize; | |
8bea85b4 DM |
475 | let chunk_count = tools::required_integer_param(¶m, "chunk-count")? as u64; |
476 | let size = tools::required_integer_param(¶m, "size")? as u64; | |
fb6026b6 DM |
477 | let csum_str = tools::required_string_param(¶m, "csum")?; |
478 | let csum = proxmox::tools::hex_to_digest(csum_str)?; | |
a2077252 DM |
479 | |
480 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
481 | ||
fb6026b6 | 482 | env.dynamic_writer_close(wid, chunk_count, size, csum)?; |
a2077252 | 483 | |
bb105f9d DM |
484 | env.log(format!("sucessfully closed dynamic index {}", wid)); |
485 | ||
a2077252 DM |
486 | Ok(Value::Null) |
487 | } | |
488 | ||
552c2259 | 489 | #[sortable] |
255f378a DM |
490 | pub const API_METHOD_CLOSE_FIXED_INDEX: ApiMethod = ApiMethod::new( |
491 | &ApiHandler::Sync(&close_fixed_index), | |
492 | &ObjectSchema::new( | |
493 | "Close fixed index writer.", | |
552c2259 | 494 | &sorted!([ |
255f378a DM |
495 | ( |
496 | "wid", | |
497 | false, | |
498 | &IntegerSchema::new("Fixed writer ID.") | |
499 | .minimum(1) | |
500 | .maximum(256) | |
501 | .schema() | |
502 | ), | |
503 | ( | |
504 | "chunk-count", | |
505 | false, | |
506 | &IntegerSchema::new("Chunk count. This is used to verify that the server got all chunks.") | |
507 | .minimum(1) | |
508 | .schema() | |
509 | ), | |
510 | ( | |
511 | "size", | |
512 | false, | |
513 | &IntegerSchema::new("File size. This is used to verify that the server got all data.") | |
514 | .minimum(1) | |
515 | .schema() | |
516 | ), | |
517 | ("csum", false, &StringSchema::new("Digest list checksum.").schema()), | |
552c2259 | 518 | ]), |
a42fa400 | 519 | ) |
255f378a | 520 | ); |
a42fa400 DM |
521 | |
522 | fn close_fixed_index ( | |
523 | param: Value, | |
524 | _info: &ApiMethod, | |
dd5495d6 | 525 | rpcenv: &mut dyn RpcEnvironment, |
a42fa400 DM |
526 | ) -> Result<Value, Error> { |
527 | ||
528 | let wid = tools::required_integer_param(¶m, "wid")? as usize; | |
529 | let chunk_count = tools::required_integer_param(¶m, "chunk-count")? as u64; | |
530 | let size = tools::required_integer_param(¶m, "size")? as u64; | |
fb6026b6 DM |
531 | let csum_str = tools::required_string_param(¶m, "csum")?; |
532 | let csum = proxmox::tools::hex_to_digest(csum_str)?; | |
a42fa400 DM |
533 | |
534 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
535 | ||
fb6026b6 | 536 | env.fixed_writer_close(wid, chunk_count, size, csum)?; |
a42fa400 DM |
537 | |
538 | env.log(format!("sucessfully closed fixed index {}", wid)); | |
539 | ||
540 | Ok(Value::Null) | |
541 | } | |
a2077252 | 542 | |
372724af DM |
543 | fn finish_backup ( |
544 | _param: Value, | |
545 | _info: &ApiMethod, | |
dd5495d6 | 546 | rpcenv: &mut dyn RpcEnvironment, |
372724af DM |
547 | ) -> Result<Value, Error> { |
548 | ||
549 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
550 | ||
551 | env.finish_backup()?; | |
60e589a1 | 552 | env.log("sucessfully finished backup"); |
372724af DM |
553 | |
554 | Ok(Value::Null) | |
555 | } | |
a2077252 | 556 | |
552c2259 | 557 | #[sortable] |
255f378a | 558 | pub const API_METHOD_DYNAMIC_CHUNK_INDEX: ApiMethod = ApiMethod::new( |
329d40b5 | 559 | &ApiHandler::AsyncHttp(&dynamic_chunk_index), |
255f378a DM |
560 | &ObjectSchema::new( |
561 | r###" | |
a42fa400 DM |
562 | Download the dynamic chunk index from the previous backup. |
563 | Simply returns an empty list if this is the first backup. | |
255f378a | 564 | "### , |
552c2259 DM |
565 | &sorted!([ |
566 | ("archive-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA) | |
567 | ]), | |
a42fa400 | 568 | ) |
255f378a | 569 | ); |
a42fa400 | 570 | |
d3611366 DM |
571 | fn dynamic_chunk_index( |
572 | _parts: Parts, | |
573 | _req_body: Body, | |
574 | param: Value, | |
255f378a | 575 | _info: &ApiMethod, |
dd5495d6 | 576 | rpcenv: Box<dyn RpcEnvironment>, |
ad51d02a | 577 | ) -> ApiFuture { |
d3611366 | 578 | |
ad51d02a DM |
579 | async move { |
580 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
d3611366 | 581 | |
ad51d02a | 582 | let archive_name = tools::required_string_param(¶m, "archive-name")?.to_owned(); |
d3611366 | 583 | |
ad51d02a DM |
584 | if !archive_name.ends_with(".didx") { |
585 | bail!("wrong archive extension: '{}'", archive_name); | |
a9584932 | 586 | } |
39e60bd6 | 587 | |
ad51d02a DM |
588 | let empty_response = { |
589 | Response::builder() | |
590 | .status(StatusCode::OK) | |
591 | .body(Body::empty())? | |
592 | }; | |
593 | ||
594 | let last_backup = match &env.last_backup { | |
595 | Some(info) => info, | |
596 | None => return Ok(empty_response), | |
597 | }; | |
598 | ||
599 | let mut path = last_backup.backup_dir.relative_path(); | |
600 | path.push(&archive_name); | |
601 | ||
602 | let index = match env.datastore.open_dynamic_reader(path) { | |
603 | Ok(index) => index, | |
604 | Err(_) => { | |
605 | env.log(format!("there is no last backup for archive '{}'", archive_name)); | |
606 | return Ok(empty_response); | |
607 | } | |
608 | }; | |
609 | ||
610 | env.log(format!("download last backup index for archive '{}'", archive_name)); | |
611 | ||
612 | let count = index.index_count(); | |
613 | for pos in 0..count { | |
614 | let (start, end, digest) = index.chunk_info(pos)?; | |
615 | let size = (end - start) as u32; | |
616 | env.register_chunk(digest, size)?; | |
617 | } | |
d3611366 | 618 | |
ad51d02a | 619 | let reader = DigestListEncoder::new(Box::new(index)); |
d3611366 | 620 | |
ad51d02a | 621 | let stream = WrappedReaderStream::new(reader); |
d3611366 | 622 | |
ad51d02a DM |
623 | // fixme: set size, content type? |
624 | let response = http::Response::builder() | |
625 | .status(200) | |
626 | .body(Body::wrap_stream(stream))?; | |
d3611366 | 627 | |
ad51d02a DM |
628 | Ok(response) |
629 | }.boxed() | |
d3611366 | 630 | } |
a42fa400 | 631 | |
552c2259 | 632 | #[sortable] |
255f378a | 633 | pub const API_METHOD_FIXED_CHUNK_INDEX: ApiMethod = ApiMethod::new( |
329d40b5 | 634 | &ApiHandler::AsyncHttp(&fixed_chunk_index), |
255f378a DM |
635 | &ObjectSchema::new( |
636 | r###" | |
a42fa400 DM |
637 | Download the fixed chunk index from the previous backup. |
638 | Simply returns an empty list if this is the first backup. | |
255f378a | 639 | "### , |
552c2259 DM |
640 | &sorted!([ |
641 | ("archive-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA) | |
642 | ]), | |
a42fa400 | 643 | ) |
255f378a | 644 | ); |
a42fa400 DM |
645 | |
646 | fn fixed_chunk_index( | |
647 | _parts: Parts, | |
648 | _req_body: Body, | |
649 | param: Value, | |
255f378a | 650 | _info: &ApiMethod, |
dd5495d6 | 651 | rpcenv: Box<dyn RpcEnvironment>, |
ad51d02a | 652 | ) -> ApiFuture { |
a42fa400 | 653 | |
ad51d02a DM |
654 | async move { |
655 | let env: &BackupEnvironment = rpcenv.as_ref(); | |
a42fa400 | 656 | |
ad51d02a | 657 | let archive_name = tools::required_string_param(¶m, "archive-name")?.to_owned(); |
a42fa400 | 658 | |
ad51d02a DM |
659 | if !archive_name.ends_with(".fidx") { |
660 | bail!("wrong archive extension: '{}'", archive_name); | |
661 | } | |
a42fa400 | 662 | |
ad51d02a DM |
663 | let empty_response = { |
664 | Response::builder() | |
665 | .status(StatusCode::OK) | |
666 | .body(Body::empty())? | |
667 | }; | |
668 | ||
669 | let last_backup = match &env.last_backup { | |
670 | Some(info) => info, | |
671 | None => return Ok(empty_response), | |
672 | }; | |
673 | ||
674 | let mut path = last_backup.backup_dir.relative_path(); | |
675 | path.push(&archive_name); | |
676 | ||
677 | let index = match env.datastore.open_fixed_reader(path) { | |
678 | Ok(index) => index, | |
679 | Err(_) => { | |
680 | env.log(format!("there is no last backup for archive '{}'", archive_name)); | |
681 | return Ok(empty_response); | |
682 | } | |
683 | }; | |
684 | ||
685 | env.log(format!("download last backup index for archive '{}'", archive_name)); | |
686 | ||
687 | let count = index.index_count(); | |
688 | let image_size = index.index_bytes(); | |
689 | for pos in 0..count { | |
690 | let digest = index.index_digest(pos).unwrap(); | |
691 | // Note: last chunk can be smaller | |
692 | let start = (pos*index.chunk_size) as u64; | |
693 | let mut end = start + index.chunk_size as u64; | |
694 | if end > image_size { end = image_size; } | |
695 | let size = (end - start) as u32; | |
696 | env.register_chunk(*digest, size)?; | |
a42fa400 | 697 | } |
a42fa400 | 698 | |
ad51d02a | 699 | let reader = DigestListEncoder::new(Box::new(index)); |
a42fa400 | 700 | |
ad51d02a | 701 | let stream = WrappedReaderStream::new(reader); |
a42fa400 | 702 | |
ad51d02a DM |
703 | // fixme: set size, content type? |
704 | let response = http::Response::builder() | |
705 | .status(200) | |
706 | .body(Body::wrap_stream(stream))?; | |
a42fa400 | 707 | |
ad51d02a DM |
708 | Ok(response) |
709 | }.boxed() | |
a42fa400 | 710 | } |