]> git.proxmox.com Git - proxmox-backup.git/blob - src/backup/datastore.rs
clippy: use chars / byte string literals
[proxmox-backup.git] / src / backup / datastore.rs
1 use std::collections::{HashSet, HashMap};
2 use std::io::{self, Write};
3 use std::path::{Path, PathBuf};
4 use std::sync::{Arc, Mutex};
5 use std::convert::TryFrom;
6 use std::str::FromStr;
7 use std::time::Duration;
8 use std::fs::File;
9
10 use anyhow::{bail, format_err, Error};
11 use lazy_static::lazy_static;
12
13 use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions, open_file_locked};
14
15 use super::backup_info::{BackupGroup, BackupDir};
16 use super::chunk_store::ChunkStore;
17 use super::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
18 use super::fixed_index::{FixedIndexReader, FixedIndexWriter};
19 use super::manifest::{MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest};
20 use super::index::*;
21 use super::{DataBlob, ArchiveType, archive_type};
22 use crate::config::datastore::{self, DataStoreConfig};
23 use crate::task::TaskState;
24 use crate::tools;
25 use crate::tools::format::HumanByte;
26 use crate::tools::fs::{lock_dir_noblock, DirLockGuard};
27 use crate::api2::types::{Authid, GarbageCollectionStatus};
28 use crate::server::UPID;
29
30 lazy_static! {
31 static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
32 }
33
34 /// Datastore Management
35 ///
36 /// A Datastore can store severals backups, and provides the
37 /// management interface for backup.
38 pub struct DataStore {
39 chunk_store: Arc<ChunkStore>,
40 gc_mutex: Mutex<bool>,
41 last_gc_status: Mutex<GarbageCollectionStatus>,
42 verify_new: bool,
43 }
44
45 impl DataStore {
46
47 pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> {
48
49 let (config, _digest) = datastore::config()?;
50 let config: datastore::DataStoreConfig = config.lookup("datastore", name)?;
51 let path = PathBuf::from(&config.path);
52
53 let mut map = DATASTORE_MAP.lock().unwrap();
54
55 if let Some(datastore) = map.get(name) {
56 // Compare Config - if changed, create new Datastore object!
57 if datastore.chunk_store.base == path &&
58 datastore.verify_new == config.verify_new.unwrap_or(false)
59 {
60 return Ok(datastore.clone());
61 }
62 }
63
64 let datastore = DataStore::open_with_path(name, &path, config)?;
65
66 let datastore = Arc::new(datastore);
67 map.insert(name.to_string(), datastore.clone());
68
69 Ok(datastore)
70 }
71
72 fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> {
73 let chunk_store = ChunkStore::open(store_name, path)?;
74
75 let mut gc_status_path = chunk_store.base_path();
76 gc_status_path.push(".gc-status");
77
78 let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? {
79 match serde_json::from_str(&state) {
80 Ok(state) => state,
81 Err(err) => {
82 eprintln!("error reading gc-status: {}", err);
83 GarbageCollectionStatus::default()
84 }
85 }
86 } else {
87 GarbageCollectionStatus::default()
88 };
89
90 Ok(Self {
91 chunk_store: Arc::new(chunk_store),
92 gc_mutex: Mutex::new(false),
93 last_gc_status: Mutex::new(gc_status),
94 verify_new: config.verify_new.unwrap_or(false),
95 })
96 }
97
98 pub fn get_chunk_iterator(
99 &self,
100 ) -> Result<
101 impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)>,
102 Error
103 > {
104 self.chunk_store.get_chunk_iterator()
105 }
106
107 pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> {
108
109 let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?;
110
111 Ok(index)
112 }
113
114 pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> {
115
116 let full_path = self.chunk_store.relative_path(filename.as_ref());
117
118 let index = FixedIndexReader::open(&full_path)?;
119
120 Ok(index)
121 }
122
123 pub fn create_dynamic_writer<P: AsRef<Path>>(
124 &self, filename: P,
125 ) -> Result<DynamicIndexWriter, Error> {
126
127 let index = DynamicIndexWriter::create(
128 self.chunk_store.clone(), filename.as_ref())?;
129
130 Ok(index)
131 }
132
133 pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> {
134
135 let full_path = self.chunk_store.relative_path(filename.as_ref());
136
137 let index = DynamicIndexReader::open(&full_path)?;
138
139 Ok(index)
140 }
141
142 pub fn open_index<P>(&self, filename: P) -> Result<Box<dyn IndexFile + Send>, Error>
143 where
144 P: AsRef<Path>,
145 {
146 let filename = filename.as_ref();
147 let out: Box<dyn IndexFile + Send> =
148 match archive_type(filename)? {
149 ArchiveType::DynamicIndex => Box::new(self.open_dynamic_reader(filename)?),
150 ArchiveType::FixedIndex => Box::new(self.open_fixed_reader(filename)?),
151 _ => bail!("cannot open index file of unknown type: {:?}", filename),
152 };
153 Ok(out)
154 }
155
156 pub fn name(&self) -> &str {
157 self.chunk_store.name()
158 }
159
160 pub fn base_path(&self) -> PathBuf {
161 self.chunk_store.base_path()
162 }
163
164 /// Cleanup a backup directory
165 ///
166 /// Removes all files not mentioned in the manifest.
167 pub fn cleanup_backup_dir(&self, backup_dir: &BackupDir, manifest: &BackupManifest
168 ) -> Result<(), Error> {
169
170 let mut full_path = self.base_path();
171 full_path.push(backup_dir.relative_path());
172
173 let mut wanted_files = HashSet::new();
174 wanted_files.insert(MANIFEST_BLOB_NAME.to_string());
175 wanted_files.insert(CLIENT_LOG_BLOB_NAME.to_string());
176 manifest.files().iter().for_each(|item| { wanted_files.insert(item.filename.clone()); });
177
178 for item in tools::fs::read_subdir(libc::AT_FDCWD, &full_path)? {
179 if let Ok(item) = item {
180 if let Some(file_type) = item.file_type() {
181 if file_type != nix::dir::Type::File { continue; }
182 }
183 let file_name = item.file_name().to_bytes();
184 if file_name == b"." || file_name == b".." { continue; };
185
186 if let Ok(name) = std::str::from_utf8(file_name) {
187 if wanted_files.contains(name) { continue; }
188 }
189 println!("remove unused file {:?}", item.file_name());
190 let dirfd = item.parent_fd();
191 let _res = unsafe { libc::unlinkat(dirfd, item.file_name().as_ptr(), 0) };
192 }
193 }
194
195 Ok(())
196 }
197
198 /// Returns the absolute path for a backup_group
199 pub fn group_path(&self, backup_group: &BackupGroup) -> PathBuf {
200 let mut full_path = self.base_path();
201 full_path.push(backup_group.group_path());
202 full_path
203 }
204
205 /// Returns the absolute path for backup_dir
206 pub fn snapshot_path(&self, backup_dir: &BackupDir) -> PathBuf {
207 let mut full_path = self.base_path();
208 full_path.push(backup_dir.relative_path());
209 full_path
210 }
211
212 /// Remove a complete backup group including all snapshots
213 pub fn remove_backup_group(&self, backup_group: &BackupGroup) -> Result<(), Error> {
214
215 let full_path = self.group_path(backup_group);
216
217 let _guard = tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?;
218
219 log::info!("removing backup group {:?}", full_path);
220
221 // remove all individual backup dirs first to ensure nothing is using them
222 for snap in backup_group.list_backups(&self.base_path())? {
223 self.remove_backup_dir(&snap.backup_dir, false)?;
224 }
225
226 // no snapshots left, we can now safely remove the empty folder
227 std::fs::remove_dir_all(&full_path)
228 .map_err(|err| {
229 format_err!(
230 "removing backup group directory {:?} failed - {}",
231 full_path,
232 err,
233 )
234 })?;
235
236 Ok(())
237 }
238
239 /// Remove a backup directory including all content
240 pub fn remove_backup_dir(&self, backup_dir: &BackupDir, force: bool) -> Result<(), Error> {
241
242 let full_path = self.snapshot_path(backup_dir);
243
244 let (_guard, _manifest_guard);
245 if !force {
246 _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?;
247 _manifest_guard = self.lock_manifest(backup_dir)?;
248 }
249
250 log::info!("removing backup snapshot {:?}", full_path);
251 std::fs::remove_dir_all(&full_path)
252 .map_err(|err| {
253 format_err!(
254 "removing backup snapshot {:?} failed - {}",
255 full_path,
256 err,
257 )
258 })?;
259
260 // the manifest does not exists anymore, we do not need to keep the lock
261 if let Ok(path) = self.manifest_lock_path(backup_dir) {
262 // ignore errors
263 let _ = std::fs::remove_file(path);
264 }
265
266 Ok(())
267 }
268
269 /// Returns the time of the last successful backup
270 ///
271 /// Or None if there is no backup in the group (or the group dir does not exist).
272 pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<i64>, Error> {
273 let base_path = self.base_path();
274 let mut group_path = base_path.clone();
275 group_path.push(backup_group.group_path());
276
277 if group_path.exists() {
278 backup_group.last_successful_backup(&base_path)
279 } else {
280 Ok(None)
281 }
282 }
283
284 /// Returns the backup owner.
285 ///
286 /// The backup owner is the entity who first created the backup group.
287 pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Authid, Error> {
288 let mut full_path = self.base_path();
289 full_path.push(backup_group.group_path());
290 full_path.push("owner");
291 let owner = proxmox::tools::fs::file_read_firstline(full_path)?;
292 Ok(owner.trim_end().parse()?) // remove trailing newline
293 }
294
295 /// Set the backup owner.
296 pub fn set_owner(
297 &self,
298 backup_group: &BackupGroup,
299 auth_id: &Authid,
300 force: bool,
301 ) -> Result<(), Error> {
302 let mut path = self.base_path();
303 path.push(backup_group.group_path());
304 path.push("owner");
305
306 let mut open_options = std::fs::OpenOptions::new();
307 open_options.write(true);
308 open_options.truncate(true);
309
310 if force {
311 open_options.create(true);
312 } else {
313 open_options.create_new(true);
314 }
315
316 let mut file = open_options.open(&path)
317 .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?;
318
319 writeln!(file, "{}", auth_id)
320 .map_err(|err| format_err!("unable to write owner file {:?} - {}", path, err))?;
321
322 Ok(())
323 }
324
325 /// Create (if it does not already exists) and lock a backup group
326 ///
327 /// And set the owner to 'userid'. If the group already exists, it returns the
328 /// current owner (instead of setting the owner).
329 ///
330 /// This also acquires an exclusive lock on the directory and returns the lock guard.
331 pub fn create_locked_backup_group(
332 &self,
333 backup_group: &BackupGroup,
334 auth_id: &Authid,
335 ) -> Result<(Authid, DirLockGuard), Error> {
336 // create intermediate path first:
337 let mut full_path = self.base_path();
338 full_path.push(backup_group.backup_type());
339 std::fs::create_dir_all(&full_path)?;
340
341 full_path.push(backup_group.backup_id());
342
343 // create the last component now
344 match std::fs::create_dir(&full_path) {
345 Ok(_) => {
346 let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?;
347 self.set_owner(backup_group, auth_id, false)?;
348 let owner = self.get_owner(backup_group)?; // just to be sure
349 Ok((owner, guard))
350 }
351 Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => {
352 let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?;
353 let owner = self.get_owner(backup_group)?; // just to be sure
354 Ok((owner, guard))
355 }
356 Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err),
357 }
358 }
359
360 /// Creates a new backup snapshot inside a BackupGroup
361 ///
362 /// The BackupGroup directory needs to exist.
363 pub fn create_locked_backup_dir(&self, backup_dir: &BackupDir)
364 -> Result<(PathBuf, bool, DirLockGuard), Error>
365 {
366 let relative_path = backup_dir.relative_path();
367 let mut full_path = self.base_path();
368 full_path.push(&relative_path);
369
370 let lock = ||
371 lock_dir_noblock(&full_path, "snapshot", "internal error - tried creating snapshot that's already in use");
372
373 match std::fs::create_dir(&full_path) {
374 Ok(_) => Ok((relative_path, true, lock()?)),
375 Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok((relative_path, false, lock()?)),
376 Err(e) => Err(e.into())
377 }
378 }
379
380 pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> {
381 let base = self.base_path();
382
383 let mut list = vec![];
384
385 use walkdir::WalkDir;
386
387 let walker = WalkDir::new(&base).into_iter();
388
389 // make sure we skip .chunks (and other hidden files to keep it simple)
390 fn is_hidden(entry: &walkdir::DirEntry) -> bool {
391 entry.file_name()
392 .to_str()
393 .map(|s| s.starts_with('.'))
394 .unwrap_or(false)
395 }
396 let handle_entry_err = |err: walkdir::Error| {
397 if let Some(inner) = err.io_error() {
398 let path = err.path().unwrap_or(Path::new(""));
399 match inner.kind() {
400 io::ErrorKind::PermissionDenied => {
401 // only allow to skip ext4 fsck directory, avoid GC if, for example,
402 // a user got file permissions wrong on datastore rsync to new server
403 if err.depth() > 1 || !path.ends_with("lost+found") {
404 bail!("cannot continue garbage-collection safely, permission denied on: {}", path.display())
405 }
406 },
407 _ => bail!("unexpected error on datastore traversal: {} - {}", inner, path.display()),
408 }
409 }
410 Ok(())
411 };
412 for entry in walker.filter_entry(|e| !is_hidden(e)) {
413 let path = match entry {
414 Ok(entry) => entry.into_path(),
415 Err(err) => {
416 handle_entry_err(err)?;
417 continue
418 },
419 };
420 if let Ok(archive_type) = archive_type(&path) {
421 if archive_type == ArchiveType::FixedIndex || archive_type == ArchiveType::DynamicIndex {
422 list.push(path);
423 }
424 }
425 }
426
427 Ok(list)
428 }
429
430 // mark chunks used by ``index`` as used
431 fn index_mark_used_chunks<I: IndexFile>(
432 &self,
433 index: I,
434 file_name: &Path, // only used for error reporting
435 status: &mut GarbageCollectionStatus,
436 worker: &dyn TaskState,
437 ) -> Result<(), Error> {
438
439 status.index_file_count += 1;
440 status.index_data_bytes += index.index_bytes();
441
442 for pos in 0..index.index_count() {
443 worker.check_abort()?;
444 tools::fail_on_shutdown()?;
445 let digest = index.index_digest(pos).unwrap();
446 if let Err(err) = self.chunk_store.touch_chunk(digest) {
447 crate::task_warn!(
448 worker,
449 "warning: unable to access chunk {}, required by {:?} - {}",
450 proxmox::tools::digest_to_hex(digest),
451 file_name,
452 err,
453 );
454
455 // touch any corresponding .bad files to keep them around, meaning if a chunk is
456 // rewritten correctly they will be removed automatically, as well as if no index
457 // file requires the chunk anymore (won't get to this loop then)
458 for i in 0..=9 {
459 let bad_ext = format!("{}.bad", i);
460 let mut bad_path = PathBuf::new();
461 bad_path.push(self.chunk_path(digest).0);
462 bad_path.set_extension(bad_ext);
463 self.chunk_store.cond_touch_path(&bad_path, false)?;
464 }
465 }
466 }
467 Ok(())
468 }
469
470 fn mark_used_chunks(
471 &self,
472 status: &mut GarbageCollectionStatus,
473 worker: &dyn TaskState,
474 ) -> Result<(), Error> {
475
476 let image_list = self.list_images()?;
477 let image_count = image_list.len();
478
479 let mut done = 0;
480 let mut last_percentage: usize = 0;
481
482 let mut strange_paths_count: u64 = 0;
483
484 for img in image_list {
485
486 worker.check_abort()?;
487 tools::fail_on_shutdown()?;
488
489 if let Some(backup_dir_path) = img.parent() {
490 let backup_dir_path = backup_dir_path.strip_prefix(self.base_path())?;
491 if let Some(backup_dir_str) = backup_dir_path.to_str() {
492 if BackupDir::from_str(backup_dir_str).is_err() {
493 strange_paths_count += 1;
494 }
495 }
496 }
497
498 match std::fs::File::open(&img) {
499 Ok(file) => {
500 if let Ok(archive_type) = archive_type(&img) {
501 if archive_type == ArchiveType::FixedIndex {
502 let index = FixedIndexReader::new(file).map_err(|e| {
503 format_err!("can't read index '{}' - {}", img.to_string_lossy(), e)
504 })?;
505 self.index_mark_used_chunks(index, &img, status, worker)?;
506 } else if archive_type == ArchiveType::DynamicIndex {
507 let index = DynamicIndexReader::new(file).map_err(|e| {
508 format_err!("can't read index '{}' - {}", img.to_string_lossy(), e)
509 })?;
510 self.index_mark_used_chunks(index, &img, status, worker)?;
511 }
512 }
513 }
514 Err(err) if err.kind() == io::ErrorKind::NotFound => (), // ignore vanished files
515 Err(err) => bail!("can't open index {} - {}", img.to_string_lossy(), err),
516 }
517 done += 1;
518
519 let percentage = done*100/image_count;
520 if percentage > last_percentage {
521 crate::task_log!(
522 worker,
523 "marked {}% ({} of {} index files)",
524 percentage,
525 done,
526 image_count,
527 );
528 last_percentage = percentage;
529 }
530 }
531
532 if strange_paths_count > 0 {
533 crate::task_log!(
534 worker,
535 "found (and marked) {} index files outside of expected directory scheme",
536 strange_paths_count,
537 );
538 }
539
540
541 Ok(())
542 }
543
544 pub fn last_gc_status(&self) -> GarbageCollectionStatus {
545 self.last_gc_status.lock().unwrap().clone()
546 }
547
548 pub fn garbage_collection_running(&self) -> bool {
549 if let Ok(_) = self.gc_mutex.try_lock() { false } else { true }
550 }
551
552 pub fn garbage_collection(&self, worker: &dyn TaskState, upid: &UPID) -> Result<(), Error> {
553
554 if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() {
555
556 // avoids that we run GC if an old daemon process has still a
557 // running backup writer, which is not save as we have no "oldest
558 // writer" information and thus no safe atime cutoff
559 let _exclusive_lock = self.chunk_store.try_exclusive_lock()?;
560
561 let phase1_start_time = proxmox::tools::time::epoch_i64();
562 let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
563
564 let mut gc_status = GarbageCollectionStatus::default();
565 gc_status.upid = Some(upid.to_string());
566
567 crate::task_log!(worker, "Start GC phase1 (mark used chunks)");
568
569 self.mark_used_chunks(&mut gc_status, worker)?;
570
571 crate::task_log!(worker, "Start GC phase2 (sweep unused chunks)");
572 self.chunk_store.sweep_unused_chunks(
573 oldest_writer,
574 phase1_start_time,
575 &mut gc_status,
576 worker,
577 )?;
578
579 crate::task_log!(
580 worker,
581 "Removed garbage: {}",
582 HumanByte::from(gc_status.removed_bytes),
583 );
584 crate::task_log!(worker, "Removed chunks: {}", gc_status.removed_chunks);
585 if gc_status.pending_bytes > 0 {
586 crate::task_log!(
587 worker,
588 "Pending removals: {} (in {} chunks)",
589 HumanByte::from(gc_status.pending_bytes),
590 gc_status.pending_chunks,
591 );
592 }
593 if gc_status.removed_bad > 0 {
594 crate::task_log!(worker, "Removed bad chunks: {}", gc_status.removed_bad);
595 }
596
597 if gc_status.still_bad > 0 {
598 crate::task_log!(worker, "Leftover bad chunks: {}", gc_status.still_bad);
599 }
600
601 crate::task_log!(
602 worker,
603 "Original data usage: {}",
604 HumanByte::from(gc_status.index_data_bytes),
605 );
606
607 if gc_status.index_data_bytes > 0 {
608 let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64;
609 crate::task_log!(
610 worker,
611 "On-Disk usage: {} ({:.2}%)",
612 HumanByte::from(gc_status.disk_bytes),
613 comp_per,
614 );
615 }
616
617 crate::task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks);
618
619 let deduplication_factor = if gc_status.disk_bytes > 0 {
620 (gc_status.index_data_bytes as f64)/(gc_status.disk_bytes as f64)
621 } else {
622 1.0
623 };
624
625 crate::task_log!(worker, "Deduplication factor: {:.2}", deduplication_factor);
626
627 if gc_status.disk_chunks > 0 {
628 let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64);
629 crate::task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk));
630 }
631
632 if let Ok(serialized) = serde_json::to_string(&gc_status) {
633 let mut path = self.base_path();
634 path.push(".gc-status");
635
636 let backup_user = crate::backup::backup_user()?;
637 let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644);
638 // set the correct owner/group/permissions while saving file
639 // owner(rw) = backup, group(r)= backup
640 let options = CreateOptions::new()
641 .perm(mode)
642 .owner(backup_user.uid)
643 .group(backup_user.gid);
644
645 // ignore errors
646 let _ = replace_file(path, serialized.as_bytes(), options);
647 }
648
649 *self.last_gc_status.lock().unwrap() = gc_status;
650
651 } else {
652 bail!("Start GC failed - (already running/locked)");
653 }
654
655 Ok(())
656 }
657
658 pub fn try_shared_chunk_store_lock(&self) -> Result<tools::ProcessLockSharedGuard, Error> {
659 self.chunk_store.try_shared_lock()
660 }
661
662 pub fn chunk_path(&self, digest:&[u8; 32]) -> (PathBuf, String) {
663 self.chunk_store.chunk_path(digest)
664 }
665
666 pub fn cond_touch_chunk(&self, digest: &[u8; 32], fail_if_not_exist: bool) -> Result<bool, Error> {
667 self.chunk_store.cond_touch_chunk(digest, fail_if_not_exist)
668 }
669
670 pub fn insert_chunk(
671 &self,
672 chunk: &DataBlob,
673 digest: &[u8; 32],
674 ) -> Result<(bool, u64), Error> {
675 self.chunk_store.insert_chunk(chunk, digest)
676 }
677
678 pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> {
679 let mut path = self.base_path();
680 path.push(backup_dir.relative_path());
681 path.push(filename);
682
683 proxmox::try_block!({
684 let mut file = std::fs::File::open(&path)?;
685 DataBlob::load_from_reader(&mut file)
686 }).map_err(|err| format_err!("unable to load blob '{:?}' - {}", path, err))
687 }
688
689
690 pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
691
692 let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest);
693
694 proxmox::try_block!({
695 let mut file = std::fs::File::open(&chunk_path)?;
696 DataBlob::load_from_reader(&mut file)
697 }).map_err(|err| format_err!(
698 "store '{}', unable to load chunk '{}' - {}",
699 self.name(),
700 digest_str,
701 err,
702 ))
703 }
704
705 /// Returns the filename to lock a manifest
706 ///
707 /// Also creates the basedir. The lockfile is located in
708 /// '/run/proxmox-backup/locks/{datastore}/{type}/{id}/{timestamp}.index.json.lck'
709 fn manifest_lock_path(
710 &self,
711 backup_dir: &BackupDir,
712 ) -> Result<String, Error> {
713 let mut path = format!(
714 "/run/proxmox-backup/locks/{}/{}/{}",
715 self.name(),
716 backup_dir.group().backup_type(),
717 backup_dir.group().backup_id(),
718 );
719 std::fs::create_dir_all(&path)?;
720 use std::fmt::Write;
721 write!(path, "/{}{}", backup_dir.backup_time_string(), &MANIFEST_LOCK_NAME)?;
722
723 Ok(path)
724 }
725
726 fn lock_manifest(
727 &self,
728 backup_dir: &BackupDir,
729 ) -> Result<File, Error> {
730 let path = self.manifest_lock_path(backup_dir)?;
731
732 // update_manifest should never take a long time, so if someone else has
733 // the lock we can simply block a bit and should get it soon
734 open_file_locked(&path, Duration::from_secs(5), true)
735 .map_err(|err| {
736 format_err!(
737 "unable to acquire manifest lock {:?} - {}", &path, err
738 )
739 })
740 }
741
742 /// Load the manifest without a lock. Must not be written back.
743 pub fn load_manifest(
744 &self,
745 backup_dir: &BackupDir,
746 ) -> Result<(BackupManifest, u64), Error> {
747 let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?;
748 let raw_size = blob.raw_size();
749 let manifest = BackupManifest::try_from(blob)?;
750 Ok((manifest, raw_size))
751 }
752
753 /// Update the manifest of the specified snapshot. Never write a manifest directly,
754 /// only use this method - anything else may break locking guarantees.
755 pub fn update_manifest(
756 &self,
757 backup_dir: &BackupDir,
758 update_fn: impl FnOnce(&mut BackupManifest),
759 ) -> Result<(), Error> {
760
761 let _guard = self.lock_manifest(backup_dir)?;
762 let (mut manifest, _) = self.load_manifest(&backup_dir)?;
763
764 update_fn(&mut manifest);
765
766 let manifest = serde_json::to_value(manifest)?;
767 let manifest = serde_json::to_string_pretty(&manifest)?;
768 let blob = DataBlob::encode(manifest.as_bytes(), None, true)?;
769 let raw_data = blob.raw_data();
770
771 let mut path = self.base_path();
772 path.push(backup_dir.relative_path());
773 path.push(MANIFEST_BLOB_NAME);
774
775 // atomic replace invalidates flock - no other writes past this point!
776 replace_file(&path, raw_data, CreateOptions::new())?;
777
778 Ok(())
779 }
780
781 pub fn verify_new(&self) -> bool {
782 self.verify_new
783 }
784 }
785