]>
Commit | Line | Data |
---|---|---|
7759eef5 | 1 | use std::collections::{HashSet, HashMap}; |
54552dda | 2 | use std::io::{self, Write}; |
367f002e WB |
3 | use std::path::{Path, PathBuf}; |
4 | use std::sync::{Arc, Mutex}; | |
60f9a6ea | 5 | use std::convert::TryFrom; |
367f002e | 6 | |
f7d4e4b5 | 7 | use anyhow::{bail, format_err, Error}; |
2c32fdde | 8 | use lazy_static::lazy_static; |
e4439025 DM |
9 | use serde_json::Value; |
10 | ||
11 | use proxmox::tools::fs::{replace_file, CreateOptions}; | |
529de6c7 | 12 | |
6d6b4e72 | 13 | use super::backup_info::{BackupGroup, BackupDir}; |
a92830dc | 14 | use super::chunk_store::ChunkStore; |
367f002e WB |
15 | use super::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; |
16 | use super::fixed_index::{FixedIndexReader, FixedIndexWriter}; | |
1610c45a | 17 | use super::manifest::{MANIFEST_BLOB_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest}; |
5de2bced | 18 | use super::index::*; |
1e8da0a7 | 19 | use super::{DataBlob, ArchiveType, archive_type}; |
367f002e | 20 | use crate::config::datastore; |
d4b59ae0 | 21 | use crate::server::WorkerTask; |
367f002e | 22 | use crate::tools; |
49a92084 | 23 | use crate::tools::format::HumanByte; |
e4342585 | 24 | use crate::tools::fs::{lock_dir_noblock, DirLockGuard}; |
e7cb4dc5 | 25 | use crate::api2::types::{GarbageCollectionStatus, Userid}; |
529de6c7 | 26 | |
367f002e WB |
27 | lazy_static! { |
28 | static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new()); | |
b3483782 | 29 | } |
ff3d3100 | 30 | |
e5064ba6 DM |
31 | /// Datastore Management |
32 | /// | |
33 | /// A Datastore can store severals backups, and provides the | |
34 | /// management interface for backup. | |
529de6c7 | 35 | pub struct DataStore { |
1629d2ad | 36 | chunk_store: Arc<ChunkStore>, |
64e53b28 | 37 | gc_mutex: Mutex<bool>, |
f2b99c34 | 38 | last_gc_status: Mutex<GarbageCollectionStatus>, |
529de6c7 DM |
39 | } |
40 | ||
41 | impl DataStore { | |
42 | ||
2c32fdde DM |
43 | pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> { |
44 | ||
d0187a51 DM |
45 | let (config, _digest) = datastore::config()?; |
46 | let config: datastore::DataStoreConfig = config.lookup("datastore", name)?; | |
2c32fdde | 47 | |
515688d1 | 48 | let mut map = DATASTORE_MAP.lock().unwrap(); |
2c32fdde DM |
49 | |
50 | if let Some(datastore) = map.get(name) { | |
51 | // Compare Config - if changed, create new Datastore object! | |
d0187a51 | 52 | if datastore.chunk_store.base == PathBuf::from(&config.path) { |
2c32fdde DM |
53 | return Ok(datastore.clone()); |
54 | } | |
55 | } | |
56 | ||
f0a61124 DM |
57 | let datastore = DataStore::open(name)?; |
58 | ||
59 | let datastore = Arc::new(datastore); | |
60 | map.insert(name.to_string(), datastore.clone()); | |
2c32fdde | 61 | |
f0a61124 | 62 | Ok(datastore) |
2c32fdde DM |
63 | } |
64 | ||
af6f80d3 | 65 | pub fn open(store_name: &str) -> Result<Self, Error> { |
529de6c7 | 66 | |
d0187a51 | 67 | let (config, _digest) = datastore::config()?; |
529de6c7 DM |
68 | let (_, store_config) = config.sections.get(store_name) |
69 | .ok_or(format_err!("no such datastore '{}'", store_name))?; | |
70 | ||
71 | let path = store_config["path"].as_str().unwrap(); | |
72 | ||
ba37f356 | 73 | Self::open_with_path(store_name, Path::new(path)) |
fce46593 DM |
74 | } |
75 | ||
ba37f356 | 76 | pub fn open_with_path(store_name: &str, path: &Path) -> Result<Self, Error> { |
277fc5a3 | 77 | let chunk_store = ChunkStore::open(store_name, path)?; |
529de6c7 | 78 | |
f2b99c34 DM |
79 | let gc_status = GarbageCollectionStatus::default(); |
80 | ||
529de6c7 | 81 | Ok(Self { |
1629d2ad | 82 | chunk_store: Arc::new(chunk_store), |
64e53b28 | 83 | gc_mutex: Mutex::new(false), |
f2b99c34 | 84 | last_gc_status: Mutex::new(gc_status), |
529de6c7 DM |
85 | }) |
86 | } | |
87 | ||
d59397e6 WB |
88 | pub fn get_chunk_iterator( |
89 | &self, | |
90 | ) -> Result< | |
a9767cf7 | 91 | impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)>, |
d59397e6 WB |
92 | Error |
93 | > { | |
a5736098 | 94 | self.chunk_store.get_chunk_iterator() |
d59397e6 WB |
95 | } |
96 | ||
91a905b6 | 97 | pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> { |
529de6c7 | 98 | |
91a905b6 | 99 | let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?; |
529de6c7 DM |
100 | |
101 | Ok(index) | |
102 | } | |
103 | ||
91a905b6 | 104 | pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> { |
529de6c7 | 105 | |
a7c72ad9 DM |
106 | let full_path = self.chunk_store.relative_path(filename.as_ref()); |
107 | ||
108 | let index = FixedIndexReader::open(&full_path)?; | |
529de6c7 DM |
109 | |
110 | Ok(index) | |
111 | } | |
3d5c11e5 | 112 | |
93d5d779 | 113 | pub fn create_dynamic_writer<P: AsRef<Path>>( |
0433db19 | 114 | &self, filename: P, |
93d5d779 | 115 | ) -> Result<DynamicIndexWriter, Error> { |
0433db19 | 116 | |
93d5d779 | 117 | let index = DynamicIndexWriter::create( |
976595e1 | 118 | self.chunk_store.clone(), filename.as_ref())?; |
0433db19 DM |
119 | |
120 | Ok(index) | |
121 | } | |
ff3d3100 | 122 | |
93d5d779 | 123 | pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> { |
77703d95 | 124 | |
d48a9955 DM |
125 | let full_path = self.chunk_store.relative_path(filename.as_ref()); |
126 | ||
127 | let index = DynamicIndexReader::open(&full_path)?; | |
77703d95 DM |
128 | |
129 | Ok(index) | |
130 | } | |
131 | ||
5de2bced WB |
132 | pub fn open_index<P>(&self, filename: P) -> Result<Box<dyn IndexFile + Send>, Error> |
133 | where | |
134 | P: AsRef<Path>, | |
135 | { | |
136 | let filename = filename.as_ref(); | |
137 | let out: Box<dyn IndexFile + Send> = | |
1e8da0a7 DM |
138 | match archive_type(filename)? { |
139 | ArchiveType::DynamicIndex => Box::new(self.open_dynamic_reader(filename)?), | |
140 | ArchiveType::FixedIndex => Box::new(self.open_fixed_reader(filename)?), | |
5de2bced WB |
141 | _ => bail!("cannot open index file of unknown type: {:?}", filename), |
142 | }; | |
143 | Ok(out) | |
144 | } | |
145 | ||
60f9a6ea DM |
146 | pub fn name(&self) -> &str { |
147 | self.chunk_store.name() | |
148 | } | |
149 | ||
ff3d3100 DM |
150 | pub fn base_path(&self) -> PathBuf { |
151 | self.chunk_store.base_path() | |
152 | } | |
153 | ||
c47e294e | 154 | /// Cleanup a backup directory |
7759eef5 DM |
155 | /// |
156 | /// Removes all files not mentioned in the manifest. | |
157 | pub fn cleanup_backup_dir(&self, backup_dir: &BackupDir, manifest: &BackupManifest | |
158 | ) -> Result<(), Error> { | |
159 | ||
160 | let mut full_path = self.base_path(); | |
161 | full_path.push(backup_dir.relative_path()); | |
162 | ||
163 | let mut wanted_files = HashSet::new(); | |
164 | wanted_files.insert(MANIFEST_BLOB_NAME.to_string()); | |
1610c45a | 165 | wanted_files.insert(CLIENT_LOG_BLOB_NAME.to_string()); |
7759eef5 DM |
166 | manifest.files().iter().for_each(|item| { wanted_files.insert(item.filename.clone()); }); |
167 | ||
168 | for item in tools::fs::read_subdir(libc::AT_FDCWD, &full_path)? { | |
169 | if let Ok(item) = item { | |
170 | if let Some(file_type) = item.file_type() { | |
171 | if file_type != nix::dir::Type::File { continue; } | |
172 | } | |
173 | let file_name = item.file_name().to_bytes(); | |
174 | if file_name == b"." || file_name == b".." { continue; }; | |
175 | ||
176 | if let Ok(name) = std::str::from_utf8(file_name) { | |
177 | if wanted_files.contains(name) { continue; } | |
178 | } | |
179 | println!("remove unused file {:?}", item.file_name()); | |
180 | let dirfd = item.parent_fd(); | |
181 | let _res = unsafe { libc::unlinkat(dirfd, item.file_name().as_ptr(), 0) }; | |
182 | } | |
183 | } | |
184 | ||
185 | Ok(()) | |
186 | } | |
4b4eba0b | 187 | |
41b373ec DM |
188 | /// Returns the absolute path for a backup_group |
189 | pub fn group_path(&self, backup_group: &BackupGroup) -> PathBuf { | |
4b4eba0b DM |
190 | let mut full_path = self.base_path(); |
191 | full_path.push(backup_group.group_path()); | |
41b373ec DM |
192 | full_path |
193 | } | |
194 | ||
195 | /// Returns the absolute path for backup_dir | |
196 | pub fn snapshot_path(&self, backup_dir: &BackupDir) -> PathBuf { | |
197 | let mut full_path = self.base_path(); | |
198 | full_path.push(backup_dir.relative_path()); | |
199 | full_path | |
200 | } | |
201 | ||
202 | /// Remove a complete backup group including all snapshots | |
6abce6c2 | 203 | pub fn remove_backup_group(&self, backup_group: &BackupGroup) -> Result<(), Error> { |
41b373ec DM |
204 | |
205 | let full_path = self.group_path(backup_group); | |
4b4eba0b | 206 | |
6d6b4e72 | 207 | let _guard = tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?; |
c9756b40 | 208 | |
4b4eba0b | 209 | log::info!("removing backup group {:?}", full_path); |
6abce6c2 | 210 | std::fs::remove_dir_all(&full_path) |
8a1d68c8 DM |
211 | .map_err(|err| { |
212 | format_err!( | |
213 | "removing backup group {:?} failed - {}", | |
214 | full_path, | |
215 | err, | |
216 | ) | |
217 | })?; | |
4b4eba0b DM |
218 | |
219 | Ok(()) | |
220 | } | |
221 | ||
8f579717 | 222 | /// Remove a backup directory including all content |
c9756b40 | 223 | pub fn remove_backup_dir(&self, backup_dir: &BackupDir, force: bool) -> Result<(), Error> { |
8f579717 | 224 | |
41b373ec | 225 | let full_path = self.snapshot_path(backup_dir); |
8f579717 | 226 | |
6d6b4e72 | 227 | let _guard; |
c9756b40 | 228 | if !force { |
6d6b4e72 | 229 | _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or used as base")?; |
c9756b40 SR |
230 | } |
231 | ||
8a1d68c8 | 232 | log::info!("removing backup snapshot {:?}", full_path); |
6abce6c2 | 233 | std::fs::remove_dir_all(&full_path) |
8a1d68c8 DM |
234 | .map_err(|err| { |
235 | format_err!( | |
236 | "removing backup snapshot {:?} failed - {}", | |
237 | full_path, | |
238 | err, | |
239 | ) | |
240 | })?; | |
8f579717 DM |
241 | |
242 | Ok(()) | |
243 | } | |
244 | ||
41b373ec DM |
245 | /// Returns the time of the last successful backup |
246 | /// | |
247 | /// Or None if there is no backup in the group (or the group dir does not exist). | |
6a7be83e | 248 | pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<i64>, Error> { |
41b373ec DM |
249 | let base_path = self.base_path(); |
250 | let mut group_path = base_path.clone(); | |
251 | group_path.push(backup_group.group_path()); | |
252 | ||
253 | if group_path.exists() { | |
254 | backup_group.last_successful_backup(&base_path) | |
255 | } else { | |
256 | Ok(None) | |
257 | } | |
258 | } | |
259 | ||
54552dda DM |
260 | /// Returns the backup owner. |
261 | /// | |
262 | /// The backup owner is the user who first created the backup group. | |
e7cb4dc5 | 263 | pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Userid, Error> { |
54552dda DM |
264 | let mut full_path = self.base_path(); |
265 | full_path.push(backup_group.group_path()); | |
266 | full_path.push("owner"); | |
267 | let owner = proxmox::tools::fs::file_read_firstline(full_path)?; | |
e7cb4dc5 | 268 | Ok(owner.trim_end().parse()?) // remove trailing newline |
54552dda DM |
269 | } |
270 | ||
271 | /// Set the backup owner. | |
e7cb4dc5 WB |
272 | pub fn set_owner( |
273 | &self, | |
274 | backup_group: &BackupGroup, | |
275 | userid: &Userid, | |
276 | force: bool, | |
277 | ) -> Result<(), Error> { | |
54552dda DM |
278 | let mut path = self.base_path(); |
279 | path.push(backup_group.group_path()); | |
280 | path.push("owner"); | |
281 | ||
282 | let mut open_options = std::fs::OpenOptions::new(); | |
283 | open_options.write(true); | |
284 | open_options.truncate(true); | |
285 | ||
286 | if force { | |
287 | open_options.create(true); | |
288 | } else { | |
289 | open_options.create_new(true); | |
290 | } | |
291 | ||
292 | let mut file = open_options.open(&path) | |
293 | .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?; | |
294 | ||
295 | write!(file, "{}\n", userid) | |
296 | .map_err(|err| format_err!("unable to write owner file {:?} - {}", path, err))?; | |
297 | ||
298 | Ok(()) | |
299 | } | |
300 | ||
1fc82c41 | 301 | /// Create (if it does not already exists) and lock a backup group |
54552dda DM |
302 | /// |
303 | /// And set the owner to 'userid'. If the group already exists, it returns the | |
304 | /// current owner (instead of setting the owner). | |
1fc82c41 | 305 | /// |
1ffe0301 | 306 | /// This also acquires an exclusive lock on the directory and returns the lock guard. |
e7cb4dc5 WB |
307 | pub fn create_locked_backup_group( |
308 | &self, | |
309 | backup_group: &BackupGroup, | |
310 | userid: &Userid, | |
e4342585 | 311 | ) -> Result<(Userid, DirLockGuard), Error> { |
8731e40a | 312 | // create intermediate path first: |
54552dda DM |
313 | let base_path = self.base_path(); |
314 | ||
315 | let mut full_path = base_path.clone(); | |
316 | full_path.push(backup_group.backup_type()); | |
8731e40a WB |
317 | std::fs::create_dir_all(&full_path)?; |
318 | ||
54552dda DM |
319 | full_path.push(backup_group.backup_id()); |
320 | ||
321 | // create the last component now | |
322 | match std::fs::create_dir(&full_path) { | |
323 | Ok(_) => { | |
e4342585 | 324 | let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?; |
54552dda DM |
325 | self.set_owner(backup_group, userid, false)?; |
326 | let owner = self.get_owner(backup_group)?; // just to be sure | |
1fc82c41 | 327 | Ok((owner, guard)) |
54552dda DM |
328 | } |
329 | Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => { | |
e4342585 | 330 | let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?; |
54552dda | 331 | let owner = self.get_owner(backup_group)?; // just to be sure |
1fc82c41 | 332 | Ok((owner, guard)) |
54552dda DM |
333 | } |
334 | Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err), | |
335 | } | |
336 | } | |
337 | ||
338 | /// Creates a new backup snapshot inside a BackupGroup | |
339 | /// | |
340 | /// The BackupGroup directory needs to exist. | |
f23f7543 SR |
341 | pub fn create_locked_backup_dir(&self, backup_dir: &BackupDir) |
342 | -> Result<(PathBuf, bool, DirLockGuard), Error> | |
343 | { | |
b3483782 DM |
344 | let relative_path = backup_dir.relative_path(); |
345 | let mut full_path = self.base_path(); | |
346 | full_path.push(&relative_path); | |
ff3d3100 | 347 | |
f23f7543 SR |
348 | let lock = || |
349 | lock_dir_noblock(&full_path, "snapshot", "internal error - tried creating snapshot that's already in use"); | |
350 | ||
8731e40a | 351 | match std::fs::create_dir(&full_path) { |
f23f7543 SR |
352 | Ok(_) => Ok((relative_path, true, lock()?)), |
353 | Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok((relative_path, false, lock()?)), | |
354 | Err(e) => Err(e.into()) | |
8731e40a | 355 | } |
ff3d3100 DM |
356 | } |
357 | ||
3d5c11e5 | 358 | pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> { |
ff3d3100 | 359 | let base = self.base_path(); |
3d5c11e5 DM |
360 | |
361 | let mut list = vec![]; | |
362 | ||
95cea65b DM |
363 | use walkdir::WalkDir; |
364 | ||
365 | let walker = WalkDir::new(&base).same_file_system(true).into_iter(); | |
366 | ||
367 | // make sure we skip .chunks (and other hidden files to keep it simple) | |
368 | fn is_hidden(entry: &walkdir::DirEntry) -> bool { | |
369 | entry.file_name() | |
370 | .to_str() | |
371 | .map(|s| s.starts_with(".")) | |
372 | .unwrap_or(false) | |
373 | } | |
c3b090ac TL |
374 | let handle_entry_err = |err: walkdir::Error| { |
375 | if let Some(inner) = err.io_error() { | |
376 | let path = err.path().unwrap_or(Path::new("")); | |
377 | match inner.kind() { | |
378 | io::ErrorKind::PermissionDenied => { | |
379 | // only allow to skip ext4 fsck directory, avoid GC if, for example, | |
380 | // a user got file permissions wrong on datastore rsync to new server | |
381 | if err.depth() > 1 || !path.ends_with("lost+found") { | |
382 | bail!("cannot continue garbage-collection safely, permission denied on: {}", path.display()) | |
383 | } | |
384 | }, | |
385 | _ => bail!("unexpected error on datastore traversal: {} - {}", inner, path.display()), | |
386 | } | |
387 | } | |
388 | Ok(()) | |
389 | }; | |
95cea65b | 390 | for entry in walker.filter_entry(|e| !is_hidden(e)) { |
c3b090ac TL |
391 | let path = match entry { |
392 | Ok(entry) => entry.into_path(), | |
393 | Err(err) => { | |
394 | handle_entry_err(err)?; | |
395 | continue | |
396 | }, | |
397 | }; | |
1e8da0a7 DM |
398 | if let Ok(archive_type) = archive_type(&path) { |
399 | if archive_type == ArchiveType::FixedIndex || archive_type == ArchiveType::DynamicIndex { | |
95cea65b | 400 | list.push(path); |
3d5c11e5 DM |
401 | } |
402 | } | |
403 | } | |
404 | ||
405 | Ok(list) | |
406 | } | |
407 | ||
a660978c DM |
408 | // mark chunks used by ``index`` as used |
409 | fn index_mark_used_chunks<I: IndexFile>( | |
410 | &self, | |
411 | index: I, | |
412 | file_name: &Path, // only used for error reporting | |
413 | status: &mut GarbageCollectionStatus, | |
99641a6b | 414 | worker: &WorkerTask, |
a660978c DM |
415 | ) -> Result<(), Error> { |
416 | ||
417 | status.index_file_count += 1; | |
418 | status.index_data_bytes += index.index_bytes(); | |
419 | ||
420 | for pos in 0..index.index_count() { | |
99641a6b | 421 | worker.fail_on_abort()?; |
a660978c DM |
422 | tools::fail_on_shutdown()?; |
423 | let digest = index.index_digest(pos).unwrap(); | |
424 | if let Err(err) = self.chunk_store.touch_chunk(digest) { | |
2f57a433 OB |
425 | worker.warn(&format!("warning: unable to access chunk {}, required by {:?} - {}", |
426 | proxmox::tools::digest_to_hex(digest), file_name, err)); | |
a660978c DM |
427 | } |
428 | } | |
429 | Ok(()) | |
430 | } | |
431 | ||
99641a6b | 432 | fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus, worker: &WorkerTask) -> Result<(), Error> { |
3d5c11e5 DM |
433 | |
434 | let image_list = self.list_images()?; | |
435 | ||
8317873c DM |
436 | let image_count = image_list.len(); |
437 | ||
438 | let mut done = 0; | |
439 | ||
440 | let mut last_percentage: usize = 0; | |
441 | ||
3d5c11e5 | 442 | for path in image_list { |
92da93b2 | 443 | |
99641a6b | 444 | worker.fail_on_abort()?; |
92da93b2 DM |
445 | tools::fail_on_shutdown()?; |
446 | ||
1e8da0a7 DM |
447 | if let Ok(archive_type) = archive_type(&path) { |
448 | if archive_type == ArchiveType::FixedIndex { | |
91a905b6 | 449 | let index = self.open_fixed_reader(&path)?; |
99641a6b | 450 | self.index_mark_used_chunks(index, &path, status, worker)?; |
1e8da0a7 | 451 | } else if archive_type == ArchiveType::DynamicIndex { |
93d5d779 | 452 | let index = self.open_dynamic_reader(&path)?; |
99641a6b | 453 | self.index_mark_used_chunks(index, &path, status, worker)?; |
77703d95 DM |
454 | } |
455 | } | |
8317873c DM |
456 | done += 1; |
457 | ||
458 | let percentage = done*100/image_count; | |
459 | if percentage > last_percentage { | |
460 | worker.log(format!("percentage done: phase1 {}% ({} of {} index files)", | |
461 | percentage, done, image_count)); | |
462 | last_percentage = percentage; | |
463 | } | |
3d5c11e5 DM |
464 | } |
465 | ||
466 | Ok(()) | |
f2b99c34 DM |
467 | } |
468 | ||
469 | pub fn last_gc_status(&self) -> GarbageCollectionStatus { | |
470 | self.last_gc_status.lock().unwrap().clone() | |
471 | } | |
3d5c11e5 | 472 | |
8545480a DM |
473 | pub fn garbage_collection_running(&self) -> bool { |
474 | if let Ok(_) = self.gc_mutex.try_lock() { false } else { true } | |
475 | } | |
476 | ||
99641a6b | 477 | pub fn garbage_collection(&self, worker: &WorkerTask) -> Result<(), Error> { |
3d5c11e5 | 478 | |
a198d74f | 479 | if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() { |
e95950e4 | 480 | |
43b13033 DM |
481 | let _exclusive_lock = self.chunk_store.try_exclusive_lock()?; |
482 | ||
49a92084 TL |
483 | let phase1_start_time = unsafe { libc::time(std::ptr::null_mut()) }; |
484 | let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time); | |
11861a48 | 485 | |
64e53b28 | 486 | let mut gc_status = GarbageCollectionStatus::default(); |
f2b99c34 | 487 | gc_status.upid = Some(worker.to_string()); |
6ea3a0b7 | 488 | |
73e57f24 | 489 | worker.log("Start GC phase1 (mark used chunks)"); |
64e53b28 | 490 | |
99641a6b | 491 | self.mark_used_chunks(&mut gc_status, &worker)?; |
64e53b28 | 492 | |
d4b59ae0 | 493 | worker.log("Start GC phase2 (sweep unused chunks)"); |
49a92084 | 494 | self.chunk_store.sweep_unused_chunks(oldest_writer, phase1_start_time, &mut gc_status, &worker)?; |
64e53b28 | 495 | |
49a92084 | 496 | worker.log(&format!("Removed garbage: {}", HumanByte::from(gc_status.removed_bytes))); |
a660978c | 497 | worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks)); |
cf459b19 | 498 | if gc_status.pending_bytes > 0 { |
49a92084 | 499 | worker.log(&format!("Pending removals: {} (in {} chunks)", HumanByte::from(gc_status.pending_bytes), gc_status.pending_chunks)); |
cf459b19 | 500 | } |
a9767cf7 SR |
501 | if gc_status.removed_bad > 0 { |
502 | worker.log(&format!("Removed bad files: {}", gc_status.removed_bad)); | |
503 | } | |
cf459b19 | 504 | |
49a92084 | 505 | worker.log(&format!("Original data usage: {}", HumanByte::from(gc_status.index_data_bytes))); |
868c5852 DM |
506 | |
507 | if gc_status.index_data_bytes > 0 { | |
49a92084 TL |
508 | let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64; |
509 | worker.log(&format!("On-Disk usage: {} ({:.2}%)", HumanByte::from(gc_status.disk_bytes), comp_per)); | |
868c5852 DM |
510 | } |
511 | ||
49a92084 | 512 | worker.log(&format!("On-Disk chunks: {}", gc_status.disk_chunks)); |
868c5852 DM |
513 | |
514 | if gc_status.disk_chunks > 0 { | |
515 | let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64); | |
49a92084 | 516 | worker.log(&format!("Average chunk size: {}", HumanByte::from(avg_chunk))); |
868c5852 | 517 | } |
64e53b28 | 518 | |
f2b99c34 DM |
519 | *self.last_gc_status.lock().unwrap() = gc_status; |
520 | ||
64e53b28 | 521 | } else { |
d4b59ae0 | 522 | bail!("Start GC failed - (already running/locked)"); |
64e53b28 | 523 | } |
3d5c11e5 DM |
524 | |
525 | Ok(()) | |
526 | } | |
3b7ade9e | 527 | |
1cf5178a DM |
528 | pub fn try_shared_chunk_store_lock(&self) -> Result<tools::ProcessLockSharedGuard, Error> { |
529 | self.chunk_store.try_shared_lock() | |
530 | } | |
531 | ||
d48a9955 DM |
532 | pub fn chunk_path(&self, digest:&[u8; 32]) -> (PathBuf, String) { |
533 | self.chunk_store.chunk_path(digest) | |
534 | } | |
535 | ||
2585a8a4 DM |
536 | pub fn cond_touch_chunk(&self, digest: &[u8; 32], fail_if_not_exist: bool) -> Result<bool, Error> { |
537 | self.chunk_store.cond_touch_chunk(digest, fail_if_not_exist) | |
538 | } | |
539 | ||
f98ac774 | 540 | pub fn insert_chunk( |
3b7ade9e | 541 | &self, |
4ee8f53d DM |
542 | chunk: &DataBlob, |
543 | digest: &[u8; 32], | |
3b7ade9e | 544 | ) -> Result<(bool, u64), Error> { |
4ee8f53d | 545 | self.chunk_store.insert_chunk(chunk, digest) |
3b7ade9e | 546 | } |
60f9a6ea | 547 | |
39f18b30 | 548 | pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> { |
60f9a6ea DM |
549 | let mut path = self.base_path(); |
550 | path.push(backup_dir.relative_path()); | |
551 | path.push(filename); | |
552 | ||
39f18b30 DM |
553 | proxmox::try_block!({ |
554 | let mut file = std::fs::File::open(&path)?; | |
555 | DataBlob::load_from_reader(&mut file) | |
556 | }).map_err(|err| format_err!("unable to load blob '{:?}' - {}", path, err)) | |
557 | } | |
e4439025 DM |
558 | |
559 | ||
39f18b30 DM |
560 | pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> { |
561 | ||
562 | let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest); | |
563 | ||
564 | proxmox::try_block!({ | |
565 | let mut file = std::fs::File::open(&chunk_path)?; | |
566 | DataBlob::load_from_reader(&mut file) | |
567 | }).map_err(|err| format_err!( | |
568 | "store '{}', unable to load chunk '{}' - {}", | |
569 | self.name(), | |
570 | digest_str, | |
571 | err, | |
572 | )) | |
573 | } | |
e4439025 | 574 | |
521a0acb WB |
575 | pub fn load_manifest( |
576 | &self, | |
577 | backup_dir: &BackupDir, | |
ff86ef00 | 578 | ) -> Result<(BackupManifest, u64), Error> { |
39f18b30 DM |
579 | let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?; |
580 | let raw_size = blob.raw_size(); | |
60f9a6ea | 581 | let manifest = BackupManifest::try_from(blob)?; |
ff86ef00 | 582 | Ok((manifest, raw_size)) |
60f9a6ea | 583 | } |
e4439025 DM |
584 | |
585 | pub fn load_manifest_json( | |
586 | &self, | |
587 | backup_dir: &BackupDir, | |
588 | ) -> Result<Value, Error> { | |
589 | let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?; | |
8819d1f2 FG |
590 | // no expected digest available |
591 | let manifest_data = blob.decode(None, None)?; | |
e4439025 DM |
592 | let manifest: Value = serde_json::from_slice(&manifest_data[..])?; |
593 | Ok(manifest) | |
594 | } | |
595 | ||
596 | pub fn store_manifest( | |
597 | &self, | |
598 | backup_dir: &BackupDir, | |
599 | manifest: Value, | |
600 | ) -> Result<(), Error> { | |
601 | let manifest = serde_json::to_string_pretty(&manifest)?; | |
602 | let blob = DataBlob::encode(manifest.as_bytes(), None, true)?; | |
603 | let raw_data = blob.raw_data(); | |
604 | ||
605 | let mut path = self.base_path(); | |
606 | path.push(backup_dir.relative_path()); | |
607 | path.push(MANIFEST_BLOB_NAME); | |
608 | ||
609 | replace_file(&path, raw_data, CreateOptions::new())?; | |
610 | ||
611 | Ok(()) | |
612 | } | |
529de6c7 | 613 | } |