]>
Commit | Line | Data |
---|---|---|
7759eef5 | 1 | use std::collections::{HashSet, HashMap}; |
54552dda | 2 | use std::io::{self, Write}; |
367f002e WB |
3 | use std::path::{Path, PathBuf}; |
4 | use std::sync::{Arc, Mutex}; | |
60f9a6ea | 5 | use std::convert::TryFrom; |
1a374fcf SR |
6 | use std::time::Duration; |
7 | use std::fs::File; | |
367f002e | 8 | |
f7d4e4b5 | 9 | use anyhow::{bail, format_err, Error}; |
2c32fdde | 10 | use lazy_static::lazy_static; |
e4439025 | 11 | |
b683fd58 | 12 | use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions, open_file_locked}; |
529de6c7 | 13 | |
6d6b4e72 | 14 | use super::backup_info::{BackupGroup, BackupDir}; |
a92830dc | 15 | use super::chunk_store::ChunkStore; |
367f002e WB |
16 | use super::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; |
17 | use super::fixed_index::{FixedIndexReader, FixedIndexWriter}; | |
1a374fcf | 18 | use super::manifest::{MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest}; |
5de2bced | 19 | use super::index::*; |
1e8da0a7 | 20 | use super::{DataBlob, ArchiveType, archive_type}; |
115d927c | 21 | use crate::config::datastore::{self, DataStoreConfig}; |
f6b1d1cc | 22 | use crate::task::TaskState; |
367f002e | 23 | use crate::tools; |
49a92084 | 24 | use crate::tools::format::HumanByte; |
e4342585 | 25 | use crate::tools::fs::{lock_dir_noblock, DirLockGuard}; |
e7cb4dc5 | 26 | use crate::api2::types::{GarbageCollectionStatus, Userid}; |
f6b1d1cc | 27 | use crate::server::UPID; |
529de6c7 | 28 | |
367f002e WB |
29 | lazy_static! { |
30 | static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new()); | |
b3483782 | 31 | } |
ff3d3100 | 32 | |
e5064ba6 DM |
33 | /// Datastore Management |
34 | /// | |
35 | /// A Datastore can store severals backups, and provides the | |
36 | /// management interface for backup. | |
529de6c7 | 37 | pub struct DataStore { |
1629d2ad | 38 | chunk_store: Arc<ChunkStore>, |
64e53b28 | 39 | gc_mutex: Mutex<bool>, |
f2b99c34 | 40 | last_gc_status: Mutex<GarbageCollectionStatus>, |
0698f78d | 41 | verify_new: bool, |
529de6c7 DM |
42 | } |
43 | ||
44 | impl DataStore { | |
45 | ||
2c32fdde DM |
46 | pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> { |
47 | ||
d0187a51 DM |
48 | let (config, _digest) = datastore::config()?; |
49 | let config: datastore::DataStoreConfig = config.lookup("datastore", name)?; | |
df729017 | 50 | let path = PathBuf::from(&config.path); |
2c32fdde | 51 | |
515688d1 | 52 | let mut map = DATASTORE_MAP.lock().unwrap(); |
2c32fdde DM |
53 | |
54 | if let Some(datastore) = map.get(name) { | |
55 | // Compare Config - if changed, create new Datastore object! | |
0698f78d SR |
56 | if datastore.chunk_store.base == path && |
57 | datastore.verify_new == config.verify_new.unwrap_or(false) | |
58 | { | |
2c32fdde DM |
59 | return Ok(datastore.clone()); |
60 | } | |
61 | } | |
62 | ||
df729017 | 63 | let datastore = DataStore::open_with_path(name, &path, config)?; |
f0a61124 DM |
64 | |
65 | let datastore = Arc::new(datastore); | |
66 | map.insert(name.to_string(), datastore.clone()); | |
2c32fdde | 67 | |
f0a61124 | 68 | Ok(datastore) |
2c32fdde DM |
69 | } |
70 | ||
0698f78d | 71 | fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> { |
277fc5a3 | 72 | let chunk_store = ChunkStore::open(store_name, path)?; |
529de6c7 | 73 | |
b683fd58 DC |
74 | let mut gc_status_path = chunk_store.base_path(); |
75 | gc_status_path.push(".gc-status"); | |
76 | ||
77 | let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? { | |
78 | match serde_json::from_str(&state) { | |
79 | Ok(state) => state, | |
80 | Err(err) => { | |
81 | eprintln!("error reading gc-status: {}", err); | |
82 | GarbageCollectionStatus::default() | |
83 | } | |
84 | } | |
85 | } else { | |
86 | GarbageCollectionStatus::default() | |
87 | }; | |
f2b99c34 | 88 | |
529de6c7 | 89 | Ok(Self { |
1629d2ad | 90 | chunk_store: Arc::new(chunk_store), |
64e53b28 | 91 | gc_mutex: Mutex::new(false), |
f2b99c34 | 92 | last_gc_status: Mutex::new(gc_status), |
0698f78d | 93 | verify_new: config.verify_new.unwrap_or(false), |
529de6c7 DM |
94 | }) |
95 | } | |
96 | ||
d59397e6 WB |
97 | pub fn get_chunk_iterator( |
98 | &self, | |
99 | ) -> Result< | |
a9767cf7 | 100 | impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)>, |
d59397e6 WB |
101 | Error |
102 | > { | |
a5736098 | 103 | self.chunk_store.get_chunk_iterator() |
d59397e6 WB |
104 | } |
105 | ||
91a905b6 | 106 | pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> { |
529de6c7 | 107 | |
91a905b6 | 108 | let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?; |
529de6c7 DM |
109 | |
110 | Ok(index) | |
111 | } | |
112 | ||
91a905b6 | 113 | pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> { |
529de6c7 | 114 | |
a7c72ad9 DM |
115 | let full_path = self.chunk_store.relative_path(filename.as_ref()); |
116 | ||
117 | let index = FixedIndexReader::open(&full_path)?; | |
529de6c7 DM |
118 | |
119 | Ok(index) | |
120 | } | |
3d5c11e5 | 121 | |
93d5d779 | 122 | pub fn create_dynamic_writer<P: AsRef<Path>>( |
0433db19 | 123 | &self, filename: P, |
93d5d779 | 124 | ) -> Result<DynamicIndexWriter, Error> { |
0433db19 | 125 | |
93d5d779 | 126 | let index = DynamicIndexWriter::create( |
976595e1 | 127 | self.chunk_store.clone(), filename.as_ref())?; |
0433db19 DM |
128 | |
129 | Ok(index) | |
130 | } | |
ff3d3100 | 131 | |
93d5d779 | 132 | pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> { |
77703d95 | 133 | |
d48a9955 DM |
134 | let full_path = self.chunk_store.relative_path(filename.as_ref()); |
135 | ||
136 | let index = DynamicIndexReader::open(&full_path)?; | |
77703d95 DM |
137 | |
138 | Ok(index) | |
139 | } | |
140 | ||
5de2bced WB |
141 | pub fn open_index<P>(&self, filename: P) -> Result<Box<dyn IndexFile + Send>, Error> |
142 | where | |
143 | P: AsRef<Path>, | |
144 | { | |
145 | let filename = filename.as_ref(); | |
146 | let out: Box<dyn IndexFile + Send> = | |
1e8da0a7 DM |
147 | match archive_type(filename)? { |
148 | ArchiveType::DynamicIndex => Box::new(self.open_dynamic_reader(filename)?), | |
149 | ArchiveType::FixedIndex => Box::new(self.open_fixed_reader(filename)?), | |
5de2bced WB |
150 | _ => bail!("cannot open index file of unknown type: {:?}", filename), |
151 | }; | |
152 | Ok(out) | |
153 | } | |
154 | ||
60f9a6ea DM |
155 | pub fn name(&self) -> &str { |
156 | self.chunk_store.name() | |
157 | } | |
158 | ||
ff3d3100 DM |
159 | pub fn base_path(&self) -> PathBuf { |
160 | self.chunk_store.base_path() | |
161 | } | |
162 | ||
c47e294e | 163 | /// Cleanup a backup directory |
7759eef5 DM |
164 | /// |
165 | /// Removes all files not mentioned in the manifest. | |
166 | pub fn cleanup_backup_dir(&self, backup_dir: &BackupDir, manifest: &BackupManifest | |
167 | ) -> Result<(), Error> { | |
168 | ||
169 | let mut full_path = self.base_path(); | |
170 | full_path.push(backup_dir.relative_path()); | |
171 | ||
172 | let mut wanted_files = HashSet::new(); | |
173 | wanted_files.insert(MANIFEST_BLOB_NAME.to_string()); | |
1610c45a | 174 | wanted_files.insert(CLIENT_LOG_BLOB_NAME.to_string()); |
7759eef5 DM |
175 | manifest.files().iter().for_each(|item| { wanted_files.insert(item.filename.clone()); }); |
176 | ||
177 | for item in tools::fs::read_subdir(libc::AT_FDCWD, &full_path)? { | |
178 | if let Ok(item) = item { | |
179 | if let Some(file_type) = item.file_type() { | |
180 | if file_type != nix::dir::Type::File { continue; } | |
181 | } | |
182 | let file_name = item.file_name().to_bytes(); | |
183 | if file_name == b"." || file_name == b".." { continue; }; | |
184 | ||
185 | if let Ok(name) = std::str::from_utf8(file_name) { | |
186 | if wanted_files.contains(name) { continue; } | |
187 | } | |
188 | println!("remove unused file {:?}", item.file_name()); | |
189 | let dirfd = item.parent_fd(); | |
190 | let _res = unsafe { libc::unlinkat(dirfd, item.file_name().as_ptr(), 0) }; | |
191 | } | |
192 | } | |
193 | ||
194 | Ok(()) | |
195 | } | |
4b4eba0b | 196 | |
41b373ec DM |
197 | /// Returns the absolute path for a backup_group |
198 | pub fn group_path(&self, backup_group: &BackupGroup) -> PathBuf { | |
4b4eba0b DM |
199 | let mut full_path = self.base_path(); |
200 | full_path.push(backup_group.group_path()); | |
41b373ec DM |
201 | full_path |
202 | } | |
203 | ||
204 | /// Returns the absolute path for backup_dir | |
205 | pub fn snapshot_path(&self, backup_dir: &BackupDir) -> PathBuf { | |
206 | let mut full_path = self.base_path(); | |
207 | full_path.push(backup_dir.relative_path()); | |
208 | full_path | |
209 | } | |
210 | ||
211 | /// Remove a complete backup group including all snapshots | |
6abce6c2 | 212 | pub fn remove_backup_group(&self, backup_group: &BackupGroup) -> Result<(), Error> { |
41b373ec DM |
213 | |
214 | let full_path = self.group_path(backup_group); | |
4b4eba0b | 215 | |
6d6b4e72 | 216 | let _guard = tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?; |
c9756b40 | 217 | |
4b4eba0b | 218 | log::info!("removing backup group {:?}", full_path); |
4c0ae82e SR |
219 | |
220 | // remove all individual backup dirs first to ensure nothing is using them | |
221 | for snap in backup_group.list_backups(&self.base_path())? { | |
222 | self.remove_backup_dir(&snap.backup_dir, false)?; | |
223 | } | |
224 | ||
225 | // no snapshots left, we can now safely remove the empty folder | |
6abce6c2 | 226 | std::fs::remove_dir_all(&full_path) |
8a1d68c8 DM |
227 | .map_err(|err| { |
228 | format_err!( | |
4c0ae82e | 229 | "removing backup group directory {:?} failed - {}", |
8a1d68c8 DM |
230 | full_path, |
231 | err, | |
232 | ) | |
233 | })?; | |
4b4eba0b DM |
234 | |
235 | Ok(()) | |
236 | } | |
237 | ||
8f579717 | 238 | /// Remove a backup directory including all content |
c9756b40 | 239 | pub fn remove_backup_dir(&self, backup_dir: &BackupDir, force: bool) -> Result<(), Error> { |
8f579717 | 240 | |
41b373ec | 241 | let full_path = self.snapshot_path(backup_dir); |
8f579717 | 242 | |
1a374fcf | 243 | let (_guard, _manifest_guard); |
c9756b40 | 244 | if !force { |
238a872d | 245 | _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?; |
1a374fcf | 246 | _manifest_guard = self.lock_manifest(backup_dir); |
c9756b40 SR |
247 | } |
248 | ||
8a1d68c8 | 249 | log::info!("removing backup snapshot {:?}", full_path); |
6abce6c2 | 250 | std::fs::remove_dir_all(&full_path) |
8a1d68c8 DM |
251 | .map_err(|err| { |
252 | format_err!( | |
253 | "removing backup snapshot {:?} failed - {}", | |
254 | full_path, | |
255 | err, | |
256 | ) | |
257 | })?; | |
8f579717 DM |
258 | |
259 | Ok(()) | |
260 | } | |
261 | ||
41b373ec DM |
262 | /// Returns the time of the last successful backup |
263 | /// | |
264 | /// Or None if there is no backup in the group (or the group dir does not exist). | |
6a7be83e | 265 | pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<i64>, Error> { |
41b373ec DM |
266 | let base_path = self.base_path(); |
267 | let mut group_path = base_path.clone(); | |
268 | group_path.push(backup_group.group_path()); | |
269 | ||
270 | if group_path.exists() { | |
271 | backup_group.last_successful_backup(&base_path) | |
272 | } else { | |
273 | Ok(None) | |
274 | } | |
275 | } | |
276 | ||
54552dda DM |
277 | /// Returns the backup owner. |
278 | /// | |
279 | /// The backup owner is the user who first created the backup group. | |
e7cb4dc5 | 280 | pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Userid, Error> { |
54552dda DM |
281 | let mut full_path = self.base_path(); |
282 | full_path.push(backup_group.group_path()); | |
283 | full_path.push("owner"); | |
284 | let owner = proxmox::tools::fs::file_read_firstline(full_path)?; | |
e7cb4dc5 | 285 | Ok(owner.trim_end().parse()?) // remove trailing newline |
54552dda DM |
286 | } |
287 | ||
288 | /// Set the backup owner. | |
e7cb4dc5 WB |
289 | pub fn set_owner( |
290 | &self, | |
291 | backup_group: &BackupGroup, | |
292 | userid: &Userid, | |
293 | force: bool, | |
294 | ) -> Result<(), Error> { | |
54552dda DM |
295 | let mut path = self.base_path(); |
296 | path.push(backup_group.group_path()); | |
297 | path.push("owner"); | |
298 | ||
299 | let mut open_options = std::fs::OpenOptions::new(); | |
300 | open_options.write(true); | |
301 | open_options.truncate(true); | |
302 | ||
303 | if force { | |
304 | open_options.create(true); | |
305 | } else { | |
306 | open_options.create_new(true); | |
307 | } | |
308 | ||
309 | let mut file = open_options.open(&path) | |
310 | .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?; | |
311 | ||
8db14689 | 312 | writeln!(file, "{}", userid) |
54552dda DM |
313 | .map_err(|err| format_err!("unable to write owner file {:?} - {}", path, err))?; |
314 | ||
315 | Ok(()) | |
316 | } | |
317 | ||
1fc82c41 | 318 | /// Create (if it does not already exists) and lock a backup group |
54552dda DM |
319 | /// |
320 | /// And set the owner to 'userid'. If the group already exists, it returns the | |
321 | /// current owner (instead of setting the owner). | |
1fc82c41 | 322 | /// |
1ffe0301 | 323 | /// This also acquires an exclusive lock on the directory and returns the lock guard. |
e7cb4dc5 WB |
324 | pub fn create_locked_backup_group( |
325 | &self, | |
326 | backup_group: &BackupGroup, | |
327 | userid: &Userid, | |
e4342585 | 328 | ) -> Result<(Userid, DirLockGuard), Error> { |
8731e40a | 329 | // create intermediate path first: |
54552dda DM |
330 | let base_path = self.base_path(); |
331 | ||
332 | let mut full_path = base_path.clone(); | |
333 | full_path.push(backup_group.backup_type()); | |
8731e40a WB |
334 | std::fs::create_dir_all(&full_path)?; |
335 | ||
54552dda DM |
336 | full_path.push(backup_group.backup_id()); |
337 | ||
338 | // create the last component now | |
339 | match std::fs::create_dir(&full_path) { | |
340 | Ok(_) => { | |
e4342585 | 341 | let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?; |
54552dda DM |
342 | self.set_owner(backup_group, userid, false)?; |
343 | let owner = self.get_owner(backup_group)?; // just to be sure | |
1fc82c41 | 344 | Ok((owner, guard)) |
54552dda DM |
345 | } |
346 | Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => { | |
e4342585 | 347 | let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?; |
54552dda | 348 | let owner = self.get_owner(backup_group)?; // just to be sure |
1fc82c41 | 349 | Ok((owner, guard)) |
54552dda DM |
350 | } |
351 | Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err), | |
352 | } | |
353 | } | |
354 | ||
355 | /// Creates a new backup snapshot inside a BackupGroup | |
356 | /// | |
357 | /// The BackupGroup directory needs to exist. | |
f23f7543 SR |
358 | pub fn create_locked_backup_dir(&self, backup_dir: &BackupDir) |
359 | -> Result<(PathBuf, bool, DirLockGuard), Error> | |
360 | { | |
b3483782 DM |
361 | let relative_path = backup_dir.relative_path(); |
362 | let mut full_path = self.base_path(); | |
363 | full_path.push(&relative_path); | |
ff3d3100 | 364 | |
f23f7543 SR |
365 | let lock = || |
366 | lock_dir_noblock(&full_path, "snapshot", "internal error - tried creating snapshot that's already in use"); | |
367 | ||
8731e40a | 368 | match std::fs::create_dir(&full_path) { |
f23f7543 SR |
369 | Ok(_) => Ok((relative_path, true, lock()?)), |
370 | Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok((relative_path, false, lock()?)), | |
371 | Err(e) => Err(e.into()) | |
8731e40a | 372 | } |
ff3d3100 DM |
373 | } |
374 | ||
3d5c11e5 | 375 | pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> { |
ff3d3100 | 376 | let base = self.base_path(); |
3d5c11e5 DM |
377 | |
378 | let mut list = vec![]; | |
379 | ||
95cea65b DM |
380 | use walkdir::WalkDir; |
381 | ||
382 | let walker = WalkDir::new(&base).same_file_system(true).into_iter(); | |
383 | ||
384 | // make sure we skip .chunks (and other hidden files to keep it simple) | |
385 | fn is_hidden(entry: &walkdir::DirEntry) -> bool { | |
386 | entry.file_name() | |
387 | .to_str() | |
388 | .map(|s| s.starts_with(".")) | |
389 | .unwrap_or(false) | |
390 | } | |
c3b090ac TL |
391 | let handle_entry_err = |err: walkdir::Error| { |
392 | if let Some(inner) = err.io_error() { | |
393 | let path = err.path().unwrap_or(Path::new("")); | |
394 | match inner.kind() { | |
395 | io::ErrorKind::PermissionDenied => { | |
396 | // only allow to skip ext4 fsck directory, avoid GC if, for example, | |
397 | // a user got file permissions wrong on datastore rsync to new server | |
398 | if err.depth() > 1 || !path.ends_with("lost+found") { | |
399 | bail!("cannot continue garbage-collection safely, permission denied on: {}", path.display()) | |
400 | } | |
401 | }, | |
402 | _ => bail!("unexpected error on datastore traversal: {} - {}", inner, path.display()), | |
403 | } | |
404 | } | |
405 | Ok(()) | |
406 | }; | |
95cea65b | 407 | for entry in walker.filter_entry(|e| !is_hidden(e)) { |
c3b090ac TL |
408 | let path = match entry { |
409 | Ok(entry) => entry.into_path(), | |
410 | Err(err) => { | |
411 | handle_entry_err(err)?; | |
412 | continue | |
413 | }, | |
414 | }; | |
1e8da0a7 DM |
415 | if let Ok(archive_type) = archive_type(&path) { |
416 | if archive_type == ArchiveType::FixedIndex || archive_type == ArchiveType::DynamicIndex { | |
95cea65b | 417 | list.push(path); |
3d5c11e5 DM |
418 | } |
419 | } | |
420 | } | |
421 | ||
422 | Ok(list) | |
423 | } | |
424 | ||
a660978c DM |
425 | // mark chunks used by ``index`` as used |
426 | fn index_mark_used_chunks<I: IndexFile>( | |
427 | &self, | |
428 | index: I, | |
429 | file_name: &Path, // only used for error reporting | |
430 | status: &mut GarbageCollectionStatus, | |
f6b1d1cc | 431 | worker: &dyn TaskState, |
a660978c DM |
432 | ) -> Result<(), Error> { |
433 | ||
434 | status.index_file_count += 1; | |
435 | status.index_data_bytes += index.index_bytes(); | |
436 | ||
437 | for pos in 0..index.index_count() { | |
f6b1d1cc | 438 | worker.check_abort()?; |
a660978c DM |
439 | tools::fail_on_shutdown()?; |
440 | let digest = index.index_digest(pos).unwrap(); | |
441 | if let Err(err) = self.chunk_store.touch_chunk(digest) { | |
f6b1d1cc WB |
442 | crate::task_warn!( |
443 | worker, | |
444 | "warning: unable to access chunk {}, required by {:?} - {}", | |
445 | proxmox::tools::digest_to_hex(digest), | |
446 | file_name, | |
447 | err, | |
448 | ); | |
a660978c DM |
449 | } |
450 | } | |
451 | Ok(()) | |
452 | } | |
453 | ||
f6b1d1cc WB |
454 | fn mark_used_chunks( |
455 | &self, | |
456 | status: &mut GarbageCollectionStatus, | |
457 | worker: &dyn TaskState, | |
458 | ) -> Result<(), Error> { | |
3d5c11e5 DM |
459 | |
460 | let image_list = self.list_images()?; | |
461 | ||
8317873c DM |
462 | let image_count = image_list.len(); |
463 | ||
464 | let mut done = 0; | |
465 | ||
466 | let mut last_percentage: usize = 0; | |
467 | ||
3d5c11e5 | 468 | for path in image_list { |
92da93b2 | 469 | |
f6b1d1cc | 470 | worker.check_abort()?; |
92da93b2 DM |
471 | tools::fail_on_shutdown()?; |
472 | ||
e0762002 DM |
473 | let full_path = self.chunk_store.relative_path(&path); |
474 | match std::fs::File::open(&full_path) { | |
475 | Ok(file) => { | |
476 | if let Ok(archive_type) = archive_type(&path) { | |
477 | if archive_type == ArchiveType::FixedIndex { | |
478 | let index = FixedIndexReader::new(file)?; | |
479 | self.index_mark_used_chunks(index, &path, status, worker)?; | |
480 | } else if archive_type == ArchiveType::DynamicIndex { | |
481 | let index = DynamicIndexReader::new(file)?; | |
482 | self.index_mark_used_chunks(index, &path, status, worker)?; | |
483 | } | |
484 | } | |
485 | } | |
486 | Err(err) => { | |
487 | if err.kind() == std::io::ErrorKind::NotFound { | |
488 | // simply ignore vanished files | |
489 | } else { | |
490 | return Err(err.into()); | |
491 | } | |
77703d95 DM |
492 | } |
493 | } | |
8317873c DM |
494 | done += 1; |
495 | ||
496 | let percentage = done*100/image_count; | |
497 | if percentage > last_percentage { | |
f6b1d1cc WB |
498 | crate::task_log!( |
499 | worker, | |
500 | "percentage done: phase1 {}% ({} of {} index files)", | |
501 | percentage, | |
502 | done, | |
503 | image_count, | |
504 | ); | |
8317873c DM |
505 | last_percentage = percentage; |
506 | } | |
3d5c11e5 DM |
507 | } |
508 | ||
509 | Ok(()) | |
f2b99c34 DM |
510 | } |
511 | ||
512 | pub fn last_gc_status(&self) -> GarbageCollectionStatus { | |
513 | self.last_gc_status.lock().unwrap().clone() | |
514 | } | |
3d5c11e5 | 515 | |
8545480a DM |
516 | pub fn garbage_collection_running(&self) -> bool { |
517 | if let Ok(_) = self.gc_mutex.try_lock() { false } else { true } | |
518 | } | |
519 | ||
f6b1d1cc | 520 | pub fn garbage_collection(&self, worker: &dyn TaskState, upid: &UPID) -> Result<(), Error> { |
3d5c11e5 | 521 | |
a198d74f | 522 | if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() { |
e95950e4 | 523 | |
c6772c92 TL |
524 | // avoids that we run GC if an old daemon process has still a |
525 | // running backup writer, which is not save as we have no "oldest | |
526 | // writer" information and thus no safe atime cutoff | |
43b13033 DM |
527 | let _exclusive_lock = self.chunk_store.try_exclusive_lock()?; |
528 | ||
823867f5 | 529 | let phase1_start_time = proxmox::tools::time::epoch_i64(); |
49a92084 | 530 | let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time); |
11861a48 | 531 | |
64e53b28 | 532 | let mut gc_status = GarbageCollectionStatus::default(); |
f6b1d1cc WB |
533 | gc_status.upid = Some(upid.to_string()); |
534 | ||
535 | crate::task_log!(worker, "Start GC phase1 (mark used chunks)"); | |
536 | ||
537 | self.mark_used_chunks(&mut gc_status, worker)?; | |
538 | ||
539 | crate::task_log!(worker, "Start GC phase2 (sweep unused chunks)"); | |
540 | self.chunk_store.sweep_unused_chunks( | |
541 | oldest_writer, | |
542 | phase1_start_time, | |
543 | &mut gc_status, | |
544 | worker, | |
545 | )?; | |
546 | ||
547 | crate::task_log!( | |
548 | worker, | |
549 | "Removed garbage: {}", | |
550 | HumanByte::from(gc_status.removed_bytes), | |
551 | ); | |
552 | crate::task_log!(worker, "Removed chunks: {}", gc_status.removed_chunks); | |
cf459b19 | 553 | if gc_status.pending_bytes > 0 { |
f6b1d1cc WB |
554 | crate::task_log!( |
555 | worker, | |
556 | "Pending removals: {} (in {} chunks)", | |
557 | HumanByte::from(gc_status.pending_bytes), | |
558 | gc_status.pending_chunks, | |
559 | ); | |
cf459b19 | 560 | } |
a9767cf7 | 561 | if gc_status.removed_bad > 0 { |
f6b1d1cc | 562 | crate::task_log!(worker, "Removed bad files: {}", gc_status.removed_bad); |
a9767cf7 | 563 | } |
cf459b19 | 564 | |
f6b1d1cc WB |
565 | crate::task_log!( |
566 | worker, | |
567 | "Original data usage: {}", | |
568 | HumanByte::from(gc_status.index_data_bytes), | |
569 | ); | |
868c5852 DM |
570 | |
571 | if gc_status.index_data_bytes > 0 { | |
49a92084 | 572 | let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64; |
f6b1d1cc WB |
573 | crate::task_log!( |
574 | worker, | |
575 | "On-Disk usage: {} ({:.2}%)", | |
576 | HumanByte::from(gc_status.disk_bytes), | |
577 | comp_per, | |
578 | ); | |
868c5852 DM |
579 | } |
580 | ||
f6b1d1cc | 581 | crate::task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks); |
868c5852 DM |
582 | |
583 | if gc_status.disk_chunks > 0 { | |
584 | let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64); | |
f6b1d1cc | 585 | crate::task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk)); |
868c5852 | 586 | } |
64e53b28 | 587 | |
b683fd58 DC |
588 | if let Ok(serialized) = serde_json::to_string(&gc_status) { |
589 | let mut path = self.base_path(); | |
590 | path.push(".gc-status"); | |
591 | ||
592 | let backup_user = crate::backup::backup_user()?; | |
593 | let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); | |
594 | // set the correct owner/group/permissions while saving file | |
595 | // owner(rw) = backup, group(r)= backup | |
596 | let options = CreateOptions::new() | |
597 | .perm(mode) | |
598 | .owner(backup_user.uid) | |
599 | .group(backup_user.gid); | |
600 | ||
601 | // ignore errors | |
602 | let _ = replace_file(path, serialized.as_bytes(), options); | |
603 | } | |
604 | ||
f2b99c34 DM |
605 | *self.last_gc_status.lock().unwrap() = gc_status; |
606 | ||
64e53b28 | 607 | } else { |
d4b59ae0 | 608 | bail!("Start GC failed - (already running/locked)"); |
64e53b28 | 609 | } |
3d5c11e5 DM |
610 | |
611 | Ok(()) | |
612 | } | |
3b7ade9e | 613 | |
1cf5178a DM |
614 | pub fn try_shared_chunk_store_lock(&self) -> Result<tools::ProcessLockSharedGuard, Error> { |
615 | self.chunk_store.try_shared_lock() | |
616 | } | |
617 | ||
d48a9955 DM |
618 | pub fn chunk_path(&self, digest:&[u8; 32]) -> (PathBuf, String) { |
619 | self.chunk_store.chunk_path(digest) | |
620 | } | |
621 | ||
2585a8a4 DM |
622 | pub fn cond_touch_chunk(&self, digest: &[u8; 32], fail_if_not_exist: bool) -> Result<bool, Error> { |
623 | self.chunk_store.cond_touch_chunk(digest, fail_if_not_exist) | |
624 | } | |
625 | ||
f98ac774 | 626 | pub fn insert_chunk( |
3b7ade9e | 627 | &self, |
4ee8f53d DM |
628 | chunk: &DataBlob, |
629 | digest: &[u8; 32], | |
3b7ade9e | 630 | ) -> Result<(bool, u64), Error> { |
4ee8f53d | 631 | self.chunk_store.insert_chunk(chunk, digest) |
3b7ade9e | 632 | } |
60f9a6ea | 633 | |
39f18b30 | 634 | pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> { |
60f9a6ea DM |
635 | let mut path = self.base_path(); |
636 | path.push(backup_dir.relative_path()); | |
637 | path.push(filename); | |
638 | ||
39f18b30 DM |
639 | proxmox::try_block!({ |
640 | let mut file = std::fs::File::open(&path)?; | |
641 | DataBlob::load_from_reader(&mut file) | |
642 | }).map_err(|err| format_err!("unable to load blob '{:?}' - {}", path, err)) | |
643 | } | |
e4439025 DM |
644 | |
645 | ||
39f18b30 DM |
646 | pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> { |
647 | ||
648 | let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest); | |
649 | ||
650 | proxmox::try_block!({ | |
651 | let mut file = std::fs::File::open(&chunk_path)?; | |
652 | DataBlob::load_from_reader(&mut file) | |
653 | }).map_err(|err| format_err!( | |
654 | "store '{}', unable to load chunk '{}' - {}", | |
655 | self.name(), | |
656 | digest_str, | |
657 | err, | |
658 | )) | |
1a374fcf SR |
659 | } |
660 | ||
661 | fn lock_manifest( | |
662 | &self, | |
663 | backup_dir: &BackupDir, | |
664 | ) -> Result<File, Error> { | |
665 | let mut path = self.base_path(); | |
666 | path.push(backup_dir.relative_path()); | |
667 | path.push(&MANIFEST_LOCK_NAME); | |
668 | ||
669 | // update_manifest should never take a long time, so if someone else has | |
670 | // the lock we can simply block a bit and should get it soon | |
671 | open_file_locked(&path, Duration::from_secs(5), true) | |
672 | .map_err(|err| { | |
673 | format_err!( | |
674 | "unable to acquire manifest lock {:?} - {}", &path, err | |
675 | ) | |
676 | }) | |
677 | } | |
e4439025 | 678 | |
1a374fcf | 679 | /// Load the manifest without a lock. Must not be written back. |
521a0acb WB |
680 | pub fn load_manifest( |
681 | &self, | |
682 | backup_dir: &BackupDir, | |
ff86ef00 | 683 | ) -> Result<(BackupManifest, u64), Error> { |
39f18b30 DM |
684 | let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?; |
685 | let raw_size = blob.raw_size(); | |
60f9a6ea | 686 | let manifest = BackupManifest::try_from(blob)?; |
ff86ef00 | 687 | Ok((manifest, raw_size)) |
60f9a6ea | 688 | } |
e4439025 | 689 | |
1a374fcf SR |
690 | /// Update the manifest of the specified snapshot. Never write a manifest directly, |
691 | /// only use this method - anything else may break locking guarantees. | |
692 | pub fn update_manifest( | |
e4439025 DM |
693 | &self, |
694 | backup_dir: &BackupDir, | |
1a374fcf | 695 | update_fn: impl FnOnce(&mut BackupManifest), |
e4439025 | 696 | ) -> Result<(), Error> { |
1a374fcf SR |
697 | |
698 | let _guard = self.lock_manifest(backup_dir)?; | |
699 | let (mut manifest, _) = self.load_manifest(&backup_dir)?; | |
700 | ||
701 | update_fn(&mut manifest); | |
702 | ||
883aa6d5 | 703 | let manifest = serde_json::to_value(manifest)?; |
e4439025 DM |
704 | let manifest = serde_json::to_string_pretty(&manifest)?; |
705 | let blob = DataBlob::encode(manifest.as_bytes(), None, true)?; | |
706 | let raw_data = blob.raw_data(); | |
707 | ||
708 | let mut path = self.base_path(); | |
709 | path.push(backup_dir.relative_path()); | |
710 | path.push(MANIFEST_BLOB_NAME); | |
711 | ||
1a374fcf | 712 | // atomic replace invalidates flock - no other writes past this point! |
e4439025 DM |
713 | replace_file(&path, raw_data, CreateOptions::new())?; |
714 | ||
715 | Ok(()) | |
716 | } | |
0698f78d SR |
717 | |
718 | pub fn verify_new(&self) -> bool { | |
719 | self.verify_new | |
720 | } | |
529de6c7 | 721 | } |