]> git.proxmox.com Git - proxmox-backup.git/blob - src/tape/media_pool.rs
tape: media_pool: implement guess_next_writable_media()
[proxmox-backup.git] / src / tape / media_pool.rs
1 //! Media Pool
2 //!
3 //! A set of backup medias.
4 //!
5 //! This struct manages backup media state during backup. The main
6 //! purpose is to allocate media sets and assign new tapes to it.
7 //!
8 //!
9
10 use std::path::{PathBuf, Path};
11
12 use anyhow::{bail, Error};
13 use ::serde::{Deserialize, Serialize};
14
15 use proxmox::tools::Uuid;
16
17 use crate::{
18 backup::{Fingerprint, BackupLockGuard},
19 api2::types::{
20 MediaStatus,
21 MediaLocation,
22 MediaSetPolicy,
23 RetentionPolicy,
24 MediaPoolConfig,
25 },
26 tools::systemd::time::compute_next_event,
27 tape::{
28 MediaId,
29 MediaSet,
30 Inventory,
31 MediaCatalog,
32 lock_media_set,
33 lock_media_pool,
34 lock_unassigned_media_pool,
35 file_formats::{
36 MediaLabel,
37 MediaSetLabel,
38 },
39 }
40 };
41
42 /// Media Pool
43 pub struct MediaPool {
44
45 name: String,
46 state_path: PathBuf,
47
48 media_set_policy: MediaSetPolicy,
49 retention: RetentionPolicy,
50
51 changer_name: Option<String>,
52 force_media_availability: bool,
53
54 // Set this if you do not need to allocate writeable media - this
55 // is useful for list_media()
56 no_media_set_locking: bool,
57
58 encrypt_fingerprint: Option<Fingerprint>,
59
60 inventory: Inventory,
61
62 current_media_set: MediaSet,
63 current_media_set_lock: Option<BackupLockGuard>,
64 }
65
66 impl MediaPool {
67
68 /// Creates a new instance
69 ///
70 /// If you specify a `changer_name`, only media accessible via
71 /// that changer is considered available. If you pass `None` for
72 /// `changer`, all offline media is considered available (backups
73 /// to standalone drives may not use media from inside a tape
74 /// library).
75 pub fn new(
76 name: &str,
77 state_path: &Path,
78 media_set_policy: MediaSetPolicy,
79 retention: RetentionPolicy,
80 changer_name: Option<String>,
81 encrypt_fingerprint: Option<Fingerprint>,
82 no_media_set_locking: bool, // for list_media()
83 ) -> Result<Self, Error> {
84
85 let _pool_lock = if no_media_set_locking {
86 None
87 } else {
88 Some(lock_media_pool(state_path, name)?)
89 };
90
91 let inventory = Inventory::load(state_path)?;
92
93 let current_media_set = match inventory.latest_media_set(name) {
94 Some(set_uuid) => inventory.compute_media_set_members(&set_uuid)?,
95 None => MediaSet::new(),
96 };
97
98 let current_media_set_lock = if no_media_set_locking {
99 None
100 } else {
101 Some(lock_media_set(state_path, current_media_set.uuid(), None)?)
102 };
103
104 Ok(MediaPool {
105 name: String::from(name),
106 state_path: state_path.to_owned(),
107 media_set_policy,
108 retention,
109 changer_name,
110 inventory,
111 current_media_set,
112 current_media_set_lock,
113 encrypt_fingerprint,
114 force_media_availability: false,
115 no_media_set_locking,
116 })
117 }
118
119 /// Pretend all Online(x) and Offline media is available
120 ///
121 /// Only media in Vault(y) is considered unavailable.
122 pub fn force_media_availability(&mut self) {
123 self.force_media_availability = true;
124 }
125
126 /// Returns the the current media set
127 pub fn current_media_set(&self) -> &MediaSet {
128 &self.current_media_set
129 }
130
131 /// Creates a new instance using the media pool configuration
132 pub fn with_config(
133 state_path: &Path,
134 config: &MediaPoolConfig,
135 changer_name: Option<String>,
136 no_media_set_locking: bool, // for list_media()
137 ) -> Result<Self, Error> {
138
139 let allocation = config.allocation.clone().unwrap_or_else(|| String::from("continue")).parse()?;
140
141 let retention = config.retention.clone().unwrap_or_else(|| String::from("keep")).parse()?;
142
143 let encrypt_fingerprint = match config.encrypt {
144 Some(ref fingerprint) => Some(fingerprint.parse()?),
145 None => None,
146 };
147
148 MediaPool::new(
149 &config.name,
150 state_path,
151 allocation,
152 retention,
153 changer_name,
154 encrypt_fingerprint,
155 no_media_set_locking,
156 )
157 }
158
159 /// Returns the pool name
160 pub fn name(&self) -> &str {
161 &self.name
162 }
163
164 /// Returns encryption settings
165 pub fn encrypt_fingerprint(&self) -> Option<Fingerprint> {
166 self.encrypt_fingerprint.clone()
167 }
168
169 pub fn set_media_status_damaged(&mut self, uuid: &Uuid) -> Result<(), Error> {
170 self.inventory.set_media_status_damaged(uuid)
171 }
172
173 fn compute_media_state(&self, media_id: &MediaId) -> (MediaStatus, MediaLocation) {
174
175 let (status, location) = self.inventory.status_and_location(&media_id.label.uuid);
176
177 match status {
178 MediaStatus::Full | MediaStatus::Damaged | MediaStatus::Retired => {
179 return (status, location);
180 }
181 MediaStatus::Unknown | MediaStatus::Writable => {
182 /* possibly writable - fall through to check */
183 }
184 }
185
186 let set = match media_id.media_set_label {
187 None => return (MediaStatus::Writable, location), // not assigned to any pool
188 Some(ref set) => set,
189 };
190
191 if set.pool != self.name { // should never trigger
192 return (MediaStatus::Unknown, location); // belong to another pool
193 }
194 if set.uuid.as_ref() == [0u8;16] { // not assigned to any pool
195 return (MediaStatus::Writable, location);
196 }
197
198 if &set.uuid != self.current_media_set.uuid() {
199 return (MediaStatus::Full, location); // assume FULL
200 }
201
202 // media is member of current set
203 if self.current_media_set.is_last_media(&media_id.label.uuid) {
204 (MediaStatus::Writable, location) // last set member is writable
205 } else {
206 (MediaStatus::Full, location)
207 }
208 }
209
210 /// Returns the 'MediaId' with associated state
211 pub fn lookup_media(&self, uuid: &Uuid) -> Result<BackupMedia, Error> {
212 let media_id = match self.inventory.lookup_media(uuid) {
213 None => bail!("unable to lookup media {}", uuid),
214 Some(media_id) => media_id.clone(),
215 };
216
217 if let Some(ref set) = media_id.media_set_label {
218 if set.pool != self.name {
219 bail!("media does not belong to pool ({} != {})", set.pool, self.name);
220 }
221 }
222
223 let (status, location) = self.compute_media_state(&media_id);
224
225 Ok(BackupMedia::with_media_id(
226 media_id,
227 location,
228 status,
229 ))
230 }
231
232 /// List all media associated with this pool
233 pub fn list_media(&self) -> Vec<BackupMedia> {
234 let media_id_list = self.inventory.list_pool_media(&self.name);
235
236 media_id_list.into_iter()
237 .map(|media_id| {
238 let (status, location) = self.compute_media_state(&media_id);
239 BackupMedia::with_media_id(
240 media_id,
241 location,
242 status,
243 )
244 })
245 .collect()
246 }
247
248 /// Set media status to FULL.
249 pub fn set_media_status_full(&mut self, uuid: &Uuid) -> Result<(), Error> {
250 let media = self.lookup_media(uuid)?; // check if media belongs to this pool
251 if media.status() != &MediaStatus::Full {
252 self.inventory.set_media_status_full(uuid)?;
253 }
254 Ok(())
255 }
256
257 /// Make sure the current media set is usable for writing
258 ///
259 /// If not, starts a new media set. Also creates a new
260 /// set if media_set_policy implies it, or if 'force' is true.
261 ///
262 /// Note: We also call this in list_media to compute correct media
263 /// status, so this must not change persistent/saved state.
264 ///
265 /// Returns the reason why we started a new media set (if we do)
266 pub fn start_write_session(
267 &mut self,
268 current_time: i64,
269 force: bool,
270 ) -> Result<Option<String>, Error> {
271
272 let _pool_lock = if self.no_media_set_locking {
273 None
274 } else {
275 Some(lock_media_pool(&self.state_path, &self.name)?)
276 };
277
278 self.inventory.reload()?;
279
280 let mut create_new_set = if force {
281 Some(String::from("forced"))
282 } else {
283 match self.current_set_usable() {
284 Err(err) => {
285 Some(err.to_string())
286 }
287 Ok(_) => None,
288 }
289 };
290
291 if create_new_set.is_none() {
292 match &self.media_set_policy {
293 MediaSetPolicy::AlwaysCreate => {
294 create_new_set = Some(String::from("policy is AlwaysCreate"));
295 }
296 MediaSetPolicy::CreateAt(event) => {
297 if let Some(set_start_time) = self.inventory.media_set_start_time(&self.current_media_set.uuid()) {
298 if let Ok(Some(alloc_time)) = compute_next_event(event, set_start_time as i64, false) {
299 if current_time >= alloc_time {
300 create_new_set = Some(String::from("policy CreateAt event triggered"));
301 }
302 }
303 }
304 }
305 MediaSetPolicy::ContinueCurrent => { /* do nothing here */ }
306 }
307 }
308
309 if create_new_set.is_some() {
310 let media_set = MediaSet::new();
311
312 let current_media_set_lock = if self.no_media_set_locking {
313 None
314 } else {
315 Some(lock_media_set(&self.state_path, media_set.uuid(), None)?)
316 };
317
318 self.current_media_set_lock = current_media_set_lock;
319 self.current_media_set = media_set;
320 }
321
322 Ok(create_new_set)
323 }
324
325 /// List media in current media set
326 pub fn current_media_list(&self) -> Result<Vec<&Uuid>, Error> {
327 let mut list = Vec::new();
328 for opt_uuid in self.current_media_set.media_list().iter() {
329 match opt_uuid {
330 Some(ref uuid) => list.push(uuid),
331 None => bail!("current_media_list failed - media set is incomplete"),
332 }
333 }
334 Ok(list)
335 }
336
337 // tests if the media data is considered as expired at specified time
338 pub fn media_is_expired(&self, media: &BackupMedia, current_time: i64) -> bool {
339 if media.status() != &MediaStatus::Full {
340 return false;
341 }
342
343 let expire_time = self.inventory.media_expire_time(
344 media.id(), &self.media_set_policy, &self.retention);
345
346 current_time >= expire_time
347 }
348
349 // check if a location is considered on site
350 pub fn location_is_available(&self, location: &MediaLocation) -> bool {
351 match location {
352 MediaLocation::Online(name) => {
353 if self.force_media_availability {
354 true
355 } else {
356 if let Some(ref changer_name) = self.changer_name {
357 name == changer_name
358 } else {
359 // a standalone drive cannot use media currently inside a library
360 false
361 }
362 }
363 }
364 MediaLocation::Offline => {
365 if self.force_media_availability {
366 true
367 } else {
368 // consider available for standalone drives
369 self.changer_name.is_none()
370 }
371 }
372 MediaLocation::Vault(_) => false,
373 }
374 }
375
376 fn add_media_to_current_set(&mut self, mut media_id: MediaId, current_time: i64) -> Result<(), Error> {
377
378 if self.current_media_set_lock.is_none() {
379 bail!("add_media_to_current_set: media set is not locked - internal error");
380 }
381
382 let seq_nr = self.current_media_set.media_list().len() as u64;
383
384 let pool = self.name.clone();
385
386 let encrypt_fingerprint = self.encrypt_fingerprint();
387
388 let set = MediaSetLabel::with_data(
389 &pool,
390 self.current_media_set.uuid().clone(),
391 seq_nr,
392 current_time,
393 encrypt_fingerprint,
394 );
395
396 media_id.media_set_label = Some(set);
397
398 let uuid = media_id.label.uuid.clone();
399
400 MediaCatalog::overwrite(&self.state_path, &media_id, false)?; // overwite catalog
401 let clear_media_status = true; // remove Full status
402 self.inventory.store(media_id, clear_media_status)?; // store persistently
403
404 self.current_media_set.add_media(uuid);
405
406 Ok(())
407 }
408
409 // Get next unassigned media (media not assigned to any pool)
410 pub fn next_unassigned_media(&self, media_list: &[MediaId]) -> Option<MediaId> {
411 let mut free_media = Vec::new();
412
413 for media_id in media_list {
414
415 let (status, location) = self.compute_media_state(&media_id);
416 if media_id.media_set_label.is_some() { continue; } // should not happen
417
418 if !self.location_is_available(&location) {
419 continue;
420 }
421
422 // only consider writable media
423 if status != MediaStatus::Writable { continue; }
424
425 free_media.push(media_id);
426 }
427
428 // sort free_media, newest first -> oldest last
429 free_media.sort_unstable_by(|a, b| {
430 let mut res = b.label.ctime.cmp(&a.label.ctime);
431 if res == std::cmp::Ordering::Equal {
432 res = b.label.label_text.cmp(&a.label.label_text);
433 }
434 res
435 });
436
437 free_media.pop().map(|e| e.clone())
438 }
439
440 // Get next empty media
441 pub fn next_empty_media(&self, media_list: &[BackupMedia]) -> Option<MediaId> {
442 let mut empty_media = Vec::new();
443
444 for media in media_list.into_iter() {
445 if !self.location_is_available(media.location()) {
446 continue;
447 }
448 // already part of a media set?
449 if media.media_set_label().is_none() {
450 // only consider writable empty media
451 if media.status() == &MediaStatus::Writable {
452 empty_media.push(media);
453 }
454 }
455 }
456
457 // sort empty_media, newest first -> oldest last
458 empty_media.sort_unstable_by(|a, b| {
459 let mut res = b.label().ctime.cmp(&a.label().ctime);
460 if res == std::cmp::Ordering::Equal {
461 res = b.label().label_text.cmp(&a.label().label_text);
462 }
463 res
464 });
465
466 empty_media.pop().map(|e| e.clone().into_id())
467 }
468
469 // Get next expired media
470 pub fn next_expired_media(&self, current_time: i64, media_list: &[BackupMedia]) -> Option<MediaId> {
471 let mut used_media = Vec::new();
472
473 for media in media_list.into_iter() {
474 if !self.location_is_available(media.location()) {
475 continue;
476 }
477 // already part of a media set?
478 if media.media_set_label().is_some() {
479 used_media.push(media);
480 }
481 }
482
483 let mut expired_media = Vec::new();
484
485 for media in used_media.into_iter() {
486 if let Some(set) = media.media_set_label() {
487 if &set.uuid == self.current_media_set.uuid() {
488 continue;
489 }
490 } else {
491 continue;
492 }
493
494 if !self.media_is_expired(&media, current_time) {
495 continue;
496 }
497
498 expired_media.push(media);
499 }
500
501 // sort expired_media, newest first -> oldest last
502 expired_media.sort_unstable_by(|a, b| {
503 let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime);
504 if res == std::cmp::Ordering::Equal {
505 res = b.label().label_text.cmp(&a.label().label_text);
506 }
507 res
508 });
509
510 if self.no_media_set_locking {
511 expired_media.pop().map(|e| e.clone().into_id())
512 } else {
513 while let Some(media) = expired_media.pop() {
514 // check if we can modify the media-set (i.e. skip
515 // media used by a restore job)
516 if let Ok(_media_set_lock) = lock_media_set(
517 &self.state_path,
518 &media.media_set_label().unwrap().uuid,
519 Some(std::time::Duration::new(0, 0)), // do not wait
520 ) {
521 return Some(media.clone().into_id());
522 }
523 }
524 None
525 }
526 }
527
528 /// Guess next writable media
529 ///
530 /// Like alloc_writable_media(), but does not really allocate
531 /// anything (thus it does not need any locks)
532 // Note: Please keep in sync with alloc_writable_media()
533 pub fn guess_next_writable_media(&self, current_time: i64) -> Result<MediaId, Error> {
534 let last_is_writable = self.current_set_usable()?;
535
536 if last_is_writable {
537 let last_uuid = self.current_media_set.last_media_uuid().unwrap();
538 let media = self.lookup_media(last_uuid)?;
539 return Ok(media.into_id());
540 }
541
542 let media_list = self.list_media();
543 if let Some(media_id) = self.next_empty_media(&media_list) {
544 return Ok(media_id);
545 }
546
547 if let Some(media_id) = self.next_expired_media(current_time, &media_list) {
548 return Ok(media_id);
549 }
550
551 let unassigned_list = self.inventory.list_unassigned_media();
552
553 if let Some(media_id) = self.next_unassigned_media(&unassigned_list) {
554 return Ok(media_id);
555 }
556
557 bail!("guess_next_writable_media in pool '{}' failed: no usable media found", self.name());
558 }
559
560 /// Allocates a writable media to the current media set
561 // Note: Please keep in sync with guess_next_writable_media()
562 pub fn alloc_writable_media(&mut self, current_time: i64) -> Result<Uuid, Error> {
563
564 if self.current_media_set_lock.is_none() {
565 bail!("alloc_writable_media: media set is not locked - internal error");
566 }
567
568 let last_is_writable = self.current_set_usable()?;
569
570 if last_is_writable {
571 let last_uuid = self.current_media_set.last_media_uuid().unwrap();
572 let media = self.lookup_media(last_uuid)?;
573 return Ok(media.uuid().clone());
574 }
575
576 { // limit pool lock scope
577 let _pool_lock = lock_media_pool(&self.state_path, &self.name)?;
578
579 self.inventory.reload()?;
580
581 let media_list = self.list_media();
582
583 // try to find empty media in pool, add to media set
584
585 if let Some(media_id) = self.next_empty_media(&media_list) {
586 // found empty media, add to media set an use it
587 println!("found empty media '{}'", media_id.label.label_text);
588 let uuid = media_id.label.uuid.clone();
589 self.add_media_to_current_set(media_id, current_time)?;
590 return Ok(uuid);
591 }
592
593 println!("no empty media in pool, try to reuse expired media");
594
595 if let Some(media_id) = self.next_expired_media(current_time, &media_list) {
596 // found expired media, add to media set an use it
597 println!("reuse expired media '{}'", media_id.label.label_text);
598 let uuid = media_id.label.uuid.clone();
599 self.add_media_to_current_set(media_id, current_time)?;
600 return Ok(uuid);
601 }
602 }
603
604 println!("no empty or expired media in pool, try to find unassigned/free media");
605
606 // try unassigned media
607 let _lock = lock_unassigned_media_pool(&self.state_path)?;
608
609 self.inventory.reload()?;
610
611 let unassigned_list = self.inventory.list_unassigned_media();
612
613 if let Some(media_id) = self.next_unassigned_media(&unassigned_list) {
614 println!("use free/unassigned media '{}'", media_id.label.label_text);
615 let uuid = media_id.label.uuid.clone();
616 self.add_media_to_current_set(media_id, current_time)?;
617 return Ok(uuid);
618 }
619
620 bail!("alloc writable media in pool '{}' failed: no usable media found", self.name());
621 }
622
623 /// check if the current media set is usable for writing
624 ///
625 /// This does several consistency checks, and return if
626 /// the last media in the current set is in writable state.
627 ///
628 /// This return error when the media set must not be used any
629 /// longer because of consistency errors.
630 pub fn current_set_usable(&self) -> Result<bool, Error> {
631
632 let media_list = self.current_media_set.media_list();
633
634 let media_count = media_list.len();
635 if media_count == 0 {
636 return Ok(false);
637 }
638
639 let set_uuid = self.current_media_set.uuid();
640 let mut last_is_writable = false;
641
642 let mut last_enc: Option<Option<Fingerprint>> = None;
643
644 for (seq, opt_uuid) in media_list.iter().enumerate() {
645 let uuid = match opt_uuid {
646 None => bail!("media set is incomplete (missing media information)"),
647 Some(uuid) => uuid,
648 };
649 let media = self.lookup_media(uuid)?;
650 match media.media_set_label() {
651 Some(MediaSetLabel { seq_nr, uuid, ..}) if *seq_nr == seq as u64 && uuid == set_uuid => { /* OK */ },
652 Some(MediaSetLabel { seq_nr, uuid, ..}) if uuid == set_uuid => {
653 bail!("media sequence error ({} != {})", *seq_nr, seq);
654 },
655 Some(MediaSetLabel { uuid, ..}) => bail!("media owner error ({} != {}", uuid, set_uuid),
656 None => bail!("media owner error (no owner)"),
657 }
658
659 if let Some(set) = media.media_set_label() { // always true here
660 if set.encryption_key_fingerprint != self.encrypt_fingerprint {
661 bail!("pool encryption key changed");
662 }
663 match last_enc {
664 None => {
665 last_enc = Some(set.encryption_key_fingerprint.clone());
666 }
667 Some(ref last_enc) => {
668 if last_enc != &set.encryption_key_fingerprint {
669 bail!("inconsistent media encryption key");
670 }
671 }
672 }
673 }
674
675 match media.status() {
676 MediaStatus::Full => { /* OK */ },
677 MediaStatus::Writable if (seq + 1) == media_count => {
678 let media_location = media.location();
679 if self.location_is_available(media_location) {
680 last_is_writable = true;
681 } else {
682 if let MediaLocation::Vault(vault) = media_location {
683 bail!("writable media offsite in vault '{}'", vault);
684 }
685 }
686 },
687 _ => bail!("unable to use media set - wrong media status {:?}", media.status()),
688 }
689 }
690
691 Ok(last_is_writable)
692 }
693
694 /// Generate a human readable name for the media set
695 pub fn generate_media_set_name(
696 &self,
697 media_set_uuid: &Uuid,
698 template: Option<String>,
699 ) -> Result<String, Error> {
700 self.inventory.generate_media_set_name(media_set_uuid, template)
701 }
702
703 }
704
705 /// Backup media
706 ///
707 /// Combines 'MediaId' with 'MediaLocation' and 'MediaStatus'
708 /// information.
709 #[derive(Debug,Serialize,Deserialize,Clone)]
710 pub struct BackupMedia {
711 /// Media ID
712 id: MediaId,
713 /// Media location
714 location: MediaLocation,
715 /// Media status
716 status: MediaStatus,
717 }
718
719 impl BackupMedia {
720
721 /// Creates a new instance
722 pub fn with_media_id(
723 id: MediaId,
724 location: MediaLocation,
725 status: MediaStatus,
726 ) -> Self {
727 Self { id, location, status }
728 }
729
730 /// Returns the media location
731 pub fn location(&self) -> &MediaLocation {
732 &self.location
733 }
734
735 /// Returns the media status
736 pub fn status(&self) -> &MediaStatus {
737 &self.status
738 }
739
740 /// Returns the media uuid
741 pub fn uuid(&self) -> &Uuid {
742 &self.id.label.uuid
743 }
744
745 /// Returns the media set label
746 pub fn media_set_label(&self) -> Option<&MediaSetLabel> {
747 self.id.media_set_label.as_ref()
748 }
749
750 /// Returns the media creation time
751 pub fn ctime(&self) -> i64 {
752 self.id.label.ctime
753 }
754
755 /// Updates the media set label
756 pub fn set_media_set_label(&mut self, set_label: MediaSetLabel) {
757 self.id.media_set_label = Some(set_label);
758 }
759
760 /// Returns the drive label
761 pub fn label(&self) -> &MediaLabel {
762 &self.id.label
763 }
764
765 /// Returns the media id (drive label + media set label)
766 pub fn id(&self) -> &MediaId {
767 &self.id
768 }
769
770 /// Returns the media id, consumes self)
771 pub fn into_id(self) -> MediaId {
772 self.id
773 }
774
775 /// Returns the media label (Barcode)
776 pub fn label_text(&self) -> &str {
777 &self.id.label.label_text
778 }
779 }