tape/helpers/snapshot_reader: sort chunks by inode (per index)

sort the chunks we want to backup to tape by inode, to gain some
speed on spinning disks. this is done per index, not globally.

costs a bit memory, but not too much, about 16 bytes per chunk which
would mean ~4MiB for a 1TiB index with 4MiB chunks.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>

diff --git a/src/tape/helpers/snapshot_reader.rs b/src/tape/helpers/snapshot_reader.rs
index 7b272e3783e126de7f8e36423d18fc62d0bf806a..416c88c1499fc7e84a916a2f3508be1097353c3a 100644 (file)
--- a/src/tape/helpers/snapshot_reader.rs
+++ b/src/tape/helpers/snapshot_reader.rs
@@ -107,7 +107,7 @@ impl SnapshotReader {
 pub struct SnapshotChunkIterator<'a> {
     snapshot_reader: &'a SnapshotReader,
     todo_list: Vec<String>,
-    current_index: Option<(Arc<Box<dyn IndexFile>>, usize)>,
+    current_index: Option<(Arc<Box<dyn IndexFile + Send>>, usize, Vec<(usize, u64)>)>,
 }
 
 impl <'a> Iterator for SnapshotChunkIterator<'a> {
@@ -119,20 +119,26 @@ impl <'a> Iterator for SnapshotChunkIterator<'a> {
                 if self.current_index.is_none() {
                     if let Some(filename) = self.todo_list.pop() {
                         let file = self.snapshot_reader.open_file(&filename)?;
-                        let index: Box<dyn IndexFile> = match archive_type(&filename)? {
+                        let index: Box<dyn IndexFile + Send> = match archive_type(&filename)? {
                             ArchiveType::FixedIndex => Box::new(FixedIndexReader::new(file)?),
                             ArchiveType::DynamicIndex => Box::new(DynamicIndexReader::new(file)?),
                             _ => bail!("SnapshotChunkIterator: got unknown file type - internal error"),
                         };
-                        self.current_index = Some((Arc::new(index), 0));
+
+                        let datastore =
+                            DataStore::lookup_datastore(self.snapshot_reader.datastore_name())?;
+                        let order = datastore.get_chunks_in_order(&index, |_| false, |_| Ok(()))?;
+
+                        self.current_index = Some((Arc::new(index), 0, order));
                     } else {
                         return Ok(None);
                     }
                 }
-                let (index, pos) = self.current_index.take().unwrap();
-                if pos < index.index_count() {
-                    let digest = *index.index_digest(pos).unwrap();
-                    self.current_index = Some((index, pos + 1));
+                let (index, pos, list) = self.current_index.take().unwrap();
+                if pos < list.len() {
+                    let (real_pos, _) = list[pos];
+                    let digest = *index.index_digest(real_pos).unwrap();
+                    self.current_index = Some((index, pos + 1, list));
                     return Ok(Some(digest));
                 } else {
                     // pop next index