use super::chunk_stat::*;
use super::chunk_store::*;
-use super::{IndexFile, ChunkReadInfo};
-use crate::tools::{self, epoch_now_u64};
+use super::{ChunkReadInfo, IndexFile};
+use crate::tools;
-use chrono::{Local, TimeZone};
use std::fs::File;
use std::io::Write;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::Arc;
-use super::read_chunk::*;
use super::ChunkInfo;
use proxmox::tools::io::ReadExt;
pub struct FixedIndexHeader {
pub magic: [u8; 8],
pub uuid: [u8; 16],
- pub ctime: u64,
+ pub ctime: i64,
/// Sha256 over the index ``SHA256(digest1||digest2||...)``
pub index_csum: [u8; 32],
pub size: u64,
index_length: usize,
index: *mut u8,
pub uuid: [u8; 16],
- pub ctime: u64,
+ pub ctime: i64,
pub index_csum: [u8; 32],
}
pub fn open(path: &Path) -> Result<Self, Error> {
File::open(path)
.map_err(Error::from)
- .and_then(|file| Self::new(file))
+ .and_then(Self::new)
.map_err(|err| format_err!("Unable to open fixed index {:?} - {}", path, err))
}
pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
- if let Err(err) =
- nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
- {
- bail!("unable to get shared lock - {}", err);
- }
-
file.seek(SeekFrom::Start(0))?;
let header_size = std::mem::size_of::<FixedIndexHeader>();
+
+ let stat = match nix::sys::stat::fstat(file.as_raw_fd()) {
+ Ok(stat) => stat,
+ Err(err) => bail!("fstat failed - {}", err),
+ };
+
+ let size = stat.st_size as usize;
+
+ if size < header_size {
+ bail!("index too small ({})", stat.st_size);
+ }
+
let header: Box<FixedIndexHeader> = unsafe { file.read_host_value_boxed()? };
if header.magic != super::FIXED_SIZED_CHUNK_INDEX_1_0 {
}
let size = u64::from_le(header.size);
- let ctime = u64::from_le(header.ctime);
+ let ctime = i64::from_le(header.ctime);
let chunk_size = u64::from_le(header.chunk_size);
let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
let index_size = index_length * 32;
- let rawfd = file.as_raw_fd();
-
- let stat = match nix::sys::stat::fstat(rawfd) {
- Ok(stat) => stat,
- Err(err) => bail!("fstat failed - {}", err),
- };
-
let expected_index_size = (stat.st_size as usize) - header_size;
if index_size != expected_index_size {
bail!(
Ok(())
}
- #[inline]
- fn chunk_end(&self, pos: usize) -> u64 {
- if pos >= self.index_length {
- panic!("chunk index out of range");
- }
-
- let end = ((pos + 1) * self.chunk_size) as u64;
- if end > self.size {
- self.size
- } else {
- end
- }
- }
-
pub fn print_info(&self) {
println!("Size: {}", self.size);
println!("ChunkSize: {}", self.chunk_size);
- println!(
- "CTime: {}",
- Local.timestamp(self.ctime as i64, 0).format("%c")
- );
+
+ let mut ctime_str = self.ctime.to_string();
+ if let Ok(s) = proxmox::tools::time::strftime_local("%c", self.ctime) {
+ ctime_str = s;
+ }
+
+ println!("CTime: {}", ctime_str);
println!("UUID: {:?}", self.uuid);
}
}
digest: *digest,
})
}
+
+ fn compute_csum(&self) -> ([u8; 32], u64) {
+ let mut csum = openssl::sha::Sha256::new();
+ let mut chunk_end = 0;
+ for pos in 0..self.index_count() {
+ let info = self.chunk_info(pos).unwrap();
+ chunk_end = info.range.end;
+ csum.update(&info.digest);
+ }
+ let csum = csum.finish();
+
+ (csum, chunk_end)
+ }
+
+ fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
+ if offset >= self.size {
+ return None;
+ }
+
+ Some((
+ (offset / self.chunk_size as u64) as usize,
+ offset & (self.chunk_size - 1) as u64, // fast modulo, valid for 2^x chunk_size
+ ))
+ }
}
pub struct FixedIndexWriter {
index_length: usize,
index: *mut u8,
pub uuid: [u8; 16],
- pub ctime: u64,
+ pub ctime: i64,
}
// `index` is mmap()ed which cannot be thread-local so should be sendable
panic!("got unexpected header size");
}
- let ctime = epoch_now_u64()?;
+ let ctime = proxmox::tools::time::epoch_i64();
let uuid = Uuid::generate();
let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0;
- header.ctime = u64::to_le(ctime);
+ header.ctime = i64::to_le(ctime);
header.size = u64::to_le(size as u64);
header.chunk_size = u64::to_le(chunk_size as u64);
header.uuid = *uuid.as_bytes();
Ok(())
}
}
-
-pub struct BufferedFixedReader<S> {
- store: S,
- index: FixedIndexReader,
- archive_size: u64,
- read_buffer: Vec<u8>,
- buffered_chunk_idx: usize,
- buffered_chunk_start: u64,
- read_offset: u64,
-}
-
-impl<S: ReadChunk> BufferedFixedReader<S> {
- pub fn new(index: FixedIndexReader, store: S) -> Self {
- let archive_size = index.size;
- Self {
- store,
- index,
- archive_size,
- read_buffer: Vec::with_capacity(1024 * 1024),
- buffered_chunk_idx: 0,
- buffered_chunk_start: 0,
- read_offset: 0,
- }
- }
-
- pub fn archive_size(&self) -> u64 {
- self.archive_size
- }
-
- fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
- let index = &self.index;
- let info = match index.chunk_info(idx) {
- Some(info) => info,
- None => bail!("chunk index out of range"),
- };
-
- // fixme: avoid copy
-
- let data = self.store.read_chunk(&info.digest)?;
- let size = info.range.end - info.range.start;
- if size != data.len() as u64 {
- bail!("read chunk with wrong size ({} != {}", size, data.len());
- }
-
- self.read_buffer.clear();
- self.read_buffer.extend_from_slice(&data);
-
- self.buffered_chunk_idx = idx;
-
- self.buffered_chunk_start = info.range.start as u64;
- Ok(())
- }
-}
-
-impl<S: ReadChunk> crate::tools::BufferedRead for BufferedFixedReader<S> {
- fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error> {
- if offset == self.archive_size {
- return Ok(&self.read_buffer[0..0]);
- }
-
- let buffer_len = self.read_buffer.len();
- let index = &self.index;
-
- // optimization for sequential read
- if buffer_len > 0
- && ((self.buffered_chunk_idx + 1) < index.index_length)
- && (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
- {
- let next_idx = self.buffered_chunk_idx + 1;
- let next_end = index.chunk_end(next_idx);
- if offset < next_end {
- self.buffer_chunk(next_idx)?;
- let buffer_offset = (offset - self.buffered_chunk_start) as usize;
- return Ok(&self.read_buffer[buffer_offset..]);
- }
- }
-
- if (buffer_len == 0)
- || (offset < self.buffered_chunk_start)
- || (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
- {
- let idx = (offset / index.chunk_size as u64) as usize;
- self.buffer_chunk(idx)?;
- }
-
- let buffer_offset = (offset - self.buffered_chunk_start) as usize;
- Ok(&self.read_buffer[buffer_offset..])
- }
-}
-
-impl<S: ReadChunk> std::io::Read for BufferedFixedReader<S> {
- fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
- use crate::tools::BufferedRead;
- use std::io::{Error, ErrorKind};
-
- let data = match self.buffered_read(self.read_offset) {
- Ok(v) => v,
- Err(err) => return Err(Error::new(ErrorKind::Other, err.to_string())),
- };
-
- let n = if data.len() > buf.len() {
- buf.len()
- } else {
- data.len()
- };
-
- unsafe {
- std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n);
- }
-
- self.read_offset += n as u64;
-
- Ok(n)
- }
-}
-
-impl<S: ReadChunk> Seek for BufferedFixedReader<S> {
- fn seek(&mut self, pos: SeekFrom) -> Result<u64, std::io::Error> {
- let new_offset = match pos {
- SeekFrom::Start(start_offset) => start_offset as i64,
- SeekFrom::End(end_offset) => (self.archive_size as i64) + end_offset,
- SeekFrom::Current(offset) => (self.read_offset as i64) + offset,
- };
-
- use std::io::{Error, ErrorKind};
- if (new_offset < 0) || (new_offset > (self.archive_size as i64)) {
- return Err(Error::new(
- ErrorKind::Other,
- format!(
- "seek is out of range {} ([0..{}])",
- new_offset, self.archive_size
- ),
- ));
- }
- self.read_offset = new_offset as u64;
-
- Ok(self.read_offset)
- }
-}
projects / proxmox-backup.git / blobdiff