--- /dev/null
+pub struct ChunkStat {
+ pub size: u64,
+ pub compressed_size: u64,
+ pub disk_size: u64,
+
+ pub chunk_count: usize,
+ pub duplicate_chunks: usize,
+
+ start_time: std::time::SystemTime,
+}
+
+impl ChunkStat {
+
+ pub fn new(size: u64) -> Self {
+ ChunkStat {
+ size,
+ compressed_size: 0,
+ disk_size: 0,
+
+ chunk_count: 0,
+ duplicate_chunks: 0,
+
+ start_time: std::time::SystemTime::now(),
+ }
+ }
+}
+
+impl std::fmt::Debug for ChunkStat {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ let avg = ((self.size as f64)/(self.chunk_count as f64)) as usize;
+ let compression = (self.compressed_size*100)/(self.size as u64);
+ let rate = (self.disk_size*100)/(self.size as u64);
+
+ let elapsed = self.start_time.elapsed().unwrap();
+ let elapsed = (elapsed.as_secs() as f64) +
+ (elapsed.subsec_millis() as f64)/1000.0;
+
+ let write_speed = ((self.size as f64)/(1024.0*1024.0))/elapsed;
+
+ write!(f, "Size: {}, average chunk size: {}, compression rate: {}%, disk_size: {} ({}%), speed: {:.2} MB/s",
+ self.size, avg, compression, self.disk_size, rate, write_speed)
+ }
+}
pub mod catalog;
pub mod checksum_reader;
pub mod checksum_writer;
+pub mod chunk_stat;
pub mod chunk_store;
pub mod chunker;
pub mod crypt_config;
pub mod index;
pub mod key_derivation;
pub mod manifest;
+pub mod read_chunk;
pub mod task;
pub use backup_info::{BackupDir, BackupGroup, BackupInfo};
--- /dev/null
+use std::future::Future;
+use std::pin::Pin;
+
+use anyhow::Error;
+
+use crate::data_blob::DataBlob;
+
+/// The ReadChunk trait allows reading backup data chunks (local or remote)
+pub trait ReadChunk {
+ /// Returns the encoded chunk data
+ fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error>;
+
+ /// Returns the decoded chunk data
+ fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error>;
+}
+
+pub trait AsyncReadChunk: Send {
+ /// Returns the encoded chunk data
+ fn read_raw_chunk<'a>(
+ &'a self,
+ digest: &'a [u8; 32],
+ ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>>;
+
+ /// Returns the decoded chunk data
+ fn read_chunk<'a>(
+ &'a self,
+ digest: &'a [u8; 32],
+ ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>>;
+}
//! An async and concurrency safe data reader backed by a local LRU cache.
-use anyhow::Error;
-use futures::future::Future;
-use futures::ready;
-use tokio::io::{AsyncRead, AsyncSeek, ReadBuf};
-
+use std::future::Future;
use std::io::SeekFrom;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
-use super::{AsyncReadChunk, IndexFile};
-use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache};
+use anyhow::Error;
+use futures::ready;
+use tokio::io::{AsyncRead, AsyncSeek, ReadBuf};
+
use proxmox::io_format_err;
use proxmox::sys::error::io_err_other;
+use pbs_datastore::read_chunk::AsyncReadChunk;
+use super::IndexFile;
+use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache};
+
struct AsyncChunkCacher<T> {
reader: Arc<T>,
}
+++ /dev/null
-pub struct ChunkStat {
- pub size: u64,
- pub compressed_size: u64,
- pub disk_size: u64,
-
- pub chunk_count: usize,
- pub duplicate_chunks: usize,
-
- start_time: std::time::SystemTime,
-}
-
-impl ChunkStat {
-
- pub fn new(size: u64) -> Self {
- ChunkStat {
- size,
- compressed_size: 0,
- disk_size: 0,
-
- chunk_count: 0,
- duplicate_chunks: 0,
-
- start_time: std::time::SystemTime::now(),
- }
- }
-}
-
-impl std::fmt::Debug for ChunkStat {
- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
- let avg = ((self.size as f64)/(self.chunk_count as f64)) as usize;
- let compression = (self.compressed_size*100)/(self.size as u64);
- let rate = (self.disk_size*100)/(self.size as u64);
-
- let elapsed = self.start_time.elapsed().unwrap();
- let elapsed = (elapsed.as_secs() as f64) +
- (elapsed.subsec_millis() as f64)/1000.0;
-
- let write_speed = ((self.size as f64)/(1024.0*1024.0))/elapsed;
-
- write!(f, "Size: {}, average chunk size: {}, compression rate: {}%, disk_size: {} ({}%), speed: {:.2} MB/s",
- self.size, avg, compression, self.disk_size, rate, write_speed)
- }
-}
use proxmox::tools::mmap::Mmap;
use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
-use super::chunk_stat::ChunkStat;
-use super::chunk_store::ChunkStore;
-use super::index::ChunkReadInfo;
-use super::read_chunk::ReadChunk;
-use super::Chunker;
-use super::IndexFile;
-use super::{DataBlob, DataChunkBuilder};
-use crate::tools;
+use pbs_datastore::Chunker;
+use pbs_datastore::index::{IndexFile, ChunkReadInfo};
+use pbs_datastore::chunk_stat::ChunkStat;
+use pbs_datastore::data_blob::{DataBlob, DataChunkBuilder};
+use pbs_datastore::chunk_store::ChunkStore;
+use pbs_datastore::read_chunk::ReadChunk;
+use pbs_tools::process_locker::ProcessLockSharedGuard;
/// Header format definition for dynamic index files (`.dixd`)
#[repr(C)]
/// Create dynamic index files (`.dixd`)
pub struct DynamicIndexWriter {
store: Arc<ChunkStore>,
- _lock: tools::ProcessLockSharedGuard,
+ _lock: ProcessLockSharedGuard,
writer: BufWriter<File>,
closed: bool,
filename: PathBuf,
-use anyhow::{bail, format_err, Error};
-use std::io::{Seek, SeekFrom};
-
-use super::chunk_stat::*;
-use super::chunk_store::*;
-use super::{ChunkReadInfo, IndexFile};
-use crate::tools;
-
use std::fs::File;
use std::io::Write;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::Arc;
+use std::io::{Seek, SeekFrom};
+
+use anyhow::{bail, format_err, Error};
-use super::ChunkInfo;
+use pbs_datastore::chunk_stat::ChunkStat;
+use pbs_datastore::chunk_store::ChunkStore;
+use pbs_datastore::data_blob::ChunkInfo;
+use pbs_datastore::index::{ChunkReadInfo, IndexFile};
+use pbs_tools::process_locker::ProcessLockSharedGuard;
use proxmox::tools::io::ReadExt;
use proxmox::tools::Uuid;
pub struct FixedIndexWriter {
store: Arc<ChunkStore>,
file: File,
- _lock: tools::ProcessLockSharedGuard,
+ _lock: ProcessLockSharedGuard,
filename: PathBuf,
tmp_filename: PathBuf,
chunk_size: usize,
#[macro_export]
macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
- () => { "proxmox-backup-protocol-v1" }
+ () => {
+ "proxmox-backup-protocol-v1"
+ };
}
#[macro_export]
macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
- () => { "proxmox-backup-reader-protocol-v1" }
+ () => {
+ "proxmox-backup-reader-protocol-v1"
+ };
}
/// Unix system user used by proxmox-backup-proxy
pub use pbs_datastore::checksum_reader::*;
pub use pbs_datastore::checksum_writer;
pub use pbs_datastore::checksum_writer::*;
+pub use pbs_datastore::chunk_stat;
+pub use pbs_datastore::chunk_stat::*;
pub use pbs_datastore::chunk_store;
pub use pbs_datastore::chunk_store::*;
pub use pbs_datastore::chunker;
pub use pbs_datastore::key_derivation::*;
pub use pbs_datastore::manifest;
pub use pbs_datastore::manifest::*;
+pub use pbs_datastore::read_chunk::*;
mod chunk_stream;
pub use chunk_stream::*;
-mod chunk_stat;
-pub use chunk_stat::*;
-
+// Split
mod read_chunk;
pub use read_chunk::*;
mod verify;
pub use verify::*;
+// Move to client
mod catalog_shell;
pub use catalog_shell::*;
use anyhow::{bail, Error};
-use super::crypt_config::{CryptConfig, CryptMode};
-use super::data_blob::DataBlob;
-use super::datastore::DataStore;
-
-/// The ReadChunk trait allows reading backup data chunks (local or remote)
-pub trait ReadChunk {
- /// Returns the encoded chunk data
- fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error>;
+use pbs_datastore::crypt_config::{CryptConfig, CryptMode};
+use pbs_datastore::data_blob::DataBlob;
+use pbs_datastore::read_chunk::{ReadChunk, AsyncReadChunk};
- /// Returns the decoded chunk data
- fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error>;
-}
+use super::datastore::DataStore;
#[derive(Clone)]
pub struct LocalChunkReader {
}
}
-pub trait AsyncReadChunk: Send {
- /// Returns the encoded chunk data
- fn read_raw_chunk<'a>(
- &'a self,
- digest: &'a [u8; 32],
- ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>>;
-
- /// Returns the decoded chunk data
- fn read_chunk<'a>(
- &'a self,
- digest: &'a [u8; 32],
- ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>>;
-}
-
impl AsyncReadChunk for LocalChunkReader {
fn read_raw_chunk<'a>(
&'a self,
projects / proxmox-backup.git / commitdiff