-//! This module implements the proxmox backup chunked data storage
+//! This module implements the proxmox backup data storage
//!
-//! A chunk is simply defined as binary blob. We store them inside a
-//! `ChunkStore`, addressed by the SHA256 digest of the binary
-//! blob. This technology is also known as content-addressable
-//! storage.
+//! Proxmox backup splits large files into chunks, and stores them
+//! deduplicated using a content addressable storage format.
//!
-//! We store larger files by splitting them into chunks. The resulting
-//! SHA256 digest list is stored as separate index file. The
-//! `DynamicIndex*` format is able to deal with dynamic chunk sizes,
-//! whereas the `FixedIndex*` format is an optimization to store a
-//! list of equal sized chunks.
+//! A chunk is simply defined as binary blob, which is stored inside a
+//! `ChunkStore`, addressed by the SHA256 digest of the binary blob.
+//!
+//! Index files are used to reconstruct the original file. They
+//! basically contain a list of SHA256 checksums. The `DynamicIndex*`
+//! format is able to deal with dynamic chunk sizes, whereas the
+//! `FixedIndex*` format is an optimization to store a list of equal
+//! sized chunks.
//!
//! # ChunkStore Locking
//!
//!
//! Acquire shared lock for ChunkStore (process wide).
//!
-//! Note: When creating .idx files, we create temporary (.tmp) file,
+//! Note: When creating .idx files, we create temporary a (.tmp) file,
//! then do an atomic rename ...
//!
//!
//! * Garbage Collect:
//!
//! Acquire exclusive lock for ChunkStore (process wide). If we have
-//! already an shared lock for ChunkStore, try to updraged that
+//! already a shared lock for the ChunkStore, try to upgrade that
//! lock.
//!
//!
//! * Server Restart
//!
-//! Try to abort running garbage collection to release exclusive
-//! ChunkStore lock asap. Start new service with existing listening
+//! Try to abort the running garbage collection to release exclusive
+//! ChunkStore locks ASAP. Start the new service with the existing listening
//! socket.
//!
//!
//!
//! Deleting backups is as easy as deleting the corresponding .idx
//! files. Unfortunately, this does not free up any storage, because
-//! those files just contains references to chunks.
+//! those files just contain references to chunks.
//!
//! To free up some storage, we run a garbage collection process at
-//! regular intervals. The collector uses an mark and sweep
+//! regular intervals. The collector uses a mark and sweep
//! approach. In the first phase, it scans all .idx files to mark used
//! chunks. The second phase then removes all unmarked chunks from the
//! store.
//! amount of time ago (by default 24h). So we may only delete chunks
//! with `atime` older than 24 hours.
//!
-//! Another problem arise from running backups. The mark phase does
+//! Another problem arises from running backups. The mark phase does
//! not find any chunks from those backups, because there is no .idx
//! file for them (created after the backup). Chunks created or
//! touched by those backups may have an `atime` as old as the start
-//! time of those backup. Please not that the backup start time may
-//! predate the GC start time. Se we may only delete chunk older than
+//! time of those backups. Please note that the backup start time may
+//! predate the GC start time. So we may only delete chunks older than
//! the start time of those running backup jobs.
//!
//!
//!
//! Not sure if this is better. TODO
+use anyhow::{bail, Error};
+
+// Note: .pcat1 => Proxmox Catalog Format version 1
+pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
+
#[macro_export]
macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
() => { "proxmox-backup-protocol-v1" }
}
-// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES
-
-// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8]
-pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239];
-
-// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8]
-pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151];
-
-// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8]
-pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235];
-
-// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8]
-pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196];
-
-// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8]
-pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161];
-
-//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8]
-pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127];
+#[macro_export]
+macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
+ () => { "proxmox-backup-reader-protocol-v1" }
+}
-// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8]
-pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240];
+/// Unix system user used by proxmox-backup-proxy
+pub const BACKUP_USER_NAME: &str = "backup";
+/// Unix system group used by proxmox-backup-proxy
+pub const BACKUP_GROUP_NAME: &str = "backup";
+
+/// Return User info for the 'backup' user (``getpwnam_r(3)``)
+pub fn backup_user() -> Result<nix::unistd::User, Error> {
+ match nix::unistd::User::from_name(BACKUP_USER_NAME)? {
+ Some(user) => Ok(user),
+ None => bail!("Unable to lookup backup user."),
+ }
+}
-// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
-pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
+/// Return Group info for the 'backup' group (``getgrnam(3)``)
+pub fn backup_group() -> Result<nix::unistd::Group, Error> {
+ match nix::unistd::Group::from_name(BACKUP_GROUP_NAME)? {
+ Some(group) => Ok(group),
+ None => bail!("Unable to lookup backup user."),
+ }
+}
-// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
-pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
+mod file_formats;
+pub use file_formats::*;
-// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8]
-pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205];
+mod manifest;
+pub use manifest::*;
mod crypt_config;
pub use crypt_config::*;
mod key_derivation;
pub use key_derivation::*;
-mod data_chunk;
-pub use data_chunk::*;
+mod crypt_reader;
+pub use crypt_reader::*;
+
+mod crypt_writer;
+pub use crypt_writer::*;
+
+mod checksum_reader;
+pub use checksum_reader::*;
+
+mod checksum_writer;
+pub use checksum_writer::*;
+
+mod chunker;
+pub use chunker::*;
mod data_blob;
pub use data_blob::*;
+mod data_blob_reader;
+pub use data_blob_reader::*;
+
+mod data_blob_writer;
+pub use data_blob_writer::*;
+
+mod catalog;
+pub use catalog::*;
+
mod chunk_stream;
pub use chunk_stream::*;
mod chunk_stat;
pub use chunk_stat::*;
-pub use proxmox_protocol::Chunker;
+mod read_chunk;
+pub use read_chunk::*;
mod chunk_store;
pub use chunk_store::*;
mod backup_info;
pub use backup_info::*;
+mod prune;
+pub use prune::*;
+
mod datastore;
pub use datastore::*;
+
+mod verify;
+pub use verify::*;
+
+mod catalog_shell;
+pub use catalog_shell::*;
+
+mod async_index_reader;
+pub use async_index_reader::*;