mod data_blob_writer;
pub use data_blob_writer::*;
-mod catalog_blob;
-pub use catalog_blob::*;
+mod catalog;
+pub use catalog::*;
mod chunk_stream;
pub use chunk_stream::*;
--- /dev/null
+use failure::*;
+use std::ffi::{CStr, CString};
+use std::os::unix::ffi::OsStringExt;
+use std::convert::TryInto;
+use std::io::{Read, Write, Seek, SeekFrom};
+use std::convert::TryFrom;
+
+use chrono::offset::{TimeZone, Local};
+
+use proxmox::tools::io::ReadExt;
+
+use crate::pxar::catalog::{BackupCatalogWriter, CatalogEntryType};
+
+enum DirEntry {
+ Directory { name: Vec<u8>, start: u64 },
+ File { name: Vec<u8>, size: u64, mtime: u64 },
+ Symlink { name: Vec<u8> },
+ Hardlink { name: Vec<u8> },
+ BlockDevice { name: Vec<u8> },
+ CharDevice { name: Vec<u8> },
+ Fifo { name: Vec<u8> },
+ Socket { name: Vec<u8> },
+}
+
+struct DirInfo {
+ name: CString,
+ entries: Vec<DirEntry>,
+}
+
+impl DirInfo {
+
+ fn new(name: CString) -> Self {
+ DirInfo { name, entries: Vec::new() }
+ }
+
+ fn new_rootdir() -> Self {
+ DirInfo::new(CString::new(b"/".to_vec()).unwrap())
+ }
+
+ fn encode_entry(data: &mut Vec<u8>, entry: &DirEntry, pos: u64) {
+ match entry {
+ DirEntry::Directory { name, start } => {
+ data.push(CatalogEntryType::Directory as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ data.extend_from_slice(&(pos-start).to_le_bytes());
+ }
+ DirEntry::File { name, size, mtime } => {
+ data.push(CatalogEntryType::File as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ data.extend_from_slice(&size.to_le_bytes());
+ data.extend_from_slice(&mtime.to_le_bytes());
+ }
+ DirEntry::Symlink { name } => {
+ data.push(CatalogEntryType::Symlink as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ }
+ DirEntry::Hardlink { name } => {
+ data.push(CatalogEntryType::Hardlink as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ }
+ DirEntry::BlockDevice { name } => {
+ data.push(CatalogEntryType::BlockDevice as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ }
+ DirEntry::CharDevice { name } => {
+ data.push(CatalogEntryType::CharDevice as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ }
+ DirEntry::Fifo { name } => {
+ data.push(CatalogEntryType::Fifo as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ }
+ DirEntry::Socket { name } => {
+ data.push(CatalogEntryType::Socket as u8);
+ data.extend_from_slice(&(name.len() as u32).to_le_bytes());
+ data.extend_from_slice(name);
+ }
+ }
+ }
+
+ fn encode(self, start: u64) -> Result<(CString, Vec<u8>), Error> {
+ let mut table = Vec::new();
+ let count: u32 = self.entries.len().try_into()?;
+ for entry in self.entries {
+ Self::encode_entry(&mut table, &entry, start);
+ }
+
+ let data = Vec::new();
+ let mut writer = std::io::Cursor::new(data);
+ let size: u32 = (4 + 4 + table.len()).try_into()?;
+ writer.write_all(&size.to_le_bytes())?;
+ writer.write_all(&count.to_le_bytes())?;
+ writer.write_all(&table)?;
+ Ok((self.name, writer.into_inner()))
+ }
+}
+
+pub struct CatalogWriter<W> {
+ writer: W,
+ dirstack: Vec<DirInfo>,
+ pos: u64,
+}
+
+impl <W: Write> CatalogWriter<W> {
+
+ pub fn new(writer: W) -> Result<Self, Error> {
+ Ok(Self { writer, dirstack: vec![ DirInfo::new_rootdir() ], pos: 0 })
+ }
+
+ pub fn finish(&mut self) -> Result<(), Error> {
+ if self.dirstack.len() != 1 {
+ bail!("unable to finish catalog at level {}", self.dirstack.len());
+ }
+
+ let dir = self.dirstack.pop().unwrap();
+
+ let start = self.pos;
+ let (_, data) = dir.encode(start)?;
+ self.write_all(&data)?;
+
+ self.write_all(&start.to_le_bytes())?;
+
+ self.writer.flush()?;
+
+ Ok(())
+ }
+}
+
+impl <W: Write> BackupCatalogWriter for CatalogWriter<W> {
+
+ fn start_directory(&mut self, name: &CStr) -> Result<(), Error> {
+ let new = DirInfo::new(name.to_owned());
+ self.dirstack.push(new);
+ Ok(())
+ }
+
+ fn end_directory(&mut self) -> Result<(), Error> {
+ let (start, name) = match self.dirstack.pop() {
+ Some(dir) => {
+ let start = self.pos;
+ let (name, data) = dir.encode(start)?;
+ self.write_all(&data)?;
+ (start, name)
+ }
+ None => {
+ bail!("got unexpected end_directory level 0");
+ }
+ };
+
+ let current = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ current.entries.push(DirEntry::Directory { name, start });
+
+ Ok(())
+ }
+
+ fn add_file(&mut self, name: &CStr, size: u64, mtime: u64) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::File { name, size, mtime });
+ Ok(())
+ }
+
+ fn add_symlink(&mut self, name: &CStr) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::Symlink { name });
+ Ok(())
+ }
+
+ fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::Hardlink { name });
+ Ok(())
+ }
+
+ fn add_block_device(&mut self, name: &CStr) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::BlockDevice { name });
+ Ok(())
+ }
+
+ fn add_char_device(&mut self, name: &CStr) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::CharDevice { name });
+ Ok(())
+ }
+
+ fn add_fifo(&mut self, name: &CStr) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::Fifo { name });
+ Ok(())
+ }
+
+ fn add_socket(&mut self, name: &CStr) -> Result<(), Error> {
+ let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ dir.entries.push(DirEntry::Socket { name });
+ Ok(())
+ }
+}
+
+impl<W: Write> CatalogWriter<W> {
+ fn write_all(&mut self, data: &[u8]) -> Result<(), Error> {
+ self.writer.write_all(data)?;
+ self.pos += u64::try_from(data.len())?;
+ Ok(())
+ }
+}
+
+// fixme: move to somehere else?
+/// Implement Write to tokio mpsc channel Sender
+pub struct SenderWriter(tokio::sync::mpsc::Sender<Result<Vec<u8>, Error>>);
+
+impl SenderWriter {
+ pub fn new(sender: tokio::sync::mpsc::Sender<Result<Vec<u8>, Error>>) -> Self {
+ Self(sender)
+ }
+}
+
+impl Write for SenderWriter {
+ fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
+ futures::executor::block_on(async move {
+ self.0.send(Ok(buf.to_vec())).await
+ .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))?;
+ Ok(buf.len())
+ })
+ }
+
+ fn flush(&mut self) -> Result<(), std::io::Error> {
+ Ok(())
+ }
+}
+
+pub struct CatalogReader<R> {
+ reader: R,
+}
+
+impl <R: Read + Seek> CatalogReader<R> {
+
+ pub fn new(reader: R) -> Self {
+ Self { reader }
+ }
+
+ fn next_byte<C: Read>(mut reader: C) -> Result<u8, std::io::Error> {
+ let mut buf = [0u8; 1];
+ reader.read_exact(&mut buf)?;
+ Ok(buf[0])
+ }
+
+ pub fn dump(&mut self) -> Result<(), Error> {
+
+ self.reader.seek(SeekFrom::End(-8))?;
+
+ let start = unsafe { self.reader.read_le_value::<u64>()? };
+
+ self.dump_dir(std::path::Path::new("./"), start)
+ }
+
+ pub fn dump_dir(&mut self, prefix: &std::path::Path, start: u64) -> Result<(), Error> {
+
+ self.reader.seek(SeekFrom::Start(start))?;
+
+ let size = unsafe { self.reader.read_le_value::<u32>()? } as usize;
+
+ if size < 8 { bail!("got small directory size {}", size) };
+
+ let data = self.reader.read_exact_allocated(size - 4)?;
+
+ let mut cursor = &data[..];
+
+ let entries = unsafe { cursor.read_le_value::<u32>()? };
+
+ //println!("TEST {} {} size {}", start, entries, size);
+
+ for _ in 0..entries {
+ let etype = CatalogEntryType::try_from(Self::next_byte(&mut cursor)?)?;
+ let name_len = unsafe { cursor.read_le_value::<u32>()? };
+ let name = cursor.read_exact_allocated(name_len as usize)?;
+
+ let mut path = std::path::PathBuf::from(prefix);
+ path.push(std::ffi::OsString::from_vec(name));
+
+ match etype {
+ CatalogEntryType::Directory => {
+ println!("{} {:?}", char::from(etype as u8), path);
+ let offset = unsafe { cursor.read_le_value::<u64>()? };
+ if offset > start {
+ bail!("got wrong directory offset ({} > {})", offset, start);
+ }
+ let pos = start - offset;
+ self.dump_dir(&path, pos)?;
+ }
+ CatalogEntryType::File => {
+ let size = unsafe { cursor.read_le_value::<u64>()? };
+ let mtime = unsafe { cursor.read_le_value::<u64>()? };
+
+ let dt = Local.timestamp(mtime as i64, 0);
+
+ println!("{} {:?} {} {}",
+ char::from(etype as u8),
+ path,
+ size,
+ dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, false)
+ );
+ }
+ _ => {
+ println!("{} {:?}", char::from(etype as u8), path);
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+}
+++ /dev/null
-use failure::*;
-use std::ffi::{CStr, CString};
-use std::os::unix::ffi::OsStringExt;
-use std::convert::TryInto;
-use std::io::{Read, Write, Seek, SeekFrom};
-use std::convert::TryFrom;
-
-use chrono::offset::{TimeZone, Local};
-
-use proxmox::tools::io::ReadExt;
-
-use crate::pxar::catalog::{BackupCatalogWriter, CatalogEntryType};
-
-enum DirEntry {
- Directory { name: Vec<u8>, start: u64 },
- File { name: Vec<u8>, size: u64, mtime: u64 },
- Symlink { name: Vec<u8> },
- Hardlink { name: Vec<u8> },
- BlockDevice { name: Vec<u8> },
- CharDevice { name: Vec<u8> },
- Fifo { name: Vec<u8> },
- Socket { name: Vec<u8> },
-}
-
-struct DirInfo {
- name: CString,
- entries: Vec<DirEntry>,
-}
-
-impl DirInfo {
-
- fn new(name: CString) -> Self {
- DirInfo { name, entries: Vec::new() }
- }
-
- fn new_rootdir() -> Self {
- DirInfo::new(CString::new(b"/".to_vec()).unwrap())
- }
-
- fn encode_entry(data: &mut Vec<u8>, entry: &DirEntry, pos: u64) {
- match entry {
- DirEntry::Directory { name, start } => {
- data.push(CatalogEntryType::Directory as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- data.extend_from_slice(&(pos-start).to_le_bytes());
- }
- DirEntry::File { name, size, mtime } => {
- data.push(CatalogEntryType::File as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- data.extend_from_slice(&size.to_le_bytes());
- data.extend_from_slice(&mtime.to_le_bytes());
- }
- DirEntry::Symlink { name } => {
- data.push(CatalogEntryType::Symlink as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- }
- DirEntry::Hardlink { name } => {
- data.push(CatalogEntryType::Hardlink as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- }
- DirEntry::BlockDevice { name } => {
- data.push(CatalogEntryType::BlockDevice as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- }
- DirEntry::CharDevice { name } => {
- data.push(CatalogEntryType::CharDevice as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- }
- DirEntry::Fifo { name } => {
- data.push(CatalogEntryType::Fifo as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- }
- DirEntry::Socket { name } => {
- data.push(CatalogEntryType::Socket as u8);
- data.extend_from_slice(&(name.len() as u32).to_le_bytes());
- data.extend_from_slice(name);
- }
- }
- }
-
- fn encode(self, start: u64) -> Result<(CString, Vec<u8>), Error> {
- let mut table = Vec::new();
- let count: u32 = self.entries.len().try_into()?;
- for entry in self.entries {
- Self::encode_entry(&mut table, &entry, start);
- }
-
- let data = Vec::new();
- let mut writer = std::io::Cursor::new(data);
- let size: u32 = (4 + 4 + table.len()).try_into()?;
- writer.write_all(&size.to_le_bytes())?;
- writer.write_all(&count.to_le_bytes())?;
- writer.write_all(&table)?;
- Ok((self.name, writer.into_inner()))
- }
-}
-
-pub struct CatalogWriter<W> {
- writer: W,
- dirstack: Vec<DirInfo>,
- pos: u64,
-}
-
-impl <W: Write> CatalogWriter<W> {
-
- pub fn new(writer: W) -> Result<Self, Error> {
- Ok(Self { writer, dirstack: vec![ DirInfo::new_rootdir() ], pos: 0 })
- }
-
- pub fn finish(&mut self) -> Result<(), Error> {
- if self.dirstack.len() != 1 {
- bail!("unable to finish catalog at level {}", self.dirstack.len());
- }
-
- let dir = self.dirstack.pop().unwrap();
-
- let start = self.pos;
- let (_, data) = dir.encode(start)?;
- self.write_all(&data)?;
-
- self.write_all(&start.to_le_bytes())?;
-
- self.writer.flush()?;
-
- Ok(())
- }
-}
-
-impl <W: Write> BackupCatalogWriter for CatalogWriter<W> {
-
- fn start_directory(&mut self, name: &CStr) -> Result<(), Error> {
- let new = DirInfo::new(name.to_owned());
- self.dirstack.push(new);
- Ok(())
- }
-
- fn end_directory(&mut self) -> Result<(), Error> {
- let (start, name) = match self.dirstack.pop() {
- Some(dir) => {
- let start = self.pos;
- let (name, data) = dir.encode(start)?;
- self.write_all(&data)?;
- (start, name)
- }
- None => {
- bail!("got unexpected end_directory level 0");
- }
- };
-
- let current = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- current.entries.push(DirEntry::Directory { name, start });
-
- Ok(())
- }
-
- fn add_file(&mut self, name: &CStr, size: u64, mtime: u64) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::File { name, size, mtime });
- Ok(())
- }
-
- fn add_symlink(&mut self, name: &CStr) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::Symlink { name });
- Ok(())
- }
-
- fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::Hardlink { name });
- Ok(())
- }
-
- fn add_block_device(&mut self, name: &CStr) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::BlockDevice { name });
- Ok(())
- }
-
- fn add_char_device(&mut self, name: &CStr) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::CharDevice { name });
- Ok(())
- }
-
- fn add_fifo(&mut self, name: &CStr) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::Fifo { name });
- Ok(())
- }
-
- fn add_socket(&mut self, name: &CStr) -> Result<(), Error> {
- let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
- let name = name.to_bytes().to_vec();
- dir.entries.push(DirEntry::Socket { name });
- Ok(())
- }
-}
-
-impl<W: Write> CatalogWriter<W> {
- fn write_all(&mut self, data: &[u8]) -> Result<(), Error> {
- self.writer.write_all(data)?;
- self.pos += u64::try_from(data.len())?;
- Ok(())
- }
-}
-
-// fixme: move to somehere else?
-/// Implement Write to tokio mpsc channel Sender
-pub struct SenderWriter(tokio::sync::mpsc::Sender<Result<Vec<u8>, Error>>);
-
-impl SenderWriter {
- pub fn new(sender: tokio::sync::mpsc::Sender<Result<Vec<u8>, Error>>) -> Self {
- Self(sender)
- }
-}
-
-impl Write for SenderWriter {
- fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
- futures::executor::block_on(async move {
- self.0.send(Ok(buf.to_vec())).await
- .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))?;
- Ok(buf.len())
- })
- }
-
- fn flush(&mut self) -> Result<(), std::io::Error> {
- Ok(())
- }
-}
-
-pub struct CatalogReader<R> {
- reader: R,
-}
-
-impl <R: Read + Seek> CatalogReader<R> {
-
- pub fn new(reader: R) -> Self {
- Self { reader }
- }
-
- fn next_byte<C: Read>(mut reader: C) -> Result<u8, std::io::Error> {
- let mut buf = [0u8; 1];
- reader.read_exact(&mut buf)?;
- Ok(buf[0])
- }
-
- pub fn dump(&mut self) -> Result<(), Error> {
-
- self.reader.seek(SeekFrom::End(-8))?;
-
- let start = unsafe { self.reader.read_le_value::<u64>()? };
-
- self.dump_dir(std::path::Path::new("./"), start)
- }
-
- pub fn dump_dir(&mut self, prefix: &std::path::Path, start: u64) -> Result<(), Error> {
-
- self.reader.seek(SeekFrom::Start(start))?;
-
- let size = unsafe { self.reader.read_le_value::<u32>()? } as usize;
-
- if size < 8 { bail!("got small directory size {}", size) };
-
- let data = self.reader.read_exact_allocated(size - 4)?;
-
- let mut cursor = &data[..];
-
- let entries = unsafe { cursor.read_le_value::<u32>()? };
-
- //println!("TEST {} {} size {}", start, entries, size);
-
- for _ in 0..entries {
- let etype = CatalogEntryType::try_from(Self::next_byte(&mut cursor)?)?;
- let name_len = unsafe { cursor.read_le_value::<u32>()? };
- let name = cursor.read_exact_allocated(name_len as usize)?;
-
- let mut path = std::path::PathBuf::from(prefix);
- path.push(std::ffi::OsString::from_vec(name));
-
- match etype {
- CatalogEntryType::Directory => {
- println!("{} {:?}", char::from(etype as u8), path);
- let offset = unsafe { cursor.read_le_value::<u64>()? };
- if offset > start {
- bail!("got wrong directory offset ({} > {})", offset, start);
- }
- let pos = start - offset;
- self.dump_dir(&path, pos)?;
- }
- CatalogEntryType::File => {
- let size = unsafe { cursor.read_le_value::<u64>()? };
- let mtime = unsafe { cursor.read_le_value::<u64>()? };
-
- let dt = Local.timestamp(mtime as i64, 0);
-
- println!("{} {:?} {} {}",
- char::from(etype as u8),
- path,
- size,
- dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, false)
- );
- }
- _ => {
- println!("{} {:?}", char::from(etype as u8), path);
- }
- }
- }
-
- Ok(())
- }
-
-}
projects / proxmox-backup.git / commitdiff