3 use std
::ffi
::{CStr, CString, OsStr}
;
4 use std
::os
::unix
::ffi
::OsStrExt
;
5 use std
::io
::{Read, Write, Seek, SeekFrom}
;
6 use std
::convert
::TryFrom
;
8 use chrono
::offset
::{TimeZone, Local}
;
10 use proxmox
::tools
::io
::ReadExt
;
11 use proxmox
::sys
::error
::io_err_other
;
13 use crate::pxar
::catalog
::BackupCatalogWriter
;
14 use crate::pxar
::{MatchPattern, MatchPatternSlice, MatchType}
;
15 use crate::backup
::file_formats
::PROXMOX_CATALOG_FILE_MAGIC_1_0
;
16 use crate::tools
::runtime
::block_on
;
19 #[derive(Copy,Clone,PartialEq)]
20 enum CatalogEntryType
{
27 Fifo
= b'p'
, // Fifo,Pipe
31 impl TryFrom
<u8> for CatalogEntryType
{
34 fn try_from(value
: u8) -> Result
<Self, Error
> {
36 b'd'
=> CatalogEntryType
::Directory
,
37 b'f'
=> CatalogEntryType
::File
,
38 b'l'
=> CatalogEntryType
::Symlink
,
39 b'h'
=> CatalogEntryType
::Hardlink
,
40 b'b'
=> CatalogEntryType
::BlockDevice
,
41 b'c'
=> CatalogEntryType
::CharDevice
,
42 b'p'
=> CatalogEntryType
::Fifo
,
43 b's'
=> CatalogEntryType
::Socket
,
44 _
=> bail
!("invalid CatalogEntryType value '{}'", char::from(value
)),
49 impl fmt
::Display
for CatalogEntryType
{
50 fn fmt(&self, f
: &mut fmt
::Formatter
<'_
>) -> fmt
::Result
{
51 write
!(f
, "{}", char::from(*self as u8))
55 /// Represents a named directory entry
57 /// The ``attr`` property contain the exact type with type specific
59 #[derive(Clone, PartialEq)]
62 pub attr
: DirEntryAttribute
,
65 /// Used to specific additional attributes inside DirEntry
66 #[derive(Clone, PartialEq)]
67 pub enum DirEntryAttribute
{
68 Directory { start: u64 }
,
69 File { size: u64, mtime: u64 }
,
80 fn new(etype
: CatalogEntryType
, name
: Vec
<u8>, start
: u64, size
: u64, mtime
:u64) -> Self {
82 CatalogEntryType
::Directory
=> {
83 DirEntry { name, attr: DirEntryAttribute::Directory { start }
}
85 CatalogEntryType
::File
=> {
86 DirEntry { name, attr: DirEntryAttribute::File { size, mtime }
}
88 CatalogEntryType
::Symlink
=> {
89 DirEntry { name, attr: DirEntryAttribute::Symlink }
91 CatalogEntryType
::Hardlink
=> {
92 DirEntry { name, attr: DirEntryAttribute::Hardlink }
94 CatalogEntryType
::BlockDevice
=> {
95 DirEntry { name, attr: DirEntryAttribute::BlockDevice }
97 CatalogEntryType
::CharDevice
=> {
98 DirEntry { name, attr: DirEntryAttribute::CharDevice }
100 CatalogEntryType
::Fifo
=> {
101 DirEntry { name, attr: DirEntryAttribute::Fifo }
103 CatalogEntryType
::Socket
=> {
104 DirEntry { name, attr: DirEntryAttribute::Socket }
109 /// Check if DirEntry is a directory
110 pub fn is_directory(&self) -> bool
{
112 DirEntryAttribute
::Directory { .. }
=> true,
117 /// Check if DirEntry is a symlink
118 pub fn is_symlink(&self) -> bool
{
120 DirEntryAttribute
::Symlink { .. }
=> true,
128 entries
: Vec
<DirEntry
>,
133 fn new(name
: CString
) -> Self {
134 DirInfo { name, entries: Vec::new() }
137 fn new_rootdir() -> Self {
138 DirInfo
::new(CString
::new(b
"/".to_vec()).unwrap())
141 fn encode_entry
<W
: Write
>(
145 ) -> Result
<(), Error
> {
147 DirEntry { name, attr: DirEntryAttribute::Directory { start }
} => {
148 writer
.write_all(&[CatalogEntryType
::Directory
as u8])?
;
149 catalog_encode_u64(writer
, name
.len() as u64)?
;
150 writer
.write_all(name
)?
;
151 catalog_encode_u64(writer
, pos
- start
)?
;
153 DirEntry { name, attr: DirEntryAttribute::File { size, mtime }
} => {
154 writer
.write_all(&[CatalogEntryType
::File
as u8])?
;
155 catalog_encode_u64(writer
, name
.len() as u64)?
;
156 writer
.write_all(name
)?
;
157 catalog_encode_u64(writer
, *size
)?
;
158 catalog_encode_u64(writer
, *mtime
)?
;
160 DirEntry { name, attr: DirEntryAttribute::Symlink }
=> {
161 writer
.write_all(&[CatalogEntryType
::Symlink
as u8])?
;
162 catalog_encode_u64(writer
, name
.len() as u64)?
;
163 writer
.write_all(name
)?
;
165 DirEntry { name, attr: DirEntryAttribute::Hardlink }
=> {
166 writer
.write_all(&[CatalogEntryType
::Hardlink
as u8])?
;
167 catalog_encode_u64(writer
, name
.len() as u64)?
;
168 writer
.write_all(name
)?
;
170 DirEntry { name, attr: DirEntryAttribute::BlockDevice }
=> {
171 writer
.write_all(&[CatalogEntryType
::BlockDevice
as u8])?
;
172 catalog_encode_u64(writer
, name
.len() as u64)?
;
173 writer
.write_all(name
)?
;
175 DirEntry { name, attr: DirEntryAttribute::CharDevice }
=> {
176 writer
.write_all(&[CatalogEntryType
::CharDevice
as u8])?
;
177 catalog_encode_u64(writer
, name
.len() as u64)?
;
178 writer
.write_all(name
)?
;
180 DirEntry { name, attr: DirEntryAttribute::Fifo }
=> {
181 writer
.write_all(&[CatalogEntryType
::Fifo
as u8])?
;
182 catalog_encode_u64(writer
, name
.len() as u64)?
;
183 writer
.write_all(name
)?
;
185 DirEntry { name, attr: DirEntryAttribute::Socket }
=> {
186 writer
.write_all(&[CatalogEntryType
::Socket
as u8])?
;
187 catalog_encode_u64(writer
, name
.len() as u64)?
;
188 writer
.write_all(name
)?
;
194 fn encode(self, start
: u64) -> Result
<(CString
, Vec
<u8>), Error
> {
195 let mut table
= Vec
::new();
196 catalog_encode_u64(&mut table
, self.entries
.len() as u64)?
;
197 for entry
in self.entries
{
198 Self::encode_entry(&mut table
, &entry
, start
)?
;
201 let mut data
= Vec
::new();
202 catalog_encode_u64(&mut data
, table
.len() as u64)?
;
203 data
.extend_from_slice(&table
);
205 Ok((self.name
, data
))
208 fn parse
<C
: FnMut(CatalogEntryType
, &[u8], u64, u64, u64) -> Result
<bool
, Error
>>(
211 ) -> Result
<(), Error
> {
213 let mut cursor
= data
;
215 let entries
= catalog_decode_u64(&mut cursor
)?
;
217 let mut name_buf
= vec
![0u8; 4096];
219 for _
in 0..entries
{
221 let mut buf
= [ 0u8 ];
222 cursor
.read_exact(&mut buf
)?
;
223 let etype
= CatalogEntryType
::try_from(buf
[0])?
;
225 let name_len
= catalog_decode_u64(&mut cursor
)?
as usize;
226 if name_len
>= name_buf
.len() {
227 bail
!("directory entry name too long ({} >= {})", name_len
, name_buf
.len());
229 let name
= &mut name_buf
[0..name_len
];
230 cursor
.read_exact(name
)?
;
232 let cont
= match etype
{
233 CatalogEntryType
::Directory
=> {
234 let offset
= catalog_decode_u64(&mut cursor
)?
;
235 callback(etype
, name
, offset
, 0, 0)?
237 CatalogEntryType
::File
=> {
238 let size
= catalog_decode_u64(&mut cursor
)?
;
239 let mtime
= catalog_decode_u64(&mut cursor
)?
;
240 callback(etype
, name
, 0, size
, mtime
)?
243 callback(etype
, name
, 0, 0, 0)?
251 if !cursor
.is_empty() {
252 bail
!("unable to parse whole catalog data block");
259 /// Write small catalog files
261 /// A Catalogs simply contains list of files and directories
262 /// (directory tree). They are use to find content without having to
263 /// search the real archive (which may be large). For files, they
264 /// include the last modification time and file size.
265 pub struct CatalogWriter
<W
> {
267 dirstack
: Vec
<DirInfo
>,
271 impl <W
: Write
> CatalogWriter
<W
> {
273 /// Create a new CatalogWriter instance
274 pub fn new(writer
: W
) -> Result
<Self, Error
> {
275 let mut me
= Self { writer, dirstack: vec![ DirInfo::new_rootdir() ], pos: 0 }
;
276 me
.write_all(&PROXMOX_CATALOG_FILE_MAGIC_1_0
)?
;
280 fn write_all(&mut self, data
: &[u8]) -> Result
<(), Error
> {
281 self.writer
.write_all(data
)?
;
282 self.pos
+= u64::try_from(data
.len())?
;
286 /// Finish writing, flush all data
288 /// This need to be called before drop.
289 pub fn finish(&mut self) -> Result
<(), Error
> {
290 if self.dirstack
.len() != 1 {
291 bail
!("unable to finish catalog at level {}", self.dirstack
.len());
294 let dir
= self.dirstack
.pop().unwrap();
296 let start
= self.pos
;
297 let (_
, data
) = dir
.encode(start
)?
;
298 self.write_all(&data
)?
;
300 self.write_all(&start
.to_le_bytes())?
;
302 self.writer
.flush()?
;
308 impl <W
: Write
> BackupCatalogWriter
for CatalogWriter
<W
> {
310 fn start_directory(&mut self, name
: &CStr
) -> Result
<(), Error
> {
311 let new
= DirInfo
::new(name
.to_owned());
312 self.dirstack
.push(new
);
316 fn end_directory(&mut self) -> Result
<(), Error
> {
317 let (start
, name
) = match self.dirstack
.pop() {
319 let start
= self.pos
;
320 let (name
, data
) = dir
.encode(start
)?
;
321 self.write_all(&data
)?
;
325 bail
!("got unexpected end_directory level 0");
329 let current
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
330 let name
= name
.to_bytes().to_vec();
331 current
.entries
.push(DirEntry { name, attr: DirEntryAttribute::Directory { start }
});
336 fn add_file(&mut self, name
: &CStr
, size
: u64, mtime
: u64) -> Result
<(), Error
> {
337 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
338 let name
= name
.to_bytes().to_vec();
339 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::File { size, mtime }
});
343 fn add_symlink(&mut self, name
: &CStr
) -> Result
<(), Error
> {
344 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
345 let name
= name
.to_bytes().to_vec();
346 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::Symlink }
);
350 fn add_hardlink(&mut self, name
: &CStr
) -> Result
<(), Error
> {
351 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
352 let name
= name
.to_bytes().to_vec();
353 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::Hardlink }
);
357 fn add_block_device(&mut self, name
: &CStr
) -> Result
<(), Error
> {
358 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
359 let name
= name
.to_bytes().to_vec();
360 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::BlockDevice }
);
364 fn add_char_device(&mut self, name
: &CStr
) -> Result
<(), Error
> {
365 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
366 let name
= name
.to_bytes().to_vec();
367 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::CharDevice }
);
371 fn add_fifo(&mut self, name
: &CStr
) -> Result
<(), Error
> {
372 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
373 let name
= name
.to_bytes().to_vec();
374 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::Fifo }
);
378 fn add_socket(&mut self, name
: &CStr
) -> Result
<(), Error
> {
379 let dir
= self.dirstack
.last_mut().ok_or_else(|| format_err
!("outside root"))?
;
380 let name
= name
.to_bytes().to_vec();
381 dir
.entries
.push(DirEntry { name, attr: DirEntryAttribute::Socket }
);
386 // fixme: move to somehere else?
387 /// Implement Write to tokio mpsc channel Sender
388 pub struct SenderWriter(tokio
::sync
::mpsc
::Sender
<Result
<Vec
<u8>, Error
>>);
391 pub fn new(sender
: tokio
::sync
::mpsc
::Sender
<Result
<Vec
<u8>, Error
>>) -> Self {
396 impl Write
for SenderWriter
{
397 fn write(&mut self, buf
: &[u8]) -> Result
<usize, std
::io
::Error
> {
398 block_on(async
move {
400 .send(Ok(buf
.to_vec()))
402 .map_err(io_err_other
)
407 fn flush(&mut self) -> Result
<(), std
::io
::Error
> {
412 /// Read Catalog files
413 pub struct CatalogReader
<R
> {
417 impl <R
: Read
+ Seek
> CatalogReader
<R
> {
419 /// Create a new CatalogReader instance
420 pub fn new(reader
: R
) -> Self {
424 /// Print whole catalog to stdout
425 pub fn dump(&mut self) -> Result
<(), Error
> {
427 let root
= self.root()?
;
429 DirEntry { attr: DirEntryAttribute::Directory { start }
, .. }=> {
430 self.dump_dir(std
::path
::Path
::new("./"), start
)
436 /// Get the root DirEntry
437 pub fn root(&mut self) -> Result
<DirEntry
, Error
> {
438 // Root dir is special
439 self.reader
.seek(SeekFrom
::Start(0))?
;
440 let mut magic
= [ 0u8; 8];
441 self.reader
.read_exact(&mut magic
)?
;
442 if magic
!= PROXMOX_CATALOG_FILE_MAGIC_1_0
{
443 bail
!("got unexpected magic number for catalog");
445 self.reader
.seek(SeekFrom
::End(-8))?
;
446 let start
= unsafe { self.reader.read_le_value::<u64>()? }
;
447 Ok(DirEntry { name: b"".to_vec(), attr: DirEntryAttribute::Directory { start }
})
450 /// Read all directory entries
454 ) -> Result
<Vec
<DirEntry
>, Error
> {
456 let start
= match parent
.attr
{
457 DirEntryAttribute
::Directory { start }
=> start
,
458 _
=> bail
!("parent is not a directory - internal error"),
461 let data
= self.read_raw_dirinfo_block(start
)?
;
463 let mut entry_list
= Vec
::new();
465 DirInfo
::parse(&data
, |etype
, name
, offset
, size
, mtime
| {
466 let entry
= DirEntry
::new(etype
, name
.to_vec(), start
- offset
, size
, mtime
);
467 entry_list
.push(entry
);
474 /// Lockup a DirEntry inside a parent directory
479 ) -> Result
<DirEntry
, Error
> {
481 let start
= match parent
.attr
{
482 DirEntryAttribute
::Directory { start }
=> start
,
483 _
=> bail
!("parent is not a directory - internal error"),
486 let data
= self.read_raw_dirinfo_block(start
)?
;
489 DirInfo
::parse(&data
, |etype
, name
, offset
, size
, mtime
| {
490 if name
!= filename
{
494 let entry
= DirEntry
::new(etype
, name
.to_vec(), start
- offset
, size
, mtime
);
496 Ok(false) // stop parsing
500 None
=> bail
!("no such file"),
501 Some(entry
) => Ok(entry
),
505 /// Read the raw directory info block from current reader position.
506 fn read_raw_dirinfo_block(&mut self, start
: u64) -> Result
<Vec
<u8>, Error
> {
507 self.reader
.seek(SeekFrom
::Start(start
))?
;
508 let size
= catalog_decode_u64(&mut self.reader
)?
;
509 if size
< 1 { bail!("got small directory size {}
", size) };
510 let data = self.reader.read_exact_allocated(size as usize)?;
514 /// Print the content of a directory to stdout
515 pub fn dump_dir(&mut self, prefix: &std::path::Path, start: u64) -> Result<(), Error> {
517 let data = self.read_raw_dirinfo_block(start)?;
519 DirInfo::parse(&data, |etype, name, offset, size, mtime| {
521 let mut path = std::path::PathBuf::from(prefix);
522 let name: &OsStr = OsStrExt::from_bytes(name);
526 CatalogEntryType::Directory => {
527 println!("{} {:?}
", etype, path);
529 bail!("got wrong directory
offset ({}
> {}
)", offset, start);
531 let pos = start - offset;
532 self.dump_dir(&path, pos)?;
534 CatalogEntryType::File => {
535 let dt = Local.timestamp(mtime as i64, 0);
542 dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, false),
546 println!("{} {:?}
", etype, path);
554 /// Finds all entries matching the given match patterns and calls the
555 /// provided callback on them.
558 mut entry: &mut Vec<DirEntry>,
559 pattern: &[MatchPatternSlice],
560 callback: &Box<fn(&[DirEntry])>,
561 ) -> Result<(), Error> {
562 let parent = entry.last().unwrap();
563 if !parent.is_directory() {
567 for e in self.read_dir(parent)? {
568 match MatchPatternSlice::match_filename_include(
569 &CString::new(e.name.clone())?,
573 (MatchType::Positive, _) => {
576 let pattern = MatchPattern::from_line(b"**/
*").unwrap().unwrap();
577 let child_pattern = vec![pattern.as_slice()];
578 self.find(&mut entry, &child_pattern, callback)?;
581 (MatchType::PartialPositive, child_pattern)
582 | (MatchType::PartialNegative, child_pattern) => {
584 self.find(&mut entry, &child_pattern, callback)?;
595 /// Serialize u64 as short, variable length byte sequence
597 /// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
598 /// We limit values to a maximum of 2^63.
599 pub fn catalog_encode_u64<W: Write>(writer: &mut W, v: u64) -> Result<(), Error> {
600 let mut enc = Vec::new();
602 if (v & (1<<63)) != 0 { bail!("catalog_encode_u64 failed - value >= 2^63"); }
609 enc.push((128 | (d & 127)) as u8);
612 writer.write_all(&enc)?;
617 /// Deserialize u64 from variable length byte sequence
619 /// We currently read maximal 9 bytes, which give a maximum of 63 bits.
620 pub fn catalog_decode_u64<R: Read>(reader: &mut R) -> Result<u64, Error> {
625 for i in 0..9 { // only allow 9 bytes (63 bits)
627 bail!("decode_u64 failed
- unexpected EOB
");
629 reader.read_exact(&mut buf)?;
632 v |= (t as u64) << (i*7);
635 v |= ((t & 127) as u64) << (i*7);
639 bail!("decode_u64 failed
- missing end marker
");
643 fn test_catalog_u64_encoder() {
645 fn test_encode_decode(value: u64) {
647 let mut data = Vec::new();
648 catalog_encode_u64(&mut data, value).unwrap();
650 //println!("ENCODE {} {:?}
", value, data);
652 let slice = &mut &data[..];
653 let decoded = catalog_decode_u64(slice).unwrap();
655 //println!("DECODE {}
", decoded);
657 assert!(decoded == value);
660 test_encode_decode(126);
661 test_encode_decode((1<<12)-1);
662 test_encode_decode((1<<20)-1);
663 test_encode_decode((1<<50)-1);
664 test_encode_decode((1<<63)-1);