1 //! *pxar* binary format definition
3 //! Please note the all values are stored in little endian ordering.
5 //! The Archive contains a list of items. Each item starts with a `Header`, followed by the
8 //! An archive contains items in the following order:
9 //! * `ENTRY` -- containing general stat() data and related bits
10 //! * `XATTR` -- one extended attribute
11 //! * ... -- more of these when there are multiple defined
12 //! * `ACL_USER` -- one `USER ACL` entry
13 //! * ... -- more of these when there are multiple defined
14 //! * `ACL_GROUP` -- one `GROUP ACL` entry
15 //! * ... -- more of these when there are multiple defined
16 //! * `ACL_GROUP_OBJ` -- The `ACL_GROUP_OBJ`
17 //! * `ACL_DEFAULT` -- The various default ACL fields if there's one defined
18 //! * `ACL_DEFAULT_USER` -- one USER ACL entry
19 //! * ... -- more of these when multiple are defined
20 //! * `ACL_DEFAULT_GROUP` -- one GROUP ACL entry
21 //! * ... -- more of these when multiple are defined
22 //! * `FCAPS` -- file capability in Linux disk format
23 //! * `QUOTA_PROJECT_ID` -- the ext4/xfs quota project ID
24 //! * `PAYLOAD` -- file contents, if it is one
25 //! * `SYMLINK` -- symlink target, if it is one
26 //! * `DEVICE` -- device major/minor, if it is a block/char device
28 //! If we are serializing a directory, then this is followed by:
30 //! * `FILENAME` -- name of the first directory entry (strictly ordered!)
31 //! * `<archive>` -- serialization of the first directory entry's metadata and contents,
32 //! following the exact same archive format
33 //! * `FILENAME` -- name of the second directory entry (strictly ordered!)
34 //! * `<archive>` -- serialization of the second directory entry
36 //! * `GOODBYE` -- lookup table at the end of a list of directory entries
38 use std
::cmp
::Ordering
;
39 use std
::ffi
::{CStr, OsStr}
;
41 use std
::fmt
::Display
;
43 use std
::mem
::size_of
;
44 use std
::os
::unix
::ffi
::OsStrExt
;
46 use std
::time
::{Duration, SystemTime}
;
48 use endian_trait
::Endian
;
49 use siphasher
::sip
::SipHasher24
;
54 // $ echo -n 'PROXMOX ARCHIVE FORMAT' | sha1sum | sed -re 's/^(.{16})(.{16}).*$/0x\1, 0x\2/'
55 pub const PXAR_HASH_KEY_1
: u64 = 0x83ac3f1cfbb450db;
56 pub const PXAR_HASH_KEY_2
: u64 = 0xaa4f1b6879369fbd;
58 /// While these constants correspond to `libc::S_` constants, we need these to be fixed for the
59 /// format itself, so we redefine them here.
61 /// Additionally this gets rid of a bunch of casts between u32 and u64.
63 /// You can usually find the values for these in `/usr/include/linux/stat.h`.
66 pub const IFMT
: u64 = 0o0170000;
68 pub const IFSOCK
: u64 = 0o0140000;
69 pub const IFLNK
: u64 = 0o0120000;
70 pub const IFREG
: u64 = 0o0100000;
71 pub const IFBLK
: u64 = 0o0060000;
72 pub const IFDIR
: u64 = 0o0040000;
73 pub const IFCHR
: u64 = 0o0020000;
74 pub const IFIFO
: u64 = 0o0010000;
76 pub const ISUID
: u64 = 0o0004000;
77 pub const ISGID
: u64 = 0o0002000;
78 pub const ISVTX
: u64 = 0o0001000;
81 // Generated by `cargo run --example mk-format-hashes`
82 /// Beginning of an entry (current version).
83 pub const PXAR_ENTRY
: u64 = 0xd5956474e588acef;
84 /// Previous version of the entry struct
85 pub const PXAR_ENTRY_V1
: u64 = 0x11da850a1c1cceff;
86 pub const PXAR_FILENAME
: u64 = 0x16701121063917b3;
87 pub const PXAR_SYMLINK
: u64 = 0x27f971e7dbf5dc5f;
88 pub const PXAR_DEVICE
: u64 = 0x9fc9e906586d5ce9;
89 pub const PXAR_XATTR
: u64 = 0x0dab0229b57dcd03;
90 pub const PXAR_ACL_USER
: u64 = 0x2ce8540a457d55b8;
91 pub const PXAR_ACL_GROUP
: u64 = 0x136e3eceb04c03ab;
92 pub const PXAR_ACL_GROUP_OBJ
: u64 = 0x10868031e9582876;
93 pub const PXAR_ACL_DEFAULT
: u64 = 0xbbbb13415a6896f5;
94 pub const PXAR_ACL_DEFAULT_USER
: u64 = 0xc89357b40532cd1f;
95 pub const PXAR_ACL_DEFAULT_GROUP
: u64 = 0xf90a8a5816038ffe;
96 pub const PXAR_FCAPS
: u64 = 0x2da9dd9db5f7fb67;
97 pub const PXAR_QUOTA_PROJID
: u64 = 0xe07540e82f7d1cbb;
98 /// Marks item as hardlink
99 pub const PXAR_HARDLINK
: u64 = 0x51269c8422bd7275;
100 /// Marks the beginnig of the payload (actual content) of regular files
101 pub const PXAR_PAYLOAD
: u64 = 0x28147a1b0b7c1a25;
102 /// Marks item as entry of goodbye table
103 pub const PXAR_GOODBYE
: u64 = 0x2fec4fa642d5731d;
104 /// The end marker used in the GOODBYE object
105 pub const PXAR_GOODBYE_TAIL_MARKER
: u64 = 0xef5eed5b753e1555;
107 #[derive(Debug, Endian)]
110 /// The item type (see `PXAR_` constants).
112 /// The size of the item, including the size of `Header`.
118 pub fn with_full_size(htype
: u64, full_size
: u64) -> Self {
119 Self { htype, full_size }
123 pub fn with_content_size(htype
: u64, content_size
: u64) -> Self {
124 Self::with_full_size(htype
, content_size
+ size_of
::<Header
>() as u64)
128 pub fn full_size(&self) -> u64 {
133 pub fn content_size(&self) -> u64 {
134 self.full_size() - (size_of
::<Self>() as u64)
138 pub fn max_content_size(&self) -> u64 {
140 // + null-termination
141 PXAR_FILENAME
=> crate::util
::MAX_FILENAME_LEN
+ 1,
142 // + null-termination
143 PXAR_SYMLINK
=> crate::util
::MAX_PATH_LEN
+ 1,
144 // + null-termination + offset
145 PXAR_HARDLINK
=> crate::util
::MAX_PATH_LEN
+ 1 + (size_of
::<u64>() as u64),
146 PXAR_DEVICE
=> size_of
::<Device
>() as u64,
147 PXAR_XATTR
| PXAR_FCAPS
=> crate::util
::MAX_XATTR_LEN
,
148 PXAR_ACL_USER
| PXAR_ACL_DEFAULT_USER
=> size_of
::<acl
::User
>() as u64,
149 PXAR_ACL_GROUP
| PXAR_ACL_DEFAULT_GROUP
=> size_of
::<acl
::Group
>() as u64,
150 PXAR_ACL_DEFAULT
=> size_of
::<acl
::Default
>() as u64,
151 PXAR_ACL_GROUP_OBJ
=> size_of
::<acl
::GroupObject
>() as u64,
152 PXAR_QUOTA_PROJID
=> size_of
::<QuotaProjectId
>() as u64,
153 PXAR_ENTRY
=> size_of
::<Stat
>() as u64,
154 PXAR_PAYLOAD
| PXAR_GOODBYE
=> u64::MAX
- (size_of
::<Self>() as u64),
155 _
=> u64::MAX
- (size_of
::<Self>() as u64),
160 pub fn check_header_size(&self) -> io
::Result
<()> {
161 if self.full_size() < size_of
::<Header
>() as u64 {
162 io_bail
!("invalid header {} - too small ({})", self, self.full_size());
165 if self.content_size() > self.max_content_size() {
167 "invalid content size ({} > {}) of entry with {}",
169 self.max_content_size(),
177 impl Display
for Header
{
178 fn fmt(&self, f
: &mut fmt
::Formatter
<'_
>) -> fmt
::Result
{
179 let readable
= match self.htype
{
180 PXAR_FILENAME
=> "FILENAME",
181 PXAR_SYMLINK
=> "SYMLINK",
182 PXAR_HARDLINK
=> "HARDLINK",
183 PXAR_DEVICE
=> "DEVICE",
184 PXAR_XATTR
=> "XATTR",
185 PXAR_FCAPS
=> "FCAPS",
186 PXAR_ACL_USER
=> "ACL_USER",
187 PXAR_ACL_DEFAULT_USER
=> "ACL_DEFAULT_USER",
188 PXAR_ACL_GROUP
=> "ACL_GROUP",
189 PXAR_ACL_DEFAULT_GROUP
=> "ACL_DEFAULT_GROUP",
190 PXAR_ACL_DEFAULT
=> "ACL_DEFAULT",
191 PXAR_ACL_GROUP_OBJ
=> "ACL_GROUP_OBJ",
192 PXAR_QUOTA_PROJID
=> "QUOTA_PROJID",
193 PXAR_ENTRY
=> "ENTRY",
194 PXAR_PAYLOAD
=> "PAYLOAD",
195 PXAR_GOODBYE
=> "GOODBYE",
198 write
!(f
, "{} header ({:x})", readable
, self.htype
)
202 #[derive(Clone, Debug, Eq, PartialEq)]
203 pub enum SignedDuration
{
208 #[derive(Clone, Debug, Default, Endian, Eq, PartialEq)]
210 pub struct StatxTimestamp
{
211 /// Seconds since the epoch (unix time).
214 /// Nanoseconds since this struct's `secs`.
217 /// (Erroneously introduced padding...)
221 impl From
<SystemTime
> for StatxTimestamp
{
222 fn from(time
: SystemTime
) -> Self {
223 match time
.duration_since(SystemTime
::UNIX_EPOCH
) {
224 Ok(positive
) => Self::from_duration_since_epoch(positive
),
225 Err(negative
) => Self::from_duration_before_epoch(negative
.duration()),
230 impl StatxTimestamp
{
231 /// Create a timestamp from seconds and nanoseconds.
232 pub const fn new(secs
: i64, nanos
: u32) -> Self {
240 /// `const` version of `Default`
241 pub const fn zero() -> Self {
249 #[cfg(all(test, target_os = "linux"))]
250 /// From data found in `struct stat` (`libc::stat`).
251 pub fn from_stat(sec
: i64, nsec
: u32) -> Self {
253 Self::from_duration_before_epoch(Duration
::new((-sec
) as u64, nsec
))
255 Self::from_duration_since_epoch(Duration
::new(sec
as u64, nsec
))
259 /// Turn a positive duration relative to the unix epoch into a time stamp.
260 pub fn from_duration_since_epoch(duration
: Duration
) -> Self {
262 secs
: duration
.as_secs() as i64,
263 nanos
: duration
.subsec_nanos(),
268 /// Turn a *negative* duration from relative to the unix epoch into a time stamp.
269 pub fn from_duration_before_epoch(duration
: Duration
) -> Self {
270 match duration
.subsec_nanos() {
272 secs
: -(duration
.as_secs() as i64),
277 secs
: -(duration
.as_secs() as i64) - 1,
278 nanos
: 1_000_000_000 - nanos
,
284 /// Get the duration since the epoch. an `Ok` value is a positive duration, an `Err` value is a
285 /// negative duration.
286 pub fn to_duration(&self) -> SignedDuration
{
288 SignedDuration
::Positive(Duration
::new(self.secs
as u64, self.nanos
))
290 // this handles the nanos=0 case as `Duration::new()` performs the carry-over.
291 SignedDuration
::Negative(Duration
::new(
292 -(self.secs
+ 1) as u64,
293 1_000_000_000 - self.nanos
,
298 /// Get a `std::time::SystemTime` from this time stamp.
299 pub fn system_time(&self) -> SystemTime
{
300 match self.to_duration() {
301 SignedDuration
::Positive(positive
) => SystemTime
::UNIX_EPOCH
+ positive
,
302 SignedDuration
::Negative(negative
) => SystemTime
::UNIX_EPOCH
- negative
,
308 fn test_statx_timestamp() {
310 size_of
::<StatxTimestamp
>(),
312 "StatxTimestamp size needs to be 16 bytes"
314 const MAY_1_2015_1530
: i64 = 1430487000;
315 let system_time
= SystemTime
::UNIX_EPOCH
+ Duration
::new(MAY_1_2015_1530
as u64, 1_000_000);
316 let tx
= StatxTimestamp
::from(system_time
);
320 secs
: MAY_1_2015_1530
,
325 assert_eq
!(tx
.system_time(), system_time
);
327 const MAY_1_1960_1530
: i64 = -305112600;
328 let system_time
= SystemTime
::UNIX_EPOCH
- Duration
::new(-MAY_1_1960_1530
as u64, 1_000_000);
329 let tx
= StatxTimestamp
::from(system_time
);
333 secs
: MAY_1_1960_1530
- 1,
338 assert_eq
!(tx
.system_time(), system_time
);
340 let system_time
= SystemTime
::UNIX_EPOCH
- Duration
::new(-MAY_1_1960_1530
as u64, 0);
341 let tx
= StatxTimestamp
::from(system_time
);
345 secs
: MAY_1_1960_1530
,
350 assert_eq
!(tx
.system_time(), system_time
);
352 #[derive(Clone, Debug, Default, Endian)]
353 #[cfg_attr(feature = "test-harness", derive(Eq, PartialEq))]
363 impl From
<Stat_V1
> for Stat
{
364 fn from(v1
: Stat_V1
) -> Stat
{
370 mtime
: StatxTimestamp
::from_duration_since_epoch(Duration
::from_nanos(v1
.mtime
)),
375 #[derive(Clone, Debug, Default, Endian, PartialEq)]
376 #[cfg_attr(feature = "test-harness", derive(Eq))]
383 pub mtime
: StatxTimestamp
,
386 /// Builder pattern methods.
388 pub const fn mode(self, mode
: u64) -> Self {
389 Self { mode, ..self }
392 pub const fn flags(self, flags
: u64) -> Self {
393 Self { flags, ..self }
396 pub const fn uid(self, uid
: u32) -> Self {
400 pub const fn gid(self, gid
: u32) -> Self {
404 pub const fn mtime(self, mtime
: StatxTimestamp
) -> Self {
405 Self { mtime, ..self }
408 pub const fn set_dir(self) -> Self {
409 let mode
= self.mode
;
410 self.mode((mode
& !mode
::IFMT
) | mode
::IFDIR
)
413 pub const fn set_regular_file(self) -> Self {
414 let mode
= self.mode
;
415 self.mode((mode
& !mode
::IFMT
) | mode
::IFREG
)
418 pub const fn set_symlink(self) -> Self {
419 let mode
= self.mode
;
420 self.mode((mode
& !mode
::IFMT
) | mode
::IFLNK
)
423 pub const fn set_blockdev(self) -> Self {
424 let mode
= self.mode
;
425 self.mode((mode
& !mode
::IFMT
) | mode
::IFBLK
)
428 pub const fn set_chardev(self) -> Self {
429 let mode
= self.mode
;
430 self.mode((mode
& !mode
::IFMT
) | mode
::IFCHR
)
433 pub const fn set_fifo(self) -> Self {
434 let mode
= self.mode
;
435 self.mode((mode
& !mode
::IFMT
) | mode
::IFIFO
)
439 /// Convenience accessor methods.
441 /// Get the mtime as duration since the epoch.
442 pub fn mtime_as_duration(&self) -> SignedDuration
{
443 self.mtime
.to_duration()
446 /// Get the file type portion of the mode bitfield.
447 pub fn get_file_bits(&self) -> u64 {
448 self.mode
& mode
::IFMT
451 /// Get the permission portion of the mode bitfield.
452 pub fn get_permission_bits(&self) -> u64 {
453 self.mode
& !mode
::IFMT
457 /// Convenience methods.
459 /// Get the file type (`mode & mode::IFMT`).
460 pub fn file_type(&self) -> u64 {
461 self.mode
& mode
::IFMT
464 /// Get the file mode bits (`mode & !mode::IFMT`).
465 pub fn file_mode(&self) -> u64 {
466 self.mode
& !mode
::IFMT
469 /// Check whether this is a directory.
470 pub fn is_dir(&self) -> bool
{
471 (self.mode
& mode
::IFMT
) == mode
::IFDIR
474 /// Check whether this is a symbolic link.
475 pub fn is_symlink(&self) -> bool
{
476 (self.mode
& mode
::IFMT
) == mode
::IFLNK
479 /// Check whether this is a device node.
480 pub fn is_device(&self) -> bool
{
481 let fmt
= self.mode
& mode
::IFMT
;
482 fmt
== mode
::IFCHR
|| fmt
== mode
::IFBLK
485 /// Check whether this is a block device node.
486 pub fn is_blockdev(&self) -> bool
{
487 let fmt
= self.mode
& mode
::IFMT
;
491 /// Check whether this is a character device node.
492 pub fn is_chardev(&self) -> bool
{
493 let fmt
= self.mode
& mode
::IFMT
;
497 /// Check whether this is a regular file.
498 pub fn is_regular_file(&self) -> bool
{
499 (self.mode
& mode
::IFMT
) == mode
::IFREG
502 /// Check whether this is a named pipe (FIFO).
503 pub fn is_fifo(&self) -> bool
{
504 (self.mode
& mode
::IFMT
) == mode
::IFIFO
507 /// Check whether this is a named socket.
508 pub fn is_socket(&self) -> bool
{
509 (self.mode
& mode
::IFMT
) == mode
::IFSOCK
513 impl From
<&std
::fs
::Metadata
> for Stat
{
514 fn from(meta
: &std
::fs
::Metadata
) -> Stat
{
516 use std
::os
::unix
::fs
::MetadataExt
;
518 let this
= Stat
::default();
524 .mode(meta
.mode() as u64);
526 let this
= match meta
.modified() {
527 Ok(mtime
) => this
.mtime(mtime
.into()),
531 let file_type
= meta
.file_type();
532 let mode
= this
.mode
;
533 if file_type
.is_dir() {
534 this
.mode(mode
| mode
::IFDIR
)
535 } else if file_type
.is_symlink() {
536 this
.mode(mode
| mode
::IFLNK
)
538 this
.mode(mode
| mode
::IFREG
)
543 #[derive(Clone, Debug)]
544 pub struct Filename
{
548 #[derive(Clone, Debug)]
554 pub fn as_os_str(&self) -> &OsStr
{
559 impl AsRef
<[u8]> for Symlink
{
560 fn as_ref(&self) -> &[u8] {
565 impl AsRef
<OsStr
> for Symlink
{
566 fn as_ref(&self) -> &OsStr
{
567 OsStr
::from_bytes(&self.data
[..self.data
.len().max(1) - 1])
571 #[derive(Clone, Debug)]
572 pub struct Hardlink
{
578 pub fn as_os_str(&self) -> &OsStr
{
583 impl AsRef
<[u8]> for Hardlink
{
584 fn as_ref(&self) -> &[u8] {
589 impl AsRef
<OsStr
> for Hardlink
{
590 fn as_ref(&self) -> &OsStr
{
591 OsStr
::from_bytes(&self.data
[..self.data
.len().max(1) - 1])
595 #[derive(Clone, Debug, Eq)]
598 pub(crate) data
: Vec
<u8>,
599 pub(crate) name_len
: usize,
603 pub fn new
<N
: AsRef
<[u8]>, V
: AsRef
<[u8]>>(name
: N
, value
: V
) -> Self {
604 let name
= name
.as_ref();
605 let value
= value
.as_ref();
606 let mut data
= Vec
::with_capacity(name
.len() + value
.len() + 1);
612 name_len
: name
.len(),
616 pub fn name(&self) -> &CStr
{
617 unsafe { CStr::from_bytes_with_nul_unchecked(&self.data[..self.name_len + 1]) }
620 pub fn value(&self) -> &[u8] {
621 &self.data
[(self.name_len
+ 1)..]
626 fn cmp(&self, other
: &XAttr
) -> Ordering
{
627 self.name().cmp(other
.name())
631 impl PartialOrd
for XAttr
{
632 fn partial_cmp(&self, other
: &XAttr
) -> Option
<Ordering
> {
633 Some(self.cmp(other
))
637 impl PartialEq
for XAttr
{
638 fn eq(&self, other
: &XAttr
) -> bool
{
639 self.name() == other
.name()
643 #[derive(Clone, Debug, Endian, Eq, PartialEq)]
650 #[cfg(target_os = "linux")]
652 /// Get a `dev_t` value for this device.
654 pub fn to_dev_t(&self) -> u64 {
655 // see bits/sysmacros.h
656 ((self.major
& 0x0000_0fff) << 8) |
657 ((self.major
& 0xffff_f000) << 32) |
658 (self.minor
& 0x0000_00ff) |
659 ((self.minor
& 0xffff_ff00) << 12)
662 /// Get a `Device` from a `dev_t` value.
664 pub fn from_dev_t(dev
: u64) -> Self {
667 major
: (dev
>> 8) & 0x0000_0fff |
668 (dev
>> 32) & 0xffff_f000,
669 minor
: dev
& 0x0000_00ff |
670 (dev
>> 12) & 0xffff_ff00,
675 #[cfg(all(test, target_os = "linux"))]
677 fn test_linux_devices() {
678 let c_dev
= unsafe { ::libc::makedev(0xabcd_1234, 0xdcba_5678) }
;
679 let dev
= Device
::from_dev_t(c_dev
);
680 assert_eq
!(dev
.to_dev_t(), c_dev
);
683 #[derive(Clone, Debug, PartialEq)]
684 #[cfg_attr(feature = "test-harness", derive(Eq))]
690 #[derive(Clone, Copy, Debug, Endian, Eq, PartialEq)]
692 pub struct QuotaProjectId
{
696 /// An entry in the "goodbye table" in a pxar archive. This is required for random access inside
698 #[derive(Clone, Debug, Endian)]
700 pub struct GoodbyeItem
{
701 /// SipHash24 of the directory item name. The last GOODBYE item uses the special hash value
702 /// `PXAR_GOODBYE_TAIL_MARKER`.
705 /// The offset from the start of the GOODBYE object to the start of the matching directory item
706 /// (point to a FILENAME). The last GOODBYE item points to the start of the matching ENTRY
710 /// The overall size of the directory item. This includes the FILENAME header. In other words,
711 /// `goodbye_start - offset + size` points to the end of the directory.
713 /// The last GOODBYE item repeats the size of the GOODBYE item.
718 /// Create a new [`GoodbyeItem`] by hashing the name, and storing the hash along with the
719 /// offset and size information.
720 pub fn new(name
: &[u8], offset
: u64, size
: u64) -> Self {
721 let hash
= hash_filename(name
);
722 Self { hash, offset, size }
726 /// Hash a file name for use in the goodbye table.
727 pub fn hash_filename(name
: &[u8]) -> u64 {
728 use std
::hash
::Hasher
;
730 let mut hasher
= SipHasher24
::new_with_keys(PXAR_HASH_KEY_1
, PXAR_HASH_KEY_2
);
735 /// Returns `true` if the path consists only of [`Normal`](std::path::Component::Normal)
737 pub fn path_is_legal_component(path
: &Path
) -> bool
{
738 let mut components
= path
.components();
739 match components
.next() {
740 Some(std
::path
::Component
::Normal(_
)) => (),
743 components
.next().is_none()
746 /// Assert sure the path consists only of [`Normal`](std::path::Component::Normal) components.
748 /// Returns an [`io::Error`](std::io::Error) of type [`Other`](std::io::ErrorKind::Other) if that's
750 pub fn check_file_name(path
: &Path
) -> io
::Result
<()> {
751 if !path_is_legal_component(path
) {
752 io_bail
!("invalid file name in archive: {:?}", path
);