3 //! Provides an interface to create a ZIP File from ZipEntries
4 //! for a more detailed description of the ZIP format, see:
5 //! https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
7 use std
::convert
::TryInto
;
8 use std
::ffi
::OsString
;
10 use std
::mem
::size_of
;
11 use std
::os
::unix
::ffi
::OsStrExt
;
12 use std
::path
::{Component, Path, PathBuf}
;
14 use anyhow
::{Error, Result}
;
15 use endian_trait
::Endian
;
16 use tokio
::io
::{AsyncRead, AsyncWrite, AsyncWriteExt}
;
18 use crc32fast
::Hasher
;
19 use proxmox
::tools
::time
::gmtime
;
20 use proxmox
::tools
::byte_buffer
::ByteBuffer
;
22 const LOCAL_FH_SIG
: u32 = 0x04034B50;
23 const LOCAL_FF_SIG
: u32 = 0x08074B50;
24 const CENTRAL_DIRECTORY_FH_SIG
: u32 = 0x02014B50;
25 const END_OF_CENTRAL_DIR
: u32 = 0x06054B50;
26 const VERSION_NEEDED
: u16 = 0x002d;
27 const VERSION_MADE_BY
: u16 = 0x032d;
29 const ZIP64_EOCD_RECORD
: u32 = 0x06064B50;
30 const ZIP64_EOCD_LOCATOR
: u32 = 0x07064B50;
33 // 0-4: day of the month (1-31)
34 // 5-8: month: (1 = jan, etc.)
35 // 9-15: year offset from 1980
39 // 5-10: minute (0-59)
42 // see https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-filetimetodosdatetime
43 fn epoch_to_dos(epoch
: i64) -> (u16, u16) {
44 let gmtime
= match gmtime(epoch
) {
46 Err(_
) => return (0, 0),
49 let seconds
= (gmtime
.tm_sec
/ 2) & 0b11111;
50 let minutes
= gmtime
.tm_min
& 0xb111111;
51 let hours
= gmtime
.tm_hour
& 0b11111;
52 let time
: u16 = ((hours
<< 11) | (minutes
<< 5) | (seconds
)) as u16;
54 let date
: u16 = if gmtime
.tm_year
> (2108 - 1900) || gmtime
.tm_year
< (1980 - 1900) {
57 let day
= gmtime
.tm_mday
& 0b11111;
58 let month
= (gmtime
.tm_mon
+ 1) & 0b1111;
59 let year
= (gmtime
.tm_year
+ 1900 - 1980) & 0b1111111;
60 ((year
<< 9) | (month
<< 5) | (day
)) as u16
71 uncompressed_size
: u64,
77 struct Zip64FieldWithOffset
{
80 uncompressed_size
: u64,
87 struct LocalFileHeader
{
96 uncompressed_size
: u32,
103 struct LocalFileFooter
{
106 compressed_size
: u64,
107 uncompressed_size
: u64,
112 struct CentralDirectoryFileHeader
{
114 version_made_by
: u16,
121 compressed_size
: u32,
122 uncompressed_size
: u32,
124 extra_field_len
: u16,
134 struct EndOfCentralDir
{
138 disk_record_count
: u16,
139 total_record_count
: u16,
141 directory_offset
: u32,
147 struct Zip64EOCDRecord
{
150 version_made_by
: u16,
153 disk_number_central_dir
: u32,
154 disk_record_count
: u64,
155 total_record_count
: u64,
157 directory_offset
: u64,
162 struct Zip64EOCDLocator
{
169 async
fn write_struct
<E
, T
>(output
: &mut T
, data
: E
) -> io
::Result
<()>
171 T
: AsyncWrite
+ ?Sized
+ Unpin
,
174 let data
= data
.to_le();
177 std
::slice
::from_raw_parts(
178 &data
as *const E
as *const u8,
179 core
::mem
::size_of_val(&data
),
182 output
.write_all(data
).await
185 /// Represents an Entry in a ZIP File
187 /// used to add to a ZipEncoder
188 pub struct ZipEntry
{
193 uncompressed_size
: u64,
194 compressed_size
: u64,
200 /// Creates a new ZipEntry
202 /// if is_file is false the path will contain an trailing separator,
203 /// so that the zip file understands that it is a directory
204 pub fn new
<P
: AsRef
<Path
>>(path
: P
, mtime
: i64, mode
: u16, is_file
: bool
) -> Self {
205 let mut relpath
= PathBuf
::new();
207 for comp
in path
.as_ref().components() {
208 if let Component
::Normal(_
) = comp
{
214 relpath
.push(""); // adds trailing slash
218 filename
: relpath
.into(),
222 uncompressed_size
: 0,
229 async
fn write_local_header
<W
>(&self, mut buf
: &mut W
) -> io
::Result
<usize>
231 W
: AsyncWrite
+ Unpin
+ ?Sized
,
233 let filename
= self.filename
.as_bytes();
234 let filename_len
= filename
.len();
235 let header_size
= size_of
::<LocalFileHeader
>();
236 let zip_field_size
= size_of
::<Zip64Field
>();
237 let size
: usize = header_size
+ filename_len
+ zip_field_size
;
239 let (date
, time
) = epoch_to_dos(self.mtime
);
244 signature
: LOCAL_FH_SIG
,
245 version_needed
: 0x2d,
251 compressed_size
: 0xFFFFFFFF,
252 uncompressed_size
: 0xFFFFFFFF,
253 filename_len
: filename_len
as u16,
254 extra_field_len
: zip_field_size
as u16,
259 buf
.write_all(filename
).await?
;
266 uncompressed_size
: 0,
275 async
fn write_data_descriptor
<W
: AsyncWrite
+ Unpin
+ ?Sized
>(
278 ) -> io
::Result
<usize> {
279 let size
= size_of
::<LocalFileFooter
>();
284 signature
: LOCAL_FF_SIG
,
286 compressed_size
: self.compressed_size
,
287 uncompressed_size
: self.uncompressed_size
,
295 async
fn write_central_directory_header
<W
: AsyncWrite
+ Unpin
+ ?Sized
>(
298 ) -> io
::Result
<usize> {
299 let filename
= self.filename
.as_bytes();
300 let filename_len
= filename
.len();
301 let header_size
= size_of
::<CentralDirectoryFileHeader
>();
302 let zip_field_size
= size_of
::<Zip64FieldWithOffset
>();
303 let size
: usize = header_size
+ filename_len
+ zip_field_size
;
305 let (date
, time
) = epoch_to_dos(self.mtime
);
309 CentralDirectoryFileHeader
{
310 signature
: CENTRAL_DIRECTORY_FH_SIG
,
311 version_made_by
: VERSION_MADE_BY
,
312 version_needed
: VERSION_NEEDED
,
318 compressed_size
: 0xFFFFFFFF,
319 uncompressed_size
: 0xFFFFFFFF,
320 filename_len
: filename_len
as u16,
321 extra_field_len
: zip_field_size
as u16,
325 external_flags
: (self.mode
as u32) << 16 | (!self.is_file
as u32) << 4,
331 buf
.write_all(filename
).await?
;
335 Zip64FieldWithOffset
{
338 uncompressed_size
: self.uncompressed_size
,
339 compressed_size
: self.compressed_size
,
349 /// Wraps a writer that implements AsyncWrite for creating a ZIP archive
351 /// This will create a ZIP archive on the fly with files added with
352 /// 'add_entry'. To Finish the file, call 'finish'
355 /// use proxmox_backup::tools::zip::*;
356 /// use tokio::fs::File;
357 /// use tokio::prelude::*;
358 /// use anyhow::{Error, Result};
361 /// async fn main() -> Result<(), Error> {
362 /// let target = File::open("foo.zip").await?;
363 /// let mut source = File::open("foo.txt").await?;
365 /// let mut zip = ZipEncoder::new(target);
366 /// zip.add_entry(ZipEntry::new(
371 /// ), Some(source)).await?;
373 /// zip.finish().await?;
378 pub struct ZipEncoder
<W
>
380 W
: AsyncWrite
+ Unpin
,
383 files
: Vec
<ZipEntry
>,
388 impl<W
: AsyncWrite
+ Unpin
> ZipEncoder
<W
> {
389 pub fn new(target
: W
) -> Self {
394 buf
: ByteBuffer
::with_capacity(1024*1024),
398 pub async
fn add_entry
<R
: AsyncRead
+ Unpin
>(
402 ) -> Result
<(), Error
> {
403 entry
.offset
= self.byte_count
.try_into()?
;
404 self.byte_count
+= entry
.write_local_header(&mut self.target
).await?
;
405 if let Some(mut content
) = content
{
406 let mut hasher
= Hasher
::new();
410 let count
= self.buf
.read_from_async(&mut content
).await?
;
418 hasher
.update(&self.buf
);
419 self.target
.write_all(&self.buf
).await?
;
420 self.buf
.consume(count
);
423 self.byte_count
+= size
;
424 entry
.compressed_size
= size
.try_into()?
;
425 entry
.uncompressed_size
= size
.try_into()?
;
426 entry
.crc32
= hasher
.finalize();
428 self.byte_count
+= entry
.write_data_descriptor(&mut self.target
).await?
;
430 self.files
.push(entry
);
437 central_dir_size
: usize,
438 central_dir_offset
: usize,
439 ) -> Result
<(), Error
> {
440 let entrycount
= self.files
.len();
442 let mut count
= entrycount
as u16;
443 let mut directory_size
= central_dir_size
as u32;
444 let mut directory_offset
= central_dir_offset
as u32;
446 if central_dir_size
> u32::MAX
as usize
447 || central_dir_offset
> u32::MAX
as usize
448 || entrycount
> u16::MAX
as usize
451 directory_size
= 0xFFFFFFFF;
452 directory_offset
= 0xFFFFFFFF;
457 signature
: ZIP64_EOCD_RECORD
,
459 version_made_by
: VERSION_MADE_BY
,
460 version_needed
: VERSION_NEEDED
,
462 disk_number_central_dir
: 0,
463 disk_record_count
: entrycount
.try_into()?
,
464 total_record_count
: entrycount
.try_into()?
,
465 directory_size
: central_dir_size
.try_into()?
,
466 directory_offset
: central_dir_offset
.try_into()?
,
471 let locator_offset
= central_dir_offset
+ central_dir_size
;
476 signature
: ZIP64_EOCD_LOCATOR
,
478 offset
: locator_offset
.try_into()?
,
488 signature
: END_OF_CENTRAL_DIR
,
491 disk_record_count
: count
,
492 total_record_count
: count
,
503 pub async
fn finish(&mut self) -> Result
<(), Error
> {
504 let central_dir_offset
= self.byte_count
;
505 let mut central_dir_size
= 0;
507 for file
in &self.files
{
508 central_dir_size
+= file
509 .write_central_directory_header(&mut self.target
)
513 self.write_eocd(central_dir_size
, central_dir_offset
)
516 self.target
.flush().await?
;