src/tools/zip.rs

   1 //! ZIP Helper
   2 //!
   3 //! Provides an interface to create a ZIP File from ZipEntries
   4 //! for a more detailed description of the ZIP format, see:
   5 //! https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
   6
   7 use std::convert::TryInto;
   8 use std::ffi::OsString;
   9 use std::io;
  10 use std::mem::size_of;
  11 use std::os::unix::ffi::OsStrExt;
  12 use std::path::{Component, Path, PathBuf};
  13
  14 use anyhow::{Error, Result};
  15 use endian_trait::Endian;
  16 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
  17
  18 use crc32fast::Hasher;
  19 use proxmox::tools::time::gmtime;
  20 use proxmox::tools::byte_buffer::ByteBuffer;
  21
  22 const LOCAL_FH_SIG: u32 = 0x04034B50;
  23 const LOCAL_FF_SIG: u32 = 0x08074B50;
  24 const CENTRAL_DIRECTORY_FH_SIG: u32 = 0x02014B50;
  25 const END_OF_CENTRAL_DIR: u32 = 0x06054B50;
  26 const VERSION_NEEDED: u16 = 0x002d;
  27 const VERSION_MADE_BY: u16 = 0x032d;
  28
  29 const ZIP64_EOCD_RECORD: u32 = 0x06064B50;
  30 const ZIP64_EOCD_LOCATOR: u32 = 0x07064B50;
  31
  32 // bits for time:
  33 // 0-4: day of the month (1-31)
  34 // 5-8: month: (1 = jan, etc.)
  35 // 9-15: year offset from 1980
  36 //
  37 // bits for date:
  38 // 0-4: second / 2
  39 // 5-10: minute (0-59)
  40 // 11-15: hour (0-23)
  41 //
  42 // see https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-filetimetodosdatetime
  43 fn epoch_to_dos(epoch: i64) -> (u16, u16) {
  44     let gmtime = match gmtime(epoch) {
  45         Ok(gmtime) => gmtime,
  46         Err(_) => return (0, 0),
  47     };
  48
  49     let seconds = (gmtime.tm_sec / 2) & 0b11111;
  50     let minutes = gmtime.tm_min & 0xb111111;
  51     let hours = gmtime.tm_hour & 0b11111;
  52     let time: u16 = ((hours << 11) | (minutes << 5) | (seconds)) as u16;
  53
  54     let date: u16 = if gmtime.tm_year > (2108 - 1900) || gmtime.tm_year < (1980 - 1900) {
  55         0
  56     } else {
  57         let day = gmtime.tm_mday & 0b11111;
  58         let month = (gmtime.tm_mon + 1) & 0b1111;
  59         let year = (gmtime.tm_year + 1900 - 1980) & 0b1111111;
  60         ((year << 9) | (month << 5) | (day)) as u16
  61     };
  62
  63     (date, time)
  64 }
  65
  66 #[derive(Endian)]
  67 #[repr(C, packed)]
  68 struct Zip64Field {
  69     field_type: u16,
  70     field_size: u16,
  71     uncompressed_size: u64,
  72     compressed_size: u64,
  73 }
  74
  75 #[derive(Endian)]
  76 #[repr(C, packed)]
  77 struct Zip64FieldWithOffset {
  78     field_type: u16,
  79     field_size: u16,
  80     uncompressed_size: u64,
  81     compressed_size: u64,
  82     offset: u64,
  83 }
  84
  85 #[derive(Endian)]
  86 #[repr(C, packed)]
  87 struct LocalFileHeader {
  88     signature: u32,
  89     version_needed: u16,
  90     flags: u16,
  91     compression: u16,
  92     time: u16,
  93     date: u16,
  94     crc32: u32,
  95     compressed_size: u32,
  96     uncompressed_size: u32,
  97     filename_len: u16,
  98     extra_field_len: u16,
  99 }
 100
 101 #[derive(Endian)]
 102 #[repr(C, packed)]
 103 struct LocalFileFooter {
 104     signature: u32,
 105     crc32: u32,
 106     compressed_size: u64,
 107     uncompressed_size: u64,
 108 }
 109
 110 #[derive(Endian)]
 111 #[repr(C, packed)]
 112 struct CentralDirectoryFileHeader {
 113     signature: u32,
 114     version_made_by: u16,
 115     version_needed: u16,
 116     flags: u16,
 117     compression: u16,
 118     time: u16,
 119     date: u16,
 120     crc32: u32,
 121     compressed_size: u32,
 122     uncompressed_size: u32,
 123     filename_len: u16,
 124     extra_field_len: u16,
 125     comment_len: u16,
 126     start_disk: u16,
 127     internal_flags: u16,
 128     external_flags: u32,
 129     offset: u32,
 130 }
 131
 132 #[derive(Endian)]
 133 #[repr(C, packed)]
 134 struct EndOfCentralDir {
 135     signature: u32,
 136     disk_number: u16,
 137     start_disk: u16,
 138     disk_record_count: u16,
 139     total_record_count: u16,
 140     directory_size: u32,
 141     directory_offset: u32,
 142     comment_len: u16,
 143 }
 144
 145 #[derive(Endian)]
 146 #[repr(C, packed)]
 147 struct Zip64EOCDRecord {
 148     signature: u32,
 149     field_size: u64,
 150     version_made_by: u16,
 151     version_needed: u16,
 152     disk_number: u32,
 153     disk_number_central_dir: u32,
 154     disk_record_count: u64,
 155     total_record_count: u64,
 156     directory_size: u64,
 157     directory_offset: u64,
 158 }
 159
 160 #[derive(Endian)]
 161 #[repr(C, packed)]
 162 struct Zip64EOCDLocator {
 163     signature: u32,
 164     disk_number: u32,
 165     offset: u64,
 166     disk_count: u32,
 167 }
 168
 169 async fn write_struct<E, T>(output: &mut T, data: E) -> io::Result<()>
 170 where
 171     T: AsyncWrite + ?Sized + Unpin,
 172     E: Endian,
 173 {
 174     let data = data.to_le();
 175
 176     let data = unsafe {
 177         std::slice::from_raw_parts(
 178             &data as *const E as *const u8,
 179             core::mem::size_of_val(&data),
 180         )
 181     };
 182     output.write_all(data).await
 183 }
 184
 185 /// Represents an Entry in a ZIP File
 186 ///
 187 /// used to add to a ZipEncoder
 188 pub struct ZipEntry {
 189     filename: OsString,
 190     mtime: i64,
 191     mode: u16,
 192     crc32: u32,
 193     uncompressed_size: u64,
 194     compressed_size: u64,
 195     offset: u64,
 196     is_file: bool,
 197 }
 198
 199 impl ZipEntry {
 200     /// Creates a new ZipEntry
 201     ///
 202     /// if is_file is false the path will contain an trailing separator,
 203     /// so that the zip file understands that it is a directory
 204     pub fn new<P: AsRef<Path>>(path: P, mtime: i64, mode: u16, is_file: bool) -> Self {
 205         let mut relpath = PathBuf::new();
 206
 207         for comp in path.as_ref().components() {
 208             if let Component::Normal(_) = comp {
 209                 relpath.push(comp);
 210             }
 211         }
 212
 213         if !is_file {
 214             relpath.push(""); // adds trailing slash
 215         }
 216
 217         Self {
 218             filename: relpath.into(),
 219             crc32: 0,
 220             mtime,
 221             mode,
 222             uncompressed_size: 0,
 223             compressed_size: 0,
 224             offset: 0,
 225             is_file,
 226         }
 227     }
 228
 229     async fn write_local_header<W>(&self, mut buf: &mut W) -> io::Result<usize>
 230     where
 231         W: AsyncWrite + Unpin + ?Sized,
 232     {
 233         let filename = self.filename.as_bytes();
 234         let filename_len = filename.len();
 235         let header_size = size_of::<LocalFileHeader>();
 236         let zip_field_size = size_of::<Zip64Field>();
 237         let size: usize = header_size + filename_len + zip_field_size;
 238
 239         let (date, time) = epoch_to_dos(self.mtime);
 240
 241         write_struct(
 242             &mut buf,
 243             LocalFileHeader {
 244                 signature: LOCAL_FH_SIG,
 245                 version_needed: 0x2d,
 246                 flags: 1 << 3,
 247                 compression: 0,
 248                 time,
 249                 date,
 250                 crc32: 0,
 251                 compressed_size: 0xFFFFFFFF,
 252                 uncompressed_size: 0xFFFFFFFF,
 253                 filename_len: filename_len as u16,
 254                 extra_field_len: zip_field_size as u16,
 255             },
 256         )
 257         .await?;
 258
 259         buf.write_all(filename).await?;
 260
 261         write_struct(
 262             &mut buf,
 263             Zip64Field {
 264                 field_type: 0x0001,
 265                 field_size: 2 * 8,
 266                 uncompressed_size: 0,
 267                 compressed_size: 0,
 268             },
 269         )
 270         .await?;
 271
 272         Ok(size)
 273     }
 274
 275     async fn write_data_descriptor<W: AsyncWrite + Unpin + ?Sized>(
 276         &self,
 277         mut buf: &mut W,
 278     ) -> io::Result<usize> {
 279         let size = size_of::<LocalFileFooter>();
 280
 281         write_struct(
 282             &mut buf,
 283             LocalFileFooter {
 284                 signature: LOCAL_FF_SIG,
 285                 crc32: self.crc32,
 286                 compressed_size: self.compressed_size,
 287                 uncompressed_size: self.uncompressed_size,
 288             },
 289         )
 290         .await?;
 291
 292         Ok(size)
 293     }
 294
 295     async fn write_central_directory_header<W: AsyncWrite + Unpin + ?Sized>(
 296         &self,
 297         mut buf: &mut W,
 298     ) -> io::Result<usize> {
 299         let filename = self.filename.as_bytes();
 300         let filename_len = filename.len();
 301         let header_size = size_of::<CentralDirectoryFileHeader>();
 302         let zip_field_size = size_of::<Zip64FieldWithOffset>();
 303         let size: usize = header_size + filename_len + zip_field_size;
 304
 305         let (date, time) = epoch_to_dos(self.mtime);
 306
 307         write_struct(
 308             &mut buf,
 309             CentralDirectoryFileHeader {
 310                 signature: CENTRAL_DIRECTORY_FH_SIG,
 311                 version_made_by: VERSION_MADE_BY,
 312                 version_needed: VERSION_NEEDED,
 313                 flags: 1 << 3,
 314                 compression: 0,
 315                 time,
 316                 date,
 317                 crc32: self.crc32,
 318                 compressed_size: 0xFFFFFFFF,
 319                 uncompressed_size: 0xFFFFFFFF,
 320                 filename_len: filename_len as u16,
 321                 extra_field_len: zip_field_size as u16,
 322                 comment_len: 0,
 323                 start_disk: 0,
 324                 internal_flags: 0,
 325                 external_flags: (self.mode as u32) << 16 | (!self.is_file as u32) << 4,
 326                 offset: 0xFFFFFFFF,
 327             },
 328         )
 329         .await?;
 330
 331         buf.write_all(filename).await?;
 332
 333         write_struct(
 334             &mut buf,
 335             Zip64FieldWithOffset {
 336                 field_type: 1,
 337                 field_size: 3 * 8,
 338                 uncompressed_size: self.uncompressed_size,
 339                 compressed_size: self.compressed_size,
 340                 offset: self.offset,
 341             },
 342         )
 343         .await?;
 344
 345         Ok(size)
 346     }
 347 }
 348
 349 /// Wraps a writer that implements AsyncWrite for creating a ZIP archive
 350 ///
 351 /// This will create a ZIP archive on the fly with files added with
 352 /// 'add_entry'. To Finish the file, call 'finish'
 353 /// Example:
 354 /// ```no_run
 355 /// use proxmox_backup::tools::zip::*;
 356 /// use tokio::fs::File;
 357 /// use tokio::prelude::*;
 358 /// use anyhow::{Error, Result};
 359 ///
 360 /// #[tokio::main]
 361 /// async fn main() -> Result<(), Error> {
 362 ///     let target = File::open("foo.zip").await?;
 363 ///     let mut source = File::open("foo.txt").await?;
 364 ///
 365 ///     let mut zip = ZipEncoder::new(target);
 366 ///     zip.add_entry(ZipEntry::new(
 367 ///         "foo.txt",
 368 ///         0,
 369 ///         0o100755,
 370 ///         true,
 371 ///     ), Some(source)).await?;
 372 ///
 373 ///     zip.finish().await?;
 374 ///
 375 ///     Ok(())
 376 /// }
 377 /// ```
 378 pub struct ZipEncoder<W>
 379 where
 380     W: AsyncWrite + Unpin,
 381 {
 382     byte_count: usize,
 383     files: Vec<ZipEntry>,
 384     target: W,
 385     buf: ByteBuffer,
 386 }
 387
 388 impl<W: AsyncWrite + Unpin> ZipEncoder<W> {
 389     pub fn new(target: W) -> Self {
 390         Self {
 391             byte_count: 0,
 392             files: Vec::new(),
 393             target,
 394             buf: ByteBuffer::with_capacity(1024*1024),
 395         }
 396     }
 397
 398     pub async fn add_entry<R: AsyncRead + Unpin>(
 399         &mut self,
 400         mut entry: ZipEntry,
 401         content: Option<R>,
 402     ) -> Result<(), Error> {
 403         entry.offset = self.byte_count.try_into()?;
 404         self.byte_count += entry.write_local_header(&mut self.target).await?;
 405         if let Some(mut content) = content {
 406             let mut hasher = Hasher::new();
 407             let mut size = 0;
 408             loop {
 409
 410                 let count = self.buf.read_from_async(&mut content).await?;
 411
 412                 // end of file
 413                 if count == 0 {
 414                     break;
 415                 }
 416
 417                 size += count;
 418                 hasher.update(&self.buf);
 419                 self.target.write_all(&self.buf).await?;
 420                 self.buf.consume(count);
 421             }
 422
 423             self.byte_count += size;
 424             entry.compressed_size = size.try_into()?;
 425             entry.uncompressed_size = size.try_into()?;
 426             entry.crc32 = hasher.finalize();
 427         }
 428         self.byte_count += entry.write_data_descriptor(&mut self.target).await?;
 429
 430         self.files.push(entry);
 431
 432         Ok(())
 433     }
 434
 435     async fn write_eocd(
 436         &mut self,
 437         central_dir_size: usize,
 438         central_dir_offset: usize,
 439     ) -> Result<(), Error> {
 440         let entrycount = self.files.len();
 441
 442         let mut count = entrycount as u16;
 443         let mut directory_size = central_dir_size as u32;
 444         let mut directory_offset = central_dir_offset as u32;
 445
 446         if central_dir_size > u32::MAX as usize
 447             || central_dir_offset > u32::MAX as usize
 448             || entrycount > u16::MAX as usize
 449         {
 450             count = 0xFFFF;
 451             directory_size = 0xFFFFFFFF;
 452             directory_offset = 0xFFFFFFFF;
 453
 454             write_struct(
 455                 &mut self.target,
 456                 Zip64EOCDRecord {
 457                     signature: ZIP64_EOCD_RECORD,
 458                     field_size: 44,
 459                     version_made_by: VERSION_MADE_BY,
 460                     version_needed: VERSION_NEEDED,
 461                     disk_number: 0,
 462                     disk_number_central_dir: 0,
 463                     disk_record_count: entrycount.try_into()?,
 464                     total_record_count: entrycount.try_into()?,
 465                     directory_size: central_dir_size.try_into()?,
 466                     directory_offset: central_dir_offset.try_into()?,
 467                 },
 468             )
 469             .await?;
 470
 471             let locator_offset = central_dir_offset + central_dir_size;
 472
 473             write_struct(
 474                 &mut self.target,
 475                 Zip64EOCDLocator {
 476                     signature: ZIP64_EOCD_LOCATOR,
 477                     disk_number: 0,
 478                     offset: locator_offset.try_into()?,
 479                     disk_count: 1,
 480                 },
 481             )
 482             .await?;
 483         }
 484
 485         write_struct(
 486             &mut self.target,
 487             EndOfCentralDir {
 488                 signature: END_OF_CENTRAL_DIR,
 489                 disk_number: 0,
 490                 start_disk: 0,
 491                 disk_record_count: count,
 492                 total_record_count: count,
 493                 directory_size,
 494                 directory_offset,
 495                 comment_len: 0,
 496             },
 497         )
 498         .await?;
 499
 500         Ok(())
 501     }
 502
 503     pub async fn finish(&mut self) -> Result<(), Error> {
 504         let central_dir_offset = self.byte_count;
 505         let mut central_dir_size = 0;
 506
 507         for file in &self.files {
 508             central_dir_size += file
 509                 .write_central_directory_header(&mut self.target)
 510                 .await?;
 511         }
 512
 513         self.write_eocd(central_dir_size, central_dir_offset)
 514             .await?;
 515
 516         self.target.flush().await?;
 517
 518         Ok(())
 519     }
 520 }