pbs-tools/src/zip.rs

   1 //! ZIP Helper
   2 //!
   3 //! Provides an interface to create a ZIP File from ZipEntries
   4 //! for a more detailed description of the ZIP format, see:
   5 //! https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
   6
   7 use std::convert::TryInto;
   8 use std::ffi::OsString;
   9 use std::io;
  10 use std::mem::size_of;
  11 use std::os::unix::ffi::OsStrExt;
  12 use std::path::{Component, Path, PathBuf};
  13 use std::pin::Pin;
  14 use std::task::{Context, Poll};
  15 use std::time::SystemTime;
  16
  17 use anyhow::{format_err, Error, Result};
  18 use endian_trait::Endian;
  19 use futures::ready;
  20 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
  21
  22 use crc32fast::Hasher;
  23 use proxmox_time::gmtime;
  24
  25 use crate::compression::{DeflateEncoder, Level};
  26
  27 const LOCAL_FH_SIG: u32 = 0x04034B50;
  28 const LOCAL_FF_SIG: u32 = 0x08074B50;
  29 const CENTRAL_DIRECTORY_FH_SIG: u32 = 0x02014B50;
  30 const END_OF_CENTRAL_DIR: u32 = 0x06054B50;
  31 const VERSION_NEEDED: u16 = 0x002d;
  32 const VERSION_MADE_BY: u16 = 0x032d;
  33
  34 const ZIP64_EOCD_RECORD: u32 = 0x06064B50;
  35 const ZIP64_EOCD_LOCATOR: u32 = 0x07064B50;
  36
  37 // bits for time:
  38 // 0-4: day of the month (1-31)
  39 // 5-8: month: (1 = jan, etc.)
  40 // 9-15: year offset from 1980
  41 //
  42 // bits for date:
  43 // 0-4: second / 2
  44 // 5-10: minute (0-59)
  45 // 11-15: hour (0-23)
  46 //
  47 // see https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-filetimetodosdatetime
  48 fn epoch_to_dos(epoch: i64) -> (u16, u16) {
  49     let gmtime = match gmtime(epoch) {
  50         Ok(gmtime) => gmtime,
  51         Err(_) => return (0, 0),
  52     };
  53
  54     let seconds = (gmtime.tm_sec / 2) & 0b11111;
  55     let minutes = gmtime.tm_min & 0xb111111;
  56     let hours = gmtime.tm_hour & 0b11111;
  57     let time: u16 = ((hours << 11) | (minutes << 5) | (seconds)) as u16;
  58
  59     let date: u16 = if gmtime.tm_year > (2108 - 1900) || gmtime.tm_year < (1980 - 1900) {
  60         0
  61     } else {
  62         let day = gmtime.tm_mday & 0b11111;
  63         let month = (gmtime.tm_mon + 1) & 0b1111;
  64         let year = (gmtime.tm_year + 1900 - 1980) & 0b1111111;
  65         ((year << 9) | (month << 5) | (day)) as u16
  66     };
  67
  68     (date, time)
  69 }
  70
  71 #[derive(Endian)]
  72 #[repr(C, packed)]
  73 struct Zip64Field {
  74     field_type: u16,
  75     field_size: u16,
  76     uncompressed_size: u64,
  77     compressed_size: u64,
  78 }
  79
  80 #[derive(Endian)]
  81 #[repr(C, packed)]
  82 struct Zip64FieldWithOffset {
  83     field_type: u16,
  84     field_size: u16,
  85     uncompressed_size: u64,
  86     compressed_size: u64,
  87     offset: u64,
  88     start_disk: u32,
  89 }
  90
  91 #[derive(Endian)]
  92 #[repr(C, packed)]
  93 struct LocalFileHeader {
  94     signature: u32,
  95     version_needed: u16,
  96     flags: u16,
  97     compression: u16,
  98     time: u16,
  99     date: u16,
 100     crc32: u32,
 101     compressed_size: u32,
 102     uncompressed_size: u32,
 103     filename_len: u16,
 104     extra_field_len: u16,
 105 }
 106
 107 #[derive(Endian)]
 108 #[repr(C, packed)]
 109 struct LocalFileFooter {
 110     signature: u32,
 111     crc32: u32,
 112     compressed_size: u64,
 113     uncompressed_size: u64,
 114 }
 115
 116 #[derive(Endian)]
 117 #[repr(C, packed)]
 118 struct CentralDirectoryFileHeader {
 119     signature: u32,
 120     version_made_by: u16,
 121     version_needed: u16,
 122     flags: u16,
 123     compression: u16,
 124     time: u16,
 125     date: u16,
 126     crc32: u32,
 127     compressed_size: u32,
 128     uncompressed_size: u32,
 129     filename_len: u16,
 130     extra_field_len: u16,
 131     comment_len: u16,
 132     start_disk: u16,
 133     internal_flags: u16,
 134     external_flags: u32,
 135     offset: u32,
 136 }
 137
 138 #[derive(Endian)]
 139 #[repr(C, packed)]
 140 struct EndOfCentralDir {
 141     signature: u32,
 142     disk_number: u16,
 143     start_disk: u16,
 144     disk_record_count: u16,
 145     total_record_count: u16,
 146     directory_size: u32,
 147     directory_offset: u32,
 148     comment_len: u16,
 149 }
 150
 151 #[derive(Endian)]
 152 #[repr(C, packed)]
 153 struct Zip64EOCDRecord {
 154     signature: u32,
 155     field_size: u64,
 156     version_made_by: u16,
 157     version_needed: u16,
 158     disk_number: u32,
 159     disk_number_central_dir: u32,
 160     disk_record_count: u64,
 161     total_record_count: u64,
 162     directory_size: u64,
 163     directory_offset: u64,
 164 }
 165
 166 #[derive(Endian)]
 167 #[repr(C, packed)]
 168 struct Zip64EOCDLocator {
 169     signature: u32,
 170     disk_number: u32,
 171     offset: u64,
 172     disk_count: u32,
 173 }
 174
 175 async fn write_struct<E, T>(output: &mut T, data: E) -> io::Result<()>
 176 where
 177     T: AsyncWrite + ?Sized + Unpin,
 178     E: Endian,
 179 {
 180     let data = data.to_le();
 181
 182     let data = unsafe {
 183         std::slice::from_raw_parts(
 184             &data as *const E as *const u8,
 185             core::mem::size_of_val(&data),
 186         )
 187     };
 188     output.write_all(data).await
 189 }
 190
 191 /// Represents an Entry in a ZIP File
 192 ///
 193 /// used to add to a ZipEncoder
 194 pub struct ZipEntry {
 195     filename: OsString,
 196     mtime: i64,
 197     mode: u16,
 198     crc32: u32,
 199     uncompressed_size: u64,
 200     compressed_size: u64,
 201     offset: u64,
 202     is_file: bool,
 203 }
 204
 205 impl ZipEntry {
 206     /// Creates a new ZipEntry
 207     ///
 208     /// if is_file is false the path will contain an trailing separator,
 209     /// so that the zip file understands that it is a directory
 210     pub fn new<P: AsRef<Path>>(path: P, mtime: i64, mode: u16, is_file: bool) -> Self {
 211         let mut relpath = PathBuf::new();
 212
 213         for comp in path.as_ref().components() {
 214             if let Component::Normal(_) = comp {
 215                 relpath.push(comp);
 216             }
 217         }
 218
 219         if !is_file {
 220             relpath.push(""); // adds trailing slash
 221         }
 222
 223         Self {
 224             filename: relpath.into(),
 225             crc32: 0,
 226             mtime,
 227             mode,
 228             uncompressed_size: 0,
 229             compressed_size: 0,
 230             offset: 0,
 231             is_file,
 232         }
 233     }
 234
 235     async fn write_local_header<W>(&self, mut buf: &mut W) -> io::Result<usize>
 236     where
 237         W: AsyncWrite + Unpin + ?Sized,
 238     {
 239         let filename = self.filename.as_bytes();
 240         let filename_len = filename.len();
 241         let header_size = size_of::<LocalFileHeader>();
 242         let zip_field_size = size_of::<Zip64Field>();
 243         let size: usize = header_size + filename_len + zip_field_size;
 244
 245         let (date, time) = epoch_to_dos(self.mtime);
 246
 247         write_struct(
 248             &mut buf,
 249             LocalFileHeader {
 250                 signature: LOCAL_FH_SIG,
 251                 version_needed: 0x2d,
 252                 flags: 1 << 3,
 253                 compression: 0x8,
 254                 time,
 255                 date,
 256                 crc32: 0,
 257                 compressed_size: 0xFFFFFFFF,
 258                 uncompressed_size: 0xFFFFFFFF,
 259                 filename_len: filename_len as u16,
 260                 extra_field_len: zip_field_size as u16,
 261             },
 262         )
 263         .await?;
 264
 265         buf.write_all(filename).await?;
 266
 267         write_struct(
 268             &mut buf,
 269             Zip64Field {
 270                 field_type: 0x0001,
 271                 field_size: 2 * 8,
 272                 uncompressed_size: 0,
 273                 compressed_size: 0,
 274             },
 275         )
 276         .await?;
 277
 278         Ok(size)
 279     }
 280
 281     async fn write_data_descriptor<W: AsyncWrite + Unpin + ?Sized>(
 282         &self,
 283         mut buf: &mut W,
 284     ) -> io::Result<usize> {
 285         let size = size_of::<LocalFileFooter>();
 286
 287         write_struct(
 288             &mut buf,
 289             LocalFileFooter {
 290                 signature: LOCAL_FF_SIG,
 291                 crc32: self.crc32,
 292                 compressed_size: self.compressed_size,
 293                 uncompressed_size: self.uncompressed_size,
 294             },
 295         )
 296         .await?;
 297
 298         Ok(size)
 299     }
 300
 301     async fn write_central_directory_header<W: AsyncWrite + Unpin + ?Sized>(
 302         &self,
 303         mut buf: &mut W,
 304     ) -> io::Result<usize> {
 305         let filename = self.filename.as_bytes();
 306         let filename_len = filename.len();
 307         let header_size = size_of::<CentralDirectoryFileHeader>();
 308         let zip_field_size = size_of::<Zip64FieldWithOffset>();
 309         let mut size: usize = header_size + filename_len;
 310
 311         let (date, time) = epoch_to_dos(self.mtime);
 312
 313         let (compressed_size, uncompressed_size, offset, need_zip64) = if self.compressed_size
 314             >= (u32::MAX as u64)
 315             || self.uncompressed_size >= (u32::MAX as u64)
 316             || self.offset >= (u32::MAX as u64)
 317         {
 318             size += zip_field_size;
 319             (0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, true)
 320         } else {
 321             (
 322                 self.compressed_size as u32,
 323                 self.uncompressed_size as u32,
 324                 self.offset as u32,
 325                 false,
 326             )
 327         };
 328
 329         write_struct(
 330             &mut buf,
 331             CentralDirectoryFileHeader {
 332                 signature: CENTRAL_DIRECTORY_FH_SIG,
 333                 version_made_by: VERSION_MADE_BY,
 334                 version_needed: VERSION_NEEDED,
 335                 flags: 1 << 3,
 336                 compression: 0x8,
 337                 time,
 338                 date,
 339                 crc32: self.crc32,
 340                 compressed_size,
 341                 uncompressed_size,
 342                 filename_len: filename_len as u16,
 343                 extra_field_len: if need_zip64 { zip_field_size as u16 } else { 0 },
 344                 comment_len: 0,
 345                 start_disk: 0,
 346                 internal_flags: 0,
 347                 external_flags: (self.mode as u32) << 16 | (!self.is_file as u32) << 4,
 348                 offset,
 349             },
 350         )
 351         .await?;
 352
 353         buf.write_all(filename).await?;
 354
 355         if need_zip64 {
 356             write_struct(
 357                 &mut buf,
 358                 Zip64FieldWithOffset {
 359                     field_type: 1,
 360                     field_size: 3 * 8 + 4,
 361                     uncompressed_size: self.uncompressed_size,
 362                     compressed_size: self.compressed_size,
 363                     offset: self.offset,
 364                     start_disk: 0,
 365                 },
 366             )
 367             .await?;
 368         }
 369
 370         Ok(size)
 371     }
 372 }
 373
 374 // wraps an asyncreader and calculates the hash
 375 struct HashWrapper<R> {
 376     inner: R,
 377     hasher: Hasher,
 378 }
 379
 380 impl<R> HashWrapper<R> {
 381     fn new(inner: R) -> Self {
 382         Self {
 383             inner,
 384             hasher: Hasher::new(),
 385         }
 386     }
 387
 388     // consumes self and returns the hash and the reader
 389     fn finish(self) -> (u32, R) {
 390         let crc32 = self.hasher.finalize();
 391         (crc32, self.inner)
 392     }
 393 }
 394
 395 impl<R> AsyncRead for HashWrapper<R>
 396 where
 397     R: AsyncRead + Unpin,
 398 {
 399     fn poll_read(
 400         self: Pin<&mut Self>,
 401         cx: &mut Context<'_>,
 402         buf: &mut ReadBuf<'_>,
 403     ) -> Poll<Result<(), io::Error>> {
 404         let this = self.get_mut();
 405         let old_len = buf.filled().len();
 406         ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?;
 407         let new_len = buf.filled().len();
 408         if new_len > old_len {
 409             this.hasher.update(&buf.filled()[old_len..new_len]);
 410         }
 411         Poll::Ready(Ok(()))
 412     }
 413 }
 414
 415 /// Wraps a writer that implements AsyncWrite for creating a ZIP archive
 416 ///
 417 /// This will create a ZIP archive on the fly with files added with
 418 /// 'add_entry'. To Finish the file, call 'finish'
 419 /// Example:
 420 /// ```no_run
 421 /// use anyhow::{Error, Result};
 422 /// use tokio::fs::File;
 423 ///
 424 /// use pbs_tools::zip::{ZipEncoder, ZipEntry};
 425 ///
 426 /// #[tokio::main]
 427 /// async fn main() -> Result<(), Error> {
 428 ///     let target = File::open("foo.zip").await?;
 429 ///     let mut source = File::open("foo.txt").await?;
 430 ///
 431 ///     let mut zip = ZipEncoder::new(target);
 432 ///     zip.add_entry(ZipEntry::new(
 433 ///         "foo.txt",
 434 ///         0,
 435 ///         0o100755,
 436 ///         true,
 437 ///     ), Some(source)).await?;
 438 ///
 439 ///     zip.finish().await?;
 440 ///
 441 ///     Ok(())
 442 /// }
 443 /// ```
 444 pub struct ZipEncoder<W>
 445 where
 446     W: AsyncWrite + Unpin,
 447 {
 448     byte_count: usize,
 449     files: Vec<ZipEntry>,
 450     target: Option<W>,
 451 }
 452
 453 impl<W: AsyncWrite + Unpin> ZipEncoder<W> {
 454     pub fn new(target: W) -> Self {
 455         Self {
 456             byte_count: 0,
 457             files: Vec::new(),
 458             target: Some(target),
 459         }
 460     }
 461
 462     pub async fn add_entry<R: AsyncRead + Unpin>(
 463         &mut self,
 464         mut entry: ZipEntry,
 465         content: Option<R>,
 466     ) -> Result<(), Error> {
 467         let mut target = self
 468             .target
 469             .take()
 470             .ok_or_else(|| format_err!("had no target during add entry"))?;
 471         entry.offset = self.byte_count.try_into()?;
 472         self.byte_count += entry.write_local_header(&mut target).await?;
 473         if let Some(content) = content {
 474             let mut reader = HashWrapper::new(content);
 475             let mut enc = DeflateEncoder::with_quality(target, Level::Fastest);
 476
 477             enc.compress(&mut reader).await?;
 478             let total_in = enc.total_in();
 479             let total_out = enc.total_out();
 480             target = enc.into_inner();
 481
 482             let (crc32, _reader) = reader.finish();
 483
 484             self.byte_count += total_out as usize;
 485             entry.compressed_size = total_out;
 486             entry.uncompressed_size = total_in;
 487
 488             entry.crc32 = crc32;
 489         }
 490         self.byte_count += entry.write_data_descriptor(&mut target).await?;
 491         self.target = Some(target);
 492
 493         self.files.push(entry);
 494
 495         Ok(())
 496     }
 497
 498     async fn write_eocd(
 499         &mut self,
 500         central_dir_size: usize,
 501         central_dir_offset: usize,
 502     ) -> Result<(), Error> {
 503         let entrycount = self.files.len();
 504         let mut target = self
 505             .target
 506             .take()
 507             .ok_or_else(|| format_err!("had no target during write_eocd"))?;
 508
 509         let mut count = entrycount as u16;
 510         let mut directory_size = central_dir_size as u32;
 511         let mut directory_offset = central_dir_offset as u32;
 512
 513         if central_dir_size > u32::MAX as usize
 514             || central_dir_offset > u32::MAX as usize
 515             || entrycount > u16::MAX as usize
 516         {
 517             count = 0xFFFF;
 518             directory_size = 0xFFFFFFFF;
 519             directory_offset = 0xFFFFFFFF;
 520
 521             write_struct(
 522                 &mut target,
 523                 Zip64EOCDRecord {
 524                     signature: ZIP64_EOCD_RECORD,
 525                     field_size: 44,
 526                     version_made_by: VERSION_MADE_BY,
 527                     version_needed: VERSION_NEEDED,
 528                     disk_number: 0,
 529                     disk_number_central_dir: 0,
 530                     disk_record_count: entrycount.try_into()?,
 531                     total_record_count: entrycount.try_into()?,
 532                     directory_size: central_dir_size.try_into()?,
 533                     directory_offset: central_dir_offset.try_into()?,
 534                 },
 535             )
 536             .await?;
 537
 538             let locator_offset = central_dir_offset + central_dir_size;
 539
 540             write_struct(
 541                 &mut target,
 542                 Zip64EOCDLocator {
 543                     signature: ZIP64_EOCD_LOCATOR,
 544                     disk_number: 0,
 545                     offset: locator_offset.try_into()?,
 546                     disk_count: 1,
 547                 },
 548             )
 549             .await?;
 550         }
 551
 552         write_struct(
 553             &mut target,
 554             EndOfCentralDir {
 555                 signature: END_OF_CENTRAL_DIR,
 556                 disk_number: 0,
 557                 start_disk: 0,
 558                 disk_record_count: count,
 559                 total_record_count: count,
 560                 directory_size,
 561                 directory_offset,
 562                 comment_len: 0,
 563             },
 564         )
 565         .await?;
 566
 567         self.target = Some(target);
 568
 569         Ok(())
 570     }
 571
 572     pub async fn finish(&mut self) -> Result<(), Error> {
 573         let mut target = self
 574             .target
 575             .take()
 576             .ok_or_else(|| format_err!("had no target during finish"))?;
 577         let central_dir_offset = self.byte_count;
 578         let mut central_dir_size = 0;
 579
 580         for file in &self.files {
 581             central_dir_size += file.write_central_directory_header(&mut target).await?;
 582         }
 583
 584         self.target = Some(target);
 585         self.write_eocd(central_dir_size, central_dir_offset)
 586             .await?;
 587
 588         self.target
 589             .take()
 590             .ok_or_else(|| format_err!("had no target for flush"))?
 591             .flush()
 592             .await?;
 593
 594         Ok(())
 595     }
 596 }
 597
 598 /// Zip a local directory and write encoded data to target. "source" has to point to a valid
 599 /// directory, it's name will be the root of the zip file - e.g.:
 600 /// source:
 601 ///         /foo/bar
 602 /// zip file:
 603 ///         /bar/file1
 604 ///         /bar/dir1
 605 ///         /bar/dir1/file2
 606 ///         ...
 607 /// ...except if "source" is the root directory
 608 pub async fn zip_directory<W>(target: W, source: &Path) -> Result<(), Error>
 609 where
 610     W: AsyncWrite + Unpin + Send,
 611 {
 612     use walkdir::WalkDir;
 613     use std::os::unix::fs::MetadataExt;
 614
 615     let base_path = source.parent().unwrap_or_else(|| Path::new("/"));
 616     let mut encoder = ZipEncoder::new(target);
 617
 618     for entry in WalkDir::new(&source).into_iter() {
 619         match entry {
 620             Ok(entry) => {
 621                 let entry_path = entry.path().to_owned();
 622                 let encoder = &mut encoder;
 623
 624                 if let Err(err) = async move {
 625                     let entry_path_no_base = entry.path().strip_prefix(base_path)?;
 626                     let metadata = entry.metadata()?;
 627                     let mtime = match metadata.modified().unwrap_or_else(|_| SystemTime::now()).duration_since(SystemTime::UNIX_EPOCH) {
 628                         Ok(dur) => dur.as_secs() as i64,
 629                         Err(time_error) => -(time_error.duration().as_secs() as i64)
 630                     };
 631                     let mode = metadata.mode() as u16;
 632
 633                     if entry.file_type().is_file() {
 634                         let file = tokio::fs::File::open(entry.path()).await?;
 635                         let ze = ZipEntry::new(
 636                             &entry_path_no_base,
 637                             mtime,
 638                             mode,
 639                             true,
 640                         );
 641                         encoder.add_entry(ze, Some(file)).await?;
 642                     } else if entry.file_type().is_dir() {
 643                         let ze = ZipEntry::new(
 644                             &entry_path_no_base,
 645                             mtime,
 646                             mode,
 647                             false,
 648                         );
 649                         let content: Option<tokio::fs::File> = None;
 650                         encoder.add_entry(ze, content).await?;
 651                     }
 652                     // ignore other file types
 653                     let ok: Result<(), Error> = Ok(());
 654                     ok
 655                 }
 656                 .await
 657                 {
 658                     eprintln!(
 659                         "zip: error encoding file or directory '{}': {}",
 660                         entry_path.display(),
 661                         err
 662                     );
 663                 }
 664             }
 665             Err(err) => {
 666                 eprintln!("zip: error reading directory entry: {}", err);
 667             }
 668         }
 669     }
 670
 671     encoder.finish().await
 672 }