"proxmox-api-macro",
"proxmox-borrow",
"proxmox-http",
+ "proxmox-io",
"proxmox-lang",
"proxmox-sortable-macro",
"proxmox-tfa",
proxmox-api-macro \
proxmox-borrow \
proxmox-http \
+ proxmox-io \
proxmox-lang \
proxmox-sortable-macro \
proxmox-tfa \
--- /dev/null
+[package]
+name = "proxmox-io"
+version = "1.0.0"
+authors = ["Proxmox Support Team <support@proxmox.com>"]
+edition = "2018"
+license = "AGPL-3"
+description = "extension traits for Read and Write"
+
+exclude = [ "debian" ]
+
+[dependencies]
+endian_trait = { version = "0.6", features = ["arrays"] }
+# this is also the feature name:
+tokio = { version = "1.0", features = [ "io-util" ], optional = true }
--- /dev/null
+rust-proxmox-io (1.0.0-1) stable; urgency=medium
+
+ * initial split out of `librust-proxmox-dev`
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 06 Oct 2021 11:04:36 +0200
--- /dev/null
+Source: rust-proxmox-io
+Section: rust
+Priority: optional
+Build-Depends: debhelper (>= 12),
+ dh-cargo (>= 24),
+ cargo:native <!nocheck>,
+ rustc:native <!nocheck>,
+ libstd-rust-dev <!nocheck>,
+ librust-endian-trait-0.6+arrays-dev <!nocheck>,
+ librust-endian-trait-0.6+default-dev <!nocheck>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.5.1
+Vcs-Git: git://git.proxmox.com/git/proxmox.git
+Vcs-Browser: https://git.proxmox.com/?p=proxmox.git
+Rules-Requires-Root: no
+
+Package: librust-proxmox-io-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-endian-trait-0.6+arrays-dev,
+ librust-endian-trait-0.6+default-dev
+Suggests:
+ librust-proxmox-io+tokio-dev (= ${binary:Version})
+Provides:
+ librust-proxmox-io+default-dev (= ${binary:Version}),
+ librust-proxmox-io-1-dev (= ${binary:Version}),
+ librust-proxmox-io-1+default-dev (= ${binary:Version}),
+ librust-proxmox-io-1.0-dev (= ${binary:Version}),
+ librust-proxmox-io-1.0+default-dev (= ${binary:Version}),
+ librust-proxmox-io-1.0.0-dev (= ${binary:Version}),
+ librust-proxmox-io-1.0.0+default-dev (= ${binary:Version})
+Description: Extension traits for Read and Write - Rust source code
+ This package contains the source for the Rust proxmox-io crate, packaged by
+ debcargo for use with cargo and dh-cargo.
+
+Package: librust-proxmox-io+tokio-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-proxmox-io-dev (= ${binary:Version}),
+ librust-tokio-1+default-dev,
+ librust-tokio-1+io-util-dev
+Provides:
+ librust-proxmox-io-1+tokio-dev (= ${binary:Version}),
+ librust-proxmox-io-1.0+tokio-dev (= ${binary:Version}),
+ librust-proxmox-io-1.0.0+tokio-dev (= ${binary:Version})
+Description: Extension traits for Read and Write - feature "tokio"
+ This metapackage enables feature "tokio" for the Rust proxmox-io crate, by
+ pulling in any additional dependencies needed by that feature.
--- /dev/null
+Copyright (C) 2021 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
--- /dev/null
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+
+[source]
+vcs_git = "git://git.proxmox.com/git/proxmox.git"
+vcs_browser = "https://git.proxmox.com/?p=proxmox.git"
--- /dev/null
+//! ByteBuffer
+//!
+//! a simple buffer for u8 with a practical api for reading and appending
+//! and consuming from the front
+//! Example:
+//! ```
+//! # use std::io::Read;
+//! # use proxmox_io::ByteBuffer;
+//! fn code<T: Read + ?Sized>(input: &mut T) -> std::io::Result<Box<[u8]>> {
+//! let mut buffer = ByteBuffer::new();
+//! let amount = buffer.read_from(input)?;
+//! let data = buffer.remove_data(amount);
+//! assert_eq!(data.len(), amount);
+//! Ok(data)
+//! }
+//! # code(&mut &b"testdata"[..]).expect("byte buffer test failed");
+//! ```
+
+use std::io::{Read, Result};
+
+use crate::vec;
+
+#[cfg(feature = "tokio")]
+use tokio::io::{AsyncRead, AsyncReadExt};
+
+/// A Buffer that holds bytes (u8)
+/// has convenience methods for reading (with any reader that
+/// implements std::io::Read or tokio::io::AsyncRead) onto the back
+/// and consuming from the front
+pub struct ByteBuffer {
+ buf: Box<[u8]>,
+ data_size: usize,
+ capacity: usize,
+}
+
+impl ByteBuffer {
+ /// Creates a new instance with a default capacity of 4096
+ pub fn new() -> Self {
+ Self::with_capacity(4096)
+ }
+
+ pub fn with_capacity(capacity: usize) -> Self {
+ Self {
+ buf: vec::undefined(capacity).into_boxed_slice(),
+ data_size: 0,
+ capacity,
+ }
+ }
+
+ pub fn free_size(&self) -> usize {
+ self.capacity - self.data_size
+ }
+
+ pub fn is_full(&self) -> bool {
+ self.data_size >= self.capacity
+ }
+
+ pub fn clear(&mut self) {
+ self.data_size = 0
+ }
+
+ /// Sets the length of the data. Useful if data was manually added
+ /// with a mutable slice (e.g. from [get_free_mut_slice](#method.get_free_mut_slice)).
+ ///
+ /// Panics when new size would be greater than capacity.
+ ///
+ /// Example:
+ /// ```
+ /// # use proxmox_io::ByteBuffer;
+ /// let mut buf = ByteBuffer::new();
+ /// buf.get_free_mut_slice()[..1].copy_from_slice(&[1u8]);
+ /// buf.add_size(1);
+ /// ```
+ ///
+ /// This code will panic:
+ /// ```should_panic
+ /// # use proxmox_io::ByteBuffer;
+ /// let mut buf = ByteBuffer::with_capacity(128);
+ /// buf.add_size(256);
+ /// ```
+ pub fn add_size(&mut self, size: usize) {
+ if self.data_size + size > self.capacity {
+ panic!("size bigger than capacity!");
+ }
+ self.data_size += size;
+ }
+
+ /// Returns a mutable reference to the free section of the
+ /// Buffer. There are no guarantees about the content of the
+ /// free part of the Buffer (may even be uninitialized).
+ /// see [add_size](#method.add_size) for a usage example.
+ pub fn get_free_mut_slice(&mut self) -> &mut [u8] {
+ &mut self.buf[self.data_size..self.capacity]
+ }
+
+ /// Removes up to max_amount of data from the front
+ /// of the buffer and returns. If there was less than max_amount present,
+ /// it will return as much data as there was in the buffer.
+ /// The rest of the data will be moved to the front, and
+ /// the data size will be updated accordingly.
+ ///
+ /// Example:
+ /// ```
+ /// # use proxmox_io::ByteBuffer;
+ /// let mut buf = ByteBuffer::new();
+ /// buf.get_free_mut_slice()[..2].copy_from_slice(&[1u8, 2u8]);
+ /// buf.add_size(2);
+ /// assert_eq!(buf.len(), 2);
+ ///
+ /// let data = buf.remove_data(100);
+ /// assert_eq!(&data[..], &[1u8, 2u8]);
+ /// assert!(buf.is_empty());
+ /// ```
+ #[must_use]
+ pub fn remove_data(&mut self, max_amount: usize) -> Box<[u8]> {
+ let size = max_amount.min(self.data_size);
+ let tmp: Box<[u8]> = self.buf[..size].into();
+ self.buf.copy_within(size..self.capacity, 0);
+ self.data_size -= size;
+ tmp
+ }
+
+ /// Removes up to max_amount of data from the front and returns
+ /// the amount of data removed. If there was less than max_amount present,
+ /// it will empty out the buffer and return the amount removed.
+ ///
+ /// Example:
+ /// ```
+ /// # use proxmox_io::ByteBuffer;
+ /// let mut buf = ByteBuffer::new();
+ /// buf.get_free_mut_slice()[..2].copy_from_slice(&[1u8, 2u8]);
+ /// buf.add_size(2);
+ /// assert_eq!(buf.len(), 2);
+ ///
+ /// let amount = buf.consume(1);
+ /// assert_eq!(amount, 1);
+ /// let amount = buf.consume(100);
+ /// assert_eq!(amount, 1);
+ /// assert!(buf.is_empty());
+ /// ```
+ pub fn consume(&mut self, max_amount: usize) -> usize {
+ let size = max_amount.min(self.data_size);
+ if size < max_amount {
+ self.clear()
+ } else {
+ self.buf.copy_within(size..self.capacity, 0);
+ self.data_size -= size;
+ }
+ size
+ }
+
+ /// Takes a reader and reads into the back of the buffer (up to the
+ /// free space in the buffer) and updates its size accordingly.
+ ///
+ /// Example:
+ /// ```
+ /// # use std::io;
+ /// # use proxmox_io::ByteBuffer;
+ /// fn code<R: io::Read>(mut reader: R) -> io::Result<()> {
+ /// let mut buf = ByteBuffer::new();
+ /// let res = buf.read_from(&mut reader)?;
+ /// // do something with the buffer
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn read_from<T: Read + ?Sized>(&mut self, input: &mut T) -> Result<usize> {
+ let amount = input.read(self.get_free_mut_slice())?;
+ self.add_size(amount);
+ Ok(amount)
+ }
+
+ /// Same as read_from, but for reader that implement tokio::io::AsyncRead.
+ /// See [read_from](#method.read_from) for an example
+ #[cfg(feature = "tokio")]
+ pub async fn read_from_async<T: AsyncRead + Unpin>(&mut self, input: &mut T) -> Result<usize> {
+ let amount = input.read(self.get_free_mut_slice()).await?;
+ self.add_size(amount);
+ Ok(amount)
+ }
+}
+
+impl Default for ByteBuffer {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl std::ops::Deref for ByteBuffer {
+ type Target = [u8];
+
+ fn deref(&self) -> &Self::Target {
+ &self.buf[..self.data_size]
+ }
+}
+
+impl std::ops::DerefMut for ByteBuffer {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.buf[..self.data_size]
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use super::ByteBuffer;
+
+ #[test]
+ fn test1() {
+ let mut buffer = ByteBuffer::new();
+ let slice = buffer.get_free_mut_slice();
+ for i in 0..slice.len() {
+ slice[i] = (i % 255) as u8;
+ }
+ buffer.add_size(5);
+
+ let slice2 = &buffer[..];
+
+ assert_eq!(slice2, &[0, 1, 2, 3, 4]);
+ }
+
+ #[test]
+ fn test2() {
+ let mut buffer = ByteBuffer::with_capacity(1024);
+ let size = buffer.read_from(&mut std::io::repeat(54)).unwrap();
+ assert_eq!(buffer.len(), size);
+ assert_eq!(buffer[0], 54);
+ }
+}
--- /dev/null
+//! Module providing I/O helpers (sync and async).
+//!
+//! The [`ReadExt`] trait provides additional operations for handling byte buffers for types
+//! implementing [`Read`](std::io::Read).
+
+mod read;
+pub use read::ReadExt;
+
+mod write;
+pub use write::WriteExt;
+
+mod sparse_copy;
+pub use sparse_copy::{buffer_is_zero, sparse_copy, SparseCopyResult};
+
+#[cfg(feature = "tokio")]
+pub use sparse_copy::sparse_copy_async;
+
+mod byte_buffer;
+pub use byte_buffer::ByteBuffer;
+
+pub mod vec;
--- /dev/null
+//! Helpers for `Read`.
+
+use std::io;
+use std::mem;
+
+use endian_trait::Endian;
+
+use crate::vec::{self, ByteVecExt};
+
+/// Adds some additional related functionality for types implementing [`Read`](std::io::Read).
+///
+/// Particularly for reading into a newly allocated buffer, appending to a `Vec<u8>` or reading
+/// values of a specific endianess (types implementing [`Endian`]).
+///
+/// Examples:
+/// ```no_run
+/// use proxmox_io::ReadExt;
+///
+/// # fn code() -> std::io::Result<()> {
+/// let mut file = std::fs::File::open("some.data")?;
+///
+/// // read some bytes into a newly allocated Vec<u8>:
+/// let mut data = file.read_exact_allocated(64)?;
+///
+/// // appending data to a vector:
+/// let actually_appended = file.append_to_vec(&mut data, 64)?; // .read() version
+/// file.append_exact_to_vec(&mut data, 64)?; // .read_exact() version
+/// # Ok(())
+/// # }
+/// ```
+///
+/// Or for reading values of a defined representation and endianess:
+///
+/// ```no_run
+/// # use endian_trait::Endian;
+/// # use proxmox_io::ReadExt;
+///
+/// #[derive(Endian)]
+/// #[repr(C)]
+/// struct Header {
+/// version: u16,
+/// data_size: u16,
+/// }
+///
+/// # fn code(mut file: std::fs::File) -> std::io::Result<()> {
+/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe:
+/// let header: Header = unsafe { file.read_le_value()? };
+/// let mut blob = file.read_exact_allocated(header.data_size as usize)?;
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+pub trait ReadExt {
+ /// Read data into a newly allocated vector. This is a shortcut for:
+ /// ```ignore
+ /// let mut data = Vec::with_capacity(len);
+ /// unsafe {
+ /// data.set_len(len);
+ /// }
+ /// reader.read_exact(&mut data)?;
+ /// ```
+ ///
+ /// With this trait, we just use:
+ /// ```no_run
+ /// use proxmox_io::ReadExt;
+ /// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> {
+ /// let data = reader.read_exact_allocated(len)?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>>;
+
+ /// Append data to a vector, growing it as necessary. Returns the amount of data appended.
+ fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize>;
+
+ /// Append an exact amount of data to a vector, growing it as necessary.
+ fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()>;
+
+ /// Read a value with host endianess.
+ ///
+ /// This is limited to types implementing the [`Endian`] trait under the assumption that
+ /// this is only done for types which are supposed to be read/writable directly.
+ ///
+ /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
+ /// this is considered unsafe.
+ ///
+ /// ```no_run
+ /// # use endian_trait::Endian;
+ /// use proxmox_io::ReadExt;
+ ///
+ /// #[derive(Endian)]
+ /// #[repr(C, packed)]
+ /// struct Data {
+ /// value: u16,
+ /// count: u32,
+ /// }
+ ///
+ /// # fn code() -> std::io::Result<()> {
+ /// let mut file = std::fs::File::open("my-raw.dat")?;
+ /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
+ /// // safely use our helper:
+ /// let data: Data = unsafe { file.read_host_value()? };
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ ///
+ /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+ unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T>;
+
+ /// Read a little endian value.
+ ///
+ /// The return type is required to implement the [`Endian`] trait, and we make the
+ /// assumption that this is only done for types which are supposed to be read/writable
+ /// directly.
+ ///
+ /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
+ /// this is considered unsafe.
+ ///
+ /// ```no_run
+ /// # use endian_trait::Endian;
+ /// use proxmox_io::ReadExt;
+ ///
+ /// #[derive(Endian)]
+ /// #[repr(C, packed)]
+ /// struct Data {
+ /// value: u16,
+ /// count: u32,
+ /// }
+ ///
+ /// # fn code() -> std::io::Result<()> {
+ /// let mut file = std::fs::File::open("my-little-endian.dat")?;
+ /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
+ /// // safely use our helper:
+ /// let data: Data = unsafe { file.read_le_value()? };
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ ///
+ /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+ unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T>;
+
+ /// Read a big endian value.
+ ///
+ /// The return type is required to implement the [`Endian`] trait, and we make the
+ /// assumption that this is only done for types which are supposed to be read/writable
+ /// directly.
+ ///
+ /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
+ /// this is considered unsafe.
+ ///
+ /// ```no_run
+ /// # use endian_trait::Endian;
+ /// use proxmox_io::ReadExt;
+ ///
+ /// #[derive(Endian)]
+ /// #[repr(C, packed)]
+ /// struct Data {
+ /// value: u16,
+ /// count: u32,
+ /// }
+ ///
+ /// # fn code() -> std::io::Result<()> {
+ /// let mut file = std::fs::File::open("my-big-endian.dat")?;
+ /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
+ /// // safely use our helper:
+ /// let data: Data = unsafe { file.read_be_value()? };
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ ///
+ /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+ unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T>;
+
+ /// Read a boxed value with host endianess.
+ ///
+ /// This is currently not limited to types implementing the [`Endian`] trait as in our use
+ /// cases we use this for types which are too big to want to always perform endian swaps
+ /// immediately on all values.
+ ///
+ /// ```
+ /// # use proxmox_io::vec;
+ /// use proxmox_io::ReadExt;
+ ///
+ /// #[repr(C)]
+ /// struct Data {
+ /// v1: u64,
+ /// buf: [u8; 4088],
+ /// }
+ ///
+ /// # let mut input = [0u8; 4096];
+ /// # use proxmox_io::WriteExt;
+ /// # let mut writer = &mut input[..];
+ /// # unsafe { writer.write_host_value(32u64).unwrap() };
+ /// # let mut file = &input[..];
+ ///
+ /// # fn code<T: std::io::Read>(mut file: T) -> std::io::Result<()> {
+ /// let data: Box<Data> = unsafe { file.read_host_value_boxed()? };
+ /// assert_eq!(data.v1, 32);
+ /// # Ok(())
+ /// # }
+ /// # code(&input[..]).unwrap();
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ unsafe fn read_host_value_boxed<T>(&mut self) -> io::Result<Box<T>>;
+
+ /// Try to read the exact number of bytes required to fill buf.
+ ///
+ /// This function reads as many bytes as necessary to completely
+ /// fill the specified buffer buf. If this function encounters an
+ /// "end of file" before getting any data, it returns Ok(false).
+ /// If there is some data, but not enough, it return an error of
+ /// the kind ErrorKind::UnexpectedEof. The contents of buf are
+ /// unspecified in this case.
+ fn read_exact_or_eof(&mut self, buf: &mut [u8]) -> io::Result<bool>;
+
+ /// Read until EOF
+ fn skip_to_end(&mut self) -> io::Result<usize>;
+}
+
+impl<R: io::Read> ReadExt for R {
+ fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>> {
+ let mut out = unsafe { vec::uninitialized(size) };
+ self.read_exact(&mut out)?;
+ Ok(out)
+ }
+
+ fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize> {
+ let pos = out.len();
+ unsafe {
+ out.grow_uninitialized(size);
+ }
+ let got = self.read(&mut out[pos..])?;
+ unsafe {
+ out.set_len(pos + got);
+ }
+ Ok(got)
+ }
+
+ fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()> {
+ let pos = out.len();
+ unsafe {
+ out.grow_uninitialized(size);
+ }
+ self.read_exact(&mut out[pos..])?;
+ Ok(())
+ }
+
+ unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T> {
+ let mut value = std::mem::MaybeUninit::<T>::uninit();
+ self.read_exact(std::slice::from_raw_parts_mut(
+ value.as_mut_ptr() as *mut u8,
+ mem::size_of::<T>(),
+ ))?;
+ Ok(value.assume_init())
+ }
+
+ unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T> {
+ Ok(self.read_host_value::<T>()?.from_le())
+ }
+
+ unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T> {
+ Ok(self.read_host_value::<T>()?.from_be())
+ }
+
+ unsafe fn read_host_value_boxed<T>(&mut self) -> io::Result<Box<T>> {
+ // FIXME: Change this once #![feature(new_uninit)] lands for Box<T>!
+
+ let ptr = std::alloc::alloc(std::alloc::Layout::new::<T>()) as *mut T;
+ self.read_exact(std::slice::from_raw_parts_mut(
+ ptr as *mut u8,
+ mem::size_of::<T>(),
+ ))?;
+ Ok(Box::from_raw(ptr))
+ }
+
+ fn read_exact_or_eof(&mut self, mut buf: &mut [u8]) -> io::Result<bool> {
+ let mut read_bytes = 0;
+ loop {
+ match self.read(&mut buf) {
+ Ok(0) => {
+ if read_bytes == 0 {
+ return Ok(false);
+ }
+ return Err(io::Error::new(
+ io::ErrorKind::UnexpectedEof,
+ "failed to fill whole buffer",
+ ));
+ }
+ Ok(n) => {
+ let tmp = buf;
+ buf = &mut tmp[n..];
+ read_bytes += n;
+ if buf.is_empty() {
+ return Ok(true);
+ }
+ }
+ Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+ Err(e) => return Err(e),
+ }
+ }
+ }
+
+ fn skip_to_end(&mut self) -> io::Result<usize> {
+ let mut skipped_bytes = 0;
+ let mut buf = unsafe { vec::uninitialized(32 * 1024) };
+ loop {
+ match self.read(&mut buf) {
+ Ok(0) => return Ok(skipped_bytes),
+ Ok(n) => skipped_bytes += n,
+ Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+ Err(e) => return Err(e),
+ }
+ }
+ }
+}
--- /dev/null
+use std::io::{self, ErrorKind, Read, Seek, SeekFrom, Write};
+
+#[cfg(feature = "tokio")]
+use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
+
+/// Efficiently check whether a byte slice contains only zeroes.
+///
+/// This is implemented in a way which allows the compiler to utilize SIMD instructions.
+pub fn buffer_is_zero(buf: &[u8]) -> bool {
+ !buf.chunks(128)
+ .map(|aa| aa.iter().fold(0, |a, b| a | b) != 0)
+ .any(|a| a)
+}
+
+/// Result of a sparse copy call.
+///
+/// Contains the amount of written/seeked bytes and if the last operation was a seek.
+#[must_use = "if sparse_copy ended with a seek(), the output file's size may need to be adjusted"]
+pub struct SparseCopyResult {
+ pub written: u64,
+ pub seeked_last: bool,
+}
+
+/// Copy similar to [`io::copy`](std::io::copy), but seeks the target when encountering zero bytes
+/// instead of writing them.
+///
+/// Example use:
+/// ```
+/// # use std::io;
+/// # use proxmox_io::sparse_copy;
+/// fn code<R, W>(mut reader: R, mut writer: W) -> io::Result<()>
+/// where
+/// R: io::Read,
+/// W: io::Write + io::Seek,
+/// {
+/// let res = sparse_copy(&mut reader, &mut writer)?;
+///
+/// println!("last part was seeked: {}", res.seeked_last);
+/// println!("written: {}", res.written);
+///
+/// Ok(())
+/// }
+/// ```
+pub fn sparse_copy<R: Read + ?Sized, W: Write + Seek + ?Sized>(
+ reader: &mut R,
+ writer: &mut W,
+) -> Result<SparseCopyResult, io::Error> {
+ let mut buf = crate::byte_buffer::ByteBuffer::with_capacity(4096);
+ let mut written = 0;
+ let mut seek_amount: i64 = 0;
+ let mut seeked_last = false;
+ loop {
+ buf.clear();
+ let len = match buf.read_from(reader) {
+ Ok(len) => len,
+ Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
+ Err(e) => return Err(e),
+ };
+
+ if len > 0 && buffer_is_zero(&buf[..]) {
+ seek_amount += len as i64;
+ continue;
+ }
+
+ if seek_amount > 0 {
+ writer.seek(SeekFrom::Current(seek_amount))?;
+ written += seek_amount as u64;
+ seek_amount = 0;
+ seeked_last = true;
+ }
+
+ if len > 0 {
+ writer.write_all(&buf[..])?;
+ written += len as u64;
+ seeked_last = false;
+ } else {
+ return Ok(SparseCopyResult {
+ written,
+ seeked_last,
+ });
+ }
+ }
+}
+
+/// copy similar to tokio::io::copy, but seeks the target when encountering
+/// zero bytes instead of writing them
+///
+/// Example:
+/// ```no_run
+/// # use std::io;
+/// # use tokio::io::{AsyncRead, AsyncWrite, AsyncSeek};
+/// # use proxmox_io::sparse_copy_async;
+/// async fn code<R, W>(mut reader: R, mut writer: W) -> io::Result<()>
+/// where
+/// R: AsyncRead + Unpin,
+/// W: AsyncWrite + AsyncSeek + Unpin,
+/// {
+/// let res = sparse_copy_async(&mut reader, &mut writer).await?;
+///
+/// println!("last part was seeked: {}", res.seeked_last);
+/// println!("written: {}", res.written);
+///
+/// Ok(())
+/// }
+/// ```
+#[cfg(feature = "tokio")]
+pub async fn sparse_copy_async<R, W>(
+ reader: &mut R,
+ writer: &mut W,
+) -> Result<SparseCopyResult, io::Error>
+where
+ R: AsyncRead + Unpin,
+ W: AsyncWrite + AsyncSeek + Unpin,
+{
+ let mut buf = crate::byte_buffer::ByteBuffer::with_capacity(4096);
+ let mut written = 0;
+ let mut seek_amount: i64 = 0;
+ let mut seeked_last = false;
+ loop {
+ buf.clear();
+ let len = match buf.read_from_async(reader).await {
+ Ok(len) => len,
+ Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
+ Err(e) => return Err(e),
+ };
+
+ if len > 0 && buffer_is_zero(&buf[..]) {
+ seek_amount += len as i64;
+ continue;
+ }
+
+ if seek_amount > 0 {
+ writer.seek(SeekFrom::Current(seek_amount)).await?;
+ written += seek_amount as u64;
+ seek_amount = 0;
+ seeked_last = true;
+ }
+
+ if len > 0 {
+ writer.write_all(&buf[..]).await?;
+ written += len as u64;
+ seeked_last = false;
+ } else {
+ return Ok(SparseCopyResult {
+ written,
+ seeked_last,
+ });
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use std::io::Cursor;
+
+ use super::sparse_copy;
+
+ const LEN: usize = 10000;
+
+ #[test]
+ fn test_sparse_copy() {
+ // test sparse
+ let mut test_data = Vec::new();
+ for _ in 0..LEN / 2 {
+ test_data.push(1u8);
+ }
+ for _ in 0..LEN / 2 {
+ test_data.push(0u8);
+ }
+ let mut test_data = Cursor::new(test_data);
+ let mut result_data = Cursor::new(vec![0; LEN]);
+
+ let result =
+ sparse_copy(&mut test_data, &mut result_data).expect("error during sparse copy");
+ assert_eq!(result.written, LEN as u64);
+ assert_eq!(result.seeked_last, true);
+ for i in 0..LEN {
+ if i < LEN / 2 {
+ assert_eq!(result_data.get_ref()[i], 1);
+ } else {
+ assert_eq!(result_data.get_ref()[i], 0);
+ }
+ }
+
+ // test non sparse
+ let mut test_data = Cursor::new(vec![1; LEN]);
+ let mut result_data = Cursor::new(vec![0; LEN]);
+
+ let result =
+ sparse_copy(&mut test_data, &mut result_data).expect("error during sparse copy");
+ assert_eq!(result.written, LEN as u64);
+ assert_eq!(result.seeked_last, false);
+ for i in 0..LEN {
+ assert_eq!(result_data.get_ref()[i], 1);
+ }
+ }
+
+ #[cfg(feature = "tokio")]
+ #[test]
+ fn test_sparse_copy_async() {
+ use std::future::Future;
+ use std::sync::Arc;
+ use std::task::{Context, Poll};
+
+ use super::sparse_copy_async;
+
+ struct PanicWaker;
+ impl std::task::Wake for PanicWaker {
+ fn wake(self: Arc<Self>) {
+ panic!("this test should not block");
+ }
+ }
+
+ let mut fut = async {
+ // test sparse
+ let mut test_data = Vec::new();
+ for _ in 0..LEN / 2 {
+ test_data.push(1u8);
+ }
+ for _ in 0..LEN / 2 {
+ test_data.push(0u8);
+ }
+ let mut test_data = Cursor::new(test_data);
+ let mut result_data = Cursor::new(vec![0; LEN]);
+
+ let result = sparse_copy_async(&mut test_data, &mut result_data)
+ .await
+ .expect("error during sparse copy");
+
+ assert_eq!(result.written, LEN as u64);
+ assert_eq!(result.seeked_last, true);
+ for i in 0..LEN {
+ if i < LEN / 2 {
+ assert_eq!(result_data.get_ref()[i], 1);
+ } else {
+ assert_eq!(result_data.get_ref()[i], 0);
+ }
+ }
+
+ // test non sparse
+ let mut test_data = Cursor::new(vec![1; LEN]);
+ let mut result_data = Cursor::new(vec![0; LEN]);
+
+ let result = sparse_copy_async(&mut test_data, &mut result_data)
+ .await
+ .expect("error during sparse copy");
+
+ assert_eq!(result.written, LEN as u64);
+ assert_eq!(result.seeked_last, false);
+ for i in 0..LEN {
+ assert_eq!(result_data.get_ref()[i], 1);
+ }
+ Ok::<(), std::io::Error>(())
+ };
+
+ let fut = unsafe { std::pin::Pin::new_unchecked(&mut fut) };
+ let waker = std::task::Waker::from(Arc::new(PanicWaker));
+ let mut context = Context::from_waker(&waker);
+ match fut.poll(&mut context) {
+ Poll::Ready(res) => res.expect("expected ok"),
+ Poll::Pending => panic!("yielded on blocking implementation"),
+ }
+ }
+}
--- /dev/null
+//! This module provides additional operations for `Vec<u8>`.
+//!
+//! Example:
+//! ```
+//! # use std::io::Read;
+//! use proxmox_io::vec::{self, ByteVecExt};
+//!
+//! fn append_1024_to_vec<T: Read>(mut input: T, buffer: &mut Vec<u8>) -> std::io::Result<()> {
+//! input.read_exact(unsafe { buffer.grow_uninitialized(1024) })
+//! }
+//! ```
+
+/// Some additional byte vector operations useful for I/O code.
+/// Example:
+/// ```
+/// # use std::io::Read;
+/// # use proxmox_io::ReadExt;
+/// use proxmox_io::vec::{self, ByteVecExt};
+///
+/// # fn code(mut file: std::fs::File, mut data: Vec<u8>) -> std::io::Result<()> {
+/// file.read_exact(unsafe {
+/// data.grow_uninitialized(1024)
+/// })?;
+/// # Ok(())
+/// # }
+/// ```
+///
+/// Note that this module also provides a safe alternative for the case where
+/// `grow_uninitialized()` is directly followed by a `read_exact()` call via the [`ReadExt`]
+/// trait:
+/// ```ignore
+/// file.append_to_vec(&mut data, 1024)?;
+/// ```
+///
+/// [`ReadExt`]: crate::ReadExt
+pub trait ByteVecExt {
+ /// Grow a vector without initializing its elements. The difference to simply using `reserve`
+ /// is that it also updates the actual length, making the newly allocated data part of the
+ /// slice.
+ ///
+ /// This is a shortcut for:
+ /// ```ignore
+ /// vec.reserve(more);
+ /// let total = vec.len() + more;
+ /// unsafe {
+ /// vec.set_len(total);
+ /// }
+ /// ```
+ ///
+ /// This returns a mutable slice to the newly allocated space, so it can be used inline:
+ /// ```
+ /// # use std::io::Read;
+ /// # use proxmox_io::vec::ByteVecExt;
+ /// # fn test(mut file: std::fs::File, buffer: &mut Vec<u8>) -> std::io::Result<()> {
+ /// file.read_exact(unsafe { buffer.grow_uninitialized(1024) })?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// Although for the above case it is recommended to use the even shorter version from the
+ /// [`ReadExt`] trait:
+ /// ```ignore
+ /// // use crate::tools::vec::ByteVecExt;
+ /// file.append_to_vec(&mut buffer, 1024)?;
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// When increasing the size, the new contents are uninitialized and have nothing to do with
+ /// the previously contained content. Since we cannot track this state through the type system,
+ /// this method is marked as an unsafe API for good measure.
+ ///
+ /// [`ReadExt`]: crate::ReadExt
+ unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8];
+
+ /// Resize a vector to a specific size without initializing its data. This is a shortcut for:
+ /// ```ignore
+ /// if new_size <= vec.len() {
+ /// vec.truncate(new_size);
+ /// } else {
+ /// unsafe {
+ /// vec.grow_uninitialized(new_size - vec.len());
+ /// }
+ /// }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// When increasing the size, the new contents are uninitialized and have nothing to do with
+ /// the previously contained content. Since we cannot track this state through the type system,
+ /// this method is marked as an unsafe API for good measure.
+ unsafe fn resize_uninitialized(&mut self, total: usize);
+}
+
+impl ByteVecExt for Vec<u8> {
+ unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8] {
+ let old_len = self.len();
+ self.reserve(more);
+ let total = old_len + more;
+ self.set_len(total);
+ &mut self[old_len..]
+ }
+
+ unsafe fn resize_uninitialized(&mut self, new_size: usize) {
+ if new_size <= self.len() {
+ self.truncate(new_size);
+ } else {
+ self.grow_uninitialized(new_size - self.len());
+ }
+ }
+}
--- /dev/null
+//! Byte vector helpers.
+//!
+//! We have a lot of I/O code such as:
+//! ```ignore
+//! let mut buffer = vec![0u8; header_size];
+//! file.read_exact(&mut buffer)?;
+//! ```
+//! (We even have this case with a 4M buffer!)
+//!
+//! This needlessly initializes the buffer to zero (which not only wastes time (an insane amount of
+//! time on debug builds, actually) but also prevents tools such as valgrind from pointing out
+//! access to actually uninitialized data, which may hide bugs...)
+//!
+//! This module provides some helpers for this kind of code. Many of these are supposed to stay on
+//! a lower level, with I/O helpers for types implementing [`Read`](std::io::Read) being available
+//! in this module.
+//!
+//! Examples:
+//! ```no_run
+//! use proxmox_io::vec::{self, ByteVecExt};
+//!
+//! # let size = 64usize;
+//! # let more = 64usize;
+//! let mut buffer = vec::undefined(size); // A zero-initialized buffer with valgrind support
+//!
+//! let mut buffer = unsafe { vec::uninitialized(size) }; // an actually uninitialized buffer
+//! vec::clear(&mut buffer); // zero out an &mut [u8]
+//!
+//! vec::clear(unsafe {
+//! buffer.grow_uninitialized(more) // grow the buffer with uninitialized bytes
+//! });
+//! ```
+
+mod byte_vec;
+pub use byte_vec::ByteVecExt;
+
+/// Create an uninitialized byte vector of a specific size.
+///
+/// This is just a shortcut for:
+/// ```no_run
+/// # let len = 64usize;
+/// let mut v = Vec::<u8>::with_capacity(len);
+/// unsafe {
+/// v.set_len(len);
+/// }
+/// ```
+///
+/// # Safety
+///
+/// It's generally not unsafe to use this method, but the contents are uninitialized, and since
+/// this does not return a `MaybeUninit` type to track the initialization state, this is simply
+/// marked as unsafe for good measure.
+#[inline]
+pub unsafe fn uninitialized(len: usize) -> Vec<u8> {
+ let mut out = Vec::with_capacity(len);
+ out.set_len(len);
+ out
+}
+
+/// Shortcut to zero out a slice of bytes.
+#[inline]
+pub fn clear(data: &mut [u8]) {
+ unsafe {
+ std::ptr::write_bytes(data.as_mut_ptr(), 0, data.len());
+ }
+}
+
+/// Create a newly allocated, zero initialized byte vector.
+#[inline]
+pub fn zeroed(len: usize) -> Vec<u8> {
+ unsafe {
+ let mut out = uninitialized(len);
+ clear(&mut out);
+ out
+ }
+}
+
+/// Create a newly allocated byte vector of a specific size with "undefined" content.
+///
+/// The data will be zero initialized, but, if the `valgrind` feature is activated, it will be
+/// marked as uninitialized for debugging.
+#[inline]
+pub fn undefined(len: usize) -> Vec<u8> {
+ undefined_impl(len)
+}
+
+#[cfg(not(feature = "valgrind"))]
+fn undefined_impl(len: usize) -> Vec<u8> {
+ zeroed(len)
+}
+
+#[cfg(feature = "valgrind")]
+fn undefined_impl(len: usize) -> Vec<u8> {
+ let out = zeroed(len);
+ vg::make_slice_undefined(&out[..]);
+ out
+}
+
+#[cfg(feature = "valgrind")]
+mod vg {
+ type ValgrindValue = valgrind_request::Value;
+
+ /// Mark a memory region as undefined when using valgrind, causing it to treat read access to
+ /// it as error.
+ #[inline]
+ pub(crate) fn make_mem_undefined(addr: *const u8, len: usize) -> ValgrindValue {
+ const MAKE_MEM_UNDEFINED: ValgrindValue =
+ (((b'M' as ValgrindValue) << 24) | ((b'C' as ValgrindValue) << 16)) + 1;
+ unsafe {
+ valgrind_request::do_client_request(
+ 0,
+ &[
+ MAKE_MEM_UNDEFINED,
+ addr as usize as ValgrindValue,
+ len as ValgrindValue,
+ 0,
+ 0,
+ 0,
+ ],
+ )
+ }
+ }
+
+ /// Mark a slice of bytes as undefined when using valgrind, causing it to treat read access to
+ /// it as error.
+ #[inline]
+ pub(crate) fn make_slice_undefined(data: &[u8]) -> ValgrindValue {
+ make_mem_undefined(data.as_ptr(), data.len())
+ }
+}
--- /dev/null
+//! Helpers for `Write`.
+
+use std::io;
+
+use endian_trait::Endian;
+
+/// Adds some additional related functionality for types implementing [`Write`](std::io::Write).
+///
+/// Particularly for writing values of a specific endianess (types implementing [`Endian`]).
+///
+/// Examples:
+/// ```no_run
+/// # use endian_trait::Endian;
+/// # use proxmox_io::WriteExt;
+///
+/// #[derive(Endian)]
+/// #[repr(C)]
+/// struct Header {
+/// version: u16,
+/// data_size: u16,
+/// }
+///
+/// # fn code(mut file: std::fs::File) -> std::io::Result<()> {
+/// let header = Header {
+/// version: 1,
+/// data_size: 16,
+/// };
+/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe:
+/// unsafe {
+/// file.write_le_value(header)?;
+/// }
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+pub trait WriteExt {
+ /// Write a value with host endianess.
+ ///
+ /// This is limited to types implementing the [`Endian`] trait under the assumption that this
+ /// is only done for types which are supposed to be read/writable directly.
+ ///
+ /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
+ /// this is considered unsafe.
+ ///
+ /// The underlying write call is `.write_all()`, so there are no partial writes.
+ ///
+ /// ```no_run
+ /// # use endian_trait::Endian;
+ /// use proxmox_io::WriteExt;
+ ///
+ /// #[derive(Endian)]
+ /// #[repr(C, packed)]
+ /// struct Data {
+ /// value: u16,
+ /// count: u32,
+ /// }
+ ///
+ /// # fn code() -> std::io::Result<()> {
+ /// let mut file = std::fs::File::create("my-raw.dat")?;
+ /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
+ /// // safely use our helper:
+ /// unsafe {
+ /// file.write_host_value(Data {
+ /// value: 1,
+ /// count: 2,
+ /// })?;
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ ///
+ /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+ unsafe fn write_host_value<T: Endian>(&mut self, value: T) -> io::Result<()>;
+
+ /// Write a little endian value.
+ ///
+ /// The input type is required to implement the [`Endian`] trait, and we make the assumption
+ /// that this is only done for types which are supposed to be read/writable directly.
+ ///
+ /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
+ /// this is considered unsafe.
+ ///
+ /// The underlying write call is `.write_all()`, so there are no partial writes.
+ ///
+ /// ```no_run
+ /// # use endian_trait::Endian;
+ /// use proxmox_io::WriteExt;
+ ///
+ /// #[derive(Endian)]
+ /// #[repr(C, packed)]
+ /// struct Data {
+ /// value: u16,
+ /// count: u32,
+ /// }
+ ///
+ /// # fn code() -> std::io::Result<()> {
+ /// let mut file = std::fs::File::create("my-raw.dat")?;
+ /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
+ /// // safely use our helper:
+ /// unsafe {
+ /// file.write_le_value(Data {
+ /// value: 1,
+ /// count: 2,
+ /// })?;
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ ///
+ /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+ unsafe fn write_le_value<T: Endian>(&mut self, value: T) -> io::Result<()>;
+
+ /// Read a big endian value.
+ ///
+ /// The input type is required to implement the [`Endian`] trait, and we make the assumption
+ /// that this is only done for types which are supposed to be read/writable directly.
+ ///
+ /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
+ /// this is considered unsafe.
+ ///
+ /// The underlying write call is `.write_all()`, so there are no partial writes.
+ ///
+ /// ```no_run
+ /// # use endian_trait::Endian;
+ /// use proxmox_io::WriteExt;
+ ///
+ /// #[derive(Endian)]
+ /// #[repr(C, packed)]
+ /// struct Data {
+ /// value: u16,
+ /// count: u32,
+ /// }
+ ///
+ /// # fn code() -> std::io::Result<()> {
+ /// let mut file = std::fs::File::create("my-raw.dat")?;
+ /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
+ /// // safely use our helper:
+ /// unsafe {
+ /// file.write_be_value(Data {
+ /// value: 1,
+ /// count: 2,
+ /// })?;
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Safety
+ ///
+ /// This should only used for types with a defined storage representation, usually
+ /// `#[repr(C)]`, otherwise the results may be inconsistent.
+ ///
+ /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
+ unsafe fn write_be_value<T: Endian>(&mut self, value: T) -> io::Result<()>;
+}
+
+impl<W: io::Write> WriteExt for W {
+ unsafe fn write_host_value<T: Endian>(&mut self, value: T) -> io::Result<()> {
+ self.write_all(std::slice::from_raw_parts(
+ &value as *const T as *const u8,
+ std::mem::size_of::<T>(),
+ ))
+ }
+
+ unsafe fn write_le_value<T: Endian>(&mut self, value: T) -> io::Result<()> {
+ self.write_host_value::<T>(value.to_le())
+ }
+
+ unsafe fn write_be_value<T: Endian>(&mut self, value: T) -> io::Result<()> {
+ self.write_host_value::<T>(value.to_be())
+ }
+}
+++ /dev/null
-//! ByteBuffer
-//!
-//! a simple buffer for u8 with a practical api for reading and appending
-//! and consuming from the front
-//! Example:
-//! ```
-//! # use std::io::Read;
-//! # use proxmox::tools::byte_buffer::ByteBuffer;
-//! fn code<T: Read + ?Sized>(input: &mut T) -> std::io::Result<Box<[u8]>> {
-//! let mut buffer = ByteBuffer::new();
-//! let amount = buffer.read_from(input)?;
-//! let data = buffer.remove_data(amount);
-//! assert_eq!(data.len(), amount);
-//! Ok(data)
-//! }
-//! # code(&mut &b"testdata"[..]).expect("byte buffer test failed");
-//! ```
-
-use std::io::{Read, Result};
-
-use crate::tools::vec;
-
-#[cfg(feature = "tokio")]
-use tokio::io::{AsyncRead, AsyncReadExt};
-
-/// A Buffer that holds bytes (u8)
-/// has convenience methods for reading (with any reader that
-/// implements std::io::Read or tokio::io::AsyncRead) onto the back
-/// and consuming from the front
-pub struct ByteBuffer {
- buf: Box<[u8]>,
- data_size: usize,
- capacity: usize,
-}
-
-impl ByteBuffer {
- /// Creates a new instance with a default capacity of 4096
- pub fn new() -> Self {
- Self::with_capacity(4096)
- }
-
- pub fn with_capacity(capacity: usize) -> Self {
- Self {
- buf: vec::undefined(capacity).into_boxed_slice(),
- data_size: 0,
- capacity,
- }
- }
-
- pub fn free_size(&self) -> usize {
- self.capacity - self.data_size
- }
-
- pub fn is_full(&self) -> bool {
- self.data_size >= self.capacity
- }
-
- pub fn clear(&mut self) {
- self.data_size = 0
- }
-
- /// Sets the length of the data. Useful if data was manually added
- /// with a mutable slice (e.g. from [get_free_mut_slice](#method.get_free_mut_slice)).
- ///
- /// Panics when new size would be greater than capacity.
- ///
- /// Example:
- /// ```
- /// # use proxmox::tools::byte_buffer::ByteBuffer;
- /// let mut buf = ByteBuffer::new();
- /// buf.get_free_mut_slice()[..1].copy_from_slice(&[1u8]);
- /// buf.add_size(1);
- /// ```
- ///
- /// This code will panic:
- /// ```should_panic
- /// # use proxmox::tools::byte_buffer::ByteBuffer;
- /// let mut buf = ByteBuffer::with_capacity(128);
- /// buf.add_size(256);
- /// ```
- pub fn add_size(&mut self, size: usize) {
- if self.data_size + size > self.capacity {
- panic!("size bigger than capacity!");
- }
- self.data_size += size;
- }
-
- /// Returns a mutable reference to the free section of the
- /// Buffer. There are no guarantees about the content of the
- /// free part of the Buffer (may even be uninitialized).
- /// see [add_size](#method.add_size) for a usage example.
- pub fn get_free_mut_slice(&mut self) -> &mut [u8] {
- &mut self.buf[self.data_size..self.capacity]
- }
-
- /// Removes up to max_amount of data from the front
- /// of the buffer and returns. If there was less than max_amount present,
- /// it will return as much data as there was in the buffer.
- /// The rest of the data will be moved to the front, and
- /// the data size will be updated accordingly.
- ///
- /// Example:
- /// ```
- /// # use proxmox::tools::byte_buffer::ByteBuffer;
- /// let mut buf = ByteBuffer::new();
- /// buf.get_free_mut_slice()[..2].copy_from_slice(&[1u8, 2u8]);
- /// buf.add_size(2);
- /// assert_eq!(buf.len(), 2);
- ///
- /// let data = buf.remove_data(100);
- /// assert_eq!(&data[..], &[1u8, 2u8]);
- /// assert!(buf.is_empty());
- /// ```
- #[must_use]
- pub fn remove_data(&mut self, max_amount: usize) -> Box<[u8]> {
- let size = max_amount.min(self.data_size);
- let tmp: Box<[u8]> = self.buf[..size].into();
- self.buf.copy_within(size..self.capacity, 0);
- self.data_size -= size;
- tmp
- }
-
- /// Removes up to max_amount of data from the front and returns
- /// the amount of data removed. If there was less than max_amount present,
- /// it will empty out the buffer and return the amount removed.
- ///
- /// Example:
- /// ```
- /// # use proxmox::tools::byte_buffer::ByteBuffer;
- /// let mut buf = ByteBuffer::new();
- /// buf.get_free_mut_slice()[..2].copy_from_slice(&[1u8, 2u8]);
- /// buf.add_size(2);
- /// assert_eq!(buf.len(), 2);
- ///
- /// let amount = buf.consume(1);
- /// assert_eq!(amount, 1);
- /// let amount = buf.consume(100);
- /// assert_eq!(amount, 1);
- /// assert!(buf.is_empty());
- /// ```
- pub fn consume(&mut self, max_amount: usize) -> usize {
- let size = max_amount.min(self.data_size);
- if size < max_amount {
- self.clear()
- } else {
- self.buf.copy_within(size..self.capacity, 0);
- self.data_size -= size;
- }
- size
- }
-
- /// Takes a reader and reads into the back of the buffer (up to the
- /// free space in the buffer) and updates its size accordingly.
- ///
- /// Example:
- /// ```
- /// # use std::io;
- /// # use proxmox::tools::byte_buffer::ByteBuffer;
- /// fn code<R: io::Read>(mut reader: R) -> io::Result<()> {
- /// let mut buf = ByteBuffer::new();
- /// let res = buf.read_from(&mut reader)?;
- /// // do something with the buffer
- /// Ok(())
- /// }
- /// ```
- pub fn read_from<T: Read + ?Sized>(&mut self, input: &mut T) -> Result<usize> {
- let amount = input.read(self.get_free_mut_slice())?;
- self.add_size(amount);
- Ok(amount)
- }
-
- /// Same as read_from, but for reader that implement tokio::io::AsyncRead.
- /// See [read_from](#method.read_from) for an example
- #[cfg(feature = "tokio")]
- pub async fn read_from_async<T: AsyncRead + Unpin>(&mut self, input: &mut T) -> Result<usize> {
- let amount = input.read(self.get_free_mut_slice()).await?;
- self.add_size(amount);
- Ok(amount)
- }
-}
-
-impl Default for ByteBuffer {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl std::ops::Deref for ByteBuffer {
- type Target = [u8];
-
- fn deref(&self) -> &Self::Target {
- &self.buf[..self.data_size]
- }
-}
-
-impl std::ops::DerefMut for ByteBuffer {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.buf[..self.data_size]
- }
-}
-
-#[cfg(test)]
-mod test {
- use crate::tools::byte_buffer::ByteBuffer;
-
- #[test]
- fn test1() {
- let mut buffer = ByteBuffer::new();
- let slice = buffer.get_free_mut_slice();
- for i in 0..slice.len() {
- slice[i] = (i % 255) as u8;
- }
- buffer.add_size(5);
-
- let slice2 = &buffer[..];
-
- assert_eq!(slice2, &[0, 1, 2, 3, 4]);
- }
-
- #[test]
- fn test2() {
- let mut buffer = ByteBuffer::with_capacity(1024);
- let size = buffer.read_from(&mut std::io::repeat(54)).unwrap();
- assert_eq!(buffer.len(), size);
- assert_eq!(buffer[0], 54);
- }
-}
+++ /dev/null
-//! Module providing I/O helpers (sync and async).
-//!
-//! The [`ReadExt`] trait provides additional operations for handling byte buffers for types
-//! implementing [`Read`](std::io::Read).
-
-use std::io::{self, ErrorKind, Read, Seek, SeekFrom, Write};
-
-mod read;
-pub use read::*;
-
-mod write;
-pub use write::*;
-
-fn buffer_is_zero(buf: &[u8]) -> bool {
- !buf.chunks(128)
- .map(|aa| aa.iter().fold(0, |a, b| a | b) != 0)
- .any(|a| a)
-}
-
-/// Result of a sparse copy call
-/// contains the amount of written/seeked bytes
-/// and if the last operation was a seek
-pub struct SparseCopyResult {
- pub written: u64,
- pub seeked_last: bool,
-}
-
-/// copy similar to io::copy, but seeks the target when encountering
-/// zero bytes instead of writing them
-///
-/// Example use:
-/// ```
-/// # use std::io;
-/// # use proxmox::tools::io::sparse_copy;
-/// fn code<R, W>(mut reader: R, mut writer: W) -> io::Result<()>
-/// where
-/// R: io::Read,
-/// W: io::Write + io::Seek,
-/// {
-/// let res = sparse_copy(&mut reader, &mut writer)?;
-///
-/// println!("last part was seeked: {}", res.seeked_last);
-/// println!("written: {}", res.written);
-///
-/// Ok(())
-/// }
-/// ```
-pub fn sparse_copy<R: Read + ?Sized, W: Write + Seek + ?Sized>(
- reader: &mut R,
- writer: &mut W,
-) -> Result<SparseCopyResult, io::Error> {
- let mut buf = crate::tools::byte_buffer::ByteBuffer::with_capacity(4096);
- let mut written = 0;
- let mut seek_amount: i64 = 0;
- let mut seeked_last = false;
- loop {
- buf.clear();
- let len = match buf.read_from(reader) {
- Ok(len) => len,
- Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
- Err(e) => return Err(e),
- };
-
- if len > 0 && buffer_is_zero(&buf[..]) {
- seek_amount += len as i64;
- continue;
- }
-
- if seek_amount > 0 {
- writer.seek(SeekFrom::Current(seek_amount))?;
- written += seek_amount as u64;
- seek_amount = 0;
- seeked_last = true;
- }
-
- if len > 0 {
- writer.write_all(&buf[..])?;
- written += len as u64;
- seeked_last = false;
- } else {
- return Ok(SparseCopyResult {
- written,
- seeked_last,
- });
- }
- }
-}
-
-#[cfg(feature = "tokio")]
-use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
-
-#[cfg(feature = "tokio")]
-/// copy similar to tokio::io::copy, but seeks the target when encountering
-/// zero bytes instead of writing them
-///
-/// Example:
-/// ```no_run
-/// # use std::io;
-/// # use tokio::io::{AsyncRead, AsyncWrite, AsyncSeek};
-/// # use proxmox::tools::io::sparse_copy_async;
-/// async fn code<R, W>(mut reader: R, mut writer: W) -> io::Result<()>
-/// where
-/// R: AsyncRead + Unpin,
-/// W: AsyncWrite + AsyncSeek + Unpin,
-/// {
-/// let res = sparse_copy_async(&mut reader, &mut writer).await?;
-///
-/// println!("last part was seeked: {}", res.seeked_last);
-/// println!("written: {}", res.written);
-///
-/// Ok(())
-/// }
-/// ```
-pub async fn sparse_copy_async<R, W>(
- reader: &mut R,
- writer: &mut W,
-) -> Result<SparseCopyResult, io::Error>
-where
- R: AsyncRead + Unpin,
- W: AsyncWrite + AsyncSeek + Unpin,
-{
- let mut buf = crate::tools::byte_buffer::ByteBuffer::with_capacity(4096);
- let mut written = 0;
- let mut seek_amount: i64 = 0;
- let mut seeked_last = false;
- loop {
- buf.clear();
- let len = match buf.read_from_async(reader).await {
- Ok(len) => len,
- Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
- Err(e) => return Err(e),
- };
-
- if len > 0 && buffer_is_zero(&buf[..]) {
- seek_amount += len as i64;
- continue;
- }
-
- if seek_amount > 0 {
- writer.seek(SeekFrom::Current(seek_amount)).await?;
- written += seek_amount as u64;
- seek_amount = 0;
- seeked_last = true;
- }
-
- if len > 0 {
- writer.write_all(&buf[..]).await?;
- written += len as u64;
- seeked_last = false;
- } else {
- return Ok(SparseCopyResult {
- written,
- seeked_last,
- });
- }
- }
-}
-
-#[cfg(test)]
-mod test {
- use std::io::Cursor;
-
- use crate::test::io::{AsyncBlockingReader, AsyncBlockingWriter};
- use crate::tools::io::{sparse_copy, sparse_copy_async};
-
- const LEN: usize = 10000;
-
- #[test]
- fn test_sparse_copy() {
- // test sparse
- let mut test_data = Vec::new();
- for _ in 0..LEN / 2 {
- test_data.push(1u8);
- }
- for _ in 0..LEN / 2 {
- test_data.push(0u8);
- }
- let mut test_data = Cursor::new(test_data);
- let mut result_data = Cursor::new(vec![0; LEN]);
-
- let result =
- sparse_copy(&mut test_data, &mut result_data).expect("error during sparse copy");
- assert_eq!(result.written, LEN as u64);
- assert_eq!(result.seeked_last, true);
- for i in 0..LEN {
- if i < LEN / 2 {
- assert_eq!(result_data.get_ref()[i], 1);
- } else {
- assert_eq!(result_data.get_ref()[i], 0);
- }
- }
-
- // test non sparse
- let mut test_data = Cursor::new(vec![1; LEN]);
- let mut result_data = Cursor::new(vec![0; LEN]);
-
- let result =
- sparse_copy(&mut test_data, &mut result_data).expect("error during sparse copy");
- assert_eq!(result.written, LEN as u64);
- assert_eq!(result.seeked_last, false);
- for i in 0..LEN {
- assert_eq!(result_data.get_ref()[i], 1);
- }
- }
-
- #[test]
- fn test_sparse_copy_async() {
- let fut = async {
- // test sparse
- let mut test_data = Vec::new();
- for _ in 0..LEN / 2 {
- test_data.push(1u8);
- }
- for _ in 0..LEN / 2 {
- test_data.push(0u8);
- }
- let mut test_data = AsyncBlockingReader::new(Cursor::new(test_data));
- let mut result_data = AsyncBlockingWriter::new(Cursor::new(vec![0; LEN]));
-
- let result = sparse_copy_async(&mut test_data, &mut result_data)
- .await
- .expect("error during sparse copy");
-
- assert_eq!(result.written, LEN as u64);
- assert_eq!(result.seeked_last, true);
- for i in 0..LEN {
- if i < LEN / 2 {
- assert_eq!(result_data.inner().get_ref()[i], 1);
- } else {
- assert_eq!(result_data.inner().get_ref()[i], 0);
- }
- }
-
- // test non sparse
- let mut test_data = AsyncBlockingReader::new(Cursor::new(vec![1; LEN]));
- let mut result_data = AsyncBlockingWriter::new(Cursor::new(vec![0; LEN]));
-
- let result = sparse_copy_async(&mut test_data, &mut result_data)
- .await
- .expect("error during sparse copy");
-
- assert_eq!(result.written, LEN as u64);
- assert_eq!(result.seeked_last, false);
- for i in 0..LEN {
- assert_eq!(result_data.inner().get_ref()[i], 1);
- }
- Ok(())
- };
-
- crate::test::task::poll_result_once(fut).expect("ok")
- }
-}
+++ /dev/null
-//! Helpers for `Read`.
-
-use std::io;
-use std::mem;
-
-use endian_trait::Endian;
-
-use crate::tools::vec::{self, ByteVecExt};
-
-/// Adds some additional related functionality for types implementing [`Read`](std::io::Read).
-///
-/// Particularly for reading into a newly allocated buffer, appending to a `Vec<u8>` or reading
-/// values of a specific endianess (types implementing [`Endian`]).
-///
-/// Examples:
-/// ```no_run
-/// use proxmox::tools::io::ReadExt;
-///
-/// # fn code() -> std::io::Result<()> {
-/// let mut file = std::fs::File::open("some.data")?;
-///
-/// // read some bytes into a newly allocated Vec<u8>:
-/// let mut data = file.read_exact_allocated(64)?;
-///
-/// // appending data to a vector:
-/// let actually_appended = file.append_to_vec(&mut data, 64)?; // .read() version
-/// file.append_exact_to_vec(&mut data, 64)?; // .read_exact() version
-/// # Ok(())
-/// # }
-/// ```
-///
-/// Or for reading values of a defined representation and endianess:
-///
-/// ```no_run
-/// # use endian_trait::Endian;
-/// # use proxmox::tools::io::ReadExt;
-///
-/// #[derive(Endian)]
-/// #[repr(C)]
-/// struct Header {
-/// version: u16,
-/// data_size: u16,
-/// }
-///
-/// # fn code(mut file: std::fs::File) -> std::io::Result<()> {
-/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe:
-/// let header: Header = unsafe { file.read_le_value()? };
-/// let mut blob = file.read_exact_allocated(header.data_size as usize)?;
-/// # Ok(())
-/// # }
-/// ```
-///
-/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
-pub trait ReadExt {
- /// Read data into a newly allocated vector. This is a shortcut for:
- /// ```ignore
- /// let mut data = Vec::with_capacity(len);
- /// unsafe {
- /// data.set_len(len);
- /// }
- /// reader.read_exact(&mut data)?;
- /// ```
- ///
- /// With this trait, we just use:
- /// ```no_run
- /// use proxmox::tools::io::ReadExt;
- /// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> {
- /// let data = reader.read_exact_allocated(len)?;
- /// # Ok(())
- /// # }
- /// ```
- fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>>;
-
- /// Append data to a vector, growing it as necessary. Returns the amount of data appended.
- fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize>;
-
- /// Append an exact amount of data to a vector, growing it as necessary.
- fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()>;
-
- /// Read a value with host endianess.
- ///
- /// This is limited to types implementing the [`Endian`] trait under the assumption that
- /// this is only done for types which are supposed to be read/writable directly.
- ///
- /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
- /// this is considered unsafe.
- ///
- /// ```no_run
- /// # use endian_trait::Endian;
- /// use proxmox::tools::io::ReadExt;
- ///
- /// #[derive(Endian)]
- /// #[repr(C, packed)]
- /// struct Data {
- /// value: u16,
- /// count: u32,
- /// }
- ///
- /// # fn code() -> std::io::Result<()> {
- /// let mut file = std::fs::File::open("my-raw.dat")?;
- /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
- /// // safely use our helper:
- /// let data: Data = unsafe { file.read_host_value()? };
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- ///
- /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
- unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T>;
-
- /// Read a little endian value.
- ///
- /// The return type is required to implement the [`Endian`] trait, and we make the
- /// assumption that this is only done for types which are supposed to be read/writable
- /// directly.
- ///
- /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
- /// this is considered unsafe.
- ///
- /// ```no_run
- /// # use endian_trait::Endian;
- /// use proxmox::tools::io::ReadExt;
- ///
- /// #[derive(Endian)]
- /// #[repr(C, packed)]
- /// struct Data {
- /// value: u16,
- /// count: u32,
- /// }
- ///
- /// # fn code() -> std::io::Result<()> {
- /// let mut file = std::fs::File::open("my-little-endian.dat")?;
- /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
- /// // safely use our helper:
- /// let data: Data = unsafe { file.read_le_value()? };
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- ///
- /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
- unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T>;
-
- /// Read a big endian value.
- ///
- /// The return type is required to implement the [`Endian`] trait, and we make the
- /// assumption that this is only done for types which are supposed to be read/writable
- /// directly.
- ///
- /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
- /// this is considered unsafe.
- ///
- /// ```no_run
- /// # use endian_trait::Endian;
- /// use proxmox::tools::io::ReadExt;
- ///
- /// #[derive(Endian)]
- /// #[repr(C, packed)]
- /// struct Data {
- /// value: u16,
- /// count: u32,
- /// }
- ///
- /// # fn code() -> std::io::Result<()> {
- /// let mut file = std::fs::File::open("my-big-endian.dat")?;
- /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
- /// // safely use our helper:
- /// let data: Data = unsafe { file.read_be_value()? };
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- ///
- /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
- unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T>;
-
- /// Read a boxed value with host endianess.
- ///
- /// This is currently not limited to types implementing the [`Endian`] trait as in our use
- /// cases we use this for types which are too big to want to always perform endian swaps
- /// immediately on all values.
- ///
- /// ```
- /// # use proxmox::tools::vec;
- /// use proxmox::tools::io::ReadExt;
- ///
- /// #[repr(C)]
- /// struct Data {
- /// v1: u64,
- /// buf: [u8; 4088],
- /// }
- ///
- /// # let mut input = [0u8; 4096];
- /// # use proxmox::tools::io::WriteExt;
- /// # let mut writer = &mut input[..];
- /// # unsafe { writer.write_host_value(32u64).unwrap() };
- /// # let mut file = &input[..];
- ///
- /// # fn code<T: std::io::Read>(mut file: T) -> std::io::Result<()> {
- /// let data: Box<Data> = unsafe { file.read_host_value_boxed()? };
- /// assert_eq!(data.v1, 32);
- /// # Ok(())
- /// # }
- /// # code(&input[..]).unwrap();
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- unsafe fn read_host_value_boxed<T>(&mut self) -> io::Result<Box<T>>;
-
- /// Try to read the exact number of bytes required to fill buf.
- ///
- /// This function reads as many bytes as necessary to completely
- /// fill the specified buffer buf. If this function encounters an
- /// "end of file" before getting any data, it returns Ok(false).
- /// If there is some data, but not enough, it return an error of
- /// the kind ErrorKind::UnexpectedEof. The contents of buf are
- /// unspecified in this case.
- fn read_exact_or_eof(&mut self, buf: &mut [u8]) -> io::Result<bool>;
-
- /// Read until EOF
- fn skip_to_end(&mut self) -> io::Result<usize>;
-}
-
-impl<R: io::Read> ReadExt for R {
- fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>> {
- let mut out = unsafe { vec::uninitialized(size) };
- self.read_exact(&mut out)?;
- Ok(out)
- }
-
- fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize> {
- let pos = out.len();
- unsafe {
- out.grow_uninitialized(size);
- }
- let got = self.read(&mut out[pos..])?;
- unsafe {
- out.set_len(pos + got);
- }
- Ok(got)
- }
-
- fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()> {
- let pos = out.len();
- unsafe {
- out.grow_uninitialized(size);
- }
- self.read_exact(&mut out[pos..])?;
- Ok(())
- }
-
- unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T> {
- let mut value = std::mem::MaybeUninit::<T>::uninit();
- self.read_exact(std::slice::from_raw_parts_mut(
- value.as_mut_ptr() as *mut u8,
- mem::size_of::<T>(),
- ))?;
- Ok(value.assume_init())
- }
-
- unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T> {
- Ok(self.read_host_value::<T>()?.from_le())
- }
-
- unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T> {
- Ok(self.read_host_value::<T>()?.from_be())
- }
-
- unsafe fn read_host_value_boxed<T>(&mut self) -> io::Result<Box<T>> {
- // FIXME: Change this once #![feature(new_uninit)] lands for Box<T>!
-
- let ptr = std::alloc::alloc(std::alloc::Layout::new::<T>()) as *mut T;
- self.read_exact(std::slice::from_raw_parts_mut(
- ptr as *mut u8,
- mem::size_of::<T>(),
- ))?;
- Ok(Box::from_raw(ptr))
- }
-
- fn read_exact_or_eof(&mut self, mut buf: &mut [u8]) -> io::Result<bool> {
- let mut read_bytes = 0;
- loop {
- match self.read(&mut buf) {
- Ok(0) => {
- if read_bytes == 0 {
- return Ok(false);
- }
- return Err(io::Error::new(
- io::ErrorKind::UnexpectedEof,
- "failed to fill whole buffer",
- ));
- }
- Ok(n) => {
- let tmp = buf;
- buf = &mut tmp[n..];
- read_bytes += n;
- if buf.is_empty() {
- return Ok(true);
- }
- }
- Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
- Err(e) => return Err(e),
- }
- }
- }
-
- fn skip_to_end(&mut self) -> io::Result<usize> {
- let mut skipped_bytes = 0;
- let mut buf = unsafe { vec::uninitialized(32 * 1024) };
- loop {
- match self.read(&mut buf) {
- Ok(0) => return Ok(skipped_bytes),
- Ok(n) => skipped_bytes += n,
- Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
- Err(e) => return Err(e),
- }
- }
- }
-}
+++ /dev/null
-//! Helpers for `Write`.
-
-use std::io;
-
-use endian_trait::Endian;
-
-/// Adds some additional related functionality for types implementing [`Write`](std::io::Write).
-///
-/// Particularly for writing values of a specific endianess (types implementing [`Endian`]).
-///
-/// Examples:
-/// ```no_run
-/// # use endian_trait::Endian;
-/// # use proxmox::tools::io::WriteExt;
-///
-/// #[derive(Endian)]
-/// #[repr(C)]
-/// struct Header {
-/// version: u16,
-/// data_size: u16,
-/// }
-///
-/// # fn code(mut file: std::fs::File) -> std::io::Result<()> {
-/// let header = Header {
-/// version: 1,
-/// data_size: 16,
-/// };
-/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe:
-/// unsafe {
-/// file.write_le_value(header)?;
-/// }
-/// # Ok(())
-/// # }
-/// ```
-///
-/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
-pub trait WriteExt {
- /// Write a value with host endianess.
- ///
- /// This is limited to types implementing the [`Endian`] trait under the assumption that this
- /// is only done for types which are supposed to be read/writable directly.
- ///
- /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
- /// this is considered unsafe.
- ///
- /// The underlying write call is `.write_all()`, so there are no partial writes.
- ///
- /// ```no_run
- /// # use endian_trait::Endian;
- /// use proxmox::tools::io::WriteExt;
- ///
- /// #[derive(Endian)]
- /// #[repr(C, packed)]
- /// struct Data {
- /// value: u16,
- /// count: u32,
- /// }
- ///
- /// # fn code() -> std::io::Result<()> {
- /// let mut file = std::fs::File::create("my-raw.dat")?;
- /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
- /// // safely use our helper:
- /// unsafe {
- /// file.write_host_value(Data {
- /// value: 1,
- /// count: 2,
- /// })?;
- /// }
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- ///
- /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
- unsafe fn write_host_value<T: Endian>(&mut self, value: T) -> io::Result<()>;
-
- /// Write a little endian value.
- ///
- /// The input type is required to implement the [`Endian`] trait, and we make the assumption
- /// that this is only done for types which are supposed to be read/writable directly.
- ///
- /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
- /// this is considered unsafe.
- ///
- /// The underlying write call is `.write_all()`, so there are no partial writes.
- ///
- /// ```no_run
- /// # use endian_trait::Endian;
- /// use proxmox::tools::io::WriteExt;
- ///
- /// #[derive(Endian)]
- /// #[repr(C, packed)]
- /// struct Data {
- /// value: u16,
- /// count: u32,
- /// }
- ///
- /// # fn code() -> std::io::Result<()> {
- /// let mut file = std::fs::File::create("my-raw.dat")?;
- /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
- /// // safely use our helper:
- /// unsafe {
- /// file.write_le_value(Data {
- /// value: 1,
- /// count: 2,
- /// })?;
- /// }
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- ///
- /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
- unsafe fn write_le_value<T: Endian>(&mut self, value: T) -> io::Result<()>;
-
- /// Read a big endian value.
- ///
- /// The input type is required to implement the [`Endian`] trait, and we make the assumption
- /// that this is only done for types which are supposed to be read/writable directly.
- ///
- /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
- /// this is considered unsafe.
- ///
- /// The underlying write call is `.write_all()`, so there are no partial writes.
- ///
- /// ```no_run
- /// # use endian_trait::Endian;
- /// use proxmox::tools::io::WriteExt;
- ///
- /// #[derive(Endian)]
- /// #[repr(C, packed)]
- /// struct Data {
- /// value: u16,
- /// count: u32,
- /// }
- ///
- /// # fn code() -> std::io::Result<()> {
- /// let mut file = std::fs::File::create("my-raw.dat")?;
- /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
- /// // safely use our helper:
- /// unsafe {
- /// file.write_be_value(Data {
- /// value: 1,
- /// count: 2,
- /// })?;
- /// }
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// # Safety
- ///
- /// This should only used for types with a defined storage representation, usually
- /// `#[repr(C)]`, otherwise the results may be inconsistent.
- ///
- /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
- unsafe fn write_be_value<T: Endian>(&mut self, value: T) -> io::Result<()>;
-}
-
-impl<W: io::Write> WriteExt for W {
- unsafe fn write_host_value<T: Endian>(&mut self, value: T) -> io::Result<()> {
- self.write_all(std::slice::from_raw_parts(
- &value as *const T as *const u8,
- std::mem::size_of::<T>(),
- ))
- }
-
- unsafe fn write_le_value<T: Endian>(&mut self, value: T) -> io::Result<()> {
- self.write_host_value::<T>(value.to_le())
- }
-
- unsafe fn write_be_value<T: Endian>(&mut self, value: T) -> io::Result<()> {
- self.write_host_value::<T>(value.to_be())
- }
-}
use lazy_static::lazy_static;
pub mod as_any;
-pub mod byte_buffer;
pub mod common_regex;
pub mod email;
pub mod fd;
pub mod fs;
-pub mod io;
pub mod mmap;
pub mod parse;
pub mod serde;
-pub mod vec;
pub mod systemd;
#[doc(inline)]
+++ /dev/null
-//! This module provides additional operations for `Vec<u8>`.
-//!
-//! Example:
-//! ```
-//! # use std::io::Read;
-//! use proxmox::tools::vec::{self, ByteVecExt};
-//!
-//! fn append_1024_to_vec<T: Read>(mut input: T, buffer: &mut Vec<u8>) -> std::io::Result<()> {
-//! input.read_exact(unsafe { buffer.grow_uninitialized(1024) })
-//! }
-//! ```
-
-/// Some additional byte vector operations useful for I/O code.
-/// Example:
-/// ```
-/// # use std::io::Read;
-/// # use proxmox::tools::io::{self, ReadExt};
-/// use proxmox::tools::vec::{self, ByteVecExt};
-///
-/// # fn code(mut file: std::fs::File, mut data: Vec<u8>) -> std::io::Result<()> {
-/// file.read_exact(unsafe {
-/// data.grow_uninitialized(1024)
-/// })?;
-/// # Ok(())
-/// # }
-/// ```
-///
-/// Note that this module also provides a safe alternative for the case where
-/// `grow_uninitialized()` is directly followed by a `read_exact()` call via the [`ReadExt`]
-/// trait:
-/// ```ignore
-/// file.append_to_vec(&mut data, 1024)?;
-/// ```
-///
-/// [`ReadExt`]: crate::tools::io::ReadExt
-pub trait ByteVecExt {
- /// Grow a vector without initializing its elements. The difference to simply using `reserve`
- /// is that it also updates the actual length, making the newly allocated data part of the
- /// slice.
- ///
- /// This is a shortcut for:
- /// ```ignore
- /// vec.reserve(more);
- /// let total = vec.len() + more;
- /// unsafe {
- /// vec.set_len(total);
- /// }
- /// ```
- ///
- /// This returns a mutable slice to the newly allocated space, so it can be used inline:
- /// ```
- /// # use std::io::Read;
- /// # use proxmox::tools::vec::ByteVecExt;
- /// # fn test(mut file: std::fs::File, buffer: &mut Vec<u8>) -> std::io::Result<()> {
- /// file.read_exact(unsafe { buffer.grow_uninitialized(1024) })?;
- /// # Ok(())
- /// # }
- /// ```
- ///
- /// Although for the above case it is recommended to use the even shorter version from the
- /// [`ReadExt`] trait:
- /// ```ignore
- /// // use crate::tools::vec::ByteVecExt;
- /// file.append_to_vec(&mut buffer, 1024)?;
- /// ```
- ///
- /// # Safety
- ///
- /// When increasing the size, the new contents are uninitialized and have nothing to do with
- /// the previously contained content. Since we cannot track this state through the type system,
- /// this method is marked as an unsafe API for good measure.
- ///
- /// [`ReadExt`]: crate::tools::io::ReadExt
- unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8];
-
- /// Resize a vector to a specific size without initializing its data. This is a shortcut for:
- /// ```ignore
- /// if new_size <= vec.len() {
- /// vec.truncate(new_size);
- /// } else {
- /// unsafe {
- /// vec.grow_uninitialized(new_size - vec.len());
- /// }
- /// }
- /// ```
- ///
- /// # Safety
- ///
- /// When increasing the size, the new contents are uninitialized and have nothing to do with
- /// the previously contained content. Since we cannot track this state through the type system,
- /// this method is marked as an unsafe API for good measure.
- unsafe fn resize_uninitialized(&mut self, total: usize);
-}
-
-impl ByteVecExt for Vec<u8> {
- unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8] {
- let old_len = self.len();
- self.reserve(more);
- let total = old_len + more;
- self.set_len(total);
- &mut self[old_len..]
- }
-
- unsafe fn resize_uninitialized(&mut self, new_size: usize) {
- if new_size <= self.len() {
- self.truncate(new_size);
- } else {
- self.grow_uninitialized(new_size - self.len());
- }
- }
-}
+++ /dev/null
-//! Byte vector helpers.
-//!
-//! We have a lot of I/O code such as:
-//! ```ignore
-//! let mut buffer = vec![0u8; header_size];
-//! file.read_exact(&mut buffer)?;
-//! ```
-//! (We even have this case with a 4M buffer!)
-//!
-//! This needlessly initializes the buffer to zero (which not only wastes time (an insane amount of
-//! time on debug builds, actually) but also prevents tools such as valgrind from pointing out
-//! access to actually uninitialized data, which may hide bugs...)
-//!
-//! This module provides some helpers for this kind of code. Many of these are supposed to stay on
-//! a lower level, with I/O helpers for types implementing [`Read`](std::io::Read) being available
-//! in the [`tools::io`](crate::tools::io) module.
-//!
-//! Examples:
-//! ```no_run
-//! use proxmox::tools::vec::{self, ByteVecExt};
-//!
-//! # let size = 64usize;
-//! # let more = 64usize;
-//! let mut buffer = vec::undefined(size); // A zero-initialized buffer with valgrind support
-//!
-//! let mut buffer = unsafe { vec::uninitialized(size) }; // an actually uninitialized buffer
-//! vec::clear(&mut buffer); // zero out an &mut [u8]
-//!
-//! vec::clear(unsafe {
-//! buffer.grow_uninitialized(more) // grow the buffer with uninitialized bytes
-//! });
-//! ```
-
-mod byte_vec;
-pub use byte_vec::*;
-
-/// Create an uninitialized byte vector of a specific size.
-///
-/// This is just a shortcut for:
-/// ```no_run
-/// # let len = 64usize;
-/// let mut v = Vec::<u8>::with_capacity(len);
-/// unsafe {
-/// v.set_len(len);
-/// }
-/// ```
-///
-/// # Safety
-///
-/// It's generally not unsafe to use this method, but the contents are uninitialized, and since
-/// this does not return a `MaybeUninit` type to track the initialization state, this is simply
-/// marked as unsafe for good measure.
-#[inline]
-pub unsafe fn uninitialized(len: usize) -> Vec<u8> {
- let mut out = Vec::with_capacity(len);
- out.set_len(len);
- out
-}
-
-/// Shortcut to zero out a slice of bytes.
-#[inline]
-pub fn clear(data: &mut [u8]) {
- unsafe {
- std::ptr::write_bytes(data.as_mut_ptr(), 0, data.len());
- }
-}
-
-/// Create a newly allocated, zero initialized byte vector.
-#[inline]
-pub fn zeroed(len: usize) -> Vec<u8> {
- unsafe {
- let mut out = uninitialized(len);
- clear(&mut out);
- out
- }
-}
-
-/// Create a newly allocated byte vector of a specific size with "undefined" content.
-///
-/// The data will be zero initialized, but, if the `valgrind` feature is activated, it will be
-/// marked as uninitialized for debugging.
-#[inline]
-pub fn undefined(len: usize) -> Vec<u8> {
- undefined_impl(len)
-}
-
-#[cfg(not(feature = "valgrind"))]
-fn undefined_impl(len: usize) -> Vec<u8> {
- zeroed(len)
-}
-
-#[cfg(feature = "valgrind")]
-fn undefined_impl(len: usize) -> Vec<u8> {
- let out = zeroed(len);
- vg::make_slice_undefined(&out[..]);
- out
-}
-
-#[cfg(feature = "valgrind")]
-mod vg {
- type ValgrindValue = valgrind_request::Value;
-
- /// Mark a memory region as undefined when using valgrind, causing it to treat read access to
- /// it as error.
- #[inline]
- pub(crate) fn make_mem_undefined(addr: *const u8, len: usize) -> ValgrindValue {
- const MAKE_MEM_UNDEFINED: ValgrindValue =
- (((b'M' as ValgrindValue) << 24) | ((b'C' as ValgrindValue) << 16)) + 1;
- unsafe {
- valgrind_request::do_client_request(
- 0,
- &[
- MAKE_MEM_UNDEFINED,
- addr as usize as ValgrindValue,
- len as ValgrindValue,
- 0,
- 0,
- 0,
- ],
- )
- }
- }
-
- /// Mark a slice of bytes as undefined when using valgrind, causing it to treat read access to
- /// it as error.
- #[inline]
- pub(crate) fn make_slice_undefined(data: &[u8]) -> ValgrindValue {
- make_mem_undefined(data.as_ptr(), data.len())
- }
-}
projects / proxmox.git / commitdiff