pbs-tools/src/fs.rs

   1 //! File system helper utilities.
   2
   3 use std::borrow::{Borrow, BorrowMut};
   4 use std::fs::File;
   5 use std::io::{self, BufRead};
   6 use std::ops::{Deref, DerefMut};
   7 use std::os::unix::io::{AsRawFd, RawFd};
   8 use std::path::Path;
   9
  10 use anyhow::{bail, format_err, Error};
  11 use nix::dir;
  12 use nix::dir::Dir;
  13 use nix::fcntl::OFlag;
  14 use nix::sys::stat::Mode;
  15
  16 use regex::Regex;
  17
  18 use proxmox::sys::error::SysError;
  19
  20 use crate::borrow::Tied;
  21
  22 pub type DirLockGuard = Dir;
  23
  24 /// This wraps nix::dir::Entry with the parent directory's file descriptor.
  25 pub struct ReadDirEntry {
  26     entry: dir::Entry,
  27     parent_fd: RawFd,
  28 }
  29
  30 impl Into<dir::Entry> for ReadDirEntry {
  31     fn into(self) -> dir::Entry {
  32         self.entry
  33     }
  34 }
  35
  36 impl Deref for ReadDirEntry {
  37     type Target = dir::Entry;
  38
  39     fn deref(&self) -> &Self::Target {
  40         &self.entry
  41     }
  42 }
  43
  44 impl DerefMut for ReadDirEntry {
  45     fn deref_mut(&mut self) -> &mut Self::Target {
  46         &mut self.entry
  47     }
  48 }
  49
  50 impl AsRef<dir::Entry> for ReadDirEntry {
  51     fn as_ref(&self) -> &dir::Entry {
  52         &self.entry
  53     }
  54 }
  55
  56 impl AsMut<dir::Entry> for ReadDirEntry {
  57     fn as_mut(&mut self) -> &mut dir::Entry {
  58         &mut self.entry
  59     }
  60 }
  61
  62 impl Borrow<dir::Entry> for ReadDirEntry {
  63     fn borrow(&self) -> &dir::Entry {
  64         &self.entry
  65     }
  66 }
  67
  68 impl BorrowMut<dir::Entry> for ReadDirEntry {
  69     fn borrow_mut(&mut self) -> &mut dir::Entry {
  70         &mut self.entry
  71     }
  72 }
  73
  74 impl ReadDirEntry {
  75     #[inline]
  76     pub fn parent_fd(&self) -> RawFd {
  77         self.parent_fd
  78     }
  79
  80     pub unsafe fn file_name_utf8_unchecked(&self) -> &str {
  81         std::str::from_utf8_unchecked(self.file_name().to_bytes())
  82     }
  83 }
  84
  85 // Since Tied<T, U> implements Deref to U, a Tied<Dir, Iterator> already implements Iterator.
  86 // This is simply a wrapper with a shorter type name mapping nix::Error to anyhow::Error.
  87 /// Wrapper over a pair of `nix::dir::Dir` and `nix::dir::Iter`, returned by `read_subdir()`.
  88 pub struct ReadDir {
  89     iter: Tied<Dir, dyn Iterator<Item = nix::Result<dir::Entry>> + Send>,
  90     dir_fd: RawFd,
  91 }
  92
  93 impl Iterator for ReadDir {
  94     type Item = Result<ReadDirEntry, Error>;
  95
  96     fn next(&mut self) -> Option<Self::Item> {
  97         self.iter.next().map(|res| {
  98             res.map(|entry| ReadDirEntry { entry, parent_fd: self.dir_fd })
  99                 .map_err(Error::from)
 100         })
 101     }
 102 }
 103
 104 /// Create an iterator over sub directory entries.
 105 /// This uses `openat` on `dirfd`, so `path` can be relative to that or an absolute path.
 106 pub fn read_subdir<P: ?Sized + nix::NixPath>(dirfd: RawFd, path: &P) -> nix::Result<ReadDir> {
 107     let dir = Dir::openat(dirfd, path, OFlag::O_RDONLY, Mode::empty())?;
 108     let fd = dir.as_raw_fd();
 109     let iter = Tied::new(dir, |dir| {
 110         Box::new(unsafe { (*dir).iter() })
 111             as Box<dyn Iterator<Item = nix::Result<dir::Entry>> + Send>
 112     });
 113     Ok(ReadDir { iter, dir_fd: fd })
 114 }
 115
 116 /// Scan through a directory with a regular expression. This is simply a shortcut filtering the
 117 /// results of `read_subdir`. Non-UTF8 compatible file names are silently ignored.
 118 pub fn scan_subdir<'a, P: ?Sized + nix::NixPath>(
 119     dirfd: RawFd,
 120     path: &P,
 121     regex: &'a regex::Regex,
 122 ) -> Result<impl Iterator<Item = Result<ReadDirEntry, Error>> + 'a, nix::Error> {
 123     Ok(read_subdir(dirfd, path)?.filter_file_name_regex(regex))
 124 }
 125
 126 /// Scan directory for matching file names with a callback.
 127 ///
 128 /// Scan through all directory entries and call `callback()` function
 129 /// if the entry name matches the regular expression. This function
 130 /// used unix `openat()`, so you can pass absolute or relative file
 131 /// names. This function simply skips non-UTF8 encoded names.
 132 pub fn scandir<P, F>(
 133     dirfd: RawFd,
 134     path: &P,
 135     regex: &regex::Regex,
 136     mut callback: F,
 137 ) -> Result<(), Error>
 138 where
 139     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 140     P: ?Sized + nix::NixPath,
 141 {
 142     for entry in scan_subdir(dirfd, path, regex)? {
 143         let entry = entry?;
 144         let file_type = match entry.file_type() {
 145             Some(file_type) => file_type,
 146             None => bail!("unable to detect file type"),
 147         };
 148
 149         callback(
 150             entry.parent_fd(),
 151             unsafe { entry.file_name_utf8_unchecked() },
 152             file_type,
 153         )?;
 154     }
 155     Ok(())
 156 }
 157
 158
 159 /// Helper trait to provide a combinators for directory entry iterators.
 160 pub trait FileIterOps<T, E>
 161 where
 162     Self: Sized + Iterator<Item = Result<T, E>>,
 163     T: Borrow<dir::Entry>,
 164     E: Into<Error> + Send + Sync,
 165 {
 166     /// Filter by file type. This is more convenient than using the `filter` method alone as this
 167     /// also includes error handling and handling of files without a type (via an error).
 168     fn filter_file_type(self, ty: dir::Type) -> FileTypeFilter<Self, T, E> {
 169         FileTypeFilter { inner: self, ty }
 170     }
 171
 172     /// Filter by file name. Note that file names which aren't valid utf-8 will be treated as if
 173     /// they do not match the pattern.
 174     fn filter_file_name_regex(self, regex: &Regex) -> FileNameRegexFilter<Self, T, E> {
 175         FileNameRegexFilter { inner: self, regex }
 176     }
 177 }
 178
 179 impl<I, T, E> FileIterOps<T, E> for I
 180 where
 181     I: Iterator<Item = Result<T, E>>,
 182     T: Borrow<dir::Entry>,
 183     E: Into<Error> + Send + Sync,
 184 {
 185 }
 186
 187 /// This filters files from its inner iterator by a file type. Files with no type produce an error.
 188 pub struct FileTypeFilter<I, T, E>
 189 where
 190     I: Iterator<Item = Result<T, E>>,
 191     T: Borrow<dir::Entry>,
 192     E: Into<Error> + Send + Sync,
 193 {
 194     inner: I,
 195     ty: nix::dir::Type,
 196 }
 197
 198 impl<I, T, E> Iterator for FileTypeFilter<I, T, E>
 199 where
 200     I: Iterator<Item = Result<T, E>>,
 201     T: Borrow<dir::Entry>,
 202     E: Into<Error> + Send + Sync,
 203 {
 204     type Item = Result<T, Error>;
 205
 206     fn next(&mut self) -> Option<Self::Item> {
 207         loop {
 208             let item = self.inner.next()?.map_err(|e| e.into());
 209             match item {
 210                 Ok(ref entry) => match entry.borrow().file_type() {
 211                     Some(ty) => {
 212                         if ty == self.ty {
 213                             return Some(item);
 214                         } else {
 215                             continue;
 216                         }
 217                     }
 218                     None => return Some(Err(format_err!("unable to detect file type"))),
 219                 },
 220                 Err(_) => return Some(item),
 221             }
 222         }
 223     }
 224 }
 225
 226 /// This filters files by name via a Regex. Files whose file name aren't valid utf-8 are skipped
 227 /// silently.
 228 pub struct FileNameRegexFilter<'a, I, T, E>
 229 where
 230     I: Iterator<Item = Result<T, E>>,
 231     T: Borrow<dir::Entry>,
 232 {
 233     inner: I,
 234     regex: &'a Regex,
 235 }
 236
 237 impl<I, T, E> Iterator for FileNameRegexFilter<'_, I, T, E>
 238 where
 239     I: Iterator<Item = Result<T, E>>,
 240     T: Borrow<dir::Entry>,
 241 {
 242     type Item = Result<T, E>;
 243
 244     fn next(&mut self) -> Option<Self::Item> {
 245         loop {
 246             let item = self.inner.next()?;
 247             match item {
 248                 Ok(ref entry) => {
 249                     if let Ok(name) = entry.borrow().file_name().to_str() {
 250                         if self.regex.is_match(name) {
 251                             return Some(item);
 252                         }
 253                     }
 254                     // file did not match regex or isn't valid utf-8
 255                     continue;
 256                 },
 257                 Err(_) => return Some(item),
 258             }
 259         }
 260     }
 261 }
 262
 263 // /usr/include/linux/fs.h: #define FS_IOC_GETFLAGS _IOR('f', 1, long)
 264 // read Linux file system attributes (see man chattr)
 265 nix::ioctl_read!(read_attr_fd, b'f', 1, libc::c_long);
 266 nix::ioctl_write_ptr!(write_attr_fd, b'f', 2, libc::c_long);
 267
 268 // /usr/include/linux/msdos_fs.h: #define FAT_IOCTL_GET_ATTRIBUTES _IOR('r', 0x10, __u32)
 269 // read FAT file system attributes
 270 nix::ioctl_read!(read_fat_attr_fd, b'r', 0x10, u32);
 271 nix::ioctl_write_ptr!(write_fat_attr_fd, b'r', 0x11, u32);
 272
 273 // From /usr/include/linux/fs.h
 274 // #define FS_IOC_FSGETXATTR _IOR('X', 31, struct fsxattr)
 275 // #define FS_IOC_FSSETXATTR _IOW('X', 32, struct fsxattr)
 276 nix::ioctl_read!(fs_ioc_fsgetxattr, b'X', 31, FSXAttr);
 277 nix::ioctl_write_ptr!(fs_ioc_fssetxattr, b'X', 32, FSXAttr);
 278
 279 #[repr(C)]
 280 #[derive(Debug)]
 281 pub struct FSXAttr {
 282     pub fsx_xflags: u32,
 283     pub fsx_extsize: u32,
 284     pub fsx_nextents: u32,
 285     pub fsx_projid: u32,
 286     pub fsx_cowextsize: u32,
 287     pub fsx_pad: [u8; 8],
 288 }
 289
 290 impl Default for FSXAttr {
 291     fn default() -> Self {
 292         FSXAttr {
 293             fsx_xflags: 0u32,
 294             fsx_extsize: 0u32,
 295             fsx_nextents: 0u32,
 296             fsx_projid: 0u32,
 297             fsx_cowextsize: 0u32,
 298             fsx_pad: [0u8; 8],
 299         }
 300     }
 301 }
 302
 303 /// Attempt to acquire a shared flock on the given path, 'what' and
 304 /// 'would_block_message' are used for error formatting.
 305 pub fn lock_dir_noblock_shared(
 306     path: &std::path::Path,
 307     what: &str,
 308     would_block_msg: &str,
 309 ) -> Result<DirLockGuard, Error> {
 310     do_lock_dir_noblock(path, what, would_block_msg, false)
 311 }
 312
 313 /// Attempt to acquire an exclusive flock on the given path, 'what' and
 314 /// 'would_block_message' are used for error formatting.
 315 pub fn lock_dir_noblock(
 316     path: &std::path::Path,
 317     what: &str,
 318     would_block_msg: &str,
 319 ) -> Result<DirLockGuard, Error> {
 320     do_lock_dir_noblock(path, what, would_block_msg, true)
 321 }
 322
 323 fn do_lock_dir_noblock(
 324     path: &std::path::Path,
 325     what: &str,
 326     would_block_msg: &str,
 327     exclusive: bool,
 328 ) -> Result<DirLockGuard, Error> {
 329     let mut handle = Dir::open(path, OFlag::O_RDONLY, Mode::empty())
 330         .map_err(|err| {
 331             format_err!("unable to open {} directory {:?} for locking - {}", what, path, err)
 332         })?;
 333
 334     // acquire in non-blocking mode, no point in waiting here since other
 335     // backups could still take a very long time
 336     proxmox::tools::fs::lock_file(&mut handle, exclusive, Some(std::time::Duration::from_nanos(0)))
 337         .map_err(|err| {
 338             format_err!(
 339                 "unable to acquire lock on {} directory {:?} - {}", what, path,
 340                 if err.would_block() {
 341                     String::from(would_block_msg)
 342                 } else {
 343                     err.to_string()
 344                 }
 345             )
 346         })?;
 347
 348     Ok(handle)
 349 }
 350
 351 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 352 /// `#`).
 353 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 354     path: P,
 355 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 356     let path = path.as_ref();
 357
 358     Ok(io::BufReader::new(
 359         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 360     )
 361     .lines()
 362     .filter_map(|line| match line {
 363         Ok(line) => {
 364             let line = line.trim();
 365             if line.is_empty() || line.starts_with('#') {
 366                 None
 367             } else {
 368                 Some(Ok(line.to_string()))
 369             }
 370         }
 371         Err(err) => Some(Err(err)),
 372     }))
 373 }