pbs-tools/src/fs.rs

   1 //! File system helper utilities.
   2
   3 use std::borrow::{Borrow, BorrowMut};
   4 use std::fs::File;
   5 use std::io::{self, BufRead};
   6 use std::ops::{Deref, DerefMut};
   7 use std::os::unix::io::{AsRawFd, RawFd};
   8 use std::path::Path;
   9
  10 use anyhow::{bail, format_err, Error};
  11 use nix::dir;
  12 use nix::dir::Dir;
  13 use nix::fcntl::OFlag;
  14 use nix::sys::stat::Mode;
  15
  16 use regex::Regex;
  17
  18 use proxmox::sys::error::SysError;
  19 use proxmox_borrow::Tied;
  20
  21 pub type DirLockGuard = Dir;
  22
  23 /// This wraps nix::dir::Entry with the parent directory's file descriptor.
  24 pub struct ReadDirEntry {
  25     entry: dir::Entry,
  26     parent_fd: RawFd,
  27 }
  28
  29 impl Into<dir::Entry> for ReadDirEntry {
  30     fn into(self) -> dir::Entry {
  31         self.entry
  32     }
  33 }
  34
  35 impl Deref for ReadDirEntry {
  36     type Target = dir::Entry;
  37
  38     fn deref(&self) -> &Self::Target {
  39         &self.entry
  40     }
  41 }
  42
  43 impl DerefMut for ReadDirEntry {
  44     fn deref_mut(&mut self) -> &mut Self::Target {
  45         &mut self.entry
  46     }
  47 }
  48
  49 impl AsRef<dir::Entry> for ReadDirEntry {
  50     fn as_ref(&self) -> &dir::Entry {
  51         &self.entry
  52     }
  53 }
  54
  55 impl AsMut<dir::Entry> for ReadDirEntry {
  56     fn as_mut(&mut self) -> &mut dir::Entry {
  57         &mut self.entry
  58     }
  59 }
  60
  61 impl Borrow<dir::Entry> for ReadDirEntry {
  62     fn borrow(&self) -> &dir::Entry {
  63         &self.entry
  64     }
  65 }
  66
  67 impl BorrowMut<dir::Entry> for ReadDirEntry {
  68     fn borrow_mut(&mut self) -> &mut dir::Entry {
  69         &mut self.entry
  70     }
  71 }
  72
  73 impl ReadDirEntry {
  74     #[inline]
  75     pub fn parent_fd(&self) -> RawFd {
  76         self.parent_fd
  77     }
  78
  79     pub unsafe fn file_name_utf8_unchecked(&self) -> &str {
  80         std::str::from_utf8_unchecked(self.file_name().to_bytes())
  81     }
  82 }
  83
  84 // Since Tied<T, U> implements Deref to U, a Tied<Dir, Iterator> already implements Iterator.
  85 // This is simply a wrapper with a shorter type name mapping nix::Error to anyhow::Error.
  86 /// Wrapper over a pair of `nix::dir::Dir` and `nix::dir::Iter`, returned by `read_subdir()`.
  87 pub struct ReadDir {
  88     iter: Tied<Dir, dyn Iterator<Item = nix::Result<dir::Entry>> + Send>,
  89     dir_fd: RawFd,
  90 }
  91
  92 impl Iterator for ReadDir {
  93     type Item = Result<ReadDirEntry, Error>;
  94
  95     fn next(&mut self) -> Option<Self::Item> {
  96         self.iter.next().map(|res| {
  97             res.map(|entry| ReadDirEntry { entry, parent_fd: self.dir_fd })
  98                 .map_err(Error::from)
  99         })
 100     }
 101 }
 102
 103 /// Create an iterator over sub directory entries.
 104 /// This uses `openat` on `dirfd`, so `path` can be relative to that or an absolute path.
 105 pub fn read_subdir<P: ?Sized + nix::NixPath>(dirfd: RawFd, path: &P) -> nix::Result<ReadDir> {
 106     let dir = Dir::openat(dirfd, path, OFlag::O_RDONLY, Mode::empty())?;
 107     let fd = dir.as_raw_fd();
 108     let iter = Tied::new(dir, |dir| {
 109         Box::new(unsafe { (*dir).iter() })
 110             as Box<dyn Iterator<Item = nix::Result<dir::Entry>> + Send>
 111     });
 112     Ok(ReadDir { iter, dir_fd: fd })
 113 }
 114
 115 /// Scan through a directory with a regular expression. This is simply a shortcut filtering the
 116 /// results of `read_subdir`. Non-UTF8 compatible file names are silently ignored.
 117 pub fn scan_subdir<'a, P: ?Sized + nix::NixPath>(
 118     dirfd: RawFd,
 119     path: &P,
 120     regex: &'a regex::Regex,
 121 ) -> Result<impl Iterator<Item = Result<ReadDirEntry, Error>> + 'a, nix::Error> {
 122     Ok(read_subdir(dirfd, path)?.filter_file_name_regex(regex))
 123 }
 124
 125 /// Scan directory for matching file names with a callback.
 126 ///
 127 /// Scan through all directory entries and call `callback()` function
 128 /// if the entry name matches the regular expression. This function
 129 /// used unix `openat()`, so you can pass absolute or relative file
 130 /// names. This function simply skips non-UTF8 encoded names.
 131 pub fn scandir<P, F>(
 132     dirfd: RawFd,
 133     path: &P,
 134     regex: &regex::Regex,
 135     mut callback: F,
 136 ) -> Result<(), Error>
 137 where
 138     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 139     P: ?Sized + nix::NixPath,
 140 {
 141     for entry in scan_subdir(dirfd, path, regex)? {
 142         let entry = entry?;
 143         let file_type = match entry.file_type() {
 144             Some(file_type) => file_type,
 145             None => bail!("unable to detect file type"),
 146         };
 147
 148         callback(
 149             entry.parent_fd(),
 150             unsafe { entry.file_name_utf8_unchecked() },
 151             file_type,
 152         )?;
 153     }
 154     Ok(())
 155 }
 156
 157
 158 /// Helper trait to provide a combinators for directory entry iterators.
 159 pub trait FileIterOps<T, E>
 160 where
 161     Self: Sized + Iterator<Item = Result<T, E>>,
 162     T: Borrow<dir::Entry>,
 163     E: Into<Error> + Send + Sync,
 164 {
 165     /// Filter by file type. This is more convenient than using the `filter` method alone as this
 166     /// also includes error handling and handling of files without a type (via an error).
 167     fn filter_file_type(self, ty: dir::Type) -> FileTypeFilter<Self, T, E> {
 168         FileTypeFilter { inner: self, ty }
 169     }
 170
 171     /// Filter by file name. Note that file names which aren't valid utf-8 will be treated as if
 172     /// they do not match the pattern.
 173     fn filter_file_name_regex(self, regex: &Regex) -> FileNameRegexFilter<Self, T, E> {
 174         FileNameRegexFilter { inner: self, regex }
 175     }
 176 }
 177
 178 impl<I, T, E> FileIterOps<T, E> for I
 179 where
 180     I: Iterator<Item = Result<T, E>>,
 181     T: Borrow<dir::Entry>,
 182     E: Into<Error> + Send + Sync,
 183 {
 184 }
 185
 186 /// This filters files from its inner iterator by a file type. Files with no type produce an error.
 187 pub struct FileTypeFilter<I, T, E>
 188 where
 189     I: Iterator<Item = Result<T, E>>,
 190     T: Borrow<dir::Entry>,
 191     E: Into<Error> + Send + Sync,
 192 {
 193     inner: I,
 194     ty: nix::dir::Type,
 195 }
 196
 197 impl<I, T, E> Iterator for FileTypeFilter<I, T, E>
 198 where
 199     I: Iterator<Item = Result<T, E>>,
 200     T: Borrow<dir::Entry>,
 201     E: Into<Error> + Send + Sync,
 202 {
 203     type Item = Result<T, Error>;
 204
 205     fn next(&mut self) -> Option<Self::Item> {
 206         loop {
 207             let item = self.inner.next()?.map_err(|e| e.into());
 208             match item {
 209                 Ok(ref entry) => match entry.borrow().file_type() {
 210                     Some(ty) => {
 211                         if ty == self.ty {
 212                             return Some(item);
 213                         } else {
 214                             continue;
 215                         }
 216                     }
 217                     None => return Some(Err(format_err!("unable to detect file type"))),
 218                 },
 219                 Err(_) => return Some(item),
 220             }
 221         }
 222     }
 223 }
 224
 225 /// This filters files by name via a Regex. Files whose file name aren't valid utf-8 are skipped
 226 /// silently.
 227 pub struct FileNameRegexFilter<'a, I, T, E>
 228 where
 229     I: Iterator<Item = Result<T, E>>,
 230     T: Borrow<dir::Entry>,
 231 {
 232     inner: I,
 233     regex: &'a Regex,
 234 }
 235
 236 impl<I, T, E> Iterator for FileNameRegexFilter<'_, I, T, E>
 237 where
 238     I: Iterator<Item = Result<T, E>>,
 239     T: Borrow<dir::Entry>,
 240 {
 241     type Item = Result<T, E>;
 242
 243     fn next(&mut self) -> Option<Self::Item> {
 244         loop {
 245             let item = self.inner.next()?;
 246             match item {
 247                 Ok(ref entry) => {
 248                     if let Ok(name) = entry.borrow().file_name().to_str() {
 249                         if self.regex.is_match(name) {
 250                             return Some(item);
 251                         }
 252                     }
 253                     // file did not match regex or isn't valid utf-8
 254                     continue;
 255                 },
 256                 Err(_) => return Some(item),
 257             }
 258         }
 259     }
 260 }
 261
 262 // /usr/include/linux/fs.h: #define FS_IOC_GETFLAGS _IOR('f', 1, long)
 263 // read Linux file system attributes (see man chattr)
 264 nix::ioctl_read!(read_attr_fd, b'f', 1, libc::c_long);
 265 nix::ioctl_write_ptr!(write_attr_fd, b'f', 2, libc::c_long);
 266
 267 // /usr/include/linux/msdos_fs.h: #define FAT_IOCTL_GET_ATTRIBUTES _IOR('r', 0x10, __u32)
 268 // read FAT file system attributes
 269 nix::ioctl_read!(read_fat_attr_fd, b'r', 0x10, u32);
 270 nix::ioctl_write_ptr!(write_fat_attr_fd, b'r', 0x11, u32);
 271
 272 // From /usr/include/linux/fs.h
 273 // #define FS_IOC_FSGETXATTR _IOR('X', 31, struct fsxattr)
 274 // #define FS_IOC_FSSETXATTR _IOW('X', 32, struct fsxattr)
 275 nix::ioctl_read!(fs_ioc_fsgetxattr, b'X', 31, FSXAttr);
 276 nix::ioctl_write_ptr!(fs_ioc_fssetxattr, b'X', 32, FSXAttr);
 277
 278 #[repr(C)]
 279 #[derive(Debug)]
 280 pub struct FSXAttr {
 281     pub fsx_xflags: u32,
 282     pub fsx_extsize: u32,
 283     pub fsx_nextents: u32,
 284     pub fsx_projid: u32,
 285     pub fsx_cowextsize: u32,
 286     pub fsx_pad: [u8; 8],
 287 }
 288
 289 impl Default for FSXAttr {
 290     fn default() -> Self {
 291         FSXAttr {
 292             fsx_xflags: 0u32,
 293             fsx_extsize: 0u32,
 294             fsx_nextents: 0u32,
 295             fsx_projid: 0u32,
 296             fsx_cowextsize: 0u32,
 297             fsx_pad: [0u8; 8],
 298         }
 299     }
 300 }
 301
 302 /// Attempt to acquire a shared flock on the given path, 'what' and
 303 /// 'would_block_message' are used for error formatting.
 304 pub fn lock_dir_noblock_shared(
 305     path: &std::path::Path,
 306     what: &str,
 307     would_block_msg: &str,
 308 ) -> Result<DirLockGuard, Error> {
 309     do_lock_dir_noblock(path, what, would_block_msg, false)
 310 }
 311
 312 /// Attempt to acquire an exclusive flock on the given path, 'what' and
 313 /// 'would_block_message' are used for error formatting.
 314 pub fn lock_dir_noblock(
 315     path: &std::path::Path,
 316     what: &str,
 317     would_block_msg: &str,
 318 ) -> Result<DirLockGuard, Error> {
 319     do_lock_dir_noblock(path, what, would_block_msg, true)
 320 }
 321
 322 fn do_lock_dir_noblock(
 323     path: &std::path::Path,
 324     what: &str,
 325     would_block_msg: &str,
 326     exclusive: bool,
 327 ) -> Result<DirLockGuard, Error> {
 328     let mut handle = Dir::open(path, OFlag::O_RDONLY, Mode::empty())
 329         .map_err(|err| {
 330             format_err!("unable to open {} directory {:?} for locking - {}", what, path, err)
 331         })?;
 332
 333     // acquire in non-blocking mode, no point in waiting here since other
 334     // backups could still take a very long time
 335     proxmox::tools::fs::lock_file(&mut handle, exclusive, Some(std::time::Duration::from_nanos(0)))
 336         .map_err(|err| {
 337             format_err!(
 338                 "unable to acquire lock on {} directory {:?} - {}", what, path,
 339                 if err.would_block() {
 340                     String::from(would_block_msg)
 341                 } else {
 342                     err.to_string()
 343                 }
 344             )
 345         })?;
 346
 347     Ok(handle)
 348 }
 349
 350 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 351 /// `#`).
 352 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 353     path: P,
 354 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 355     let path = path.as_ref();
 356
 357     Ok(io::BufReader::new(
 358         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 359     )
 360     .lines()
 361     .filter_map(|line| match line {
 362         Ok(line) => {
 363             let line = line.trim();
 364             if line.is_empty() || line.starts_with('#') {
 365                 None
 366             } else {
 367                 Some(Ok(line.to_string()))
 368             }
 369         }
 370         Err(err) => Some(Err(err)),
 371     }))
 372 }