src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::{File, OpenOptions};
   8 use std::io::ErrorKind;
   9 use std::io::Read;
  10 use std::os::unix::io::{AsRawFd, RawFd};
  11 use std::path::Path;
  12 use std::time::Duration;
  13
  14 use anyhow::{bail, format_err, Error};
  15 use serde_json::Value;
  16 use openssl::hash::{hash, DigestBytes, MessageDigest};
  17 use percent_encoding::AsciiSet;
  18
  19 use proxmox::tools::vec;
  20
  21 pub use proxmox::tools::fd::Fd;
  22
  23 pub mod acl;
  24 pub mod async_io;
  25 pub mod borrow;
  26 pub mod daemon;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod timer;
  33
  34 mod wrapped_reader_stream;
  35 pub use wrapped_reader_stream::*;
  36
  37 mod std_channel_writer;
  38 pub use std_channel_writer::*;
  39
  40 pub mod xattr;
  41
  42 mod process_locker;
  43 pub use process_locker::*;
  44
  45 mod file_logger;
  46 pub use file_logger::*;
  47
  48 mod broadcast_future;
  49 pub use broadcast_future::*;
  50
  51 /// The `BufferedRead` trait provides a single function
  52 /// `buffered_read`. It returns a reference to an internal buffer. The
  53 /// purpose of this traid is to avoid unnecessary data copies.
  54 pub trait BufferedRead {
  55     /// This functions tries to fill the internal buffers, then
  56     /// returns a reference to the available data. It returns an empty
  57     /// buffer if `offset` points to the end of the file.
  58     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  59 }
  60
  61 /// Directly map a type into a binary buffer. This is mostly useful
  62 /// for reading structured data from a byte stream (file). You need to
  63 /// make sure that the buffer location does not change, so please
  64 /// avoid vec resize while you use such map.
  65 ///
  66 /// This function panics if the buffer is not large enough.
  67 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  68     if buffer.len() < ::std::mem::size_of::<T>() {
  69         bail!("unable to map struct - buffer too small");
  70     }
  71     Ok(unsafe { &*(buffer.as_ptr() as *const T) })
  72 }
  73
  74 /// Directly map a type into a mutable binary buffer. This is mostly
  75 /// useful for writing structured data into a byte stream (file). You
  76 /// need to make sure that the buffer location does not change, so
  77 /// please avoid vec resize while you use such map.
  78 ///
  79 /// This function panics if the buffer is not large enough.
  80 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  81     if buffer.len() < ::std::mem::size_of::<T>() {
  82         bail!("unable to map struct - buffer too small");
  83     }
  84     Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) })
  85 }
  86
  87 /// Create a file lock using fntl. This function allows you to specify
  88 /// a timeout if you want to avoid infinite blocking.
  89 pub fn lock_file<F: AsRawFd>(
  90     file: &mut F,
  91     exclusive: bool,
  92     timeout: Option<Duration>,
  93 ) -> Result<(), Error> {
  94     let lockarg = if exclusive {
  95         nix::fcntl::FlockArg::LockExclusive
  96     } else {
  97         nix::fcntl::FlockArg::LockShared
  98     };
  99
 100     let timeout = match timeout {
 101         None => {
 102             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 103             return Ok(());
 104         }
 105         Some(t) => t,
 106     };
 107
 108     // unblock the timeout signal temporarily
 109     let _sigblock_guard = timer::unblock_timeout_signal();
 110
 111     // setup a timeout timer
 112     let mut timer = timer::Timer::create(
 113         timer::Clock::Realtime,
 114         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT),
 115     )?;
 116
 117     timer.arm(
 118         timer::TimerSpec::new()
 119             .value(Some(timeout))
 120             .interval(Some(Duration::from_millis(10))),
 121     )?;
 122
 123     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 124     Ok(())
 125 }
 126
 127 /// Open or create a lock file (append mode). Then try to
 128 /// aquire a lock using `lock_file()`.
 129 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<File, Error> {
 130     let path = path.as_ref();
 131     let mut file = match OpenOptions::new().create(true).append(true).open(path) {
 132         Ok(file) => file,
 133         Err(err) => bail!("Unable to open lock {:?} - {}", path, err),
 134     };
 135     match lock_file(&mut file, true, Some(timeout)) {
 136         Ok(_) => Ok(file),
 137         Err(err) => bail!("Unable to aquire lock {:?} - {}", path, err),
 138     }
 139 }
 140
 141 /// Split a file into equal sized chunks. The last chunk may be
 142 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 143 /// cannot return a borrowed buffer ref (we want zero-copy)
 144 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
 145 where
 146     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 147     R: Read,
 148 {
 149     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
 150
 151     if chunk_size > READ_BUFFER_SIZE {
 152         bail!("chunk size too large!");
 153     }
 154
 155     let mut buf = vec::undefined(READ_BUFFER_SIZE);
 156
 157     let mut pos = 0;
 158     let mut file_pos = 0;
 159     loop {
 160         let mut eof = false;
 161         let mut tmp = &mut buf[..];
 162         // try to read large portions, at least chunk_size
 163         while pos < chunk_size {
 164             match file.read(tmp) {
 165                 Ok(0) => {
 166                     eof = true;
 167                     break;
 168                 }
 169                 Ok(n) => {
 170                     pos += n;
 171                     if pos > chunk_size {
 172                         break;
 173                     }
 174                     tmp = &mut tmp[n..];
 175                 }
 176                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 177                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 178             }
 179         }
 180         let mut start = 0;
 181         while start + chunk_size <= pos {
 182             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 183                 break;
 184             }
 185             file_pos += chunk_size;
 186             start += chunk_size;
 187         }
 188         if eof {
 189             if start < pos {
 190                 (chunk_cb)(file_pos, &buf[start..pos])?;
 191                 //file_pos += pos - start;
 192             }
 193             break;
 194         } else {
 195             let rest = pos - start;
 196             if rest > 0 {
 197                 let ptr = buf.as_mut_ptr();
 198                 unsafe {
 199                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 200                 }
 201                 pos = rest;
 202             } else {
 203                 pos = 0;
 204             }
 205         }
 206     }
 207
 208     Ok(())
 209 }
 210
 211 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 212     let mut query = url::form_urlencoded::Serializer::new(String::new());
 213
 214     let object = data.as_object().ok_or_else(|| {
 215         format_err!("json_object_to_query: got wrong data type (expected object).")
 216     })?;
 217
 218     for (key, value) in object {
 219         match value {
 220             Value::Bool(b) => {
 221                 query.append_pair(key, &b.to_string());
 222             }
 223             Value::Number(n) => {
 224                 query.append_pair(key, &n.to_string());
 225             }
 226             Value::String(s) => {
 227                 query.append_pair(key, &s);
 228             }
 229             Value::Array(arr) => {
 230                 for element in arr {
 231                     match element {
 232                         Value::Bool(b) => {
 233                             query.append_pair(key, &b.to_string());
 234                         }
 235                         Value::Number(n) => {
 236                             query.append_pair(key, &n.to_string());
 237                         }
 238                         Value::String(s) => {
 239                             query.append_pair(key, &s);
 240                         }
 241                         _ => bail!(
 242                             "json_object_to_query: unable to handle complex array data types."
 243                         ),
 244                     }
 245                 }
 246             }
 247             _ => bail!("json_object_to_query: unable to handle complex data types."),
 248         }
 249     }
 250
 251     Ok(query.finish())
 252 }
 253
 254 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 255     match param[name].as_str() {
 256         Some(s) => Ok(s),
 257         None => bail!("missing parameter '{}'", name),
 258     }
 259 }
 260
 261 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 262     match param[name].as_str() {
 263         Some(s) => Ok(s),
 264         None => bail!("missing property '{}'", name),
 265     }
 266 }
 267
 268 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 269     match param[name].as_i64() {
 270         Some(s) => Ok(s),
 271         None => bail!("missing parameter '{}'", name),
 272     }
 273 }
 274
 275 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 276     match param[name].as_i64() {
 277         Some(s) => Ok(s),
 278         None => bail!("missing property '{}'", name),
 279     }
 280 }
 281
 282 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 283     match param[name].as_array() {
 284         Some(s) => Ok(s.to_vec()),
 285         None => bail!("missing parameter '{}'", name),
 286     }
 287 }
 288
 289 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 290     match param[name].as_array() {
 291         Some(s) => Ok(s.to_vec()),
 292         None => bail!("missing property '{}'", name),
 293     }
 294 }
 295
 296 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 297     let mut result = vec![];
 298
 299     use nix::fcntl::AtFlags;
 300     use nix::fcntl::OFlag;
 301     use nix::sys::stat::Mode;
 302
 303     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 304
 305     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 306         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 307         Err(_) => false,
 308     };
 309
 310     if !is_dir {
 311         if let Some(parent) = dirname.parent() {
 312             dirname = parent.to_owned();
 313         }
 314     }
 315
 316     let mut dir =
 317         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 318             Ok(d) => d,
 319             Err(_) => return result,
 320         };
 321
 322     for item in dir.iter() {
 323         if let Ok(entry) = item {
 324             if let Ok(name) = entry.file_name().to_str() {
 325                 if name == "." || name == ".." {
 326                     continue;
 327                 }
 328                 let mut newpath = dirname.clone();
 329                 newpath.push(name);
 330
 331                 if let Ok(stat) =
 332                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 333                 {
 334                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 335                         newpath.push("");
 336                         if let Some(newpath) = newpath.to_str() {
 337                             result.push(newpath.to_owned());
 338                         }
 339                         continue;
 340                     }
 341                 }
 342                 if let Some(newpath) = newpath.to_str() {
 343                     result.push(newpath.to_owned());
 344                 }
 345             }
 346         }
 347     }
 348
 349     result
 350 }
 351
 352 /// Scan directory for matching file names.
 353 ///
 354 /// Scan through all directory entries and call `callback()` function
 355 /// if the entry name matches the regular expression. This function
 356 /// used unix `openat()`, so you can pass absolute or relative file
 357 /// names. This function simply skips non-UTF8 encoded names.
 358 pub fn scandir<P, F>(
 359     dirfd: RawFd,
 360     path: &P,
 361     regex: &regex::Regex,
 362     mut callback: F,
 363 ) -> Result<(), Error>
 364 where
 365     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 366     P: ?Sized + nix::NixPath,
 367 {
 368     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 369         let entry = entry?;
 370         let file_type = match entry.file_type() {
 371             Some(file_type) => file_type,
 372             None => bail!("unable to detect file type"),
 373         };
 374
 375         callback(
 376             entry.parent_fd(),
 377             unsafe { entry.file_name_utf8_unchecked() },
 378             file_type,
 379         )?;
 380     }
 381     Ok(())
 382 }
 383
 384 /// Shortcut for md5 sums.
 385 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 386     hash(MessageDigest::md5(), data).map_err(Error::from)
 387 }
 388
 389 pub fn get_hardware_address() -> Result<String, Error> {
 390     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 391
 392     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 393     let digest = md5sum(&contents)?;
 394
 395     Ok(proxmox::tools::bin_to_hex(&digest))
 396 }
 397
 398 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 399     if digest1 != digest2 {
 400         bail!("detected modified configuration - file changed by other user? Try again.");
 401     }
 402     Ok(())
 403 }
 404
 405 /// Extract authentication cookie from cookie header.
 406 /// We assume cookie_name is already url encoded.
 407 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 408     for pair in cookie.split(';') {
 409         let (name, value) = match pair.find('=') {
 410             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 411             None => return None, // Cookie format error
 412         };
 413
 414         if name == cookie_name {
 415             use percent_encoding::percent_decode;
 416             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 417                 return Some(value.into());
 418             } else {
 419                 return None; // Cookie format error
 420             }
 421         }
 422     }
 423
 424     None
 425 }
 426
 427 pub fn join(data: &Vec<String>, sep: char) -> String {
 428     let mut list = String::new();
 429
 430     for item in data {
 431         if !list.is_empty() {
 432             list.push(sep);
 433         }
 434         list.push_str(item);
 435     }
 436
 437     list
 438 }
 439
 440 /// Detect modified configuration files
 441 ///
 442 /// This function fails with a resonable error message if checksums do not match.
 443 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 444     if digest1 != digest2 {
 445         bail!("detected modified configuration - file changed by other user? Try again.");
 446     }
 447     Ok(())
 448 }
 449
 450 /// normalize uri path
 451 ///
 452 /// Do not allow ".", "..", or hidden files ".XXXX"
 453 /// Also remove empty path components
 454 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 455     let items = path.split('/');
 456
 457     let mut path = String::new();
 458     let mut components = vec![];
 459
 460     for name in items {
 461         if name.is_empty() {
 462             continue;
 463         }
 464         if name.starts_with('.') {
 465             bail!("Path contains illegal components.");
 466         }
 467         path.push('/');
 468         path.push_str(name);
 469         components.push(name);
 470     }
 471
 472     Ok((path, components))
 473 }
 474
 475 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 476     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 477     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 478         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 479     flags.set(FdFlag::FD_CLOEXEC, on);
 480     fcntl(fd, F_SETFD(flags))?;
 481     Ok(())
 482 }
 483
 484 static mut SHUTDOWN_REQUESTED: bool = false;
 485
 486 pub fn request_shutdown() {
 487     unsafe {
 488         SHUTDOWN_REQUESTED = true;
 489     }
 490     crate::server::server_shutdown();
 491 }
 492
 493 #[inline(always)]
 494 pub fn shutdown_requested() -> bool {
 495     unsafe { SHUTDOWN_REQUESTED }
 496 }
 497
 498 pub fn fail_on_shutdown() -> Result<(), Error> {
 499     if shutdown_requested() {
 500         bail!("Server shutdown requested - aborting task");
 501     }
 502     Ok(())
 503 }
 504
 505 // wrap nix::unistd::pipe2 + O_CLOEXEC into something returning guarded file descriptors
 506 pub fn pipe() -> Result<(Fd, Fd), Error> {
 507     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 508     Ok((Fd(pin), Fd(pout)))
 509 }
 510
 511 /// An easy way to convert types to Any
 512 ///
 513 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 514 pub trait AsAny {
 515     fn as_any(&self) -> &dyn Any;
 516 }
 517
 518 impl<T: Any> AsAny for T {
 519     fn as_any(&self) -> &dyn Any {
 520         self
 521     }
 522 }
 523
 524 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 525 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 526     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 527     .add(0x20)
 528     .add(0x7f)
 529     // the DEFAULT_ENCODE_SET added:
 530     .add(b' ')
 531     .add(b'"')
 532     .add(b'#')
 533     .add(b'<')
 534     .add(b'>')
 535     .add(b'`')
 536     .add(b'?')
 537     .add(b'{')
 538     .add(b'}');