src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::{File, OpenOptions};
   8 use std::io::{self, BufRead, ErrorKind, Read};
   9 use std::os::unix::io::{AsRawFd, RawFd};
  10 use std::path::Path;
  11 use std::time::Duration;
  12 use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH};
  13
  14 use anyhow::{bail, format_err, Error};
  15 use serde_json::Value;
  16 use openssl::hash::{hash, DigestBytes, MessageDigest};
  17 use percent_encoding::AsciiSet;
  18
  19 use proxmox::tools::vec;
  20
  21 pub use proxmox::tools::fd::Fd;
  22
  23 pub mod acl;
  24 pub mod async_io;
  25 pub mod borrow;
  26 pub mod daemon;
  27 pub mod disks;
  28 pub mod fs;
  29 pub mod format;
  30 pub mod lru_cache;
  31 pub mod runtime;
  32 pub mod ticket;
  33 pub mod timer;
  34 pub mod statistics;
  35 pub mod systemd;
  36 pub mod nom;
  37
  38 mod wrapped_reader_stream;
  39 pub use wrapped_reader_stream::*;
  40
  41 mod std_channel_writer;
  42 pub use std_channel_writer::*;
  43
  44 pub mod xattr;
  45
  46 mod process_locker;
  47 pub use process_locker::*;
  48
  49 mod file_logger;
  50 pub use file_logger::*;
  51
  52 mod broadcast_future;
  53 pub use broadcast_future::*;
  54
  55 /// The `BufferedRead` trait provides a single function
  56 /// `buffered_read`. It returns a reference to an internal buffer. The
  57 /// purpose of this traid is to avoid unnecessary data copies.
  58 pub trait BufferedRead {
  59     /// This functions tries to fill the internal buffers, then
  60     /// returns a reference to the available data. It returns an empty
  61     /// buffer if `offset` points to the end of the file.
  62     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  63 }
  64
  65 /// Directly map a type into a binary buffer. This is mostly useful
  66 /// for reading structured data from a byte stream (file). You need to
  67 /// make sure that the buffer location does not change, so please
  68 /// avoid vec resize while you use such map.
  69 ///
  70 /// This function panics if the buffer is not large enough.
  71 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  72     if buffer.len() < ::std::mem::size_of::<T>() {
  73         bail!("unable to map struct - buffer too small");
  74     }
  75     Ok(unsafe { &*(buffer.as_ptr() as *const T) })
  76 }
  77
  78 /// Directly map a type into a mutable binary buffer. This is mostly
  79 /// useful for writing structured data into a byte stream (file). You
  80 /// need to make sure that the buffer location does not change, so
  81 /// please avoid vec resize while you use such map.
  82 ///
  83 /// This function panics if the buffer is not large enough.
  84 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  85     if buffer.len() < ::std::mem::size_of::<T>() {
  86         bail!("unable to map struct - buffer too small");
  87     }
  88     Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) })
  89 }
  90
  91 /// Create a file lock using fntl. This function allows you to specify
  92 /// a timeout if you want to avoid infinite blocking.
  93 pub fn lock_file<F: AsRawFd>(
  94     file: &mut F,
  95     exclusive: bool,
  96     timeout: Option<Duration>,
  97 ) -> Result<(), Error> {
  98     let lockarg = if exclusive {
  99         nix::fcntl::FlockArg::LockExclusive
 100     } else {
 101         nix::fcntl::FlockArg::LockShared
 102     };
 103
 104     let timeout = match timeout {
 105         None => {
 106             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 107             return Ok(());
 108         }
 109         Some(t) => t,
 110     };
 111
 112     // unblock the timeout signal temporarily
 113     let _sigblock_guard = timer::unblock_timeout_signal();
 114
 115     // setup a timeout timer
 116     let mut timer = timer::Timer::create(
 117         timer::Clock::Realtime,
 118         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT),
 119     )?;
 120
 121     timer.arm(
 122         timer::TimerSpec::new()
 123             .value(Some(timeout))
 124             .interval(Some(Duration::from_millis(10))),
 125     )?;
 126
 127     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 128     Ok(())
 129 }
 130
 131 /// Open or create a lock file (append mode). Then try to
 132 /// acquire a lock using `lock_file()`.
 133 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<File, Error> {
 134     let path = path.as_ref();
 135     let mut file = match OpenOptions::new().create(true).append(true).open(path) {
 136         Ok(file) => file,
 137         Err(err) => bail!("Unable to open lock {:?} - {}", path, err),
 138     };
 139     match lock_file(&mut file, true, Some(timeout)) {
 140         Ok(_) => Ok(file),
 141         Err(err) => bail!("Unable to acquire lock {:?} - {}", path, err),
 142     }
 143 }
 144
 145 /// Split a file into equal sized chunks. The last chunk may be
 146 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 147 /// cannot return a borrowed buffer ref (we want zero-copy)
 148 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
 149 where
 150     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 151     R: Read,
 152 {
 153     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
 154
 155     if chunk_size > READ_BUFFER_SIZE {
 156         bail!("chunk size too large!");
 157     }
 158
 159     let mut buf = vec::undefined(READ_BUFFER_SIZE);
 160
 161     let mut pos = 0;
 162     let mut file_pos = 0;
 163     loop {
 164         let mut eof = false;
 165         let mut tmp = &mut buf[..];
 166         // try to read large portions, at least chunk_size
 167         while pos < chunk_size {
 168             match file.read(tmp) {
 169                 Ok(0) => {
 170                     eof = true;
 171                     break;
 172                 }
 173                 Ok(n) => {
 174                     pos += n;
 175                     if pos > chunk_size {
 176                         break;
 177                     }
 178                     tmp = &mut tmp[n..];
 179                 }
 180                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 181                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 182             }
 183         }
 184         let mut start = 0;
 185         while start + chunk_size <= pos {
 186             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 187                 break;
 188             }
 189             file_pos += chunk_size;
 190             start += chunk_size;
 191         }
 192         if eof {
 193             if start < pos {
 194                 (chunk_cb)(file_pos, &buf[start..pos])?;
 195                 //file_pos += pos - start;
 196             }
 197             break;
 198         } else {
 199             let rest = pos - start;
 200             if rest > 0 {
 201                 let ptr = buf.as_mut_ptr();
 202                 unsafe {
 203                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 204                 }
 205                 pos = rest;
 206             } else {
 207                 pos = 0;
 208             }
 209         }
 210     }
 211
 212     Ok(())
 213 }
 214
 215 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 216     let mut query = url::form_urlencoded::Serializer::new(String::new());
 217
 218     let object = data.as_object().ok_or_else(|| {
 219         format_err!("json_object_to_query: got wrong data type (expected object).")
 220     })?;
 221
 222     for (key, value) in object {
 223         match value {
 224             Value::Bool(b) => {
 225                 query.append_pair(key, &b.to_string());
 226             }
 227             Value::Number(n) => {
 228                 query.append_pair(key, &n.to_string());
 229             }
 230             Value::String(s) => {
 231                 query.append_pair(key, &s);
 232             }
 233             Value::Array(arr) => {
 234                 for element in arr {
 235                     match element {
 236                         Value::Bool(b) => {
 237                             query.append_pair(key, &b.to_string());
 238                         }
 239                         Value::Number(n) => {
 240                             query.append_pair(key, &n.to_string());
 241                         }
 242                         Value::String(s) => {
 243                             query.append_pair(key, &s);
 244                         }
 245                         _ => bail!(
 246                             "json_object_to_query: unable to handle complex array data types."
 247                         ),
 248                     }
 249                 }
 250             }
 251             _ => bail!("json_object_to_query: unable to handle complex data types."),
 252         }
 253     }
 254
 255     Ok(query.finish())
 256 }
 257
 258 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 259     match param[name].as_str() {
 260         Some(s) => Ok(s),
 261         None => bail!("missing parameter '{}'", name),
 262     }
 263 }
 264
 265 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 266     match param[name].as_str() {
 267         Some(s) => Ok(s),
 268         None => bail!("missing property '{}'", name),
 269     }
 270 }
 271
 272 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 273     match param[name].as_i64() {
 274         Some(s) => Ok(s),
 275         None => bail!("missing parameter '{}'", name),
 276     }
 277 }
 278
 279 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 280     match param[name].as_i64() {
 281         Some(s) => Ok(s),
 282         None => bail!("missing property '{}'", name),
 283     }
 284 }
 285
 286 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 287     match param[name].as_array() {
 288         Some(s) => Ok(s.to_vec()),
 289         None => bail!("missing parameter '{}'", name),
 290     }
 291 }
 292
 293 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 294     match param[name].as_array() {
 295         Some(s) => Ok(s.to_vec()),
 296         None => bail!("missing property '{}'", name),
 297     }
 298 }
 299
 300 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 301     let mut result = vec![];
 302
 303     use nix::fcntl::AtFlags;
 304     use nix::fcntl::OFlag;
 305     use nix::sys::stat::Mode;
 306
 307     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 308
 309     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 310         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 311         Err(_) => false,
 312     };
 313
 314     if !is_dir {
 315         if let Some(parent) = dirname.parent() {
 316             dirname = parent.to_owned();
 317         }
 318     }
 319
 320     let mut dir =
 321         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 322             Ok(d) => d,
 323             Err(_) => return result,
 324         };
 325
 326     for item in dir.iter() {
 327         if let Ok(entry) = item {
 328             if let Ok(name) = entry.file_name().to_str() {
 329                 if name == "." || name == ".." {
 330                     continue;
 331                 }
 332                 let mut newpath = dirname.clone();
 333                 newpath.push(name);
 334
 335                 if let Ok(stat) =
 336                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 337                 {
 338                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 339                         newpath.push("");
 340                         if let Some(newpath) = newpath.to_str() {
 341                             result.push(newpath.to_owned());
 342                         }
 343                         continue;
 344                     }
 345                 }
 346                 if let Some(newpath) = newpath.to_str() {
 347                     result.push(newpath.to_owned());
 348                 }
 349             }
 350         }
 351     }
 352
 353     result
 354 }
 355
 356 /// Scan directory for matching file names.
 357 ///
 358 /// Scan through all directory entries and call `callback()` function
 359 /// if the entry name matches the regular expression. This function
 360 /// used unix `openat()`, so you can pass absolute or relative file
 361 /// names. This function simply skips non-UTF8 encoded names.
 362 pub fn scandir<P, F>(
 363     dirfd: RawFd,
 364     path: &P,
 365     regex: &regex::Regex,
 366     mut callback: F,
 367 ) -> Result<(), Error>
 368 where
 369     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 370     P: ?Sized + nix::NixPath,
 371 {
 372     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 373         let entry = entry?;
 374         let file_type = match entry.file_type() {
 375             Some(file_type) => file_type,
 376             None => bail!("unable to detect file type"),
 377         };
 378
 379         callback(
 380             entry.parent_fd(),
 381             unsafe { entry.file_name_utf8_unchecked() },
 382             file_type,
 383         )?;
 384     }
 385     Ok(())
 386 }
 387
 388 /// Shortcut for md5 sums.
 389 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 390     hash(MessageDigest::md5(), data).map_err(Error::from)
 391 }
 392
 393 pub fn get_hardware_address() -> Result<String, Error> {
 394     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 395
 396     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 397     let digest = md5sum(&contents)?;
 398
 399     Ok(proxmox::tools::bin_to_hex(&digest))
 400 }
 401
 402 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 403     if digest1 != digest2 {
 404         bail!("detected modified configuration - file changed by other user? Try again.");
 405     }
 406     Ok(())
 407 }
 408
 409 /// Extract authentication cookie from cookie header.
 410 /// We assume cookie_name is already url encoded.
 411 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 412     for pair in cookie.split(';') {
 413         let (name, value) = match pair.find('=') {
 414             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 415             None => return None, // Cookie format error
 416         };
 417
 418         if name == cookie_name {
 419             use percent_encoding::percent_decode;
 420             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 421                 return Some(value.into());
 422             } else {
 423                 return None; // Cookie format error
 424             }
 425         }
 426     }
 427
 428     None
 429 }
 430
 431 pub fn join(data: &Vec<String>, sep: char) -> String {
 432     let mut list = String::new();
 433
 434     for item in data {
 435         if !list.is_empty() {
 436             list.push(sep);
 437         }
 438         list.push_str(item);
 439     }
 440
 441     list
 442 }
 443
 444 /// Detect modified configuration files
 445 ///
 446 /// This function fails with a reasonable error message if checksums do not match.
 447 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 448     if digest1 != digest2 {
 449         bail!("detected modified configuration - file changed by other user? Try again.");
 450     }
 451     Ok(())
 452 }
 453
 454 /// normalize uri path
 455 ///
 456 /// Do not allow ".", "..", or hidden files ".XXXX"
 457 /// Also remove empty path components
 458 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 459     let items = path.split('/');
 460
 461     let mut path = String::new();
 462     let mut components = vec![];
 463
 464     for name in items {
 465         if name.is_empty() {
 466             continue;
 467         }
 468         if name.starts_with('.') {
 469             bail!("Path contains illegal components.");
 470         }
 471         path.push('/');
 472         path.push_str(name);
 473         components.push(name);
 474     }
 475
 476     Ok((path, components))
 477 }
 478
 479 /// Helper to check result from std::process::Command output
 480 ///
 481 /// The exit_code_check() function should return true if the exit code
 482 /// is considered successful.
 483 pub fn command_output(
 484     output: std::process::Output,
 485     exit_code_check: Option<fn(i32) -> bool>,
 486 ) -> Result<String, Error> {
 487
 488     if !output.status.success() {
 489         match output.status.code() {
 490             Some(code) => {
 491                 let is_ok = match exit_code_check {
 492                     Some(check_fn) => check_fn(code),
 493                     None => code == 0,
 494                 };
 495                 if !is_ok {
 496                     let msg = String::from_utf8(output.stderr)
 497                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 498                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 499
 500                     bail!("status code: {} - {}", code, msg);
 501                 }
 502             }
 503             None => bail!("terminated by signal"),
 504         }
 505     }
 506
 507     let output = String::from_utf8(output.stdout)?;
 508
 509     Ok(output)
 510 }
 511
 512 pub fn run_command(
 513     mut command: std::process::Command,
 514     exit_code_check: Option<fn(i32) -> bool>,
 515 ) -> Result<String, Error> {
 516
 517    let output = command.output()
 518         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 519
 520     let output = crate::tools::command_output(output, exit_code_check)
 521         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 522
 523     Ok(output)
 524 }
 525
 526 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 527     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 528     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 529         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 530     flags.set(FdFlag::FD_CLOEXEC, on);
 531     fcntl(fd, F_SETFD(flags))?;
 532     Ok(())
 533 }
 534
 535 static mut SHUTDOWN_REQUESTED: bool = false;
 536
 537 pub fn request_shutdown() {
 538     unsafe {
 539         SHUTDOWN_REQUESTED = true;
 540     }
 541     crate::server::server_shutdown();
 542 }
 543
 544 #[inline(always)]
 545 pub fn shutdown_requested() -> bool {
 546     unsafe { SHUTDOWN_REQUESTED }
 547 }
 548
 549 pub fn fail_on_shutdown() -> Result<(), Error> {
 550     if shutdown_requested() {
 551         bail!("Server shutdown requested - aborting task");
 552     }
 553     Ok(())
 554 }
 555
 556 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 557 /// descriptors.
 558 pub fn pipe() -> Result<(Fd, Fd), Error> {
 559     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 560     Ok((Fd(pin), Fd(pout)))
 561 }
 562
 563 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 564 /// descriptors.
 565 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 566     use nix::sys::socket;
 567     let (pa, pb) = socket::socketpair(
 568         socket::AddressFamily::Unix,
 569         socket::SockType::Stream,
 570         None,
 571         socket::SockFlag::SOCK_CLOEXEC,
 572     )?;
 573     Ok((Fd(pa), Fd(pb)))
 574 }
 575
 576
 577 /// An easy way to convert types to Any
 578 ///
 579 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 580 pub trait AsAny {
 581     fn as_any(&self) -> &dyn Any;
 582 }
 583
 584 impl<T: Any> AsAny for T {
 585     fn as_any(&self) -> &dyn Any {
 586         self
 587     }
 588 }
 589
 590 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 591 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 592     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 593     .add(0x20)
 594     .add(0x7f)
 595     // the DEFAULT_ENCODE_SET added:
 596     .add(b' ')
 597     .add(b'"')
 598     .add(b'#')
 599     .add(b'<')
 600     .add(b'>')
 601     .add(b'`')
 602     .add(b'?')
 603     .add(b'{')
 604     .add(b'}');
 605
 606 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 607 /// `#`).
 608 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 609     path: P,
 610 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 611     let path = path.as_ref();
 612
 613     Ok(io::BufReader::new(
 614         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 615     )
 616     .lines()
 617     .filter_map(|line| match line {
 618         Ok(line) => {
 619             let line = line.trim();
 620             if line.is_empty() || line.starts_with('#') {
 621                 None
 622             } else {
 623                 Some(Ok(line.to_string()))
 624             }
 625         }
 626         Err(err) => Some(Err(err)),
 627     }))
 628 }
 629
 630 pub fn epoch_now() -> Result<Duration, SystemTimeError> {
 631     SystemTime::now().duration_since(UNIX_EPOCH)
 632 }
 633
 634 pub fn epoch_now_f64() -> Result<f64, SystemTimeError> {
 635     Ok(epoch_now()?.as_secs_f64())
 636 }
 637
 638 pub fn epoch_now_u64() -> Result<u64, SystemTimeError> {
 639     Ok(epoch_now()?.as_secs())
 640 }
 641
 642 pub fn setup_safe_path_env() {
 643     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 644     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 645     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 646         std::env::remove_var(name);
 647     }
 648 }