src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, ErrorKind, Read};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11 use std::time::Duration;
  12 use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH};
  13
  14 use anyhow::{bail, format_err, Error};
  15 use serde_json::Value;
  16 use openssl::hash::{hash, DigestBytes, MessageDigest};
  17 use percent_encoding::AsciiSet;
  18
  19 use proxmox::tools::vec;
  20
  21 pub use proxmox::tools::fd::Fd;
  22
  23 pub mod acl;
  24 pub mod async_io;
  25 pub mod borrow;
  26 pub mod cert;
  27 pub mod daemon;
  28 pub mod disks;
  29 pub mod fs;
  30 pub mod format;
  31 pub mod lru_cache;
  32 pub mod runtime;
  33 pub mod ticket;
  34 pub mod statistics;
  35 pub mod systemd;
  36 pub mod nom;
  37
  38 mod wrapped_reader_stream;
  39 pub use wrapped_reader_stream::*;
  40
  41 mod std_channel_writer;
  42 pub use std_channel_writer::*;
  43
  44 pub mod xattr;
  45
  46 mod process_locker;
  47 pub use process_locker::*;
  48
  49 mod file_logger;
  50 pub use file_logger::*;
  51
  52 mod broadcast_future;
  53 pub use broadcast_future::*;
  54
  55 /// The `BufferedRead` trait provides a single function
  56 /// `buffered_read`. It returns a reference to an internal buffer. The
  57 /// purpose of this traid is to avoid unnecessary data copies.
  58 pub trait BufferedRead {
  59     /// This functions tries to fill the internal buffers, then
  60     /// returns a reference to the available data. It returns an empty
  61     /// buffer if `offset` points to the end of the file.
  62     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  63 }
  64
  65 /// Directly map a type into a binary buffer. This is mostly useful
  66 /// for reading structured data from a byte stream (file). You need to
  67 /// make sure that the buffer location does not change, so please
  68 /// avoid vec resize while you use such map.
  69 ///
  70 /// This function panics if the buffer is not large enough.
  71 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  72     if buffer.len() < ::std::mem::size_of::<T>() {
  73         bail!("unable to map struct - buffer too small");
  74     }
  75     Ok(unsafe { &*(buffer.as_ptr() as *const T) })
  76 }
  77
  78 /// Directly map a type into a mutable binary buffer. This is mostly
  79 /// useful for writing structured data into a byte stream (file). You
  80 /// need to make sure that the buffer location does not change, so
  81 /// please avoid vec resize while you use such map.
  82 ///
  83 /// This function panics if the buffer is not large enough.
  84 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  85     if buffer.len() < ::std::mem::size_of::<T>() {
  86         bail!("unable to map struct - buffer too small");
  87     }
  88     Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) })
  89 }
  90
  91 /// Split a file into equal sized chunks. The last chunk may be
  92 /// smaller. Note: We cannot implement an `Iterator`, because iterators
  93 /// cannot return a borrowed buffer ref (we want zero-copy)
  94 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
  95 where
  96     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
  97     R: Read,
  98 {
  99     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
 100
 101     if chunk_size > READ_BUFFER_SIZE {
 102         bail!("chunk size too large!");
 103     }
 104
 105     let mut buf = vec::undefined(READ_BUFFER_SIZE);
 106
 107     let mut pos = 0;
 108     let mut file_pos = 0;
 109     loop {
 110         let mut eof = false;
 111         let mut tmp = &mut buf[..];
 112         // try to read large portions, at least chunk_size
 113         while pos < chunk_size {
 114             match file.read(tmp) {
 115                 Ok(0) => {
 116                     eof = true;
 117                     break;
 118                 }
 119                 Ok(n) => {
 120                     pos += n;
 121                     if pos > chunk_size {
 122                         break;
 123                     }
 124                     tmp = &mut tmp[n..];
 125                 }
 126                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 127                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 128             }
 129         }
 130         let mut start = 0;
 131         while start + chunk_size <= pos {
 132             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 133                 break;
 134             }
 135             file_pos += chunk_size;
 136             start += chunk_size;
 137         }
 138         if eof {
 139             if start < pos {
 140                 (chunk_cb)(file_pos, &buf[start..pos])?;
 141                 //file_pos += pos - start;
 142             }
 143             break;
 144         } else {
 145             let rest = pos - start;
 146             if rest > 0 {
 147                 let ptr = buf.as_mut_ptr();
 148                 unsafe {
 149                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 150                 }
 151                 pos = rest;
 152             } else {
 153                 pos = 0;
 154             }
 155         }
 156     }
 157
 158     Ok(())
 159 }
 160
 161 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 162     let mut query = url::form_urlencoded::Serializer::new(String::new());
 163
 164     let object = data.as_object().ok_or_else(|| {
 165         format_err!("json_object_to_query: got wrong data type (expected object).")
 166     })?;
 167
 168     for (key, value) in object {
 169         match value {
 170             Value::Bool(b) => {
 171                 query.append_pair(key, &b.to_string());
 172             }
 173             Value::Number(n) => {
 174                 query.append_pair(key, &n.to_string());
 175             }
 176             Value::String(s) => {
 177                 query.append_pair(key, &s);
 178             }
 179             Value::Array(arr) => {
 180                 for element in arr {
 181                     match element {
 182                         Value::Bool(b) => {
 183                             query.append_pair(key, &b.to_string());
 184                         }
 185                         Value::Number(n) => {
 186                             query.append_pair(key, &n.to_string());
 187                         }
 188                         Value::String(s) => {
 189                             query.append_pair(key, &s);
 190                         }
 191                         _ => bail!(
 192                             "json_object_to_query: unable to handle complex array data types."
 193                         ),
 194                     }
 195                 }
 196             }
 197             _ => bail!("json_object_to_query: unable to handle complex data types."),
 198         }
 199     }
 200
 201     Ok(query.finish())
 202 }
 203
 204 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 205     match param[name].as_str() {
 206         Some(s) => Ok(s),
 207         None => bail!("missing parameter '{}'", name),
 208     }
 209 }
 210
 211 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 212     match param[name].as_str() {
 213         Some(s) => Ok(s),
 214         None => bail!("missing property '{}'", name),
 215     }
 216 }
 217
 218 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 219     match param[name].as_i64() {
 220         Some(s) => Ok(s),
 221         None => bail!("missing parameter '{}'", name),
 222     }
 223 }
 224
 225 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 226     match param[name].as_i64() {
 227         Some(s) => Ok(s),
 228         None => bail!("missing property '{}'", name),
 229     }
 230 }
 231
 232 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 233     match param[name].as_array() {
 234         Some(s) => Ok(s.to_vec()),
 235         None => bail!("missing parameter '{}'", name),
 236     }
 237 }
 238
 239 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 240     match param[name].as_array() {
 241         Some(s) => Ok(s.to_vec()),
 242         None => bail!("missing property '{}'", name),
 243     }
 244 }
 245
 246 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 247     let mut result = vec![];
 248
 249     use nix::fcntl::AtFlags;
 250     use nix::fcntl::OFlag;
 251     use nix::sys::stat::Mode;
 252
 253     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 254
 255     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 256         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 257         Err(_) => false,
 258     };
 259
 260     if !is_dir {
 261         if let Some(parent) = dirname.parent() {
 262             dirname = parent.to_owned();
 263         }
 264     }
 265
 266     let mut dir =
 267         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 268             Ok(d) => d,
 269             Err(_) => return result,
 270         };
 271
 272     for item in dir.iter() {
 273         if let Ok(entry) = item {
 274             if let Ok(name) = entry.file_name().to_str() {
 275                 if name == "." || name == ".." {
 276                     continue;
 277                 }
 278                 let mut newpath = dirname.clone();
 279                 newpath.push(name);
 280
 281                 if let Ok(stat) =
 282                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 283                 {
 284                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 285                         newpath.push("");
 286                         if let Some(newpath) = newpath.to_str() {
 287                             result.push(newpath.to_owned());
 288                         }
 289                         continue;
 290                     }
 291                 }
 292                 if let Some(newpath) = newpath.to_str() {
 293                     result.push(newpath.to_owned());
 294                 }
 295             }
 296         }
 297     }
 298
 299     result
 300 }
 301
 302 /// Scan directory for matching file names.
 303 ///
 304 /// Scan through all directory entries and call `callback()` function
 305 /// if the entry name matches the regular expression. This function
 306 /// used unix `openat()`, so you can pass absolute or relative file
 307 /// names. This function simply skips non-UTF8 encoded names.
 308 pub fn scandir<P, F>(
 309     dirfd: RawFd,
 310     path: &P,
 311     regex: &regex::Regex,
 312     mut callback: F,
 313 ) -> Result<(), Error>
 314 where
 315     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 316     P: ?Sized + nix::NixPath,
 317 {
 318     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 319         let entry = entry?;
 320         let file_type = match entry.file_type() {
 321             Some(file_type) => file_type,
 322             None => bail!("unable to detect file type"),
 323         };
 324
 325         callback(
 326             entry.parent_fd(),
 327             unsafe { entry.file_name_utf8_unchecked() },
 328             file_type,
 329         )?;
 330     }
 331     Ok(())
 332 }
 333
 334 /// Shortcut for md5 sums.
 335 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 336     hash(MessageDigest::md5(), data).map_err(Error::from)
 337 }
 338
 339 pub fn get_hardware_address() -> Result<String, Error> {
 340     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 341
 342     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 343     let digest = md5sum(&contents)?;
 344
 345     Ok(proxmox::tools::bin_to_hex(&digest))
 346 }
 347
 348 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 349     if digest1 != digest2 {
 350         bail!("detected modified configuration - file changed by other user? Try again.");
 351     }
 352     Ok(())
 353 }
 354
 355 /// Extract authentication cookie from cookie header.
 356 /// We assume cookie_name is already url encoded.
 357 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 358     for pair in cookie.split(';') {
 359         let (name, value) = match pair.find('=') {
 360             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 361             None => return None, // Cookie format error
 362         };
 363
 364         if name == cookie_name {
 365             use percent_encoding::percent_decode;
 366             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 367                 return Some(value.into());
 368             } else {
 369                 return None; // Cookie format error
 370             }
 371         }
 372     }
 373
 374     None
 375 }
 376
 377 pub fn join(data: &Vec<String>, sep: char) -> String {
 378     let mut list = String::new();
 379
 380     for item in data {
 381         if !list.is_empty() {
 382             list.push(sep);
 383         }
 384         list.push_str(item);
 385     }
 386
 387     list
 388 }
 389
 390 /// Detect modified configuration files
 391 ///
 392 /// This function fails with a reasonable error message if checksums do not match.
 393 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 394     if digest1 != digest2 {
 395         bail!("detected modified configuration - file changed by other user? Try again.");
 396     }
 397     Ok(())
 398 }
 399
 400 /// normalize uri path
 401 ///
 402 /// Do not allow ".", "..", or hidden files ".XXXX"
 403 /// Also remove empty path components
 404 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 405     let items = path.split('/');
 406
 407     let mut path = String::new();
 408     let mut components = vec![];
 409
 410     for name in items {
 411         if name.is_empty() {
 412             continue;
 413         }
 414         if name.starts_with('.') {
 415             bail!("Path contains illegal components.");
 416         }
 417         path.push('/');
 418         path.push_str(name);
 419         components.push(name);
 420     }
 421
 422     Ok((path, components))
 423 }
 424
 425 /// Helper to check result from std::process::Command output
 426 ///
 427 /// The exit_code_check() function should return true if the exit code
 428 /// is considered successful.
 429 pub fn command_output(
 430     output: std::process::Output,
 431     exit_code_check: Option<fn(i32) -> bool>,
 432 ) -> Result<String, Error> {
 433
 434     if !output.status.success() {
 435         match output.status.code() {
 436             Some(code) => {
 437                 let is_ok = match exit_code_check {
 438                     Some(check_fn) => check_fn(code),
 439                     None => code == 0,
 440                 };
 441                 if !is_ok {
 442                     let msg = String::from_utf8(output.stderr)
 443                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 444                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 445
 446                     bail!("status code: {} - {}", code, msg);
 447                 }
 448             }
 449             None => bail!("terminated by signal"),
 450         }
 451     }
 452
 453     let output = String::from_utf8(output.stdout)?;
 454
 455     Ok(output)
 456 }
 457
 458 pub fn run_command(
 459     mut command: std::process::Command,
 460     exit_code_check: Option<fn(i32) -> bool>,
 461 ) -> Result<String, Error> {
 462
 463    let output = command.output()
 464         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 465
 466     let output = crate::tools::command_output(output, exit_code_check)
 467         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 468
 469     Ok(output)
 470 }
 471
 472 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 473     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 474     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 475         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 476     flags.set(FdFlag::FD_CLOEXEC, on);
 477     fcntl(fd, F_SETFD(flags))?;
 478     Ok(())
 479 }
 480
 481 static mut SHUTDOWN_REQUESTED: bool = false;
 482
 483 pub fn request_shutdown() {
 484     unsafe {
 485         SHUTDOWN_REQUESTED = true;
 486     }
 487     crate::server::server_shutdown();
 488 }
 489
 490 #[inline(always)]
 491 pub fn shutdown_requested() -> bool {
 492     unsafe { SHUTDOWN_REQUESTED }
 493 }
 494
 495 pub fn fail_on_shutdown() -> Result<(), Error> {
 496     if shutdown_requested() {
 497         bail!("Server shutdown requested - aborting task");
 498     }
 499     Ok(())
 500 }
 501
 502 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 503 /// descriptors.
 504 pub fn pipe() -> Result<(Fd, Fd), Error> {
 505     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 506     Ok((Fd(pin), Fd(pout)))
 507 }
 508
 509 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 510 /// descriptors.
 511 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 512     use nix::sys::socket;
 513     let (pa, pb) = socket::socketpair(
 514         socket::AddressFamily::Unix,
 515         socket::SockType::Stream,
 516         None,
 517         socket::SockFlag::SOCK_CLOEXEC,
 518     )?;
 519     Ok((Fd(pa), Fd(pb)))
 520 }
 521
 522
 523 /// An easy way to convert types to Any
 524 ///
 525 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 526 pub trait AsAny {
 527     fn as_any(&self) -> &dyn Any;
 528 }
 529
 530 impl<T: Any> AsAny for T {
 531     fn as_any(&self) -> &dyn Any {
 532         self
 533     }
 534 }
 535
 536 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 537 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 538     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 539     .add(0x20)
 540     .add(0x7f)
 541     // the DEFAULT_ENCODE_SET added:
 542     .add(b' ')
 543     .add(b'"')
 544     .add(b'#')
 545     .add(b'<')
 546     .add(b'>')
 547     .add(b'`')
 548     .add(b'?')
 549     .add(b'{')
 550     .add(b'}');
 551
 552 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 553 /// `#`).
 554 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 555     path: P,
 556 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 557     let path = path.as_ref();
 558
 559     Ok(io::BufReader::new(
 560         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 561     )
 562     .lines()
 563     .filter_map(|line| match line {
 564         Ok(line) => {
 565             let line = line.trim();
 566             if line.is_empty() || line.starts_with('#') {
 567                 None
 568             } else {
 569                 Some(Ok(line.to_string()))
 570             }
 571         }
 572         Err(err) => Some(Err(err)),
 573     }))
 574 }
 575
 576 pub fn epoch_now() -> Result<Duration, SystemTimeError> {
 577     SystemTime::now().duration_since(UNIX_EPOCH)
 578 }
 579
 580 pub fn epoch_now_f64() -> Result<f64, SystemTimeError> {
 581     Ok(epoch_now()?.as_secs_f64())
 582 }
 583
 584 pub fn epoch_now_u64() -> Result<u64, SystemTimeError> {
 585     Ok(epoch_now()?.as_secs())
 586 }
 587
 588 pub fn setup_safe_path_env() {
 589     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 590     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 591     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 592         std::env::remove_var(name);
 593     }
 594 }
 595
 596 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 597     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 598         Some(n) => &line[n..],
 599         None => return &[],
 600     };
 601     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 602         Some(n) => &line[..(line.len() - n)],
 603         None => &[],
 604     }
 605 }