src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::AsciiSet;
  16
  17 use proxmox::tools::vec;
  18
  19 pub use proxmox::tools::fd::Fd;
  20
  21 pub mod acl;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod cert;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod statistics;
  33 pub mod systemd;
  34 pub mod nom;
  35 pub mod logrotate;
  36 pub mod loopdev;
  37 pub mod fuse_loop;
  38 pub mod socket;
  39
  40 mod parallel_handler;
  41 pub use parallel_handler::*;
  42
  43 mod wrapped_reader_stream;
  44 pub use wrapped_reader_stream::*;
  45
  46 mod std_channel_writer;
  47 pub use std_channel_writer::*;
  48
  49 pub mod xattr;
  50
  51 mod process_locker;
  52 pub use process_locker::*;
  53
  54 mod file_logger;
  55 pub use file_logger::*;
  56
  57 mod broadcast_future;
  58 pub use broadcast_future::*;
  59
  60 /// The `BufferedRead` trait provides a single function
  61 /// `buffered_read`. It returns a reference to an internal buffer. The
  62 /// purpose of this traid is to avoid unnecessary data copies.
  63 pub trait BufferedRead {
  64     /// This functions tries to fill the internal buffers, then
  65     /// returns a reference to the available data. It returns an empty
  66     /// buffer if `offset` points to the end of the file.
  67     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  68 }
  69
  70 /// Split a file into equal sized chunks. The last chunk may be
  71 /// smaller. Note: We cannot implement an `Iterator`, because iterators
  72 /// cannot return a borrowed buffer ref (we want zero-copy)
  73 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
  74 where
  75     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
  76     R: Read,
  77 {
  78     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
  79
  80     if chunk_size > READ_BUFFER_SIZE {
  81         bail!("chunk size too large!");
  82     }
  83
  84     let mut buf = vec::undefined(READ_BUFFER_SIZE);
  85
  86     let mut pos = 0;
  87     let mut file_pos = 0;
  88     loop {
  89         let mut eof = false;
  90         let mut tmp = &mut buf[..];
  91         // try to read large portions, at least chunk_size
  92         while pos < chunk_size {
  93             match file.read(tmp) {
  94                 Ok(0) => {
  95                     eof = true;
  96                     break;
  97                 }
  98                 Ok(n) => {
  99                     pos += n;
 100                     if pos > chunk_size {
 101                         break;
 102                     }
 103                     tmp = &mut tmp[n..];
 104                 }
 105                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 106                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 107             }
 108         }
 109         let mut start = 0;
 110         while start + chunk_size <= pos {
 111             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 112                 break;
 113             }
 114             file_pos += chunk_size;
 115             start += chunk_size;
 116         }
 117         if eof {
 118             if start < pos {
 119                 (chunk_cb)(file_pos, &buf[start..pos])?;
 120                 //file_pos += pos - start;
 121             }
 122             break;
 123         } else {
 124             let rest = pos - start;
 125             if rest > 0 {
 126                 let ptr = buf.as_mut_ptr();
 127                 unsafe {
 128                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 129                 }
 130                 pos = rest;
 131             } else {
 132                 pos = 0;
 133             }
 134         }
 135     }
 136
 137     Ok(())
 138 }
 139
 140 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 141     let mut query = url::form_urlencoded::Serializer::new(String::new());
 142
 143     let object = data.as_object().ok_or_else(|| {
 144         format_err!("json_object_to_query: got wrong data type (expected object).")
 145     })?;
 146
 147     for (key, value) in object {
 148         match value {
 149             Value::Bool(b) => {
 150                 query.append_pair(key, &b.to_string());
 151             }
 152             Value::Number(n) => {
 153                 query.append_pair(key, &n.to_string());
 154             }
 155             Value::String(s) => {
 156                 query.append_pair(key, &s);
 157             }
 158             Value::Array(arr) => {
 159                 for element in arr {
 160                     match element {
 161                         Value::Bool(b) => {
 162                             query.append_pair(key, &b.to_string());
 163                         }
 164                         Value::Number(n) => {
 165                             query.append_pair(key, &n.to_string());
 166                         }
 167                         Value::String(s) => {
 168                             query.append_pair(key, &s);
 169                         }
 170                         _ => bail!(
 171                             "json_object_to_query: unable to handle complex array data types."
 172                         ),
 173                     }
 174                 }
 175             }
 176             _ => bail!("json_object_to_query: unable to handle complex data types."),
 177         }
 178     }
 179
 180     Ok(query.finish())
 181 }
 182
 183 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 184     match param[name].as_str() {
 185         Some(s) => Ok(s),
 186         None => bail!("missing parameter '{}'", name),
 187     }
 188 }
 189
 190 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 191     match param[name].as_str() {
 192         Some(s) => Ok(s),
 193         None => bail!("missing property '{}'", name),
 194     }
 195 }
 196
 197 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 198     match param[name].as_i64() {
 199         Some(s) => Ok(s),
 200         None => bail!("missing parameter '{}'", name),
 201     }
 202 }
 203
 204 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 205     match param[name].as_i64() {
 206         Some(s) => Ok(s),
 207         None => bail!("missing property '{}'", name),
 208     }
 209 }
 210
 211 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 212     match param[name].as_array() {
 213         Some(s) => Ok(s.to_vec()),
 214         None => bail!("missing parameter '{}'", name),
 215     }
 216 }
 217
 218 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 219     match param[name].as_array() {
 220         Some(s) => Ok(s.to_vec()),
 221         None => bail!("missing property '{}'", name),
 222     }
 223 }
 224
 225 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 226     let mut result = vec![];
 227
 228     use nix::fcntl::AtFlags;
 229     use nix::fcntl::OFlag;
 230     use nix::sys::stat::Mode;
 231
 232     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 233
 234     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 235         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 236         Err(_) => false,
 237     };
 238
 239     if !is_dir {
 240         if let Some(parent) = dirname.parent() {
 241             dirname = parent.to_owned();
 242         }
 243     }
 244
 245     let mut dir =
 246         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 247             Ok(d) => d,
 248             Err(_) => return result,
 249         };
 250
 251     for item in dir.iter() {
 252         if let Ok(entry) = item {
 253             if let Ok(name) = entry.file_name().to_str() {
 254                 if name == "." || name == ".." {
 255                     continue;
 256                 }
 257                 let mut newpath = dirname.clone();
 258                 newpath.push(name);
 259
 260                 if let Ok(stat) =
 261                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 262                 {
 263                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 264                         newpath.push("");
 265                         if let Some(newpath) = newpath.to_str() {
 266                             result.push(newpath.to_owned());
 267                         }
 268                         continue;
 269                     }
 270                 }
 271                 if let Some(newpath) = newpath.to_str() {
 272                     result.push(newpath.to_owned());
 273                 }
 274             }
 275         }
 276     }
 277
 278     result
 279 }
 280
 281 /// Scan directory for matching file names.
 282 ///
 283 /// Scan through all directory entries and call `callback()` function
 284 /// if the entry name matches the regular expression. This function
 285 /// used unix `openat()`, so you can pass absolute or relative file
 286 /// names. This function simply skips non-UTF8 encoded names.
 287 pub fn scandir<P, F>(
 288     dirfd: RawFd,
 289     path: &P,
 290     regex: &regex::Regex,
 291     mut callback: F,
 292 ) -> Result<(), Error>
 293 where
 294     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 295     P: ?Sized + nix::NixPath,
 296 {
 297     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 298         let entry = entry?;
 299         let file_type = match entry.file_type() {
 300             Some(file_type) => file_type,
 301             None => bail!("unable to detect file type"),
 302         };
 303
 304         callback(
 305             entry.parent_fd(),
 306             unsafe { entry.file_name_utf8_unchecked() },
 307             file_type,
 308         )?;
 309     }
 310     Ok(())
 311 }
 312
 313 /// Shortcut for md5 sums.
 314 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 315     hash(MessageDigest::md5(), data).map_err(Error::from)
 316 }
 317
 318 pub fn get_hardware_address() -> Result<String, Error> {
 319     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 320
 321     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 322     let digest = md5sum(&contents)?;
 323
 324     Ok(proxmox::tools::bin_to_hex(&digest))
 325 }
 326
 327 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 328     if digest1 != digest2 {
 329         bail!("detected modified configuration - file changed by other user? Try again.");
 330     }
 331     Ok(())
 332 }
 333
 334 /// Extract a specific cookie from cookie header.
 335 /// We assume cookie_name is already url encoded.
 336 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 337     for pair in cookie.split(';') {
 338         let (name, value) = match pair.find('=') {
 339             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 340             None => return None, // Cookie format error
 341         };
 342
 343         if name == cookie_name {
 344             use percent_encoding::percent_decode;
 345             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 346                 return Some(value.into());
 347             } else {
 348                 return None; // Cookie format error
 349             }
 350         }
 351     }
 352
 353     None
 354 }
 355
 356 pub fn join(data: &Vec<String>, sep: char) -> String {
 357     let mut list = String::new();
 358
 359     for item in data {
 360         if !list.is_empty() {
 361             list.push(sep);
 362         }
 363         list.push_str(item);
 364     }
 365
 366     list
 367 }
 368
 369 /// Detect modified configuration files
 370 ///
 371 /// This function fails with a reasonable error message if checksums do not match.
 372 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 373     if digest1 != digest2 {
 374         bail!("detected modified configuration - file changed by other user? Try again.");
 375     }
 376     Ok(())
 377 }
 378
 379 /// normalize uri path
 380 ///
 381 /// Do not allow ".", "..", or hidden files ".XXXX"
 382 /// Also remove empty path components
 383 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 384     let items = path.split('/');
 385
 386     let mut path = String::new();
 387     let mut components = vec![];
 388
 389     for name in items {
 390         if name.is_empty() {
 391             continue;
 392         }
 393         if name.starts_with('.') {
 394             bail!("Path contains illegal components.");
 395         }
 396         path.push('/');
 397         path.push_str(name);
 398         components.push(name);
 399     }
 400
 401     Ok((path, components))
 402 }
 403
 404 /// Helper to check result from std::process::Command output
 405 ///
 406 /// The exit_code_check() function should return true if the exit code
 407 /// is considered successful.
 408 pub fn command_output(
 409     output: std::process::Output,
 410     exit_code_check: Option<fn(i32) -> bool>,
 411 ) -> Result<Vec<u8>, Error> {
 412
 413     if !output.status.success() {
 414         match output.status.code() {
 415             Some(code) => {
 416                 let is_ok = match exit_code_check {
 417                     Some(check_fn) => check_fn(code),
 418                     None => code == 0,
 419                 };
 420                 if !is_ok {
 421                     let msg = String::from_utf8(output.stderr)
 422                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 423                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 424
 425                     bail!("status code: {} - {}", code, msg);
 426                 }
 427             }
 428             None => bail!("terminated by signal"),
 429         }
 430     }
 431
 432     Ok(output.stdout)
 433 }
 434
 435 /// Helper to check result from std::process::Command output, returns String.
 436 ///
 437 /// The exit_code_check() function should return true if the exit code
 438 /// is considered successful.
 439 pub fn command_output_as_string(
 440     output: std::process::Output,
 441     exit_code_check: Option<fn(i32) -> bool>,
 442 ) -> Result<String, Error> {
 443     let output = command_output(output, exit_code_check)?;
 444     let output = String::from_utf8(output)?;
 445     Ok(output)
 446 }
 447
 448 pub fn run_command(
 449     mut command: std::process::Command,
 450     exit_code_check: Option<fn(i32) -> bool>,
 451 ) -> Result<String, Error> {
 452
 453    let output = command.output()
 454         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 455
 456     let output = crate::tools::command_output_as_string(output, exit_code_check)
 457         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 458
 459     Ok(output)
 460 }
 461
 462 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 463     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 464     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 465         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 466     flags.set(FdFlag::FD_CLOEXEC, on);
 467     fcntl(fd, F_SETFD(flags))?;
 468     Ok(())
 469 }
 470
 471 static mut SHUTDOWN_REQUESTED: bool = false;
 472
 473 pub fn request_shutdown() {
 474     unsafe {
 475         SHUTDOWN_REQUESTED = true;
 476     }
 477     crate::server::server_shutdown();
 478 }
 479
 480 #[inline(always)]
 481 pub fn shutdown_requested() -> bool {
 482     unsafe { SHUTDOWN_REQUESTED }
 483 }
 484
 485 pub fn fail_on_shutdown() -> Result<(), Error> {
 486     if shutdown_requested() {
 487         bail!("Server shutdown requested - aborting task");
 488     }
 489     Ok(())
 490 }
 491
 492 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 493 /// descriptors.
 494 pub fn pipe() -> Result<(Fd, Fd), Error> {
 495     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 496     Ok((Fd(pin), Fd(pout)))
 497 }
 498
 499 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 500 /// descriptors.
 501 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 502     use nix::sys::socket;
 503     let (pa, pb) = socket::socketpair(
 504         socket::AddressFamily::Unix,
 505         socket::SockType::Stream,
 506         None,
 507         socket::SockFlag::SOCK_CLOEXEC,
 508     )?;
 509     Ok((Fd(pa), Fd(pb)))
 510 }
 511
 512
 513 /// An easy way to convert types to Any
 514 ///
 515 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 516 pub trait AsAny {
 517     fn as_any(&self) -> &dyn Any;
 518 }
 519
 520 impl<T: Any> AsAny for T {
 521     fn as_any(&self) -> &dyn Any {
 522         self
 523     }
 524 }
 525
 526 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 527 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 528     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 529     .add(0x20)
 530     .add(0x7f)
 531     // the DEFAULT_ENCODE_SET added:
 532     .add(b' ')
 533     .add(b'"')
 534     .add(b'#')
 535     .add(b'<')
 536     .add(b'>')
 537     .add(b'`')
 538     .add(b'?')
 539     .add(b'{')
 540     .add(b'}');
 541
 542 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 543 /// `#`).
 544 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 545     path: P,
 546 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 547     let path = path.as_ref();
 548
 549     Ok(io::BufReader::new(
 550         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 551     )
 552     .lines()
 553     .filter_map(|line| match line {
 554         Ok(line) => {
 555             let line = line.trim();
 556             if line.is_empty() || line.starts_with('#') {
 557                 None
 558             } else {
 559                 Some(Ok(line.to_string()))
 560             }
 561         }
 562         Err(err) => Some(Err(err)),
 563     }))
 564 }
 565
 566 pub fn setup_safe_path_env() {
 567     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 568     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 569     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 570         std::env::remove_var(name);
 571     }
 572 }
 573
 574 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 575     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 576         Some(n) => &line[n..],
 577         None => return &[],
 578     };
 579     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 580         Some(n) => &line[..(line.len() - n)],
 581         None => &[],
 582     }
 583 }
 584
 585 /// Seeks to start of file and computes the SHA256 hash
 586 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 587
 588     file.seek(SeekFrom::Start(0))?;
 589
 590     let mut hasher = openssl::sha::Sha256::new();
 591     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 592     let mut size: u64 = 0;
 593
 594     loop {
 595         let count = match file.read(&mut buffer) {
 596             Ok(count) => count,
 597             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 598                 continue;
 599             }
 600             Err(err) => return Err(err.into()),
 601         };
 602         if count == 0 {
 603             break;
 604         }
 605         size += count as u64;
 606         hasher.update(&buffer[..count]);
 607     }
 608
 609     let csum = hasher.finish();
 610
 611     Ok((csum, size))
 612 }