src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::AsciiSet;
  16
  17 use proxmox::tools::vec;
  18
  19 pub use proxmox::tools::fd::Fd;
  20
  21 pub mod acl;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod cert;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod statistics;
  33 pub mod systemd;
  34 pub mod nom;
  35 pub mod logrotate;
  36 pub mod loopdev;
  37 pub mod fuse_loop;
  38 pub mod socket;
  39 pub mod subscription;
  40 pub mod zip;
  41 pub mod http;
  42
  43 mod parallel_handler;
  44 pub use parallel_handler::*;
  45
  46 mod wrapped_reader_stream;
  47 pub use wrapped_reader_stream::*;
  48
  49 mod async_channel_writer;
  50 pub use async_channel_writer::*;
  51
  52
  53 mod std_channel_writer;
  54 pub use std_channel_writer::*;
  55
  56 pub mod xattr;
  57
  58 mod process_locker;
  59 pub use process_locker::*;
  60
  61 mod file_logger;
  62 pub use file_logger::*;
  63
  64 mod broadcast_future;
  65 pub use broadcast_future::*;
  66
  67 /// The `BufferedRead` trait provides a single function
  68 /// `buffered_read`. It returns a reference to an internal buffer. The
  69 /// purpose of this traid is to avoid unnecessary data copies.
  70 pub trait BufferedRead {
  71     /// This functions tries to fill the internal buffers, then
  72     /// returns a reference to the available data. It returns an empty
  73     /// buffer if `offset` points to the end of the file.
  74     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  75 }
  76
  77 /// Split a file into equal sized chunks. The last chunk may be
  78 /// smaller. Note: We cannot implement an `Iterator`, because iterators
  79 /// cannot return a borrowed buffer ref (we want zero-copy)
  80 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
  81 where
  82     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
  83     R: Read,
  84 {
  85     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
  86
  87     if chunk_size > READ_BUFFER_SIZE {
  88         bail!("chunk size too large!");
  89     }
  90
  91     let mut buf = vec::undefined(READ_BUFFER_SIZE);
  92
  93     let mut pos = 0;
  94     let mut file_pos = 0;
  95     loop {
  96         let mut eof = false;
  97         let mut tmp = &mut buf[..];
  98         // try to read large portions, at least chunk_size
  99         while pos < chunk_size {
 100             match file.read(tmp) {
 101                 Ok(0) => {
 102                     eof = true;
 103                     break;
 104                 }
 105                 Ok(n) => {
 106                     pos += n;
 107                     if pos > chunk_size {
 108                         break;
 109                     }
 110                     tmp = &mut tmp[n..];
 111                 }
 112                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 113                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 114             }
 115         }
 116         let mut start = 0;
 117         while start + chunk_size <= pos {
 118             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 119                 break;
 120             }
 121             file_pos += chunk_size;
 122             start += chunk_size;
 123         }
 124         if eof {
 125             if start < pos {
 126                 (chunk_cb)(file_pos, &buf[start..pos])?;
 127                 //file_pos += pos - start;
 128             }
 129             break;
 130         } else {
 131             let rest = pos - start;
 132             if rest > 0 {
 133                 let ptr = buf.as_mut_ptr();
 134                 unsafe {
 135                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 136                 }
 137                 pos = rest;
 138             } else {
 139                 pos = 0;
 140             }
 141         }
 142     }
 143
 144     Ok(())
 145 }
 146
 147 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 148     let mut query = url::form_urlencoded::Serializer::new(String::new());
 149
 150     let object = data.as_object().ok_or_else(|| {
 151         format_err!("json_object_to_query: got wrong data type (expected object).")
 152     })?;
 153
 154     for (key, value) in object {
 155         match value {
 156             Value::Bool(b) => {
 157                 query.append_pair(key, &b.to_string());
 158             }
 159             Value::Number(n) => {
 160                 query.append_pair(key, &n.to_string());
 161             }
 162             Value::String(s) => {
 163                 query.append_pair(key, &s);
 164             }
 165             Value::Array(arr) => {
 166                 for element in arr {
 167                     match element {
 168                         Value::Bool(b) => {
 169                             query.append_pair(key, &b.to_string());
 170                         }
 171                         Value::Number(n) => {
 172                             query.append_pair(key, &n.to_string());
 173                         }
 174                         Value::String(s) => {
 175                             query.append_pair(key, &s);
 176                         }
 177                         _ => bail!(
 178                             "json_object_to_query: unable to handle complex array data types."
 179                         ),
 180                     }
 181                 }
 182             }
 183             _ => bail!("json_object_to_query: unable to handle complex data types."),
 184         }
 185     }
 186
 187     Ok(query.finish())
 188 }
 189
 190 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 191     match param[name].as_str() {
 192         Some(s) => Ok(s),
 193         None => bail!("missing parameter '{}'", name),
 194     }
 195 }
 196
 197 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 198     match param[name].as_str() {
 199         Some(s) => Ok(s),
 200         None => bail!("missing property '{}'", name),
 201     }
 202 }
 203
 204 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 205     match param[name].as_i64() {
 206         Some(s) => Ok(s),
 207         None => bail!("missing parameter '{}'", name),
 208     }
 209 }
 210
 211 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 212     match param[name].as_i64() {
 213         Some(s) => Ok(s),
 214         None => bail!("missing property '{}'", name),
 215     }
 216 }
 217
 218 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 219     match param[name].as_array() {
 220         Some(s) => Ok(s.to_vec()),
 221         None => bail!("missing parameter '{}'", name),
 222     }
 223 }
 224
 225 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 226     match param[name].as_array() {
 227         Some(s) => Ok(s.to_vec()),
 228         None => bail!("missing property '{}'", name),
 229     }
 230 }
 231
 232 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 233     let mut result = vec![];
 234
 235     use nix::fcntl::AtFlags;
 236     use nix::fcntl::OFlag;
 237     use nix::sys::stat::Mode;
 238
 239     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 240
 241     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 242         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 243         Err(_) => false,
 244     };
 245
 246     if !is_dir {
 247         if let Some(parent) = dirname.parent() {
 248             dirname = parent.to_owned();
 249         }
 250     }
 251
 252     let mut dir =
 253         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 254             Ok(d) => d,
 255             Err(_) => return result,
 256         };
 257
 258     for item in dir.iter() {
 259         if let Ok(entry) = item {
 260             if let Ok(name) = entry.file_name().to_str() {
 261                 if name == "." || name == ".." {
 262                     continue;
 263                 }
 264                 let mut newpath = dirname.clone();
 265                 newpath.push(name);
 266
 267                 if let Ok(stat) =
 268                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 269                 {
 270                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 271                         newpath.push("");
 272                         if let Some(newpath) = newpath.to_str() {
 273                             result.push(newpath.to_owned());
 274                         }
 275                         continue;
 276                     }
 277                 }
 278                 if let Some(newpath) = newpath.to_str() {
 279                     result.push(newpath.to_owned());
 280                 }
 281             }
 282         }
 283     }
 284
 285     result
 286 }
 287
 288 /// Scan directory for matching file names.
 289 ///
 290 /// Scan through all directory entries and call `callback()` function
 291 /// if the entry name matches the regular expression. This function
 292 /// used unix `openat()`, so you can pass absolute or relative file
 293 /// names. This function simply skips non-UTF8 encoded names.
 294 pub fn scandir<P, F>(
 295     dirfd: RawFd,
 296     path: &P,
 297     regex: &regex::Regex,
 298     mut callback: F,
 299 ) -> Result<(), Error>
 300 where
 301     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 302     P: ?Sized + nix::NixPath,
 303 {
 304     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 305         let entry = entry?;
 306         let file_type = match entry.file_type() {
 307             Some(file_type) => file_type,
 308             None => bail!("unable to detect file type"),
 309         };
 310
 311         callback(
 312             entry.parent_fd(),
 313             unsafe { entry.file_name_utf8_unchecked() },
 314             file_type,
 315         )?;
 316     }
 317     Ok(())
 318 }
 319
 320 /// Shortcut for md5 sums.
 321 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 322     hash(MessageDigest::md5(), data).map_err(Error::from)
 323 }
 324
 325 pub fn get_hardware_address() -> Result<String, Error> {
 326     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 327
 328     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 329         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 330     let digest = md5sum(&contents)
 331         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 332
 333     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 334 }
 335
 336 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 337     if digest1 != digest2 {
 338         bail!("detected modified configuration - file changed by other user? Try again.");
 339     }
 340     Ok(())
 341 }
 342
 343 /// Extract a specific cookie from cookie header.
 344 /// We assume cookie_name is already url encoded.
 345 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 346     for pair in cookie.split(';') {
 347         let (name, value) = match pair.find('=') {
 348             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 349             None => return None, // Cookie format error
 350         };
 351
 352         if name == cookie_name {
 353             use percent_encoding::percent_decode;
 354             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 355                 return Some(value.into());
 356             } else {
 357                 return None; // Cookie format error
 358             }
 359         }
 360     }
 361
 362     None
 363 }
 364
 365 pub fn join(data: &Vec<String>, sep: char) -> String {
 366     let mut list = String::new();
 367
 368     for item in data {
 369         if !list.is_empty() {
 370             list.push(sep);
 371         }
 372         list.push_str(item);
 373     }
 374
 375     list
 376 }
 377
 378 /// Detect modified configuration files
 379 ///
 380 /// This function fails with a reasonable error message if checksums do not match.
 381 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 382     if digest1 != digest2 {
 383         bail!("detected modified configuration - file changed by other user? Try again.");
 384     }
 385     Ok(())
 386 }
 387
 388 /// normalize uri path
 389 ///
 390 /// Do not allow ".", "..", or hidden files ".XXXX"
 391 /// Also remove empty path components
 392 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 393     let items = path.split('/');
 394
 395     let mut path = String::new();
 396     let mut components = vec![];
 397
 398     for name in items {
 399         if name.is_empty() {
 400             continue;
 401         }
 402         if name.starts_with('.') {
 403             bail!("Path contains illegal components.");
 404         }
 405         path.push('/');
 406         path.push_str(name);
 407         components.push(name);
 408     }
 409
 410     Ok((path, components))
 411 }
 412
 413 /// Helper to check result from std::process::Command output
 414 ///
 415 /// The exit_code_check() function should return true if the exit code
 416 /// is considered successful.
 417 pub fn command_output(
 418     output: std::process::Output,
 419     exit_code_check: Option<fn(i32) -> bool>,
 420 ) -> Result<Vec<u8>, Error> {
 421
 422     if !output.status.success() {
 423         match output.status.code() {
 424             Some(code) => {
 425                 let is_ok = match exit_code_check {
 426                     Some(check_fn) => check_fn(code),
 427                     None => code == 0,
 428                 };
 429                 if !is_ok {
 430                     let msg = String::from_utf8(output.stderr)
 431                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 432                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 433
 434                     bail!("status code: {} - {}", code, msg);
 435                 }
 436             }
 437             None => bail!("terminated by signal"),
 438         }
 439     }
 440
 441     Ok(output.stdout)
 442 }
 443
 444 /// Helper to check result from std::process::Command output, returns String.
 445 ///
 446 /// The exit_code_check() function should return true if the exit code
 447 /// is considered successful.
 448 pub fn command_output_as_string(
 449     output: std::process::Output,
 450     exit_code_check: Option<fn(i32) -> bool>,
 451 ) -> Result<String, Error> {
 452     let output = command_output(output, exit_code_check)?;
 453     let output = String::from_utf8(output)?;
 454     Ok(output)
 455 }
 456
 457 pub fn run_command(
 458     mut command: std::process::Command,
 459     exit_code_check: Option<fn(i32) -> bool>,
 460 ) -> Result<String, Error> {
 461
 462    let output = command.output()
 463         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 464
 465     let output = crate::tools::command_output_as_string(output, exit_code_check)
 466         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 467
 468     Ok(output)
 469 }
 470
 471 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 472     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 473     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 474         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 475     flags.set(FdFlag::FD_CLOEXEC, on);
 476     fcntl(fd, F_SETFD(flags))?;
 477     Ok(())
 478 }
 479
 480 static mut SHUTDOWN_REQUESTED: bool = false;
 481
 482 pub fn request_shutdown() {
 483     unsafe {
 484         SHUTDOWN_REQUESTED = true;
 485     }
 486     crate::server::server_shutdown();
 487 }
 488
 489 #[inline(always)]
 490 pub fn shutdown_requested() -> bool {
 491     unsafe { SHUTDOWN_REQUESTED }
 492 }
 493
 494 pub fn fail_on_shutdown() -> Result<(), Error> {
 495     if shutdown_requested() {
 496         bail!("Server shutdown requested - aborting task");
 497     }
 498     Ok(())
 499 }
 500
 501 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 502 /// descriptors.
 503 pub fn pipe() -> Result<(Fd, Fd), Error> {
 504     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 505     Ok((Fd(pin), Fd(pout)))
 506 }
 507
 508 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 509 /// descriptors.
 510 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 511     use nix::sys::socket;
 512     let (pa, pb) = socket::socketpair(
 513         socket::AddressFamily::Unix,
 514         socket::SockType::Stream,
 515         None,
 516         socket::SockFlag::SOCK_CLOEXEC,
 517     )?;
 518     Ok((Fd(pa), Fd(pb)))
 519 }
 520
 521
 522 /// An easy way to convert types to Any
 523 ///
 524 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 525 pub trait AsAny {
 526     fn as_any(&self) -> &dyn Any;
 527 }
 528
 529 impl<T: Any> AsAny for T {
 530     fn as_any(&self) -> &dyn Any {
 531         self
 532     }
 533 }
 534
 535 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 536 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 537     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 538     .add(0x20)
 539     .add(0x7f)
 540     // the DEFAULT_ENCODE_SET added:
 541     .add(b' ')
 542     .add(b'"')
 543     .add(b'#')
 544     .add(b'<')
 545     .add(b'>')
 546     .add(b'`')
 547     .add(b'?')
 548     .add(b'{')
 549     .add(b'}');
 550
 551 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 552 /// `#`).
 553 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 554     path: P,
 555 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 556     let path = path.as_ref();
 557
 558     Ok(io::BufReader::new(
 559         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 560     )
 561     .lines()
 562     .filter_map(|line| match line {
 563         Ok(line) => {
 564             let line = line.trim();
 565             if line.is_empty() || line.starts_with('#') {
 566                 None
 567             } else {
 568                 Some(Ok(line.to_string()))
 569             }
 570         }
 571         Err(err) => Some(Err(err)),
 572     }))
 573 }
 574
 575 pub fn setup_safe_path_env() {
 576     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 577     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 578     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 579         std::env::remove_var(name);
 580     }
 581 }
 582
 583 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 584     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 585         Some(n) => &line[n..],
 586         None => return &[],
 587     };
 588     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 589         Some(n) => &line[..(line.len() - n)],
 590         None => &[],
 591     }
 592 }
 593
 594 /// Seeks to start of file and computes the SHA256 hash
 595 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 596
 597     file.seek(SeekFrom::Start(0))?;
 598
 599     let mut hasher = openssl::sha::Sha256::new();
 600     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 601     let mut size: u64 = 0;
 602
 603     loop {
 604         let count = match file.read(&mut buffer) {
 605             Ok(count) => count,
 606             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 607                 continue;
 608             }
 609             Err(err) => return Err(err.into()),
 610         };
 611         if count == 0 {
 612             break;
 613         }
 614         size += count as u64;
 615         hasher.update(&buffer[..count]);
 616     }
 617
 618     let csum = hasher.finish();
 619
 620     Ok((csum, size))
 621 }