src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::AsciiSet;
  16
  17 use proxmox::tools::vec;
  18
  19 pub use proxmox::tools::fd::Fd;
  20
  21 pub mod acl;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod cert;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod statistics;
  33 pub mod systemd;
  34 pub mod nom;
  35
  36 mod wrapped_reader_stream;
  37 pub use wrapped_reader_stream::*;
  38
  39 mod std_channel_writer;
  40 pub use std_channel_writer::*;
  41
  42 pub mod xattr;
  43
  44 mod process_locker;
  45 pub use process_locker::*;
  46
  47 mod file_logger;
  48 pub use file_logger::*;
  49
  50 mod broadcast_future;
  51 pub use broadcast_future::*;
  52
  53 /// The `BufferedRead` trait provides a single function
  54 /// `buffered_read`. It returns a reference to an internal buffer. The
  55 /// purpose of this traid is to avoid unnecessary data copies.
  56 pub trait BufferedRead {
  57     /// This functions tries to fill the internal buffers, then
  58     /// returns a reference to the available data. It returns an empty
  59     /// buffer if `offset` points to the end of the file.
  60     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  61 }
  62
  63 /// Split a file into equal sized chunks. The last chunk may be
  64 /// smaller. Note: We cannot implement an `Iterator`, because iterators
  65 /// cannot return a borrowed buffer ref (we want zero-copy)
  66 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
  67 where
  68     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
  69     R: Read,
  70 {
  71     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
  72
  73     if chunk_size > READ_BUFFER_SIZE {
  74         bail!("chunk size too large!");
  75     }
  76
  77     let mut buf = vec::undefined(READ_BUFFER_SIZE);
  78
  79     let mut pos = 0;
  80     let mut file_pos = 0;
  81     loop {
  82         let mut eof = false;
  83         let mut tmp = &mut buf[..];
  84         // try to read large portions, at least chunk_size
  85         while pos < chunk_size {
  86             match file.read(tmp) {
  87                 Ok(0) => {
  88                     eof = true;
  89                     break;
  90                 }
  91                 Ok(n) => {
  92                     pos += n;
  93                     if pos > chunk_size {
  94                         break;
  95                     }
  96                     tmp = &mut tmp[n..];
  97                 }
  98                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
  99                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 100             }
 101         }
 102         let mut start = 0;
 103         while start + chunk_size <= pos {
 104             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 105                 break;
 106             }
 107             file_pos += chunk_size;
 108             start += chunk_size;
 109         }
 110         if eof {
 111             if start < pos {
 112                 (chunk_cb)(file_pos, &buf[start..pos])?;
 113                 //file_pos += pos - start;
 114             }
 115             break;
 116         } else {
 117             let rest = pos - start;
 118             if rest > 0 {
 119                 let ptr = buf.as_mut_ptr();
 120                 unsafe {
 121                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 122                 }
 123                 pos = rest;
 124             } else {
 125                 pos = 0;
 126             }
 127         }
 128     }
 129
 130     Ok(())
 131 }
 132
 133 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 134     let mut query = url::form_urlencoded::Serializer::new(String::new());
 135
 136     let object = data.as_object().ok_or_else(|| {
 137         format_err!("json_object_to_query: got wrong data type (expected object).")
 138     })?;
 139
 140     for (key, value) in object {
 141         match value {
 142             Value::Bool(b) => {
 143                 query.append_pair(key, &b.to_string());
 144             }
 145             Value::Number(n) => {
 146                 query.append_pair(key, &n.to_string());
 147             }
 148             Value::String(s) => {
 149                 query.append_pair(key, &s);
 150             }
 151             Value::Array(arr) => {
 152                 for element in arr {
 153                     match element {
 154                         Value::Bool(b) => {
 155                             query.append_pair(key, &b.to_string());
 156                         }
 157                         Value::Number(n) => {
 158                             query.append_pair(key, &n.to_string());
 159                         }
 160                         Value::String(s) => {
 161                             query.append_pair(key, &s);
 162                         }
 163                         _ => bail!(
 164                             "json_object_to_query: unable to handle complex array data types."
 165                         ),
 166                     }
 167                 }
 168             }
 169             _ => bail!("json_object_to_query: unable to handle complex data types."),
 170         }
 171     }
 172
 173     Ok(query.finish())
 174 }
 175
 176 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 177     match param[name].as_str() {
 178         Some(s) => Ok(s),
 179         None => bail!("missing parameter '{}'", name),
 180     }
 181 }
 182
 183 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 184     match param[name].as_str() {
 185         Some(s) => Ok(s),
 186         None => bail!("missing property '{}'", name),
 187     }
 188 }
 189
 190 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 191     match param[name].as_i64() {
 192         Some(s) => Ok(s),
 193         None => bail!("missing parameter '{}'", name),
 194     }
 195 }
 196
 197 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 198     match param[name].as_i64() {
 199         Some(s) => Ok(s),
 200         None => bail!("missing property '{}'", name),
 201     }
 202 }
 203
 204 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 205     match param[name].as_array() {
 206         Some(s) => Ok(s.to_vec()),
 207         None => bail!("missing parameter '{}'", name),
 208     }
 209 }
 210
 211 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 212     match param[name].as_array() {
 213         Some(s) => Ok(s.to_vec()),
 214         None => bail!("missing property '{}'", name),
 215     }
 216 }
 217
 218 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 219     let mut result = vec![];
 220
 221     use nix::fcntl::AtFlags;
 222     use nix::fcntl::OFlag;
 223     use nix::sys::stat::Mode;
 224
 225     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 226
 227     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 228         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 229         Err(_) => false,
 230     };
 231
 232     if !is_dir {
 233         if let Some(parent) = dirname.parent() {
 234             dirname = parent.to_owned();
 235         }
 236     }
 237
 238     let mut dir =
 239         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 240             Ok(d) => d,
 241             Err(_) => return result,
 242         };
 243
 244     for item in dir.iter() {
 245         if let Ok(entry) = item {
 246             if let Ok(name) = entry.file_name().to_str() {
 247                 if name == "." || name == ".." {
 248                     continue;
 249                 }
 250                 let mut newpath = dirname.clone();
 251                 newpath.push(name);
 252
 253                 if let Ok(stat) =
 254                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 255                 {
 256                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 257                         newpath.push("");
 258                         if let Some(newpath) = newpath.to_str() {
 259                             result.push(newpath.to_owned());
 260                         }
 261                         continue;
 262                     }
 263                 }
 264                 if let Some(newpath) = newpath.to_str() {
 265                     result.push(newpath.to_owned());
 266                 }
 267             }
 268         }
 269     }
 270
 271     result
 272 }
 273
 274 /// Scan directory for matching file names.
 275 ///
 276 /// Scan through all directory entries and call `callback()` function
 277 /// if the entry name matches the regular expression. This function
 278 /// used unix `openat()`, so you can pass absolute or relative file
 279 /// names. This function simply skips non-UTF8 encoded names.
 280 pub fn scandir<P, F>(
 281     dirfd: RawFd,
 282     path: &P,
 283     regex: &regex::Regex,
 284     mut callback: F,
 285 ) -> Result<(), Error>
 286 where
 287     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 288     P: ?Sized + nix::NixPath,
 289 {
 290     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 291         let entry = entry?;
 292         let file_type = match entry.file_type() {
 293             Some(file_type) => file_type,
 294             None => bail!("unable to detect file type"),
 295         };
 296
 297         callback(
 298             entry.parent_fd(),
 299             unsafe { entry.file_name_utf8_unchecked() },
 300             file_type,
 301         )?;
 302     }
 303     Ok(())
 304 }
 305
 306 /// Shortcut for md5 sums.
 307 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 308     hash(MessageDigest::md5(), data).map_err(Error::from)
 309 }
 310
 311 pub fn get_hardware_address() -> Result<String, Error> {
 312     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 313
 314     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 315     let digest = md5sum(&contents)?;
 316
 317     Ok(proxmox::tools::bin_to_hex(&digest))
 318 }
 319
 320 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 321     if digest1 != digest2 {
 322         bail!("detected modified configuration - file changed by other user? Try again.");
 323     }
 324     Ok(())
 325 }
 326
 327 /// Extract a specific cookie from cookie header.
 328 /// We assume cookie_name is already url encoded.
 329 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 330     for pair in cookie.split(';') {
 331         let (name, value) = match pair.find('=') {
 332             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 333             None => return None, // Cookie format error
 334         };
 335
 336         if name == cookie_name {
 337             use percent_encoding::percent_decode;
 338             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 339                 return Some(value.into());
 340             } else {
 341                 return None; // Cookie format error
 342             }
 343         }
 344     }
 345
 346     None
 347 }
 348
 349 pub fn join(data: &Vec<String>, sep: char) -> String {
 350     let mut list = String::new();
 351
 352     for item in data {
 353         if !list.is_empty() {
 354             list.push(sep);
 355         }
 356         list.push_str(item);
 357     }
 358
 359     list
 360 }
 361
 362 /// Detect modified configuration files
 363 ///
 364 /// This function fails with a reasonable error message if checksums do not match.
 365 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 366     if digest1 != digest2 {
 367         bail!("detected modified configuration - file changed by other user? Try again.");
 368     }
 369     Ok(())
 370 }
 371
 372 /// normalize uri path
 373 ///
 374 /// Do not allow ".", "..", or hidden files ".XXXX"
 375 /// Also remove empty path components
 376 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 377     let items = path.split('/');
 378
 379     let mut path = String::new();
 380     let mut components = vec![];
 381
 382     for name in items {
 383         if name.is_empty() {
 384             continue;
 385         }
 386         if name.starts_with('.') {
 387             bail!("Path contains illegal components.");
 388         }
 389         path.push('/');
 390         path.push_str(name);
 391         components.push(name);
 392     }
 393
 394     Ok((path, components))
 395 }
 396
 397 /// Helper to check result from std::process::Command output
 398 ///
 399 /// The exit_code_check() function should return true if the exit code
 400 /// is considered successful.
 401 pub fn command_output(
 402     output: std::process::Output,
 403     exit_code_check: Option<fn(i32) -> bool>,
 404 ) -> Result<String, Error> {
 405
 406     if !output.status.success() {
 407         match output.status.code() {
 408             Some(code) => {
 409                 let is_ok = match exit_code_check {
 410                     Some(check_fn) => check_fn(code),
 411                     None => code == 0,
 412                 };
 413                 if !is_ok {
 414                     let msg = String::from_utf8(output.stderr)
 415                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 416                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 417
 418                     bail!("status code: {} - {}", code, msg);
 419                 }
 420             }
 421             None => bail!("terminated by signal"),
 422         }
 423     }
 424
 425     let output = String::from_utf8(output.stdout)?;
 426
 427     Ok(output)
 428 }
 429
 430 pub fn run_command(
 431     mut command: std::process::Command,
 432     exit_code_check: Option<fn(i32) -> bool>,
 433 ) -> Result<String, Error> {
 434
 435    let output = command.output()
 436         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 437
 438     let output = crate::tools::command_output(output, exit_code_check)
 439         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 440
 441     Ok(output)
 442 }
 443
 444 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 445     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 446     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 447         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 448     flags.set(FdFlag::FD_CLOEXEC, on);
 449     fcntl(fd, F_SETFD(flags))?;
 450     Ok(())
 451 }
 452
 453 static mut SHUTDOWN_REQUESTED: bool = false;
 454
 455 pub fn request_shutdown() {
 456     unsafe {
 457         SHUTDOWN_REQUESTED = true;
 458     }
 459     crate::server::server_shutdown();
 460 }
 461
 462 #[inline(always)]
 463 pub fn shutdown_requested() -> bool {
 464     unsafe { SHUTDOWN_REQUESTED }
 465 }
 466
 467 pub fn fail_on_shutdown() -> Result<(), Error> {
 468     if shutdown_requested() {
 469         bail!("Server shutdown requested - aborting task");
 470     }
 471     Ok(())
 472 }
 473
 474 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 475 /// descriptors.
 476 pub fn pipe() -> Result<(Fd, Fd), Error> {
 477     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 478     Ok((Fd(pin), Fd(pout)))
 479 }
 480
 481 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 482 /// descriptors.
 483 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 484     use nix::sys::socket;
 485     let (pa, pb) = socket::socketpair(
 486         socket::AddressFamily::Unix,
 487         socket::SockType::Stream,
 488         None,
 489         socket::SockFlag::SOCK_CLOEXEC,
 490     )?;
 491     Ok((Fd(pa), Fd(pb)))
 492 }
 493
 494
 495 /// An easy way to convert types to Any
 496 ///
 497 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 498 pub trait AsAny {
 499     fn as_any(&self) -> &dyn Any;
 500 }
 501
 502 impl<T: Any> AsAny for T {
 503     fn as_any(&self) -> &dyn Any {
 504         self
 505     }
 506 }
 507
 508 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 509 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 510     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 511     .add(0x20)
 512     .add(0x7f)
 513     // the DEFAULT_ENCODE_SET added:
 514     .add(b' ')
 515     .add(b'"')
 516     .add(b'#')
 517     .add(b'<')
 518     .add(b'>')
 519     .add(b'`')
 520     .add(b'?')
 521     .add(b'{')
 522     .add(b'}');
 523
 524 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 525 /// `#`).
 526 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 527     path: P,
 528 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 529     let path = path.as_ref();
 530
 531     Ok(io::BufReader::new(
 532         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 533     )
 534     .lines()
 535     .filter_map(|line| match line {
 536         Ok(line) => {
 537             let line = line.trim();
 538             if line.is_empty() || line.starts_with('#') {
 539                 None
 540             } else {
 541                 Some(Ok(line.to_string()))
 542             }
 543         }
 544         Err(err) => Some(Err(err)),
 545     }))
 546 }
 547
 548 pub fn setup_safe_path_env() {
 549     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 550     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 551     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 552         std::env::remove_var(name);
 553     }
 554 }
 555
 556 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 557     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 558         Some(n) => &line[n..],
 559         None => return &[],
 560     };
 561     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 562         Some(n) => &line[..(line.len() - n)],
 563         None => &[],
 564     }
 565 }
 566
 567 /// Seeks to start of file and computes the SHA256 hash
 568 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 569
 570     file.seek(SeekFrom::Start(0))?;
 571
 572     let mut hasher = openssl::sha::Sha256::new();
 573     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 574     let mut size: u64 = 0;
 575
 576     loop {
 577         let count = match file.read(&mut buffer) {
 578             Ok(count) => count,
 579             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 580                 continue;
 581             }
 582             Err(err) => return Err(err.into()),
 583         };
 584         if count == 0 {
 585             break;
 586         }
 587         size += count as u64;
 588         hasher.update(&buffer[..count]);
 589     }
 590
 591     let csum = hasher.finish();
 592
 593     Ok((csum, size))
 594 }