src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::AsciiSet;
  16
  17 use proxmox::tools::vec;
  18
  19 pub use proxmox::tools::fd::Fd;
  20
  21 pub mod acl;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod cert;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod statistics;
  33 pub mod systemd;
  34 pub mod nom;
  35 pub mod logrotate;
  36 pub mod loopdev;
  37 pub mod fuse_loop;
  38 pub mod socket;
  39 pub mod zip;
  40
  41 mod parallel_handler;
  42 pub use parallel_handler::*;
  43
  44 mod wrapped_reader_stream;
  45 pub use wrapped_reader_stream::*;
  46
  47 mod async_channel_writer;
  48 pub use async_channel_writer::*;
  49
  50
  51 mod std_channel_writer;
  52 pub use std_channel_writer::*;
  53
  54 pub mod xattr;
  55
  56 mod process_locker;
  57 pub use process_locker::*;
  58
  59 mod file_logger;
  60 pub use file_logger::*;
  61
  62 mod broadcast_future;
  63 pub use broadcast_future::*;
  64
  65 /// The `BufferedRead` trait provides a single function
  66 /// `buffered_read`. It returns a reference to an internal buffer. The
  67 /// purpose of this traid is to avoid unnecessary data copies.
  68 pub trait BufferedRead {
  69     /// This functions tries to fill the internal buffers, then
  70     /// returns a reference to the available data. It returns an empty
  71     /// buffer if `offset` points to the end of the file.
  72     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  73 }
  74
  75 /// Split a file into equal sized chunks. The last chunk may be
  76 /// smaller. Note: We cannot implement an `Iterator`, because iterators
  77 /// cannot return a borrowed buffer ref (we want zero-copy)
  78 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
  79 where
  80     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
  81     R: Read,
  82 {
  83     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
  84
  85     if chunk_size > READ_BUFFER_SIZE {
  86         bail!("chunk size too large!");
  87     }
  88
  89     let mut buf = vec::undefined(READ_BUFFER_SIZE);
  90
  91     let mut pos = 0;
  92     let mut file_pos = 0;
  93     loop {
  94         let mut eof = false;
  95         let mut tmp = &mut buf[..];
  96         // try to read large portions, at least chunk_size
  97         while pos < chunk_size {
  98             match file.read(tmp) {
  99                 Ok(0) => {
 100                     eof = true;
 101                     break;
 102                 }
 103                 Ok(n) => {
 104                     pos += n;
 105                     if pos > chunk_size {
 106                         break;
 107                     }
 108                     tmp = &mut tmp[n..];
 109                 }
 110                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 111                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 112             }
 113         }
 114         let mut start = 0;
 115         while start + chunk_size <= pos {
 116             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 117                 break;
 118             }
 119             file_pos += chunk_size;
 120             start += chunk_size;
 121         }
 122         if eof {
 123             if start < pos {
 124                 (chunk_cb)(file_pos, &buf[start..pos])?;
 125                 //file_pos += pos - start;
 126             }
 127             break;
 128         } else {
 129             let rest = pos - start;
 130             if rest > 0 {
 131                 let ptr = buf.as_mut_ptr();
 132                 unsafe {
 133                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 134                 }
 135                 pos = rest;
 136             } else {
 137                 pos = 0;
 138             }
 139         }
 140     }
 141
 142     Ok(())
 143 }
 144
 145 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 146     let mut query = url::form_urlencoded::Serializer::new(String::new());
 147
 148     let object = data.as_object().ok_or_else(|| {
 149         format_err!("json_object_to_query: got wrong data type (expected object).")
 150     })?;
 151
 152     for (key, value) in object {
 153         match value {
 154             Value::Bool(b) => {
 155                 query.append_pair(key, &b.to_string());
 156             }
 157             Value::Number(n) => {
 158                 query.append_pair(key, &n.to_string());
 159             }
 160             Value::String(s) => {
 161                 query.append_pair(key, &s);
 162             }
 163             Value::Array(arr) => {
 164                 for element in arr {
 165                     match element {
 166                         Value::Bool(b) => {
 167                             query.append_pair(key, &b.to_string());
 168                         }
 169                         Value::Number(n) => {
 170                             query.append_pair(key, &n.to_string());
 171                         }
 172                         Value::String(s) => {
 173                             query.append_pair(key, &s);
 174                         }
 175                         _ => bail!(
 176                             "json_object_to_query: unable to handle complex array data types."
 177                         ),
 178                     }
 179                 }
 180             }
 181             _ => bail!("json_object_to_query: unable to handle complex data types."),
 182         }
 183     }
 184
 185     Ok(query.finish())
 186 }
 187
 188 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 189     match param[name].as_str() {
 190         Some(s) => Ok(s),
 191         None => bail!("missing parameter '{}'", name),
 192     }
 193 }
 194
 195 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 196     match param[name].as_str() {
 197         Some(s) => Ok(s),
 198         None => bail!("missing property '{}'", name),
 199     }
 200 }
 201
 202 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 203     match param[name].as_i64() {
 204         Some(s) => Ok(s),
 205         None => bail!("missing parameter '{}'", name),
 206     }
 207 }
 208
 209 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 210     match param[name].as_i64() {
 211         Some(s) => Ok(s),
 212         None => bail!("missing property '{}'", name),
 213     }
 214 }
 215
 216 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 217     match param[name].as_array() {
 218         Some(s) => Ok(s.to_vec()),
 219         None => bail!("missing parameter '{}'", name),
 220     }
 221 }
 222
 223 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 224     match param[name].as_array() {
 225         Some(s) => Ok(s.to_vec()),
 226         None => bail!("missing property '{}'", name),
 227     }
 228 }
 229
 230 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 231     let mut result = vec![];
 232
 233     use nix::fcntl::AtFlags;
 234     use nix::fcntl::OFlag;
 235     use nix::sys::stat::Mode;
 236
 237     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 238
 239     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 240         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 241         Err(_) => false,
 242     };
 243
 244     if !is_dir {
 245         if let Some(parent) = dirname.parent() {
 246             dirname = parent.to_owned();
 247         }
 248     }
 249
 250     let mut dir =
 251         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 252             Ok(d) => d,
 253             Err(_) => return result,
 254         };
 255
 256     for item in dir.iter() {
 257         if let Ok(entry) = item {
 258             if let Ok(name) = entry.file_name().to_str() {
 259                 if name == "." || name == ".." {
 260                     continue;
 261                 }
 262                 let mut newpath = dirname.clone();
 263                 newpath.push(name);
 264
 265                 if let Ok(stat) =
 266                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 267                 {
 268                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 269                         newpath.push("");
 270                         if let Some(newpath) = newpath.to_str() {
 271                             result.push(newpath.to_owned());
 272                         }
 273                         continue;
 274                     }
 275                 }
 276                 if let Some(newpath) = newpath.to_str() {
 277                     result.push(newpath.to_owned());
 278                 }
 279             }
 280         }
 281     }
 282
 283     result
 284 }
 285
 286 /// Scan directory for matching file names.
 287 ///
 288 /// Scan through all directory entries and call `callback()` function
 289 /// if the entry name matches the regular expression. This function
 290 /// used unix `openat()`, so you can pass absolute or relative file
 291 /// names. This function simply skips non-UTF8 encoded names.
 292 pub fn scandir<P, F>(
 293     dirfd: RawFd,
 294     path: &P,
 295     regex: &regex::Regex,
 296     mut callback: F,
 297 ) -> Result<(), Error>
 298 where
 299     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 300     P: ?Sized + nix::NixPath,
 301 {
 302     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 303         let entry = entry?;
 304         let file_type = match entry.file_type() {
 305             Some(file_type) => file_type,
 306             None => bail!("unable to detect file type"),
 307         };
 308
 309         callback(
 310             entry.parent_fd(),
 311             unsafe { entry.file_name_utf8_unchecked() },
 312             file_type,
 313         )?;
 314     }
 315     Ok(())
 316 }
 317
 318 /// Shortcut for md5 sums.
 319 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 320     hash(MessageDigest::md5(), data).map_err(Error::from)
 321 }
 322
 323 pub fn get_hardware_address() -> Result<String, Error> {
 324     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 325
 326     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 327     let digest = md5sum(&contents)?;
 328
 329     Ok(proxmox::tools::bin_to_hex(&digest))
 330 }
 331
 332 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 333     if digest1 != digest2 {
 334         bail!("detected modified configuration - file changed by other user? Try again.");
 335     }
 336     Ok(())
 337 }
 338
 339 /// Extract a specific cookie from cookie header.
 340 /// We assume cookie_name is already url encoded.
 341 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 342     for pair in cookie.split(';') {
 343         let (name, value) = match pair.find('=') {
 344             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 345             None => return None, // Cookie format error
 346         };
 347
 348         if name == cookie_name {
 349             use percent_encoding::percent_decode;
 350             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 351                 return Some(value.into());
 352             } else {
 353                 return None; // Cookie format error
 354             }
 355         }
 356     }
 357
 358     None
 359 }
 360
 361 pub fn join(data: &Vec<String>, sep: char) -> String {
 362     let mut list = String::new();
 363
 364     for item in data {
 365         if !list.is_empty() {
 366             list.push(sep);
 367         }
 368         list.push_str(item);
 369     }
 370
 371     list
 372 }
 373
 374 /// Detect modified configuration files
 375 ///
 376 /// This function fails with a reasonable error message if checksums do not match.
 377 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 378     if digest1 != digest2 {
 379         bail!("detected modified configuration - file changed by other user? Try again.");
 380     }
 381     Ok(())
 382 }
 383
 384 /// normalize uri path
 385 ///
 386 /// Do not allow ".", "..", or hidden files ".XXXX"
 387 /// Also remove empty path components
 388 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 389     let items = path.split('/');
 390
 391     let mut path = String::new();
 392     let mut components = vec![];
 393
 394     for name in items {
 395         if name.is_empty() {
 396             continue;
 397         }
 398         if name.starts_with('.') {
 399             bail!("Path contains illegal components.");
 400         }
 401         path.push('/');
 402         path.push_str(name);
 403         components.push(name);
 404     }
 405
 406     Ok((path, components))
 407 }
 408
 409 /// Helper to check result from std::process::Command output
 410 ///
 411 /// The exit_code_check() function should return true if the exit code
 412 /// is considered successful.
 413 pub fn command_output(
 414     output: std::process::Output,
 415     exit_code_check: Option<fn(i32) -> bool>,
 416 ) -> Result<Vec<u8>, Error> {
 417
 418     if !output.status.success() {
 419         match output.status.code() {
 420             Some(code) => {
 421                 let is_ok = match exit_code_check {
 422                     Some(check_fn) => check_fn(code),
 423                     None => code == 0,
 424                 };
 425                 if !is_ok {
 426                     let msg = String::from_utf8(output.stderr)
 427                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 428                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 429
 430                     bail!("status code: {} - {}", code, msg);
 431                 }
 432             }
 433             None => bail!("terminated by signal"),
 434         }
 435     }
 436
 437     Ok(output.stdout)
 438 }
 439
 440 /// Helper to check result from std::process::Command output, returns String.
 441 ///
 442 /// The exit_code_check() function should return true if the exit code
 443 /// is considered successful.
 444 pub fn command_output_as_string(
 445     output: std::process::Output,
 446     exit_code_check: Option<fn(i32) -> bool>,
 447 ) -> Result<String, Error> {
 448     let output = command_output(output, exit_code_check)?;
 449     let output = String::from_utf8(output)?;
 450     Ok(output)
 451 }
 452
 453 pub fn run_command(
 454     mut command: std::process::Command,
 455     exit_code_check: Option<fn(i32) -> bool>,
 456 ) -> Result<String, Error> {
 457
 458    let output = command.output()
 459         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 460
 461     let output = crate::tools::command_output_as_string(output, exit_code_check)
 462         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 463
 464     Ok(output)
 465 }
 466
 467 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 468     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 469     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 470         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 471     flags.set(FdFlag::FD_CLOEXEC, on);
 472     fcntl(fd, F_SETFD(flags))?;
 473     Ok(())
 474 }
 475
 476 static mut SHUTDOWN_REQUESTED: bool = false;
 477
 478 pub fn request_shutdown() {
 479     unsafe {
 480         SHUTDOWN_REQUESTED = true;
 481     }
 482     crate::server::server_shutdown();
 483 }
 484
 485 #[inline(always)]
 486 pub fn shutdown_requested() -> bool {
 487     unsafe { SHUTDOWN_REQUESTED }
 488 }
 489
 490 pub fn fail_on_shutdown() -> Result<(), Error> {
 491     if shutdown_requested() {
 492         bail!("Server shutdown requested - aborting task");
 493     }
 494     Ok(())
 495 }
 496
 497 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 498 /// descriptors.
 499 pub fn pipe() -> Result<(Fd, Fd), Error> {
 500     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 501     Ok((Fd(pin), Fd(pout)))
 502 }
 503
 504 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 505 /// descriptors.
 506 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 507     use nix::sys::socket;
 508     let (pa, pb) = socket::socketpair(
 509         socket::AddressFamily::Unix,
 510         socket::SockType::Stream,
 511         None,
 512         socket::SockFlag::SOCK_CLOEXEC,
 513     )?;
 514     Ok((Fd(pa), Fd(pb)))
 515 }
 516
 517
 518 /// An easy way to convert types to Any
 519 ///
 520 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 521 pub trait AsAny {
 522     fn as_any(&self) -> &dyn Any;
 523 }
 524
 525 impl<T: Any> AsAny for T {
 526     fn as_any(&self) -> &dyn Any {
 527         self
 528     }
 529 }
 530
 531 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 532 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 533     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 534     .add(0x20)
 535     .add(0x7f)
 536     // the DEFAULT_ENCODE_SET added:
 537     .add(b' ')
 538     .add(b'"')
 539     .add(b'#')
 540     .add(b'<')
 541     .add(b'>')
 542     .add(b'`')
 543     .add(b'?')
 544     .add(b'{')
 545     .add(b'}');
 546
 547 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 548 /// `#`).
 549 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 550     path: P,
 551 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 552     let path = path.as_ref();
 553
 554     Ok(io::BufReader::new(
 555         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 556     )
 557     .lines()
 558     .filter_map(|line| match line {
 559         Ok(line) => {
 560             let line = line.trim();
 561             if line.is_empty() || line.starts_with('#') {
 562                 None
 563             } else {
 564                 Some(Ok(line.to_string()))
 565             }
 566         }
 567         Err(err) => Some(Err(err)),
 568     }))
 569 }
 570
 571 pub fn setup_safe_path_env() {
 572     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 573     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 574     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 575         std::env::remove_var(name);
 576     }
 577 }
 578
 579 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 580     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 581         Some(n) => &line[n..],
 582         None => return &[],
 583     };
 584     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 585         Some(n) => &line[..(line.len() - n)],
 586         None => &[],
 587     }
 588 }
 589
 590 /// Seeks to start of file and computes the SHA256 hash
 591 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 592
 593     file.seek(SeekFrom::Start(0))?;
 594
 595     let mut hasher = openssl::sha::Sha256::new();
 596     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 597     let mut size: u64 = 0;
 598
 599     loop {
 600         let count = match file.read(&mut buffer) {
 601             Ok(count) => count,
 602             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 603                 continue;
 604             }
 605             Err(err) => return Err(err.into()),
 606         };
 607         if count == 0 {
 608             break;
 609         }
 610         size += count as u64;
 611         hasher.update(&buffer[..count]);
 612     }
 613
 614     let csum = hasher.finish();
 615
 616     Ok((csum, size))
 617 }