src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::AsciiSet;
  16
  17 use proxmox::tools::vec;
  18
  19 pub use proxmox::tools::fd::Fd;
  20
  21 pub mod acl;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod cert;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod statistics;
  33 pub mod systemd;
  34 pub mod nom;
  35 pub mod logrotate;
  36 pub mod loopdev;
  37 pub mod fuse_loop;
  38 pub mod socket;
  39 pub mod zip;
  40 pub mod http;
  41
  42 mod parallel_handler;
  43 pub use parallel_handler::*;
  44
  45 mod wrapped_reader_stream;
  46 pub use wrapped_reader_stream::*;
  47
  48 mod async_channel_writer;
  49 pub use async_channel_writer::*;
  50
  51
  52 mod std_channel_writer;
  53 pub use std_channel_writer::*;
  54
  55 pub mod xattr;
  56
  57 mod process_locker;
  58 pub use process_locker::*;
  59
  60 mod file_logger;
  61 pub use file_logger::*;
  62
  63 mod broadcast_future;
  64 pub use broadcast_future::*;
  65
  66 /// The `BufferedRead` trait provides a single function
  67 /// `buffered_read`. It returns a reference to an internal buffer. The
  68 /// purpose of this traid is to avoid unnecessary data copies.
  69 pub trait BufferedRead {
  70     /// This functions tries to fill the internal buffers, then
  71     /// returns a reference to the available data. It returns an empty
  72     /// buffer if `offset` points to the end of the file.
  73     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  74 }
  75
  76 /// Split a file into equal sized chunks. The last chunk may be
  77 /// smaller. Note: We cannot implement an `Iterator`, because iterators
  78 /// cannot return a borrowed buffer ref (we want zero-copy)
  79 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
  80 where
  81     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
  82     R: Read,
  83 {
  84     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
  85
  86     if chunk_size > READ_BUFFER_SIZE {
  87         bail!("chunk size too large!");
  88     }
  89
  90     let mut buf = vec::undefined(READ_BUFFER_SIZE);
  91
  92     let mut pos = 0;
  93     let mut file_pos = 0;
  94     loop {
  95         let mut eof = false;
  96         let mut tmp = &mut buf[..];
  97         // try to read large portions, at least chunk_size
  98         while pos < chunk_size {
  99             match file.read(tmp) {
 100                 Ok(0) => {
 101                     eof = true;
 102                     break;
 103                 }
 104                 Ok(n) => {
 105                     pos += n;
 106                     if pos > chunk_size {
 107                         break;
 108                     }
 109                     tmp = &mut tmp[n..];
 110                 }
 111                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 112                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 113             }
 114         }
 115         let mut start = 0;
 116         while start + chunk_size <= pos {
 117             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 118                 break;
 119             }
 120             file_pos += chunk_size;
 121             start += chunk_size;
 122         }
 123         if eof {
 124             if start < pos {
 125                 (chunk_cb)(file_pos, &buf[start..pos])?;
 126                 //file_pos += pos - start;
 127             }
 128             break;
 129         } else {
 130             let rest = pos - start;
 131             if rest > 0 {
 132                 let ptr = buf.as_mut_ptr();
 133                 unsafe {
 134                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 135                 }
 136                 pos = rest;
 137             } else {
 138                 pos = 0;
 139             }
 140         }
 141     }
 142
 143     Ok(())
 144 }
 145
 146 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 147     let mut query = url::form_urlencoded::Serializer::new(String::new());
 148
 149     let object = data.as_object().ok_or_else(|| {
 150         format_err!("json_object_to_query: got wrong data type (expected object).")
 151     })?;
 152
 153     for (key, value) in object {
 154         match value {
 155             Value::Bool(b) => {
 156                 query.append_pair(key, &b.to_string());
 157             }
 158             Value::Number(n) => {
 159                 query.append_pair(key, &n.to_string());
 160             }
 161             Value::String(s) => {
 162                 query.append_pair(key, &s);
 163             }
 164             Value::Array(arr) => {
 165                 for element in arr {
 166                     match element {
 167                         Value::Bool(b) => {
 168                             query.append_pair(key, &b.to_string());
 169                         }
 170                         Value::Number(n) => {
 171                             query.append_pair(key, &n.to_string());
 172                         }
 173                         Value::String(s) => {
 174                             query.append_pair(key, &s);
 175                         }
 176                         _ => bail!(
 177                             "json_object_to_query: unable to handle complex array data types."
 178                         ),
 179                     }
 180                 }
 181             }
 182             _ => bail!("json_object_to_query: unable to handle complex data types."),
 183         }
 184     }
 185
 186     Ok(query.finish())
 187 }
 188
 189 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 190     match param[name].as_str() {
 191         Some(s) => Ok(s),
 192         None => bail!("missing parameter '{}'", name),
 193     }
 194 }
 195
 196 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 197     match param[name].as_str() {
 198         Some(s) => Ok(s),
 199         None => bail!("missing property '{}'", name),
 200     }
 201 }
 202
 203 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 204     match param[name].as_i64() {
 205         Some(s) => Ok(s),
 206         None => bail!("missing parameter '{}'", name),
 207     }
 208 }
 209
 210 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 211     match param[name].as_i64() {
 212         Some(s) => Ok(s),
 213         None => bail!("missing property '{}'", name),
 214     }
 215 }
 216
 217 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 218     match param[name].as_array() {
 219         Some(s) => Ok(s.to_vec()),
 220         None => bail!("missing parameter '{}'", name),
 221     }
 222 }
 223
 224 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 225     match param[name].as_array() {
 226         Some(s) => Ok(s.to_vec()),
 227         None => bail!("missing property '{}'", name),
 228     }
 229 }
 230
 231 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 232     let mut result = vec![];
 233
 234     use nix::fcntl::AtFlags;
 235     use nix::fcntl::OFlag;
 236     use nix::sys::stat::Mode;
 237
 238     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 239
 240     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 241         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 242         Err(_) => false,
 243     };
 244
 245     if !is_dir {
 246         if let Some(parent) = dirname.parent() {
 247             dirname = parent.to_owned();
 248         }
 249     }
 250
 251     let mut dir =
 252         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 253             Ok(d) => d,
 254             Err(_) => return result,
 255         };
 256
 257     for item in dir.iter() {
 258         if let Ok(entry) = item {
 259             if let Ok(name) = entry.file_name().to_str() {
 260                 if name == "." || name == ".." {
 261                     continue;
 262                 }
 263                 let mut newpath = dirname.clone();
 264                 newpath.push(name);
 265
 266                 if let Ok(stat) =
 267                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 268                 {
 269                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 270                         newpath.push("");
 271                         if let Some(newpath) = newpath.to_str() {
 272                             result.push(newpath.to_owned());
 273                         }
 274                         continue;
 275                     }
 276                 }
 277                 if let Some(newpath) = newpath.to_str() {
 278                     result.push(newpath.to_owned());
 279                 }
 280             }
 281         }
 282     }
 283
 284     result
 285 }
 286
 287 /// Scan directory for matching file names.
 288 ///
 289 /// Scan through all directory entries and call `callback()` function
 290 /// if the entry name matches the regular expression. This function
 291 /// used unix `openat()`, so you can pass absolute or relative file
 292 /// names. This function simply skips non-UTF8 encoded names.
 293 pub fn scandir<P, F>(
 294     dirfd: RawFd,
 295     path: &P,
 296     regex: &regex::Regex,
 297     mut callback: F,
 298 ) -> Result<(), Error>
 299 where
 300     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 301     P: ?Sized + nix::NixPath,
 302 {
 303     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 304         let entry = entry?;
 305         let file_type = match entry.file_type() {
 306             Some(file_type) => file_type,
 307             None => bail!("unable to detect file type"),
 308         };
 309
 310         callback(
 311             entry.parent_fd(),
 312             unsafe { entry.file_name_utf8_unchecked() },
 313             file_type,
 314         )?;
 315     }
 316     Ok(())
 317 }
 318
 319 /// Shortcut for md5 sums.
 320 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 321     hash(MessageDigest::md5(), data).map_err(Error::from)
 322 }
 323
 324 pub fn get_hardware_address() -> Result<String, Error> {
 325     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 326
 327     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 328     let digest = md5sum(&contents)?;
 329
 330     Ok(proxmox::tools::bin_to_hex(&digest))
 331 }
 332
 333 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 334     if digest1 != digest2 {
 335         bail!("detected modified configuration - file changed by other user? Try again.");
 336     }
 337     Ok(())
 338 }
 339
 340 /// Extract a specific cookie from cookie header.
 341 /// We assume cookie_name is already url encoded.
 342 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 343     for pair in cookie.split(';') {
 344         let (name, value) = match pair.find('=') {
 345             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 346             None => return None, // Cookie format error
 347         };
 348
 349         if name == cookie_name {
 350             use percent_encoding::percent_decode;
 351             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 352                 return Some(value.into());
 353             } else {
 354                 return None; // Cookie format error
 355             }
 356         }
 357     }
 358
 359     None
 360 }
 361
 362 pub fn join(data: &Vec<String>, sep: char) -> String {
 363     let mut list = String::new();
 364
 365     for item in data {
 366         if !list.is_empty() {
 367             list.push(sep);
 368         }
 369         list.push_str(item);
 370     }
 371
 372     list
 373 }
 374
 375 /// Detect modified configuration files
 376 ///
 377 /// This function fails with a reasonable error message if checksums do not match.
 378 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 379     if digest1 != digest2 {
 380         bail!("detected modified configuration - file changed by other user? Try again.");
 381     }
 382     Ok(())
 383 }
 384
 385 /// normalize uri path
 386 ///
 387 /// Do not allow ".", "..", or hidden files ".XXXX"
 388 /// Also remove empty path components
 389 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 390     let items = path.split('/');
 391
 392     let mut path = String::new();
 393     let mut components = vec![];
 394
 395     for name in items {
 396         if name.is_empty() {
 397             continue;
 398         }
 399         if name.starts_with('.') {
 400             bail!("Path contains illegal components.");
 401         }
 402         path.push('/');
 403         path.push_str(name);
 404         components.push(name);
 405     }
 406
 407     Ok((path, components))
 408 }
 409
 410 /// Helper to check result from std::process::Command output
 411 ///
 412 /// The exit_code_check() function should return true if the exit code
 413 /// is considered successful.
 414 pub fn command_output(
 415     output: std::process::Output,
 416     exit_code_check: Option<fn(i32) -> bool>,
 417 ) -> Result<Vec<u8>, Error> {
 418
 419     if !output.status.success() {
 420         match output.status.code() {
 421             Some(code) => {
 422                 let is_ok = match exit_code_check {
 423                     Some(check_fn) => check_fn(code),
 424                     None => code == 0,
 425                 };
 426                 if !is_ok {
 427                     let msg = String::from_utf8(output.stderr)
 428                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 429                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 430
 431                     bail!("status code: {} - {}", code, msg);
 432                 }
 433             }
 434             None => bail!("terminated by signal"),
 435         }
 436     }
 437
 438     Ok(output.stdout)
 439 }
 440
 441 /// Helper to check result from std::process::Command output, returns String.
 442 ///
 443 /// The exit_code_check() function should return true if the exit code
 444 /// is considered successful.
 445 pub fn command_output_as_string(
 446     output: std::process::Output,
 447     exit_code_check: Option<fn(i32) -> bool>,
 448 ) -> Result<String, Error> {
 449     let output = command_output(output, exit_code_check)?;
 450     let output = String::from_utf8(output)?;
 451     Ok(output)
 452 }
 453
 454 pub fn run_command(
 455     mut command: std::process::Command,
 456     exit_code_check: Option<fn(i32) -> bool>,
 457 ) -> Result<String, Error> {
 458
 459    let output = command.output()
 460         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 461
 462     let output = crate::tools::command_output_as_string(output, exit_code_check)
 463         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 464
 465     Ok(output)
 466 }
 467
 468 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 469     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 470     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 471         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 472     flags.set(FdFlag::FD_CLOEXEC, on);
 473     fcntl(fd, F_SETFD(flags))?;
 474     Ok(())
 475 }
 476
 477 static mut SHUTDOWN_REQUESTED: bool = false;
 478
 479 pub fn request_shutdown() {
 480     unsafe {
 481         SHUTDOWN_REQUESTED = true;
 482     }
 483     crate::server::server_shutdown();
 484 }
 485
 486 #[inline(always)]
 487 pub fn shutdown_requested() -> bool {
 488     unsafe { SHUTDOWN_REQUESTED }
 489 }
 490
 491 pub fn fail_on_shutdown() -> Result<(), Error> {
 492     if shutdown_requested() {
 493         bail!("Server shutdown requested - aborting task");
 494     }
 495     Ok(())
 496 }
 497
 498 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 499 /// descriptors.
 500 pub fn pipe() -> Result<(Fd, Fd), Error> {
 501     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 502     Ok((Fd(pin), Fd(pout)))
 503 }
 504
 505 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 506 /// descriptors.
 507 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 508     use nix::sys::socket;
 509     let (pa, pb) = socket::socketpair(
 510         socket::AddressFamily::Unix,
 511         socket::SockType::Stream,
 512         None,
 513         socket::SockFlag::SOCK_CLOEXEC,
 514     )?;
 515     Ok((Fd(pa), Fd(pb)))
 516 }
 517
 518
 519 /// An easy way to convert types to Any
 520 ///
 521 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 522 pub trait AsAny {
 523     fn as_any(&self) -> &dyn Any;
 524 }
 525
 526 impl<T: Any> AsAny for T {
 527     fn as_any(&self) -> &dyn Any {
 528         self
 529     }
 530 }
 531
 532 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 533 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 534     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 535     .add(0x20)
 536     .add(0x7f)
 537     // the DEFAULT_ENCODE_SET added:
 538     .add(b' ')
 539     .add(b'"')
 540     .add(b'#')
 541     .add(b'<')
 542     .add(b'>')
 543     .add(b'`')
 544     .add(b'?')
 545     .add(b'{')
 546     .add(b'}');
 547
 548 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 549 /// `#`).
 550 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 551     path: P,
 552 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 553     let path = path.as_ref();
 554
 555     Ok(io::BufReader::new(
 556         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 557     )
 558     .lines()
 559     .filter_map(|line| match line {
 560         Ok(line) => {
 561             let line = line.trim();
 562             if line.is_empty() || line.starts_with('#') {
 563                 None
 564             } else {
 565                 Some(Ok(line.to_string()))
 566             }
 567         }
 568         Err(err) => Some(Err(err)),
 569     }))
 570 }
 571
 572 pub fn setup_safe_path_env() {
 573     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 574     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 575     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 576         std::env::remove_var(name);
 577     }
 578 }
 579
 580 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 581     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 582         Some(n) => &line[n..],
 583         None => return &[],
 584     };
 585     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 586         Some(n) => &line[..(line.len() - n)],
 587         None => &[],
 588     }
 589 }
 590
 591 /// Seeks to start of file and computes the SHA256 hash
 592 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 593
 594     file.seek(SeekFrom::Start(0))?;
 595
 596     let mut hasher = openssl::sha::Sha256::new();
 597     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 598     let mut size: u64 = 0;
 599
 600     loop {
 601         let count = match file.read(&mut buffer) {
 602             Ok(count) => count,
 603             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 604                 continue;
 605             }
 606             Err(err) => return Err(err.into()),
 607         };
 608         if count == 0 {
 609             break;
 610         }
 611         size += count as u64;
 612         hasher.update(&buffer[..count]);
 613     }
 614
 615     let csum = hasher.finish();
 616
 617     Ok((csum, size))
 618 }