src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::{File, OpenOptions};
   8 use std::io::{self, BufRead, ErrorKind, Read};
   9 use std::os::unix::io::{AsRawFd, RawFd};
  10 use std::path::Path;
  11 use std::time::Duration;
  12
  13 use anyhow::{bail, format_err, Error};
  14 use serde_json::Value;
  15 use openssl::hash::{hash, DigestBytes, MessageDigest};
  16 use percent_encoding::AsciiSet;
  17
  18 use proxmox::tools::vec;
  19
  20 pub use proxmox::tools::fd::Fd;
  21
  22 pub mod acl;
  23 pub mod async_io;
  24 pub mod borrow;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod fs;
  28 pub mod format;
  29 pub mod lru_cache;
  30 pub mod runtime;
  31 pub mod ticket;
  32 pub mod timer;
  33 pub mod systemd;
  34
  35 mod wrapped_reader_stream;
  36 pub use wrapped_reader_stream::*;
  37
  38 mod std_channel_writer;
  39 pub use std_channel_writer::*;
  40
  41 pub mod xattr;
  42
  43 mod process_locker;
  44 pub use process_locker::*;
  45
  46 mod file_logger;
  47 pub use file_logger::*;
  48
  49 mod broadcast_future;
  50 pub use broadcast_future::*;
  51
  52 /// The `BufferedRead` trait provides a single function
  53 /// `buffered_read`. It returns a reference to an internal buffer. The
  54 /// purpose of this traid is to avoid unnecessary data copies.
  55 pub trait BufferedRead {
  56     /// This functions tries to fill the internal buffers, then
  57     /// returns a reference to the available data. It returns an empty
  58     /// buffer if `offset` points to the end of the file.
  59     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  60 }
  61
  62 /// Directly map a type into a binary buffer. This is mostly useful
  63 /// for reading structured data from a byte stream (file). You need to
  64 /// make sure that the buffer location does not change, so please
  65 /// avoid vec resize while you use such map.
  66 ///
  67 /// This function panics if the buffer is not large enough.
  68 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  69     if buffer.len() < ::std::mem::size_of::<T>() {
  70         bail!("unable to map struct - buffer too small");
  71     }
  72     Ok(unsafe { &*(buffer.as_ptr() as *const T) })
  73 }
  74
  75 /// Directly map a type into a mutable binary buffer. This is mostly
  76 /// useful for writing structured data into a byte stream (file). You
  77 /// need to make sure that the buffer location does not change, so
  78 /// please avoid vec resize while you use such map.
  79 ///
  80 /// This function panics if the buffer is not large enough.
  81 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  82     if buffer.len() < ::std::mem::size_of::<T>() {
  83         bail!("unable to map struct - buffer too small");
  84     }
  85     Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) })
  86 }
  87
  88 /// Create a file lock using fntl. This function allows you to specify
  89 /// a timeout if you want to avoid infinite blocking.
  90 pub fn lock_file<F: AsRawFd>(
  91     file: &mut F,
  92     exclusive: bool,
  93     timeout: Option<Duration>,
  94 ) -> Result<(), Error> {
  95     let lockarg = if exclusive {
  96         nix::fcntl::FlockArg::LockExclusive
  97     } else {
  98         nix::fcntl::FlockArg::LockShared
  99     };
 100
 101     let timeout = match timeout {
 102         None => {
 103             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 104             return Ok(());
 105         }
 106         Some(t) => t,
 107     };
 108
 109     // unblock the timeout signal temporarily
 110     let _sigblock_guard = timer::unblock_timeout_signal();
 111
 112     // setup a timeout timer
 113     let mut timer = timer::Timer::create(
 114         timer::Clock::Realtime,
 115         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT),
 116     )?;
 117
 118     timer.arm(
 119         timer::TimerSpec::new()
 120             .value(Some(timeout))
 121             .interval(Some(Duration::from_millis(10))),
 122     )?;
 123
 124     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 125     Ok(())
 126 }
 127
 128 /// Open or create a lock file (append mode). Then try to
 129 /// acquire a lock using `lock_file()`.
 130 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<File, Error> {
 131     let path = path.as_ref();
 132     let mut file = match OpenOptions::new().create(true).append(true).open(path) {
 133         Ok(file) => file,
 134         Err(err) => bail!("Unable to open lock {:?} - {}", path, err),
 135     };
 136     match lock_file(&mut file, true, Some(timeout)) {
 137         Ok(_) => Ok(file),
 138         Err(err) => bail!("Unable to acquire lock {:?} - {}", path, err),
 139     }
 140 }
 141
 142 /// Split a file into equal sized chunks. The last chunk may be
 143 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 144 /// cannot return a borrowed buffer ref (we want zero-copy)
 145 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
 146 where
 147     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 148     R: Read,
 149 {
 150     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
 151
 152     if chunk_size > READ_BUFFER_SIZE {
 153         bail!("chunk size too large!");
 154     }
 155
 156     let mut buf = vec::undefined(READ_BUFFER_SIZE);
 157
 158     let mut pos = 0;
 159     let mut file_pos = 0;
 160     loop {
 161         let mut eof = false;
 162         let mut tmp = &mut buf[..];
 163         // try to read large portions, at least chunk_size
 164         while pos < chunk_size {
 165             match file.read(tmp) {
 166                 Ok(0) => {
 167                     eof = true;
 168                     break;
 169                 }
 170                 Ok(n) => {
 171                     pos += n;
 172                     if pos > chunk_size {
 173                         break;
 174                     }
 175                     tmp = &mut tmp[n..];
 176                 }
 177                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 178                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 179             }
 180         }
 181         let mut start = 0;
 182         while start + chunk_size <= pos {
 183             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 184                 break;
 185             }
 186             file_pos += chunk_size;
 187             start += chunk_size;
 188         }
 189         if eof {
 190             if start < pos {
 191                 (chunk_cb)(file_pos, &buf[start..pos])?;
 192                 //file_pos += pos - start;
 193             }
 194             break;
 195         } else {
 196             let rest = pos - start;
 197             if rest > 0 {
 198                 let ptr = buf.as_mut_ptr();
 199                 unsafe {
 200                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 201                 }
 202                 pos = rest;
 203             } else {
 204                 pos = 0;
 205             }
 206         }
 207     }
 208
 209     Ok(())
 210 }
 211
 212 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 213     let mut query = url::form_urlencoded::Serializer::new(String::new());
 214
 215     let object = data.as_object().ok_or_else(|| {
 216         format_err!("json_object_to_query: got wrong data type (expected object).")
 217     })?;
 218
 219     for (key, value) in object {
 220         match value {
 221             Value::Bool(b) => {
 222                 query.append_pair(key, &b.to_string());
 223             }
 224             Value::Number(n) => {
 225                 query.append_pair(key, &n.to_string());
 226             }
 227             Value::String(s) => {
 228                 query.append_pair(key, &s);
 229             }
 230             Value::Array(arr) => {
 231                 for element in arr {
 232                     match element {
 233                         Value::Bool(b) => {
 234                             query.append_pair(key, &b.to_string());
 235                         }
 236                         Value::Number(n) => {
 237                             query.append_pair(key, &n.to_string());
 238                         }
 239                         Value::String(s) => {
 240                             query.append_pair(key, &s);
 241                         }
 242                         _ => bail!(
 243                             "json_object_to_query: unable to handle complex array data types."
 244                         ),
 245                     }
 246                 }
 247             }
 248             _ => bail!("json_object_to_query: unable to handle complex data types."),
 249         }
 250     }
 251
 252     Ok(query.finish())
 253 }
 254
 255 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 256     match param[name].as_str() {
 257         Some(s) => Ok(s),
 258         None => bail!("missing parameter '{}'", name),
 259     }
 260 }
 261
 262 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 263     match param[name].as_str() {
 264         Some(s) => Ok(s),
 265         None => bail!("missing property '{}'", name),
 266     }
 267 }
 268
 269 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 270     match param[name].as_i64() {
 271         Some(s) => Ok(s),
 272         None => bail!("missing parameter '{}'", name),
 273     }
 274 }
 275
 276 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 277     match param[name].as_i64() {
 278         Some(s) => Ok(s),
 279         None => bail!("missing property '{}'", name),
 280     }
 281 }
 282
 283 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 284     match param[name].as_array() {
 285         Some(s) => Ok(s.to_vec()),
 286         None => bail!("missing parameter '{}'", name),
 287     }
 288 }
 289
 290 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 291     match param[name].as_array() {
 292         Some(s) => Ok(s.to_vec()),
 293         None => bail!("missing property '{}'", name),
 294     }
 295 }
 296
 297 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 298     let mut result = vec![];
 299
 300     use nix::fcntl::AtFlags;
 301     use nix::fcntl::OFlag;
 302     use nix::sys::stat::Mode;
 303
 304     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 305
 306     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 307         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 308         Err(_) => false,
 309     };
 310
 311     if !is_dir {
 312         if let Some(parent) = dirname.parent() {
 313             dirname = parent.to_owned();
 314         }
 315     }
 316
 317     let mut dir =
 318         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 319             Ok(d) => d,
 320             Err(_) => return result,
 321         };
 322
 323     for item in dir.iter() {
 324         if let Ok(entry) = item {
 325             if let Ok(name) = entry.file_name().to_str() {
 326                 if name == "." || name == ".." {
 327                     continue;
 328                 }
 329                 let mut newpath = dirname.clone();
 330                 newpath.push(name);
 331
 332                 if let Ok(stat) =
 333                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 334                 {
 335                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 336                         newpath.push("");
 337                         if let Some(newpath) = newpath.to_str() {
 338                             result.push(newpath.to_owned());
 339                         }
 340                         continue;
 341                     }
 342                 }
 343                 if let Some(newpath) = newpath.to_str() {
 344                     result.push(newpath.to_owned());
 345                 }
 346             }
 347         }
 348     }
 349
 350     result
 351 }
 352
 353 /// Scan directory for matching file names.
 354 ///
 355 /// Scan through all directory entries and call `callback()` function
 356 /// if the entry name matches the regular expression. This function
 357 /// used unix `openat()`, so you can pass absolute or relative file
 358 /// names. This function simply skips non-UTF8 encoded names.
 359 pub fn scandir<P, F>(
 360     dirfd: RawFd,
 361     path: &P,
 362     regex: &regex::Regex,
 363     mut callback: F,
 364 ) -> Result<(), Error>
 365 where
 366     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 367     P: ?Sized + nix::NixPath,
 368 {
 369     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 370         let entry = entry?;
 371         let file_type = match entry.file_type() {
 372             Some(file_type) => file_type,
 373             None => bail!("unable to detect file type"),
 374         };
 375
 376         callback(
 377             entry.parent_fd(),
 378             unsafe { entry.file_name_utf8_unchecked() },
 379             file_type,
 380         )?;
 381     }
 382     Ok(())
 383 }
 384
 385 /// Shortcut for md5 sums.
 386 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 387     hash(MessageDigest::md5(), data).map_err(Error::from)
 388 }
 389
 390 pub fn get_hardware_address() -> Result<String, Error> {
 391     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 392
 393     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 394     let digest = md5sum(&contents)?;
 395
 396     Ok(proxmox::tools::bin_to_hex(&digest))
 397 }
 398
 399 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 400     if digest1 != digest2 {
 401         bail!("detected modified configuration - file changed by other user? Try again.");
 402     }
 403     Ok(())
 404 }
 405
 406 /// Extract authentication cookie from cookie header.
 407 /// We assume cookie_name is already url encoded.
 408 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 409     for pair in cookie.split(';') {
 410         let (name, value) = match pair.find('=') {
 411             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 412             None => return None, // Cookie format error
 413         };
 414
 415         if name == cookie_name {
 416             use percent_encoding::percent_decode;
 417             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 418                 return Some(value.into());
 419             } else {
 420                 return None; // Cookie format error
 421             }
 422         }
 423     }
 424
 425     None
 426 }
 427
 428 pub fn join(data: &Vec<String>, sep: char) -> String {
 429     let mut list = String::new();
 430
 431     for item in data {
 432         if !list.is_empty() {
 433             list.push(sep);
 434         }
 435         list.push_str(item);
 436     }
 437
 438     list
 439 }
 440
 441 /// Detect modified configuration files
 442 ///
 443 /// This function fails with a reasonable error message if checksums do not match.
 444 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 445     if digest1 != digest2 {
 446         bail!("detected modified configuration - file changed by other user? Try again.");
 447     }
 448     Ok(())
 449 }
 450
 451 /// normalize uri path
 452 ///
 453 /// Do not allow ".", "..", or hidden files ".XXXX"
 454 /// Also remove empty path components
 455 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 456     let items = path.split('/');
 457
 458     let mut path = String::new();
 459     let mut components = vec![];
 460
 461     for name in items {
 462         if name.is_empty() {
 463             continue;
 464         }
 465         if name.starts_with('.') {
 466             bail!("Path contains illegal components.");
 467         }
 468         path.push('/');
 469         path.push_str(name);
 470         components.push(name);
 471     }
 472
 473     Ok((path, components))
 474 }
 475
 476 /// Helper to check result from std::process::Command output
 477 ///
 478 /// The exit_code_check() function should return true if the exit code
 479 /// is considered successful.
 480 pub fn command_output(
 481     output: std::process::Output,
 482     exit_code_check: Option<fn(i32) -> bool>
 483 ) -> Result<String, Error> {
 484
 485     if !output.status.success() {
 486         match output.status.code() {
 487             Some(code) => {
 488                 let is_ok = match exit_code_check {
 489                     Some(check_fn) => check_fn(code),
 490                     None => code == 0,
 491                 };
 492                 if !is_ok {
 493                     let msg = String::from_utf8(output.stderr)
 494                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 495                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 496
 497                     bail!("status code: {} - {}", code, msg);
 498                 }
 499             }
 500             None => bail!("terminated by signal"),
 501         }
 502     }
 503
 504     let output = String::from_utf8(output.stdout)?;
 505
 506     Ok(output)
 507 }
 508
 509
 510 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 511     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 512     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 513         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 514     flags.set(FdFlag::FD_CLOEXEC, on);
 515     fcntl(fd, F_SETFD(flags))?;
 516     Ok(())
 517 }
 518
 519 static mut SHUTDOWN_REQUESTED: bool = false;
 520
 521 pub fn request_shutdown() {
 522     unsafe {
 523         SHUTDOWN_REQUESTED = true;
 524     }
 525     crate::server::server_shutdown();
 526 }
 527
 528 #[inline(always)]
 529 pub fn shutdown_requested() -> bool {
 530     unsafe { SHUTDOWN_REQUESTED }
 531 }
 532
 533 pub fn fail_on_shutdown() -> Result<(), Error> {
 534     if shutdown_requested() {
 535         bail!("Server shutdown requested - aborting task");
 536     }
 537     Ok(())
 538 }
 539
 540 // wrap nix::unistd::pipe2 + O_CLOEXEC into something returning guarded file descriptors
 541 pub fn pipe() -> Result<(Fd, Fd), Error> {
 542     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 543     Ok((Fd(pin), Fd(pout)))
 544 }
 545
 546 /// An easy way to convert types to Any
 547 ///
 548 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 549 pub trait AsAny {
 550     fn as_any(&self) -> &dyn Any;
 551 }
 552
 553 impl<T: Any> AsAny for T {
 554     fn as_any(&self) -> &dyn Any {
 555         self
 556     }
 557 }
 558
 559 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 560 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 561     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 562     .add(0x20)
 563     .add(0x7f)
 564     // the DEFAULT_ENCODE_SET added:
 565     .add(b' ')
 566     .add(b'"')
 567     .add(b'#')
 568     .add(b'<')
 569     .add(b'>')
 570     .add(b'`')
 571     .add(b'?')
 572     .add(b'{')
 573     .add(b'}');
 574
 575 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 576 /// `#`).
 577 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 578     path: P,
 579 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 580     let path = path.as_ref();
 581
 582     Ok(io::BufReader::new(
 583         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 584     )
 585     .lines()
 586     .filter_map(|line| match line {
 587         Ok(line) => {
 588             let line = line.trim();
 589             if line.is_empty() || line.starts_with('#') {
 590                 None
 591             } else {
 592                 Some(Ok(line.to_string()))
 593             }
 594         }
 595         Err(err) => Some(Err(err)),
 596     }))
 597 }