src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::{File, OpenOptions};
   8 use std::io::ErrorKind;
   9 use std::io::Read;
  10 use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
  11 use std::path::Path;
  12 use std::time::Duration;
  13
  14 use failure::*;
  15 use serde_json::Value;
  16 use openssl::hash::{hash, DigestBytes, MessageDigest};
  17 use percent_encoding::AsciiSet;
  18
  19 use proxmox::tools::vec;
  20
  21 pub mod acl;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod daemon;
  25 pub mod fs;
  26 pub mod runtime;
  27 pub mod ticket;
  28 pub mod timer;
  29 pub mod tty;
  30 pub mod wrapped_reader_stream;
  31 pub mod xattr;
  32
  33 mod process_locker;
  34 pub use process_locker::*;
  35
  36 mod file_logger;
  37 pub use file_logger::*;
  38
  39 mod broadcast_future;
  40 pub use broadcast_future::*;
  41
  42 /// The `BufferedRead` trait provides a single function
  43 /// `buffered_read`. It returns a reference to an internal buffer. The
  44 /// purpose of this traid is to avoid unnecessary data copies.
  45 pub trait BufferedRead {
  46     /// This functions tries to fill the internal buffers, then
  47     /// returns a reference to the available data. It returns an empty
  48     /// buffer if `offset` points to the end of the file.
  49     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  50 }
  51
  52 /// Directly map a type into a binary buffer. This is mostly useful
  53 /// for reading structured data from a byte stream (file). You need to
  54 /// make sure that the buffer location does not change, so please
  55 /// avoid vec resize while you use such map.
  56 ///
  57 /// This function panics if the buffer is not large enough.
  58 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  59     if buffer.len() < ::std::mem::size_of::<T>() {
  60         bail!("unable to map struct - buffer too small");
  61     }
  62     Ok(unsafe { &*(buffer.as_ptr() as *const T) })
  63 }
  64
  65 /// Directly map a type into a mutable binary buffer. This is mostly
  66 /// useful for writing structured data into a byte stream (file). You
  67 /// need to make sure that the buffer location does not change, so
  68 /// please avoid vec resize while you use such map.
  69 ///
  70 /// This function panics if the buffer is not large enough.
  71 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  72     if buffer.len() < ::std::mem::size_of::<T>() {
  73         bail!("unable to map struct - buffer too small");
  74     }
  75     Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) })
  76 }
  77
  78 /// Create a file lock using fntl. This function allows you to specify
  79 /// a timeout if you want to avoid infinite blocking.
  80 pub fn lock_file<F: AsRawFd>(
  81     file: &mut F,
  82     exclusive: bool,
  83     timeout: Option<Duration>,
  84 ) -> Result<(), Error> {
  85     let lockarg = if exclusive {
  86         nix::fcntl::FlockArg::LockExclusive
  87     } else {
  88         nix::fcntl::FlockArg::LockShared
  89     };
  90
  91     let timeout = match timeout {
  92         None => {
  93             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
  94             return Ok(());
  95         }
  96         Some(t) => t,
  97     };
  98
  99     // unblock the timeout signal temporarily
 100     let _sigblock_guard = timer::unblock_timeout_signal();
 101
 102     // setup a timeout timer
 103     let mut timer = timer::Timer::create(
 104         timer::Clock::Realtime,
 105         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT),
 106     )?;
 107
 108     timer.arm(
 109         timer::TimerSpec::new()
 110             .value(Some(timeout))
 111             .interval(Some(Duration::from_millis(10))),
 112     )?;
 113
 114     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 115     Ok(())
 116 }
 117
 118 /// Open or create a lock file (append mode). Then try to
 119 /// aquire a lock using `lock_file()`.
 120 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<File, Error> {
 121     let path = path.as_ref();
 122     let mut file = match OpenOptions::new().create(true).append(true).open(path) {
 123         Ok(file) => file,
 124         Err(err) => bail!("Unable to open lock {:?} - {}", path, err),
 125     };
 126     match lock_file(&mut file, true, Some(timeout)) {
 127         Ok(_) => Ok(file),
 128         Err(err) => bail!("Unable to aquire lock {:?} - {}", path, err),
 129     }
 130 }
 131
 132 /// Split a file into equal sized chunks. The last chunk may be
 133 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 134 /// cannot return a borrowed buffer ref (we want zero-copy)
 135 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
 136 where
 137     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 138     R: Read,
 139 {
 140     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
 141
 142     if chunk_size > READ_BUFFER_SIZE {
 143         bail!("chunk size too large!");
 144     }
 145
 146     let mut buf = vec::undefined(READ_BUFFER_SIZE);
 147
 148     let mut pos = 0;
 149     let mut file_pos = 0;
 150     loop {
 151         let mut eof = false;
 152         let mut tmp = &mut buf[..];
 153         // try to read large portions, at least chunk_size
 154         while pos < chunk_size {
 155             match file.read(tmp) {
 156                 Ok(0) => {
 157                     eof = true;
 158                     break;
 159                 }
 160                 Ok(n) => {
 161                     pos += n;
 162                     if pos > chunk_size {
 163                         break;
 164                     }
 165                     tmp = &mut tmp[n..];
 166                 }
 167                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 168                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 169             }
 170         }
 171         let mut start = 0;
 172         while start + chunk_size <= pos {
 173             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 174                 break;
 175             }
 176             file_pos += chunk_size;
 177             start += chunk_size;
 178         }
 179         if eof {
 180             if start < pos {
 181                 (chunk_cb)(file_pos, &buf[start..pos])?;
 182                 //file_pos += pos - start;
 183             }
 184             break;
 185         } else {
 186             let rest = pos - start;
 187             if rest > 0 {
 188                 let ptr = buf.as_mut_ptr();
 189                 unsafe {
 190                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 191                 }
 192                 pos = rest;
 193             } else {
 194                 pos = 0;
 195             }
 196         }
 197     }
 198
 199     Ok(())
 200 }
 201
 202 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 203     let mut query = url::form_urlencoded::Serializer::new(String::new());
 204
 205     let object = data.as_object().ok_or_else(|| {
 206         format_err!("json_object_to_query: got wrong data type (expected object).")
 207     })?;
 208
 209     for (key, value) in object {
 210         match value {
 211             Value::Bool(b) => {
 212                 query.append_pair(key, &b.to_string());
 213             }
 214             Value::Number(n) => {
 215                 query.append_pair(key, &n.to_string());
 216             }
 217             Value::String(s) => {
 218                 query.append_pair(key, &s);
 219             }
 220             Value::Array(arr) => {
 221                 for element in arr {
 222                     match element {
 223                         Value::Bool(b) => {
 224                             query.append_pair(key, &b.to_string());
 225                         }
 226                         Value::Number(n) => {
 227                             query.append_pair(key, &n.to_string());
 228                         }
 229                         Value::String(s) => {
 230                             query.append_pair(key, &s);
 231                         }
 232                         _ => bail!(
 233                             "json_object_to_query: unable to handle complex array data types."
 234                         ),
 235                     }
 236                 }
 237             }
 238             _ => bail!("json_object_to_query: unable to handle complex data types."),
 239         }
 240     }
 241
 242     Ok(query.finish())
 243 }
 244
 245 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 246     match param[name].as_str() {
 247         Some(s) => Ok(s),
 248         None => bail!("missing parameter '{}'", name),
 249     }
 250 }
 251
 252 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 253     match param[name].as_str() {
 254         Some(s) => Ok(s),
 255         None => bail!("missing property '{}'", name),
 256     }
 257 }
 258
 259 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 260     match param[name].as_i64() {
 261         Some(s) => Ok(s),
 262         None => bail!("missing parameter '{}'", name),
 263     }
 264 }
 265
 266 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 267     match param[name].as_i64() {
 268         Some(s) => Ok(s),
 269         None => bail!("missing property '{}'", name),
 270     }
 271 }
 272
 273 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 274     match param[name].as_array() {
 275         Some(s) => Ok(s.to_vec()),
 276         None => bail!("missing parameter '{}'", name),
 277     }
 278 }
 279
 280 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 281     match param[name].as_array() {
 282         Some(s) => Ok(s.to_vec()),
 283         None => bail!("missing property '{}'", name),
 284     }
 285 }
 286
 287 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 288     let mut result = vec![];
 289
 290     use nix::fcntl::AtFlags;
 291     use nix::fcntl::OFlag;
 292     use nix::sys::stat::Mode;
 293
 294     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 295
 296     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 297         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 298         Err(_) => false,
 299     };
 300
 301     if !is_dir {
 302         if let Some(parent) = dirname.parent() {
 303             dirname = parent.to_owned();
 304         }
 305     }
 306
 307     let mut dir =
 308         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 309             Ok(d) => d,
 310             Err(_) => return result,
 311         };
 312
 313     for item in dir.iter() {
 314         if let Ok(entry) = item {
 315             if let Ok(name) = entry.file_name().to_str() {
 316                 if name == "." || name == ".." {
 317                     continue;
 318                 }
 319                 let mut newpath = dirname.clone();
 320                 newpath.push(name);
 321
 322                 if let Ok(stat) =
 323                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 324                 {
 325                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 326                         newpath.push("");
 327                         if let Some(newpath) = newpath.to_str() {
 328                             result.push(newpath.to_owned());
 329                         }
 330                         continue;
 331                     }
 332                 }
 333                 if let Some(newpath) = newpath.to_str() {
 334                     result.push(newpath.to_owned());
 335                 }
 336             }
 337         }
 338     }
 339
 340     result
 341 }
 342
 343 /// Scan directory for matching file names.
 344 ///
 345 /// Scan through all directory entries and call `callback()` function
 346 /// if the entry name matches the regular expression. This function
 347 /// used unix `openat()`, so you can pass absolute or relative file
 348 /// names. This function simply skips non-UTF8 encoded names.
 349 pub fn scandir<P, F>(
 350     dirfd: RawFd,
 351     path: &P,
 352     regex: &regex::Regex,
 353     mut callback: F,
 354 ) -> Result<(), Error>
 355 where
 356     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 357     P: ?Sized + nix::NixPath,
 358 {
 359     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 360         let entry = entry?;
 361         let file_type = match entry.file_type() {
 362             Some(file_type) => file_type,
 363             None => bail!("unable to detect file type"),
 364         };
 365
 366         callback(
 367             entry.parent_fd(),
 368             unsafe { entry.file_name_utf8_unchecked() },
 369             file_type,
 370         )?;
 371     }
 372     Ok(())
 373 }
 374
 375 /// Shortcut for md5 sums.
 376 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 377     hash(MessageDigest::md5(), data).map_err(Error::from)
 378 }
 379
 380 pub fn get_hardware_address() -> Result<String, Error> {
 381     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 382
 383     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 384     let digest = md5sum(&contents)?;
 385
 386     Ok(proxmox::tools::bin_to_hex(&digest))
 387 }
 388
 389 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 390     if digest1 != digest2 {
 391         bail!("detected modified configuration - file changed by other user? Try again.");
 392     }
 393     Ok(())
 394 }
 395
 396 /// Extract authentication cookie from cookie header.
 397 /// We assume cookie_name is already url encoded.
 398 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 399     for pair in cookie.split(';') {
 400         let (name, value) = match pair.find('=') {
 401             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 402             None => return None, // Cookie format error
 403         };
 404
 405         if name == cookie_name {
 406             use percent_encoding::percent_decode;
 407             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 408                 return Some(value.into());
 409             } else {
 410                 return None; // Cookie format error
 411             }
 412         }
 413     }
 414
 415     None
 416 }
 417
 418 pub fn join(data: &Vec<String>, sep: char) -> String {
 419     let mut list = String::new();
 420
 421     for item in data {
 422         if !list.is_empty() {
 423             list.push(sep);
 424         }
 425         list.push_str(item);
 426     }
 427
 428     list
 429 }
 430
 431 /// normalize uri path
 432 ///
 433 /// Do not allow ".", "..", or hidden files ".XXXX"
 434 /// Also remove empty path components
 435 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 436     let items = path.split('/');
 437
 438     let mut path = String::new();
 439     let mut components = vec![];
 440
 441     for name in items {
 442         if name.is_empty() {
 443             continue;
 444         }
 445         if name.starts_with('.') {
 446             bail!("Path contains illegal components.");
 447         }
 448         path.push('/');
 449         path.push_str(name);
 450         components.push(name);
 451     }
 452
 453     Ok((path, components))
 454 }
 455
 456 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 457     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 458     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 459         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 460     flags.set(FdFlag::FD_CLOEXEC, on);
 461     fcntl(fd, F_SETFD(flags))?;
 462     Ok(())
 463 }
 464
 465 static mut SHUTDOWN_REQUESTED: bool = false;
 466
 467 pub fn request_shutdown() {
 468     unsafe {
 469         SHUTDOWN_REQUESTED = true;
 470     }
 471     crate::server::server_shutdown();
 472 }
 473
 474 #[inline(always)]
 475 pub fn shutdown_requested() -> bool {
 476     unsafe { SHUTDOWN_REQUESTED }
 477 }
 478
 479 pub fn fail_on_shutdown() -> Result<(), Error> {
 480     if shutdown_requested() {
 481         bail!("Server shutdown requested - aborting task");
 482     }
 483     Ok(())
 484 }
 485
 486 /// Guard a raw file descriptor with a drop handler. This is mostly useful when access to an owned
 487 /// `RawFd` is required without the corresponding handler object (such as when only the file
 488 /// descriptor number is required in a closure which may be dropped instead of being executed).
 489 pub struct Fd(pub RawFd);
 490
 491 impl Drop for Fd {
 492     fn drop(&mut self) {
 493         if self.0 != -1 {
 494             unsafe {
 495                 libc::close(self.0);
 496             }
 497         }
 498     }
 499 }
 500
 501 impl AsRawFd for Fd {
 502     fn as_raw_fd(&self) -> RawFd {
 503         self.0
 504     }
 505 }
 506
 507 impl IntoRawFd for Fd {
 508     fn into_raw_fd(mut self) -> RawFd {
 509         let fd = self.0;
 510         self.0 = -1;
 511         fd
 512     }
 513 }
 514
 515 impl FromRawFd for Fd {
 516     unsafe fn from_raw_fd(fd: RawFd) -> Self {
 517         Self(fd)
 518     }
 519 }
 520
 521 // wrap nix::unistd::pipe2 + O_CLOEXEC into something returning guarded file descriptors
 522 pub fn pipe() -> Result<(Fd, Fd), Error> {
 523     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 524     Ok((Fd(pin), Fd(pout)))
 525 }
 526
 527 /// An easy way to convert types to Any
 528 ///
 529 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 530 pub trait AsAny {
 531     fn as_any(&self) -> &dyn Any;
 532 }
 533
 534 impl<T: Any> AsAny for T {
 535     fn as_any(&self) -> &dyn Any {
 536         self
 537     }
 538 }
 539
 540 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 541 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 542     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 543     .add(0x20)
 544     .add(0x7f)
 545     // the DEFAULT_ENCODE_SET added:
 546     .add(b' ')
 547     .add(b'"')
 548     .add(b'#')
 549     .add(b'<')
 550     .add(b'>')
 551     .add(b'`')
 552     .add(b'?')
 553     .add(b'{')
 554     .add(b'}');