src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::{File, OpenOptions};
   8 use std::io::ErrorKind;
   9 use std::io::Read;
  10 use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
  11 use std::path::Path;
  12 use std::time::Duration;
  13
  14 use failure::*;
  15 use serde_json::Value;
  16 use openssl::hash::{hash, DigestBytes, MessageDigest};
  17
  18 use proxmox::tools::vec;
  19
  20 pub mod acl;
  21 pub mod async_io;
  22 pub mod async_mutex;
  23 pub mod borrow;
  24 pub mod daemon;
  25 pub mod fs;
  26 pub mod futures;
  27 pub mod runtime;
  28 pub mod ticket;
  29 pub mod timer;
  30 pub mod tty;
  31 pub mod wrapped_reader_stream;
  32 pub mod xattr;
  33
  34 mod process_locker;
  35 pub use process_locker::*;
  36
  37 mod file_logger;
  38 pub use file_logger::*;
  39
  40 mod broadcast_future;
  41 pub use broadcast_future::*;
  42
  43 /// The `BufferedRead` trait provides a single function
  44 /// `buffered_read`. It returns a reference to an internal buffer. The
  45 /// purpose of this traid is to avoid unnecessary data copies.
  46 pub trait BufferedRead {
  47     /// This functions tries to fill the internal buffers, then
  48     /// returns a reference to the available data. It returns an empty
  49     /// buffer if `offset` points to the end of the file.
  50     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  51 }
  52
  53 /// Directly map a type into a binary buffer. This is mostly useful
  54 /// for reading structured data from a byte stream (file). You need to
  55 /// make sure that the buffer location does not change, so please
  56 /// avoid vec resize while you use such map.
  57 ///
  58 /// This function panics if the buffer is not large enough.
  59 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  60     if buffer.len() < ::std::mem::size_of::<T>() {
  61         bail!("unable to map struct - buffer too small");
  62     }
  63     Ok(unsafe { &*(buffer.as_ptr() as *const T) })
  64 }
  65
  66 /// Directly map a type into a mutable binary buffer. This is mostly
  67 /// useful for writing structured data into a byte stream (file). You
  68 /// need to make sure that the buffer location does not change, so
  69 /// please avoid vec resize while you use such map.
  70 ///
  71 /// This function panics if the buffer is not large enough.
  72 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  73     if buffer.len() < ::std::mem::size_of::<T>() {
  74         bail!("unable to map struct - buffer too small");
  75     }
  76     Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) })
  77 }
  78
  79 /// Create a file lock using fntl. This function allows you to specify
  80 /// a timeout if you want to avoid infinite blocking.
  81 pub fn lock_file<F: AsRawFd>(
  82     file: &mut F,
  83     exclusive: bool,
  84     timeout: Option<Duration>,
  85 ) -> Result<(), Error> {
  86     let lockarg = if exclusive {
  87         nix::fcntl::FlockArg::LockExclusive
  88     } else {
  89         nix::fcntl::FlockArg::LockShared
  90     };
  91
  92     let timeout = match timeout {
  93         None => {
  94             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
  95             return Ok(());
  96         }
  97         Some(t) => t,
  98     };
  99
 100     // unblock the timeout signal temporarily
 101     let _sigblock_guard = timer::unblock_timeout_signal();
 102
 103     // setup a timeout timer
 104     let mut timer = timer::Timer::create(
 105         timer::Clock::Realtime,
 106         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT),
 107     )?;
 108
 109     timer.arm(
 110         timer::TimerSpec::new()
 111             .value(Some(timeout))
 112             .interval(Some(Duration::from_millis(10))),
 113     )?;
 114
 115     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 116     Ok(())
 117 }
 118
 119 /// Open or create a lock file (append mode). Then try to
 120 /// aquire a lock using `lock_file()`.
 121 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<File, Error> {
 122     let path = path.as_ref();
 123     let mut file = match OpenOptions::new().create(true).append(true).open(path) {
 124         Ok(file) => file,
 125         Err(err) => bail!("Unable to open lock {:?} - {}", path, err),
 126     };
 127     match lock_file(&mut file, true, Some(timeout)) {
 128         Ok(_) => Ok(file),
 129         Err(err) => bail!("Unable to aquire lock {:?} - {}", path, err),
 130     }
 131 }
 132
 133 /// Split a file into equal sized chunks. The last chunk may be
 134 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 135 /// cannot return a borrowed buffer ref (we want zero-copy)
 136 pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error>
 137 where
 138     C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 139     R: Read,
 140 {
 141     const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M
 142
 143     if chunk_size > READ_BUFFER_SIZE {
 144         bail!("chunk size too large!");
 145     }
 146
 147     let mut buf = vec::undefined(READ_BUFFER_SIZE);
 148
 149     let mut pos = 0;
 150     let mut file_pos = 0;
 151     loop {
 152         let mut eof = false;
 153         let mut tmp = &mut buf[..];
 154         // try to read large portions, at least chunk_size
 155         while pos < chunk_size {
 156             match file.read(tmp) {
 157                 Ok(0) => {
 158                     eof = true;
 159                     break;
 160                 }
 161                 Ok(n) => {
 162                     pos += n;
 163                     if pos > chunk_size {
 164                         break;
 165                     }
 166                     tmp = &mut tmp[n..];
 167                 }
 168                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 169                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 170             }
 171         }
 172         let mut start = 0;
 173         while start + chunk_size <= pos {
 174             if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? {
 175                 break;
 176             }
 177             file_pos += chunk_size;
 178             start += chunk_size;
 179         }
 180         if eof {
 181             if start < pos {
 182                 (chunk_cb)(file_pos, &buf[start..pos])?;
 183                 //file_pos += pos - start;
 184             }
 185             break;
 186         } else {
 187             let rest = pos - start;
 188             if rest > 0 {
 189                 let ptr = buf.as_mut_ptr();
 190                 unsafe {
 191                     std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest);
 192                 }
 193                 pos = rest;
 194             } else {
 195                 pos = 0;
 196             }
 197         }
 198     }
 199
 200     Ok(())
 201 }
 202
 203 /// Returns the Unix uid/gid for the sepcified system user.
 204 pub fn getpwnam_ugid(username: &str) -> Result<(libc::uid_t, libc::gid_t), Error> {
 205     let c_username = std::ffi::CString::new(username).unwrap();
 206     let info = unsafe { libc::getpwnam(c_username.as_ptr()) };
 207     if info.is_null() {
 208         bail!("getwpnam '{}' failed", username);
 209     }
 210
 211     let info = unsafe { *info };
 212
 213     Ok((info.pw_uid, info.pw_gid))
 214 }
 215
 216 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 217     let mut query = url::form_urlencoded::Serializer::new(String::new());
 218
 219     let object = data.as_object().ok_or_else(|| {
 220         format_err!("json_object_to_query: got wrong data type (expected object).")
 221     })?;
 222
 223     for (key, value) in object {
 224         match value {
 225             Value::Bool(b) => {
 226                 query.append_pair(key, &b.to_string());
 227             }
 228             Value::Number(n) => {
 229                 query.append_pair(key, &n.to_string());
 230             }
 231             Value::String(s) => {
 232                 query.append_pair(key, &s);
 233             }
 234             Value::Array(arr) => {
 235                 for element in arr {
 236                     match element {
 237                         Value::Bool(b) => {
 238                             query.append_pair(key, &b.to_string());
 239                         }
 240                         Value::Number(n) => {
 241                             query.append_pair(key, &n.to_string());
 242                         }
 243                         Value::String(s) => {
 244                             query.append_pair(key, &s);
 245                         }
 246                         _ => bail!(
 247                             "json_object_to_query: unable to handle complex array data types."
 248                         ),
 249                     }
 250                 }
 251             }
 252             _ => bail!("json_object_to_query: unable to handle complex data types."),
 253         }
 254     }
 255
 256     Ok(query.finish())
 257 }
 258
 259 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 260     match param[name].as_str() {
 261         Some(s) => Ok(s),
 262         None => bail!("missing parameter '{}'", name),
 263     }
 264 }
 265
 266 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 267     match param[name].as_str() {
 268         Some(s) => Ok(s),
 269         None => bail!("missing property '{}'", name),
 270     }
 271 }
 272
 273 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 274     match param[name].as_i64() {
 275         Some(s) => Ok(s),
 276         None => bail!("missing parameter '{}'", name),
 277     }
 278 }
 279
 280 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 281     match param[name].as_i64() {
 282         Some(s) => Ok(s),
 283         None => bail!("missing property '{}'", name),
 284     }
 285 }
 286
 287 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 288     match param[name].as_array() {
 289         Some(s) => Ok(s.to_vec()),
 290         None => bail!("missing parameter '{}'", name),
 291     }
 292 }
 293
 294 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 295     match param[name].as_array() {
 296         Some(s) => Ok(s.to_vec()),
 297         None => bail!("missing property '{}'", name),
 298     }
 299 }
 300
 301 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 302     let mut result = vec![];
 303
 304     use nix::fcntl::AtFlags;
 305     use nix::fcntl::OFlag;
 306     use nix::sys::stat::Mode;
 307
 308     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 309
 310     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 311         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 312         Err(_) => false,
 313     };
 314
 315     if !is_dir {
 316         if let Some(parent) = dirname.parent() {
 317             dirname = parent.to_owned();
 318         }
 319     }
 320
 321     let mut dir =
 322         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 323             Ok(d) => d,
 324             Err(_) => return result,
 325         };
 326
 327     for item in dir.iter() {
 328         if let Ok(entry) = item {
 329             if let Ok(name) = entry.file_name().to_str() {
 330                 if name == "." || name == ".." {
 331                     continue;
 332                 }
 333                 let mut newpath = dirname.clone();
 334                 newpath.push(name);
 335
 336                 if let Ok(stat) =
 337                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 338                 {
 339                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 340                         newpath.push("");
 341                         if let Some(newpath) = newpath.to_str() {
 342                             result.push(newpath.to_owned());
 343                         }
 344                         continue;
 345                     }
 346                 }
 347                 if let Some(newpath) = newpath.to_str() {
 348                     result.push(newpath.to_owned());
 349                 }
 350             }
 351         }
 352     }
 353
 354     result
 355 }
 356
 357 /// Scan directory for matching file names.
 358 ///
 359 /// Scan through all directory entries and call `callback()` function
 360 /// if the entry name matches the regular expression. This function
 361 /// used unix `openat()`, so you can pass absolute or relative file
 362 /// names. This function simply skips non-UTF8 encoded names.
 363 pub fn scandir<P, F>(
 364     dirfd: RawFd,
 365     path: &P,
 366     regex: &regex::Regex,
 367     mut callback: F,
 368 ) -> Result<(), Error>
 369 where
 370     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 371     P: ?Sized + nix::NixPath,
 372 {
 373     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 374         let entry = entry?;
 375         let file_type = match entry.file_type() {
 376             Some(file_type) => file_type,
 377             None => bail!("unable to detect file type"),
 378         };
 379
 380         callback(
 381             entry.parent_fd(),
 382             unsafe { entry.file_name_utf8_unchecked() },
 383             file_type,
 384         )?;
 385     }
 386     Ok(())
 387 }
 388
 389 /// Shortcut for md5 sums.
 390 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 391     hash(MessageDigest::md5(), data).map_err(Error::from)
 392 }
 393
 394 pub fn get_hardware_address() -> Result<String, Error> {
 395     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 396
 397     let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
 398     let digest = md5sum(&contents)?;
 399
 400     Ok(proxmox::tools::bin_to_hex(&digest))
 401 }
 402
 403 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 404     if digest1 != digest2 {
 405         bail!("detected modified configuration - file changed by other user? Try again.");
 406     }
 407     Ok(())
 408 }
 409
 410 /// Extract authentication cookie from cookie header.
 411 /// We assume cookie_name is already url encoded.
 412 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 413     for pair in cookie.split(';') {
 414         let (name, value) = match pair.find('=') {
 415             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 416             None => return None, // Cookie format error
 417         };
 418
 419         if name == cookie_name {
 420             use url::percent_encoding::percent_decode;
 421             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 422                 return Some(value.into());
 423             } else {
 424                 return None; // Cookie format error
 425             }
 426         }
 427     }
 428
 429     None
 430 }
 431
 432 pub fn join(data: &Vec<String>, sep: char) -> String {
 433     let mut list = String::new();
 434
 435     for item in data {
 436         if !list.is_empty() {
 437             list.push(sep);
 438         }
 439         list.push_str(item);
 440     }
 441
 442     list
 443 }
 444
 445 /// normalize uri path
 446 ///
 447 /// Do not allow ".", "..", or hidden files ".XXXX"
 448 /// Also remove empty path components
 449 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 450     let items = path.split('/');
 451
 452     let mut path = String::new();
 453     let mut components = vec![];
 454
 455     for name in items {
 456         if name.is_empty() {
 457             continue;
 458         }
 459         if name.starts_with('.') {
 460             bail!("Path contains illegal components.");
 461         }
 462         path.push('/');
 463         path.push_str(name);
 464         components.push(name);
 465     }
 466
 467     Ok((path, components))
 468 }
 469
 470 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 471     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 472     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 473         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 474     flags.set(FdFlag::FD_CLOEXEC, on);
 475     fcntl(fd, F_SETFD(flags))?;
 476     Ok(())
 477 }
 478
 479 static mut SHUTDOWN_REQUESTED: bool = false;
 480
 481 pub fn request_shutdown() {
 482     unsafe {
 483         SHUTDOWN_REQUESTED = true;
 484     }
 485     crate::server::server_shutdown();
 486 }
 487
 488 #[inline(always)]
 489 pub fn shutdown_requested() -> bool {
 490     unsafe { SHUTDOWN_REQUESTED }
 491 }
 492
 493 pub fn fail_on_shutdown() -> Result<(), Error> {
 494     if shutdown_requested() {
 495         bail!("Server shutdown requested - aborting task");
 496     }
 497     Ok(())
 498 }
 499
 500 /// Guard a raw file descriptor with a drop handler. This is mostly useful when access to an owned
 501 /// `RawFd` is required without the corresponding handler object (such as when only the file
 502 /// descriptor number is required in a closure which may be dropped instead of being executed).
 503 pub struct Fd(pub RawFd);
 504
 505 impl Drop for Fd {
 506     fn drop(&mut self) {
 507         if self.0 != -1 {
 508             unsafe {
 509                 libc::close(self.0);
 510             }
 511         }
 512     }
 513 }
 514
 515 impl AsRawFd for Fd {
 516     fn as_raw_fd(&self) -> RawFd {
 517         self.0
 518     }
 519 }
 520
 521 impl IntoRawFd for Fd {
 522     fn into_raw_fd(mut self) -> RawFd {
 523         let fd = self.0;
 524         self.0 = -1;
 525         fd
 526     }
 527 }
 528
 529 impl FromRawFd for Fd {
 530     unsafe fn from_raw_fd(fd: RawFd) -> Self {
 531         Self(fd)
 532     }
 533 }
 534
 535 // wrap nix::unistd::pipe2 + O_CLOEXEC into something returning guarded file descriptors
 536 pub fn pipe() -> Result<(Fd, Fd), Error> {
 537     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 538     Ok((Fd(pin), Fd(pout)))
 539 }
 540
 541 /// An easy way to convert types to Any
 542 ///
 543 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 544 pub trait AsAny {
 545     fn as_any(&self) -> &dyn Any;
 546 }
 547
 548 impl<T: Any> AsAny for T {
 549     fn as_any(&self) -> &dyn Any {
 550         self
 551     }
 552 }