src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use failure::*;
   5 use nix::unistd;
   6 use nix::sys::stat;
   7
   8 use lazy_static::lazy_static;
   9
  10 use std::fs::{File, OpenOptions};
  11 use std::io::Write;
  12 use std::path::Path;
  13 use std::io::Read;
  14 use std::io::ErrorKind;
  15 use std::time::Duration;
  16
  17 use std::os::unix::io::RawFd;
  18 use std::os::unix::io::AsRawFd;
  19
  20 use std::collections::HashMap;
  21
  22 use serde_json::Value;
  23
  24 pub mod timer;
  25 pub mod wrapped_reader_stream;
  26 #[macro_use]
  27 pub mod common_regex;
  28 pub mod ticket;
  29 pub mod borrow;
  30 pub mod fs;
  31 pub mod tty;
  32 pub mod signalfd;
  33
  34 #[macro_use]
  35 mod file_logger;
  36 pub use file_logger::*;
  37
  38 /// Macro to write error-handling blocks (like perl eval {})
  39 ///
  40 /// #### Example:
  41 /// ```
  42 /// # #[macro_use] extern crate proxmox_backup;
  43 /// # use failure::*;
  44 /// # let some_condition = false;
  45 /// let result = try_block!({
  46 ///     if (some_condition) {
  47 ///         bail!("some error");
  48 ///     }
  49 ///     Ok(())
  50 /// })
  51 /// .map_err(|e| format_err!("my try block returned an error - {}", e));
  52 /// ```
  53
  54 #[macro_export]
  55 macro_rules! try_block {
  56     { $($token:tt)* } => {{ (|| -> Result<_,_> { $($token)* })() }}
  57 }
  58
  59
  60 /// The `BufferedRead` trait provides a single function
  61 /// `buffered_read`. It returns a reference to an internal buffer. The
  62 /// purpose of this traid is to avoid unnecessary data copies.
  63 pub trait BufferedRead {
  64     /// This functions tries to fill the internal buffers, then
  65     /// returns a reference to the available data. It returns an empty
  66     /// buffer if `offset` points to the end of the file.
  67     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  68 }
  69
  70 /// Directly map a type into a binary buffer. This is mostly useful
  71 /// for reading structured data from a byte stream (file). You need to
  72 /// make sure that the buffer location does not change, so please
  73 /// avoid vec resize while you use such map.
  74 ///
  75 /// This function panics if the buffer is not large enough.
  76 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  77     if buffer.len() < ::std::mem::size_of::<T>() {
  78         bail!("unable to map struct - buffer too small");
  79     }
  80     Ok(unsafe { & * (buffer.as_ptr() as *const T) })
  81 }
  82
  83 /// Directly map a type into a mutable binary buffer. This is mostly
  84 /// useful for writing structured data into a byte stream (file). You
  85 /// need to make sure that the buffer location does not change, so
  86 /// please avoid vec resize while you use such map.
  87 ///
  88 /// This function panics if the buffer is not large enough.
  89 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  90     if buffer.len() < ::std::mem::size_of::<T>() {
  91         bail!("unable to map struct - buffer too small");
  92     }
  93     Ok(unsafe { &mut * (buffer.as_ptr() as *mut T) })
  94 }
  95
  96 pub fn file_read_firstline<P: AsRef<Path>>(path: P) -> Result<String, Error> {
  97
  98     let path = path.as_ref();
  99
 100     try_block!({
 101         let file = std::fs::File::open(path)?;
 102
 103         use std::io::{BufRead, BufReader};
 104
 105         let mut reader = BufReader::new(file);
 106
 107         let mut line = String::new();
 108
 109         let _ = reader.read_line(&mut line)?;
 110
 111         Ok(line)
 112     }).map_err(|err: Error| format_err!("unable to read {:?} - {}", path, err))
 113 }
 114
 115 pub fn file_get_contents<P: AsRef<Path>>(path: P) -> Result<Vec<u8>, Error> {
 116
 117     let path = path.as_ref();
 118
 119     try_block!({
 120         std::fs::read(path)
 121     }).map_err(|err| format_err!("unable to read {:?} - {}", path, err))
 122 }
 123
 124 pub fn file_get_json<P: AsRef<Path>>(path: P) -> Result<Value, Error> {
 125
 126     let path = path.as_ref();
 127
 128     let raw = file_get_contents(path)?;
 129
 130     try_block!({
 131         let data = String::from_utf8(raw)?;
 132         let json = serde_json::from_str(&data)?;
 133         Ok(json)
 134     }).map_err(|err: Error| format_err!("unable to read json from {:?} - {}", path, err))
 135 }
 136
 137 /// Atomically write a file. We first create a temporary file, which
 138 /// is then renamed.
 139 pub fn file_set_contents<P: AsRef<Path>>(
 140     path: P,
 141     data: &[u8],
 142     perm: Option<stat::Mode>,
 143 ) -> Result<(), Error> {
 144
 145     let path = path.as_ref();
 146
 147     // Note: we use mkstemp heŕe, because this worka with different
 148     // processes, threads, and even tokio tasks.
 149     let mut template = path.to_owned();
 150     template.set_extension("tmp_XXXXXX");
 151     let (fd, tmp_path) = match unistd::mkstemp(&template) {
 152         Ok((fd, path)) => (fd, path),
 153         Err(err) => bail!("mkstemp {:?} failed: {}", template, err),
 154     };
 155
 156     let tmp_path = tmp_path.as_path();
 157
 158     let mode : stat::Mode = perm.unwrap_or(stat::Mode::from(
 159         stat::Mode::S_IRUSR | stat::Mode::S_IWUSR |
 160         stat::Mode::S_IRGRP | stat::Mode::S_IROTH
 161     ));
 162
 163     if let Err(err) = stat::fchmod(fd, mode) {
 164         let _ = unistd::unlink(tmp_path);
 165         bail!("fchmod {:?} failed: {}", tmp_path, err);
 166     }
 167
 168     use std::os::unix::io::FromRawFd;
 169     let mut file = unsafe { File::from_raw_fd(fd) };
 170
 171     if let Err(err) = file.write_all(data) {
 172         let _ = unistd::unlink(tmp_path);
 173         bail!("write failed: {}", err);
 174     }
 175
 176     if let Err(err) = std::fs::rename(tmp_path, path) {
 177         let _ = unistd::unlink(tmp_path);
 178         bail!("Atomic rename failed for file {:?} - {}", path, err);
 179     }
 180
 181     Ok(())
 182 }
 183
 184 /// Create a file lock using fntl. This function allows you to specify
 185 /// a timeout if you want to avoid infinite blocking.
 186 pub fn lock_file<F: AsRawFd>(
 187     file: &mut F,
 188     exclusive: bool,
 189     timeout: Option<Duration>,
 190     ) -> Result<(), Error>
 191 {
 192     let lockarg =
 193         if exclusive {
 194             nix::fcntl::FlockArg::LockExclusive
 195         } else {
 196             nix::fcntl::FlockArg::LockShared
 197         };
 198
 199     let timeout = match timeout {
 200         None => {
 201             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 202             return Ok(());
 203         }
 204         Some(t) => t,
 205     };
 206
 207     // unblock the timeout signal temporarily
 208     let _sigblock_guard = timer::unblock_timeout_signal();
 209
 210     // setup a timeout timer
 211     let mut timer = timer::Timer::create(
 212         timer::Clock::Realtime,
 213         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT))?;
 214
 215     timer.arm(timer::TimerSpec::new()
 216         .value(Some(timeout))
 217         .interval(Some(Duration::from_millis(10))))?;
 218
 219     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 220     Ok(())
 221 }
 222
 223 /// Open or create a lock file (append mode). Then try to
 224 /// aquire a lock using `lock_file()`.
 225 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration)
 226     -> Result<File, Error>
 227 {
 228     let path = path.as_ref();
 229     let mut file =
 230         match OpenOptions::new()
 231             .create(true)
 232             .append(true)
 233             .open(path)
 234         {
 235             Ok(file) => file,
 236             Err(err) => bail!("Unable to open lock {:?} - {}",
 237                               path, err),
 238         };
 239     match lock_file(&mut file, true, Some(timeout)) {
 240         Ok(_) => Ok(file),
 241         Err(err) => bail!("Unable to aquire lock {:?} - {}",
 242                           path, err),
 243     }
 244 }
 245
 246 /// Split a file into equal sized chunks. The last chunk may be
 247 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 248 /// cannot return a borrowed buffer ref (we want zero-copy)
 249 pub fn file_chunker<C, R>(
 250     mut file: R,
 251     chunk_size: usize,
 252     mut chunk_cb: C
 253 ) -> Result<(), Error>
 254     where C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 255           R: Read,
 256 {
 257
 258     const READ_BUFFER_SIZE: usize = 4*1024*1024; // 4M
 259
 260     if chunk_size > READ_BUFFER_SIZE { bail!("chunk size too large!"); }
 261
 262     let mut buf = vec![0u8; READ_BUFFER_SIZE];
 263
 264     let mut pos = 0;
 265     let mut file_pos = 0;
 266     loop {
 267         let mut eof = false;
 268         let mut tmp = &mut buf[..];
 269        // try to read large portions, at least chunk_size
 270         while pos < chunk_size {
 271             match file.read(tmp) {
 272                 Ok(0) => { eof = true; break; },
 273                 Ok(n) => {
 274                     pos += n;
 275                     if pos > chunk_size { break; }
 276                     tmp = &mut tmp[n..];
 277                 }
 278                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 279                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 280             }
 281         }
 282         let mut start = 0;
 283         while start + chunk_size <= pos {
 284             if !(chunk_cb)(file_pos, &buf[start..start+chunk_size])? { break; }
 285             file_pos += chunk_size;
 286             start += chunk_size;
 287         }
 288         if eof {
 289             if start < pos {
 290                 (chunk_cb)(file_pos, &buf[start..pos])?;
 291                 //file_pos += pos - start;
 292             }
 293             break;
 294         } else {
 295             let rest = pos - start;
 296             if rest > 0 {
 297                 let ptr = buf.as_mut_ptr();
 298                 unsafe { std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest); }
 299                 pos = rest;
 300             } else {
 301                 pos = 0;
 302             }
 303         }
 304     }
 305
 306     Ok(())
 307 }
 308
 309 // Returns the Unix uid/gid for the sepcified system user.
 310 pub fn getpwnam_ugid(username: &str) -> Result<(libc::uid_t,libc::gid_t), Error> {
 311     let info = unsafe { libc::getpwnam(std::ffi::CString::new(username).unwrap().as_ptr()) };
 312     if info == std::ptr::null_mut() {
 313         bail!("getwpnam '{}' failed", username);
 314     }
 315
 316     let info = unsafe { *info };
 317
 318     Ok((info.pw_uid, info.pw_gid))
 319 }
 320
 321 // Returns the hosts node name (UTS node name)
 322 pub fn nodename() -> &'static str {
 323
 324     lazy_static!{
 325         static ref NODENAME: String = {
 326
 327             nix::sys::utsname::uname()
 328                 .nodename()
 329                 .split('.')
 330                 .next()
 331                 .unwrap()
 332                 .to_owned()
 333         };
 334     }
 335
 336     &NODENAME
 337 }
 338
 339 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 340
 341     let mut query = url::form_urlencoded::Serializer::new(String::new());
 342
 343     let object = data.as_object().ok_or_else(|| {
 344         format_err!("json_object_to_query: got wrong data type (expected object).")
 345     })?;
 346
 347     for (key, value) in object {
 348         match value {
 349             Value::Bool(b) => { query.append_pair(key, &b.to_string()); }
 350             Value::Number(n) => { query.append_pair(key, &n.to_string()); }
 351             Value::String(s) => { query.append_pair(key, &s); }
 352             Value::Array(arr) => {
 353                 for element in arr {
 354                     match element {
 355                         Value::Bool(b) => { query.append_pair(key, &b.to_string()); }
 356                         Value::Number(n) => { query.append_pair(key, &n.to_string()); }
 357                         Value::String(s) => { query.append_pair(key, &s); }
 358                         _ => bail!("json_object_to_query: unable to handle complex array data types."),
 359                     }
 360                 }
 361             }
 362             _ => bail!("json_object_to_query: unable to handle complex data types."),
 363         }
 364     }
 365
 366     Ok(query.finish())
 367 }
 368
 369 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 370     match param[name].as_str()   {
 371         Some(s) => Ok(s),
 372         None => bail!("missing parameter '{}'", name),
 373     }
 374 }
 375
 376 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 377     match param[name].as_i64()   {
 378         Some(s) => Ok(s),
 379         None => bail!("missing parameter '{}'", name),
 380     }
 381 }
 382
 383 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 384     match param[name].as_array()   {
 385         Some(s) => Ok(s.to_vec()),
 386         None => bail!("missing parameter '{}'", name),
 387     }
 388 }
 389
 390 pub fn complete_file_name(arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 391
 392     let mut result = vec![];
 393
 394     use nix::fcntl::OFlag;
 395     use nix::sys::stat::Mode;
 396     use nix::fcntl::AtFlags;
 397
 398     let mut dirname = std::path::PathBuf::from(if arg.len() == 0 { "./" } else { arg });
 399
 400     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 401         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 402         Err(_) => false,
 403     };
 404
 405     if !is_dir {
 406         if let Some(parent) = dirname.parent() {
 407             dirname = parent.to_owned();
 408         }
 409     }
 410
 411     let mut dir = match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 412         Ok(d) => d,
 413         Err(_) => return result,
 414     };
 415
 416     for item in dir.iter() {
 417         if let Ok(entry) = item {
 418             if let Ok(name) = entry.file_name().to_str() {
 419                 if name == "." || name == ".." { continue; }
 420                 let mut newpath = dirname.clone();
 421                 newpath.push(name);
 422
 423                 if let Ok(stat) = nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty()) {
 424                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 425                         newpath.push("");
 426                         if let Some(newpath) = newpath.to_str() {
 427                             result.push(newpath.to_owned());
 428                         }
 429                         continue;
 430                      }
 431                 }
 432                 if let Some(newpath) = newpath.to_str() {
 433                     result.push(newpath.to_owned());
 434                 }
 435
 436              }
 437         }
 438     }
 439
 440     result
 441 }
 442
 443 /// Scan directory for matching file names.
 444 ///
 445 /// Scan through all directory entries and call `callback()` function
 446 /// if the entry name matches the regular expression. This function
 447 /// used unix `openat()`, so you can pass absolute or relative file
 448 /// names. This function simply skips non-UTF8 encoded names.
 449 pub fn scandir<P, F>(
 450     dirfd: RawFd,
 451     path: &P,
 452     regex: &regex::Regex,
 453     mut callback: F
 454 ) -> Result<(), Error>
 455     where F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 456           P: ?Sized + nix::NixPath,
 457 {
 458     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 459         let entry = entry?;
 460         let file_type = match entry.file_type() {
 461             Some(file_type) => file_type,
 462             None => bail!("unable to detect file type"),
 463         };
 464
 465         callback(entry.parent_fd(), unsafe { entry.file_name_utf8_unchecked() }, file_type)?;
 466     }
 467     Ok(())
 468 }
 469
 470 pub fn get_hardware_address() -> Result<String, Error> {
 471
 472     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 473
 474     let contents = file_get_contents(FILENAME)?;
 475     let digest = md5::compute(contents);
 476
 477     Ok(format!("{:0x}", digest))
 478 }
 479
 480 pub fn digest_to_hex(digest: &[u8]) -> String {
 481
 482     const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef";
 483
 484     let mut buf = Vec::<u8>::with_capacity(digest.len()*2);
 485
 486     for i in 0..digest.len() {
 487         buf.push(HEX_CHARS[(digest[i] >> 4) as usize]);
 488         buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]);
 489     }
 490
 491     unsafe { String::from_utf8_unchecked(buf) }
 492 }
 493
 494 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 495     if digest1 != digest2 {
 496         bail!("detected modified configuration - file changed by other user? Try again.");
 497     }
 498     Ok(())
 499 }
 500
 501 /// Extract authentication cookie from cookie header.
 502 /// We assume cookie_name is already url encoded.
 503 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 504
 505     for pair in cookie.split(';') {
 506
 507         let (name, value) = match pair.find('=') {
 508             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 509             None => return None, // Cookie format error
 510         };
 511
 512         if name == cookie_name {
 513             use url::percent_encoding::percent_decode;
 514             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 515                 return Some(value.into());
 516             } else {
 517                 return None; // Cookie format error
 518             }
 519         }
 520     }
 521
 522     None
 523 }
 524
 525 pub fn join(data: &Vec<String>, sep: char) -> String {
 526
 527     let mut list = String::new();
 528
 529     for item in data {
 530         if list.len() != 0 { list.push(sep); }
 531         list.push_str(item);
 532     }
 533
 534     list
 535 }