src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use failure::*;
   5 use nix::unistd;
   6 use nix::sys::stat;
   7
   8 use lazy_static::lazy_static;
   9
  10 use std::fs::{File, OpenOptions};
  11 use std::io::Write;
  12 use std::path::Path;
  13 use std::io::Read;
  14 use std::io::ErrorKind;
  15 use std::time::Duration;
  16
  17 use std::os::unix::io::RawFd;
  18 use std::os::unix::io::AsRawFd;
  19
  20 use std::collections::HashMap;
  21
  22 use serde_json::Value;
  23
  24 pub mod timer;
  25 pub mod wrapped_reader_stream;
  26 #[macro_use]
  27 pub mod common_regex;
  28 pub mod ticket;
  29 pub mod borrow;
  30 pub mod fs;
  31 pub mod tty;
  32 pub mod signalfd;
  33 pub mod daemon;
  34
  35 mod process_locker;
  36 pub use process_locker::*;
  37
  38 #[macro_use]
  39 mod file_logger;
  40 pub use file_logger::*;
  41
  42 /// Macro to write error-handling blocks (like perl eval {})
  43 ///
  44 /// #### Example:
  45 /// ```
  46 /// # #[macro_use] extern crate proxmox_backup;
  47 /// # use failure::*;
  48 /// # let some_condition = false;
  49 /// let result = try_block!({
  50 ///     if (some_condition) {
  51 ///         bail!("some error");
  52 ///     }
  53 ///     Ok(())
  54 /// })
  55 /// .map_err(|e| format_err!("my try block returned an error - {}", e));
  56 /// ```
  57
  58 #[macro_export]
  59 macro_rules! try_block {
  60     { $($token:tt)* } => {{ (|| -> Result<_,_> { $($token)* })() }}
  61 }
  62
  63
  64 /// The `BufferedRead` trait provides a single function
  65 /// `buffered_read`. It returns a reference to an internal buffer. The
  66 /// purpose of this traid is to avoid unnecessary data copies.
  67 pub trait BufferedRead {
  68     /// This functions tries to fill the internal buffers, then
  69     /// returns a reference to the available data. It returns an empty
  70     /// buffer if `offset` points to the end of the file.
  71     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  72 }
  73
  74 /// Directly map a type into a binary buffer. This is mostly useful
  75 /// for reading structured data from a byte stream (file). You need to
  76 /// make sure that the buffer location does not change, so please
  77 /// avoid vec resize while you use such map.
  78 ///
  79 /// This function panics if the buffer is not large enough.
  80 pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> {
  81     if buffer.len() < ::std::mem::size_of::<T>() {
  82         bail!("unable to map struct - buffer too small");
  83     }
  84     Ok(unsafe { & * (buffer.as_ptr() as *const T) })
  85 }
  86
  87 /// Directly map a type into a mutable binary buffer. This is mostly
  88 /// useful for writing structured data into a byte stream (file). You
  89 /// need to make sure that the buffer location does not change, so
  90 /// please avoid vec resize while you use such map.
  91 ///
  92 /// This function panics if the buffer is not large enough.
  93 pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> {
  94     if buffer.len() < ::std::mem::size_of::<T>() {
  95         bail!("unable to map struct - buffer too small");
  96     }
  97     Ok(unsafe { &mut * (buffer.as_ptr() as *mut T) })
  98 }
  99
 100 pub fn file_read_firstline<P: AsRef<Path>>(path: P) -> Result<String, Error> {
 101
 102     let path = path.as_ref();
 103
 104     try_block!({
 105         let file = std::fs::File::open(path)?;
 106
 107         use std::io::{BufRead, BufReader};
 108
 109         let mut reader = BufReader::new(file);
 110
 111         let mut line = String::new();
 112
 113         let _ = reader.read_line(&mut line)?;
 114
 115         Ok(line)
 116     }).map_err(|err: Error| format_err!("unable to read {:?} - {}", path, err))
 117 }
 118
 119 pub fn file_get_contents<P: AsRef<Path>>(path: P) -> Result<Vec<u8>, Error> {
 120
 121     let path = path.as_ref();
 122
 123     try_block!({
 124         std::fs::read(path)
 125     }).map_err(|err| format_err!("unable to read {:?} - {}", path, err))
 126 }
 127
 128 pub fn file_get_json<P: AsRef<Path>>(path: P) -> Result<Value, Error> {
 129
 130     let path = path.as_ref();
 131
 132     let raw = file_get_contents(path)?;
 133
 134     try_block!({
 135         let data = String::from_utf8(raw)?;
 136         let json = serde_json::from_str(&data)?;
 137         Ok(json)
 138     }).map_err(|err: Error| format_err!("unable to read json from {:?} - {}", path, err))
 139 }
 140
 141 /// Atomically write a file. We first create a temporary file, which
 142 /// is then renamed.
 143 pub fn file_set_contents<P: AsRef<Path>>(
 144     path: P,
 145     data: &[u8],
 146     perm: Option<stat::Mode>,
 147 ) -> Result<(), Error> {
 148
 149     let path = path.as_ref();
 150
 151     // Note: we use mkstemp heŕe, because this worka with different
 152     // processes, threads, and even tokio tasks.
 153     let mut template = path.to_owned();
 154     template.set_extension("tmp_XXXXXX");
 155     let (fd, tmp_path) = match unistd::mkstemp(&template) {
 156         Ok((fd, path)) => (fd, path),
 157         Err(err) => bail!("mkstemp {:?} failed: {}", template, err),
 158     };
 159
 160     let tmp_path = tmp_path.as_path();
 161
 162     let mode : stat::Mode = perm.unwrap_or(stat::Mode::from(
 163         stat::Mode::S_IRUSR | stat::Mode::S_IWUSR |
 164         stat::Mode::S_IRGRP | stat::Mode::S_IROTH
 165     ));
 166
 167     if let Err(err) = stat::fchmod(fd, mode) {
 168         let _ = unistd::unlink(tmp_path);
 169         bail!("fchmod {:?} failed: {}", tmp_path, err);
 170     }
 171
 172     use std::os::unix::io::FromRawFd;
 173     let mut file = unsafe { File::from_raw_fd(fd) };
 174
 175     if let Err(err) = file.write_all(data) {
 176         let _ = unistd::unlink(tmp_path);
 177         bail!("write failed: {}", err);
 178     }
 179
 180     if let Err(err) = std::fs::rename(tmp_path, path) {
 181         let _ = unistd::unlink(tmp_path);
 182         bail!("Atomic rename failed for file {:?} - {}", path, err);
 183     }
 184
 185     Ok(())
 186 }
 187
 188 /// Create a file lock using fntl. This function allows you to specify
 189 /// a timeout if you want to avoid infinite blocking.
 190 pub fn lock_file<F: AsRawFd>(
 191     file: &mut F,
 192     exclusive: bool,
 193     timeout: Option<Duration>,
 194 ) -> Result<(), Error> {
 195     let lockarg =
 196         if exclusive {
 197             nix::fcntl::FlockArg::LockExclusive
 198         } else {
 199             nix::fcntl::FlockArg::LockShared
 200         };
 201
 202     let timeout = match timeout {
 203         None => {
 204             nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 205             return Ok(());
 206         }
 207         Some(t) => t,
 208     };
 209
 210     // unblock the timeout signal temporarily
 211     let _sigblock_guard = timer::unblock_timeout_signal();
 212
 213     // setup a timeout timer
 214     let mut timer = timer::Timer::create(
 215         timer::Clock::Realtime,
 216         timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT))?;
 217
 218     timer.arm(timer::TimerSpec::new()
 219         .value(Some(timeout))
 220         .interval(Some(Duration::from_millis(10))))?;
 221
 222     nix::fcntl::flock(file.as_raw_fd(), lockarg)?;
 223     Ok(())
 224 }
 225
 226 /// Open or create a lock file (append mode). Then try to
 227 /// aquire a lock using `lock_file()`.
 228 pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration)
 229     -> Result<File, Error>
 230 {
 231     let path = path.as_ref();
 232     let mut file =
 233         match OpenOptions::new()
 234             .create(true)
 235             .append(true)
 236             .open(path)
 237         {
 238             Ok(file) => file,
 239             Err(err) => bail!("Unable to open lock {:?} - {}",
 240                               path, err),
 241         };
 242     match lock_file(&mut file, true, Some(timeout)) {
 243         Ok(_) => Ok(file),
 244         Err(err) => bail!("Unable to aquire lock {:?} - {}",
 245                           path, err),
 246     }
 247 }
 248
 249 /// Split a file into equal sized chunks. The last chunk may be
 250 /// smaller. Note: We cannot implement an `Iterator`, because iterators
 251 /// cannot return a borrowed buffer ref (we want zero-copy)
 252 pub fn file_chunker<C, R>(
 253     mut file: R,
 254     chunk_size: usize,
 255     mut chunk_cb: C
 256 ) -> Result<(), Error>
 257     where C: FnMut(usize, &[u8]) -> Result<bool, Error>,
 258           R: Read,
 259 {
 260
 261     const READ_BUFFER_SIZE: usize = 4*1024*1024; // 4M
 262
 263     if chunk_size > READ_BUFFER_SIZE { bail!("chunk size too large!"); }
 264
 265     let mut buf = vec![0u8; READ_BUFFER_SIZE];
 266
 267     let mut pos = 0;
 268     let mut file_pos = 0;
 269     loop {
 270         let mut eof = false;
 271         let mut tmp = &mut buf[..];
 272        // try to read large portions, at least chunk_size
 273         while pos < chunk_size {
 274             match file.read(tmp) {
 275                 Ok(0) => { eof = true; break; },
 276                 Ok(n) => {
 277                     pos += n;
 278                     if pos > chunk_size { break; }
 279                     tmp = &mut tmp[n..];
 280                 }
 281                 Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
 282                 Err(e) => bail!("read chunk failed - {}", e.to_string()),
 283             }
 284         }
 285         let mut start = 0;
 286         while start + chunk_size <= pos {
 287             if !(chunk_cb)(file_pos, &buf[start..start+chunk_size])? { break; }
 288             file_pos += chunk_size;
 289             start += chunk_size;
 290         }
 291         if eof {
 292             if start < pos {
 293                 (chunk_cb)(file_pos, &buf[start..pos])?;
 294                 //file_pos += pos - start;
 295             }
 296             break;
 297         } else {
 298             let rest = pos - start;
 299             if rest > 0 {
 300                 let ptr = buf.as_mut_ptr();
 301                 unsafe { std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest); }
 302                 pos = rest;
 303             } else {
 304                 pos = 0;
 305             }
 306         }
 307     }
 308
 309     Ok(())
 310 }
 311
 312 // Returns the Unix uid/gid for the sepcified system user.
 313 pub fn getpwnam_ugid(username: &str) -> Result<(libc::uid_t,libc::gid_t), Error> {
 314     let info = unsafe { libc::getpwnam(std::ffi::CString::new(username).unwrap().as_ptr()) };
 315     if info == std::ptr::null_mut() {
 316         bail!("getwpnam '{}' failed", username);
 317     }
 318
 319     let info = unsafe { *info };
 320
 321     Ok((info.pw_uid, info.pw_gid))
 322 }
 323
 324 // Returns the hosts node name (UTS node name)
 325 pub fn nodename() -> &'static str {
 326
 327     lazy_static!{
 328         static ref NODENAME: String = {
 329
 330             nix::sys::utsname::uname()
 331                 .nodename()
 332                 .split('.')
 333                 .next()
 334                 .unwrap()
 335                 .to_owned()
 336         };
 337     }
 338
 339     &NODENAME
 340 }
 341
 342 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
 343
 344     let mut query = url::form_urlencoded::Serializer::new(String::new());
 345
 346     let object = data.as_object().ok_or_else(|| {
 347         format_err!("json_object_to_query: got wrong data type (expected object).")
 348     })?;
 349
 350     for (key, value) in object {
 351         match value {
 352             Value::Bool(b) => { query.append_pair(key, &b.to_string()); }
 353             Value::Number(n) => { query.append_pair(key, &n.to_string()); }
 354             Value::String(s) => { query.append_pair(key, &s); }
 355             Value::Array(arr) => {
 356                 for element in arr {
 357                     match element {
 358                         Value::Bool(b) => { query.append_pair(key, &b.to_string()); }
 359                         Value::Number(n) => { query.append_pair(key, &n.to_string()); }
 360                         Value::String(s) => { query.append_pair(key, &s); }
 361                         _ => bail!("json_object_to_query: unable to handle complex array data types."),
 362                     }
 363                 }
 364             }
 365             _ => bail!("json_object_to_query: unable to handle complex data types."),
 366         }
 367     }
 368
 369     Ok(query.finish())
 370 }
 371
 372 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 373     match param[name].as_str()   {
 374         Some(s) => Ok(s),
 375         None => bail!("missing parameter '{}'", name),
 376     }
 377 }
 378
 379 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 380     match param[name].as_i64()   {
 381         Some(s) => Ok(s),
 382         None => bail!("missing parameter '{}'", name),
 383     }
 384 }
 385
 386 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 387     match param[name].as_array()   {
 388         Some(s) => Ok(s.to_vec()),
 389         None => bail!("missing parameter '{}'", name),
 390     }
 391 }
 392
 393 pub fn complete_file_name(arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 394
 395     let mut result = vec![];
 396
 397     use nix::fcntl::OFlag;
 398     use nix::sys::stat::Mode;
 399     use nix::fcntl::AtFlags;
 400
 401     let mut dirname = std::path::PathBuf::from(if arg.len() == 0 { "./" } else { arg });
 402
 403     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 404         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 405         Err(_) => false,
 406     };
 407
 408     if !is_dir {
 409         if let Some(parent) = dirname.parent() {
 410             dirname = parent.to_owned();
 411         }
 412     }
 413
 414     let mut dir = match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 415         Ok(d) => d,
 416         Err(_) => return result,
 417     };
 418
 419     for item in dir.iter() {
 420         if let Ok(entry) = item {
 421             if let Ok(name) = entry.file_name().to_str() {
 422                 if name == "." || name == ".." { continue; }
 423                 let mut newpath = dirname.clone();
 424                 newpath.push(name);
 425
 426                 if let Ok(stat) = nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty()) {
 427                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 428                         newpath.push("");
 429                         if let Some(newpath) = newpath.to_str() {
 430                             result.push(newpath.to_owned());
 431                         }
 432                         continue;
 433                      }
 434                 }
 435                 if let Some(newpath) = newpath.to_str() {
 436                     result.push(newpath.to_owned());
 437                 }
 438
 439              }
 440         }
 441     }
 442
 443     result
 444 }
 445
 446 /// Scan directory for matching file names.
 447 ///
 448 /// Scan through all directory entries and call `callback()` function
 449 /// if the entry name matches the regular expression. This function
 450 /// used unix `openat()`, so you can pass absolute or relative file
 451 /// names. This function simply skips non-UTF8 encoded names.
 452 pub fn scandir<P, F>(
 453     dirfd: RawFd,
 454     path: &P,
 455     regex: &regex::Regex,
 456     mut callback: F
 457 ) -> Result<(), Error>
 458     where F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 459           P: ?Sized + nix::NixPath,
 460 {
 461     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 462         let entry = entry?;
 463         let file_type = match entry.file_type() {
 464             Some(file_type) => file_type,
 465             None => bail!("unable to detect file type"),
 466         };
 467
 468         callback(entry.parent_fd(), unsafe { entry.file_name_utf8_unchecked() }, file_type)?;
 469     }
 470     Ok(())
 471 }
 472
 473 pub fn get_hardware_address() -> Result<String, Error> {
 474
 475     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 476
 477     let contents = file_get_contents(FILENAME)?;
 478     let digest = md5::compute(contents);
 479
 480     Ok(format!("{:0x}", digest))
 481 }
 482
 483 pub fn digest_to_hex(digest: &[u8]) -> String {
 484
 485     const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef";
 486
 487     let mut buf = Vec::<u8>::with_capacity(digest.len()*2);
 488
 489     for i in 0..digest.len() {
 490         buf.push(HEX_CHARS[(digest[i] >> 4) as usize]);
 491         buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]);
 492     }
 493
 494     unsafe { String::from_utf8_unchecked(buf) }
 495 }
 496
 497 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 498     if digest1 != digest2 {
 499         bail!("detected modified configuration - file changed by other user? Try again.");
 500     }
 501     Ok(())
 502 }
 503
 504 /// Extract authentication cookie from cookie header.
 505 /// We assume cookie_name is already url encoded.
 506 pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 507
 508     for pair in cookie.split(';') {
 509
 510         let (name, value) = match pair.find('=') {
 511             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 512             None => return None, // Cookie format error
 513         };
 514
 515         if name == cookie_name {
 516             use url::percent_encoding::percent_decode;
 517             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 518                 return Some(value.into());
 519             } else {
 520                 return None; // Cookie format error
 521             }
 522         }
 523     }
 524
 525     None
 526 }
 527
 528 pub fn join(data: &Vec<String>, sep: char) -> String {
 529
 530     let mut list = String::new();
 531
 532     for item in data {
 533         if list.len() != 0 { list.push(sep); }
 534         list.push_str(item);
 535     }
 536
 537     list
 538 }
 539
 540 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 541     use nix::fcntl::{fcntl, F_GETFD, F_SETFD, FdFlag};
 542     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 543         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 544     flags.set(FdFlag::FD_CLOEXEC, on);
 545     fcntl(fd, F_SETFD(flags))?;
 546     Ok(())
 547 }