src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::{utf8_percent_encode, AsciiSet};
  16
  17 pub use proxmox::tools::fd::Fd;
  18
  19 pub mod acl;
  20 pub mod apt;
  21 pub mod async_io;
  22 pub mod borrow;
  23 pub mod cert;
  24 pub mod daemon;
  25 pub mod disks;
  26 pub mod format;
  27 pub mod fs;
  28 pub mod fuse_loop;
  29 pub mod http;
  30 pub mod logrotate;
  31 pub mod loopdev;
  32 pub mod lru_cache;
  33 pub mod nom;
  34 pub mod runtime;
  35 pub mod socket;
  36 pub mod statistics;
  37 pub mod subscription;
  38 pub mod systemd;
  39 pub mod ticket;
  40 pub mod xattr;
  41 pub mod zip;
  42
  43 pub mod parallel_handler;
  44 pub use parallel_handler::ParallelHandler;
  45
  46 mod wrapped_reader_stream;
  47 pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedReaderStream};
  48
  49 mod async_channel_writer;
  50 pub use async_channel_writer::AsyncChannelWriter;
  51
  52 mod std_channel_writer;
  53 pub use std_channel_writer::StdChannelWriter;
  54
  55 mod process_locker;
  56 pub use process_locker::{ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard};
  57
  58 mod file_logger;
  59 pub use file_logger::{FileLogger, FileLogOptions};
  60
  61 mod broadcast_future;
  62 pub use broadcast_future::{BroadcastData, BroadcastFuture};
  63
  64 /// The `BufferedRead` trait provides a single function
  65 /// `buffered_read`. It returns a reference to an internal buffer. The
  66 /// purpose of this traid is to avoid unnecessary data copies.
  67 pub trait BufferedRead {
  68     /// This functions tries to fill the internal buffers, then
  69     /// returns a reference to the available data. It returns an empty
  70     /// buffer if `offset` points to the end of the file.
  71     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  72 }
  73
  74 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
  75     let mut query = url::form_urlencoded::Serializer::new(String::new());
  76
  77     let object = data.as_object().ok_or_else(|| {
  78         format_err!("json_object_to_query: got wrong data type (expected object).")
  79     })?;
  80
  81     for (key, value) in object {
  82         match value {
  83             Value::Bool(b) => {
  84                 query.append_pair(key, &b.to_string());
  85             }
  86             Value::Number(n) => {
  87                 query.append_pair(key, &n.to_string());
  88             }
  89             Value::String(s) => {
  90                 query.append_pair(key, &s);
  91             }
  92             Value::Array(arr) => {
  93                 for element in arr {
  94                     match element {
  95                         Value::Bool(b) => {
  96                             query.append_pair(key, &b.to_string());
  97                         }
  98                         Value::Number(n) => {
  99                             query.append_pair(key, &n.to_string());
 100                         }
 101                         Value::String(s) => {
 102                             query.append_pair(key, &s);
 103                         }
 104                         _ => bail!(
 105                             "json_object_to_query: unable to handle complex array data types."
 106                         ),
 107                     }
 108                 }
 109             }
 110             _ => bail!("json_object_to_query: unable to handle complex data types."),
 111         }
 112     }
 113
 114     Ok(query.finish())
 115 }
 116
 117 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 118     match param[name].as_str() {
 119         Some(s) => Ok(s),
 120         None => bail!("missing parameter '{}'", name),
 121     }
 122 }
 123
 124 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 125     match param[name].as_str() {
 126         Some(s) => Ok(s),
 127         None => bail!("missing property '{}'", name),
 128     }
 129 }
 130
 131 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 132     match param[name].as_i64() {
 133         Some(s) => Ok(s),
 134         None => bail!("missing parameter '{}'", name),
 135     }
 136 }
 137
 138 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 139     match param[name].as_i64() {
 140         Some(s) => Ok(s),
 141         None => bail!("missing property '{}'", name),
 142     }
 143 }
 144
 145 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 146     match param[name].as_array() {
 147         Some(s) => Ok(s.to_vec()),
 148         None => bail!("missing parameter '{}'", name),
 149     }
 150 }
 151
 152 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 153     match param[name].as_array() {
 154         Some(s) => Ok(s.to_vec()),
 155         None => bail!("missing property '{}'", name),
 156     }
 157 }
 158
 159 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 160     let mut result = vec![];
 161
 162     use nix::fcntl::AtFlags;
 163     use nix::fcntl::OFlag;
 164     use nix::sys::stat::Mode;
 165
 166     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 167
 168     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 169         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 170         Err(_) => false,
 171     };
 172
 173     if !is_dir {
 174         if let Some(parent) = dirname.parent() {
 175             dirname = parent.to_owned();
 176         }
 177     }
 178
 179     let mut dir =
 180         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 181             Ok(d) => d,
 182             Err(_) => return result,
 183         };
 184
 185     for item in dir.iter() {
 186         if let Ok(entry) = item {
 187             if let Ok(name) = entry.file_name().to_str() {
 188                 if name == "." || name == ".." {
 189                     continue;
 190                 }
 191                 let mut newpath = dirname.clone();
 192                 newpath.push(name);
 193
 194                 if let Ok(stat) =
 195                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 196                 {
 197                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 198                         newpath.push("");
 199                         if let Some(newpath) = newpath.to_str() {
 200                             result.push(newpath.to_owned());
 201                         }
 202                         continue;
 203                     }
 204                 }
 205                 if let Some(newpath) = newpath.to_str() {
 206                     result.push(newpath.to_owned());
 207                 }
 208             }
 209         }
 210     }
 211
 212     result
 213 }
 214
 215 /// Scan directory for matching file names.
 216 ///
 217 /// Scan through all directory entries and call `callback()` function
 218 /// if the entry name matches the regular expression. This function
 219 /// used unix `openat()`, so you can pass absolute or relative file
 220 /// names. This function simply skips non-UTF8 encoded names.
 221 pub fn scandir<P, F>(
 222     dirfd: RawFd,
 223     path: &P,
 224     regex: &regex::Regex,
 225     mut callback: F,
 226 ) -> Result<(), Error>
 227 where
 228     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 229     P: ?Sized + nix::NixPath,
 230 {
 231     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 232         let entry = entry?;
 233         let file_type = match entry.file_type() {
 234             Some(file_type) => file_type,
 235             None => bail!("unable to detect file type"),
 236         };
 237
 238         callback(
 239             entry.parent_fd(),
 240             unsafe { entry.file_name_utf8_unchecked() },
 241             file_type,
 242         )?;
 243     }
 244     Ok(())
 245 }
 246
 247 /// Shortcut for md5 sums.
 248 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 249     hash(MessageDigest::md5(), data).map_err(Error::from)
 250 }
 251
 252 pub fn get_hardware_address() -> Result<String, Error> {
 253     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 254
 255     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 256         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 257     let digest = md5sum(&contents)
 258         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 259
 260     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 261 }
 262
 263 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 264     if digest1 != digest2 {
 265         bail!("detected modified configuration - file changed by other user? Try again.");
 266     }
 267     Ok(())
 268 }
 269
 270 /// Extract a specific cookie from cookie header.
 271 /// We assume cookie_name is already url encoded.
 272 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 273     for pair in cookie.split(';') {
 274         let (name, value) = match pair.find('=') {
 275             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 276             None => return None, // Cookie format error
 277         };
 278
 279         if name == cookie_name {
 280             use percent_encoding::percent_decode;
 281             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 282                 return Some(value.into());
 283             } else {
 284                 return None; // Cookie format error
 285             }
 286         }
 287     }
 288
 289     None
 290 }
 291
 292 /// percent encode a url component
 293 pub fn percent_encode_component(comp: &str) -> String {
 294     utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
 295 }
 296
 297 pub fn join(data: &Vec<String>, sep: char) -> String {
 298     let mut list = String::new();
 299
 300     for item in data {
 301         if !list.is_empty() {
 302             list.push(sep);
 303         }
 304         list.push_str(item);
 305     }
 306
 307     list
 308 }
 309
 310 /// Detect modified configuration files
 311 ///
 312 /// This function fails with a reasonable error message if checksums do not match.
 313 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 314     if digest1 != digest2 {
 315         bail!("detected modified configuration - file changed by other user? Try again.");
 316     }
 317     Ok(())
 318 }
 319
 320 /// normalize uri path
 321 ///
 322 /// Do not allow ".", "..", or hidden files ".XXXX"
 323 /// Also remove empty path components
 324 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 325     let items = path.split('/');
 326
 327     let mut path = String::new();
 328     let mut components = vec![];
 329
 330     for name in items {
 331         if name.is_empty() {
 332             continue;
 333         }
 334         if name.starts_with('.') {
 335             bail!("Path contains illegal components.");
 336         }
 337         path.push('/');
 338         path.push_str(name);
 339         components.push(name);
 340     }
 341
 342     Ok((path, components))
 343 }
 344
 345 /// Helper to check result from std::process::Command output
 346 ///
 347 /// The exit_code_check() function should return true if the exit code
 348 /// is considered successful.
 349 pub fn command_output(
 350     output: std::process::Output,
 351     exit_code_check: Option<fn(i32) -> bool>,
 352 ) -> Result<Vec<u8>, Error> {
 353
 354     if !output.status.success() {
 355         match output.status.code() {
 356             Some(code) => {
 357                 let is_ok = match exit_code_check {
 358                     Some(check_fn) => check_fn(code),
 359                     None => code == 0,
 360                 };
 361                 if !is_ok {
 362                     let msg = String::from_utf8(output.stderr)
 363                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 364                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 365
 366                     bail!("status code: {} - {}", code, msg);
 367                 }
 368             }
 369             None => bail!("terminated by signal"),
 370         }
 371     }
 372
 373     Ok(output.stdout)
 374 }
 375
 376 /// Helper to check result from std::process::Command output, returns String.
 377 ///
 378 /// The exit_code_check() function should return true if the exit code
 379 /// is considered successful.
 380 pub fn command_output_as_string(
 381     output: std::process::Output,
 382     exit_code_check: Option<fn(i32) -> bool>,
 383 ) -> Result<String, Error> {
 384     let output = command_output(output, exit_code_check)?;
 385     let output = String::from_utf8(output)?;
 386     Ok(output)
 387 }
 388
 389 pub fn run_command(
 390     mut command: std::process::Command,
 391     exit_code_check: Option<fn(i32) -> bool>,
 392 ) -> Result<String, Error> {
 393
 394    let output = command.output()
 395         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 396
 397     let output = command_output_as_string(output, exit_code_check)
 398         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 399
 400     Ok(output)
 401 }
 402
 403 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 404     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 405     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 406         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 407     flags.set(FdFlag::FD_CLOEXEC, on);
 408     fcntl(fd, F_SETFD(flags))?;
 409     Ok(())
 410 }
 411
 412 static mut SHUTDOWN_REQUESTED: bool = false;
 413
 414 pub fn request_shutdown() {
 415     unsafe {
 416         SHUTDOWN_REQUESTED = true;
 417     }
 418     crate::server::server_shutdown();
 419 }
 420
 421 #[inline(always)]
 422 pub fn shutdown_requested() -> bool {
 423     unsafe { SHUTDOWN_REQUESTED }
 424 }
 425
 426 pub fn fail_on_shutdown() -> Result<(), Error> {
 427     if shutdown_requested() {
 428         bail!("Server shutdown requested - aborting task");
 429     }
 430     Ok(())
 431 }
 432
 433 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 434 /// descriptors.
 435 pub fn pipe() -> Result<(Fd, Fd), Error> {
 436     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 437     Ok((Fd(pin), Fd(pout)))
 438 }
 439
 440 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 441 /// descriptors.
 442 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 443     use nix::sys::socket;
 444     let (pa, pb) = socket::socketpair(
 445         socket::AddressFamily::Unix,
 446         socket::SockType::Stream,
 447         None,
 448         socket::SockFlag::SOCK_CLOEXEC,
 449     )?;
 450     Ok((Fd(pa), Fd(pb)))
 451 }
 452
 453
 454 /// An easy way to convert types to Any
 455 ///
 456 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 457 pub trait AsAny {
 458     fn as_any(&self) -> &dyn Any;
 459 }
 460
 461 impl<T: Any> AsAny for T {
 462     fn as_any(&self) -> &dyn Any {
 463         self
 464     }
 465 }
 466
 467 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 468 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 469     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 470     .add(0x20)
 471     .add(0x7f)
 472     // the DEFAULT_ENCODE_SET added:
 473     .add(b' ')
 474     .add(b'"')
 475     .add(b'#')
 476     .add(b'<')
 477     .add(b'>')
 478     .add(b'`')
 479     .add(b'?')
 480     .add(b'{')
 481     .add(b'}');
 482
 483 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 484 /// `#`).
 485 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 486     path: P,
 487 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 488     let path = path.as_ref();
 489
 490     Ok(io::BufReader::new(
 491         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 492     )
 493     .lines()
 494     .filter_map(|line| match line {
 495         Ok(line) => {
 496             let line = line.trim();
 497             if line.is_empty() || line.starts_with('#') {
 498                 None
 499             } else {
 500                 Some(Ok(line.to_string()))
 501             }
 502         }
 503         Err(err) => Some(Err(err)),
 504     }))
 505 }
 506
 507 pub fn setup_safe_path_env() {
 508     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 509     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 510     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 511         std::env::remove_var(name);
 512     }
 513 }
 514
 515 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 516     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 517         Some(n) => &line[n..],
 518         None => return &[],
 519     };
 520     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 521         Some(n) => &line[..(line.len() - n)],
 522         None => &[],
 523     }
 524 }
 525
 526 /// Seeks to start of file and computes the SHA256 hash
 527 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 528
 529     file.seek(SeekFrom::Start(0))?;
 530
 531     let mut hasher = openssl::sha::Sha256::new();
 532     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 533     let mut size: u64 = 0;
 534
 535     loop {
 536         let count = match file.read(&mut buffer) {
 537             Ok(count) => count,
 538             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 539                 continue;
 540             }
 541             Err(err) => return Err(err.into()),
 542         };
 543         if count == 0 {
 544             break;
 545         }
 546         size += count as u64;
 547         hasher.update(&buffer[..count]);
 548     }
 549
 550     let csum = hasher.finish();
 551
 552     Ok((csum, size))
 553 }