src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::AsciiSet;
  16
  17 pub use proxmox::tools::fd::Fd;
  18
  19 pub mod acl;
  20 pub mod apt;
  21 pub mod async_io;
  22 pub mod borrow;
  23 pub mod cert;
  24 pub mod daemon;
  25 pub mod disks;
  26 pub mod fs;
  27 pub mod format;
  28 pub mod lru_cache;
  29 pub mod runtime;
  30 pub mod ticket;
  31 pub mod statistics;
  32 pub mod systemd;
  33 pub mod nom;
  34 pub mod logrotate;
  35 pub mod loopdev;
  36 pub mod fuse_loop;
  37 pub mod socket;
  38 pub mod subscription;
  39 pub mod zip;
  40 pub mod http;
  41
  42 mod parallel_handler;
  43 pub use parallel_handler::*;
  44
  45 mod wrapped_reader_stream;
  46 pub use wrapped_reader_stream::*;
  47
  48 mod async_channel_writer;
  49 pub use async_channel_writer::*;
  50
  51
  52 mod std_channel_writer;
  53 pub use std_channel_writer::*;
  54
  55 pub mod xattr;
  56
  57 mod process_locker;
  58 pub use process_locker::*;
  59
  60 mod file_logger;
  61 pub use file_logger::*;
  62
  63 mod broadcast_future;
  64 pub use broadcast_future::*;
  65
  66 /// The `BufferedRead` trait provides a single function
  67 /// `buffered_read`. It returns a reference to an internal buffer. The
  68 /// purpose of this traid is to avoid unnecessary data copies.
  69 pub trait BufferedRead {
  70     /// This functions tries to fill the internal buffers, then
  71     /// returns a reference to the available data. It returns an empty
  72     /// buffer if `offset` points to the end of the file.
  73     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  74 }
  75
  76 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
  77     let mut query = url::form_urlencoded::Serializer::new(String::new());
  78
  79     let object = data.as_object().ok_or_else(|| {
  80         format_err!("json_object_to_query: got wrong data type (expected object).")
  81     })?;
  82
  83     for (key, value) in object {
  84         match value {
  85             Value::Bool(b) => {
  86                 query.append_pair(key, &b.to_string());
  87             }
  88             Value::Number(n) => {
  89                 query.append_pair(key, &n.to_string());
  90             }
  91             Value::String(s) => {
  92                 query.append_pair(key, &s);
  93             }
  94             Value::Array(arr) => {
  95                 for element in arr {
  96                     match element {
  97                         Value::Bool(b) => {
  98                             query.append_pair(key, &b.to_string());
  99                         }
 100                         Value::Number(n) => {
 101                             query.append_pair(key, &n.to_string());
 102                         }
 103                         Value::String(s) => {
 104                             query.append_pair(key, &s);
 105                         }
 106                         _ => bail!(
 107                             "json_object_to_query: unable to handle complex array data types."
 108                         ),
 109                     }
 110                 }
 111             }
 112             _ => bail!("json_object_to_query: unable to handle complex data types."),
 113         }
 114     }
 115
 116     Ok(query.finish())
 117 }
 118
 119 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 120     match param[name].as_str() {
 121         Some(s) => Ok(s),
 122         None => bail!("missing parameter '{}'", name),
 123     }
 124 }
 125
 126 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 127     match param[name].as_str() {
 128         Some(s) => Ok(s),
 129         None => bail!("missing property '{}'", name),
 130     }
 131 }
 132
 133 pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 134     match param[name].as_i64() {
 135         Some(s) => Ok(s),
 136         None => bail!("missing parameter '{}'", name),
 137     }
 138 }
 139
 140 pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> {
 141     match param[name].as_i64() {
 142         Some(s) => Ok(s),
 143         None => bail!("missing property '{}'", name),
 144     }
 145 }
 146
 147 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 148     match param[name].as_array() {
 149         Some(s) => Ok(s.to_vec()),
 150         None => bail!("missing parameter '{}'", name),
 151     }
 152 }
 153
 154 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> {
 155     match param[name].as_array() {
 156         Some(s) => Ok(s.to_vec()),
 157         None => bail!("missing property '{}'", name),
 158     }
 159 }
 160
 161 pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> {
 162     let mut result = vec![];
 163
 164     use nix::fcntl::AtFlags;
 165     use nix::fcntl::OFlag;
 166     use nix::sys::stat::Mode;
 167
 168     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 169
 170     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 171         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 172         Err(_) => false,
 173     };
 174
 175     if !is_dir {
 176         if let Some(parent) = dirname.parent() {
 177             dirname = parent.to_owned();
 178         }
 179     }
 180
 181     let mut dir =
 182         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 183             Ok(d) => d,
 184             Err(_) => return result,
 185         };
 186
 187     for item in dir.iter() {
 188         if let Ok(entry) = item {
 189             if let Ok(name) = entry.file_name().to_str() {
 190                 if name == "." || name == ".." {
 191                     continue;
 192                 }
 193                 let mut newpath = dirname.clone();
 194                 newpath.push(name);
 195
 196                 if let Ok(stat) =
 197                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 198                 {
 199                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 200                         newpath.push("");
 201                         if let Some(newpath) = newpath.to_str() {
 202                             result.push(newpath.to_owned());
 203                         }
 204                         continue;
 205                     }
 206                 }
 207                 if let Some(newpath) = newpath.to_str() {
 208                     result.push(newpath.to_owned());
 209                 }
 210             }
 211         }
 212     }
 213
 214     result
 215 }
 216
 217 /// Scan directory for matching file names.
 218 ///
 219 /// Scan through all directory entries and call `callback()` function
 220 /// if the entry name matches the regular expression. This function
 221 /// used unix `openat()`, so you can pass absolute or relative file
 222 /// names. This function simply skips non-UTF8 encoded names.
 223 pub fn scandir<P, F>(
 224     dirfd: RawFd,
 225     path: &P,
 226     regex: &regex::Regex,
 227     mut callback: F,
 228 ) -> Result<(), Error>
 229 where
 230     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 231     P: ?Sized + nix::NixPath,
 232 {
 233     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 234         let entry = entry?;
 235         let file_type = match entry.file_type() {
 236             Some(file_type) => file_type,
 237             None => bail!("unable to detect file type"),
 238         };
 239
 240         callback(
 241             entry.parent_fd(),
 242             unsafe { entry.file_name_utf8_unchecked() },
 243             file_type,
 244         )?;
 245     }
 246     Ok(())
 247 }
 248
 249 /// Shortcut for md5 sums.
 250 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 251     hash(MessageDigest::md5(), data).map_err(Error::from)
 252 }
 253
 254 pub fn get_hardware_address() -> Result<String, Error> {
 255     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 256
 257     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 258         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 259     let digest = md5sum(&contents)
 260         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 261
 262     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 263 }
 264
 265 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 266     if digest1 != digest2 {
 267         bail!("detected modified configuration - file changed by other user? Try again.");
 268     }
 269     Ok(())
 270 }
 271
 272 /// Extract a specific cookie from cookie header.
 273 /// We assume cookie_name is already url encoded.
 274 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 275     for pair in cookie.split(';') {
 276         let (name, value) = match pair.find('=') {
 277             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 278             None => return None, // Cookie format error
 279         };
 280
 281         if name == cookie_name {
 282             use percent_encoding::percent_decode;
 283             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 284                 return Some(value.into());
 285             } else {
 286                 return None; // Cookie format error
 287             }
 288         }
 289     }
 290
 291     None
 292 }
 293
 294 pub fn join(data: &Vec<String>, sep: char) -> String {
 295     let mut list = String::new();
 296
 297     for item in data {
 298         if !list.is_empty() {
 299             list.push(sep);
 300         }
 301         list.push_str(item);
 302     }
 303
 304     list
 305 }
 306
 307 /// Detect modified configuration files
 308 ///
 309 /// This function fails with a reasonable error message if checksums do not match.
 310 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 311     if digest1 != digest2 {
 312         bail!("detected modified configuration - file changed by other user? Try again.");
 313     }
 314     Ok(())
 315 }
 316
 317 /// normalize uri path
 318 ///
 319 /// Do not allow ".", "..", or hidden files ".XXXX"
 320 /// Also remove empty path components
 321 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 322     let items = path.split('/');
 323
 324     let mut path = String::new();
 325     let mut components = vec![];
 326
 327     for name in items {
 328         if name.is_empty() {
 329             continue;
 330         }
 331         if name.starts_with('.') {
 332             bail!("Path contains illegal components.");
 333         }
 334         path.push('/');
 335         path.push_str(name);
 336         components.push(name);
 337     }
 338
 339     Ok((path, components))
 340 }
 341
 342 /// Helper to check result from std::process::Command output
 343 ///
 344 /// The exit_code_check() function should return true if the exit code
 345 /// is considered successful.
 346 pub fn command_output(
 347     output: std::process::Output,
 348     exit_code_check: Option<fn(i32) -> bool>,
 349 ) -> Result<Vec<u8>, Error> {
 350
 351     if !output.status.success() {
 352         match output.status.code() {
 353             Some(code) => {
 354                 let is_ok = match exit_code_check {
 355                     Some(check_fn) => check_fn(code),
 356                     None => code == 0,
 357                 };
 358                 if !is_ok {
 359                     let msg = String::from_utf8(output.stderr)
 360                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 361                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 362
 363                     bail!("status code: {} - {}", code, msg);
 364                 }
 365             }
 366             None => bail!("terminated by signal"),
 367         }
 368     }
 369
 370     Ok(output.stdout)
 371 }
 372
 373 /// Helper to check result from std::process::Command output, returns String.
 374 ///
 375 /// The exit_code_check() function should return true if the exit code
 376 /// is considered successful.
 377 pub fn command_output_as_string(
 378     output: std::process::Output,
 379     exit_code_check: Option<fn(i32) -> bool>,
 380 ) -> Result<String, Error> {
 381     let output = command_output(output, exit_code_check)?;
 382     let output = String::from_utf8(output)?;
 383     Ok(output)
 384 }
 385
 386 pub fn run_command(
 387     mut command: std::process::Command,
 388     exit_code_check: Option<fn(i32) -> bool>,
 389 ) -> Result<String, Error> {
 390
 391    let output = command.output()
 392         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 393
 394     let output = command_output_as_string(output, exit_code_check)
 395         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 396
 397     Ok(output)
 398 }
 399
 400 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 401     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 402     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 403         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 404     flags.set(FdFlag::FD_CLOEXEC, on);
 405     fcntl(fd, F_SETFD(flags))?;
 406     Ok(())
 407 }
 408
 409 static mut SHUTDOWN_REQUESTED: bool = false;
 410
 411 pub fn request_shutdown() {
 412     unsafe {
 413         SHUTDOWN_REQUESTED = true;
 414     }
 415     crate::server::server_shutdown();
 416 }
 417
 418 #[inline(always)]
 419 pub fn shutdown_requested() -> bool {
 420     unsafe { SHUTDOWN_REQUESTED }
 421 }
 422
 423 pub fn fail_on_shutdown() -> Result<(), Error> {
 424     if shutdown_requested() {
 425         bail!("Server shutdown requested - aborting task");
 426     }
 427     Ok(())
 428 }
 429
 430 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 431 /// descriptors.
 432 pub fn pipe() -> Result<(Fd, Fd), Error> {
 433     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 434     Ok((Fd(pin), Fd(pout)))
 435 }
 436
 437 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 438 /// descriptors.
 439 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 440     use nix::sys::socket;
 441     let (pa, pb) = socket::socketpair(
 442         socket::AddressFamily::Unix,
 443         socket::SockType::Stream,
 444         None,
 445         socket::SockFlag::SOCK_CLOEXEC,
 446     )?;
 447     Ok((Fd(pa), Fd(pb)))
 448 }
 449
 450
 451 /// An easy way to convert types to Any
 452 ///
 453 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 454 pub trait AsAny {
 455     fn as_any(&self) -> &dyn Any;
 456 }
 457
 458 impl<T: Any> AsAny for T {
 459     fn as_any(&self) -> &dyn Any {
 460         self
 461     }
 462 }
 463
 464 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 465 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 466     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 467     .add(0x20)
 468     .add(0x7f)
 469     // the DEFAULT_ENCODE_SET added:
 470     .add(b' ')
 471     .add(b'"')
 472     .add(b'#')
 473     .add(b'<')
 474     .add(b'>')
 475     .add(b'`')
 476     .add(b'?')
 477     .add(b'{')
 478     .add(b'}');
 479
 480 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 481 /// `#`).
 482 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 483     path: P,
 484 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 485     let path = path.as_ref();
 486
 487     Ok(io::BufReader::new(
 488         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 489     )
 490     .lines()
 491     .filter_map(|line| match line {
 492         Ok(line) => {
 493             let line = line.trim();
 494             if line.is_empty() || line.starts_with('#') {
 495                 None
 496             } else {
 497                 Some(Ok(line.to_string()))
 498             }
 499         }
 500         Err(err) => Some(Err(err)),
 501     }))
 502 }
 503
 504 pub fn setup_safe_path_env() {
 505     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 506     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 507     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 508         std::env::remove_var(name);
 509     }
 510 }
 511
 512 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 513     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 514         Some(n) => &line[n..],
 515         None => return &[],
 516     };
 517     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 518         Some(n) => &line[..(line.len() - n)],
 519         None => &[],
 520     }
 521 }
 522
 523 /// Seeks to start of file and computes the SHA256 hash
 524 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 525
 526     file.seek(SeekFrom::Start(0))?;
 527
 528     let mut hasher = openssl::sha::Sha256::new();
 529     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 530     let mut size: u64 = 0;
 531
 532     loop {
 533         let count = match file.read(&mut buffer) {
 534             Ok(count) => count,
 535             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 536                 continue;
 537             }
 538             Err(err) => return Err(err.into()),
 539         };
 540         if count == 0 {
 541             break;
 542         }
 543         size += count as u64;
 544         hasher.update(&buffer[..count]);
 545     }
 546
 547     let csum = hasher.finish();
 548
 549     Ok((csum, size))
 550 }