src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::borrow::Borrow;
   6 use std::collections::HashMap;
   7 use std::hash::BuildHasher;
   8 use std::fs::File;
   9 use std::io::{self, BufRead, Read, Seek, SeekFrom};
  10 use std::os::unix::io::RawFd;
  11 use std::path::Path;
  12
  13 use anyhow::{bail, format_err, Error};
  14 use serde_json::Value;
  15 use openssl::hash::{hash, DigestBytes, MessageDigest};
  16 use percent_encoding::{utf8_percent_encode, AsciiSet};
  17
  18 pub use proxmox::tools::fd::Fd;
  19
  20 pub mod acl;
  21 pub mod apt;
  22 pub mod async_io;
  23 pub mod borrow;
  24 pub mod cert;
  25 pub mod daemon;
  26 pub mod disks;
  27 pub mod format;
  28 pub mod fs;
  29 pub mod fuse_loop;
  30 pub mod http;
  31 pub mod logrotate;
  32 pub mod loopdev;
  33 pub mod lru_cache;
  34 pub mod nom;
  35 pub mod runtime;
  36 pub mod serde_filter;
  37 pub mod socket;
  38 pub mod statistics;
  39 pub mod subscription;
  40 pub mod systemd;
  41 pub mod ticket;
  42 pub mod xattr;
  43 pub mod zip;
  44
  45 pub mod parallel_handler;
  46 pub use parallel_handler::ParallelHandler;
  47
  48 mod wrapped_reader_stream;
  49 pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedReaderStream};
  50
  51 mod async_channel_writer;
  52 pub use async_channel_writer::AsyncChannelWriter;
  53
  54 mod std_channel_writer;
  55 pub use std_channel_writer::StdChannelWriter;
  56
  57 mod process_locker;
  58 pub use process_locker::{ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard};
  59
  60 mod file_logger;
  61 pub use file_logger::{FileLogger, FileLogOptions};
  62
  63 mod broadcast_future;
  64 pub use broadcast_future::{BroadcastData, BroadcastFuture};
  65
  66 /// The `BufferedRead` trait provides a single function
  67 /// `buffered_read`. It returns a reference to an internal buffer. The
  68 /// purpose of this traid is to avoid unnecessary data copies.
  69 pub trait BufferedRead {
  70     /// This functions tries to fill the internal buffers, then
  71     /// returns a reference to the available data. It returns an empty
  72     /// buffer if `offset` points to the end of the file.
  73     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  74 }
  75
  76 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
  77     let mut query = url::form_urlencoded::Serializer::new(String::new());
  78
  79     let object = data.as_object().ok_or_else(|| {
  80         format_err!("json_object_to_query: got wrong data type (expected object).")
  81     })?;
  82
  83     for (key, value) in object {
  84         match value {
  85             Value::Bool(b) => {
  86                 query.append_pair(key, &b.to_string());
  87             }
  88             Value::Number(n) => {
  89                 query.append_pair(key, &n.to_string());
  90             }
  91             Value::String(s) => {
  92                 query.append_pair(key, &s);
  93             }
  94             Value::Array(arr) => {
  95                 for element in arr {
  96                     match element {
  97                         Value::Bool(b) => {
  98                             query.append_pair(key, &b.to_string());
  99                         }
 100                         Value::Number(n) => {
 101                             query.append_pair(key, &n.to_string());
 102                         }
 103                         Value::String(s) => {
 104                             query.append_pair(key, &s);
 105                         }
 106                         _ => bail!(
 107                             "json_object_to_query: unable to handle complex array data types."
 108                         ),
 109                     }
 110                 }
 111             }
 112             _ => bail!("json_object_to_query: unable to handle complex data types."),
 113         }
 114     }
 115
 116     Ok(query.finish())
 117 }
 118
 119 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 120     match param[name].as_str() {
 121         Some(s) => Ok(s),
 122         None => bail!("missing parameter '{}'", name),
 123     }
 124 }
 125
 126 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 127     match param[name].as_str() {
 128         Some(s) => Ok(s),
 129         None => bail!("missing property '{}'", name),
 130     }
 131 }
 132
 133 pub fn required_integer_param(param: &Value, name: &str) -> Result<i64, Error> {
 134     match param[name].as_i64() {
 135         Some(s) => Ok(s),
 136         None => bail!("missing parameter '{}'", name),
 137     }
 138 }
 139
 140 pub fn required_integer_property(param: &Value, name: &str) -> Result<i64, Error> {
 141     match param[name].as_i64() {
 142         Some(s) => Ok(s),
 143         None => bail!("missing property '{}'", name),
 144     }
 145 }
 146
 147 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 148     match param[name].as_array() {
 149         Some(s) => Ok(&s),
 150         None => bail!("missing parameter '{}'", name),
 151     }
 152 }
 153
 154 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 155     match param[name].as_array() {
 156         Some(s) => Ok(&s),
 157         None => bail!("missing property '{}'", name),
 158     }
 159 }
 160
 161 pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
 162 where
 163     S: BuildHasher,
 164 {
 165     let mut result = vec![];
 166
 167     use nix::fcntl::AtFlags;
 168     use nix::fcntl::OFlag;
 169     use nix::sys::stat::Mode;
 170
 171     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 172
 173     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 174         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 175         Err(_) => false,
 176     };
 177
 178     if !is_dir {
 179         if let Some(parent) = dirname.parent() {
 180             dirname = parent.to_owned();
 181         }
 182     }
 183
 184     let mut dir =
 185         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 186             Ok(d) => d,
 187             Err(_) => return result,
 188         };
 189
 190     for item in dir.iter() {
 191         if let Ok(entry) = item {
 192             if let Ok(name) = entry.file_name().to_str() {
 193                 if name == "." || name == ".." {
 194                     continue;
 195                 }
 196                 let mut newpath = dirname.clone();
 197                 newpath.push(name);
 198
 199                 if let Ok(stat) =
 200                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 201                 {
 202                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 203                         newpath.push("");
 204                         if let Some(newpath) = newpath.to_str() {
 205                             result.push(newpath.to_owned());
 206                         }
 207                         continue;
 208                     }
 209                 }
 210                 if let Some(newpath) = newpath.to_str() {
 211                     result.push(newpath.to_owned());
 212                 }
 213             }
 214         }
 215     }
 216
 217     result
 218 }
 219
 220 /// Scan directory for matching file names.
 221 ///
 222 /// Scan through all directory entries and call `callback()` function
 223 /// if the entry name matches the regular expression. This function
 224 /// used unix `openat()`, so you can pass absolute or relative file
 225 /// names. This function simply skips non-UTF8 encoded names.
 226 pub fn scandir<P, F>(
 227     dirfd: RawFd,
 228     path: &P,
 229     regex: &regex::Regex,
 230     mut callback: F,
 231 ) -> Result<(), Error>
 232 where
 233     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 234     P: ?Sized + nix::NixPath,
 235 {
 236     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 237         let entry = entry?;
 238         let file_type = match entry.file_type() {
 239             Some(file_type) => file_type,
 240             None => bail!("unable to detect file type"),
 241         };
 242
 243         callback(
 244             entry.parent_fd(),
 245             unsafe { entry.file_name_utf8_unchecked() },
 246             file_type,
 247         )?;
 248     }
 249     Ok(())
 250 }
 251
 252 /// Shortcut for md5 sums.
 253 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 254     hash(MessageDigest::md5(), data).map_err(Error::from)
 255 }
 256
 257 pub fn get_hardware_address() -> Result<String, Error> {
 258     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 259
 260     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 261         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 262     let digest = md5sum(&contents)
 263         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 264
 265     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 266 }
 267
 268 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 269     if digest1 != digest2 {
 270         bail!("detected modified configuration - file changed by other user? Try again.");
 271     }
 272     Ok(())
 273 }
 274
 275 /// Extract a specific cookie from cookie header.
 276 /// We assume cookie_name is already url encoded.
 277 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 278     for pair in cookie.split(';') {
 279         let (name, value) = match pair.find('=') {
 280             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 281             None => return None, // Cookie format error
 282         };
 283
 284         if name == cookie_name {
 285             use percent_encoding::percent_decode;
 286             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 287                 return Some(value.into());
 288             } else {
 289                 return None; // Cookie format error
 290             }
 291         }
 292     }
 293
 294     None
 295 }
 296
 297 /// percent encode a url component
 298 pub fn percent_encode_component(comp: &str) -> String {
 299     utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
 300 }
 301
 302 pub fn join<S: Borrow<str>>(data: &[S], sep: char) -> String {
 303     let mut list = String::new();
 304
 305     for item in data {
 306         if !list.is_empty() {
 307             list.push(sep);
 308         }
 309         list.push_str(item.borrow());
 310     }
 311
 312     list
 313 }
 314
 315 /// Detect modified configuration files
 316 ///
 317 /// This function fails with a reasonable error message if checksums do not match.
 318 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 319     if digest1 != digest2 {
 320         bail!("detected modified configuration - file changed by other user? Try again.");
 321     }
 322     Ok(())
 323 }
 324
 325 /// normalize uri path
 326 ///
 327 /// Do not allow ".", "..", or hidden files ".XXXX"
 328 /// Also remove empty path components
 329 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 330     let items = path.split('/');
 331
 332     let mut path = String::new();
 333     let mut components = vec![];
 334
 335     for name in items {
 336         if name.is_empty() {
 337             continue;
 338         }
 339         if name.starts_with('.') {
 340             bail!("Path contains illegal components.");
 341         }
 342         path.push('/');
 343         path.push_str(name);
 344         components.push(name);
 345     }
 346
 347     Ok((path, components))
 348 }
 349
 350 /// Helper to check result from std::process::Command output
 351 ///
 352 /// The exit_code_check() function should return true if the exit code
 353 /// is considered successful.
 354 pub fn command_output(
 355     output: std::process::Output,
 356     exit_code_check: Option<fn(i32) -> bool>,
 357 ) -> Result<Vec<u8>, Error> {
 358
 359     if !output.status.success() {
 360         match output.status.code() {
 361             Some(code) => {
 362                 let is_ok = match exit_code_check {
 363                     Some(check_fn) => check_fn(code),
 364                     None => code == 0,
 365                 };
 366                 if !is_ok {
 367                     let msg = String::from_utf8(output.stderr)
 368                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 369                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 370
 371                     bail!("status code: {} - {}", code, msg);
 372                 }
 373             }
 374             None => bail!("terminated by signal"),
 375         }
 376     }
 377
 378     Ok(output.stdout)
 379 }
 380
 381 /// Helper to check result from std::process::Command output, returns String.
 382 ///
 383 /// The exit_code_check() function should return true if the exit code
 384 /// is considered successful.
 385 pub fn command_output_as_string(
 386     output: std::process::Output,
 387     exit_code_check: Option<fn(i32) -> bool>,
 388 ) -> Result<String, Error> {
 389     let output = command_output(output, exit_code_check)?;
 390     let output = String::from_utf8(output)?;
 391     Ok(output)
 392 }
 393
 394 pub fn run_command(
 395     mut command: std::process::Command,
 396     exit_code_check: Option<fn(i32) -> bool>,
 397 ) -> Result<String, Error> {
 398
 399    let output = command.output()
 400         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 401
 402     let output = command_output_as_string(output, exit_code_check)
 403         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 404
 405     Ok(output)
 406 }
 407
 408 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 409     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 410     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 411         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 412     flags.set(FdFlag::FD_CLOEXEC, on);
 413     fcntl(fd, F_SETFD(flags))?;
 414     Ok(())
 415 }
 416
 417 static mut SHUTDOWN_REQUESTED: bool = false;
 418
 419 pub fn request_shutdown() {
 420     unsafe {
 421         SHUTDOWN_REQUESTED = true;
 422     }
 423     crate::server::server_shutdown();
 424 }
 425
 426 #[inline(always)]
 427 pub fn shutdown_requested() -> bool {
 428     unsafe { SHUTDOWN_REQUESTED }
 429 }
 430
 431 pub fn fail_on_shutdown() -> Result<(), Error> {
 432     if shutdown_requested() {
 433         bail!("Server shutdown requested - aborting task");
 434     }
 435     Ok(())
 436 }
 437
 438 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 439 /// descriptors.
 440 pub fn pipe() -> Result<(Fd, Fd), Error> {
 441     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 442     Ok((Fd(pin), Fd(pout)))
 443 }
 444
 445 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 446 /// descriptors.
 447 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 448     use nix::sys::socket;
 449     let (pa, pb) = socket::socketpair(
 450         socket::AddressFamily::Unix,
 451         socket::SockType::Stream,
 452         None,
 453         socket::SockFlag::SOCK_CLOEXEC,
 454     )?;
 455     Ok((Fd(pa), Fd(pb)))
 456 }
 457
 458
 459 /// An easy way to convert types to Any
 460 ///
 461 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 462 pub trait AsAny {
 463     fn as_any(&self) -> &dyn Any;
 464 }
 465
 466 impl<T: Any> AsAny for T {
 467     fn as_any(&self) -> &dyn Any {
 468         self
 469     }
 470 }
 471
 472 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 473 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 474     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 475     .add(0x20)
 476     .add(0x7f)
 477     // the DEFAULT_ENCODE_SET added:
 478     .add(b' ')
 479     .add(b'"')
 480     .add(b'#')
 481     .add(b'<')
 482     .add(b'>')
 483     .add(b'`')
 484     .add(b'?')
 485     .add(b'{')
 486     .add(b'}');
 487
 488 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 489 /// `#`).
 490 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 491     path: P,
 492 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 493     let path = path.as_ref();
 494
 495     Ok(io::BufReader::new(
 496         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 497     )
 498     .lines()
 499     .filter_map(|line| match line {
 500         Ok(line) => {
 501             let line = line.trim();
 502             if line.is_empty() || line.starts_with('#') {
 503                 None
 504             } else {
 505                 Some(Ok(line.to_string()))
 506             }
 507         }
 508         Err(err) => Some(Err(err)),
 509     }))
 510 }
 511
 512 pub fn setup_safe_path_env() {
 513     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 514     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 515     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 516         std::env::remove_var(name);
 517     }
 518 }
 519
 520 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 521     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 522         Some(n) => &line[n..],
 523         None => return &[],
 524     };
 525     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 526         Some(n) => &line[..(line.len() - n)],
 527         None => &[],
 528     }
 529 }
 530
 531 /// Seeks to start of file and computes the SHA256 hash
 532 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 533
 534     file.seek(SeekFrom::Start(0))?;
 535
 536     let mut hasher = openssl::sha::Sha256::new();
 537     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 538     let mut size: u64 = 0;
 539
 540     loop {
 541         let count = match file.read(&mut buffer) {
 542             Ok(0) => break,
 543             Ok(count) => count,
 544             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 545                 continue;
 546             }
 547             Err(err) => return Err(err.into()),
 548         };
 549         size += count as u64;
 550         hasher.update(&buffer[..count]);
 551     }
 552
 553     let csum = hasher.finish();
 554
 555     Ok((csum, size))
 556 }
 557
 558 /// Create the base run-directory.
 559 ///
 560 /// This exists to fixate the permissions for the run *base* directory while allowing intermediate
 561 /// directories after it to have different permissions.
 562 pub fn create_run_dir() -> Result<(), Error> {
 563     let _: bool = proxmox::tools::fs::create_path(PROXMOX_BACKUP_RUN_DIR_M!(), None, None)?;
 564     Ok(())
 565 }