src/tools.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::borrow::Borrow;
   6 use std::collections::HashMap;
   7 use std::hash::BuildHasher;
   8 use std::fs::File;
   9 use std::io::{self, BufRead, Read, Seek, SeekFrom};
  10 use std::os::unix::io::RawFd;
  11 use std::path::Path;
  12
  13 use anyhow::{bail, format_err, Error};
  14 use serde_json::Value;
  15 use openssl::hash::{hash, DigestBytes, MessageDigest};
  16 use percent_encoding::{utf8_percent_encode, AsciiSet};
  17
  18 pub use proxmox::tools::fd::Fd;
  19 use proxmox::tools::fs::{create_path, CreateOptions};
  20
  21 use proxmox_http::http::{
  22     client::SimpleHttp,
  23     client::SimpleHttpOptions,
  24     ProxyConfig,
  25 };
  26
  27 pub mod acl;
  28 pub mod apt;
  29 pub mod async_io;
  30 pub mod borrow;
  31 pub mod cert;
  32 pub mod compression;
  33 pub mod config;
  34 pub mod cpio;
  35 pub mod daemon;
  36 pub mod disks;
  37 pub mod format;
  38 pub mod fs;
  39 pub mod fuse_loop;
  40
  41 pub mod json;
  42 pub mod logrotate;
  43 pub mod loopdev;
  44 pub mod lru_cache;
  45 pub mod nom;
  46 pub mod runtime;
  47 pub mod serde_filter;
  48 pub mod statistics;
  49 pub mod subscription;
  50 pub mod systemd;
  51 pub mod ticket;
  52 pub mod xattr;
  53 pub mod zip;
  54 pub mod sgutils2;
  55 pub mod paperkey;
  56
  57 pub mod parallel_handler;
  58 pub use parallel_handler::ParallelHandler;
  59
  60 mod wrapped_reader_stream;
  61 pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedReaderStream};
  62
  63 mod async_channel_writer;
  64 pub use async_channel_writer::AsyncChannelWriter;
  65
  66 mod std_channel_writer;
  67 pub use std_channel_writer::StdChannelWriter;
  68
  69 mod tokio_writer_adapter;
  70 pub use tokio_writer_adapter::TokioWriterAdapter;
  71
  72 mod process_locker;
  73 pub use process_locker::{ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard};
  74
  75 mod file_logger;
  76 pub use file_logger::{FileLogger, FileLogOptions};
  77
  78 mod broadcast_future;
  79 pub use broadcast_future::{BroadcastData, BroadcastFuture};
  80
  81 /// The `BufferedRead` trait provides a single function
  82 /// `buffered_read`. It returns a reference to an internal buffer. The
  83 /// purpose of this traid is to avoid unnecessary data copies.
  84 pub trait BufferedRead {
  85     /// This functions tries to fill the internal buffers, then
  86     /// returns a reference to the available data. It returns an empty
  87     /// buffer if `offset` points to the end of the file.
  88     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  89 }
  90
  91 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
  92     let mut query = url::form_urlencoded::Serializer::new(String::new());
  93
  94     let object = data.as_object().ok_or_else(|| {
  95         format_err!("json_object_to_query: got wrong data type (expected object).")
  96     })?;
  97
  98     for (key, value) in object {
  99         match value {
 100             Value::Bool(b) => {
 101                 query.append_pair(key, &b.to_string());
 102             }
 103             Value::Number(n) => {
 104                 query.append_pair(key, &n.to_string());
 105             }
 106             Value::String(s) => {
 107                 query.append_pair(key, &s);
 108             }
 109             Value::Array(arr) => {
 110                 for element in arr {
 111                     match element {
 112                         Value::Bool(b) => {
 113                             query.append_pair(key, &b.to_string());
 114                         }
 115                         Value::Number(n) => {
 116                             query.append_pair(key, &n.to_string());
 117                         }
 118                         Value::String(s) => {
 119                             query.append_pair(key, &s);
 120                         }
 121                         _ => bail!(
 122                             "json_object_to_query: unable to handle complex array data types."
 123                         ),
 124                     }
 125                 }
 126             }
 127             _ => bail!("json_object_to_query: unable to handle complex data types."),
 128         }
 129     }
 130
 131     Ok(query.finish())
 132 }
 133
 134 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 135     match param[name].as_str() {
 136         Some(s) => Ok(s),
 137         None => bail!("missing parameter '{}'", name),
 138     }
 139 }
 140
 141 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 142     match param[name].as_str() {
 143         Some(s) => Ok(s),
 144         None => bail!("missing property '{}'", name),
 145     }
 146 }
 147
 148 pub fn required_integer_param(param: &Value, name: &str) -> Result<i64, Error> {
 149     match param[name].as_i64() {
 150         Some(s) => Ok(s),
 151         None => bail!("missing parameter '{}'", name),
 152     }
 153 }
 154
 155 pub fn required_integer_property(param: &Value, name: &str) -> Result<i64, Error> {
 156     match param[name].as_i64() {
 157         Some(s) => Ok(s),
 158         None => bail!("missing property '{}'", name),
 159     }
 160 }
 161
 162 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 163     match param[name].as_array() {
 164         Some(s) => Ok(&s),
 165         None => bail!("missing parameter '{}'", name),
 166     }
 167 }
 168
 169 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 170     match param[name].as_array() {
 171         Some(s) => Ok(&s),
 172         None => bail!("missing property '{}'", name),
 173     }
 174 }
 175
 176 pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
 177 where
 178     S: BuildHasher,
 179 {
 180     let mut result = vec![];
 181
 182     use nix::fcntl::AtFlags;
 183     use nix::fcntl::OFlag;
 184     use nix::sys::stat::Mode;
 185
 186     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 187
 188     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 189         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 190         Err(_) => false,
 191     };
 192
 193     if !is_dir {
 194         if let Some(parent) = dirname.parent() {
 195             dirname = parent.to_owned();
 196         }
 197     }
 198
 199     let mut dir =
 200         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 201             Ok(d) => d,
 202             Err(_) => return result,
 203         };
 204
 205     for item in dir.iter() {
 206         if let Ok(entry) = item {
 207             if let Ok(name) = entry.file_name().to_str() {
 208                 if name == "." || name == ".." {
 209                     continue;
 210                 }
 211                 let mut newpath = dirname.clone();
 212                 newpath.push(name);
 213
 214                 if let Ok(stat) =
 215                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 216                 {
 217                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 218                         newpath.push("");
 219                         if let Some(newpath) = newpath.to_str() {
 220                             result.push(newpath.to_owned());
 221                         }
 222                         continue;
 223                     }
 224                 }
 225                 if let Some(newpath) = newpath.to_str() {
 226                     result.push(newpath.to_owned());
 227                 }
 228             }
 229         }
 230     }
 231
 232     result
 233 }
 234
 235 /// Scan directory for matching file names.
 236 ///
 237 /// Scan through all directory entries and call `callback()` function
 238 /// if the entry name matches the regular expression. This function
 239 /// used unix `openat()`, so you can pass absolute or relative file
 240 /// names. This function simply skips non-UTF8 encoded names.
 241 pub fn scandir<P, F>(
 242     dirfd: RawFd,
 243     path: &P,
 244     regex: &regex::Regex,
 245     mut callback: F,
 246 ) -> Result<(), Error>
 247 where
 248     F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>,
 249     P: ?Sized + nix::NixPath,
 250 {
 251     for entry in self::fs::scan_subdir(dirfd, path, regex)? {
 252         let entry = entry?;
 253         let file_type = match entry.file_type() {
 254             Some(file_type) => file_type,
 255             None => bail!("unable to detect file type"),
 256         };
 257
 258         callback(
 259             entry.parent_fd(),
 260             unsafe { entry.file_name_utf8_unchecked() },
 261             file_type,
 262         )?;
 263     }
 264     Ok(())
 265 }
 266
 267 /// Shortcut for md5 sums.
 268 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 269     hash(MessageDigest::md5(), data).map_err(Error::from)
 270 }
 271
 272 pub fn get_hardware_address() -> Result<String, Error> {
 273     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 274
 275     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 276         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 277     let digest = md5sum(&contents)
 278         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 279
 280     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 281 }
 282
 283 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 284     if digest1 != digest2 {
 285         bail!("detected modified configuration - file changed by other user? Try again.");
 286     }
 287     Ok(())
 288 }
 289
 290 /// Extract a specific cookie from cookie header.
 291 /// We assume cookie_name is already url encoded.
 292 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 293     for pair in cookie.split(';') {
 294         let (name, value) = match pair.find('=') {
 295             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 296             None => return None, // Cookie format error
 297         };
 298
 299         if name == cookie_name {
 300             use percent_encoding::percent_decode;
 301             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 302                 return Some(value.into());
 303             } else {
 304                 return None; // Cookie format error
 305             }
 306         }
 307     }
 308
 309     None
 310 }
 311
 312 /// percent encode a url component
 313 pub fn percent_encode_component(comp: &str) -> String {
 314     utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
 315 }
 316
 317 pub fn join<S: Borrow<str>>(data: &[S], sep: char) -> String {
 318     let mut list = String::new();
 319
 320     for item in data {
 321         if !list.is_empty() {
 322             list.push(sep);
 323         }
 324         list.push_str(item.borrow());
 325     }
 326
 327     list
 328 }
 329
 330 /// Detect modified configuration files
 331 ///
 332 /// This function fails with a reasonable error message if checksums do not match.
 333 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 334     if digest1 != digest2 {
 335         bail!("detected modified configuration - file changed by other user? Try again.");
 336     }
 337     Ok(())
 338 }
 339
 340 /// normalize uri path
 341 ///
 342 /// Do not allow ".", "..", or hidden files ".XXXX"
 343 /// Also remove empty path components
 344 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 345     let items = path.split('/');
 346
 347     let mut path = String::new();
 348     let mut components = vec![];
 349
 350     for name in items {
 351         if name.is_empty() {
 352             continue;
 353         }
 354         if name.starts_with('.') {
 355             bail!("Path contains illegal components.");
 356         }
 357         path.push('/');
 358         path.push_str(name);
 359         components.push(name);
 360     }
 361
 362     Ok((path, components))
 363 }
 364
 365 /// Helper to check result from std::process::Command output
 366 ///
 367 /// The exit_code_check() function should return true if the exit code
 368 /// is considered successful.
 369 pub fn command_output(
 370     output: std::process::Output,
 371     exit_code_check: Option<fn(i32) -> bool>,
 372 ) -> Result<Vec<u8>, Error> {
 373
 374     if !output.status.success() {
 375         match output.status.code() {
 376             Some(code) => {
 377                 let is_ok = match exit_code_check {
 378                     Some(check_fn) => check_fn(code),
 379                     None => code == 0,
 380                 };
 381                 if !is_ok {
 382                     let msg = String::from_utf8(output.stderr)
 383                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 384                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 385
 386                     bail!("status code: {} - {}", code, msg);
 387                 }
 388             }
 389             None => bail!("terminated by signal"),
 390         }
 391     }
 392
 393     Ok(output.stdout)
 394 }
 395
 396 /// Helper to check result from std::process::Command output, returns String.
 397 ///
 398 /// The exit_code_check() function should return true if the exit code
 399 /// is considered successful.
 400 pub fn command_output_as_string(
 401     output: std::process::Output,
 402     exit_code_check: Option<fn(i32) -> bool>,
 403 ) -> Result<String, Error> {
 404     let output = command_output(output, exit_code_check)?;
 405     let output = String::from_utf8(output)?;
 406     Ok(output)
 407 }
 408
 409 pub fn run_command(
 410     mut command: std::process::Command,
 411     exit_code_check: Option<fn(i32) -> bool>,
 412 ) -> Result<String, Error> {
 413
 414    let output = command.output()
 415         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 416
 417     let output = command_output_as_string(output, exit_code_check)
 418         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 419
 420     Ok(output)
 421 }
 422
 423 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 424     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 425     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 426         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 427     flags.set(FdFlag::FD_CLOEXEC, on);
 428     fcntl(fd, F_SETFD(flags))?;
 429     Ok(())
 430 }
 431
 432 static mut SHUTDOWN_REQUESTED: bool = false;
 433
 434 pub fn request_shutdown() {
 435     unsafe {
 436         SHUTDOWN_REQUESTED = true;
 437     }
 438     crate::server::server_shutdown();
 439 }
 440
 441 #[inline(always)]
 442 pub fn shutdown_requested() -> bool {
 443     unsafe { SHUTDOWN_REQUESTED }
 444 }
 445
 446 pub fn fail_on_shutdown() -> Result<(), Error> {
 447     if shutdown_requested() {
 448         bail!("Server shutdown requested - aborting task");
 449     }
 450     Ok(())
 451 }
 452
 453 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 454 /// descriptors.
 455 pub fn pipe() -> Result<(Fd, Fd), Error> {
 456     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 457     Ok((Fd(pin), Fd(pout)))
 458 }
 459
 460 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 461 /// descriptors.
 462 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 463     use nix::sys::socket;
 464     let (pa, pb) = socket::socketpair(
 465         socket::AddressFamily::Unix,
 466         socket::SockType::Stream,
 467         None,
 468         socket::SockFlag::SOCK_CLOEXEC,
 469     )?;
 470     Ok((Fd(pa), Fd(pb)))
 471 }
 472
 473
 474 /// An easy way to convert types to Any
 475 ///
 476 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 477 pub trait AsAny {
 478     fn as_any(&self) -> &dyn Any;
 479 }
 480
 481 impl<T: Any> AsAny for T {
 482     fn as_any(&self) -> &dyn Any {
 483         self
 484     }
 485 }
 486
 487 /// The default 2 hours are far too long for PBS
 488 pub const PROXMOX_BACKUP_TCP_KEEPALIVE_TIME: u32 = 120;
 489 pub const DEFAULT_USER_AGENT_STRING: &'static str = "proxmox-backup-client/1.0";
 490
 491 /// Returns a new instance of `SimpleHttp` configured for PBS usage.
 492 pub fn pbs_simple_http(proxy_config: Option<ProxyConfig>) -> SimpleHttp {
 493     let options = SimpleHttpOptions {
 494         proxy_config,
 495         user_agent: Some(DEFAULT_USER_AGENT_STRING.to_string()),
 496         tcp_keepalive: Some(PROXMOX_BACKUP_TCP_KEEPALIVE_TIME),
 497         ..Default::default()
 498     };
 499
 500     SimpleHttp::with_options(options)
 501 }
 502
 503 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 504 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 505     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 506     .add(0x20)
 507     .add(0x7f)
 508     // the DEFAULT_ENCODE_SET added:
 509     .add(b' ')
 510     .add(b'"')
 511     .add(b'#')
 512     .add(b'<')
 513     .add(b'>')
 514     .add(b'`')
 515     .add(b'?')
 516     .add(b'{')
 517     .add(b'}');
 518
 519 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 520 /// `#`).
 521 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 522     path: P,
 523 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 524     let path = path.as_ref();
 525
 526     Ok(io::BufReader::new(
 527         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 528     )
 529     .lines()
 530     .filter_map(|line| match line {
 531         Ok(line) => {
 532             let line = line.trim();
 533             if line.is_empty() || line.starts_with('#') {
 534                 None
 535             } else {
 536                 Some(Ok(line.to_string()))
 537             }
 538         }
 539         Err(err) => Some(Err(err)),
 540     }))
 541 }
 542
 543 pub fn setup_safe_path_env() {
 544     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 545     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 546     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 547         std::env::remove_var(name);
 548     }
 549 }
 550
 551 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 552     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 553         Some(n) => &line[n..],
 554         None => return &[],
 555     };
 556     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 557         Some(n) => &line[..(line.len() - n)],
 558         None => &[],
 559     }
 560 }
 561
 562 /// Seeks to start of file and computes the SHA256 hash
 563 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 564
 565     file.seek(SeekFrom::Start(0))?;
 566
 567     let mut hasher = openssl::sha::Sha256::new();
 568     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 569     let mut size: u64 = 0;
 570
 571     loop {
 572         let count = match file.read(&mut buffer) {
 573             Ok(0) => break,
 574             Ok(count) => count,
 575             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 576                 continue;
 577             }
 578             Err(err) => return Err(err.into()),
 579         };
 580         size += count as u64;
 581         hasher.update(&buffer[..count]);
 582     }
 583
 584     let csum = hasher.finish();
 585
 586     Ok((csum, size))
 587 }
 588
 589 /// Create the base run-directory.
 590 ///
 591 /// This exists to fixate the permissions for the run *base* directory while allowing intermediate
 592 /// directories after it to have different permissions.
 593 pub fn create_run_dir() -> Result<(), Error> {
 594     let backup_user = crate::backup::backup_user()?;
 595     let opts = CreateOptions::new()
 596         .owner(backup_user.uid)
 597         .group(backup_user.gid);
 598     let _: bool = create_path(PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
 599     Ok(())
 600 }
 601
 602 /// Modeled after the nightly `std::ops::ControlFlow`.
 603 #[derive(Clone, Copy, Debug, PartialEq)]
 604 pub enum ControlFlow<B, C = ()> {
 605     Continue(C),
 606     Break(B),
 607 }
 608
 609 impl<B> ControlFlow<B> {
 610     pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
 611 }