src/tools/mod.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::{utf8_percent_encode, AsciiSet};
  16
  17 pub use proxmox::tools::fd::Fd;
  18 use proxmox::tools::fs::{create_path, CreateOptions};
  19
  20 use proxmox_http::{
  21     client::SimpleHttp,
  22     client::SimpleHttpOptions,
  23     ProxyConfig,
  24 };
  25
  26 pub mod acl;
  27 pub mod apt;
  28 pub mod async_io;
  29 pub mod cert;
  30 pub mod compression;
  31 pub mod config;
  32 pub mod cpio;
  33 pub mod daemon;
  34 pub mod disks;
  35 pub mod fuse_loop;
  36
  37 mod memcom;
  38 pub use memcom::Memcom;
  39
  40 pub mod json;
  41 pub mod logrotate;
  42 pub mod loopdev;
  43 pub mod lru_cache;
  44 pub mod async_lru_cache;
  45 pub mod nom;
  46 pub mod serde_filter;
  47 pub mod statistics;
  48 pub mod subscription;
  49 pub mod systemd;
  50 pub mod ticket;
  51 pub mod xattr;
  52 pub mod zip;
  53 pub mod sgutils2;
  54 pub mod paperkey;
  55
  56 pub mod parallel_handler;
  57 pub use parallel_handler::ParallelHandler;
  58
  59 mod wrapped_reader_stream;
  60 pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedReaderStream};
  61
  62 mod async_channel_writer;
  63 pub use async_channel_writer::AsyncChannelWriter;
  64
  65 mod std_channel_writer;
  66 pub use std_channel_writer::StdChannelWriter;
  67
  68 mod tokio_writer_adapter;
  69 pub use tokio_writer_adapter::TokioWriterAdapter;
  70
  71 mod process_locker;
  72 pub use process_locker::{ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard};
  73
  74 mod file_logger;
  75 pub use file_logger::{FileLogger, FileLogOptions};
  76
  77 mod broadcast_future;
  78 pub use broadcast_future::{BroadcastData, BroadcastFuture};
  79
  80 /// The `BufferedRead` trait provides a single function
  81 /// `buffered_read`. It returns a reference to an internal buffer. The
  82 /// purpose of this traid is to avoid unnecessary data copies.
  83 pub trait BufferedRead {
  84     /// This functions tries to fill the internal buffers, then
  85     /// returns a reference to the available data. It returns an empty
  86     /// buffer if `offset` points to the end of the file.
  87     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  88 }
  89
  90 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
  91     let mut query = url::form_urlencoded::Serializer::new(String::new());
  92
  93     let object = data.as_object().ok_or_else(|| {
  94         format_err!("json_object_to_query: got wrong data type (expected object).")
  95     })?;
  96
  97     for (key, value) in object {
  98         match value {
  99             Value::Bool(b) => {
 100                 query.append_pair(key, &b.to_string());
 101             }
 102             Value::Number(n) => {
 103                 query.append_pair(key, &n.to_string());
 104             }
 105             Value::String(s) => {
 106                 query.append_pair(key, &s);
 107             }
 108             Value::Array(arr) => {
 109                 for element in arr {
 110                     match element {
 111                         Value::Bool(b) => {
 112                             query.append_pair(key, &b.to_string());
 113                         }
 114                         Value::Number(n) => {
 115                             query.append_pair(key, &n.to_string());
 116                         }
 117                         Value::String(s) => {
 118                             query.append_pair(key, &s);
 119                         }
 120                         _ => bail!(
 121                             "json_object_to_query: unable to handle complex array data types."
 122                         ),
 123                     }
 124                 }
 125             }
 126             _ => bail!("json_object_to_query: unable to handle complex data types."),
 127         }
 128     }
 129
 130     Ok(query.finish())
 131 }
 132
 133 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 134     match param[name].as_str() {
 135         Some(s) => Ok(s),
 136         None => bail!("missing parameter '{}'", name),
 137     }
 138 }
 139
 140 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 141     match param[name].as_str() {
 142         Some(s) => Ok(s),
 143         None => bail!("missing property '{}'", name),
 144     }
 145 }
 146
 147 pub fn required_integer_param(param: &Value, name: &str) -> Result<i64, Error> {
 148     match param[name].as_i64() {
 149         Some(s) => Ok(s),
 150         None => bail!("missing parameter '{}'", name),
 151     }
 152 }
 153
 154 pub fn required_integer_property(param: &Value, name: &str) -> Result<i64, Error> {
 155     match param[name].as_i64() {
 156         Some(s) => Ok(s),
 157         None => bail!("missing property '{}'", name),
 158     }
 159 }
 160
 161 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 162     match param[name].as_array() {
 163         Some(s) => Ok(&s),
 164         None => bail!("missing parameter '{}'", name),
 165     }
 166 }
 167
 168 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 169     match param[name].as_array() {
 170         Some(s) => Ok(&s),
 171         None => bail!("missing property '{}'", name),
 172     }
 173 }
 174
 175 pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
 176 where
 177     S: BuildHasher,
 178 {
 179     let mut result = vec![];
 180
 181     use nix::fcntl::AtFlags;
 182     use nix::fcntl::OFlag;
 183     use nix::sys::stat::Mode;
 184
 185     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 186
 187     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 188         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 189         Err(_) => false,
 190     };
 191
 192     if !is_dir {
 193         if let Some(parent) = dirname.parent() {
 194             dirname = parent.to_owned();
 195         }
 196     }
 197
 198     let mut dir =
 199         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 200             Ok(d) => d,
 201             Err(_) => return result,
 202         };
 203
 204     for item in dir.iter() {
 205         if let Ok(entry) = item {
 206             if let Ok(name) = entry.file_name().to_str() {
 207                 if name == "." || name == ".." {
 208                     continue;
 209                 }
 210                 let mut newpath = dirname.clone();
 211                 newpath.push(name);
 212
 213                 if let Ok(stat) =
 214                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 215                 {
 216                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 217                         newpath.push("");
 218                         if let Some(newpath) = newpath.to_str() {
 219                             result.push(newpath.to_owned());
 220                         }
 221                         continue;
 222                     }
 223                 }
 224                 if let Some(newpath) = newpath.to_str() {
 225                     result.push(newpath.to_owned());
 226                 }
 227             }
 228         }
 229     }
 230
 231     result
 232 }
 233
 234 /// Shortcut for md5 sums.
 235 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 236     hash(MessageDigest::md5(), data).map_err(Error::from)
 237 }
 238
 239 pub fn get_hardware_address() -> Result<String, Error> {
 240     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 241
 242     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 243         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 244     let digest = md5sum(&contents)
 245         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 246
 247     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 248 }
 249
 250 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 251     if digest1 != digest2 {
 252         bail!("detected modified configuration - file changed by other user? Try again.");
 253     }
 254     Ok(())
 255 }
 256
 257 /// Extract a specific cookie from cookie header.
 258 /// We assume cookie_name is already url encoded.
 259 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 260     for pair in cookie.split(';') {
 261         let (name, value) = match pair.find('=') {
 262             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 263             None => return None, // Cookie format error
 264         };
 265
 266         if name == cookie_name {
 267             use percent_encoding::percent_decode;
 268             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 269                 return Some(value.into());
 270             } else {
 271                 return None; // Cookie format error
 272             }
 273         }
 274     }
 275
 276     None
 277 }
 278
 279 /// percent encode a url component
 280 pub fn percent_encode_component(comp: &str) -> String {
 281     utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
 282 }
 283
 284 /// Detect modified configuration files
 285 ///
 286 /// This function fails with a reasonable error message if checksums do not match.
 287 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 288     if digest1 != digest2 {
 289         bail!("detected modified configuration - file changed by other user? Try again.");
 290     }
 291     Ok(())
 292 }
 293
 294 /// normalize uri path
 295 ///
 296 /// Do not allow ".", "..", or hidden files ".XXXX"
 297 /// Also remove empty path components
 298 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 299     let items = path.split('/');
 300
 301     let mut path = String::new();
 302     let mut components = vec![];
 303
 304     for name in items {
 305         if name.is_empty() {
 306             continue;
 307         }
 308         if name.starts_with('.') {
 309             bail!("Path contains illegal components.");
 310         }
 311         path.push('/');
 312         path.push_str(name);
 313         components.push(name);
 314     }
 315
 316     Ok((path, components))
 317 }
 318
 319 /// Helper to check result from std::process::Command output
 320 ///
 321 /// The exit_code_check() function should return true if the exit code
 322 /// is considered successful.
 323 pub fn command_output(
 324     output: std::process::Output,
 325     exit_code_check: Option<fn(i32) -> bool>,
 326 ) -> Result<Vec<u8>, Error> {
 327
 328     if !output.status.success() {
 329         match output.status.code() {
 330             Some(code) => {
 331                 let is_ok = match exit_code_check {
 332                     Some(check_fn) => check_fn(code),
 333                     None => code == 0,
 334                 };
 335                 if !is_ok {
 336                     let msg = String::from_utf8(output.stderr)
 337                         .map(|m| if m.is_empty() { String::from("no error message") } else { m })
 338                         .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
 339
 340                     bail!("status code: {} - {}", code, msg);
 341                 }
 342             }
 343             None => bail!("terminated by signal"),
 344         }
 345     }
 346
 347     Ok(output.stdout)
 348 }
 349
 350 /// Helper to check result from std::process::Command output, returns String.
 351 ///
 352 /// The exit_code_check() function should return true if the exit code
 353 /// is considered successful.
 354 pub fn command_output_as_string(
 355     output: std::process::Output,
 356     exit_code_check: Option<fn(i32) -> bool>,
 357 ) -> Result<String, Error> {
 358     let output = command_output(output, exit_code_check)?;
 359     let output = String::from_utf8(output)?;
 360     Ok(output)
 361 }
 362
 363 pub fn run_command(
 364     mut command: std::process::Command,
 365     exit_code_check: Option<fn(i32) -> bool>,
 366 ) -> Result<String, Error> {
 367
 368    let output = command.output()
 369         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 370
 371     let output = command_output_as_string(output, exit_code_check)
 372         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 373
 374     Ok(output)
 375 }
 376
 377 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 378     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 379     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 380         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 381     flags.set(FdFlag::FD_CLOEXEC, on);
 382     fcntl(fd, F_SETFD(flags))?;
 383     Ok(())
 384 }
 385
 386 static mut SHUTDOWN_REQUESTED: bool = false;
 387
 388 pub fn request_shutdown() {
 389     unsafe {
 390         SHUTDOWN_REQUESTED = true;
 391     }
 392     crate::server::server_shutdown();
 393 }
 394
 395 #[inline(always)]
 396 pub fn shutdown_requested() -> bool {
 397     unsafe { SHUTDOWN_REQUESTED }
 398 }
 399
 400 pub fn fail_on_shutdown() -> Result<(), Error> {
 401     if shutdown_requested() {
 402         bail!("Server shutdown requested - aborting task");
 403     }
 404     Ok(())
 405 }
 406
 407 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 408 /// descriptors.
 409 pub fn pipe() -> Result<(Fd, Fd), Error> {
 410     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 411     Ok((Fd(pin), Fd(pout)))
 412 }
 413
 414 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 415 /// descriptors.
 416 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 417     use nix::sys::socket;
 418     let (pa, pb) = socket::socketpair(
 419         socket::AddressFamily::Unix,
 420         socket::SockType::Stream,
 421         None,
 422         socket::SockFlag::SOCK_CLOEXEC,
 423     )?;
 424     Ok((Fd(pa), Fd(pb)))
 425 }
 426
 427
 428 /// An easy way to convert types to Any
 429 ///
 430 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 431 pub trait AsAny {
 432     fn as_any(&self) -> &dyn Any;
 433 }
 434
 435 impl<T: Any> AsAny for T {
 436     fn as_any(&self) -> &dyn Any {
 437         self
 438     }
 439 }
 440
 441 /// The default 2 hours are far too long for PBS
 442 pub const PROXMOX_BACKUP_TCP_KEEPALIVE_TIME: u32 = 120;
 443 pub const DEFAULT_USER_AGENT_STRING: &'static str = "proxmox-backup-client/1.0";
 444
 445 /// Returns a new instance of `SimpleHttp` configured for PBS usage.
 446 pub fn pbs_simple_http(proxy_config: Option<ProxyConfig>) -> SimpleHttp {
 447     let options = SimpleHttpOptions {
 448         proxy_config,
 449         user_agent: Some(DEFAULT_USER_AGENT_STRING.to_string()),
 450         tcp_keepalive: Some(PROXMOX_BACKUP_TCP_KEEPALIVE_TIME),
 451         ..Default::default()
 452     };
 453
 454     SimpleHttp::with_options(options)
 455 }
 456
 457 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 458 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 459     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 460     .add(0x20)
 461     .add(0x7f)
 462     // the DEFAULT_ENCODE_SET added:
 463     .add(b' ')
 464     .add(b'"')
 465     .add(b'#')
 466     .add(b'<')
 467     .add(b'>')
 468     .add(b'`')
 469     .add(b'?')
 470     .add(b'{')
 471     .add(b'}');
 472
 473 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 474 /// `#`).
 475 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 476     path: P,
 477 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 478     let path = path.as_ref();
 479
 480     Ok(io::BufReader::new(
 481         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 482     )
 483     .lines()
 484     .filter_map(|line| match line {
 485         Ok(line) => {
 486             let line = line.trim();
 487             if line.is_empty() || line.starts_with('#') {
 488                 None
 489             } else {
 490                 Some(Ok(line.to_string()))
 491             }
 492         }
 493         Err(err) => Some(Err(err)),
 494     }))
 495 }
 496
 497 pub fn setup_safe_path_env() {
 498     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 499     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 500     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 501         std::env::remove_var(name);
 502     }
 503 }
 504
 505 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 506     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 507         Some(n) => &line[n..],
 508         None => return &[],
 509     };
 510     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 511         Some(n) => &line[..(line.len() - n)],
 512         None => &[],
 513     }
 514 }
 515
 516 /// Seeks to start of file and computes the SHA256 hash
 517 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 518
 519     file.seek(SeekFrom::Start(0))?;
 520
 521     let mut hasher = openssl::sha::Sha256::new();
 522     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 523     let mut size: u64 = 0;
 524
 525     loop {
 526         let count = match file.read(&mut buffer) {
 527             Ok(0) => break,
 528             Ok(count) => count,
 529             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 530                 continue;
 531             }
 532             Err(err) => return Err(err.into()),
 533         };
 534         size += count as u64;
 535         hasher.update(&buffer[..count]);
 536     }
 537
 538     let csum = hasher.finish();
 539
 540     Ok((csum, size))
 541 }
 542
 543 /// Create the base run-directory.
 544 ///
 545 /// This exists to fixate the permissions for the run *base* directory while allowing intermediate
 546 /// directories after it to have different permissions.
 547 pub fn create_run_dir() -> Result<(), Error> {
 548     let backup_user = crate::backup::backup_user()?;
 549     let opts = CreateOptions::new()
 550         .owner(backup_user.uid)
 551         .group(backup_user.gid);
 552     let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
 553     Ok(())
 554 }
 555
 556 /// Modeled after the nightly `std::ops::ControlFlow`.
 557 #[derive(Clone, Copy, Debug, PartialEq)]
 558 pub enum ControlFlow<B, C = ()> {
 559     Continue(C),
 560     Break(B),
 561 }
 562
 563 impl<B> ControlFlow<B> {
 564     pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
 565 }