src/tools/mod.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead, Read, Seek, SeekFrom};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15 use percent_encoding::{utf8_percent_encode, AsciiSet};
  16
  17 pub use proxmox::tools::fd::Fd;
  18 use proxmox::tools::fs::{create_path, CreateOptions};
  19
  20 use proxmox_http::{
  21     client::SimpleHttp,
  22     client::SimpleHttpOptions,
  23     ProxyConfig,
  24 };
  25
  26 pub use pbs_tools::{run_command, command_output, command_output_as_string};
  27
  28 pub mod acl;
  29 pub mod apt;
  30 pub mod async_io;
  31 pub mod cert;
  32 pub mod compression;
  33 pub mod config;
  34 pub mod cpio;
  35 pub mod daemon;
  36 pub mod disks;
  37 pub mod fuse_loop;
  38
  39 mod memcom;
  40 pub use memcom::Memcom;
  41
  42 pub mod json;
  43 pub mod logrotate;
  44 pub mod loopdev;
  45 pub mod lru_cache;
  46 pub mod async_lru_cache;
  47 pub mod nom;
  48 pub mod serde_filter;
  49 pub mod statistics;
  50 pub mod subscription;
  51 pub mod systemd;
  52 pub mod ticket;
  53 pub mod xattr;
  54 pub mod zip;
  55 pub mod sgutils2;
  56 pub mod paperkey;
  57
  58 pub mod parallel_handler;
  59 pub use parallel_handler::ParallelHandler;
  60
  61 mod wrapped_reader_stream;
  62 pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedReaderStream};
  63
  64 mod async_channel_writer;
  65 pub use async_channel_writer::AsyncChannelWriter;
  66
  67 mod std_channel_writer;
  68 pub use std_channel_writer::StdChannelWriter;
  69
  70 mod tokio_writer_adapter;
  71 pub use tokio_writer_adapter::TokioWriterAdapter;
  72
  73 mod process_locker;
  74 pub use process_locker::{ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard};
  75
  76 mod file_logger;
  77 pub use file_logger::{FileLogger, FileLogOptions};
  78
  79 mod broadcast_future;
  80 pub use broadcast_future::{BroadcastData, BroadcastFuture};
  81
  82 /// The `BufferedRead` trait provides a single function
  83 /// `buffered_read`. It returns a reference to an internal buffer. The
  84 /// purpose of this traid is to avoid unnecessary data copies.
  85 pub trait BufferedRead {
  86     /// This functions tries to fill the internal buffers, then
  87     /// returns a reference to the available data. It returns an empty
  88     /// buffer if `offset` points to the end of the file.
  89     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  90 }
  91
  92 pub fn json_object_to_query(data: Value) -> Result<String, Error> {
  93     let mut query = url::form_urlencoded::Serializer::new(String::new());
  94
  95     let object = data.as_object().ok_or_else(|| {
  96         format_err!("json_object_to_query: got wrong data type (expected object).")
  97     })?;
  98
  99     for (key, value) in object {
 100         match value {
 101             Value::Bool(b) => {
 102                 query.append_pair(key, &b.to_string());
 103             }
 104             Value::Number(n) => {
 105                 query.append_pair(key, &n.to_string());
 106             }
 107             Value::String(s) => {
 108                 query.append_pair(key, &s);
 109             }
 110             Value::Array(arr) => {
 111                 for element in arr {
 112                     match element {
 113                         Value::Bool(b) => {
 114                             query.append_pair(key, &b.to_string());
 115                         }
 116                         Value::Number(n) => {
 117                             query.append_pair(key, &n.to_string());
 118                         }
 119                         Value::String(s) => {
 120                             query.append_pair(key, &s);
 121                         }
 122                         _ => bail!(
 123                             "json_object_to_query: unable to handle complex array data types."
 124                         ),
 125                     }
 126                 }
 127             }
 128             _ => bail!("json_object_to_query: unable to handle complex data types."),
 129         }
 130     }
 131
 132     Ok(query.finish())
 133 }
 134
 135 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 136     match param[name].as_str() {
 137         Some(s) => Ok(s),
 138         None => bail!("missing parameter '{}'", name),
 139     }
 140 }
 141
 142 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
 143     match param[name].as_str() {
 144         Some(s) => Ok(s),
 145         None => bail!("missing property '{}'", name),
 146     }
 147 }
 148
 149 pub fn required_integer_param(param: &Value, name: &str) -> Result<i64, Error> {
 150     match param[name].as_i64() {
 151         Some(s) => Ok(s),
 152         None => bail!("missing parameter '{}'", name),
 153     }
 154 }
 155
 156 pub fn required_integer_property(param: &Value, name: &str) -> Result<i64, Error> {
 157     match param[name].as_i64() {
 158         Some(s) => Ok(s),
 159         None => bail!("missing property '{}'", name),
 160     }
 161 }
 162
 163 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 164     match param[name].as_array() {
 165         Some(s) => Ok(&s),
 166         None => bail!("missing parameter '{}'", name),
 167     }
 168 }
 169
 170 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 171     match param[name].as_array() {
 172         Some(s) => Ok(&s),
 173         None => bail!("missing property '{}'", name),
 174     }
 175 }
 176
 177 pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
 178 where
 179     S: BuildHasher,
 180 {
 181     let mut result = vec![];
 182
 183     use nix::fcntl::AtFlags;
 184     use nix::fcntl::OFlag;
 185     use nix::sys::stat::Mode;
 186
 187     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 188
 189     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 190         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 191         Err(_) => false,
 192     };
 193
 194     if !is_dir {
 195         if let Some(parent) = dirname.parent() {
 196             dirname = parent.to_owned();
 197         }
 198     }
 199
 200     let mut dir =
 201         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 202             Ok(d) => d,
 203             Err(_) => return result,
 204         };
 205
 206     for item in dir.iter() {
 207         if let Ok(entry) = item {
 208             if let Ok(name) = entry.file_name().to_str() {
 209                 if name == "." || name == ".." {
 210                     continue;
 211                 }
 212                 let mut newpath = dirname.clone();
 213                 newpath.push(name);
 214
 215                 if let Ok(stat) =
 216                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 217                 {
 218                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 219                         newpath.push("");
 220                         if let Some(newpath) = newpath.to_str() {
 221                             result.push(newpath.to_owned());
 222                         }
 223                         continue;
 224                     }
 225                 }
 226                 if let Some(newpath) = newpath.to_str() {
 227                     result.push(newpath.to_owned());
 228                 }
 229             }
 230         }
 231     }
 232
 233     result
 234 }
 235
 236 /// Shortcut for md5 sums.
 237 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 238     hash(MessageDigest::md5(), data).map_err(Error::from)
 239 }
 240
 241 pub fn get_hardware_address() -> Result<String, Error> {
 242     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 243
 244     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 245         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 246     let digest = md5sum(&contents)
 247         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 248
 249     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 250 }
 251
 252 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 253     if digest1 != digest2 {
 254         bail!("detected modified configuration - file changed by other user? Try again.");
 255     }
 256     Ok(())
 257 }
 258
 259 /// Extract a specific cookie from cookie header.
 260 /// We assume cookie_name is already url encoded.
 261 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 262     for pair in cookie.split(';') {
 263         let (name, value) = match pair.find('=') {
 264             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 265             None => return None, // Cookie format error
 266         };
 267
 268         if name == cookie_name {
 269             use percent_encoding::percent_decode;
 270             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 271                 return Some(value.into());
 272             } else {
 273                 return None; // Cookie format error
 274             }
 275         }
 276     }
 277
 278     None
 279 }
 280
 281 /// percent encode a url component
 282 pub fn percent_encode_component(comp: &str) -> String {
 283     utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
 284 }
 285
 286 /// Detect modified configuration files
 287 ///
 288 /// This function fails with a reasonable error message if checksums do not match.
 289 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 290     if digest1 != digest2 {
 291         bail!("detected modified configuration - file changed by other user? Try again.");
 292     }
 293     Ok(())
 294 }
 295
 296 /// normalize uri path
 297 ///
 298 /// Do not allow ".", "..", or hidden files ".XXXX"
 299 /// Also remove empty path components
 300 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 301     let items = path.split('/');
 302
 303     let mut path = String::new();
 304     let mut components = vec![];
 305
 306     for name in items {
 307         if name.is_empty() {
 308             continue;
 309         }
 310         if name.starts_with('.') {
 311             bail!("Path contains illegal components.");
 312         }
 313         path.push('/');
 314         path.push_str(name);
 315         components.push(name);
 316     }
 317
 318     Ok((path, components))
 319 }
 320
 321 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 322     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 323     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 324         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 325     flags.set(FdFlag::FD_CLOEXEC, on);
 326     fcntl(fd, F_SETFD(flags))?;
 327     Ok(())
 328 }
 329
 330 static mut SHUTDOWN_REQUESTED: bool = false;
 331
 332 pub fn request_shutdown() {
 333     unsafe {
 334         SHUTDOWN_REQUESTED = true;
 335     }
 336     crate::server::server_shutdown();
 337 }
 338
 339 #[inline(always)]
 340 pub fn shutdown_requested() -> bool {
 341     unsafe { SHUTDOWN_REQUESTED }
 342 }
 343
 344 pub fn fail_on_shutdown() -> Result<(), Error> {
 345     if shutdown_requested() {
 346         bail!("Server shutdown requested - aborting task");
 347     }
 348     Ok(())
 349 }
 350
 351 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 352 /// descriptors.
 353 pub fn pipe() -> Result<(Fd, Fd), Error> {
 354     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 355     Ok((Fd(pin), Fd(pout)))
 356 }
 357
 358 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 359 /// descriptors.
 360 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 361     use nix::sys::socket;
 362     let (pa, pb) = socket::socketpair(
 363         socket::AddressFamily::Unix,
 364         socket::SockType::Stream,
 365         None,
 366         socket::SockFlag::SOCK_CLOEXEC,
 367     )?;
 368     Ok((Fd(pa), Fd(pb)))
 369 }
 370
 371
 372 /// An easy way to convert types to Any
 373 ///
 374 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 375 pub trait AsAny {
 376     fn as_any(&self) -> &dyn Any;
 377 }
 378
 379 impl<T: Any> AsAny for T {
 380     fn as_any(&self) -> &dyn Any {
 381         self
 382     }
 383 }
 384
 385 /// The default 2 hours are far too long for PBS
 386 pub const PROXMOX_BACKUP_TCP_KEEPALIVE_TIME: u32 = 120;
 387 pub const DEFAULT_USER_AGENT_STRING: &'static str = "proxmox-backup-client/1.0";
 388
 389 /// Returns a new instance of `SimpleHttp` configured for PBS usage.
 390 pub fn pbs_simple_http(proxy_config: Option<ProxyConfig>) -> SimpleHttp {
 391     let options = SimpleHttpOptions {
 392         proxy_config,
 393         user_agent: Some(DEFAULT_USER_AGENT_STRING.to_string()),
 394         tcp_keepalive: Some(PROXMOX_BACKUP_TCP_KEEPALIVE_TIME),
 395         ..Default::default()
 396     };
 397
 398     SimpleHttp::with_options(options)
 399 }
 400
 401 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
 402 pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
 403     // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
 404     .add(0x20)
 405     .add(0x7f)
 406     // the DEFAULT_ENCODE_SET added:
 407     .add(b' ')
 408     .add(b'"')
 409     .add(b'#')
 410     .add(b'<')
 411     .add(b'>')
 412     .add(b'`')
 413     .add(b'?')
 414     .add(b'{')
 415     .add(b'}');
 416
 417 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 418 /// `#`).
 419 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 420     path: P,
 421 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 422     let path = path.as_ref();
 423
 424     Ok(io::BufReader::new(
 425         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 426     )
 427     .lines()
 428     .filter_map(|line| match line {
 429         Ok(line) => {
 430             let line = line.trim();
 431             if line.is_empty() || line.starts_with('#') {
 432                 None
 433             } else {
 434                 Some(Ok(line.to_string()))
 435             }
 436         }
 437         Err(err) => Some(Err(err)),
 438     }))
 439 }
 440
 441 pub fn setup_safe_path_env() {
 442     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 443     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 444     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 445         std::env::remove_var(name);
 446     }
 447 }
 448
 449 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 450     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 451         Some(n) => &line[n..],
 452         None => return &[],
 453     };
 454     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 455         Some(n) => &line[..(line.len() - n)],
 456         None => &[],
 457     }
 458 }
 459
 460 /// Seeks to start of file and computes the SHA256 hash
 461 pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
 462
 463     file.seek(SeekFrom::Start(0))?;
 464
 465     let mut hasher = openssl::sha::Sha256::new();
 466     let mut buffer = proxmox::tools::vec::undefined(256*1024);
 467     let mut size: u64 = 0;
 468
 469     loop {
 470         let count = match file.read(&mut buffer) {
 471             Ok(0) => break,
 472             Ok(count) => count,
 473             Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
 474                 continue;
 475             }
 476             Err(err) => return Err(err.into()),
 477         };
 478         size += count as u64;
 479         hasher.update(&buffer[..count]);
 480     }
 481
 482     let csum = hasher.finish();
 483
 484     Ok((csum, size))
 485 }
 486
 487 /// Create the base run-directory.
 488 ///
 489 /// This exists to fixate the permissions for the run *base* directory while allowing intermediate
 490 /// directories after it to have different permissions.
 491 pub fn create_run_dir() -> Result<(), Error> {
 492     let backup_user = crate::backup::backup_user()?;
 493     let opts = CreateOptions::new()
 494         .owner(backup_user.uid)
 495         .group(backup_user.gid);
 496     let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
 497     Ok(())
 498 }
 499
 500 /// Modeled after the nightly `std::ops::ControlFlow`.
 501 #[derive(Clone, Copy, Debug, PartialEq)]
 502 pub enum ControlFlow<B, C = ()> {
 503     Continue(C),
 504     Break(B),
 505 }
 506
 507 impl<B> ControlFlow<B> {
 508     pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
 509 }