src/tools/mod.rs

   1 //! Tools and utilities
   2 //!
   3 //! This is a collection of small and useful tools.
   4 use std::any::Any;
   5 use std::collections::HashMap;
   6 use std::hash::BuildHasher;
   7 use std::fs::File;
   8 use std::io::{self, BufRead};
   9 use std::os::unix::io::RawFd;
  10 use std::path::Path;
  11
  12 use anyhow::{bail, format_err, Error};
  13 use serde_json::Value;
  14 use openssl::hash::{hash, DigestBytes, MessageDigest};
  15
  16 pub use proxmox::tools::fd::Fd;
  17 use proxmox::tools::fs::{create_path, CreateOptions};
  18
  19 use proxmox_http::{
  20     client::SimpleHttp,
  21     client::SimpleHttpOptions,
  22     ProxyConfig,
  23 };
  24
  25 pub use pbs_tools::json;
  26 pub use pbs_tools::nom;
  27 pub use pbs_tools::{run_command, command_output, command_output_as_string};
  28 pub use pbs_tools::process_locker::{
  29     ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard
  30 };
  31
  32 pub mod acl;
  33 pub mod apt;
  34 pub mod async_io;
  35 pub mod compression;
  36 pub mod config;
  37 pub mod cpio;
  38 pub mod daemon;
  39 pub mod disks;
  40 pub mod fuse_loop;
  41
  42 mod memcom;
  43 pub use memcom::Memcom;
  44
  45 pub mod logrotate;
  46 pub mod loopdev;
  47 pub mod lru_cache;
  48 pub mod async_lru_cache;
  49 pub mod serde_filter;
  50 pub mod statistics;
  51 pub mod subscription;
  52 pub mod systemd;
  53 pub mod ticket;
  54 pub mod xattr;
  55 pub mod zip;
  56 pub mod sgutils2;
  57 pub mod paperkey;
  58
  59 pub mod parallel_handler;
  60 pub use parallel_handler::ParallelHandler;
  61
  62 mod wrapped_reader_stream;
  63 pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedReaderStream};
  64
  65 mod async_channel_writer;
  66 pub use async_channel_writer::AsyncChannelWriter;
  67
  68 mod file_logger;
  69 pub use file_logger::{FileLogger, FileLogOptions};
  70
  71 pub use pbs_tools::broadcast_future::{BroadcastData, BroadcastFuture};
  72
  73 /// The `BufferedRead` trait provides a single function
  74 /// `buffered_read`. It returns a reference to an internal buffer. The
  75 /// purpose of this traid is to avoid unnecessary data copies.
  76 pub trait BufferedRead {
  77     /// This functions tries to fill the internal buffers, then
  78     /// returns a reference to the available data. It returns an empty
  79     /// buffer if `offset` points to the end of the file.
  80     fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>;
  81 }
  82
  83 pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
  84     match param[name].as_str() {
  85         Some(s) => Ok(s),
  86         None => bail!("missing parameter '{}'", name),
  87     }
  88 }
  89
  90 pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> {
  91     match param[name].as_str() {
  92         Some(s) => Ok(s),
  93         None => bail!("missing property '{}'", name),
  94     }
  95 }
  96
  97 pub fn required_integer_param(param: &Value, name: &str) -> Result<i64, Error> {
  98     match param[name].as_i64() {
  99         Some(s) => Ok(s),
 100         None => bail!("missing parameter '{}'", name),
 101     }
 102 }
 103
 104 pub fn required_integer_property(param: &Value, name: &str) -> Result<i64, Error> {
 105     match param[name].as_i64() {
 106         Some(s) => Ok(s),
 107         None => bail!("missing property '{}'", name),
 108     }
 109 }
 110
 111 pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 112     match param[name].as_array() {
 113         Some(s) => Ok(&s),
 114         None => bail!("missing parameter '{}'", name),
 115     }
 116 }
 117
 118 pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [Value], Error> {
 119     match param[name].as_array() {
 120         Some(s) => Ok(&s),
 121         None => bail!("missing property '{}'", name),
 122     }
 123 }
 124
 125 pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
 126 where
 127     S: BuildHasher,
 128 {
 129     let mut result = vec![];
 130
 131     use nix::fcntl::AtFlags;
 132     use nix::fcntl::OFlag;
 133     use nix::sys::stat::Mode;
 134
 135     let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
 136
 137     let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
 138         Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
 139         Err(_) => false,
 140     };
 141
 142     if !is_dir {
 143         if let Some(parent) = dirname.parent() {
 144             dirname = parent.to_owned();
 145         }
 146     }
 147
 148     let mut dir =
 149         match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
 150             Ok(d) => d,
 151             Err(_) => return result,
 152         };
 153
 154     for item in dir.iter() {
 155         if let Ok(entry) = item {
 156             if let Ok(name) = entry.file_name().to_str() {
 157                 if name == "." || name == ".." {
 158                     continue;
 159                 }
 160                 let mut newpath = dirname.clone();
 161                 newpath.push(name);
 162
 163                 if let Ok(stat) =
 164                     nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
 165                 {
 166                     if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
 167                         newpath.push("");
 168                         if let Some(newpath) = newpath.to_str() {
 169                             result.push(newpath.to_owned());
 170                         }
 171                         continue;
 172                     }
 173                 }
 174                 if let Some(newpath) = newpath.to_str() {
 175                     result.push(newpath.to_owned());
 176                 }
 177             }
 178         }
 179     }
 180
 181     result
 182 }
 183
 184 /// Shortcut for md5 sums.
 185 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 186     hash(MessageDigest::md5(), data).map_err(Error::from)
 187 }
 188
 189 pub fn get_hardware_address() -> Result<String, Error> {
 190     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 191
 192     let contents = proxmox::tools::fs::file_get_contents(FILENAME)
 193         .map_err(|e| format_err!("Error getting host key - {}", e))?;
 194     let digest = md5sum(&contents)
 195         .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 196
 197     Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 198 }
 199
 200 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
 201     if digest1 != digest2 {
 202         bail!("detected modified configuration - file changed by other user? Try again.");
 203     }
 204     Ok(())
 205 }
 206
 207 /// Extract a specific cookie from cookie header.
 208 /// We assume cookie_name is already url encoded.
 209 pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
 210     for pair in cookie.split(';') {
 211         let (name, value) = match pair.find('=') {
 212             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
 213             None => return None, // Cookie format error
 214         };
 215
 216         if name == cookie_name {
 217             use percent_encoding::percent_decode;
 218             if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() {
 219                 return Some(value.into());
 220             } else {
 221                 return None; // Cookie format error
 222             }
 223         }
 224     }
 225
 226     None
 227 }
 228
 229 /// Detect modified configuration files
 230 ///
 231 /// This function fails with a reasonable error message if checksums do not match.
 232 pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> {
 233     if digest1 != digest2 {
 234         bail!("detected modified configuration - file changed by other user? Try again.");
 235     }
 236     Ok(())
 237 }
 238
 239 /// normalize uri path
 240 ///
 241 /// Do not allow ".", "..", or hidden files ".XXXX"
 242 /// Also remove empty path components
 243 pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 244     let items = path.split('/');
 245
 246     let mut path = String::new();
 247     let mut components = vec![];
 248
 249     for name in items {
 250         if name.is_empty() {
 251             continue;
 252         }
 253         if name.starts_with('.') {
 254             bail!("Path contains illegal components.");
 255         }
 256         path.push('/');
 257         path.push_str(name);
 258         components.push(name);
 259     }
 260
 261     Ok((path, components))
 262 }
 263
 264 pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> {
 265     use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD};
 266     let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?)
 267         .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way...
 268     flags.set(FdFlag::FD_CLOEXEC, on);
 269     fcntl(fd, F_SETFD(flags))?;
 270     Ok(())
 271 }
 272
 273 static mut SHUTDOWN_REQUESTED: bool = false;
 274
 275 pub fn request_shutdown() {
 276     unsafe {
 277         SHUTDOWN_REQUESTED = true;
 278     }
 279     crate::server::server_shutdown();
 280 }
 281
 282 #[inline(always)]
 283 pub fn shutdown_requested() -> bool {
 284     unsafe { SHUTDOWN_REQUESTED }
 285 }
 286
 287 pub fn fail_on_shutdown() -> Result<(), Error> {
 288     if shutdown_requested() {
 289         bail!("Server shutdown requested - aborting task");
 290     }
 291     Ok(())
 292 }
 293
 294 /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file
 295 /// descriptors.
 296 pub fn pipe() -> Result<(Fd, Fd), Error> {
 297     let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?;
 298     Ok((Fd(pin), Fd(pout)))
 299 }
 300
 301 /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file
 302 /// descriptors.
 303 pub fn socketpair() -> Result<(Fd, Fd), Error> {
 304     use nix::sys::socket;
 305     let (pa, pb) = socket::socketpair(
 306         socket::AddressFamily::Unix,
 307         socket::SockType::Stream,
 308         None,
 309         socket::SockFlag::SOCK_CLOEXEC,
 310     )?;
 311     Ok((Fd(pa), Fd(pb)))
 312 }
 313
 314
 315 /// An easy way to convert types to Any
 316 ///
 317 /// Mostly useful to downcast trait objects (see RpcEnvironment).
 318 pub trait AsAny {
 319     fn as_any(&self) -> &dyn Any;
 320 }
 321
 322 impl<T: Any> AsAny for T {
 323     fn as_any(&self) -> &dyn Any {
 324         self
 325     }
 326 }
 327
 328 /// The default 2 hours are far too long for PBS
 329 pub const PROXMOX_BACKUP_TCP_KEEPALIVE_TIME: u32 = 120;
 330 pub const DEFAULT_USER_AGENT_STRING: &'static str = "proxmox-backup-client/1.0";
 331
 332 /// Returns a new instance of `SimpleHttp` configured for PBS usage.
 333 pub fn pbs_simple_http(proxy_config: Option<ProxyConfig>) -> SimpleHttp {
 334     let options = SimpleHttpOptions {
 335         proxy_config,
 336         user_agent: Some(DEFAULT_USER_AGENT_STRING.to_string()),
 337         tcp_keepalive: Some(PROXMOX_BACKUP_TCP_KEEPALIVE_TIME),
 338         ..Default::default()
 339     };
 340
 341     SimpleHttp::with_options(options)
 342 }
 343
 344 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 345 /// `#`).
 346 pub fn file_get_non_comment_lines<P: AsRef<Path>>(
 347     path: P,
 348 ) -> Result<impl Iterator<Item = io::Result<String>>, Error> {
 349     let path = path.as_ref();
 350
 351     Ok(io::BufReader::new(
 352         File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?,
 353     )
 354     .lines()
 355     .filter_map(|line| match line {
 356         Ok(line) => {
 357             let line = line.trim();
 358             if line.is_empty() || line.starts_with('#') {
 359                 None
 360             } else {
 361                 Some(Ok(line.to_string()))
 362             }
 363         }
 364         Err(err) => Some(Err(err)),
 365     }))
 366 }
 367
 368 pub fn setup_safe_path_env() {
 369     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
 370     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
 371     for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] {
 372         std::env::remove_var(name);
 373     }
 374 }
 375
 376 pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
 377     let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
 378         Some(n) => &line[n..],
 379         None => return &[],
 380     };
 381     match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
 382         Some(n) => &line[..(line.len() - n)],
 383         None => &[],
 384     }
 385 }
 386
 387 /// Create the base run-directory.
 388 ///
 389 /// This exists to fixate the permissions for the run *base* directory while allowing intermediate
 390 /// directories after it to have different permissions.
 391 pub fn create_run_dir() -> Result<(), Error> {
 392     let backup_user = crate::backup::backup_user()?;
 393     let opts = CreateOptions::new()
 394         .owner(backup_user.uid)
 395         .group(backup_user.gid);
 396     let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
 397     Ok(())
 398 }
 399
 400 /// Modeled after the nightly `std::ops::ControlFlow`.
 401 #[derive(Clone, Copy, Debug, PartialEq)]
 402 pub enum ControlFlow<B, C = ()> {
 403     Continue(C),
 404     Break(B),
 405 }
 406
 407 impl<B> ControlFlow<B> {
 408     pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
 409 }