1 //! Tools and utilities
3 //! This is a collection of small and useful tools.
7 use nix
::{convert_ioctl_res, request_code_read, ioc}
;
9 use lazy_static
::lazy_static
;
11 use std
::fs
::{File, OpenOptions}
;
15 use std
::io
::ErrorKind
;
16 use std
::time
::Duration
;
19 use std
::os
::unix
::io
::{AsRawFd, FromRawFd, IntoRawFd, RawFd}
;
21 use std
::collections
::HashMap
;
23 use serde_json
::Value
;
27 pub mod wrapped_reader_stream
;
46 pub use process_locker
::*;
50 pub use file_logger
::*;
53 pub use broadcast_future
::*;
55 /// Macro to write error-handling blocks (like perl eval {})
59 /// # #[macro_use] extern crate proxmox_backup;
61 /// # let some_condition = false;
62 /// let result = try_block!({
63 /// if (some_condition) {
64 /// bail!("some error");
68 /// .map_err(|e| format_err!("my try block returned an error - {}", e));
72 macro_rules
! try_block
{
73 { $($token:tt)* }
=> {{ (|| -> Result<_,_> { $($token)* }
)() }}
77 /// The `BufferedRead` trait provides a single function
78 /// `buffered_read`. It returns a reference to an internal buffer. The
79 /// purpose of this traid is to avoid unnecessary data copies.
80 pub trait BufferedRead
{
81 /// This functions tries to fill the internal buffers, then
82 /// returns a reference to the available data. It returns an empty
83 /// buffer if `offset` points to the end of the file.
84 fn buffered_read(&mut self, offset
: u64) -> Result
<&[u8], Error
>;
87 /// Directly map a type into a binary buffer. This is mostly useful
88 /// for reading structured data from a byte stream (file). You need to
89 /// make sure that the buffer location does not change, so please
90 /// avoid vec resize while you use such map.
92 /// This function panics if the buffer is not large enough.
93 pub fn map_struct
<T
>(buffer
: &[u8]) -> Result
<&T
, Error
> {
94 if buffer
.len() < ::std
::mem
::size_of
::<T
>() {
95 bail
!("unable to map struct - buffer too small");
97 Ok(unsafe { & * (buffer.as_ptr() as *const T) }
)
100 /// Directly map a type into a mutable binary buffer. This is mostly
101 /// useful for writing structured data into a byte stream (file). You
102 /// need to make sure that the buffer location does not change, so
103 /// please avoid vec resize while you use such map.
105 /// This function panics if the buffer is not large enough.
106 pub fn map_struct_mut
<T
>(buffer
: &mut [u8]) -> Result
<&mut T
, Error
> {
107 if buffer
.len() < ::std
::mem
::size_of
::<T
>() {
108 bail
!("unable to map struct - buffer too small");
110 Ok(unsafe { &mut * (buffer.as_ptr() as *mut T) }
)
113 pub fn file_read_firstline
<P
: AsRef
<Path
>>(path
: P
) -> Result
<String
, Error
> {
115 let path
= path
.as_ref();
118 let file
= std
::fs
::File
::open(path
)?
;
120 use std
::io
::{BufRead, BufReader}
;
122 let mut reader
= BufReader
::new(file
);
124 let mut line
= String
::new();
126 let _
= reader
.read_line(&mut line
)?
;
129 }).map_err(|err
: Error
| format_err
!("unable to read {:?} - {}", path
, err
))
132 pub fn file_get_contents
<P
: AsRef
<Path
>>(path
: P
) -> Result
<Vec
<u8>, Error
> {
134 let path
= path
.as_ref();
138 }).map_err(|err
| format_err
!("unable to read {:?} - {}", path
, err
))
141 pub fn file_get_json
<P
: AsRef
<Path
>>(path
: P
, default: Option
<Value
>) -> Result
<Value
, Error
> {
143 let path
= path
.as_ref();
145 let raw
= match std
::fs
::read(path
) {
148 if err
.kind() == std
::io
::ErrorKind
::NotFound
{
149 if let Some(v
) = default {
153 bail
!("unable to read json {:?} - {}", path
, err
);
158 let data
= String
::from_utf8(raw
)?
;
159 let json
= serde_json
::from_str(&data
)?
;
161 }).map_err(|err
: Error
| format_err
!("unable to parse json from {:?} - {}", path
, err
))
164 /// Atomically write a file
166 /// We first create a temporary file, which is then renamed.
167 pub fn file_set_contents
<P
: AsRef
<Path
>>(
170 perm
: Option
<stat
::Mode
>,
171 ) -> Result
<(), Error
> {
172 file_set_contents_full(path
, data
, perm
, None
, None
)
175 /// Atomically write a file with owner and group
176 pub fn file_set_contents_full
<P
: AsRef
<Path
>>(
179 perm
: Option
<stat
::Mode
>,
180 owner
: Option
<unistd
::Uid
>,
181 group
: Option
<unistd
::Gid
>,
182 ) -> Result
<(), Error
> {
184 let path
= path
.as_ref();
186 // Note: we use mkstemp heŕe, because this worka with different
187 // processes, threads, and even tokio tasks.
188 let mut template
= path
.to_owned();
189 template
.set_extension("tmp_XXXXXX");
190 let (fd
, tmp_path
) = match unistd
::mkstemp(&template
) {
191 Ok((fd
, path
)) => (fd
, path
),
192 Err(err
) => bail
!("mkstemp {:?} failed: {}", template
, err
),
195 let tmp_path
= tmp_path
.as_path();
197 let mode
: stat
::Mode
= perm
.unwrap_or(stat
::Mode
::from(
198 stat
::Mode
::S_IRUSR
| stat
::Mode
::S_IWUSR
|
199 stat
::Mode
::S_IRGRP
| stat
::Mode
::S_IROTH
202 if let Err(err
) = stat
::fchmod(fd
, mode
) {
203 let _
= unistd
::unlink(tmp_path
);
204 bail
!("fchmod {:?} failed: {}", tmp_path
, err
);
207 if owner
!= None
|| group
!= None
{
208 if let Err(err
) = fchown(fd
, owner
, group
) {
209 let _
= unistd
::unlink(tmp_path
);
210 bail
!("fchown {:?} failed: {}", tmp_path
, err
);
214 use std
::os
::unix
::io
::FromRawFd
;
215 let mut file
= unsafe { File::from_raw_fd(fd) }
;
217 if let Err(err
) = file
.write_all(data
) {
218 let _
= unistd
::unlink(tmp_path
);
219 bail
!("write failed: {}", err
);
222 if let Err(err
) = std
::fs
::rename(tmp_path
, path
) {
223 let _
= unistd
::unlink(tmp_path
);
224 bail
!("Atomic rename failed for file {:?} - {}", path
, err
);
230 /// Create a file lock using fntl. This function allows you to specify
231 /// a timeout if you want to avoid infinite blocking.
232 pub fn lock_file
<F
: AsRawFd
>(
235 timeout
: Option
<Duration
>,
236 ) -> Result
<(), Error
> {
239 nix
::fcntl
::FlockArg
::LockExclusive
241 nix
::fcntl
::FlockArg
::LockShared
244 let timeout
= match timeout
{
246 nix
::fcntl
::flock(file
.as_raw_fd(), lockarg
)?
;
252 // unblock the timeout signal temporarily
253 let _sigblock_guard
= timer
::unblock_timeout_signal();
255 // setup a timeout timer
256 let mut timer
= timer
::Timer
::create(
257 timer
::Clock
::Realtime
,
258 timer
::TimerEvent
::ThisThreadSignal(timer
::SIGTIMEOUT
))?
;
260 timer
.arm(timer
::TimerSpec
::new()
261 .value(Some(timeout
))
262 .interval(Some(Duration
::from_millis(10))))?
;
264 nix
::fcntl
::flock(file
.as_raw_fd(), lockarg
)?
;
268 /// Open or create a lock file (append mode). Then try to
269 /// aquire a lock using `lock_file()`.
270 pub fn open_file_locked
<P
: AsRef
<Path
>>(path
: P
, timeout
: Duration
)
271 -> Result
<File
, Error
>
273 let path
= path
.as_ref();
275 match OpenOptions
::new()
281 Err(err
) => bail
!("Unable to open lock {:?} - {}",
284 match lock_file(&mut file
, true, Some(timeout
)) {
286 Err(err
) => bail
!("Unable to aquire lock {:?} - {}",
291 /// Split a file into equal sized chunks. The last chunk may be
292 /// smaller. Note: We cannot implement an `Iterator`, because iterators
293 /// cannot return a borrowed buffer ref (we want zero-copy)
294 pub fn file_chunker
<C
, R
>(
298 ) -> Result
<(), Error
>
299 where C
: FnMut(usize, &[u8]) -> Result
<bool
, Error
>,
303 const READ_BUFFER_SIZE
: usize = 4*1024*1024; // 4M
305 if chunk_size
> READ_BUFFER_SIZE { bail!("chunk size too large!"); }
307 let mut buf
= vec
::undefined(READ_BUFFER_SIZE
);
310 let mut file_pos
= 0;
313 let mut tmp
= &mut buf
[..];
314 // try to read large portions, at least chunk_size
315 while pos
< chunk_size
{
316 match file
.read(tmp
) {
317 Ok(0) => { eof = true; break; }
,
320 if pos
> chunk_size { break; }
323 Err(ref e
) if e
.kind() == ErrorKind
::Interrupted
=> { /* try again */ }
324 Err(e
) => bail
!("read chunk failed - {}", e
.to_string()),
328 while start
+ chunk_size
<= pos
{
329 if !(chunk_cb
)(file_pos
, &buf
[start
..start
+chunk_size
])? { break; }
330 file_pos
+= chunk_size
;
335 (chunk_cb
)(file_pos
, &buf
[start
..pos
])?
;
336 //file_pos += pos - start;
340 let rest
= pos
- start
;
342 let ptr
= buf
.as_mut_ptr();
343 unsafe { std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest); }
354 /// Returns the Unix uid/gid for the sepcified system user.
355 pub fn getpwnam_ugid(username
: &str) -> Result
<(libc
::uid_t
,libc
::gid_t
), Error
> {
356 let info
= unsafe { libc::getpwnam(std::ffi::CString::new(username).unwrap().as_ptr()) }
;
357 if info
== std
::ptr
::null_mut() {
358 bail
!("getwpnam '{}' failed", username
);
361 let info
= unsafe { *info }
;
363 Ok((info
.pw_uid
, info
.pw_gid
))
366 /// Creates directory at the provided path with specified ownership
368 /// Simply returns if the directory already exists.
369 pub fn create_dir_chown
<P
: AsRef
<Path
>>(
371 perm
: Option
<stat
::Mode
>,
372 owner
: Option
<unistd
::Uid
>,
373 group
: Option
<unistd
::Gid
>,
374 ) -> Result
<(), nix
::Error
>
376 let mode
: stat
::Mode
= perm
.unwrap_or(stat
::Mode
::from_bits_truncate(0o770));
378 let path
= path
.as_ref();
380 match nix
::unistd
::mkdir(path
, mode
) {
382 Err(nix
::Error
::Sys(nix
::errno
::Errno
::EEXIST
)) => {
388 unistd
::chown(path
, owner
, group
)?
;
393 /// Change ownership of an open file handle
396 owner
: Option
<nix
::unistd
::Uid
>,
397 group
: Option
<nix
::unistd
::Gid
>
398 ) -> Result
<(), Error
> {
400 // According to the POSIX specification, -1 is used to indicate that owner and group
401 // are not to be changed. Since uid_t and gid_t are unsigned types, we have to wrap
402 // around to get -1 (copied fron nix crate).
403 let uid
= owner
.map(Into
::into
).unwrap_or((0 as libc
::uid_t
).wrapping_sub(1));
404 let gid
= group
.map(Into
::into
).unwrap_or((0 as libc
::gid_t
).wrapping_sub(1));
406 let res
= unsafe { libc::fchown(fd, uid, gid) }
;
407 nix
::errno
::Errno
::result(res
)?
;
412 // Returns the hosts node name (UTS node name)
413 pub fn nodename() -> &'
static str {
416 static ref NODENAME
: String
= {
418 nix
::sys
::utsname
::uname()
430 pub fn json_object_to_query(data
: Value
) -> Result
<String
, Error
> {
432 let mut query
= url
::form_urlencoded
::Serializer
::new(String
::new());
434 let object
= data
.as_object().ok_or_else(|| {
435 format_err
!("json_object_to_query: got wrong data type (expected object).")
438 for (key
, value
) in object
{
440 Value
::Bool(b
) => { query.append_pair(key, &b.to_string()); }
441 Value
::Number(n
) => { query.append_pair(key, &n.to_string()); }
442 Value
::String(s
) => { query.append_pair(key, &s); }
443 Value
::Array(arr
) => {
446 Value
::Bool(b
) => { query.append_pair(key, &b.to_string()); }
447 Value
::Number(n
) => { query.append_pair(key, &n.to_string()); }
448 Value
::String(s
) => { query.append_pair(key, &s); }
449 _
=> bail
!("json_object_to_query: unable to handle complex array data types."),
453 _
=> bail
!("json_object_to_query: unable to handle complex data types."),
460 pub fn required_string_param
<'a
>(param
: &'a Value
, name
: &str) -> Result
<&'a
str, Error
> {
461 match param
[name
].as_str() {
463 None
=> bail
!("missing parameter '{}'", name
),
467 pub fn required_integer_param
<'a
>(param
: &'a Value
, name
: &str) -> Result
<i64, Error
> {
468 match param
[name
].as_i64() {
470 None
=> bail
!("missing parameter '{}'", name
),
474 pub fn required_array_param
<'a
>(param
: &'a Value
, name
: &str) -> Result
<Vec
<Value
>, Error
> {
475 match param
[name
].as_array() {
476 Some(s
) => Ok(s
.to_vec()),
477 None
=> bail
!("missing parameter '{}'", name
),
481 pub fn complete_file_name(arg
: &str, _param
: &HashMap
<String
, String
>) -> Vec
<String
> {
483 let mut result
= vec
![];
485 use nix
::fcntl
::OFlag
;
486 use nix
::sys
::stat
::Mode
;
487 use nix
::fcntl
::AtFlags
;
489 let mut dirname
= std
::path
::PathBuf
::from(if arg
.len() == 0 { "./" }
else { arg }
);
491 let is_dir
= match nix
::sys
::stat
::fstatat(libc
::AT_FDCWD
, &dirname
, AtFlags
::empty()) {
492 Ok(stat
) => (stat
.st_mode
& libc
::S_IFMT
) == libc
::S_IFDIR
,
497 if let Some(parent
) = dirname
.parent() {
498 dirname
= parent
.to_owned();
502 let mut dir
= match nix
::dir
::Dir
::openat(libc
::AT_FDCWD
, &dirname
, OFlag
::O_DIRECTORY
, Mode
::empty()) {
504 Err(_
) => return result
,
507 for item
in dir
.iter() {
508 if let Ok(entry
) = item
{
509 if let Ok(name
) = entry
.file_name().to_str() {
510 if name
== "." || name
== ".." { continue; }
511 let mut newpath
= dirname
.clone();
514 if let Ok(stat
) = nix
::sys
::stat
::fstatat(libc
::AT_FDCWD
, &newpath
, AtFlags
::empty()) {
515 if (stat
.st_mode
& libc
::S_IFMT
) == libc
::S_IFDIR
{
517 if let Some(newpath
) = newpath
.to_str() {
518 result
.push(newpath
.to_owned());
523 if let Some(newpath
) = newpath
.to_str() {
524 result
.push(newpath
.to_owned());
534 /// Scan directory for matching file names.
536 /// Scan through all directory entries and call `callback()` function
537 /// if the entry name matches the regular expression. This function
538 /// used unix `openat()`, so you can pass absolute or relative file
539 /// names. This function simply skips non-UTF8 encoded names.
540 pub fn scandir
<P
, F
>(
543 regex
: ®ex
::Regex
,
545 ) -> Result
<(), Error
>
546 where F
: FnMut(RawFd
, &str, nix
::dir
::Type
) -> Result
<(), Error
>,
547 P
: ?Sized
+ nix
::NixPath
,
549 for entry
in self::fs
::scan_subdir(dirfd
, path
, regex
)?
{
551 let file_type
= match entry
.file_type() {
552 Some(file_type
) => file_type
,
553 None
=> bail
!("unable to detect file type"),
556 callback(entry
.parent_fd(), unsafe { entry.file_name_utf8_unchecked() }
, file_type
)?
;
561 pub fn get_hardware_address() -> Result
<String
, Error
> {
563 static FILENAME
: &str = "/etc/ssh/ssh_host_rsa_key.pub";
565 let contents
= file_get_contents(FILENAME
)?
;
566 let digest
= md5
::compute(contents
);
568 Ok(format
!("{:0x}", digest
))
571 const HEX_CHARS
: &'
static [u8; 16] = b
"0123456789abcdef";
573 pub fn digest_to_hex(digest
: &[u8]) -> String
{
574 let mut buf
= Vec
::<u8>::with_capacity(digest
.len()*2);
576 for i
in 0..digest
.len() {
577 buf
.push(HEX_CHARS
[(digest
[i
] >> 4) as usize]);
578 buf
.push(HEX_CHARS
[(digest
[i
] & 0xf) as usize]);
581 unsafe { String::from_utf8_unchecked(buf) }
584 pub fn hex_to_digest(hex
: &str) -> Result
<[u8; 32], Error
> {
585 let mut digest
= [0u8; 32];
587 let bytes
= hex
.as_bytes();
589 if bytes
.len() != 64 { bail!("got wrong digest length."); }
592 if c
>= b'
0'
&& c
<= b'
9' { return Ok(c - b'0'); }
593 if c
>= b'a'
&& c
<= b'f' { return Ok(c - b'a' + 10); }
594 if c
>= b'A'
&& c
<= b'F' { return Ok(c - b'A' + 10); }
595 bail
!("found illegal hex character.");
599 for pair
in bytes
.chunks(2) {
600 if pos
>= digest
.len() { bail!("hex digest too long."); }
601 let h
= val(pair
[0])?
;
602 let l
= val(pair
[1])?
;
603 digest
[pos
] = (h
<<4)|l
;
607 if pos
!= digest
.len() { bail!("hex digest too short."); }
612 pub fn assert_if_modified(digest1
: &str, digest2
: &str) -> Result
<(), Error
> {
613 if digest1
!= digest2
{
614 bail
!("detected modified configuration - file changed by other user? Try again.");
619 /// Extract authentication cookie from cookie header.
620 /// We assume cookie_name is already url encoded.
621 pub fn extract_auth_cookie(cookie
: &str, cookie_name
: &str) -> Option
<String
> {
623 for pair
in cookie
.split('
;'
) {
625 let (name
, value
) = match pair
.find('
='
) {
626 Some(i
) => (pair
[..i
].trim(), pair
[(i
+ 1)..].trim()),
627 None
=> return None
, // Cookie format error
630 if name
== cookie_name
{
631 use url
::percent_encoding
::percent_decode
;
632 if let Ok(value
) = percent_decode(value
.as_bytes()).decode_utf8() {
633 return Some(value
.into());
635 return None
; // Cookie format error
643 pub fn join(data
: &Vec
<String
>, sep
: char) -> String
{
645 let mut list
= String
::new();
648 if list
.len() != 0 { list.push(sep); }
655 /// normalize uri path
657 /// Do not allow ".", "..", or hidden files ".XXXX"
658 /// Also remove empty path components
659 pub fn normalize_uri_path(path
: &str) -> Result
<(String
, Vec
<&str>), Error
> {
661 let items
= path
.split('
/'
);
663 let mut path
= String
::new();
664 let mut components
= vec
![];
667 if name
.is_empty() { continue; }
668 if name
.starts_with(".") {
669 bail
!("Path contains illegal components.");
673 components
.push(name
);
676 Ok((path
, components
))
679 pub fn fd_change_cloexec(fd
: RawFd
, on
: bool
) -> Result
<(), Error
> {
680 use nix
::fcntl
::{fcntl, F_GETFD, F_SETFD, FdFlag}
;
681 let mut flags
= FdFlag
::from_bits(fcntl(fd
, F_GETFD
)?
)
682 .ok_or_else(|| format_err
!("unhandled file flags"))?
; // nix crate is stupid this way...
683 flags
.set(FdFlag
::FD_CLOEXEC
, on
);
684 fcntl(fd
, F_SETFD(flags
))?
;
689 static mut SHUTDOWN_REQUESTED
: bool
= false;
691 pub fn request_shutdown() {
692 unsafe { SHUTDOWN_REQUESTED = true; }
693 crate::server
::server_shutdown();
697 pub fn shutdown_requested() -> bool
{
698 unsafe { SHUTDOWN_REQUESTED }
701 pub fn fail_on_shutdown() -> Result
<(), Error
> {
702 if shutdown_requested() {
703 bail
!("Server shutdown requested - aborting task");
708 /// Guard a raw file descriptor with a drop handler. This is mostly useful when access to an owned
709 /// `RawFd` is required without the corresponding handler object (such as when only the file
710 /// descriptor number is required in a closure which may be dropped instead of being executed).
711 pub struct Fd(pub RawFd
);
723 impl AsRawFd
for Fd
{
724 fn as_raw_fd(&self) -> RawFd
{
729 impl IntoRawFd
for Fd
{
730 fn into_raw_fd(mut self) -> RawFd
{
737 impl FromRawFd
for Fd
{
738 unsafe fn from_raw_fd(fd
: RawFd
) -> Self {
743 // wrap nix::unistd::pipe2 + O_CLOEXEC into something returning guarded file descriptors
744 pub fn pipe() -> Result
<(Fd
, Fd
), Error
> {
745 let (pin
, pout
) = nix
::unistd
::pipe2(nix
::fcntl
::OFlag
::O_CLOEXEC
)?
;
746 Ok((Fd(pin
), Fd(pout
)))
749 /// An easy way to convert types to Any
751 /// Mostly useful to downcast trait objects (see RpcEnvironment).
753 fn as_any(&self) -> &Any
;
756 impl<T
: Any
> AsAny
for T
{
757 fn as_any(&self) -> &Any { self }
761 // /usr/include/linux/fs.h: #define BLKGETSIZE64 _IOR(0x12,114,size_t)
762 // return device size in bytes (u64 *arg)
763 nix
::ioctl_read
!(blkgetsize64
, 0x12, 114, u64);
765 /// Return file or block device size
766 pub fn image_size(path
: &Path
) -> Result
<u64, Error
> {
768 use std
::os
::unix
::io
::AsRawFd
;
769 use std
::os
::unix
::fs
::FileTypeExt
;
771 let file
= std
::fs
::File
::open(path
)?
;
772 let metadata
= file
.metadata()?
;
773 let file_type
= metadata
.file_type();
775 if file_type
.is_block_device() {
776 let mut size
: u64 = 0;
777 let res
= unsafe { blkgetsize64(file.as_raw_fd(), &mut size) }
;
779 if let Err(err
) = res
{
780 bail
!("blkgetsize64 failed for {:?} - {}", path
, err
);
783 } else if file_type
.is_file() {
786 bail
!("image size failed - got unexpected file type {:?}", file_type
);