1 //! Tools and utilities
3 //! This is a collection of small and useful tools.
5 use std
::collections
::HashMap
;
6 use std
::hash
::BuildHasher
;
7 use std
::fs
::{File, OpenOptions}
;
8 use std
::io
::ErrorKind
;
10 use std
::os
::unix
::io
::{AsRawFd, FromRawFd, IntoRawFd, RawFd}
;
12 use std
::time
::Duration
;
15 use serde_json
::Value
;
16 use openssl
::hash
::{hash, DigestBytes, MessageDigest}
;
17 use percent_encoding
::AsciiSet
;
19 use proxmox
::tools
::vec
;
31 pub mod wrapped_reader_stream
;
35 pub use process_locker
::*;
38 pub use file_logger
::*;
41 pub use broadcast_future
::*;
43 /// The `BufferedRead` trait provides a single function
44 /// `buffered_read`. It returns a reference to an internal buffer. The
45 /// purpose of this traid is to avoid unnecessary data copies.
46 pub trait BufferedRead
{
47 /// This functions tries to fill the internal buffers, then
48 /// returns a reference to the available data. It returns an empty
49 /// buffer if `offset` points to the end of the file.
50 fn buffered_read(&mut self, offset
: u64) -> Result
<&[u8], Error
>;
53 /// Directly map a type into a binary buffer. This is mostly useful
54 /// for reading structured data from a byte stream (file). You need to
55 /// make sure that the buffer location does not change, so please
56 /// avoid vec resize while you use such map.
58 /// This function panics if the buffer is not large enough.
59 pub fn map_struct
<T
>(buffer
: &[u8]) -> Result
<&T
, Error
> {
60 if buffer
.len() < ::std
::mem
::size_of
::<T
>() {
61 bail
!("unable to map struct - buffer too small");
63 Ok(unsafe { &*(buffer.as_ptr() as *const T) }
)
66 /// Directly map a type into a mutable binary buffer. This is mostly
67 /// useful for writing structured data into a byte stream (file). You
68 /// need to make sure that the buffer location does not change, so
69 /// please avoid vec resize while you use such map.
71 /// This function panics if the buffer is not large enough.
72 pub fn map_struct_mut
<T
>(buffer
: &mut [u8]) -> Result
<&mut T
, Error
> {
73 if buffer
.len() < ::std
::mem
::size_of
::<T
>() {
74 bail
!("unable to map struct - buffer too small");
76 Ok(unsafe { &mut *(buffer.as_ptr() as *mut T) }
)
79 /// Create a file lock using fntl. This function allows you to specify
80 /// a timeout if you want to avoid infinite blocking.
81 pub fn lock_file
<F
: AsRawFd
>(
84 timeout
: Option
<Duration
>,
85 ) -> Result
<(), Error
> {
86 let lockarg
= if exclusive
{
87 nix
::fcntl
::FlockArg
::LockExclusive
89 nix
::fcntl
::FlockArg
::LockShared
92 let timeout
= match timeout
{
94 nix
::fcntl
::flock(file
.as_raw_fd(), lockarg
)?
;
100 // unblock the timeout signal temporarily
101 let _sigblock_guard
= timer
::unblock_timeout_signal();
103 // setup a timeout timer
104 let mut timer
= timer
::Timer
::create(
105 timer
::Clock
::Realtime
,
106 timer
::TimerEvent
::ThisThreadSignal(timer
::SIGTIMEOUT
),
110 timer
::TimerSpec
::new()
111 .value(Some(timeout
))
112 .interval(Some(Duration
::from_millis(10))),
115 nix
::fcntl
::flock(file
.as_raw_fd(), lockarg
)?
;
119 /// Open or create a lock file (append mode). Then try to
120 /// aquire a lock using `lock_file()`.
121 pub fn open_file_locked
<P
: AsRef
<Path
>>(path
: P
, timeout
: Duration
) -> Result
<File
, Error
> {
122 let path
= path
.as_ref();
123 let mut file
= match OpenOptions
::new().create(true).append(true).open(path
) {
125 Err(err
) => bail
!("Unable to open lock {:?} - {}", path
, err
),
127 match lock_file(&mut file
, true, Some(timeout
)) {
129 Err(err
) => bail
!("Unable to aquire lock {:?} - {}", path
, err
),
133 /// Split a file into equal sized chunks. The last chunk may be
134 /// smaller. Note: We cannot implement an `Iterator`, because iterators
135 /// cannot return a borrowed buffer ref (we want zero-copy)
136 pub fn file_chunker
<C
, R
>(mut file
: R
, chunk_size
: usize, mut chunk_cb
: C
) -> Result
<(), Error
>
138 C
: FnMut(usize, &[u8]) -> Result
<bool
, Error
>,
141 const READ_BUFFER_SIZE
: usize = 4 * 1024 * 1024; // 4M
143 if chunk_size
> READ_BUFFER_SIZE
{
144 bail
!("chunk size too large!");
147 let mut buf
= vec
::undefined(READ_BUFFER_SIZE
);
150 let mut file_pos
= 0;
153 let mut tmp
= &mut buf
[..];
154 // try to read large portions, at least chunk_size
155 while pos
< chunk_size
{
156 match file
.read(tmp
) {
163 if pos
> chunk_size
{
168 Err(ref e
) if e
.kind() == ErrorKind
::Interrupted
=> { /* try again */ }
169 Err(e
) => bail
!("read chunk failed - {}", e
.to_string()),
173 while start
+ chunk_size
<= pos
{
174 if !(chunk_cb
)(file_pos
, &buf
[start
..start
+ chunk_size
])?
{
177 file_pos
+= chunk_size
;
182 (chunk_cb
)(file_pos
, &buf
[start
..pos
])?
;
183 //file_pos += pos - start;
187 let rest
= pos
- start
;
189 let ptr
= buf
.as_mut_ptr();
191 std
::ptr
::copy_nonoverlapping(ptr
.add(start
), ptr
, rest
);
203 /// Returns the Unix uid/gid for the sepcified system user.
204 pub fn getpwnam_ugid(username
: &str) -> Result
<(libc
::uid_t
, libc
::gid_t
), Error
> {
205 let c_username
= std
::ffi
::CString
::new(username
).unwrap();
206 let info
= unsafe { libc::getpwnam(c_username.as_ptr()) }
;
208 bail
!("getwpnam '{}' failed", username
);
211 let info
= unsafe { *info }
;
213 Ok((info
.pw_uid
, info
.pw_gid
))
216 pub fn json_object_to_query(data
: Value
) -> Result
<String
, Error
> {
217 let mut query
= url
::form_urlencoded
::Serializer
::new(String
::new());
219 let object
= data
.as_object().ok_or_else(|| {
220 format_err
!("json_object_to_query: got wrong data type (expected object).")
223 for (key
, value
) in object
{
226 query
.append_pair(key
, &b
.to_string());
228 Value
::Number(n
) => {
229 query
.append_pair(key
, &n
.to_string());
231 Value
::String(s
) => {
232 query
.append_pair(key
, &s
);
234 Value
::Array(arr
) => {
238 query
.append_pair(key
, &b
.to_string());
240 Value
::Number(n
) => {
241 query
.append_pair(key
, &n
.to_string());
243 Value
::String(s
) => {
244 query
.append_pair(key
, &s
);
247 "json_object_to_query: unable to handle complex array data types."
252 _
=> bail
!("json_object_to_query: unable to handle complex data types."),
259 pub fn required_string_param
<'a
>(param
: &'a Value
, name
: &str) -> Result
<&'a
str, Error
> {
260 match param
[name
].as_str() {
262 None
=> bail
!("missing parameter '{}'", name
),
266 pub fn required_string_property
<'a
>(param
: &'a Value
, name
: &str) -> Result
<&'a
str, Error
> {
267 match param
[name
].as_str() {
269 None
=> bail
!("missing property '{}'", name
),
273 pub fn required_integer_param
<'a
>(param
: &'a Value
, name
: &str) -> Result
<i64, Error
> {
274 match param
[name
].as_i64() {
276 None
=> bail
!("missing parameter '{}'", name
),
280 pub fn required_integer_property
<'a
>(param
: &'a Value
, name
: &str) -> Result
<i64, Error
> {
281 match param
[name
].as_i64() {
283 None
=> bail
!("missing property '{}'", name
),
287 pub fn required_array_param
<'a
>(param
: &'a Value
, name
: &str) -> Result
<Vec
<Value
>, Error
> {
288 match param
[name
].as_array() {
289 Some(s
) => Ok(s
.to_vec()),
290 None
=> bail
!("missing parameter '{}'", name
),
294 pub fn required_array_property
<'a
>(param
: &'a Value
, name
: &str) -> Result
<Vec
<Value
>, Error
> {
295 match param
[name
].as_array() {
296 Some(s
) => Ok(s
.to_vec()),
297 None
=> bail
!("missing property '{}'", name
),
301 pub fn complete_file_name
<S
: BuildHasher
>(arg
: &str, _param
: &HashMap
<String
, String
, S
>) -> Vec
<String
> {
302 let mut result
= vec
![];
304 use nix
::fcntl
::AtFlags
;
305 use nix
::fcntl
::OFlag
;
306 use nix
::sys
::stat
::Mode
;
308 let mut dirname
= std
::path
::PathBuf
::from(if arg
.is_empty() { "./" }
else { arg }
);
310 let is_dir
= match nix
::sys
::stat
::fstatat(libc
::AT_FDCWD
, &dirname
, AtFlags
::empty()) {
311 Ok(stat
) => (stat
.st_mode
& libc
::S_IFMT
) == libc
::S_IFDIR
,
316 if let Some(parent
) = dirname
.parent() {
317 dirname
= parent
.to_owned();
322 match nix
::dir
::Dir
::openat(libc
::AT_FDCWD
, &dirname
, OFlag
::O_DIRECTORY
, Mode
::empty()) {
324 Err(_
) => return result
,
327 for item
in dir
.iter() {
328 if let Ok(entry
) = item
{
329 if let Ok(name
) = entry
.file_name().to_str() {
330 if name
== "." || name
== ".." {
333 let mut newpath
= dirname
.clone();
337 nix
::sys
::stat
::fstatat(libc
::AT_FDCWD
, &newpath
, AtFlags
::empty())
339 if (stat
.st_mode
& libc
::S_IFMT
) == libc
::S_IFDIR
{
341 if let Some(newpath
) = newpath
.to_str() {
342 result
.push(newpath
.to_owned());
347 if let Some(newpath
) = newpath
.to_str() {
348 result
.push(newpath
.to_owned());
357 /// Scan directory for matching file names.
359 /// Scan through all directory entries and call `callback()` function
360 /// if the entry name matches the regular expression. This function
361 /// used unix `openat()`, so you can pass absolute or relative file
362 /// names. This function simply skips non-UTF8 encoded names.
363 pub fn scandir
<P
, F
>(
366 regex
: ®ex
::Regex
,
368 ) -> Result
<(), Error
>
370 F
: FnMut(RawFd
, &str, nix
::dir
::Type
) -> Result
<(), Error
>,
371 P
: ?Sized
+ nix
::NixPath
,
373 for entry
in self::fs
::scan_subdir(dirfd
, path
, regex
)?
{
375 let file_type
= match entry
.file_type() {
376 Some(file_type
) => file_type
,
377 None
=> bail
!("unable to detect file type"),
382 unsafe { entry.file_name_utf8_unchecked() }
,
389 /// Shortcut for md5 sums.
390 pub fn md5sum(data
: &[u8]) -> Result
<DigestBytes
, Error
> {
391 hash(MessageDigest
::md5(), data
).map_err(Error
::from
)
394 pub fn get_hardware_address() -> Result
<String
, Error
> {
395 static FILENAME
: &str = "/etc/ssh/ssh_host_rsa_key.pub";
397 let contents
= proxmox
::tools
::fs
::file_get_contents(FILENAME
)?
;
398 let digest
= md5sum(&contents
)?
;
400 Ok(proxmox
::tools
::bin_to_hex(&digest
))
403 pub fn assert_if_modified(digest1
: &str, digest2
: &str) -> Result
<(), Error
> {
404 if digest1
!= digest2
{
405 bail
!("detected modified configuration - file changed by other user? Try again.");
410 /// Extract authentication cookie from cookie header.
411 /// We assume cookie_name is already url encoded.
412 pub fn extract_auth_cookie(cookie
: &str, cookie_name
: &str) -> Option
<String
> {
413 for pair
in cookie
.split('
;'
) {
414 let (name
, value
) = match pair
.find('
='
) {
415 Some(i
) => (pair
[..i
].trim(), pair
[(i
+ 1)..].trim()),
416 None
=> return None
, // Cookie format error
419 if name
== cookie_name
{
420 use percent_encoding
::percent_decode
;
421 if let Ok(value
) = percent_decode(value
.as_bytes()).decode_utf8() {
422 return Some(value
.into());
424 return None
; // Cookie format error
432 pub fn join(data
: &Vec
<String
>, sep
: char) -> String
{
433 let mut list
= String
::new();
436 if !list
.is_empty() {
445 /// normalize uri path
447 /// Do not allow ".", "..", or hidden files ".XXXX"
448 /// Also remove empty path components
449 pub fn normalize_uri_path(path
: &str) -> Result
<(String
, Vec
<&str>), Error
> {
450 let items
= path
.split('
/'
);
452 let mut path
= String
::new();
453 let mut components
= vec
![];
459 if name
.starts_with('
.'
) {
460 bail
!("Path contains illegal components.");
464 components
.push(name
);
467 Ok((path
, components
))
470 pub fn fd_change_cloexec(fd
: RawFd
, on
: bool
) -> Result
<(), Error
> {
471 use nix
::fcntl
::{fcntl, FdFlag, F_GETFD, F_SETFD}
;
472 let mut flags
= FdFlag
::from_bits(fcntl(fd
, F_GETFD
)?
)
473 .ok_or_else(|| format_err
!("unhandled file flags"))?
; // nix crate is stupid this way...
474 flags
.set(FdFlag
::FD_CLOEXEC
, on
);
475 fcntl(fd
, F_SETFD(flags
))?
;
479 static mut SHUTDOWN_REQUESTED
: bool
= false;
481 pub fn request_shutdown() {
483 SHUTDOWN_REQUESTED
= true;
485 crate::server
::server_shutdown();
489 pub fn shutdown_requested() -> bool
{
490 unsafe { SHUTDOWN_REQUESTED }
493 pub fn fail_on_shutdown() -> Result
<(), Error
> {
494 if shutdown_requested() {
495 bail
!("Server shutdown requested - aborting task");
500 /// Guard a raw file descriptor with a drop handler. This is mostly useful when access to an owned
501 /// `RawFd` is required without the corresponding handler object (such as when only the file
502 /// descriptor number is required in a closure which may be dropped instead of being executed).
503 pub struct Fd(pub RawFd
);
515 impl AsRawFd
for Fd
{
516 fn as_raw_fd(&self) -> RawFd
{
521 impl IntoRawFd
for Fd
{
522 fn into_raw_fd(mut self) -> RawFd
{
529 impl FromRawFd
for Fd
{
530 unsafe fn from_raw_fd(fd
: RawFd
) -> Self {
535 // wrap nix::unistd::pipe2 + O_CLOEXEC into something returning guarded file descriptors
536 pub fn pipe() -> Result
<(Fd
, Fd
), Error
> {
537 let (pin
, pout
) = nix
::unistd
::pipe2(nix
::fcntl
::OFlag
::O_CLOEXEC
)?
;
538 Ok((Fd(pin
), Fd(pout
)))
541 /// An easy way to convert types to Any
543 /// Mostly useful to downcast trait objects (see RpcEnvironment).
545 fn as_any(&self) -> &dyn Any
;
548 impl<T
: Any
> AsAny
for T
{
549 fn as_any(&self) -> &dyn Any
{
554 /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
555 pub const DEFAULT_ENCODE_SET
: &AsciiSet
= &percent_encoding
::CONTROLS
// 0..1f and 7e
556 // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
559 // the DEFAULT_ENCODE_SET added: