]>
Commit | Line | Data |
---|---|---|
1 | //! Tools and utilities | |
2 | //! | |
3 | //! This is a collection of small and useful tools. | |
4 | use std::any::Any; | |
5 | use std::collections::HashMap; | |
6 | use std::hash::BuildHasher; | |
7 | use std::fs::File; | |
8 | use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom}; | |
9 | use std::os::unix::io::RawFd; | |
10 | use std::path::Path; | |
11 | ||
12 | use anyhow::{bail, format_err, Error}; | |
13 | use serde_json::Value; | |
14 | use openssl::hash::{hash, DigestBytes, MessageDigest}; | |
15 | use percent_encoding::AsciiSet; | |
16 | ||
17 | use proxmox::tools::vec; | |
18 | ||
19 | pub use proxmox::tools::fd::Fd; | |
20 | ||
21 | pub mod acl; | |
22 | pub mod async_io; | |
23 | pub mod borrow; | |
24 | pub mod cert; | |
25 | pub mod daemon; | |
26 | pub mod disks; | |
27 | pub mod fs; | |
28 | pub mod format; | |
29 | pub mod lru_cache; | |
30 | pub mod runtime; | |
31 | pub mod ticket; | |
32 | pub mod statistics; | |
33 | pub mod systemd; | |
34 | pub mod nom; | |
35 | pub mod logrotate; | |
36 | pub mod loopdev; | |
37 | pub mod fuse_loop; | |
38 | ||
39 | mod parallel_handler; | |
40 | pub use parallel_handler::*; | |
41 | ||
42 | mod wrapped_reader_stream; | |
43 | pub use wrapped_reader_stream::*; | |
44 | ||
45 | mod std_channel_writer; | |
46 | pub use std_channel_writer::*; | |
47 | ||
48 | pub mod xattr; | |
49 | ||
50 | mod process_locker; | |
51 | pub use process_locker::*; | |
52 | ||
53 | mod file_logger; | |
54 | pub use file_logger::*; | |
55 | ||
56 | mod broadcast_future; | |
57 | pub use broadcast_future::*; | |
58 | ||
59 | /// The `BufferedRead` trait provides a single function | |
60 | /// `buffered_read`. It returns a reference to an internal buffer. The | |
61 | /// purpose of this traid is to avoid unnecessary data copies. | |
62 | pub trait BufferedRead { | |
63 | /// This functions tries to fill the internal buffers, then | |
64 | /// returns a reference to the available data. It returns an empty | |
65 | /// buffer if `offset` points to the end of the file. | |
66 | fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>; | |
67 | } | |
68 | ||
69 | /// Split a file into equal sized chunks. The last chunk may be | |
70 | /// smaller. Note: We cannot implement an `Iterator`, because iterators | |
71 | /// cannot return a borrowed buffer ref (we want zero-copy) | |
72 | pub fn file_chunker<C, R>(mut file: R, chunk_size: usize, mut chunk_cb: C) -> Result<(), Error> | |
73 | where | |
74 | C: FnMut(usize, &[u8]) -> Result<bool, Error>, | |
75 | R: Read, | |
76 | { | |
77 | const READ_BUFFER_SIZE: usize = 4 * 1024 * 1024; // 4M | |
78 | ||
79 | if chunk_size > READ_BUFFER_SIZE { | |
80 | bail!("chunk size too large!"); | |
81 | } | |
82 | ||
83 | let mut buf = vec::undefined(READ_BUFFER_SIZE); | |
84 | ||
85 | let mut pos = 0; | |
86 | let mut file_pos = 0; | |
87 | loop { | |
88 | let mut eof = false; | |
89 | let mut tmp = &mut buf[..]; | |
90 | // try to read large portions, at least chunk_size | |
91 | while pos < chunk_size { | |
92 | match file.read(tmp) { | |
93 | Ok(0) => { | |
94 | eof = true; | |
95 | break; | |
96 | } | |
97 | Ok(n) => { | |
98 | pos += n; | |
99 | if pos > chunk_size { | |
100 | break; | |
101 | } | |
102 | tmp = &mut tmp[n..]; | |
103 | } | |
104 | Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ } | |
105 | Err(e) => bail!("read chunk failed - {}", e.to_string()), | |
106 | } | |
107 | } | |
108 | let mut start = 0; | |
109 | while start + chunk_size <= pos { | |
110 | if !(chunk_cb)(file_pos, &buf[start..start + chunk_size])? { | |
111 | break; | |
112 | } | |
113 | file_pos += chunk_size; | |
114 | start += chunk_size; | |
115 | } | |
116 | if eof { | |
117 | if start < pos { | |
118 | (chunk_cb)(file_pos, &buf[start..pos])?; | |
119 | //file_pos += pos - start; | |
120 | } | |
121 | break; | |
122 | } else { | |
123 | let rest = pos - start; | |
124 | if rest > 0 { | |
125 | let ptr = buf.as_mut_ptr(); | |
126 | unsafe { | |
127 | std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest); | |
128 | } | |
129 | pos = rest; | |
130 | } else { | |
131 | pos = 0; | |
132 | } | |
133 | } | |
134 | } | |
135 | ||
136 | Ok(()) | |
137 | } | |
138 | ||
139 | pub fn json_object_to_query(data: Value) -> Result<String, Error> { | |
140 | let mut query = url::form_urlencoded::Serializer::new(String::new()); | |
141 | ||
142 | let object = data.as_object().ok_or_else(|| { | |
143 | format_err!("json_object_to_query: got wrong data type (expected object).") | |
144 | })?; | |
145 | ||
146 | for (key, value) in object { | |
147 | match value { | |
148 | Value::Bool(b) => { | |
149 | query.append_pair(key, &b.to_string()); | |
150 | } | |
151 | Value::Number(n) => { | |
152 | query.append_pair(key, &n.to_string()); | |
153 | } | |
154 | Value::String(s) => { | |
155 | query.append_pair(key, &s); | |
156 | } | |
157 | Value::Array(arr) => { | |
158 | for element in arr { | |
159 | match element { | |
160 | Value::Bool(b) => { | |
161 | query.append_pair(key, &b.to_string()); | |
162 | } | |
163 | Value::Number(n) => { | |
164 | query.append_pair(key, &n.to_string()); | |
165 | } | |
166 | Value::String(s) => { | |
167 | query.append_pair(key, &s); | |
168 | } | |
169 | _ => bail!( | |
170 | "json_object_to_query: unable to handle complex array data types." | |
171 | ), | |
172 | } | |
173 | } | |
174 | } | |
175 | _ => bail!("json_object_to_query: unable to handle complex data types."), | |
176 | } | |
177 | } | |
178 | ||
179 | Ok(query.finish()) | |
180 | } | |
181 | ||
182 | pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> { | |
183 | match param[name].as_str() { | |
184 | Some(s) => Ok(s), | |
185 | None => bail!("missing parameter '{}'", name), | |
186 | } | |
187 | } | |
188 | ||
189 | pub fn required_string_property<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> { | |
190 | match param[name].as_str() { | |
191 | Some(s) => Ok(s), | |
192 | None => bail!("missing property '{}'", name), | |
193 | } | |
194 | } | |
195 | ||
196 | pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> { | |
197 | match param[name].as_i64() { | |
198 | Some(s) => Ok(s), | |
199 | None => bail!("missing parameter '{}'", name), | |
200 | } | |
201 | } | |
202 | ||
203 | pub fn required_integer_property<'a>(param: &'a Value, name: &str) -> Result<i64, Error> { | |
204 | match param[name].as_i64() { | |
205 | Some(s) => Ok(s), | |
206 | None => bail!("missing property '{}'", name), | |
207 | } | |
208 | } | |
209 | ||
210 | pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> { | |
211 | match param[name].as_array() { | |
212 | Some(s) => Ok(s.to_vec()), | |
213 | None => bail!("missing parameter '{}'", name), | |
214 | } | |
215 | } | |
216 | ||
217 | pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> { | |
218 | match param[name].as_array() { | |
219 | Some(s) => Ok(s.to_vec()), | |
220 | None => bail!("missing property '{}'", name), | |
221 | } | |
222 | } | |
223 | ||
224 | pub fn complete_file_name<S: BuildHasher>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String> { | |
225 | let mut result = vec![]; | |
226 | ||
227 | use nix::fcntl::AtFlags; | |
228 | use nix::fcntl::OFlag; | |
229 | use nix::sys::stat::Mode; | |
230 | ||
231 | let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg }); | |
232 | ||
233 | let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) { | |
234 | Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR, | |
235 | Err(_) => false, | |
236 | }; | |
237 | ||
238 | if !is_dir { | |
239 | if let Some(parent) = dirname.parent() { | |
240 | dirname = parent.to_owned(); | |
241 | } | |
242 | } | |
243 | ||
244 | let mut dir = | |
245 | match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) { | |
246 | Ok(d) => d, | |
247 | Err(_) => return result, | |
248 | }; | |
249 | ||
250 | for item in dir.iter() { | |
251 | if let Ok(entry) = item { | |
252 | if let Ok(name) = entry.file_name().to_str() { | |
253 | if name == "." || name == ".." { | |
254 | continue; | |
255 | } | |
256 | let mut newpath = dirname.clone(); | |
257 | newpath.push(name); | |
258 | ||
259 | if let Ok(stat) = | |
260 | nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty()) | |
261 | { | |
262 | if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR { | |
263 | newpath.push(""); | |
264 | if let Some(newpath) = newpath.to_str() { | |
265 | result.push(newpath.to_owned()); | |
266 | } | |
267 | continue; | |
268 | } | |
269 | } | |
270 | if let Some(newpath) = newpath.to_str() { | |
271 | result.push(newpath.to_owned()); | |
272 | } | |
273 | } | |
274 | } | |
275 | } | |
276 | ||
277 | result | |
278 | } | |
279 | ||
280 | /// Scan directory for matching file names. | |
281 | /// | |
282 | /// Scan through all directory entries and call `callback()` function | |
283 | /// if the entry name matches the regular expression. This function | |
284 | /// used unix `openat()`, so you can pass absolute or relative file | |
285 | /// names. This function simply skips non-UTF8 encoded names. | |
286 | pub fn scandir<P, F>( | |
287 | dirfd: RawFd, | |
288 | path: &P, | |
289 | regex: ®ex::Regex, | |
290 | mut callback: F, | |
291 | ) -> Result<(), Error> | |
292 | where | |
293 | F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>, | |
294 | P: ?Sized + nix::NixPath, | |
295 | { | |
296 | for entry in self::fs::scan_subdir(dirfd, path, regex)? { | |
297 | let entry = entry?; | |
298 | let file_type = match entry.file_type() { | |
299 | Some(file_type) => file_type, | |
300 | None => bail!("unable to detect file type"), | |
301 | }; | |
302 | ||
303 | callback( | |
304 | entry.parent_fd(), | |
305 | unsafe { entry.file_name_utf8_unchecked() }, | |
306 | file_type, | |
307 | )?; | |
308 | } | |
309 | Ok(()) | |
310 | } | |
311 | ||
312 | /// Shortcut for md5 sums. | |
313 | pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> { | |
314 | hash(MessageDigest::md5(), data).map_err(Error::from) | |
315 | } | |
316 | ||
317 | pub fn get_hardware_address() -> Result<String, Error> { | |
318 | static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub"; | |
319 | ||
320 | let contents = proxmox::tools::fs::file_get_contents(FILENAME)?; | |
321 | let digest = md5sum(&contents)?; | |
322 | ||
323 | Ok(proxmox::tools::bin_to_hex(&digest)) | |
324 | } | |
325 | ||
326 | pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> { | |
327 | if digest1 != digest2 { | |
328 | bail!("detected modified configuration - file changed by other user? Try again."); | |
329 | } | |
330 | Ok(()) | |
331 | } | |
332 | ||
333 | /// Extract a specific cookie from cookie header. | |
334 | /// We assume cookie_name is already url encoded. | |
335 | pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> { | |
336 | for pair in cookie.split(';') { | |
337 | let (name, value) = match pair.find('=') { | |
338 | Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()), | |
339 | None => return None, // Cookie format error | |
340 | }; | |
341 | ||
342 | if name == cookie_name { | |
343 | use percent_encoding::percent_decode; | |
344 | if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() { | |
345 | return Some(value.into()); | |
346 | } else { | |
347 | return None; // Cookie format error | |
348 | } | |
349 | } | |
350 | } | |
351 | ||
352 | None | |
353 | } | |
354 | ||
355 | pub fn join(data: &Vec<String>, sep: char) -> String { | |
356 | let mut list = String::new(); | |
357 | ||
358 | for item in data { | |
359 | if !list.is_empty() { | |
360 | list.push(sep); | |
361 | } | |
362 | list.push_str(item); | |
363 | } | |
364 | ||
365 | list | |
366 | } | |
367 | ||
368 | /// Detect modified configuration files | |
369 | /// | |
370 | /// This function fails with a reasonable error message if checksums do not match. | |
371 | pub fn detect_modified_configuration_file(digest1: &[u8;32], digest2: &[u8;32]) -> Result<(), Error> { | |
372 | if digest1 != digest2 { | |
373 | bail!("detected modified configuration - file changed by other user? Try again."); | |
374 | } | |
375 | Ok(()) | |
376 | } | |
377 | ||
378 | /// normalize uri path | |
379 | /// | |
380 | /// Do not allow ".", "..", or hidden files ".XXXX" | |
381 | /// Also remove empty path components | |
382 | pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> { | |
383 | let items = path.split('/'); | |
384 | ||
385 | let mut path = String::new(); | |
386 | let mut components = vec![]; | |
387 | ||
388 | for name in items { | |
389 | if name.is_empty() { | |
390 | continue; | |
391 | } | |
392 | if name.starts_with('.') { | |
393 | bail!("Path contains illegal components."); | |
394 | } | |
395 | path.push('/'); | |
396 | path.push_str(name); | |
397 | components.push(name); | |
398 | } | |
399 | ||
400 | Ok((path, components)) | |
401 | } | |
402 | ||
403 | /// Helper to check result from std::process::Command output | |
404 | /// | |
405 | /// The exit_code_check() function should return true if the exit code | |
406 | /// is considered successful. | |
407 | pub fn command_output( | |
408 | output: std::process::Output, | |
409 | exit_code_check: Option<fn(i32) -> bool>, | |
410 | ) -> Result<Vec<u8>, Error> { | |
411 | ||
412 | if !output.status.success() { | |
413 | match output.status.code() { | |
414 | Some(code) => { | |
415 | let is_ok = match exit_code_check { | |
416 | Some(check_fn) => check_fn(code), | |
417 | None => code == 0, | |
418 | }; | |
419 | if !is_ok { | |
420 | let msg = String::from_utf8(output.stderr) | |
421 | .map(|m| if m.is_empty() { String::from("no error message") } else { m }) | |
422 | .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)")); | |
423 | ||
424 | bail!("status code: {} - {}", code, msg); | |
425 | } | |
426 | } | |
427 | None => bail!("terminated by signal"), | |
428 | } | |
429 | } | |
430 | ||
431 | Ok(output.stdout) | |
432 | } | |
433 | ||
434 | /// Helper to check result from std::process::Command output, returns String. | |
435 | /// | |
436 | /// The exit_code_check() function should return true if the exit code | |
437 | /// is considered successful. | |
438 | pub fn command_output_as_string( | |
439 | output: std::process::Output, | |
440 | exit_code_check: Option<fn(i32) -> bool>, | |
441 | ) -> Result<String, Error> { | |
442 | let output = command_output(output, exit_code_check)?; | |
443 | let output = String::from_utf8(output)?; | |
444 | Ok(output) | |
445 | } | |
446 | ||
447 | pub fn run_command( | |
448 | mut command: std::process::Command, | |
449 | exit_code_check: Option<fn(i32) -> bool>, | |
450 | ) -> Result<String, Error> { | |
451 | ||
452 | let output = command.output() | |
453 | .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?; | |
454 | ||
455 | let output = crate::tools::command_output_as_string(output, exit_code_check) | |
456 | .map_err(|err| format_err!("command {:?} failed - {}", command, err))?; | |
457 | ||
458 | Ok(output) | |
459 | } | |
460 | ||
461 | pub fn fd_change_cloexec(fd: RawFd, on: bool) -> Result<(), Error> { | |
462 | use nix::fcntl::{fcntl, FdFlag, F_GETFD, F_SETFD}; | |
463 | let mut flags = FdFlag::from_bits(fcntl(fd, F_GETFD)?) | |
464 | .ok_or_else(|| format_err!("unhandled file flags"))?; // nix crate is stupid this way... | |
465 | flags.set(FdFlag::FD_CLOEXEC, on); | |
466 | fcntl(fd, F_SETFD(flags))?; | |
467 | Ok(()) | |
468 | } | |
469 | ||
470 | static mut SHUTDOWN_REQUESTED: bool = false; | |
471 | ||
472 | pub fn request_shutdown() { | |
473 | unsafe { | |
474 | SHUTDOWN_REQUESTED = true; | |
475 | } | |
476 | crate::server::server_shutdown(); | |
477 | } | |
478 | ||
479 | #[inline(always)] | |
480 | pub fn shutdown_requested() -> bool { | |
481 | unsafe { SHUTDOWN_REQUESTED } | |
482 | } | |
483 | ||
484 | pub fn fail_on_shutdown() -> Result<(), Error> { | |
485 | if shutdown_requested() { | |
486 | bail!("Server shutdown requested - aborting task"); | |
487 | } | |
488 | Ok(()) | |
489 | } | |
490 | ||
491 | /// safe wrapper for `nix::unistd::pipe2` defaulting to `O_CLOEXEC` and guarding the file | |
492 | /// descriptors. | |
493 | pub fn pipe() -> Result<(Fd, Fd), Error> { | |
494 | let (pin, pout) = nix::unistd::pipe2(nix::fcntl::OFlag::O_CLOEXEC)?; | |
495 | Ok((Fd(pin), Fd(pout))) | |
496 | } | |
497 | ||
498 | /// safe wrapper for `nix::sys::socket::socketpair` defaulting to `O_CLOEXEC` and guarding the file | |
499 | /// descriptors. | |
500 | pub fn socketpair() -> Result<(Fd, Fd), Error> { | |
501 | use nix::sys::socket; | |
502 | let (pa, pb) = socket::socketpair( | |
503 | socket::AddressFamily::Unix, | |
504 | socket::SockType::Stream, | |
505 | None, | |
506 | socket::SockFlag::SOCK_CLOEXEC, | |
507 | )?; | |
508 | Ok((Fd(pa), Fd(pb))) | |
509 | } | |
510 | ||
511 | ||
512 | /// An easy way to convert types to Any | |
513 | /// | |
514 | /// Mostly useful to downcast trait objects (see RpcEnvironment). | |
515 | pub trait AsAny { | |
516 | fn as_any(&self) -> &dyn Any; | |
517 | } | |
518 | ||
519 | impl<T: Any> AsAny for T { | |
520 | fn as_any(&self) -> &dyn Any { | |
521 | self | |
522 | } | |
523 | } | |
524 | ||
525 | /// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}` | |
526 | pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e | |
527 | // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above) | |
528 | .add(0x20) | |
529 | .add(0x7f) | |
530 | // the DEFAULT_ENCODE_SET added: | |
531 | .add(b' ') | |
532 | .add(b'"') | |
533 | .add(b'#') | |
534 | .add(b'<') | |
535 | .add(b'>') | |
536 | .add(b'`') | |
537 | .add(b'?') | |
538 | .add(b'{') | |
539 | .add(b'}'); | |
540 | ||
541 | /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a | |
542 | /// `#`). | |
543 | pub fn file_get_non_comment_lines<P: AsRef<Path>>( | |
544 | path: P, | |
545 | ) -> Result<impl Iterator<Item = io::Result<String>>, Error> { | |
546 | let path = path.as_ref(); | |
547 | ||
548 | Ok(io::BufReader::new( | |
549 | File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?, | |
550 | ) | |
551 | .lines() | |
552 | .filter_map(|line| match line { | |
553 | Ok(line) => { | |
554 | let line = line.trim(); | |
555 | if line.is_empty() || line.starts_with('#') { | |
556 | None | |
557 | } else { | |
558 | Some(Ok(line.to_string())) | |
559 | } | |
560 | } | |
561 | Err(err) => Some(Err(err)), | |
562 | })) | |
563 | } | |
564 | ||
565 | pub fn setup_safe_path_env() { | |
566 | std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin"); | |
567 | // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html | |
568 | for name in &["IFS", "CDPATH", "ENV", "BASH_ENV"] { | |
569 | std::env::remove_var(name); | |
570 | } | |
571 | } | |
572 | ||
573 | pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] { | |
574 | let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) { | |
575 | Some(n) => &line[n..], | |
576 | None => return &[], | |
577 | }; | |
578 | match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) { | |
579 | Some(n) => &line[..(line.len() - n)], | |
580 | None => &[], | |
581 | } | |
582 | } | |
583 | ||
584 | /// Seeks to start of file and computes the SHA256 hash | |
585 | pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> { | |
586 | ||
587 | file.seek(SeekFrom::Start(0))?; | |
588 | ||
589 | let mut hasher = openssl::sha::Sha256::new(); | |
590 | let mut buffer = proxmox::tools::vec::undefined(256*1024); | |
591 | let mut size: u64 = 0; | |
592 | ||
593 | loop { | |
594 | let count = match file.read(&mut buffer) { | |
595 | Ok(count) => count, | |
596 | Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => { | |
597 | continue; | |
598 | } | |
599 | Err(err) => return Err(err.into()), | |
600 | }; | |
601 | if count == 0 { | |
602 | break; | |
603 | } | |
604 | size += count as u64; | |
605 | hasher.update(&buffer[..count]); | |
606 | } | |
607 | ||
608 | let csum = hasher.finish(); | |
609 | ||
610 | Ok((csum, size)) | |
611 | } |