]>
Commit | Line | Data |
---|---|---|
51b499db DM |
1 | //! Tools and utilities |
2 | //! | |
3 | //! This is a collection of small and useful tools. | |
f12f8ff1 DM |
4 | use failure::*; |
5 | use nix::unistd; | |
6 | use nix::sys::stat; | |
7 | ||
74a69302 DM |
8 | use lazy_static::lazy_static; |
9 | ||
365bb90f | 10 | use std::fs::{File, OpenOptions}; |
f12f8ff1 DM |
11 | use std::io::Write; |
12 | use std::path::Path; | |
43eeef28 DM |
13 | use std::io::Read; |
14 | use std::io::ErrorKind; | |
1628a4c7 | 15 | use std::time::Duration; |
f12f8ff1 | 16 | |
443f3743 | 17 | use std::os::unix::io::RawFd; |
eae8aa3a | 18 | use std::os::unix::io::AsRawFd; |
365bb90f | 19 | |
f5f13ebc | 20 | use serde_json::{json, Value}; |
0fe5d605 | 21 | |
8cf6e764 | 22 | pub mod timer; |
7f0d67cf | 23 | pub mod wrapped_reader_stream; |
8f973f81 DM |
24 | #[macro_use] |
25 | pub mod common_regex; | |
8d04280b | 26 | pub mod ticket; |
6ed25cbe | 27 | pub mod borrow; |
b4d5787d | 28 | pub mod fs; |
c9b296f1 | 29 | pub mod tty; |
8cf6e764 | 30 | |
3b151414 DM |
31 | #[macro_use] |
32 | mod file_logger; | |
33 | pub use file_logger::*; | |
34 | ||
e80503d2 DM |
35 | /// Macro to write error-handling blocks (like perl eval {}) |
36 | /// | |
37 | /// #### Example: | |
38 | /// ``` | |
fe651dd6 DM |
39 | /// # #[macro_use] extern crate proxmox_backup; |
40 | /// # use failure::*; | |
41 | /// # let some_condition = false; | |
e80503d2 DM |
42 | /// let result = try_block!({ |
43 | /// if (some_condition) { | |
44 | /// bail!("some error"); | |
45 | /// } | |
46 | /// Ok(()) | |
47 | /// }) | |
48 | /// .map_err(|e| format_err!("my try block returned an error - {}", e)); | |
49 | /// ``` | |
50 | ||
f0dbba8c DM |
51 | #[macro_export] |
52 | macro_rules! try_block { | |
53 | { $($token:tt)* } => {{ (|| -> Result<_,_> { $($token)* })() }} | |
54 | } | |
55 | ||
5d14eb6a | 56 | |
fded74d0 | 57 | /// The `BufferedRead` trait provides a single function |
0a72e267 DM |
58 | /// `buffered_read`. It returns a reference to an internal buffer. The |
59 | /// purpose of this traid is to avoid unnecessary data copies. | |
fded74d0 | 60 | pub trait BufferedRead { |
318564ac DM |
61 | /// This functions tries to fill the internal buffers, then |
62 | /// returns a reference to the available data. It returns an empty | |
63 | /// buffer if `offset` points to the end of the file. | |
0a72e267 DM |
64 | fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error>; |
65 | } | |
66 | ||
51b499db DM |
67 | /// Directly map a type into a binary buffer. This is mostly useful |
68 | /// for reading structured data from a byte stream (file). You need to | |
69 | /// make sure that the buffer location does not change, so please | |
70 | /// avoid vec resize while you use such map. | |
71 | /// | |
72 | /// This function panics if the buffer is not large enough. | |
50ea4396 | 73 | pub fn map_struct<T>(buffer: &[u8]) -> Result<&T, Error> { |
dc3de618 DM |
74 | if buffer.len() < ::std::mem::size_of::<T>() { |
75 | bail!("unable to map struct - buffer too small"); | |
76 | } | |
95bd5dfe | 77 | Ok(unsafe { & * (buffer.as_ptr() as *const T) }) |
dc3de618 DM |
78 | } |
79 | ||
51b499db DM |
80 | /// Directly map a type into a mutable binary buffer. This is mostly |
81 | /// useful for writing structured data into a byte stream (file). You | |
82 | /// need to make sure that the buffer location does not change, so | |
83 | /// please avoid vec resize while you use such map. | |
84 | /// | |
85 | /// This function panics if the buffer is not large enough. | |
50ea4396 | 86 | pub fn map_struct_mut<T>(buffer: &mut [u8]) -> Result<&mut T, Error> { |
dc3de618 DM |
87 | if buffer.len() < ::std::mem::size_of::<T>() { |
88 | bail!("unable to map struct - buffer too small"); | |
89 | } | |
95bd5dfe | 90 | Ok(unsafe { &mut * (buffer.as_ptr() as *mut T) }) |
dc3de618 DM |
91 | } |
92 | ||
6235a418 | 93 | pub fn file_read_firstline<P: AsRef<Path>>(path: P) -> Result<String, Error> { |
447787ab DM |
94 | |
95 | let path = path.as_ref(); | |
96 | ||
6235a418 DM |
97 | try_block!({ |
98 | let file = std::fs::File::open(path)?; | |
447787ab | 99 | |
6235a418 | 100 | use std::io::{BufRead, BufReader}; |
447787ab | 101 | |
6235a418 | 102 | let mut reader = BufReader::new(file); |
447787ab | 103 | |
6235a418 | 104 | let mut line = String::new(); |
447787ab | 105 | |
6235a418 | 106 | let _ = reader.read_line(&mut line)?; |
447787ab | 107 | |
6235a418 DM |
108 | Ok(line) |
109 | }).map_err(|err: Error| format_err!("unable to read {:?} - {}", path, err)) | |
447787ab DM |
110 | } |
111 | ||
12400210 DM |
112 | pub fn file_get_contents<P: AsRef<Path>>(path: P) -> Result<Vec<u8>, Error> { |
113 | ||
114 | let path = path.as_ref(); | |
115 | ||
116 | try_block!({ | |
117 | std::fs::read(path) | |
118 | }).map_err(|err| format_err!("unable to read {:?} - {}", path, err)) | |
53157ca6 DM |
119 | } |
120 | ||
51b499db DM |
121 | /// Atomically write a file. We first create a temporary file, which |
122 | /// is then renamed. | |
f12f8ff1 DM |
123 | pub fn file_set_contents<P: AsRef<Path>>( |
124 | path: P, | |
125 | data: &[u8], | |
126 | perm: Option<stat::Mode>, | |
127 | ) -> Result<(), Error> { | |
128 | ||
129 | let path = path.as_ref(); | |
130 | ||
d64d80d2 DM |
131 | // Note: we use mkstemp heŕe, because this worka with different |
132 | // processes, threads, and even tokio tasks. | |
f12f8ff1 DM |
133 | let mut template = path.to_owned(); |
134 | template.set_extension("tmp_XXXXXX"); | |
135 | let (fd, tmp_path) = match unistd::mkstemp(&template) { | |
136 | Ok((fd, path)) => (fd, path), | |
137 | Err(err) => bail!("mkstemp {:?} failed: {}", template, err), | |
138 | }; | |
139 | ||
140 | let tmp_path = tmp_path.as_path(); | |
141 | ||
1a7bc3dd | 142 | let mode : stat::Mode = perm.unwrap_or(stat::Mode::from( |
f12f8ff1 DM |
143 | stat::Mode::S_IRUSR | stat::Mode::S_IWUSR | |
144 | stat::Mode::S_IRGRP | stat::Mode::S_IROTH | |
1a7bc3dd | 145 | )); |
f12f8ff1 DM |
146 | |
147 | if let Err(err) = stat::fchmod(fd, mode) { | |
148 | let _ = unistd::unlink(tmp_path); | |
149 | bail!("fchmod {:?} failed: {}", tmp_path, err); | |
150 | } | |
151 | ||
152 | use std::os::unix::io::FromRawFd; | |
153 | let mut file = unsafe { File::from_raw_fd(fd) }; | |
154 | ||
155 | if let Err(err) = file.write_all(data) { | |
156 | let _ = unistd::unlink(tmp_path); | |
157 | bail!("write failed: {}", err); | |
158 | } | |
159 | ||
160 | if let Err(err) = std::fs::rename(tmp_path, path) { | |
161 | let _ = unistd::unlink(tmp_path); | |
162 | bail!("Atomic rename failed for file {:?} - {}", path, err); | |
163 | } | |
164 | ||
165 | Ok(()) | |
166 | } | |
43eeef28 | 167 | |
51b499db DM |
168 | /// Create a file lock using fntl. This function allows you to specify |
169 | /// a timeout if you want to avoid infinite blocking. | |
1628a4c7 WB |
170 | pub fn lock_file<F: AsRawFd>( |
171 | file: &mut F, | |
172 | exclusive: bool, | |
173 | timeout: Option<Duration>, | |
174 | ) -> Result<(), Error> | |
175 | { | |
176 | let lockarg = | |
177 | if exclusive { | |
178 | nix::fcntl::FlockArg::LockExclusive | |
179 | } else { | |
180 | nix::fcntl::FlockArg::LockShared | |
181 | }; | |
182 | ||
183 | let timeout = match timeout { | |
184 | None => { | |
185 | nix::fcntl::flock(file.as_raw_fd(), lockarg)?; | |
186 | return Ok(()); | |
187 | } | |
188 | Some(t) => t, | |
189 | }; | |
190 | ||
191 | // unblock the timeout signal temporarily | |
192 | let _sigblock_guard = timer::unblock_timeout_signal(); | |
193 | ||
194 | // setup a timeout timer | |
195 | let mut timer = timer::Timer::create( | |
196 | timer::Clock::Realtime, | |
197 | timer::TimerEvent::ThisThreadSignal(timer::SIGTIMEOUT))?; | |
198 | ||
199 | timer.arm(timer::TimerSpec::new() | |
200 | .value(Some(timeout)) | |
201 | .interval(Some(Duration::from_millis(10))))?; | |
202 | ||
203 | nix::fcntl::flock(file.as_raw_fd(), lockarg)?; | |
204 | Ok(()) | |
205 | } | |
365bb90f | 206 | |
51b499db DM |
207 | /// Open or create a lock file (append mode). Then try to |
208 | /// aquire a lock using `lock_file()`. | |
1628a4c7 WB |
209 | pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) |
210 | -> Result<File, Error> | |
211 | { | |
212 | let path = path.as_ref(); | |
213 | let mut file = | |
214 | match OpenOptions::new() | |
215 | .create(true) | |
216 | .append(true) | |
217 | .open(path) | |
218 | { | |
365bb90f DM |
219 | Ok(file) => file, |
220 | Err(err) => bail!("Unable to open lock {:?} - {}", | |
221 | path, err), | |
222 | }; | |
28b96b56 DM |
223 | match lock_file(&mut file, true, Some(timeout)) { |
224 | Ok(_) => Ok(file), | |
225 | Err(err) => bail!("Unable to aquire lock {:?} - {}", | |
226 | path, err), | |
227 | } | |
365bb90f DM |
228 | } |
229 | ||
51b499db DM |
230 | /// Split a file into equal sized chunks. The last chunk may be |
231 | /// smaller. Note: We cannot implement an `Iterator`, because iterators | |
232 | /// cannot return a borrowed buffer ref (we want zero-copy) | |
43eeef28 DM |
233 | pub fn file_chunker<C, R>( |
234 | mut file: R, | |
235 | chunk_size: usize, | |
606ce64b | 236 | mut chunk_cb: C |
43eeef28 | 237 | ) -> Result<(), Error> |
606ce64b | 238 | where C: FnMut(usize, &[u8]) -> Result<bool, Error>, |
43eeef28 DM |
239 | R: Read, |
240 | { | |
241 | ||
242 | const READ_BUFFER_SIZE: usize = 4*1024*1024; // 4M | |
243 | ||
244 | if chunk_size > READ_BUFFER_SIZE { bail!("chunk size too large!"); } | |
245 | ||
246 | let mut buf = vec![0u8; READ_BUFFER_SIZE]; | |
247 | ||
248 | let mut pos = 0; | |
249 | let mut file_pos = 0; | |
250 | loop { | |
251 | let mut eof = false; | |
252 | let mut tmp = &mut buf[..]; | |
253 | // try to read large portions, at least chunk_size | |
254 | while pos < chunk_size { | |
255 | match file.read(tmp) { | |
256 | Ok(0) => { eof = true; break; }, | |
257 | Ok(n) => { | |
258 | pos += n; | |
259 | if pos > chunk_size { break; } | |
260 | tmp = &mut tmp[n..]; | |
261 | } | |
262 | Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ } | |
5f0c2d56 | 263 | Err(e) => bail!("read chunk failed - {}", e.to_string()), |
43eeef28 DM |
264 | } |
265 | } | |
43eeef28 DM |
266 | let mut start = 0; |
267 | while start + chunk_size <= pos { | |
268 | if !(chunk_cb)(file_pos, &buf[start..start+chunk_size])? { break; } | |
269 | file_pos += chunk_size; | |
270 | start += chunk_size; | |
271 | } | |
272 | if eof { | |
273 | if start < pos { | |
274 | (chunk_cb)(file_pos, &buf[start..pos])?; | |
275 | //file_pos += pos - start; | |
276 | } | |
277 | break; | |
278 | } else { | |
279 | let rest = pos - start; | |
280 | if rest > 0 { | |
281 | let ptr = buf.as_mut_ptr(); | |
282 | unsafe { std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest); } | |
283 | pos = rest; | |
284 | } else { | |
285 | pos = 0; | |
286 | } | |
287 | } | |
288 | } | |
289 | ||
290 | Ok(()) | |
43eeef28 | 291 | } |
0fe5d605 | 292 | |
5d14eb6a DM |
293 | // Returns the Unix uid/gid for the sepcified system user. |
294 | pub fn getpwnam_ugid(username: &str) -> Result<(libc::uid_t,libc::gid_t), Error> { | |
295 | let info = unsafe { libc::getpwnam(std::ffi::CString::new(username).unwrap().as_ptr()) }; | |
296 | if info == std::ptr::null_mut() { | |
297 | bail!("getwpnam '{}' failed", username); | |
298 | } | |
299 | ||
300 | let info = unsafe { *info }; | |
301 | ||
302 | Ok((info.pw_uid, info.pw_gid)) | |
303 | } | |
304 | ||
305 | // Returns the hosts node name (UTS node name) | |
74a69302 DM |
306 | pub fn nodename() -> &'static str { |
307 | ||
308 | lazy_static!{ | |
309 | static ref NODENAME: String = { | |
310 | ||
0d38dcb4 DM |
311 | nix::sys::utsname::uname() |
312 | .nodename() | |
313 | .split('.') | |
314 | .next() | |
315 | .unwrap() | |
316 | .to_owned() | |
74a69302 DM |
317 | }; |
318 | } | |
319 | ||
320 | &NODENAME | |
321 | } | |
322 | ||
f5f13ebc DM |
323 | pub fn json_object_to_query(data: Value) -> Result<String, Error> { |
324 | ||
325 | let mut query = url::form_urlencoded::Serializer::new(String::new()); | |
326 | ||
327 | let object = data.as_object().ok_or_else(|| { | |
328 | format_err!("json_object_to_query: got wrong data type (expected object).") | |
329 | })?; | |
330 | ||
331 | for (key, value) in object { | |
332 | match value { | |
333 | Value::Bool(b) => { query.append_pair(key, &b.to_string()); } | |
334 | Value::Number(n) => { query.append_pair(key, &n.to_string()); } | |
335 | Value::String(s) => { query.append_pair(key, &s); } | |
336 | Value::Array(arr) => { | |
337 | for element in arr { | |
338 | match element { | |
339 | Value::Bool(b) => { query.append_pair(key, &b.to_string()); } | |
340 | Value::Number(n) => { query.append_pair(key, &n.to_string()); } | |
341 | Value::String(s) => { query.append_pair(key, &s); } | |
342 | _ => bail!("json_object_to_query: unable to handle complex array data types."), | |
343 | } | |
344 | } | |
345 | } | |
346 | _ => bail!("json_object_to_query: unable to handle complex data types."), | |
347 | } | |
348 | } | |
349 | ||
350 | Ok(query.finish()) | |
351 | } | |
352 | ||
0fe5d605 DM |
353 | pub fn required_string_param<'a>(param: &'a Value, name: &str) -> Result<&'a str, Error> { |
354 | match param[name].as_str() { | |
355 | Some(s) => Ok(s), | |
356 | None => bail!("missing parameter '{}'", name), | |
357 | } | |
358 | } | |
0d38dcb4 DM |
359 | |
360 | pub fn required_integer_param<'a>(param: &'a Value, name: &str) -> Result<i64, Error> { | |
361 | match param[name].as_i64() { | |
362 | Some(s) => Ok(s), | |
363 | None => bail!("missing parameter '{}'", name), | |
f8dfbb45 DM |
364 | } |
365 | } | |
366 | ||
367 | pub fn required_array_param<'a>(param: &'a Value, name: &str) -> Result<Vec<Value>, Error> { | |
368 | match param[name].as_array() { | |
369 | Some(s) => Ok(s.to_vec()), | |
370 | None => bail!("missing parameter '{}'", name), | |
0d38dcb4 DM |
371 | } |
372 | } | |
383e8577 DM |
373 | |
374 | pub fn complete_file_name(arg: &str) -> Vec<String> { | |
375 | ||
376 | let mut result = vec![]; | |
377 | ||
378 | use nix::fcntl::OFlag; | |
379 | use nix::sys::stat::Mode; | |
380 | use nix::fcntl::AtFlags; | |
381 | ||
382 | let mut dirname = std::path::PathBuf::from(arg); | |
383 | ||
384 | let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) { | |
385 | Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR, | |
386 | Err(_) => false, | |
387 | }; | |
388 | ||
389 | if !is_dir { | |
390 | if let Some(parent) = dirname.parent() { | |
391 | dirname = parent.to_owned(); | |
392 | } | |
393 | } | |
394 | ||
395 | let mut dir = match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) { | |
396 | Ok(d) => d, | |
397 | Err(_) => return result, | |
398 | }; | |
399 | ||
400 | for item in dir.iter() { | |
401 | if let Ok(entry) = item { | |
402 | if let Ok(name) = entry.file_name().to_str() { | |
403 | if name == "." || name == ".." { continue; } | |
404 | let mut newpath = dirname.clone(); | |
405 | newpath.push(name); | |
406 | ||
407 | if let Ok(stat) = nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty()) { | |
408 | if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR { | |
409 | newpath.push(""); | |
410 | if let Some(newpath) = newpath.to_str() { | |
411 | result.push(newpath.to_owned()); | |
412 | } | |
413 | continue; | |
414 | } | |
415 | } | |
416 | if let Some(newpath) = newpath.to_str() { | |
417 | result.push(newpath.to_owned()); | |
418 | } | |
419 | ||
420 | } | |
421 | } | |
422 | } | |
423 | ||
424 | result | |
425 | } | |
443f3743 DM |
426 | |
427 | /// Scan directory for matching file names. | |
428 | /// | |
429 | /// Scan through all directory entries and call `callback()` function | |
430 | /// if the entry name matches the regular expression. This function | |
431 | /// used unix `openat()`, so you can pass absolute or relative file | |
432 | /// names. This function simply skips non-UTF8 encoded names. | |
433 | pub fn scandir<P, F>( | |
434 | dirfd: RawFd, | |
121f18ef | 435 | path: &P, |
443f3743 | 436 | regex: ®ex::Regex, |
cce1676a | 437 | mut callback: F |
443f3743 | 438 | ) -> Result<(), Error> |
cce1676a | 439 | where F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>, |
121f18ef | 440 | P: ?Sized + nix::NixPath, |
443f3743 | 441 | { |
121f18ef | 442 | for entry in self::fs::scan_subdir(dirfd, path, regex)? { |
443f3743 DM |
443 | let entry = entry?; |
444 | let file_type = match entry.file_type() { | |
445 | Some(file_type) => file_type, | |
446 | None => bail!("unable to detect file type"), | |
447 | }; | |
443f3743 | 448 | |
121f18ef | 449 | callback(entry.parent_fd(), unsafe { entry.file_name_utf8_unchecked() }, file_type)?; |
443f3743 DM |
450 | } |
451 | Ok(()) | |
452 | } | |
7e13b2d6 DM |
453 | |
454 | pub fn get_hardware_address() -> Result<String, Error> { | |
455 | ||
1631c54f | 456 | static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub"; |
7e13b2d6 | 457 | |
1631c54f | 458 | let contents = file_get_contents(FILENAME)?; |
7e13b2d6 DM |
459 | let digest = md5::compute(contents); |
460 | ||
461 | Ok(format!("{:0x}", digest)) | |
462 | } | |
22968600 DM |
463 | |
464 | pub fn digest_to_hex(digest: &[u8]) -> String { | |
465 | ||
466 | const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; | |
467 | ||
468 | let mut buf = Vec::<u8>::with_capacity(digest.len()*2); | |
469 | ||
470 | for i in 0..digest.len() { | |
471 | buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); | |
472 | buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); | |
473 | } | |
474 | ||
475 | unsafe { String::from_utf8_unchecked(buf) } | |
476 | } | |
477 | ||
af2fddea DM |
478 | pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> { |
479 | if digest1 != digest2 { | |
480 | bail!("detected modified configuration - file changed by other user? Try again."); | |
481 | } | |
482 | Ok(()) | |
483 | } | |
b9903d63 DM |
484 | |
485 | /// Extract authentication cookie from cookie header. | |
486 | /// We assume cookie_name is already url encoded. | |
487 | pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> { | |
488 | ||
489 | for pair in cookie.split(';') { | |
490 | ||
491 | let (name, value) = match pair.find('=') { | |
492 | Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()), | |
493 | None => return None, // Cookie format error | |
494 | }; | |
495 | ||
496 | if name == cookie_name { | |
497 | use url::percent_encoding::percent_decode; | |
498 | if let Ok(value) = percent_decode(value.as_bytes()).decode_utf8() { | |
499 | return Some(value.into()); | |
500 | } else { | |
501 | return None; // Cookie format error | |
502 | } | |
503 | } | |
504 | } | |
505 | ||
506 | None | |
507 | } | |
af53186e DM |
508 | |
509 | pub fn join(data: &Vec<String>, sep: char) -> String { | |
510 | ||
511 | let mut list = String::new(); | |
512 | ||
513 | for item in data { | |
514 | if list.len() != 0 { list.push(sep); } | |
515 | list.push_str(item); | |
516 | } | |
517 | ||
518 | list | |
519 | } |