From: Wolfgang Bumiller Date: Wed, 7 Jul 2021 11:47:17 +0000 (+0200) Subject: move UPID to pbs-api-types, add UPIDExt X-Git-Tag: v2.0.10~279 X-Git-Url: https://git.proxmox.com/?a=commitdiff_plain;h=95f9d67ce9b258c31522cde1a1acd0a73b7119d4;p=proxmox-backup.git move UPID to pbs-api-types, add UPIDExt pbs-server side related methods are added via the UPIDExt trait Signed-off-by: Wolfgang Bumiller --- diff --git a/pbs-api-types/Cargo.toml b/pbs-api-types/Cargo.toml index 9f3e9f5f..cd3a7073 100644 --- a/pbs-api-types/Cargo.toml +++ b/pbs-api-types/Cargo.toml @@ -8,7 +8,11 @@ description = "general API type helpers for PBS" [dependencies] anyhow = "1.0" lazy_static = "1.4" +nix = "0.19.1" +libc = "0.2" regex = "1.2" serde = { version = "1.0", features = ["derive"] } proxmox = { version = "0.11.5", default-features = false, features = [ "api-macro" ] } + +pbs-systemd = { path = "../pbs-systemd" } diff --git a/pbs-api-types/src/lib.rs b/pbs-api-types/src/lib.rs index 7775324d..50072bf4 100644 --- a/pbs-api-types/src/lib.rs +++ b/pbs-api-types/src/lib.rs @@ -36,6 +36,8 @@ pub use userid::{Tokenname, TokennameRef}; pub use userid::{Username, UsernameRef}; pub use userid::{PROXMOX_GROUP_ID_SCHEMA, PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN_NAME_SCHEMA}; +pub mod upid; + const_regex! { pub BACKUP_TYPE_REGEX = concat!(r"^(", BACKUP_TYPE_RE!(), r")$"); diff --git a/pbs-api-types/src/upid.rs b/pbs-api-types/src/upid.rs new file mode 100644 index 00000000..5666df1e --- /dev/null +++ b/pbs-api-types/src/upid.rs @@ -0,0 +1,143 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use anyhow::{bail, Error}; + +use proxmox::api::schema::{ApiStringFormat, Schema, StringSchema}; +use proxmox::const_regex; +use proxmox::sys::linux::procfs; + +use crate::Authid; + +/// Unique Process/Task Identifier +/// +/// We use this to uniquely identify worker task. UPIDs have a short +/// string repesentaion, which gives additional information about the +/// type of the task. for example: +/// ```text +/// UPID:{node}:{pid}:{pstart}:{task_id}:{starttime}:{worker_type}:{worker_id}:{userid}: +/// UPID:elsa:00004F37:0039E469:00000000:5CA78B83:garbage_collection::root@pam: +/// ``` +/// Please note that we use tokio, so a single thread can run multiple +/// tasks. +// #[api] - manually implemented API type +#[derive(Debug, Clone)] +pub struct UPID { + /// The Unix PID + pub pid: libc::pid_t, + /// The Unix process start time from `/proc/pid/stat` + pub pstart: u64, + /// The task start time (Epoch) + pub starttime: i64, + /// The task ID (inside the process/thread) + pub task_id: usize, + /// Worker type (arbitrary ASCII string) + pub worker_type: String, + /// Worker ID (arbitrary ASCII string) + pub worker_id: Option, + /// The authenticated entity who started the task + pub auth_id: Authid, + /// The node name. + pub node: String, +} + +proxmox::forward_serialize_to_display!(UPID); +proxmox::forward_deserialize_to_from_str!(UPID); + +const_regex! { + pub PROXMOX_UPID_REGEX = concat!( + r"^UPID:(?P[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?):(?P[0-9A-Fa-f]{8}):", + r"(?P[0-9A-Fa-f]{8,9}):(?P[0-9A-Fa-f]{8,16}):(?P[0-9A-Fa-f]{8}):", + r"(?P[^:\s]+):(?P[^:\s]*):(?P[^:\s]+):$" + ); +} + +pub const PROXMOX_UPID_FORMAT: ApiStringFormat = + ApiStringFormat::Pattern(&PROXMOX_UPID_REGEX); + +impl UPID { + pub const API_SCHEMA: Schema = StringSchema::new("Unique Process/Task Identifier") + .min_length("UPID:N:12345678:12345678:12345678:::".len()) + .max_length(128) // arbitrary + .format(&PROXMOX_UPID_FORMAT) + .schema(); + + /// Create a new UPID + pub fn new( + worker_type: &str, + worker_id: Option, + auth_id: Authid, + ) -> Result { + + let pid = unsafe { libc::getpid() }; + + let bad: &[_] = &['/', ':', ' ']; + + if worker_type.contains(bad) { + bail!("illegal characters in worker type '{}'", worker_type); + } + + static WORKER_TASK_NEXT_ID: AtomicUsize = AtomicUsize::new(0); + + let task_id = WORKER_TASK_NEXT_ID.fetch_add(1, Ordering::SeqCst); + + Ok(UPID { + pid, + pstart: procfs::PidStat::read_from_pid(nix::unistd::Pid::from_raw(pid))?.starttime, + starttime: proxmox::tools::time::epoch_i64(), + task_id, + worker_type: worker_type.to_owned(), + worker_id, + auth_id, + node: proxmox::tools::nodename().to_owned(), + }) + } +} + + +impl std::str::FromStr for UPID { + type Err = Error; + + fn from_str(s: &str) -> Result { + if let Some(cap) = PROXMOX_UPID_REGEX.captures(s) { + + let worker_id = if cap["wid"].is_empty() { + None + } else { + let wid = pbs_systemd::unescape_unit(&cap["wid"])?; + Some(wid) + }; + + Ok(UPID { + pid: i32::from_str_radix(&cap["pid"], 16).unwrap(), + pstart: u64::from_str_radix(&cap["pstart"], 16).unwrap(), + starttime: i64::from_str_radix(&cap["starttime"], 16).unwrap(), + task_id: usize::from_str_radix(&cap["task_id"], 16).unwrap(), + worker_type: cap["wtype"].to_string(), + worker_id, + auth_id: cap["authid"].parse()?, + node: cap["node"].to_string(), + }) + } else { + bail!("unable to parse UPID '{}'", s); + } + + } +} + +impl std::fmt::Display for UPID { + + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + + let wid = if let Some(ref id) = self.worker_id { + pbs_systemd::escape_unit(id, false) + } else { + String::new() + }; + + // Note: pstart can be > 32bit if uptime > 497 days, so this can result in + // more that 8 characters for pstart + + write!(f, "UPID:{}:{:08X}:{:08X}:{:08X}:{:08X}:{}:{}:{}:", + self.node, self.pid, self.pstart, self.task_id, self.starttime, self.worker_type, wid, self.auth_id) + } +} diff --git a/src/api2/node/tasks.rs b/src/api2/node/tasks.rs index f42a1a69..34e71af1 100644 --- a/src/api2/node/tasks.rs +++ b/src/api2/node/tasks.rs @@ -13,7 +13,7 @@ use crate::tools; use crate::api2::types::*; use crate::api2::pull::check_pull_privs; -use crate::server::{self, UPID, TaskState, TaskListInfoIterator}; +use crate::server::{self, UPID, UPIDExt, TaskState, TaskListInfoIterator}; use crate::config::acl::{ PRIV_DATASTORE_MODIFY, PRIV_DATASTORE_VERIFY, diff --git a/src/server/upid.rs b/src/server/upid.rs index 3ca7cff2..d0b0cea2 100644 --- a/src/server/upid.rs +++ b/src/server/upid.rs @@ -1,151 +1,20 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; +pub use pbs_api_types::upid::UPID; -use anyhow::{bail, Error}; - -use proxmox::api::schema::{ApiStringFormat, Schema, StringSchema}; -use proxmox::const_regex; -use proxmox::sys::linux::procfs; - -use crate::api2::types::Authid; - -/// Unique Process/Task Identifier -/// -/// We use this to uniquely identify worker task. UPIDs have a short -/// string repesentaion, which gives additional information about the -/// type of the task. for example: -/// ```text -/// UPID:{node}:{pid}:{pstart}:{task_id}:{starttime}:{worker_type}:{worker_id}:{userid}: -/// UPID:elsa:00004F37:0039E469:00000000:5CA78B83:garbage_collection::root@pam: -/// ``` -/// Please note that we use tokio, so a single thread can run multiple -/// tasks. -// #[api] - manually implemented API type -#[derive(Debug, Clone)] -pub struct UPID { - /// The Unix PID - pub pid: libc::pid_t, - /// The Unix process start time from `/proc/pid/stat` - pub pstart: u64, - /// The task start time (Epoch) - pub starttime: i64, - /// The task ID (inside the process/thread) - pub task_id: usize, - /// Worker type (arbitrary ASCII string) - pub worker_type: String, - /// Worker ID (arbitrary ASCII string) - pub worker_id: Option, - /// The authenticated entity who started the task - pub auth_id: Authid, - /// The node name. - pub node: String, +pub trait UPIDExt: private::Sealed { + /// Returns the absolute path to the task log file + fn log_path(&self) -> std::path::PathBuf; } -proxmox::forward_serialize_to_display!(UPID); -proxmox::forward_deserialize_to_from_str!(UPID); - -const_regex! { - pub PROXMOX_UPID_REGEX = concat!( - r"^UPID:(?P[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?):(?P[0-9A-Fa-f]{8}):", - r"(?P[0-9A-Fa-f]{8,9}):(?P[0-9A-Fa-f]{8,16}):(?P[0-9A-Fa-f]{8}):", - r"(?P[^:\s]+):(?P[^:\s]*):(?P[^:\s]+):$" - ); +mod private { + pub trait Sealed {} + impl Sealed for super::UPID {} } -pub const PROXMOX_UPID_FORMAT: ApiStringFormat = - ApiStringFormat::Pattern(&PROXMOX_UPID_REGEX); - -impl UPID { - pub const API_SCHEMA: Schema = StringSchema::new("Unique Process/Task Identifier") - .min_length("UPID:N:12345678:12345678:12345678:::".len()) - .max_length(128) // arbitrary - .format(&PROXMOX_UPID_FORMAT) - .schema(); - - /// Create a new UPID - pub fn new( - worker_type: &str, - worker_id: Option, - auth_id: Authid, - ) -> Result { - - let pid = unsafe { libc::getpid() }; - - let bad: &[_] = &['/', ':', ' ']; - - if worker_type.contains(bad) { - bail!("illegal characters in worker type '{}'", worker_type); - } - - static WORKER_TASK_NEXT_ID: AtomicUsize = AtomicUsize::new(0); - - let task_id = WORKER_TASK_NEXT_ID.fetch_add(1, Ordering::SeqCst); - - Ok(UPID { - pid, - pstart: procfs::PidStat::read_from_pid(nix::unistd::Pid::from_raw(pid))?.starttime, - starttime: proxmox::tools::time::epoch_i64(), - task_id, - worker_type: worker_type.to_owned(), - worker_id, - auth_id, - node: proxmox::tools::nodename().to_owned(), - }) - } - - /// Returns the absolute path to the task log file - pub fn log_path(&self) -> std::path::PathBuf { +impl UPIDExt for UPID { + fn log_path(&self) -> std::path::PathBuf { let mut path = std::path::PathBuf::from(super::PROXMOX_BACKUP_TASK_DIR); path.push(format!("{:02X}", self.pstart % 256)); path.push(self.to_string()); path } } - - -impl std::str::FromStr for UPID { - type Err = Error; - - fn from_str(s: &str) -> Result { - if let Some(cap) = PROXMOX_UPID_REGEX.captures(s) { - - let worker_id = if cap["wid"].is_empty() { - None - } else { - let wid = crate::tools::systemd::unescape_unit(&cap["wid"])?; - Some(wid) - }; - - Ok(UPID { - pid: i32::from_str_radix(&cap["pid"], 16).unwrap(), - pstart: u64::from_str_radix(&cap["pstart"], 16).unwrap(), - starttime: i64::from_str_radix(&cap["starttime"], 16).unwrap(), - task_id: usize::from_str_radix(&cap["task_id"], 16).unwrap(), - worker_type: cap["wtype"].to_string(), - worker_id, - auth_id: cap["authid"].parse()?, - node: cap["node"].to_string(), - }) - } else { - bail!("unable to parse UPID '{}'", s); - } - - } -} - -impl std::fmt::Display for UPID { - - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - - let wid = if let Some(ref id) = self.worker_id { - crate::tools::systemd::escape_unit(id, false) - } else { - String::new() - }; - - // Note: pstart can be > 32bit if uptime > 497 days, so this can result in - // more that 8 characters for pstart - - write!(f, "UPID:{}:{:08X}:{:08X}:{:08X}:{:08X}:{}:{}:{}:", - self.node, self.pid, self.pstart, self.task_id, self.starttime, self.worker_type, wid, self.auth_id) - } -} diff --git a/src/server/worker_task.rs b/src/server/worker_task.rs index ecda7d5c..f60556ef 100644 --- a/src/server/worker_task.rs +++ b/src/server/worker_task.rs @@ -16,7 +16,7 @@ use proxmox::sys::linux::procfs; use proxmox::try_block; use proxmox::tools::fs::{create_path, open_file_locked, replace_file, CreateOptions}; -use super::UPID; +use super::{UPID, UPIDExt}; use pbs_buildcfg;