pbs-client/src/tools/mod.rs

   1 //! Shared tools useful for common CLI clients.
   2 use std::collections::HashMap;
   3 use std::env::VarError::{NotPresent, NotUnicode};
   4 use std::fs::File;
   5 use std::io::{BufRead, BufReader};
   6 use std::os::unix::io::FromRawFd;
   7 use std::process::Command;
   8
   9 use anyhow::{bail, format_err, Context, Error};
  10 use serde_json::{json, Value};
  11 use xdg::BaseDirectories;
  12
  13 use proxmox_router::cli::{complete_file_name, shellword_split};
  14 use proxmox_schema::*;
  15 use proxmox_sys::fs::file_get_json;
  16
  17 use pbs_api_types::{Authid, RateLimitConfig, UserWithTokens, BACKUP_REPO_URL};
  18 use pbs_tools::json::json_object_to_query;
  19
  20 use crate::{BackupRepository, HttpClient, HttpClientOptions};
  21
  22 pub mod key_source;
  23
  24 const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
  25 const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
  26
  27 pub const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  28     .format(&BACKUP_REPO_URL)
  29     .max_length(256)
  30     .schema();
  31
  32 pub const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new("Chunk size in KB. Must be a power of 2.")
  33     .minimum(64)
  34     .maximum(4096)
  35     .default(4096)
  36     .schema();
  37
  38 /// Helper to read a secret through a environment variable (ENV).
  39 ///
  40 /// Tries the following variable names in order and returns the value
  41 /// it will resolve for the first defined one:
  42 ///
  43 /// BASE_NAME => use value from ENV(BASE_NAME) directly as secret
  44 /// BASE_NAME_FD => read the secret from the specified file descriptor
  45 /// BASE_NAME_FILE => read the secret from the specified file name
  46 /// BASE_NAME_CMD => read the secret from specified command first line of output on stdout
  47 ///
  48 /// Only return the first line of data (without CRLF).
  49 pub fn get_secret_from_env(base_name: &str) -> Result<Option<String>, Error> {
  50     let firstline = |data: String| -> String {
  51         match data.lines().next() {
  52             Some(line) => line.to_string(),
  53             None => String::new(),
  54         }
  55     };
  56
  57     let firstline_file = |file: &mut File| -> Result<String, Error> {
  58         let reader = BufReader::new(file);
  59         match reader.lines().next() {
  60             Some(Ok(line)) => Ok(line),
  61             Some(Err(err)) => Err(err.into()),
  62             None => Ok(String::new()),
  63         }
  64     };
  65
  66     match std::env::var(base_name) {
  67         Ok(p) => return Ok(Some(firstline(p))),
  68         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", base_name)),
  69         Err(NotPresent) => {}
  70     };
  71
  72     let env_name = format!("{}_FD", base_name);
  73     match std::env::var(&env_name) {
  74         Ok(fd_str) => {
  75             let fd: i32 = fd_str.parse().map_err(|err| {
  76                 format_err!(
  77                     "unable to parse file descriptor in ENV({}): {}",
  78                     env_name,
  79                     err
  80                 )
  81             })?;
  82             let mut file = unsafe { File::from_raw_fd(fd) };
  83             return Ok(Some(firstline_file(&mut file)?));
  84         }
  85         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", env_name)),
  86         Err(NotPresent) => {}
  87     }
  88
  89     let env_name = format!("{}_FILE", base_name);
  90     match std::env::var(&env_name) {
  91         Ok(filename) => {
  92             let mut file = std::fs::File::open(filename)
  93                 .map_err(|err| format_err!("unable to open file in ENV({}): {}", env_name, err))?;
  94             return Ok(Some(firstline_file(&mut file)?));
  95         }
  96         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", env_name)),
  97         Err(NotPresent) => {}
  98     }
  99
 100     let env_name = format!("{}_CMD", base_name);
 101     match std::env::var(&env_name) {
 102         Ok(ref command) => {
 103             let args = shellword_split(command)?;
 104             let mut command = Command::new(&args[0]);
 105             command.args(&args[1..]);
 106             let output = proxmox_sys::command::run_command(command, None)?;
 107             return Ok(Some(firstline(output)));
 108         }
 109         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", env_name)),
 110         Err(NotPresent) => {}
 111     }
 112
 113     Ok(None)
 114 }
 115
 116 pub fn get_default_repository() -> Option<String> {
 117     std::env::var("PBS_REPOSITORY").ok()
 118 }
 119
 120 pub fn extract_repository_from_value(param: &Value) -> Result<BackupRepository, Error> {
 121     let repo_url = param["repository"]
 122         .as_str()
 123         .map(String::from)
 124         .or_else(get_default_repository)
 125         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
 126
 127     let repo: BackupRepository = repo_url.parse()?;
 128
 129     Ok(repo)
 130 }
 131
 132 pub fn extract_repository_from_map(param: &HashMap<String, String>) -> Option<BackupRepository> {
 133     param
 134         .get("repository")
 135         .map(String::from)
 136         .or_else(get_default_repository)
 137         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
 138 }
 139
 140 pub fn connect(repo: &BackupRepository) -> Result<HttpClient, Error> {
 141     let rate_limit = RateLimitConfig::default(); // unlimited
 142     connect_do(repo.host(), repo.port(), repo.auth_id(), rate_limit)
 143         .map_err(|err| format_err!("error building client for repository {} - {}", repo, err))
 144 }
 145
 146 pub fn connect_rate_limited(
 147     repo: &BackupRepository,
 148     rate_limit: RateLimitConfig,
 149 ) -> Result<HttpClient, Error> {
 150     connect_do(repo.host(), repo.port(), repo.auth_id(), rate_limit)
 151         .map_err(|err| format_err!("error building client for repository {} - {}", repo, err))
 152 }
 153
 154 fn connect_do(
 155     server: &str,
 156     port: u16,
 157     auth_id: &Authid,
 158     rate_limit: RateLimitConfig,
 159 ) -> Result<HttpClient, Error> {
 160     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 161
 162     let password = get_secret_from_env(ENV_VAR_PBS_PASSWORD)?;
 163     let options = HttpClientOptions::new_interactive(password, fingerprint).rate_limit(rate_limit);
 164
 165     HttpClient::new(server, port, auth_id, options)
 166 }
 167
 168 /// like get, but simply ignore errors and return Null instead
 169 pub async fn try_get(repo: &BackupRepository, url: &str) -> Value {
 170     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 171     let password = get_secret_from_env(ENV_VAR_PBS_PASSWORD).unwrap_or(None);
 172
 173     // ticket cache, but no questions asked
 174     let options = HttpClientOptions::new_interactive(password, fingerprint).interactive(false);
 175
 176     let client = match HttpClient::new(repo.host(), repo.port(), repo.auth_id(), options) {
 177         Ok(v) => v,
 178         _ => return Value::Null,
 179     };
 180
 181     let mut resp = match client.get(url, None).await {
 182         Ok(v) => v,
 183         _ => return Value::Null,
 184     };
 185
 186     if let Some(map) = resp.as_object_mut() {
 187         if let Some(data) = map.remove("data") {
 188             return data;
 189         }
 190     }
 191     Value::Null
 192 }
 193
 194 pub fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 195     proxmox_async::runtime::main(async { complete_backup_group_do(param).await })
 196 }
 197
 198 pub async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
 199     let mut result = vec![];
 200
 201     let repo = match extract_repository_from_map(param) {
 202         Some(v) => v,
 203         _ => return result,
 204     };
 205
 206     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 207
 208     let data = try_get(&repo, &path).await;
 209
 210     if let Some(list) = data.as_array() {
 211         for item in list {
 212             if let (Some(backup_id), Some(backup_type)) =
 213                 (item["backup-id"].as_str(), item["backup-type"].as_str())
 214             {
 215                 result.push(format!("{}/{}", backup_type, backup_id));
 216             }
 217         }
 218     }
 219
 220     result
 221 }
 222
 223 pub fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 224     proxmox_async::runtime::main(async { complete_group_or_snapshot_do(arg, param).await })
 225 }
 226
 227 pub async fn complete_group_or_snapshot_do(
 228     arg: &str,
 229     param: &HashMap<String, String>,
 230 ) -> Vec<String> {
 231     if arg.matches('/').count() < 2 {
 232         let groups = complete_backup_group_do(param).await;
 233         let mut result = vec![];
 234         for group in groups {
 235             result.push(group.to_string());
 236             result.push(format!("{}/", group));
 237         }
 238         return result;
 239     }
 240
 241     complete_backup_snapshot_do(param).await
 242 }
 243
 244 pub fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 245     proxmox_async::runtime::main(async { complete_backup_snapshot_do(param).await })
 246 }
 247
 248 pub async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
 249     let mut result = vec![];
 250
 251     let repo = match extract_repository_from_map(param) {
 252         Some(v) => v,
 253         _ => return result,
 254     };
 255
 256     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 257
 258     let data = try_get(&repo, &path).await;
 259
 260     if let Value::Array(list) = data {
 261         for item in list {
 262             match serde_json::from_value::<pbs_api_types::BackupDir>(item) {
 263                 Ok(item) => result.push(item.to_string()),
 264                 Err(_) => {
 265                     // FIXME: print error in completion?
 266                     continue;
 267                 }
 268             };
 269         }
 270     }
 271
 272     result
 273 }
 274
 275 pub fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 276     proxmox_async::runtime::main(async { complete_server_file_name_do(param).await })
 277 }
 278
 279 pub async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
 280     let mut result = vec![];
 281
 282     let repo = match extract_repository_from_map(param) {
 283         Some(v) => v,
 284         _ => return result,
 285     };
 286
 287     let snapshot: pbs_api_types::BackupDir = match param.get("snapshot") {
 288         Some(path) => match path.parse() {
 289             Ok(v) => v,
 290             _ => return result,
 291         },
 292         _ => return result,
 293     };
 294
 295     let ns: pbs_api_types::BackupNamespace = match param.get("ns") {
 296         Some(ns) => match ns.parse() {
 297             Ok(v) => v,
 298             _ => return result,
 299         },
 300         _ => return result,
 301     };
 302
 303     let query = json_object_to_query(json!({
 304         "backup-ns": ns,
 305         "backup-type": snapshot.group.ty,
 306         "backup-id": snapshot.group.id,
 307         "backup-time": snapshot.time,
 308     }))
 309     .unwrap();
 310
 311     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
 312
 313     let data = try_get(&repo, &path).await;
 314
 315     if let Some(list) = data.as_array() {
 316         for item in list {
 317             if let Some(filename) = item["filename"].as_str() {
 318                 result.push(filename.to_owned());
 319             }
 320         }
 321     }
 322
 323     result
 324 }
 325
 326 pub fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 327     complete_server_file_name(arg, param)
 328         .iter()
 329         .map(|v| pbs_tools::format::strip_server_file_extension(v).to_owned())
 330         .collect()
 331 }
 332
 333 pub fn complete_pxar_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 334     complete_server_file_name(arg, param)
 335         .iter()
 336         .filter_map(|name| {
 337             if name.ends_with(".pxar.didx") {
 338                 Some(pbs_tools::format::strip_server_file_extension(name).to_owned())
 339             } else {
 340                 None
 341             }
 342         })
 343         .collect()
 344 }
 345
 346 pub fn complete_img_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 347     complete_server_file_name(arg, param)
 348         .iter()
 349         .filter_map(|name| {
 350             if name.ends_with(".img.fidx") {
 351                 Some(pbs_tools::format::strip_server_file_extension(name).to_owned())
 352             } else {
 353                 None
 354             }
 355         })
 356         .collect()
 357 }
 358
 359 pub fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 360     let mut result = vec![];
 361
 362     let mut size = 64;
 363     loop {
 364         result.push(size.to_string());
 365         size *= 2;
 366         if size > 4096 {
 367             break;
 368         }
 369     }
 370
 371     result
 372 }
 373
 374 pub fn complete_auth_id(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 375     proxmox_async::runtime::main(async { complete_auth_id_do(param).await })
 376 }
 377
 378 pub async fn complete_auth_id_do(param: &HashMap<String, String>) -> Vec<String> {
 379     let mut result = vec![];
 380
 381     let repo = match extract_repository_from_map(param) {
 382         Some(v) => v,
 383         _ => return result,
 384     };
 385
 386     let data = try_get(&repo, "api2/json/access/users?include_tokens=true").await;
 387
 388     if let Ok(parsed) = serde_json::from_value::<Vec<UserWithTokens>>(data) {
 389         for user in parsed {
 390             result.push(user.userid.to_string());
 391             for token in user.tokens {
 392                 result.push(token.tokenid.to_string());
 393             }
 394         }
 395     };
 396
 397     result
 398 }
 399
 400 pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 401     let mut result = vec![];
 402
 403     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 404         Ok(v) => v,
 405         _ => return result,
 406     };
 407
 408     // usually $HOME/.cache/proxmox-backup/repo-list
 409     let path = match base.place_cache_file("repo-list") {
 410         Ok(v) => v,
 411         _ => return result,
 412     };
 413
 414     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 415
 416     if let Some(map) = data.as_object() {
 417         for (repo, _count) in map {
 418             result.push(repo.to_owned());
 419         }
 420     }
 421
 422     result
 423 }
 424
 425 pub fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 426     let mut result = vec![];
 427
 428     let data: Vec<&str> = arg.splitn(2, ':').collect();
 429
 430     if data.len() != 2 {
 431         result.push(String::from("root.pxar:/"));
 432         result.push(String::from("etc.pxar:/etc"));
 433         return result;
 434     }
 435
 436     let files = complete_file_name(data[1], param);
 437
 438     for file in files {
 439         result.push(format!("{}:{}", data[0], file));
 440     }
 441
 442     result
 443 }
 444
 445 pub fn base_directories() -> Result<xdg::BaseDirectories, Error> {
 446     xdg::BaseDirectories::with_prefix("proxmox-backup").map_err(Error::from)
 447 }
 448
 449 /// Convenience helper for better error messages:
 450 pub fn find_xdg_file(
 451     file_name: impl AsRef<std::path::Path>,
 452     description: &'static str,
 453 ) -> Result<Option<std::path::PathBuf>, Error> {
 454     let file_name = file_name.as_ref();
 455     base_directories()
 456         .map(|base| base.find_config_file(file_name))
 457         .with_context(|| format!("error searching for {}", description))
 458 }
 459
 460 pub fn place_xdg_file(
 461     file_name: impl AsRef<std::path::Path>,
 462     description: &'static str,
 463 ) -> Result<std::path::PathBuf, Error> {
 464     let file_name = file_name.as_ref();
 465     base_directories()
 466         .and_then(|base| base.place_config_file(file_name).map_err(Error::from))
 467         .with_context(|| format!("failed to place {} in xdg home", description))
 468 }