pbs-client/src/tools/mod.rs

   1 //! Shared tools useful for common CLI clients.
   2 use std::collections::HashMap;
   3 use std::fs::File;
   4 use std::os::unix::io::FromRawFd;
   5 use std::env::VarError::{NotUnicode, NotPresent};
   6 use std::io::{BufReader, BufRead};
   7 use std::process::Command;
   8
   9 use anyhow::{bail, format_err, Context, Error};
  10 use serde_json::{json, Value};
  11 use xdg::BaseDirectories;
  12
  13 use proxmox_schema::*;
  14 use proxmox_router::cli::{complete_file_name, shellword_split};
  15 use proxmox_sys::fs::file_get_json;
  16
  17 use pbs_api_types::{BACKUP_REPO_URL, Authid, RateLimitConfig, UserWithTokens};
  18 use pbs_datastore::BackupDir;
  19 use pbs_tools::json::json_object_to_query;
  20
  21 use crate::{BackupRepository, HttpClient, HttpClientOptions};
  22
  23 pub mod key_source;
  24
  25 const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
  26 const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
  27
  28 pub const REPO_URL_SCHEMA: Schema = StringSchema::new("Repository URL.")
  29     .format(&BACKUP_REPO_URL)
  30     .max_length(256)
  31     .schema();
  32
  33 pub const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new("Chunk size in KB. Must be a power of 2.")
  34     .minimum(64)
  35     .maximum(4096)
  36     .default(4096)
  37     .schema();
  38
  39 /// Helper to read a secret through a environment variable (ENV).
  40 ///
  41 /// Tries the following variable names in order and returns the value
  42 /// it will resolve for the first defined one:
  43 ///
  44 /// BASE_NAME => use value from ENV(BASE_NAME) directly as secret
  45 /// BASE_NAME_FD => read the secret from the specified file descriptor
  46 /// BASE_NAME_FILE => read the secret from the specified file name
  47 /// BASE_NAME_CMD => read the secret from specified command first line of output on stdout
  48 ///
  49 /// Only return the first line of data (without CRLF).
  50 pub fn get_secret_from_env(base_name: &str) -> Result<Option<String>, Error> {
  51
  52     let firstline = |data: String| -> String {
  53         match data.lines().next() {
  54             Some(line) => line.to_string(),
  55             None => String::new(),
  56         }
  57     };
  58
  59     let firstline_file = |file: &mut File| -> Result<String, Error> {
  60         let reader = BufReader::new(file);
  61         match reader.lines().next() {
  62             Some(Ok(line)) => Ok(line),
  63             Some(Err(err)) => Err(err.into()),
  64             None => Ok(String::new()),
  65         }
  66     };
  67
  68     match std::env::var(base_name) {
  69         Ok(p) => return Ok(Some(firstline(p))),
  70         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", base_name)),
  71         Err(NotPresent) => {},
  72     };
  73
  74     let env_name = format!("{}_FD", base_name);
  75     match std::env::var(&env_name) {
  76         Ok(fd_str) => {
  77             let fd: i32 = fd_str.parse()
  78                 .map_err(|err| format_err!("unable to parse file descriptor in ENV({}): {}", env_name, err))?;
  79             let mut file = unsafe { File::from_raw_fd(fd) };
  80             return Ok(Some(firstline_file(&mut file)?));
  81         }
  82         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", env_name)),
  83         Err(NotPresent) => {},
  84     }
  85
  86     let env_name = format!("{}_FILE", base_name);
  87     match std::env::var(&env_name) {
  88         Ok(filename) => {
  89             let mut file = std::fs::File::open(filename)
  90                 .map_err(|err| format_err!("unable to open file in ENV({}): {}", env_name, err))?;
  91             return Ok(Some(firstline_file(&mut file)?));
  92         }
  93         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", env_name)),
  94         Err(NotPresent) => {},
  95     }
  96
  97     let env_name = format!("{}_CMD", base_name);
  98     match std::env::var(&env_name) {
  99         Ok(ref command) => {
 100             let args = shellword_split(command)?;
 101             let mut command = Command::new(&args[0]);
 102             command.args(&args[1..]);
 103             let output = proxmox_sys::command::run_command(command, None)?;
 104             return Ok(Some(firstline(output)));
 105         }
 106         Err(NotUnicode(_)) => bail!(format!("{} contains bad characters", env_name)),
 107         Err(NotPresent) => {},
 108     }
 109
 110     Ok(None)
 111 }
 112
 113 pub fn get_default_repository() -> Option<String> {
 114     std::env::var("PBS_REPOSITORY").ok()
 115 }
 116
 117 pub fn extract_repository_from_value(param: &Value) -> Result<BackupRepository, Error> {
 118     let repo_url = param["repository"]
 119         .as_str()
 120         .map(String::from)
 121         .or_else(get_default_repository)
 122         .ok_or_else(|| format_err!("unable to get (default) repository"))?;
 123
 124     let repo: BackupRepository = repo_url.parse()?;
 125
 126     Ok(repo)
 127 }
 128
 129 pub fn extract_repository_from_map(param: &HashMap<String, String>) -> Option<BackupRepository> {
 130     param
 131         .get("repository")
 132         .map(String::from)
 133         .or_else(get_default_repository)
 134         .and_then(|repo_url| repo_url.parse::<BackupRepository>().ok())
 135 }
 136
 137 pub fn connect(repo: &BackupRepository) -> Result<HttpClient, Error> {
 138     let rate_limit = RateLimitConfig::default(); // unlimited
 139     connect_do(repo.host(), repo.port(), repo.auth_id(), rate_limit)
 140         .map_err(|err| format_err!("error building client for repository {} - {}", repo, err))
 141 }
 142
 143 pub fn connect_rate_limited(
 144     repo: &BackupRepository,
 145     rate_limit: RateLimitConfig,
 146 ) -> Result<HttpClient, Error> {
 147     connect_do(repo.host(), repo.port(), repo.auth_id(), rate_limit)
 148         .map_err(|err| format_err!("error building client for repository {} - {}", repo, err))
 149 }
 150
 151 fn connect_do(
 152     server: &str,
 153     port: u16,
 154     auth_id: &Authid,
 155     rate_limit: RateLimitConfig,
 156 ) -> Result<HttpClient, Error> {
 157     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 158
 159     let password = get_secret_from_env(ENV_VAR_PBS_PASSWORD)?;
 160     let options = HttpClientOptions::new_interactive(password, fingerprint)
 161         .rate_limit(rate_limit);
 162
 163     HttpClient::new(server, port, auth_id, options)
 164 }
 165
 166 /// like get, but simply ignore errors and return Null instead
 167 pub async fn try_get(repo: &BackupRepository, url: &str) -> Value {
 168
 169     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 170     let password = get_secret_from_env(ENV_VAR_PBS_PASSWORD).unwrap_or(None);
 171
 172     // ticket cache, but no questions asked
 173     let options = HttpClientOptions::new_interactive(password, fingerprint)
 174         .interactive(false);
 175
 176     let client = match HttpClient::new(repo.host(), repo.port(), repo.auth_id(), options) {
 177         Ok(v) => v,
 178         _ => return Value::Null,
 179     };
 180
 181     let mut resp = match client.get(url, None).await {
 182         Ok(v) => v,
 183         _ => return Value::Null,
 184     };
 185
 186     if let Some(map) = resp.as_object_mut() {
 187         if let Some(data) = map.remove("data") {
 188             return data;
 189         }
 190     }
 191     Value::Null
 192 }
 193
 194 pub fn complete_backup_group(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 195     proxmox_async::runtime::main(async { complete_backup_group_do(param).await })
 196 }
 197
 198 pub async fn complete_backup_group_do(param: &HashMap<String, String>) -> Vec<String> {
 199
 200     let mut result = vec![];
 201
 202     let repo = match extract_repository_from_map(param) {
 203         Some(v) => v,
 204         _ => return result,
 205     };
 206
 207     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 208
 209     let data = try_get(&repo, &path).await;
 210
 211     if let Some(list) = data.as_array() {
 212         for item in list {
 213             if let (Some(backup_id), Some(backup_type)) =
 214                 (item["backup-id"].as_str(), item["backup-type"].as_str())
 215             {
 216                 result.push(format!("{}/{}", backup_type, backup_id));
 217             }
 218         }
 219     }
 220
 221     result
 222 }
 223
 224 pub fn complete_group_or_snapshot(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 225     proxmox_async::runtime::main(async { complete_group_or_snapshot_do(arg, param).await })
 226 }
 227
 228 pub async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 229
 230     if arg.matches('/').count() < 2 {
 231         let groups = complete_backup_group_do(param).await;
 232         let mut result = vec![];
 233         for group in groups {
 234             result.push(group.to_string());
 235             result.push(format!("{}/", group));
 236         }
 237         return result;
 238     }
 239
 240     complete_backup_snapshot_do(param).await
 241 }
 242
 243 pub fn complete_backup_snapshot(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 244     proxmox_async::runtime::main(async { complete_backup_snapshot_do(param).await })
 245 }
 246
 247 pub async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<String> {
 248
 249     let mut result = vec![];
 250
 251     let repo = match extract_repository_from_map(param) {
 252         Some(v) => v,
 253         _ => return result,
 254     };
 255
 256     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 257
 258     let data = try_get(&repo, &path).await;
 259
 260     if let Some(list) = data.as_array() {
 261         for item in list {
 262             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
 263                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
 264             {
 265                 if let Ok(snapshot) = BackupDir::new(backup_type, backup_id, backup_time) {
 266                     result.push(snapshot.relative_path().to_str().unwrap().to_owned());
 267                 }
 268             }
 269         }
 270     }
 271
 272     result
 273 }
 274
 275 pub fn complete_server_file_name(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 276     proxmox_async::runtime::main(async { complete_server_file_name_do(param).await })
 277 }
 278
 279 pub async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<String> {
 280
 281     let mut result = vec![];
 282
 283     let repo = match extract_repository_from_map(param) {
 284         Some(v) => v,
 285         _ => return result,
 286     };
 287
 288     let snapshot: BackupDir = match param.get("snapshot") {
 289         Some(path) => {
 290             match path.parse() {
 291                 Ok(v) => v,
 292                 _ => return result,
 293             }
 294         }
 295         _ => return result,
 296     };
 297
 298     let query = json_object_to_query(json!({
 299         "backup-type": snapshot.group().backup_type(),
 300         "backup-id": snapshot.group().backup_id(),
 301         "backup-time": snapshot.backup_time(),
 302     })).unwrap();
 303
 304     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);
 305
 306     let data = try_get(&repo, &path).await;
 307
 308     if let Some(list) = data.as_array() {
 309         for item in list {
 310             if let Some(filename) = item["filename"].as_str() {
 311                 result.push(filename.to_owned());
 312             }
 313         }
 314     }
 315
 316     result
 317 }
 318
 319 pub fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 320     complete_server_file_name(arg, param)
 321         .iter()
 322         .map(|v| pbs_tools::format::strip_server_file_extension(v).to_owned())
 323         .collect()
 324 }
 325
 326 pub fn complete_pxar_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 327     complete_server_file_name(arg, param)
 328         .iter()
 329         .filter_map(|name| {
 330             if name.ends_with(".pxar.didx") {
 331                 Some(pbs_tools::format::strip_server_file_extension(name).to_owned())
 332             } else {
 333                 None
 334             }
 335         })
 336         .collect()
 337 }
 338
 339 pub fn complete_img_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 340     complete_server_file_name(arg, param)
 341         .iter()
 342         .filter_map(|name| {
 343             if name.ends_with(".img.fidx") {
 344                 Some(pbs_tools::format::strip_server_file_extension(name).to_owned())
 345             } else {
 346                 None
 347             }
 348         })
 349         .collect()
 350 }
 351
 352 pub fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 353
 354     let mut result = vec![];
 355
 356     let mut size = 64;
 357     loop {
 358         result.push(size.to_string());
 359         size *= 2;
 360         if size > 4096 { break; }
 361     }
 362
 363     result
 364 }
 365
 366 pub fn complete_auth_id(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 367     proxmox_async::runtime::main(async { complete_auth_id_do(param).await })
 368 }
 369
 370 pub async fn complete_auth_id_do(param: &HashMap<String, String>) -> Vec<String> {
 371
 372     let mut result = vec![];
 373
 374     let repo = match extract_repository_from_map(param) {
 375         Some(v) => v,
 376         _ => return result,
 377     };
 378
 379     let data = try_get(&repo, "api2/json/access/users?include_tokens=true").await;
 380
 381     if let Ok(parsed) = serde_json::from_value::<Vec<UserWithTokens>>(data) {
 382         for user in parsed {
 383             result.push(user.userid.to_string());
 384             for token in user.tokens {
 385                 result.push(token.tokenid.to_string());
 386             }
 387         }
 388     };
 389
 390     result
 391 }
 392
 393 pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
 394     let mut result = vec![];
 395
 396     let base = match BaseDirectories::with_prefix("proxmox-backup") {
 397         Ok(v) => v,
 398         _ => return result,
 399     };
 400
 401     // usually $HOME/.cache/proxmox-backup/repo-list
 402     let path = match base.place_cache_file("repo-list") {
 403         Ok(v) => v,
 404         _ => return result,
 405     };
 406
 407     let data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
 408
 409     if let Some(map) = data.as_object() {
 410         for (repo, _count) in map {
 411             result.push(repo.to_owned());
 412         }
 413     }
 414
 415     result
 416 }
 417
 418 pub fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
 419     let mut result = vec![];
 420
 421     let data: Vec<&str> = arg.splitn(2, ':').collect();
 422
 423     if data.len() != 2 {
 424         result.push(String::from("root.pxar:/"));
 425         result.push(String::from("etc.pxar:/etc"));
 426         return result;
 427     }
 428
 429     let files = complete_file_name(data[1], param);
 430
 431     for file in files {
 432         result.push(format!("{}:{}", data[0], file));
 433     }
 434
 435     result
 436 }
 437
 438 pub fn base_directories() -> Result<xdg::BaseDirectories, Error> {
 439     xdg::BaseDirectories::with_prefix("proxmox-backup").map_err(Error::from)
 440 }
 441
 442 /// Convenience helper for better error messages:
 443 pub fn find_xdg_file(
 444     file_name: impl AsRef<std::path::Path>,
 445     description: &'static str,
 446 ) -> Result<Option<std::path::PathBuf>, Error> {
 447     let file_name = file_name.as_ref();
 448     base_directories()
 449         .map(|base| base.find_config_file(file_name))
 450         .with_context(|| format!("error searching for {}", description))
 451 }
 452
 453 pub fn place_xdg_file(
 454     file_name: impl AsRef<std::path::Path>,
 455     description: &'static str,
 456 ) -> Result<std::path::PathBuf, Error> {
 457     let file_name = file_name.as_ref();
 458     base_directories()
 459         .and_then(|base| base.place_config_file(file_name).map_err(Error::from))
 460         .with_context(|| format!("failed to place {} in xdg home", description))
 461 }