From: Fabian Grünbichler Date: Wed, 21 Sep 2022 08:12:41 +0000 (+0200) Subject: medium: add diff command X-Git-Url: https://git.proxmox.com/?a=commitdiff_plain;h=d056f823f00fe3778e22b8ac5960996454bbd7ef;p=proxmox-offline-mirror.git medium: add diff command Signed-off-by: Fabian Grünbichler --- diff --git a/src/bin/proxmox_offline_mirror_cmds/medium.rs b/src/bin/proxmox_offline_mirror_cmds/medium.rs index b76e4e6..574f748 100644 --- a/src/bin/proxmox_offline_mirror_cmds/medium.rs +++ b/src/bin/proxmox_offline_mirror_cmds/medium.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::Error; use serde_json::Value; @@ -220,6 +220,108 @@ async fn sync( Ok(Value::Null) } +#[api( + input: { + properties: { + config: { + type: String, + optional: true, + description: "Path to mirroring config file.", + }, + id: { + schema: MEDIA_ID_SCHEMA, + }, + verbose: { + type: bool, + optional: true, + default: false, + description: "Verbose output (print paths in addition to summary)." + }, + } + }, + )] +/// Diff a medium +async fn diff( + config: Option, + id: String, + verbose: bool, + _param: Value, +) -> Result { + let config = config.unwrap_or_else(get_config_path); + + let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?; + let config: MediaConfig = section_config.lookup("medium", &id)?; + let mut mirrors = Vec::with_capacity(config.mirrors.len()); + for mirror in &config.mirrors { + let mirror: MirrorConfig = section_config.lookup("mirror", mirror)?; + mirrors.push(mirror); + } + + let mut diffs = medium::diff(&config, mirrors)?; + let mut mirrors: Vec = diffs.keys().cloned().collect(); + mirrors.sort_unstable(); + + let sort_paths = + |(path, _): &(PathBuf, u64), (other_path, _): &(PathBuf, u64)| path.cmp(other_path); + + let mut first = true; + for mirror in mirrors { + if first { + first = false; + } else { + println!(); + } + + println!("Mirror '{mirror}'"); + if let Some(Some(mut diff)) = diffs.remove(&mirror) { + let mut total_size = 0; + println!("\t{} file(s) only on medium:", diff.added.paths.len()); + if verbose { + diff.added.paths.sort_unstable_by(sort_paths); + diff.changed.paths.sort_unstable_by(sort_paths); + diff.removed.paths.sort_unstable_by(sort_paths); + } + for (path, size) in diff.added.paths { + if verbose { + println!("\t\t{path:?}: +{size}b"); + } + total_size += size; + } + println!("\tTotal size: +{total_size}b"); + + total_size = 0; + println!( + "\n\t{} file(s) missing on medium:", + diff.removed.paths.len() + ); + for (path, size) in diff.removed.paths { + if verbose { + println!("\t\t{path:?}: -{size}b"); + } + total_size += size; + } + println!("\tTotal size: -{total_size}b"); + + total_size = 0; + println!( + "\n\t{} file(s) diff between source and medium:", + diff.changed.paths.len() + ); + for (path, size) in diff.changed.paths { + if verbose { + println!("\t\t{path:?}: +-{size}b"); + } + } + println!("\tSum of size differences: +-{total_size}b"); + } else { + // TODO + println!("\tNot yet synced or no longer available on source side."); + } + } + + Ok(Value::Null) +} + pub fn medium_commands() -> CommandLineInterface { let cmd_def = CliCommandMap::new() .insert( @@ -230,7 +332,8 @@ pub fn medium_commands() -> CommandLineInterface { "status", CliCommand::new(&API_METHOD_STATUS).arg_param(&["id"]), ) - .insert("sync", CliCommand::new(&API_METHOD_SYNC).arg_param(&["id"])); + .insert("sync", CliCommand::new(&API_METHOD_SYNC).arg_param(&["id"])) + .insert("diff", CliCommand::new(&API_METHOD_DIFF).arg_param(&["id"])); cmd_def.into() } diff --git a/src/medium.rs b/src/medium.rs index aa5bef8..4b1f006 100644 --- a/src/medium.rs +++ b/src/medium.rs @@ -1,5 +1,7 @@ use std::{ collections::{HashMap, HashSet}, + fs::Metadata, + os::linux::fs::MetadataExt, path::{Path, PathBuf}, }; @@ -16,7 +18,7 @@ use crate::{ generate_repo_file_line, mirror::pool, pool::Pool, - types::{Snapshot, SNAPSHOT_REGEX}, + types::{Diff, Snapshot, SNAPSHOT_REGEX}, }; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -435,3 +437,102 @@ pub fn sync( Ok(()) } + +/// Sync medium's content according to config. +pub fn diff( + medium: &crate::config::MediaConfig, + mirrors: Vec, +) -> Result>, Error> { + let medium_base = Path::new(&medium.mountpoint); + if !medium_base.exists() { + bail!("Medium mountpoint doesn't exist."); + } + + let _lock = lock(medium_base)?; + + let state = + load_state(medium_base)?.ok_or_else(|| format_err!("Medium not yet initializes."))?; + + let mirror_state = get_mirror_state(medium, &state); + + let pools: HashMap = + state + .mirrors + .iter() + .fold(HashMap::new(), |mut map, (id, info)| { + map.insert(id.clone(), info.pool.clone()); + map + }); + + let mut diffs = HashMap::new(); + + let convert_file_list_to_diff = |files: Vec<(PathBuf, Metadata)>, added: bool| -> Diff { + files + .into_iter() + .fold(Diff::default(), |mut diff, (file, meta)| { + if !meta.is_file() { + return diff; + } + + let size = meta.st_size(); + if added { + diff.added.paths.push((file, size)); + } else { + diff.removed.paths.push((file, size)); + } + diff + }) + }; + + let get_target_pool = + |mirror_id: &str, mirror: Option<&MirrorConfig>| -> Result, Error> { + let mut mirror_base = medium_base.to_path_buf(); + mirror_base.push(Path::new(mirror_id)); + + let mut mirror_pool = medium_base.to_path_buf(); + let pool_dir = match pools.get(mirror_id) { + Some(pool_dir) => pool_dir.to_owned(), + None => { + if let Some(mirror) = mirror { + mirror_pool_dir(mirror) + } else { + return Ok(None); + } + } + }; + mirror_pool.push(pool_dir); + + Ok(Some(Pool::open(&mirror_base, &mirror_pool)?)) + }; + + for mirror in mirrors.into_iter() { + let source_pool: Pool = pool(&mirror)?; + + if !mirror_state.synced.contains(&mirror.id) { + let files = source_pool.lock()?.list_files()?; + diffs.insert(mirror.id, Some(convert_file_list_to_diff(files, false))); + continue; + } + + let target_pool = get_target_pool(mirror.id.as_str(), Some(&mirror))? + .ok_or_else(|| format_err!("Failed to open target pool."))?; + diffs.insert( + mirror.id, + Some(source_pool.lock()?.diff_pools(&target_pool)?), + ); + } + + for dropped in mirror_state.target_only { + match get_target_pool(&dropped, None)? { + Some(pool) => { + let files = pool.lock()?.list_files()?; + diffs.insert(dropped, Some(convert_file_list_to_diff(files, false))); + } + None => { + diffs.insert(dropped, None); + } + } + } + + Ok(diffs) +}