}
},
)]
-/// Show full media config entry.
+/// Show full medium config entry.
async fn show_medium(config: Option<String>, id: String, param: Value) -> Result<Value, Error> {
let config = config.unwrap_or_else(|| DEFAULT_CONFIG_PATH.to_string());
},
mountpoint: {
type: String,
- description: "Path where sync target is mounted."
},
verify: {
type: bool,
- description: "Whether to verify existing files stored in pool (IO-intensive).",
},
sync: {
type: bool,
- description: "Whether to write pool updates with fsync flag.",
},
mirrors: {
type: Array,
- description: "List of mirror IDs this sync target should contain.",
items: {
schema: MIRROR_ID_SCHEMA,
},
)]
#[derive(Debug, Serialize, Deserialize, Updater)]
#[serde(rename_all = "kebab-case")]
-/// Configuration file for mirrored repositories.
+/// Configuration entry for an external medium.
pub struct MediaConfig {
#[updater(skip)]
+ /// Identifier for this entry.
pub id: String,
+ /// Mountpoint where medium is available on mirroring system.
pub mountpoint: String,
+ /// List of [MirrorConfig] IDs which should be synced to medium.
pub mirrors: Vec<String>,
+ /// Whether to verify existing files or assume they are valid (IO-intensive).
pub verify: bool,
+ /// Whether to write new files using FSYNC.
pub sync: bool,
}
config
}
+/// Lock guard for guarding modifications of config file.
+///
+/// Obtained via [lock_config], should only be dropped once config file should no longer be locked.
pub struct ConfigLockGuard(std::fs::File);
-/// Get exclusive lock
+/// Get exclusive lock for config file (in order to make or protect against modifications).
pub fn lock_config(path: &str) -> Result<ConfigLockGuard, Error> {
let path = Path::new(path);
Ok(ConfigLockGuard(file))
}
+/// Read config
pub fn config(path: &str) -> Result<(SectionConfigData, [u8; 32]), Error> {
let content =
proxmox_sys::fs::file_read_optional_string(path)?.unwrap_or_else(|| "".to_string());
Ok((data, digest))
}
+/// Write config (and verify data matches schema!)
pub fn save_config(path: &str, data: &SectionConfigData) -> Result<(), Error> {
let raw = CONFIG.write(path, data)?;
replace_file(path, raw.as_bytes(), CreateOptions::default(), true)
use anyhow::{bail, format_err, Error};
use proxmox_schema::parse_boolean;
+/// Prints `query`, reads string from terminal, defaulting to `default`.
+///
+/// Will retry if no default is given and user doesn't input any data.
pub fn read_string_from_tty(query: &str, default: Option<&str>) -> Result<String, Error> {
use std::io::{BufRead, BufReader};
}
}
+/// Prints `query`, reads boolean-string from terminal, defaulting to `default`.
+///
+/// Will retry if the user doesn't input a valid boolean string.
pub fn read_bool_from_tty(query: &str, default: Option<bool>) -> Result<bool, Error> {
let default = default.map(|v| if v { "yes" } else { "no" });
}
}
+/// Prints query and a list of options, allowing the user to select one.
+///
+/// Will retry if user input cannot be parsed as choice or is invalid.
pub fn read_selection_from_tty<'a, V>(
query: &str,
choices: &'a [(V, &str)],
}
}
}
+
+/// Verifies GPG-signed `msg` was signed by `key`, returning the verified data without signature.
pub(crate) fn verify_signature<'msg>(
msg: &'msg [u8],
key: &[u8],
+//! Proxmox mirroring tool for APT repositories.
+//!
+//! This library provides the underlying functionality of the `proxmox-apt-mirror` and
+//! `proxmox-apt-repo` binaries.
+//!
+//! It implements the following features:
+//! - local storage in a hardlink-based pool
+//! - intelligent fetching only those files of a repository that have changed since the last mirroring operation
+//! - syncing to external media
+
use std::{
fmt::Display,
ops::{Add, AddAssign},
use proxmox_apt::repositories::{APTRepository, APTRepositoryFile, APTRepositoryFileType};
use types::Snapshot;
+/// Main configuration file containing definitions of mirrors and external media.
pub mod config;
+/// Helpers
pub mod helpers;
+/// Operations concerning a medium.
pub mod medium;
+/// Operations concerning a mirror.
pub mod mirror;
-pub mod pool;
+/// Hardlink pool.
+pub(crate) mod pool;
+/// Various common types
pub mod types;
+/// Combination of data and whether it needed to be fetched or was re-used.
struct FetchResult {
+ /// Fetched/read data
data: Vec<u8>,
+ /// Number of bytes fetched (0 if re-using pool data)
fetched: usize,
}
}
#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
+/// To keep track of progress and how much data was newly fetched vs. re-used and just linked
struct Progress {
new: usize,
new_bytes: usize,
}
}
+/// Try to parse a line in sources.list format into an `APTRepository`.
pub(crate) fn convert_repo_line(line: String) -> Result<APTRepository, Error> {
let mut repository = APTRepositoryFile::with_content(line, APTRepositoryFileType::List);
repository.parse()?;
Ok(repository.repositories[0].clone())
}
+/// Generate a file-based repository line in sources.list format
pub fn generate_repo_file_line(
medium_base: &Path,
mirror_id: &str,
};
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
+/// Information about a mirror on the medium.
+///
+/// Used to generate repository lines for accessing the synced mirror.
pub struct MirrorInfo {
+ /// Original repository line
pub repository: String,
+ /// Mirrored architectures
pub architectures: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
+/// State of mirrors on the medium
pub struct MediumState {
+ /// Map of mirror ID to `MirrorInfo`.
pub mirrors: HashMap<String, MirrorInfo>,
+ /// Timestamp of last sync operation.
pub last_sync: i64,
}
+/// Information about the mirrors on a medium.
+///
+/// Derived from `MediaConfig` (supposed state) and `MediumState` (actual state)
pub struct MediumMirrorState {
+ /// Mirrors which are configured and synced
pub synced: HashSet<String>,
+ /// Mirrors which are configured
+ pub config: HashSet<String>,
+ /// Mirrors which are configured but not synced yet
pub source_only: HashSet<String>,
+ /// Mirrors which are not configured but exist on medium
pub target_only: HashSet<String>,
}
+// helper to derive `MediumMirrorState`
fn get_mirror_state(config: &MediaConfig, state: &MediumState) -> MediumMirrorState {
let synced_mirrors: HashSet<String> = state
.mirrors
MediumMirrorState {
synced: synced_mirrors,
+ config: config_mirrors,
source_only: new_mirrors,
target_only: dropped_mirrors,
}
}
+// Helper to lock medium
fn lock(base: &Path) -> Result<ConfigLockGuard, Error> {
let mut lockfile = base.to_path_buf();
lockfile.push("mirror-state");
config::lock_config(lockfile)
}
+// Helper to get statefile path
fn statefile(base: &Path) -> PathBuf {
let mut statefile = base.to_path_buf();
statefile.push(".mirror-state");
statefile
}
+// Helper to load statefile
fn load_state(base: &Path) -> Result<Option<MediumState>, Error> {
let statefile = statefile(base);
}
}
+// Helper to write statefile
fn write_state(_lock: &ConfigLockGuard, base: &Path, state: &MediumState) -> Result<(), Error> {
replace_file(
&statefile(base),
Ok(())
}
+/// List snapshots of a given mirror on a given medium.
pub fn list_snapshots(medium_base: &Path, mirror: &str) -> Result<Vec<Snapshot>, Error> {
if !medium_base.exists() {
bail!("Medium mountpoint doesn't exist.");
Ok(list)
}
+/// Generate a repository snippet for a selection of mirrors on a medium.
pub fn generate_repo_snippet(
medium_base: &Path,
repositories: &HashMap<String, (&MirrorInfo, Snapshot)>,
Ok(res)
}
+/// Run garbage collection on all mirrors on a medium.
pub fn gc(medium: &crate::config::MediaConfig) -> Result<(), Error> {
let medium_base = Path::new(&medium.mountpoint);
if !medium_base.exists() {
Ok(())
}
+/// Get `MediumState` and `MediumMirrorState` for a given medium.
pub fn status(
medium: &crate::config::MediaConfig,
) -> Result<(MediumState, MediumMirrorState), Error> {
Ok((state, mirror_state))
}
+/// Sync medium's content according to config.
pub fn sync(medium: &crate::config::MediaConfig, mirrors: Vec<MirrorConfig>) -> Result<(), Error> {
println!(
"Syncing {} mirrors {:?} to medium '{}' ({:?})",
&medium.mountpoint
);
+ if mirrors.len() != medium.mirrors.len() {
+ bail!("Number of mirrors in config and sync request don't match.");
+ }
+
let medium_base = Path::new(&medium.mountpoint);
if !medium_base.exists() {
bail!("Medium mountpoint doesn't exist.");
let mirror_state = get_mirror_state(medium, &state);
println!("Previously synced mirrors: {:?}", &mirror_state.synced);
+ let requested: HashSet<String> = mirrors.iter().map(|mirror| mirror.id.clone()).collect();
+ if requested != mirror_state.config {
+ bail!(
+ "Config and sync request don't use the same mirror list: {:?} / {:?}",
+ mirror_state.config,
+ requested
+ );
+ }
+
if !mirror_state.source_only.is_empty() {
println!(
"Adding {} new mirror(s) to target medium: {:?}",
Pool::open(Path::new(&config.dir), Path::new(&pool_dir))
}
+/// `MirrorConfig`, but some fields converted/parsed into usable types.
struct ParsedMirrorConfig {
pub repository: APTRepository,
pub architectures: Vec<String>,
}
}
+// Helper to get absolute URL for dist-specific relative `path`.
fn get_dist_url(repo: &APTRepository, path: &str) -> String {
let dist_root = format!("{}/dists/{}", repo.uris[0], repo.suites[0]);
format!("{}/{}", dist_root, path)
}
+// Helper to get dist-specific path given a `prefix` (snapshot dir) and relative `path`.
fn get_dist_path(repo: &APTRepository, prefix: &Path, path: &str) -> PathBuf {
let mut base = PathBuf::from(prefix);
base.push("dists");
base
}
+// Helper to get generic URL given a `repo` and `path`.
fn get_repo_url(repo: &APTRepository, path: &str) -> String {
format!("{}/{}", repo.uris[0], path)
}
+/// Helper to fetch file from URI and optionally verify the responses checksum.
+///
+/// Only fetches and returns data, doesn't store anything anywhere.
fn fetch_repo_file(
uri: &str,
max_size: Option<u64>,
})
}
+/// Helper to fetch InRelease (`detached` == false) or Release/Release.gpg (`detached` == true) files from repository.
+///
+/// Verifies the contained/detached signature, stores all fetched files under `prefix`, and returns the verified raw release file data.
fn fetch_release(
config: &ParsedMirrorConfig,
prefix: &Path,
})
}
+/// Helper to fetch an index file referenced by a `ReleaseFile`.
+///
+/// Since these usually come in compressed and uncompressed form, with the latter often not actually existing in the source repository as file, this fetches and if necessary decompresses to obtain a copy of the uncompressed data.
+/// Will skip fetching if both references are already available with the expected checksum in the pool, in which case they will just be re-linked under the new path.
+///
+/// Returns the uncompressed data.
fn fetch_index_file(
config: &ParsedMirrorConfig,
prefix: &Path,
})
}
+/// Helper to fetch arbitrary files like binary packages.
+///
+/// Will skip fetching if matching file already exists locally, in which case it will just be re-linked under the new path.
+///
+/// If need_data is false and the mirror config is set to skip verification, reading the file's content will be skipped as well if fetching was skipped.
fn fetch_plain_file(
config: &ParsedMirrorConfig,
url: &str,
Ok(res)
}
+/// Initialize a new mirror (by creating the corresponding pool).
pub fn init(config: &MirrorConfig) -> Result<(), Error> {
let pool_dir = format!("{}/.pool", config.dir);
Pool::create(Path::new(&config.dir), Path::new(&pool_dir))?;
Ok(())
}
+/// Destroy a mirror (by destroying the corresponding pool).
pub fn destroy(config: &MirrorConfig) -> Result<(), Error> {
let pool: Pool = pool(config)?;
pool.lock()?.destroy()?;
Ok(())
}
+/// List snapshots
pub fn list_snapshots(config: &MirrorConfig) -> Result<Vec<Snapshot>, Error> {
let _pool: Pool = pool(config)?;
Ok(list)
}
+/// Create a new snapshot of the remote repository, fetching and storing files as needed.
+///
+/// Operates in three phases:
+/// - Fetch and verify release files
+/// - Fetch referenced indices according to config
+/// - Fetch binary packages referenced by package indices
+///
+/// Files will be linked in a temporary directory and only renamed to the final, valid snapshot directory at the end. In case of error, leftover `XXX.tmp` directories at the top level of `base_dir` can be safely removed once the next snapshot was successfully created, as they only contain hardlinks.
pub fn create_snapshot(config: MirrorConfig, snapshot: &Snapshot) -> Result<(), Error> {
let config: ParsedMirrorConfig = config.try_into()?;
Ok(())
}
+/// Remove a snapshot by removing the corresponding snapshot directory. To actually free up space, a garbage collection needs to be run afterwards.
pub fn remove_snapshot(config: &MirrorConfig, snapshot: &Snapshot) -> Result<(), Error> {
let pool: Pool = pool(config)?;
let path = pool.get_path(Path::new(&snapshot.to_string()))?;
pool.lock()?.remove_dir(&path)
}
+/// Run a garbage collection on the underlying pool.
pub fn gc(config: &MirrorConfig) -> Result<(usize, u64), Error> {
let pool: Pool = pool(config)?;
use walkdir::WalkDir;
#[derive(Debug)]
+/// Pool consisting of two (possibly overlapping) directory trees:
+/// - pool_dir contains checksum files added by `add_file`
+/// - base_dir contains directories and hardlinks to checksum files created by `link_file`
+///
+/// Files are considered orphaned and eligible for GC if they either only exist in pool_dir or only exist in base_dir
pub(crate) struct Pool {
pool_dir: PathBuf,
base_dir: PathBuf,
}
+/// Lock guard used to guard against concurrent modification
pub(crate) struct PoolLockGuard<'lock> {
pool: &'lock Pool,
_lock: Option<File>,
}
impl Pool {
+ /// Create a new pool by creating `pool_dir` and `base_dir`. They must not exist before calling this function.
pub(crate) fn create(base: &Path, pool: &Path) -> Result<Self, Error> {
if base.exists() {
bail!("Pool base dir already exists.");
base_dir: base.to_path_buf(),
})
}
+
+ /// Open an existing pool. `pool_dir` and `base_dir` must exist.
pub(crate) fn open(base: &Path, pool: &Path) -> Result<Self, Error> {
if !base.exists() {
bail!("Pool base dir doesn't exist.")
})
}
+ /// Lock a pool to add/remove files or links, or protect against concurrent modifications.
pub(crate) fn lock(&self) -> Result<PoolLockGuard, Error> {
let timeout = std::time::Duration::new(10, 0);
let lock = Some(proxmox_sys::fs::open_file_locked(
_lock: lock,
})
}
+
+ /// Returns whether the pool contain a file for the given checksum.
pub(crate) fn contains(&self, checksums: &CheckSums) -> bool {
match self.get_checksum_paths(checksums) {
Ok(paths) => paths.iter().any(|path| path.exists()),
}
}
+ /// Returns the file contents for a given checksum, optionally `verify`ing whether the on-disk data matches the checksum.
pub(crate) fn get_contents(
&self,
checksums: &CheckSums,
Ok(data)
}
+ // Helper to return all possible checksum file paths for a given checksum. Checksums considered insecure will be ignored.
fn get_checksum_paths(&self, checksums: &CheckSums) -> Result<Vec<PathBuf>, Error> {
if !checksums.is_secure() {
bail!("pool cannot operate on files lacking secure checksum!");
}
impl PoolLockGuard<'_> {
+ // Helper to scan the pool for all checksum files and the total link count. The resulting HashMap can be used to check whether files in `base_dir` are properly registered in the pool or orphaned.
fn get_inode_csum_map(&self) -> Result<(HashMap<u64, CheckSums>, u64), Error> {
let mut inode_map: HashMap<u64, CheckSums> = HashMap::new();
let mut link_count = 0;
Ok((inode_map, link_count))
}
+ /// Syncs the pool into a target pool, optionally verifying file contents along the way.
+ ///
+ /// This proceeds in four phases:
+ /// - iterate over source pool checksum files, add missing ones to target pool
+ /// - iterate over source pool links, add missing ones to target pool
+ /// - iterate over target pool links, remove those which are not present in source pool
+ /// - if links were removed in phase 3, run GC on target pool
pub(crate) fn sync_pool(&self, target: &Pool, verify: bool) -> Result<(), Error> {
let target = target.lock()?;
Ok(())
}
+ /// Adds a new checksum file.
+ ///
+ /// If `checksums` contains multiple trusted checksums, they will be linked to the first checksum file.
pub(crate) fn add_file(
&self,
data: &[u8],
Ok(())
}
+ /// Links previously added file into `path` (relative to `base_dir`). Missing parent directories will be created automatically.
pub(crate) fn link_file(&self, checksums: &CheckSums, path: &Path) -> Result<bool, Error> {
let path = self.pool.get_path(path)?;
if !self.pool.path_in_base(&path) {
link_file_do(source, &path)
}
+ /// Unlink a previously linked file at `path` (absolute, must be below `base_dir`). Optionally remove any parent directories that became empty.
pub(crate) fn unlink_file(
&self,
mut path: &Path,
Ok(())
}
+ /// Remove a directory tree at `path` (absolute, must be below `base_dir`)
pub(crate) fn remove_dir(&self, path: &Path) -> Result<(), Error> {
if !self.pool.path_in_base(path) {
bail!("Cannot unlink file outside of pool.");
.map_err(|err| format_err!("Failed to remove {path:?} - {err}"))
}
+ /// Run a garbage collection, removing
+ /// - any checksum files that have no links outside of `pool_dir`
+ /// - any files in `base_dir` that have no corresponding checksum files
pub(crate) fn gc(&self) -> Result<(usize, u64), Error> {
let (inode_map, _link_count) = self.get_inode_csum_map()?;
Ok((count, size))
}
+ /// Destroy pool by removing `base_dir` and `pool_dir`.
pub(crate) fn destroy(self) -> Result<(), Error> {
// TODO - this removes the lock file..
std::fs::remove_dir_all(self.pool_dir.clone())?;
Ok(())
}
+ /// Rename a link or directory from `from` to `to` (both relative to `base_dir`).
pub(crate) fn rename(&self, from: &Path, to: &Path) -> Result<(), Error> {
let mut abs_from = self.base_dir.clone();
abs_from.push(from);
}
pub const PROXMOX_SAFE_ID_FORMAT: ApiStringFormat =
ApiStringFormat::Pattern(&PROXMOX_SAFE_ID_REGEX);
+
+/// Schema for config IDs
pub const MIRROR_ID_SCHEMA: Schema = StringSchema::new("Mirror name.")
.format(&PROXMOX_SAFE_ID_FORMAT)
.min_length(3)
projects / proxmox-offline-mirror.git / commitdiff