8 use anyhow
::{bail, format_err, Error}
;
9 use flate2
::bufread
::GzDecoder
;
11 use proxmox_http
::{client::sync::Client, HttpClient, HttpOptions}
;
12 use proxmox_sys
::fs
::file_get_contents
;
15 config
::{MirrorConfig, SubscriptionKey}
,
18 types
::{Snapshot, SNAPSHOT_REGEX}
,
19 FetchResult
, Progress
,
23 CheckSums
, CompressionType
, FileReference
, FileReferenceType
, PackagesFile
, ReleaseFile
,
25 repositories
::{APTRepository, APTRepositoryPackageType}
,
30 fn mirror_dir(config
: &MirrorConfig
) -> String
{
31 format
!("{}/{}", config
.base_dir
, config
.id
)
34 pub(crate) fn pool(config
: &MirrorConfig
) -> Result
<Pool
, Error
> {
35 let pool_dir
= format
!("{}/.pool", config
.base_dir
);
36 Pool
::open(Path
::new(&mirror_dir(config
)), Path
::new(&pool_dir
))
39 /// `MirrorConfig`, but some fields converted/parsed into usable types.
40 struct ParsedMirrorConfig
{
41 pub repository
: APTRepository
,
42 pub architectures
: Vec
<String
>,
47 pub auth
: Option
<String
>,
51 impl TryInto
<ParsedMirrorConfig
> for MirrorConfig
{
52 type Error
= anyhow
::Error
;
54 fn try_into(self) -> Result
<ParsedMirrorConfig
, Self::Error
> {
55 let pool
= pool(&self)?
;
57 let repository
= convert_repo_line(self.repository
.clone())?
;
59 let key
= file_get_contents(Path
::new(&self.key_path
))?
;
61 let options
= HttpOptions
{
62 user_agent
: Some("proxmox-offline-mirror 0.1".to_string()),
64 }; // TODO actually read version ;)
66 let client
= Client
::new(options
);
68 Ok(ParsedMirrorConfig
{
70 architectures
: self.architectures
,
81 // Helper to get absolute URL for dist-specific relative `path`.
82 fn get_dist_url(repo
: &APTRepository
, path
: &str) -> String
{
83 let dist_root
= format
!("{}/dists/{}", repo
.uris
[0], repo
.suites
[0]);
85 format
!("{}/{}", dist_root
, path
)
88 // Helper to get dist-specific path given a `prefix` (snapshot dir) and relative `path`.
89 fn get_dist_path(repo
: &APTRepository
, prefix
: &Path
, path
: &str) -> PathBuf
{
90 let mut base
= PathBuf
::from(prefix
);
92 base
.push(&repo
.suites
[0]);
97 // Helper to get generic URL given a `repo` and `path`.
98 fn get_repo_url(repo
: &APTRepository
, path
: &str) -> String
{
99 format
!("{}/{}", repo
.uris
[0], path
)
102 /// Helper to fetch file from URI and optionally verify the responses checksum.
104 /// Only fetches and returns data, doesn't store anything anywhere.
109 checksums
: Option
<&CheckSums
>,
111 ) -> Result
<FetchResult
, Error
> {
112 println
!("-> GET '{}'..", uri
);
114 let headers
= if let Some(auth
) = auth
{
115 let mut map
= HashMap
::new();
116 map
.insert("Authorization".to_string(), auth
.to_string());
122 let response
= client
.get(uri
, headers
.as_ref())?
;
124 let reader
: Box
<dyn Read
> = response
.into_body();
125 let mut reader
= reader
.take(max_size
as u64);
126 let mut data
= Vec
::new();
127 reader
.read_to_end(&mut data
)?
;
129 if let Some(checksums
) = checksums
{
130 checksums
.verify(&data
)?
;
139 /// Helper to fetch InRelease (`detached` == false) or Release/Release.gpg (`detached` == true) files from repository.
141 /// Verifies the contained/detached signature, stores all fetched files under `prefix`, and returns the verified raw release file data.
143 config
: &ParsedMirrorConfig
,
147 ) -> Result
<FetchResult
, Error
> {
148 let (name
, fetched
, sig
) = if detached
{
149 println
!("Fetching Release/Release.gpg files");
150 let sig
= fetch_repo_file(
152 &get_dist_url(&config
.repository
, "Release.gpg"),
155 config
.auth
.as_deref(),
157 let mut fetched
= fetch_repo_file(
159 &get_dist_url(&config
.repository
, "Release"),
162 config
.auth
.as_deref(),
164 fetched
.fetched
+= sig
.fetched
;
165 ("Release(.gpg)", fetched
, Some(sig
.data()))
167 println
!("Fetching InRelease file");
168 let fetched
= fetch_repo_file(
170 &get_dist_url(&config
.repository
, "InRelease"),
173 config
.auth
.as_deref(),
175 ("InRelease", fetched
, None
)
178 println
!("Verifying '{name}' signature using provided repository key..");
179 let content
= fetched
.data_ref();
180 let verified
= helpers
::verify_signature(content
, &config
.key
, sig
.as_deref())?
;
183 let sha512
= Some(openssl
::sha
::sha512(content
));
184 let csums
= CheckSums
{
190 return Ok(FetchResult
{
192 fetched
: fetched
.fetched
,
196 let locked
= &config
.pool
.lock()?
;
198 if !locked
.contains(&csums
) {
199 locked
.add_file(content
, &csums
, config
.sync
)?
;
205 Path
::new(&get_dist_path(&config
.repository
, prefix
, "Release")),
207 let sig
= sig
.unwrap();
208 let sha512
= Some(openssl
::sha
::sha512(&sig
));
209 let csums
= CheckSums
{
213 if !locked
.contains(&csums
) {
214 locked
.add_file(&sig
, &csums
, config
.sync
)?
;
218 Path
::new(&get_dist_path(&config
.repository
, prefix
, "Release.gpg")),
223 Path
::new(&get_dist_path(&config
.repository
, prefix
, "InRelease")),
229 fetched
: fetched
.fetched
,
233 /// Helper to fetch an index file referenced by a `ReleaseFile`.
235 /// Since these usually come in compressed and uncompressed form, with the latter often not actually existing in the source repository as file, this fetches and if necessary decompresses to obtain a copy of the uncompressed data.
236 /// Will skip fetching if both references are already available with the expected checksum in the pool, in which case they will just be re-linked under the new path.
238 /// Returns the uncompressed data.
240 config
: &ParsedMirrorConfig
,
242 reference
: &FileReference
,
243 uncompressed
: Option
<&FileReference
>,
246 ) -> Result
<FetchResult
, Error
> {
247 let url
= get_dist_url(&config
.repository
, &reference
.path
);
248 let path
= get_dist_path(&config
.repository
, prefix
, &reference
.path
);
250 if let Some(uncompressed
) = uncompressed
{
251 let uncompressed_path
= get_dist_path(&config
.repository
, prefix
, &uncompressed
.path
);
253 if config
.pool
.contains(&reference
.checksums
)
254 && config
.pool
.contains(&uncompressed
.checksums
)
258 .get_contents(&uncompressed
.checksums
, config
.verify
)?
;
261 return Ok(FetchResult { data, fetched: 0 }
);
263 // Ensure they're linked at current path
264 config
.pool
.lock()?
.link_file(&reference
.checksums
, &path
)?
;
268 .link_file(&uncompressed
.checksums
, &uncompressed_path
)?
;
269 return Ok(FetchResult { data, fetched: 0 }
);
273 let urls
= if by_hash
{
274 let mut urls
= Vec
::new();
275 if let Some((base_url
, _file_name
)) = url
.rsplit_once('
/'
) {
276 if let Some(sha512
) = reference
.checksums
.sha512
{
277 urls
.push(format
!("{base_url}/by-hash/SHA512/{}", hex
::encode(sha512
)));
279 if let Some(sha256
) = reference
.checksums
.sha256
{
280 urls
.push(format
!("{base_url}/by-hash/SHA256/{}", hex
::encode(sha256
)));
291 .fold(None
, |res
, url
| match res
{
292 Some(Ok(res
)) => Some(Ok(res
)),
293 _
=> Some(fetch_plain_file(
298 &reference
.checksums
,
303 .ok_or_else(|| format_err
!("Failed to retrieve {}", reference
.path
))??
;
305 let mut buf
= Vec
::new();
306 let raw
= res
.data_ref();
308 let decompressed
= match reference
.file_type
.compression() {
310 Some(CompressionType
::Gzip
) => {
311 let mut gz
= GzDecoder
::new(raw
);
312 gz
.read_to_end(&mut buf
)?
;
315 Some(CompressionType
::Bzip2
) => {
316 let mut bz
= bzip2
::read
::BzDecoder
::new(raw
);
317 bz
.read_to_end(&mut buf
)?
;
320 Some(CompressionType
::Lzma
) | Some(CompressionType
::Xz
) => {
321 let mut xz
= xz2
::read
::XzDecoder
::new_multi_decoder(raw
);
322 xz
.read_to_end(&mut buf
)?
;
326 let res
= FetchResult
{
327 data
: decompressed
.to_owned(),
328 fetched
: res
.fetched
,
335 let locked
= &config
.pool
.lock()?
;
336 if let Some(uncompressed
) = uncompressed
{
337 if !locked
.contains(&uncompressed
.checksums
) {
338 locked
.add_file(decompressed
, &uncompressed
.checksums
, config
.sync
)?
;
341 // Ensure it's linked at current path
342 let uncompressed_path
= get_dist_path(&config
.repository
, prefix
, &uncompressed
.path
);
343 locked
.link_file(&uncompressed
.checksums
, &uncompressed_path
)?
;
349 /// Helper to fetch arbitrary files like binary packages.
351 /// Will skip fetching if matching file already exists locally, in which case it will just be re-linked under the new path.
353 /// If need_data is false and the mirror config is set to skip verification, reading the file's content will be skipped as well if fetching was skipped.
355 config
: &ParsedMirrorConfig
,
359 checksums
: &CheckSums
,
362 ) -> Result
<FetchResult
, Error
> {
363 let locked
= &config
.pool
.lock()?
;
364 let res
= if locked
.contains(checksums
) {
365 if need_data
|| config
.verify
{
367 .get_contents(checksums
, config
.verify
)
368 .map(|data
| FetchResult { data, fetched: 0 }
)?
370 // performance optimization for .deb files if verify is false
371 // we never need the file contents and they make up the bulk of a repo
377 } else if dry_run
&& !need_data
{
383 let fetched
= fetch_repo_file(
388 config
.auth
.as_deref(),
390 locked
.add_file(fetched
.data_ref(), checksums
, config
.verify
)?
;
395 // Ensure it's linked at current path
396 locked
.link_file(checksums
, file
)?
;
402 /// Initialize a new mirror (by creating the corresponding pool).
403 pub fn init(config
: &MirrorConfig
) -> Result
<(), Error
> {
404 let pool_dir
= format
!("{}/.pool", config
.base_dir
);
406 let dir
= format
!("{}/{}", config
.base_dir
, config
.id
);
408 Pool
::create(Path
::new(&dir
), Path
::new(&pool_dir
))?
;
412 /// Destroy a mirror (by destroying the corresponding pool's link dir followed by GC).
413 pub fn destroy(config
: &MirrorConfig
) -> Result
<(), Error
> {
414 let pool
: Pool
= pool(config
)?
;
415 pool
.lock()?
.destroy()?
;
421 pub fn list_snapshots(config
: &MirrorConfig
) -> Result
<Vec
<Snapshot
>, Error
> {
422 let _pool
: Pool
= pool(config
)?
;
424 let mut list
: Vec
<Snapshot
> = vec
![];
426 let dir
= mirror_dir(config
);
428 let path
= Path
::new(&dir
);
430 proxmox_sys
::fs
::scandir(
434 |_l2_fd
, snapshot
, file_type
| {
435 if file_type
!= nix
::dir
::Type
::Directory
{
439 list
.push(snapshot
.parse()?
);
445 list
.sort_unstable();
450 /// Create a new snapshot of the remote repository, fetching and storing files as needed.
452 /// Operates in three phases:
453 /// - Fetch and verify release files
454 /// - Fetch referenced indices according to config
455 /// - Fetch binary packages referenced by package indices
457 /// Files will be linked in a temporary directory and only renamed to the final, valid snapshot directory at the end. In case of error, leftover `XXX.tmp` directories at the top level of `base_dir` can be safely removed once the next snapshot was successfully created, as they only contain hardlinks.
458 pub fn create_snapshot(
459 config
: MirrorConfig
,
461 subscription
: Option
<SubscriptionKey
>,
463 ) -> Result
<(), Error
> {
464 let auth
= if let Some(product
) = &config
.use_subscription
{
468 "Mirror {} requires a subscription key, but none given.",
472 Some(key
) if key
.product() == *product
=> {
473 let base64
= base64
::encode(format
!("{}:{}", key
.key
, key
.server_id
));
474 Some(format
!("basic {base64}"))
478 "Repository product type '{}' and key product type '{}' don't match.",
488 let mut config
: ParsedMirrorConfig
= config
.try_into()?
;
491 let prefix
= format
!("{snapshot}.tmp");
492 let prefix
= Path
::new(&prefix
);
494 let mut total_progress
= Progress
::new();
496 let parse_release
= |res
: FetchResult
, name
: &str| -> Result
<ReleaseFile
, Error
> {
497 println
!("Parsing {name}..");
498 let parsed
: ReleaseFile
= res
.data
[..].try_into()?
;
500 "'{name}' file has {} referenced files..",
506 // we want both on-disk for compat reasons
507 let res
= fetch_release(&config
, prefix
, true, dry_run
)?
;
508 total_progress
.update(&res
);
509 let _release
= parse_release(res
, "Release")?
;
511 let res
= fetch_release(&config
, prefix
, false, dry_run
)?
;
512 total_progress
.update(&res
);
513 let release
= parse_release(res
, "InRelease")?
;
515 let mut per_component
= HashMap
::new();
516 let mut others
= Vec
::new();
520 .contains(&APTRepositoryPackageType
::Deb
);
524 .contains(&APTRepositoryPackageType
::DebSrc
);
526 for (basename
, references
) in &release
.files
{
527 let reference
= references
.first();
528 let reference
= if let Some(reference
) = reference
{
533 let skip_components
= !&config
.repository
.components
.contains(&reference
.component
);
535 let skip
= skip_components
536 || match &reference
.file_type
{
537 FileReferenceType
::Ignored
=> true,
538 FileReferenceType
::PDiff
=> true, // would require fetching the patches as well
539 FileReferenceType
::Sources(_
) => !source
,
541 if let Some(arch
) = reference
.file_type
.architecture() {
542 !binary
|| !config
.architectures
.contains(arch
)
549 println
!("Skipping {}", reference
.path
);
550 others
.push(reference
);
552 let list
= per_component
553 .entry(reference
.component
)
554 .or_insert_with(Vec
::new
);
560 let mut indices_size
= 0_usize
;
561 let mut total_count
= 0;
563 for (component
, references
) in &per_component
{
564 println
!("Component '{component}'");
566 let mut component_indices_size
= 0;
568 for basename
in references
{
569 for reference
in release
.files
.get(*basename
).unwrap() {
570 println
!("\t{:?}: {:?}", reference
.path
, reference
.file_type
);
571 component_indices_size
+= reference
.size
;
574 indices_size
+= component_indices_size
;
576 let component_count
= references
.len();
577 total_count
+= component_count
;
579 println
!("Component references count: {component_count}");
580 println
!("Component indices size: {component_indices_size}");
581 if references
.is_empty() {
582 println
!("\tNo references found..");
585 println
!("Total indices count: {total_count}");
586 println
!("Total indices size: {indices_size}");
588 if !others
.is_empty() {
589 println
!("Skipped {} references", others
.len());
593 let mut packages_size
= 0_usize
;
594 let mut packages_indices
= HashMap
::new();
595 let mut failed_references
= Vec
::new();
596 for (component
, references
) in per_component
{
597 println
!("\nFetching indices for component '{component}'");
598 let mut component_deb_size
= 0;
599 let mut fetch_progress
= Progress
::new();
601 for basename
in references
{
602 println
!("\tFetching '{basename}'..");
603 let files
= release
.files
.get(basename
).unwrap();
604 let uncompressed_ref
= files
.iter().find(|reference
| reference
.path
== *basename
);
606 let mut package_index_data
= None
;
608 for reference
in files
{
609 // if both compressed and uncompressed are referenced, the uncompressed file may not exist on the server
610 if Some(reference
) == uncompressed_ref
&& files
.len() > 1 {
614 // this will ensure the uncompressed file will be written locally
615 let res
= match fetch_index_file(
620 release
.aquire_by_hash
,
624 Err(err
) if !reference
.file_type
.is_package_index() => {
626 "Failed to fetch '{:?}' type reference '{}', skipping - {err}",
627 reference
.file_type
, reference
.path
629 failed_references
.push(reference
);
632 Err(err
) => bail
!(err
),
634 fetch_progress
.update(&res
);
636 if package_index_data
.is_none() && reference
.file_type
.is_package_index() {
637 package_index_data
= Some(res
.data());
640 if let Some(data
) = package_index_data
{
641 let packages
: PackagesFile
= data
[..].try_into()?
;
642 let size
: usize = packages
.files
.iter().map(|p
| p
.size
).sum();
643 println
!("\t{} packages totalling {size}", packages
.files
.len());
644 component_deb_size
+= size
;
646 packages_indices
.entry(basename
).or_insert(packages
);
648 println
!("Progress: {fetch_progress}");
650 println
!("Total deb size for component: {component_deb_size}");
651 packages_size
+= component_deb_size
;
652 total_progress
+= fetch_progress
;
654 println
!("Total deb size: {packages_size}");
655 if !failed_references
.is_empty() {
656 eprintln
!("Failed to download non-package-index references:");
657 for reference
in failed_references
{
658 eprintln
!("\t{}", reference
.path
);
662 println
!("\nFetching packages..");
663 let mut dry_run_progress
= Progress
::new();
664 for (basename
, references
) in packages_indices
{
665 let total_files
= references
.files
.len();
666 if total_files
== 0 {
667 println
!("\n{basename} - no files, skipping.");
670 println
!("\n{basename} - {total_files} total file(s)");
673 let mut fetch_progress
= Progress
::new();
674 for package
in references
.files
{
675 let url
= get_repo_url(&config
.repository
, &package
.file
);
678 if config
.pool
.contains(&package
.checksums
) {
679 fetch_progress
.update(&FetchResult
{
684 println
!("\t(dry-run) GET missing '{url}' ({}b)", package
.size
);
685 fetch_progress
.update(&FetchResult
{
687 fetched
: package
.size
,
691 let mut full_path
= PathBuf
::from(prefix
);
692 full_path
.push(&package
.file
);
694 let res
= fetch_plain_file(
703 fetch_progress
.update(&res
);
706 if fetch_progress
.file_count() % (max(total_files
/ 100, 1)) == 0 {
707 println
!("\tProgress: {fetch_progress}");
710 println
!("\tProgress: {fetch_progress}");
712 dry_run_progress
+= fetch_progress
;
714 total_progress
+= fetch_progress
;
719 println
!("\nDry-run Stats (indices, downloaded but not persisted):\n{total_progress}");
720 println
!("\nDry-run stats (packages, new == missing):\n{dry_run_progress}");
722 println
!("\nStats: {total_progress}");
726 println
!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
727 let locked
= config
.pool
.lock()?
;
728 locked
.rename(prefix
, Path
::new(&format
!("{snapshot}")))?
;
734 /// Remove a snapshot by removing the corresponding snapshot directory. To actually free up space, a garbage collection needs to be run afterwards.
735 pub fn remove_snapshot(config
: &MirrorConfig
, snapshot
: &Snapshot
) -> Result
<(), Error
> {
736 let pool
: Pool
= pool(config
)?
;
737 let path
= pool
.get_path(Path
::new(&snapshot
.to_string()))?
;
739 pool
.lock()?
.remove_dir(&path
)
742 /// Run a garbage collection on the underlying pool.
743 pub fn gc(config
: &MirrorConfig
) -> Result
<(usize, u64), Error
> {
744 let pool
: Pool
= pool(config
)?
;