]> git.proxmox.com Git - proxmox-offline-mirror.git/blob - src/bin/proxmox-offline-mirror.rs
f7ef7e67afd125bbb0e428c7352752460da56e8a
[proxmox-offline-mirror.git] / src / bin / proxmox-offline-mirror.rs
1 use std::fmt::Display;
2 use std::path::Path;
3
4 use anyhow::{bail, Error};
5 use serde_json::Value;
6
7 use proxmox_router::cli::{run_cli_command, CliCommand, CliCommandMap, CliEnvironment};
8 use proxmox_schema::api;
9 use proxmox_section_config::SectionConfigData;
10 use proxmox_sys::linux::tty;
11
12 use proxmox_offline_mirror::helpers::tty::{
13 read_bool_from_tty, read_selection_from_tty, read_string_from_tty,
14 };
15 use proxmox_offline_mirror::{
16 config::{save_config, MediaConfig, MirrorConfig, SkipConfig},
17 mirror,
18 types::{ProductType, MEDIA_ID_SCHEMA, MIRROR_ID_SCHEMA},
19 };
20
21 mod proxmox_offline_mirror_cmds;
22 use proxmox_offline_mirror_cmds::*;
23
24 enum Distro {
25 Debian,
26 Pbs,
27 Pmg,
28 Pve,
29 PveCeph,
30 }
31
32 impl Display for Distro {
33 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34 match self {
35 Distro::Debian => write!(f, "debian"),
36 Distro::Pbs => write!(f, "pbs"),
37 Distro::Pmg => write!(f, "pmg"),
38 Distro::Pve => write!(f, "pve"),
39 Distro::PveCeph => write!(f, "ceph"),
40 }
41 }
42 }
43
44 enum Release {
45 Bullseye,
46 Buster,
47 }
48
49 impl Display for Release {
50 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
51 match self {
52 Release::Bullseye => write!(f, "bullseye"),
53 Release::Buster => write!(f, "buster"),
54 }
55 }
56 }
57
58 enum DebianVariant {
59 Main,
60 Security,
61 Updates,
62 Backports,
63 Debug,
64 }
65
66 impl Display for DebianVariant {
67 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68 match self {
69 DebianVariant::Main => write!(f, "main"),
70 DebianVariant::Security => write!(f, "security"),
71 DebianVariant::Updates => write!(f, "updates"),
72 DebianVariant::Backports => write!(f, "backports"),
73 DebianVariant::Debug => write!(f, "debug"),
74 }
75 }
76 }
77
78 #[derive(PartialEq)]
79 enum ProxmoxVariant {
80 Enterprise,
81 NoSubscription,
82 Test,
83 }
84
85 impl Display for ProxmoxVariant {
86 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87 match self {
88 ProxmoxVariant::Enterprise => write!(f, "enterprise"),
89 ProxmoxVariant::NoSubscription => write!(f, "no_subscription"),
90 ProxmoxVariant::Test => write!(f, "test"),
91 }
92 }
93 }
94
95 fn derive_debian_repo(
96 release: &Release,
97 variant: &DebianVariant,
98 components: &str,
99 ) -> Result<(String, String, String, SkipConfig), Error> {
100 println!("Configure filters for Debian mirror {release} / {variant}:");
101 let skip_sections = match read_string_from_tty(
102 "\tEnter list of package sections to be skipped ('-' for None)",
103 Some("debug,games"),
104 )?
105 .as_str()
106 {
107 "-" => None,
108 list => Some(
109 list.split(',')
110 .map(|v| v.trim().to_owned())
111 .collect::<Vec<String>>(),
112 ),
113 };
114 let skip_packages = match read_string_from_tty(
115 "\tEnter list of package names/name globs to be skipped ('-' for None)",
116 None,
117 )?
118 .as_str()
119 {
120 "-" => None,
121 list => Some(
122 list.split(',')
123 .map(|v| v.trim().to_owned())
124 .collect::<Vec<String>>(),
125 ),
126 };
127 let filters = SkipConfig {
128 skip_packages,
129 skip_sections,
130 };
131 let url = match (release, variant) {
132 (Release::Bullseye, DebianVariant::Main) => "http://deb.debian.org/debian bullseye",
133 (Release::Bullseye, DebianVariant::Security) => {
134 "http://deb.debian.org/debian-security bullseye-security"
135 }
136 (Release::Bullseye, DebianVariant::Updates) => {
137 "http://deb.debian.org/debian bullseye-updates"
138 }
139 (Release::Bullseye, DebianVariant::Backports) => {
140 "http://deb.debian.org/debian bullseye-backports"
141 }
142 (Release::Bullseye, DebianVariant::Debug) => {
143 "http://deb.debian.org/debian-debug bullseye-debug"
144 }
145 (Release::Buster, DebianVariant::Main) => "http://deb.debian.org/debian buster",
146 (Release::Buster, DebianVariant::Security) => {
147 "http://deb.debian.org/debian-security buster/updates"
148 }
149 (Release::Buster, DebianVariant::Updates) => "http://deb.debian.org/debian buster-updates",
150 (Release::Buster, DebianVariant::Backports) => {
151 "http://deb.debian.org/debian buster-backports"
152 }
153 (Release::Buster, DebianVariant::Debug) => {
154 "http://deb.debian.org/debian-debug buster-debug"
155 }
156 };
157
158 let url = format!("{url} {components}");
159 let key = match (release, variant) {
160 (Release::Bullseye, DebianVariant::Security) => {
161 "/usr/share/keyrings/debian-archive-bullseye-security-automatic.gpg"
162 }
163 (Release::Bullseye, _) => "/usr/share/keyrings/debian-archive-bullseye-automatic.gpg",
164 (Release::Buster, DebianVariant::Security) => {
165 "/usr/share/keyrings/debian-archive-buster-security-automatic.gpg"
166 }
167 (Release::Buster, _) => "/usr/share/keyrings/debian-archive-buster-stable.gpg",
168 };
169
170 let suggested_id = format!("debian_{release}_{variant}");
171
172 Ok((url, key.to_string(), suggested_id, filters))
173 }
174
175 fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Error> {
176 let mut use_subscription = None;
177 let mut extra_repos = Vec::new();
178
179 let (repository, key_path, architectures, suggested_id, skip) = if read_bool_from_tty(
180 "Guided Setup",
181 Some(true),
182 )? {
183 let distros = &[
184 (Distro::Pve, "Proxmox VE"),
185 (Distro::Pbs, "Proxmox Backup Server"),
186 (Distro::Pmg, "Proxmox Mail Gateway"),
187 (Distro::PveCeph, "Proxmox Ceph"),
188 (Distro::Debian, "Debian"),
189 ];
190 let dist = read_selection_from_tty("Select distro to mirror", distros, None)?;
191
192 let releases = &[(Release::Bullseye, "Bullseye"), (Release::Buster, "Buster")];
193 let release = read_selection_from_tty("Select release", releases, Some(0))?;
194
195 let mut add_debian_repo = false;
196
197 let (url, key_path, suggested_id, skip) = match dist {
198 Distro::Debian => {
199 let variants = &[
200 (DebianVariant::Main, "Main repository"),
201 (DebianVariant::Security, "Security"),
202 (DebianVariant::Updates, "Updates"),
203 (DebianVariant::Backports, "Backports"),
204 (DebianVariant::Debug, "Debug Information"),
205 ];
206 let variant =
207 read_selection_from_tty("Select repository variant", variants, Some(0))?;
208 let components = read_string_from_tty(
209 "Enter repository components",
210 Some("main contrib non-free"),
211 )?;
212
213 derive_debian_repo(release, variant, &components)?
214 }
215 Distro::PveCeph => {
216 enum CephRelease {
217 Luminous,
218 Nautilus,
219 Octopus,
220 Pacific,
221 }
222
223 let releases = match release {
224 Release::Bullseye => {
225 vec![
226 (CephRelease::Octopus, "Octopus (15.x)"),
227 (CephRelease::Pacific, "Pacific (16.x)"),
228 ]
229 }
230 Release::Buster => {
231 vec![
232 (CephRelease::Luminous, "Luminous (12.x)"),
233 (CephRelease::Nautilus, "Nautilus (14.x)"),
234 (CephRelease::Octopus, "Octopus (15.x)"),
235 ]
236 }
237 };
238
239 let ceph_release = read_selection_from_tty(
240 "Select Ceph release",
241 &releases,
242 Some(releases.len() - 1),
243 )?;
244
245 let components =
246 read_string_from_tty("Enter repository components", Some("main test"))?;
247
248 let key = match release {
249 Release::Bullseye => "/etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg",
250 Release::Buster => "/etc/apt/trusted.gpg.d/proxmox-release-buster.gpg",
251 };
252
253 let ceph_release = match ceph_release {
254 CephRelease::Luminous => "luminous",
255 CephRelease::Nautilus => "nautilus",
256 CephRelease::Octopus => "octopus",
257 CephRelease::Pacific => "pacific",
258 };
259
260 let url = format!(
261 "http://download.proxmox.com/debian/ceph-{ceph_release} {release} {components}"
262 );
263 let suggested_id = format!("ceph_{ceph_release}_{release}");
264
265 (url, key.to_string(), suggested_id, SkipConfig::default())
266 }
267 product => {
268 let variants = &[
269 (ProxmoxVariant::Enterprise, "Enterprise repository"),
270 (ProxmoxVariant::NoSubscription, "No-Subscription repository"),
271 (ProxmoxVariant::Test, "Test repository"),
272 ];
273
274 let variant =
275 read_selection_from_tty("Select repository variant", variants, Some(0))?;
276
277 // TODO enterprise query for key!
278 let url = match (release, variant) {
279 (Release::Bullseye, ProxmoxVariant::Enterprise) => format!("https://enterprise.proxmox.com/debian/{product} bullseye {product}-enterprise"),
280 (Release::Bullseye, ProxmoxVariant::NoSubscription) => format!("http://download.proxmox.com/debian/{product} bullseye {product}-no-subscription"),
281 (Release::Bullseye, ProxmoxVariant::Test) => format!("http://download.proxmox.com/debian/{product} bullseye {product}test"),
282 (Release::Buster, ProxmoxVariant::Enterprise) => format!("https://enterprise.proxmox.com/debian/{product} buster {product}-enterprise"),
283 (Release::Buster, ProxmoxVariant::NoSubscription) => format!("http://download.proxmox.com/debian/{product} buster {product}-no-subscription"),
284 (Release::Buster, ProxmoxVariant::Test) => format!("http://download.proxmox.com/debian/{product} buster {product}test"),
285 };
286
287 use_subscription = match (product, variant) {
288 (Distro::Pbs, &ProxmoxVariant::Enterprise) => Some(ProductType::Pbs),
289 (Distro::Pmg, &ProxmoxVariant::Enterprise) => Some(ProductType::Pmg),
290 (Distro::Pve, &ProxmoxVariant::Enterprise) => Some(ProductType::Pve),
291 _ => None,
292 };
293
294 let key = match release {
295 Release::Bullseye => "/etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg",
296 Release::Buster => "/etc/apt/trusted.gpg.d/proxmox-release-buster.gpg",
297 };
298
299 let suggested_id = format!("{product}_{release}_{variant}");
300
301 add_debian_repo = read_bool_from_tty(
302 "Should missing Debian mirrors for the selected product be auto-added",
303 Some(true),
304 )?;
305
306 (url, key.to_string(), suggested_id, SkipConfig::default())
307 }
308 };
309
310 let architectures = vec!["amd64".to_string(), "all".to_string()];
311
312 if add_debian_repo {
313 extra_repos.push(derive_debian_repo(
314 release,
315 &DebianVariant::Main,
316 "main contrib",
317 )?);
318 extra_repos.push(derive_debian_repo(
319 release,
320 &DebianVariant::Updates,
321 "main contrib",
322 )?);
323 extra_repos.push(derive_debian_repo(
324 release,
325 &DebianVariant::Security,
326 "main contrib",
327 )?);
328 }
329 (
330 format!("deb {url}"),
331 key_path,
332 architectures,
333 Some(suggested_id),
334 skip,
335 )
336 } else {
337 let repo = read_string_from_tty("Enter repository line in sources.list format", None)?;
338 let key_path = read_string_from_tty("Enter (absolute) path to repository key file", None)?;
339 let architectures =
340 read_string_from_tty("Enter list of architectures to mirror", Some("amd64,all"))?;
341 let architectures: Vec<String> = architectures
342 .split(|c: char| c == ',' || c.is_ascii_whitespace())
343 .filter_map(|value| {
344 if value.is_empty() {
345 None
346 } else {
347 Some(value.to_owned())
348 }
349 })
350 .collect();
351 let subscription_products = &[
352 (Some(ProductType::Pve), "PVE"),
353 (Some(ProductType::Pbs), "PBS"),
354 (Some(ProductType::Pmg), "PMG"),
355 (None, "None"),
356 ];
357 use_subscription = read_selection_from_tty(
358 "Does this repository require a valid Proxmox subscription key",
359 subscription_products,
360 None,
361 )?
362 .clone();
363
364 (repo, key_path, architectures, None, SkipConfig::default())
365 };
366
367 if !Path::new(&key_path).exists() {
368 eprintln!("Keyfile '{key_path}' doesn't exist - make sure to install relevant keyring packages or update config to provide correct path!");
369 }
370
371 let id = loop {
372 let mut id = read_string_from_tty("Enter mirror ID", suggested_id.as_deref())?;
373 while let Err(err) = MIRROR_ID_SCHEMA.parse_simple_value(&id) {
374 eprintln!("Not a valid mirror ID: {err}");
375 id = read_string_from_tty("Enter mirror ID", None)?;
376 }
377
378 if config.sections.contains_key(&id) {
379 eprintln!("Config entry '{id}' already exists!");
380 continue;
381 }
382
383 break id;
384 };
385
386 let base_dir = loop {
387 let path = read_string_from_tty(
388 "Enter (absolute) base path where mirrored repositories will be stored",
389 Some("/var/lib/proxmox-offline-mirror/mirrors/"),
390 )?;
391 if !path.starts_with('/') {
392 eprintln!("Path must start with '/'");
393 } else {
394 break path;
395 }
396 };
397
398 let verify = read_bool_from_tty(
399 "Should already mirrored files be re-verified when updating the mirror? (io-intensive!)",
400 Some(true),
401 )?;
402 let sync = read_bool_from_tty("Should newly written files be written using FSYNC to ensure crash-consistency? (io-intensive!)", Some(true))?;
403
404 let mut configs = Vec::with_capacity(extra_repos.len() + 1);
405
406 for (url, key_path, suggested_id, skip) in extra_repos {
407 if config.sections.contains_key(&suggested_id) {
408 eprintln!("config section '{suggested_id}' already exists, skipping..");
409 } else {
410 let repository = format!("deb {url}");
411
412 configs.push(MirrorConfig {
413 id: suggested_id,
414 repository,
415 architectures: architectures.clone(),
416 key_path,
417 verify,
418 sync,
419 base_dir: base_dir.clone(),
420 use_subscription: None,
421 ignore_errors: false,
422 skip,
423 });
424 }
425 }
426
427 let main_config = MirrorConfig {
428 id,
429 repository,
430 architectures,
431 key_path,
432 verify,
433 sync,
434 base_dir,
435 use_subscription,
436 ignore_errors: false,
437 skip,
438 };
439
440 configs.push(main_config);
441 Ok(configs)
442 }
443
444 fn action_add_medium(config: &SectionConfigData) -> Result<MediaConfig, Error> {
445 let id = loop {
446 let id = read_string_from_tty("Enter new medium ID", None)?;
447 if let Err(err) = MEDIA_ID_SCHEMA.parse_simple_value(&id) {
448 eprintln!("Not a valid medium ID: {err}");
449 continue;
450 }
451
452 if config.sections.contains_key(&id) {
453 eprintln!("Config entry '{id}' already exists!");
454 continue;
455 }
456
457 break id;
458 };
459
460 let mountpoint = loop {
461 let path = read_string_from_tty("Enter (absolute) path where medium is mounted", None)?;
462 if !path.starts_with('/') {
463 eprintln!("Path must start with '/'");
464 continue;
465 }
466
467 let mountpoint = Path::new(&path);
468 if !mountpoint.exists() {
469 eprintln!("Path doesn't exist.");
470 } else {
471 let mut statefile = mountpoint.to_path_buf();
472 statefile.push(".mirror-state");
473 if !statefile.exists()
474 || read_bool_from_tty(
475 &format!("Found existing statefile at {statefile:?} - proceed?"),
476 Some(false),
477 )?
478 {
479 break path;
480 }
481 }
482 };
483
484 let mirrors: Vec<MirrorConfig> = config.convert_to_typed_array("mirror")?;
485 let mut available_mirrors: Vec<String> = Vec::new();
486 for mirror_config in mirrors {
487 available_mirrors.push(mirror_config.id);
488 }
489
490 let mut selected_mirrors: Vec<String> = Vec::new();
491
492 enum Action {
493 SelectMirror,
494 SelectAllMirrors,
495 DeselectMirror,
496 DeselectAllMirrors,
497 Proceed,
498 }
499
500 loop {
501 println!();
502 let actions = if selected_mirrors.is_empty() {
503 println!("No mirrors selected for inclusion on medium so far.");
504 vec![
505 (Action::SelectMirror, "Add mirror to selection."),
506 (Action::SelectAllMirrors, "Add all mirrors to selection."),
507 (Action::Proceed, "Proceed"),
508 ]
509 } else {
510 println!("Mirrors selected for inclusion on medium:");
511 for id in &selected_mirrors {
512 println!("\t- {id}");
513 }
514 println!();
515 if available_mirrors.is_empty() {
516 println!("No more mirrors available for selection!");
517 vec![
518 (Action::DeselectMirror, "Remove mirror from selection."),
519 (
520 Action::DeselectAllMirrors,
521 "Remove all mirrors from selection.",
522 ),
523 (Action::Proceed, "Proceed"),
524 ]
525 } else {
526 vec![
527 (Action::SelectMirror, "Add mirror to selection."),
528 (Action::SelectAllMirrors, "Add all mirrors to selection."),
529 (Action::DeselectMirror, "Remove mirror from selection."),
530 (
531 Action::DeselectAllMirrors,
532 "Remove all mirrors from selection.",
533 ),
534 (Action::Proceed, "Proceed"),
535 ]
536 }
537 };
538
539 println!();
540
541 let action = read_selection_from_tty("Select action", &actions, Some(0))?;
542 println!();
543
544 match action {
545 Action::SelectMirror => {
546 if available_mirrors.is_empty() {
547 println!("No (more) unselected mirrors available.");
548 continue;
549 }
550
551 let mirrors: Vec<(&str, &str)> = available_mirrors
552 .iter()
553 .map(|v| (v.as_ref(), v.as_ref()))
554 .collect();
555
556 let selected =
557 read_selection_from_tty("Select a mirror to add", &mirrors, None)?.to_string();
558 available_mirrors.retain(|v| *v != selected);
559 selected_mirrors.push(selected);
560 }
561 Action::SelectAllMirrors => {
562 selected_mirrors.extend_from_slice(&available_mirrors);
563 available_mirrors.truncate(0);
564 }
565 Action::DeselectMirror => {
566 if selected_mirrors.is_empty() {
567 println!("No mirrors selected (yet).");
568 continue;
569 }
570
571 let mirrors: Vec<(&str, &str)> = selected_mirrors
572 .iter()
573 .map(|v| (v.as_ref(), v.as_ref()))
574 .collect();
575
576 let selected =
577 read_selection_from_tty("Select a mirror to remove", &mirrors, None)?
578 .to_string();
579 selected_mirrors.retain(|v| *v != selected);
580 available_mirrors.push(selected);
581 }
582 Action::DeselectAllMirrors => {
583 available_mirrors.extend_from_slice(&selected_mirrors);
584 selected_mirrors.truncate(0);
585 }
586 Action::Proceed => {
587 break;
588 }
589 }
590 }
591
592 let verify = read_bool_from_tty(
593 "Should mirrored files be re-verified when updating the medium? (io-intensive!)",
594 Some(true),
595 )?;
596 let sync = read_bool_from_tty("Should newly written files be written using FSYNC to ensure crash-consistency? (io-intensive!)", Some(true))?;
597
598 Ok(MediaConfig {
599 id,
600 mountpoint,
601 mirrors: selected_mirrors,
602 verify,
603 sync,
604 })
605 }
606
607 #[api(
608 input: {
609 properties: {
610 config: {
611 type: String,
612 optional: true,
613 description: "Path to mirroring config file.",
614 },
615 },
616 },
617 )]
618 /// Interactive setup wizard.
619 async fn setup(config: Option<String>, _param: Value) -> Result<(), Error> {
620 if !tty::stdin_isatty() {
621 bail!("Setup wizard can only run interactively.");
622 }
623
624 let config_file = config.unwrap_or_else(get_config_path);
625
626 let _lock = proxmox_offline_mirror::config::lock_config(&config_file)?;
627
628 let (mut config, _digest) = proxmox_offline_mirror::config::config(&config_file)?;
629
630 if config.sections.is_empty() {
631 println!("Initializing new config.");
632 } else {
633 println!("Loaded existing config.");
634 }
635
636 enum Action {
637 AddMirror,
638 AddMedium,
639 Quit,
640 }
641
642 loop {
643 println!();
644 let mut mirror_defined = false;
645 if !config.sections.is_empty() {
646 println!("Existing config entries:");
647 for (section, (section_type, _)) in config.sections.iter() {
648 if section_type == "mirror" {
649 mirror_defined = true;
650 }
651 println!("{section_type} '{section}'");
652 }
653 println!();
654 }
655
656 let actions = if mirror_defined {
657 vec![
658 (Action::AddMirror, "Add new mirror entry"),
659 (Action::AddMedium, "Add new medium entry"),
660 (Action::Quit, "Quit"),
661 ]
662 } else {
663 vec![
664 (Action::AddMirror, "Add new mirror entry"),
665 (Action::Quit, "Quit"),
666 ]
667 };
668
669 match read_selection_from_tty("Select Action:", &actions, Some(0))? {
670 Action::Quit => break,
671 Action::AddMirror => {
672 for mirror_config in action_add_mirror(&config)? {
673 let id = mirror_config.id.clone();
674 mirror::init(&mirror_config)?;
675 config.set_data(&id, "mirror", mirror_config)?;
676 save_config(&config_file, &config)?;
677 println!("Config entry '{id}' added");
678 println!("Run \"proxmox-offline-mirror mirror snapshot create --config '{config_file}' '{id}'\" to create a new mirror snapshot.");
679 }
680 }
681 Action::AddMedium => {
682 let media_config = action_add_medium(&config)?;
683 let id = media_config.id.clone();
684 config.set_data(&id, "medium", media_config)?;
685 save_config(&config_file, &config)?;
686 println!("Config entry '{id}' added");
687 println!("Run \"proxmox-offline-mirror medium sync --config '{config_file}' '{id}'\" to sync mirror snapshots to medium.");
688 }
689 }
690 }
691
692 Ok(())
693 }
694 fn main() {
695 let rpcenv = CliEnvironment::new();
696
697 let cmd_def = CliCommandMap::new()
698 .insert("setup", CliCommand::new(&API_METHOD_SETUP))
699 .insert("config", config_commands())
700 .insert("key", key_commands())
701 .insert("medium", medium_commands())
702 .insert("mirror", mirror_commands());
703
704 run_cli_command(
705 cmd_def,
706 rpcenv,
707 Some(|future| proxmox_async::runtime::main(future)),
708 );
709 }