]> git.proxmox.com Git - proxmox-offline-mirror.git/blob - src/bin/proxmox-offline-mirror.rs
fix #4259: mirror: add ignore-errors option
[proxmox-offline-mirror.git] / src / bin / proxmox-offline-mirror.rs
1 use std::fmt::Display;
2 use std::path::Path;
3
4 use anyhow::{bail, Error};
5 use serde_json::Value;
6
7 use proxmox_router::cli::{run_cli_command, CliCommand, CliCommandMap, CliEnvironment};
8 use proxmox_schema::api;
9 use proxmox_section_config::SectionConfigData;
10 use proxmox_sys::linux::tty;
11
12 use proxmox_offline_mirror::helpers::tty::{
13 read_bool_from_tty, read_selection_from_tty, read_string_from_tty,
14 };
15 use proxmox_offline_mirror::{
16 config::{save_config, MediaConfig, MirrorConfig},
17 mirror,
18 types::{ProductType, MEDIA_ID_SCHEMA, MIRROR_ID_SCHEMA},
19 };
20
21 mod proxmox_offline_mirror_cmds;
22 use proxmox_offline_mirror_cmds::*;
23
24 enum Distro {
25 Debian,
26 Pbs,
27 Pmg,
28 Pve,
29 PveCeph,
30 }
31
32 impl Display for Distro {
33 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34 match self {
35 Distro::Debian => write!(f, "debian"),
36 Distro::Pbs => write!(f, "pbs"),
37 Distro::Pmg => write!(f, "pmg"),
38 Distro::Pve => write!(f, "pve"),
39 Distro::PveCeph => write!(f, "ceph"),
40 }
41 }
42 }
43
44 enum Release {
45 Bullseye,
46 Buster,
47 }
48
49 impl Display for Release {
50 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
51 match self {
52 Release::Bullseye => write!(f, "bullseye"),
53 Release::Buster => write!(f, "buster"),
54 }
55 }
56 }
57
58 enum DebianVariant {
59 Main,
60 Security,
61 Updates,
62 Backports,
63 Debug,
64 }
65
66 impl Display for DebianVariant {
67 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68 match self {
69 DebianVariant::Main => write!(f, "main"),
70 DebianVariant::Security => write!(f, "security"),
71 DebianVariant::Updates => write!(f, "updates"),
72 DebianVariant::Backports => write!(f, "backports"),
73 DebianVariant::Debug => write!(f, "debug"),
74 }
75 }
76 }
77
78 #[derive(PartialEq)]
79 enum ProxmoxVariant {
80 Enterprise,
81 NoSubscription,
82 Test,
83 }
84
85 impl Display for ProxmoxVariant {
86 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87 match self {
88 ProxmoxVariant::Enterprise => write!(f, "enterprise"),
89 ProxmoxVariant::NoSubscription => write!(f, "no_subscription"),
90 ProxmoxVariant::Test => write!(f, "test"),
91 }
92 }
93 }
94
95 fn derive_debian_repo(
96 release: &Release,
97 variant: &DebianVariant,
98 components: &str,
99 ) -> (String, String, String) {
100 let url = match (release, variant) {
101 (Release::Bullseye, DebianVariant::Main) => "http://deb.debian.org/debian bullseye",
102 (Release::Bullseye, DebianVariant::Security) => {
103 "http://deb.debian.org/debian-security bullseye-security"
104 }
105 (Release::Bullseye, DebianVariant::Updates) => {
106 "http://deb.debian.org/debian bullseye-updates"
107 }
108 (Release::Bullseye, DebianVariant::Backports) => {
109 "http://deb.debian.org/debian bullseye-backports"
110 }
111 (Release::Bullseye, DebianVariant::Debug) => {
112 "http://deb.debian.org/debian-debug bullseye-debug"
113 }
114 (Release::Buster, DebianVariant::Main) => "http://deb.debian.org/debian buster",
115 (Release::Buster, DebianVariant::Security) => {
116 "http://deb.debian.org/debian-security buster/updates"
117 }
118 (Release::Buster, DebianVariant::Updates) => "http://deb.debian.org/debian buster-updates",
119 (Release::Buster, DebianVariant::Backports) => {
120 "http://deb.debian.org/debian buster-backports"
121 }
122 (Release::Buster, DebianVariant::Debug) => {
123 "http://deb.debian.org/debian-debug buster-debug"
124 }
125 };
126
127 let url = format!("{url} {components}");
128 let key = match (release, variant) {
129 (Release::Bullseye, DebianVariant::Security) => {
130 "/usr/share/keyrings/debian-archive-bullseye-security-automatic.gpg"
131 }
132 (Release::Bullseye, _) => "/usr/share/keyrings/debian-archive-bullseye-automatic.gpg",
133 (Release::Buster, DebianVariant::Security) => {
134 "/usr/share/keyrings/debian-archive-buster-security-automatic.gpg"
135 }
136 (Release::Buster, _) => "/usr/share/keyrings/debian-archive-buster-stable.gpg",
137 };
138
139 let suggested_id = format!("debian_{release}_{variant}");
140
141 (url, key.to_string(), suggested_id)
142 }
143
144 fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Error> {
145 let mut use_subscription = None;
146 let mut extra_repos = Vec::new();
147
148 let (repository, key_path, architectures, suggested_id) = if read_bool_from_tty(
149 "Guided Setup",
150 Some(true),
151 )? {
152 let distros = &[
153 (Distro::Pve, "Proxmox VE"),
154 (Distro::Pbs, "Proxmox Backup Server"),
155 (Distro::Pmg, "Proxmox Mail Gateway"),
156 (Distro::PveCeph, "Proxmox Ceph"),
157 (Distro::Debian, "Debian"),
158 ];
159 let dist = read_selection_from_tty("Select distro to mirror", distros, None)?;
160
161 let releases = &[(Release::Bullseye, "Bullseye"), (Release::Buster, "Buster")];
162 let release = read_selection_from_tty("Select release", releases, Some(0))?;
163
164 let mut add_debian_repo = false;
165
166 let (url, key_path, suggested_id) = match dist {
167 Distro::Debian => {
168 let variants = &[
169 (DebianVariant::Main, "Main repository"),
170 (DebianVariant::Security, "Security"),
171 (DebianVariant::Updates, "Updates"),
172 (DebianVariant::Backports, "Backports"),
173 (DebianVariant::Debug, "Debug Information"),
174 ];
175 let variant =
176 read_selection_from_tty("Select repository variant", variants, Some(0))?;
177 let components = read_string_from_tty(
178 "Enter repository components",
179 Some("main contrib non-free"),
180 )?;
181
182 derive_debian_repo(release, variant, &components)
183 }
184 Distro::PveCeph => {
185 enum CephRelease {
186 Luminous,
187 Nautilus,
188 Octopus,
189 Pacific,
190 }
191
192 let releases = match release {
193 Release::Bullseye => {
194 vec![
195 (CephRelease::Octopus, "Octopus (15.x)"),
196 (CephRelease::Pacific, "Pacific (16.x)"),
197 ]
198 }
199 Release::Buster => {
200 vec![
201 (CephRelease::Luminous, "Luminous (12.x)"),
202 (CephRelease::Nautilus, "Nautilus (14.x)"),
203 (CephRelease::Octopus, "Octopus (15.x)"),
204 ]
205 }
206 };
207
208 let ceph_release = read_selection_from_tty(
209 "Select Ceph release",
210 &releases,
211 Some(releases.len() - 1),
212 )?;
213
214 let components =
215 read_string_from_tty("Enter repository components", Some("main test"))?;
216
217 let key = match release {
218 Release::Bullseye => "/etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg",
219 Release::Buster => "/etc/apt/trusted.gpg.d/proxmox-release-buster.gpg",
220 };
221
222 let ceph_release = match ceph_release {
223 CephRelease::Luminous => "luminous",
224 CephRelease::Nautilus => "nautilus",
225 CephRelease::Octopus => "octopus",
226 CephRelease::Pacific => "pacific",
227 };
228
229 let url = format!(
230 "http://download.proxmox.com/debian/ceph-{ceph_release} {release} {components}"
231 );
232 let suggested_id = format!("ceph_{ceph_release}_{release}");
233
234 (url, key.to_string(), suggested_id)
235 }
236 product => {
237 let variants = &[
238 (ProxmoxVariant::Enterprise, "Enterprise repository"),
239 (ProxmoxVariant::NoSubscription, "No-Subscription repository"),
240 (ProxmoxVariant::Test, "Test repository"),
241 ];
242
243 let variant =
244 read_selection_from_tty("Select repository variant", variants, Some(0))?;
245
246 // TODO enterprise query for key!
247 let url = match (release, variant) {
248 (Release::Bullseye, ProxmoxVariant::Enterprise) => format!("https://enterprise.proxmox.com/debian/{product} bullseye {product}-enterprise"),
249 (Release::Bullseye, ProxmoxVariant::NoSubscription) => format!("http://download.proxmox.com/debian/{product} bullseye {product}-no-subscription"),
250 (Release::Bullseye, ProxmoxVariant::Test) => format!("http://download.proxmox.com/debian/{product} bullseye {product}test"),
251 (Release::Buster, ProxmoxVariant::Enterprise) => format!("https://enterprise.proxmox.com/debian/{product} buster {product}-enterprise"),
252 (Release::Buster, ProxmoxVariant::NoSubscription) => format!("http://download.proxmox.com/debian/{product} buster {product}-no-subscription"),
253 (Release::Buster, ProxmoxVariant::Test) => format!("http://download.proxmox.com/debian/{product} buster {product}test"),
254 };
255
256 use_subscription = match (product, variant) {
257 (Distro::Pbs, &ProxmoxVariant::Enterprise) => Some(ProductType::Pbs),
258 (Distro::Pmg, &ProxmoxVariant::Enterprise) => Some(ProductType::Pmg),
259 (Distro::Pve, &ProxmoxVariant::Enterprise) => Some(ProductType::Pve),
260 _ => None,
261 };
262
263 let key = match release {
264 Release::Bullseye => "/etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg",
265 Release::Buster => "/etc/apt/trusted.gpg.d/proxmox-release-buster.gpg",
266 };
267
268 let suggested_id = format!("{product}_{release}_{variant}");
269
270 add_debian_repo = read_bool_from_tty(
271 "Should missing Debian mirrors for the selected product be auto-added",
272 Some(true),
273 )?;
274
275 (url, key.to_string(), suggested_id)
276 }
277 };
278
279 let architectures = vec!["amd64".to_string(), "all".to_string()];
280
281 if add_debian_repo {
282 extra_repos.push(derive_debian_repo(
283 release,
284 &DebianVariant::Main,
285 "main contrib",
286 ));
287 extra_repos.push(derive_debian_repo(
288 release,
289 &DebianVariant::Updates,
290 "main contrib",
291 ));
292 extra_repos.push(derive_debian_repo(
293 release,
294 &DebianVariant::Security,
295 "main contrib",
296 ));
297 }
298 (
299 format!("deb {url}"),
300 key_path,
301 architectures,
302 Some(suggested_id),
303 )
304 } else {
305 let repo = read_string_from_tty("Enter repository line in sources.list format", None)?;
306 let key_path = read_string_from_tty("Enter (absolute) path to repository key file", None)?;
307 let architectures =
308 read_string_from_tty("Enter list of architectures to mirror", Some("amd64,all"))?;
309 let architectures: Vec<String> = architectures
310 .split(|c: char| c == ',' || c.is_ascii_whitespace())
311 .filter_map(|value| {
312 if value.is_empty() {
313 None
314 } else {
315 Some(value.to_owned())
316 }
317 })
318 .collect();
319 let subscription_products = &[
320 (Some(ProductType::Pve), "PVE"),
321 (Some(ProductType::Pbs), "PBS"),
322 (Some(ProductType::Pmg), "PMG"),
323 (None, "None"),
324 ];
325 use_subscription = read_selection_from_tty(
326 "Does this repository require a valid Proxmox subscription key",
327 subscription_products,
328 None,
329 )?
330 .clone();
331
332 (repo, key_path, architectures, None)
333 };
334
335 if !Path::new(&key_path).exists() {
336 eprintln!("Keyfile '{key_path}' doesn't exist - make sure to install relevant keyring packages or update config to provide correct path!");
337 }
338
339 let id = loop {
340 let mut id = read_string_from_tty("Enter mirror ID", suggested_id.as_deref())?;
341 while let Err(err) = MIRROR_ID_SCHEMA.parse_simple_value(&id) {
342 eprintln!("Not a valid mirror ID: {err}");
343 id = read_string_from_tty("Enter mirror ID", None)?;
344 }
345
346 if config.sections.contains_key(&id) {
347 eprintln!("Config entry '{id}' already exists!");
348 continue;
349 }
350
351 break id;
352 };
353
354 let base_dir = loop {
355 let path = read_string_from_tty(
356 "Enter (absolute) base path where mirrored repositories will be stored",
357 Some("/var/lib/proxmox-offline-mirror/mirrors/"),
358 )?;
359 if !path.starts_with('/') {
360 eprintln!("Path must start with '/'");
361 } else {
362 break path;
363 }
364 };
365
366 let verify = read_bool_from_tty(
367 "Should already mirrored files be re-verified when updating the mirror? (io-intensive!)",
368 Some(true),
369 )?;
370 let sync = read_bool_from_tty("Should newly written files be written using FSYNC to ensure crash-consistency? (io-intensive!)", Some(true))?;
371
372 let mut configs = Vec::with_capacity(extra_repos.len() + 1);
373
374 for (url, key_path, suggested_id) in extra_repos {
375 if config.sections.contains_key(&suggested_id) {
376 eprintln!("config section '{suggested_id}' already exists, skipping..");
377 } else {
378 let repository = format!("deb {url}");
379
380 configs.push(MirrorConfig {
381 id: suggested_id,
382 repository,
383 architectures: architectures.clone(),
384 key_path,
385 verify,
386 sync,
387 base_dir: base_dir.clone(),
388 use_subscription: None,
389 ignore_errors: false,
390 });
391 }
392 }
393
394 let main_config = MirrorConfig {
395 id,
396 repository,
397 architectures,
398 key_path,
399 verify,
400 sync,
401 base_dir,
402 use_subscription,
403 ignore_errors: false,
404 };
405
406 configs.push(main_config);
407 Ok(configs)
408 }
409
410 fn action_add_medium(config: &SectionConfigData) -> Result<MediaConfig, Error> {
411 let id = loop {
412 let id = read_string_from_tty("Enter new medium ID", None)?;
413 if let Err(err) = MEDIA_ID_SCHEMA.parse_simple_value(&id) {
414 eprintln!("Not a valid medium ID: {err}");
415 continue;
416 }
417
418 if config.sections.contains_key(&id) {
419 eprintln!("Config entry '{id}' already exists!");
420 continue;
421 }
422
423 break id;
424 };
425
426 let mountpoint = loop {
427 let path = read_string_from_tty("Enter (absolute) path where medium is mounted", None)?;
428 if !path.starts_with('/') {
429 eprintln!("Path must start with '/'");
430 continue;
431 }
432
433 let mountpoint = Path::new(&path);
434 if !mountpoint.exists() {
435 eprintln!("Path doesn't exist.");
436 } else {
437 let mut statefile = mountpoint.to_path_buf();
438 statefile.push(".mirror-state");
439 if !statefile.exists()
440 || read_bool_from_tty(
441 &format!("Found existing statefile at {statefile:?} - proceed?"),
442 Some(false),
443 )?
444 {
445 break path;
446 }
447 }
448 };
449
450 let mirrors: Vec<MirrorConfig> = config.convert_to_typed_array("mirror")?;
451 let mut available_mirrors: Vec<String> = Vec::new();
452 for mirror_config in mirrors {
453 available_mirrors.push(mirror_config.id);
454 }
455
456 let mut selected_mirrors: Vec<String> = Vec::new();
457
458 enum Action {
459 SelectMirror,
460 SelectAllMirrors,
461 DeselectMirror,
462 DeselectAllMirrors,
463 Proceed,
464 }
465
466 loop {
467 println!();
468 let actions = if selected_mirrors.is_empty() {
469 println!("No mirrors selected for inclusion on medium so far.");
470 vec![
471 (Action::SelectMirror, "Add mirror to selection."),
472 (Action::SelectAllMirrors, "Add all mirrors to selection."),
473 (Action::Proceed, "Proceed"),
474 ]
475 } else {
476 println!("Mirrors selected for inclusion on medium:");
477 for id in &selected_mirrors {
478 println!("\t- {id}");
479 }
480 println!();
481 if available_mirrors.is_empty() {
482 println!("No more mirrors available for selection!");
483 vec![
484 (Action::DeselectMirror, "Remove mirror from selection."),
485 (
486 Action::DeselectAllMirrors,
487 "Remove all mirrors from selection.",
488 ),
489 (Action::Proceed, "Proceed"),
490 ]
491 } else {
492 vec![
493 (Action::SelectMirror, "Add mirror to selection."),
494 (Action::SelectAllMirrors, "Add all mirrors to selection."),
495 (Action::DeselectMirror, "Remove mirror from selection."),
496 (
497 Action::DeselectAllMirrors,
498 "Remove all mirrors from selection.",
499 ),
500 (Action::Proceed, "Proceed"),
501 ]
502 }
503 };
504
505 println!();
506
507 let action = read_selection_from_tty("Select action", &actions, Some(0))?;
508 println!();
509
510 match action {
511 Action::SelectMirror => {
512 if available_mirrors.is_empty() {
513 println!("No (more) unselected mirrors available.");
514 continue;
515 }
516
517 let mirrors: Vec<(&str, &str)> = available_mirrors
518 .iter()
519 .map(|v| (v.as_ref(), v.as_ref()))
520 .collect();
521
522 let selected =
523 read_selection_from_tty("Select a mirror to add", &mirrors, None)?.to_string();
524 available_mirrors = available_mirrors
525 .into_iter()
526 .filter(|v| *v != selected)
527 .collect();
528 selected_mirrors.push(selected);
529 }
530 Action::SelectAllMirrors => {
531 selected_mirrors.extend_from_slice(&available_mirrors);
532 available_mirrors.truncate(0);
533 }
534 Action::DeselectMirror => {
535 if selected_mirrors.is_empty() {
536 println!("No mirrors selected (yet).");
537 continue;
538 }
539
540 let mirrors: Vec<(&str, &str)> = selected_mirrors
541 .iter()
542 .map(|v| (v.as_ref(), v.as_ref()))
543 .collect();
544
545 let selected =
546 read_selection_from_tty("Select a mirror to remove", &mirrors, None)?
547 .to_string();
548 selected_mirrors = selected_mirrors
549 .into_iter()
550 .filter(|v| *v != selected)
551 .collect();
552 available_mirrors.push(selected);
553 }
554 Action::DeselectAllMirrors => {
555 available_mirrors.extend_from_slice(&selected_mirrors);
556 selected_mirrors.truncate(0);
557 }
558 Action::Proceed => {
559 break;
560 }
561 }
562 }
563
564 let verify = read_bool_from_tty(
565 "Should mirrored files be re-verified when updating the medium? (io-intensive!)",
566 Some(true),
567 )?;
568 let sync = read_bool_from_tty("Should newly written files be written using FSYNC to ensure crash-consistency? (io-intensive!)", Some(true))?;
569
570 Ok(MediaConfig {
571 id,
572 mountpoint,
573 mirrors: selected_mirrors,
574 verify,
575 sync,
576 })
577 }
578
579 #[api(
580 input: {
581 properties: {
582 config: {
583 type: String,
584 optional: true,
585 description: "Path to mirroring config file.",
586 },
587 },
588 },
589 )]
590 /// Interactive setup wizard.
591 async fn setup(config: Option<String>, _param: Value) -> Result<(), Error> {
592 if !tty::stdin_isatty() {
593 bail!("Setup wizard can only run interactively.");
594 }
595
596 let config_file = config.unwrap_or_else(get_config_path);
597
598 let _lock = proxmox_offline_mirror::config::lock_config(&config_file)?;
599
600 let (mut config, _digest) = proxmox_offline_mirror::config::config(&config_file)?;
601
602 if config.sections.is_empty() {
603 println!("Initializing new config.");
604 } else {
605 println!("Loaded existing config.");
606 }
607
608 enum Action {
609 AddMirror,
610 AddMedium,
611 Quit,
612 }
613
614 loop {
615 println!();
616 let mut mirror_defined = false;
617 if !config.sections.is_empty() {
618 println!("Existing config entries:");
619 for (section, (section_type, _)) in config.sections.iter() {
620 if section_type == "mirror" {
621 mirror_defined = true;
622 }
623 println!("{section_type} '{section}'");
624 }
625 println!();
626 }
627
628 let actions = if mirror_defined {
629 vec![
630 (Action::AddMirror, "Add new mirror entry"),
631 (Action::AddMedium, "Add new medium entry"),
632 (Action::Quit, "Quit"),
633 ]
634 } else {
635 vec![
636 (Action::AddMirror, "Add new mirror entry"),
637 (Action::Quit, "Quit"),
638 ]
639 };
640
641 match read_selection_from_tty("Select Action:", &actions, Some(0))? {
642 Action::Quit => break,
643 Action::AddMirror => {
644 for mirror_config in action_add_mirror(&config)? {
645 let id = mirror_config.id.clone();
646 mirror::init(&mirror_config)?;
647 config.set_data(&id, "mirror", mirror_config)?;
648 save_config(&config_file, &config)?;
649 println!("Config entry '{id}' added");
650 println!("Run \"proxmox-offline-mirror mirror snapshot create --config '{config_file}' '{id}'\" to create a new mirror snapshot.");
651 }
652 }
653 Action::AddMedium => {
654 let media_config = action_add_medium(&config)?;
655 let id = media_config.id.clone();
656 config.set_data(&id, "medium", media_config)?;
657 save_config(&config_file, &config)?;
658 println!("Config entry '{id}' added");
659 println!("Run \"proxmox-offline-mirror medium sync --config '{config_file}' '{id}'\" to sync mirror snapshots to medium.");
660 }
661 }
662 }
663
664 Ok(())
665 }
666 fn main() {
667 let rpcenv = CliEnvironment::new();
668
669 let cmd_def = CliCommandMap::new()
670 .insert("setup", CliCommand::new(&API_METHOD_SETUP))
671 .insert("config", config_commands())
672 .insert("key", key_commands())
673 .insert("medium", medium_commands())
674 .insert("mirror", mirror_commands());
675
676 run_cli_command(
677 cmd_def,
678 rpcenv,
679 Some(|future| proxmox_async::runtime::main(future)),
680 );
681 }