]>
Commit | Line | Data |
---|---|---|
8cdd2311 FG |
1 | use std::collections::HashMap; |
2 | ||
3 | use anyhow::{bail, format_err, Error}; | |
4 | use rfc822_like::de::Deserializer; | |
5 | use serde::Deserialize; | |
6 | use serde_json::Value; | |
7 | ||
8 | use super::CheckSums; | |
9 | ||
10 | #[derive(Debug, Deserialize)] | |
11 | #[serde(rename_all = "PascalCase")] | |
12 | pub struct ReleaseFileRaw { | |
13 | pub architectures: Option<String>, | |
14 | pub changelogs: Option<String>, | |
15 | pub codename: Option<String>, | |
16 | pub components: Option<String>, | |
17 | pub date: Option<String>, | |
18 | pub description: Option<String>, | |
19 | pub label: Option<String>, | |
20 | pub origin: Option<String>, | |
21 | pub suite: Option<String>, | |
22 | pub version: Option<String>, | |
23 | ||
24 | #[serde(rename = "MD5Sum")] | |
25 | pub md5_sum: Option<String>, | |
26 | #[serde(rename = "SHA1")] | |
27 | pub sha1: Option<String>, | |
28 | #[serde(rename = "SHA256")] | |
29 | pub sha256: Option<String>, | |
30 | #[serde(rename = "SHA512")] | |
31 | pub sha512: Option<String>, | |
32 | ||
33 | #[serde(flatten)] | |
34 | pub extra_fields: HashMap<String, Value>, | |
35 | } | |
36 | ||
37 | #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] | |
38 | pub enum CompressionType { | |
39 | Bzip2, | |
40 | Gzip, | |
41 | Lzma, | |
42 | Xz, | |
43 | } | |
44 | ||
45 | pub type Architecture = String; | |
46 | pub type Component = String; | |
47 | ||
48 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] | |
49 | /// Type of file reference extraced from path. | |
50 | /// | |
51 | /// `Packages` and `Sources` will contain further reference to binary or source package files. | |
52 | /// These are handled in `PackagesFile` and `SourcesFile` respectively. | |
53 | pub enum FileReferenceType { | |
54 | /// A `Contents` index listing contents of binary packages | |
55 | Contents(Architecture, Option<CompressionType>), | |
56 | /// A `Contents` index listing contents of binary udeb packages | |
57 | ContentsUdeb(Architecture, Option<CompressionType>), | |
58 | /// A DEP11 `Components` metadata file or `icons` archive | |
59 | Dep11(Option<CompressionType>), | |
60 | /// Referenced files which are not really part of the APT repository but only signed for trust-anchor reasons | |
61 | Ignored, | |
62 | /// PDiff indices | |
63 | PDiff, | |
64 | /// A `Packages` index listing binary package metadata and references | |
65 | Packages(Architecture, Option<CompressionType>), | |
66 | /// A compat `Release` file with no relevant content | |
67 | PseudoRelease(Option<Architecture>), | |
68 | /// A `Sources` index listing source package metadata and references | |
69 | Sources(Option<CompressionType>), | |
70 | /// A `Translation` file | |
71 | Translation(Option<CompressionType>), | |
72 | /// Unknown file reference | |
73 | Unknown, | |
74 | } | |
75 | ||
76 | impl FileReferenceType { | |
77 | fn match_compression(value: &str) -> Result<Option<CompressionType>, Error> { | |
78 | if value.is_empty() { | |
79 | return Ok(None); | |
80 | } | |
81 | ||
82 | let value = if let Some(stripped) = value.strip_prefix('.') { | |
83 | stripped | |
84 | } else { | |
85 | value | |
86 | }; | |
87 | ||
88 | match value { | |
89 | "bz2" => Ok(Some(CompressionType::Bzip2)), | |
90 | "gz" => Ok(Some(CompressionType::Gzip)), | |
91 | "lzma" => Ok(Some(CompressionType::Lzma)), | |
92 | "xz" => Ok(Some(CompressionType::Xz)), | |
93 | other => bail!("Unexpected file extension '{other}'."), | |
94 | } | |
95 | } | |
26e2bce4 | 96 | |
8cdd2311 FG |
97 | pub fn parse(component: &str, path: &str) -> Result<FileReferenceType, Error> { |
98 | // everything referenced in a release file should be component-specific | |
99 | let rest = path | |
100 | .strip_prefix(&format!("{component}/")) | |
101 | .ok_or_else(|| format_err!("Doesn't start with component '{component}'"))?; | |
102 | ||
0e845221 WB |
103 | let parse_binary_dir = |
104 | |file_name: &str, arch: &str| parse_binary_dir(file_name, arch, path); | |
8cdd2311 | 105 | |
72dc88fb | 106 | if let Some((dir, rest)) = rest.split_once('/') { |
8cdd2311 FG |
107 | // reference into another subdir |
108 | match dir { | |
109 | "source" => { | |
110 | // Sources or compat-Release | |
72dc88fb | 111 | if let Some((dir, _rest)) = rest.split_once('/') { |
8cdd2311 FG |
112 | if dir == "Sources.diff" { |
113 | Ok(FileReferenceType::PDiff) | |
114 | } else { | |
115 | Ok(FileReferenceType::Unknown) | |
116 | } | |
117 | } else if rest == "Release" { | |
118 | Ok(FileReferenceType::PseudoRelease(None)) | |
119 | } else if let Some(ext) = rest.strip_prefix("Sources") { | |
120 | let comp = FileReferenceType::match_compression(ext)?; | |
121 | Ok(FileReferenceType::Sources(comp)) | |
122 | } else { | |
123 | Ok(FileReferenceType::Unknown) | |
124 | } | |
125 | } | |
126 | "dep11" => { | |
127 | if let Some((_path, ext)) = rest.rsplit_once('.') { | |
128 | Ok(FileReferenceType::Dep11( | |
129 | FileReferenceType::match_compression(ext).ok().flatten(), | |
130 | )) | |
131 | } else { | |
132 | Ok(FileReferenceType::Dep11(None)) | |
133 | } | |
134 | } | |
135 | "debian-installer" => { | |
136 | // another layer, then like regular repo but pointing at udebs | |
72dc88fb | 137 | if let Some((dir, rest)) = rest.split_once('/') { |
8cdd2311 FG |
138 | if let Some(arch) = dir.strip_prefix("binary-") { |
139 | // Packages or compat-Release | |
140 | return parse_binary_dir(rest, arch); | |
141 | } | |
142 | } | |
143 | ||
144 | // all the rest | |
145 | Ok(FileReferenceType::Unknown) | |
146 | } | |
147 | "i18n" => { | |
72dc88fb | 148 | if let Some((dir, _rest)) = rest.split_once('/') { |
8cdd2311 FG |
149 | if dir.starts_with("Translation") && dir.ends_with(".diff") { |
150 | Ok(FileReferenceType::PDiff) | |
151 | } else { | |
152 | Ok(FileReferenceType::Unknown) | |
153 | } | |
154 | } else if let Some((_, ext)) = rest.split_once('.') { | |
155 | Ok(FileReferenceType::Translation( | |
156 | FileReferenceType::match_compression(ext)?, | |
157 | )) | |
158 | } else { | |
159 | Ok(FileReferenceType::Translation(None)) | |
160 | } | |
161 | } | |
162 | _ => { | |
163 | if let Some(arch) = dir.strip_prefix("binary-") { | |
164 | // Packages or compat-Release | |
165 | parse_binary_dir(rest, arch) | |
166 | } else if let Some(_arch) = dir.strip_prefix("installer-") { | |
167 | // netboot installer checksum files | |
168 | Ok(FileReferenceType::Ignored) | |
169 | } else { | |
170 | // all the rest | |
171 | Ok(FileReferenceType::Unknown) | |
172 | } | |
173 | } | |
174 | } | |
175 | } else if let Some(rest) = rest.strip_prefix("Contents-") { | |
176 | // reference to a top-level file - Contents-* | |
177 | let (rest, udeb) = if let Some(rest) = rest.strip_prefix("udeb-") { | |
178 | (rest, true) | |
179 | } else { | |
180 | (rest, false) | |
181 | }; | |
72dc88fb | 182 | let (arch, comp) = match rest.split_once('.') { |
8cdd2311 FG |
183 | Some((arch, comp_str)) => ( |
184 | arch.to_owned(), | |
185 | FileReferenceType::match_compression(comp_str)?, | |
186 | ), | |
187 | None => (rest.to_owned(), None), | |
188 | }; | |
189 | if udeb { | |
190 | Ok(FileReferenceType::ContentsUdeb(arch, comp)) | |
191 | } else { | |
192 | Ok(FileReferenceType::Contents(arch, comp)) | |
193 | } | |
194 | } else { | |
195 | Ok(FileReferenceType::Unknown) | |
196 | } | |
197 | } | |
198 | ||
199 | pub fn compression(&self) -> Option<CompressionType> { | |
200 | match *self { | |
201 | FileReferenceType::Contents(_, comp) | |
202 | | FileReferenceType::ContentsUdeb(_, comp) | |
203 | | FileReferenceType::Packages(_, comp) | |
204 | | FileReferenceType::Sources(comp) | |
205 | | FileReferenceType::Translation(comp) | |
206 | | FileReferenceType::Dep11(comp) => comp, | |
207 | FileReferenceType::Unknown | |
208 | | FileReferenceType::PDiff | |
209 | | FileReferenceType::PseudoRelease(_) | |
210 | | FileReferenceType::Ignored => None, | |
211 | } | |
212 | } | |
213 | ||
214 | pub fn is_package_index(&self) -> bool { | |
72dc88fb | 215 | matches!(self, FileReferenceType::Packages(_, _)) |
8cdd2311 FG |
216 | } |
217 | } | |
218 | ||
219 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] | |
220 | pub struct FileReference { | |
221 | pub path: String, | |
222 | pub size: usize, | |
223 | pub checksums: CheckSums, | |
224 | pub component: Component, | |
225 | pub file_type: FileReferenceType, | |
226 | } | |
227 | ||
228 | impl FileReference { | |
229 | pub fn basename(&self) -> Result<String, Error> { | |
230 | match self.file_type.compression() { | |
231 | Some(_) => { | |
232 | let (base, _ext) = self | |
233 | .path | |
72dc88fb | 234 | .rsplit_once('.') |
8cdd2311 FG |
235 | .ok_or_else(|| format_err!("compressed file without file extension"))?; |
236 | Ok(base.to_owned()) | |
237 | } | |
238 | None => Ok(self.path.clone()), | |
239 | } | |
240 | } | |
241 | } | |
242 | ||
243 | #[derive(Debug, Default, PartialEq, Eq)] | |
244 | /// A parsed representation of a Release file | |
245 | pub struct ReleaseFile { | |
246 | /// List of architectures, e.g., `amd64` or `all`. | |
247 | pub architectures: Vec<String>, | |
248 | // TODO No-Support-for-Architecture-all | |
249 | /// URL for changelog queries via `apt changelog`. | |
250 | pub changelogs: Option<String>, | |
251 | /// Release codename - single word, e.g., `bullseye`. | |
252 | pub codename: Option<String>, | |
253 | /// List of repository areas, e.g., `main`. | |
254 | pub components: Vec<String>, | |
255 | /// UTC timestamp of release file generation | |
256 | pub date: Option<u64>, | |
257 | /// UTC timestamp of release file expiration | |
258 | pub valid_until: Option<u64>, | |
259 | /// Repository description - | |
260 | // TODO exact format? | |
261 | pub description: Option<String>, | |
262 | /// Repository label - single line | |
263 | pub label: Option<String>, | |
264 | /// Repository origin - single line | |
265 | pub origin: Option<String>, | |
266 | /// Release suite - single word, e.g., `stable`. | |
267 | pub suite: Option<String>, | |
268 | /// Release version | |
269 | pub version: Option<String>, | |
270 | ||
271 | /// Whether by-hash retrieval of referenced files is possible | |
272 | pub aquire_by_hash: bool, | |
273 | ||
274 | /// Files referenced by this `Release` file, e.g., packages indices. | |
275 | /// | |
276 | /// Grouped by basename, since only the compressed version needs to actually exist on the repository server. | |
277 | pub files: HashMap<String, Vec<FileReference>>, | |
278 | } | |
279 | ||
280 | impl TryFrom<ReleaseFileRaw> for ReleaseFile { | |
281 | type Error = Error; | |
282 | ||
283 | fn try_from(value: ReleaseFileRaw) -> Result<Self, Self::Error> { | |
284 | let mut parsed = ReleaseFile::default(); | |
285 | ||
0e845221 WB |
286 | parsed.architectures = whitespace_split_to_vec( |
287 | &value | |
8cdd2311 FG |
288 | .architectures |
289 | .ok_or_else(|| format_err!("'Architectures' field missing."))?, | |
290 | ); | |
0e845221 WB |
291 | parsed.components = whitespace_split_to_vec( |
292 | &value | |
8cdd2311 FG |
293 | .components |
294 | .ok_or_else(|| format_err!("'Components' field missing."))?, | |
295 | ); | |
296 | ||
297 | parsed.changelogs = value.changelogs; | |
298 | parsed.codename = value.codename; | |
299 | ||
0e845221 | 300 | parsed.date = value.date.as_deref().map(parse_date); |
8cdd2311 FG |
301 | parsed.valid_until = value |
302 | .extra_fields | |
303 | .get("Valid-Until") | |
0e845221 | 304 | .map(|val| parse_date(&val.to_string())); |
8cdd2311 FG |
305 | |
306 | parsed.description = value.description; | |
307 | parsed.label = value.label; | |
308 | parsed.origin = value.origin; | |
309 | parsed.suite = value.suite; | |
310 | parsed.version = value.version; | |
311 | ||
312 | parsed.aquire_by_hash = match value.extra_fields.get("Aquire-By-Hash") { | |
313 | Some(val) => *val == "yes", | |
314 | None => false, | |
315 | }; | |
316 | ||
317 | // Fixup bullseye-security release files which have invalid components | |
318 | if parsed.label.as_deref() == Some("Debian-Security") | |
319 | && parsed.codename.as_deref() == Some("bullseye-security") | |
320 | { | |
321 | parsed.components = parsed | |
322 | .components | |
323 | .into_iter() | |
324 | .map(|comp| { | |
325 | if let Some(stripped) = comp.strip_prefix("updates/") { | |
326 | stripped.to_owned() | |
327 | } else { | |
328 | comp | |
329 | } | |
330 | }) | |
331 | .collect(); | |
332 | } | |
333 | ||
334 | let mut references_map: HashMap<String, HashMap<String, FileReference>> = HashMap::new(); | |
335 | ||
8cdd2311 FG |
336 | if let Some(md5) = value.md5_sum { |
337 | for line in md5.lines() { | |
338 | let (mut file_ref, checksum) = | |
339 | parse_file_reference(line, 16, parsed.components.as_ref())?; | |
340 | ||
341 | let checksum = checksum | |
342 | .try_into() | |
343 | .map_err(|_err| format_err!("unexpected checksum length"))?; | |
344 | ||
345 | file_ref.checksums.md5 = Some(checksum); | |
346 | ||
347 | merge_references(&mut references_map, file_ref)?; | |
348 | } | |
349 | } | |
350 | ||
351 | if let Some(sha1) = value.sha1 { | |
352 | for line in sha1.lines() { | |
353 | let (mut file_ref, checksum) = | |
354 | parse_file_reference(line, 20, parsed.components.as_ref())?; | |
355 | let checksum = checksum | |
356 | .try_into() | |
357 | .map_err(|_err| format_err!("unexpected checksum length"))?; | |
358 | ||
359 | file_ref.checksums.sha1 = Some(checksum); | |
360 | merge_references(&mut references_map, file_ref)?; | |
361 | } | |
362 | } | |
363 | ||
364 | if let Some(sha256) = value.sha256 { | |
365 | for line in sha256.lines() { | |
366 | let (mut file_ref, checksum) = | |
367 | parse_file_reference(line, 32, parsed.components.as_ref())?; | |
368 | let checksum = checksum | |
369 | .try_into() | |
370 | .map_err(|_err| format_err!("unexpected checksum length"))?; | |
371 | ||
372 | file_ref.checksums.sha256 = Some(checksum); | |
373 | merge_references(&mut references_map, file_ref)?; | |
374 | } | |
375 | } | |
376 | ||
377 | if let Some(sha512) = value.sha512 { | |
378 | for line in sha512.lines() { | |
379 | let (mut file_ref, checksum) = | |
380 | parse_file_reference(line, 64, parsed.components.as_ref())?; | |
381 | let checksum = checksum | |
382 | .try_into() | |
383 | .map_err(|_err| format_err!("unexpected checksum length"))?; | |
384 | ||
385 | file_ref.checksums.sha512 = Some(checksum); | |
386 | merge_references(&mut references_map, file_ref)?; | |
387 | } | |
388 | } | |
389 | ||
390 | parsed.files = | |
391 | references_map | |
392 | .into_iter() | |
393 | .fold(HashMap::new(), |mut map, (base, inner_map)| { | |
394 | map.insert(base, inner_map.into_values().collect()); | |
395 | map | |
396 | }); | |
397 | ||
398 | if let Some(insecure) = parsed | |
399 | .files | |
400 | .values() | |
401 | .flatten() | |
402 | .find(|file| !file.checksums.is_secure()) | |
403 | { | |
404 | bail!( | |
405 | "found file reference without strong checksum: {}", | |
406 | insecure.path | |
407 | ); | |
408 | } | |
409 | ||
410 | Ok(parsed) | |
411 | } | |
412 | } | |
413 | ||
414 | impl TryFrom<String> for ReleaseFile { | |
415 | type Error = Error; | |
416 | ||
417 | fn try_from(value: String) -> Result<Self, Self::Error> { | |
418 | value.as_bytes().try_into() | |
419 | } | |
420 | } | |
421 | ||
422 | impl TryFrom<&[u8]> for ReleaseFile { | |
423 | type Error = Error; | |
424 | ||
425 | fn try_from(value: &[u8]) -> Result<Self, Self::Error> { | |
426 | let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(value))?; | |
427 | deserialized.try_into() | |
428 | } | |
429 | } | |
430 | ||
0e845221 WB |
431 | fn whitespace_split_to_vec(list_str: &str) -> Vec<String> { |
432 | list_str | |
433 | .split_ascii_whitespace() | |
434 | .map(|arch| arch.to_owned()) | |
435 | .collect() | |
436 | } | |
437 | ||
438 | fn parse_file_reference( | |
439 | line: &str, | |
440 | csum_len: usize, | |
441 | components: &[String], | |
442 | ) -> Result<(FileReference, Vec<u8>), Error> { | |
443 | let mut split = line.split_ascii_whitespace(); | |
26e2bce4 FG |
444 | let checksum = split |
445 | .next() | |
446 | .ok_or_else(|| format_err!("No 'checksum' field in the file reference line."))?; | |
0e845221 WB |
447 | if checksum.len() > csum_len * 2 { |
448 | bail!( | |
449 | "invalid checksum length: '{}', expected {} bytes", | |
450 | checksum, | |
451 | csum_len | |
452 | ); | |
453 | } | |
454 | ||
455 | let checksum = hex::decode(checksum)?; | |
456 | ||
457 | let size = split | |
458 | .next() | |
459 | .ok_or_else(|| format_err!("No 'size' field in file reference line."))? | |
460 | .parse::<usize>()?; | |
461 | ||
462 | let file = split | |
463 | .next() | |
464 | .ok_or_else(|| format_err!("No 'path' field in file reference line."))? | |
465 | .to_string(); | |
466 | ||
467 | let (component, file_type) = components | |
468 | .iter() | |
469 | .find_map(|component| { | |
470 | if !file.starts_with(&format!("{component}/")) { | |
471 | return None; | |
472 | } | |
473 | ||
474 | Some( | |
475 | FileReferenceType::parse(component, &file) | |
476 | .map(|file_type| (component.clone(), file_type)), | |
477 | ) | |
478 | }) | |
479 | .unwrap_or_else(|| Ok(("UNKNOWN".to_string(), FileReferenceType::Unknown)))?; | |
480 | ||
481 | Ok(( | |
482 | FileReference { | |
483 | path: file, | |
484 | size, | |
485 | checksums: CheckSums::default(), | |
486 | component, | |
487 | file_type, | |
488 | }, | |
489 | checksum, | |
490 | )) | |
491 | } | |
492 | ||
493 | fn parse_date(_date_str: &str) -> u64 { | |
494 | // TODO implement | |
495 | 0 | |
496 | } | |
497 | ||
498 | fn parse_binary_dir(file_name: &str, arch: &str, path: &str) -> Result<FileReferenceType, Error> { | |
499 | if let Some((dir, _rest)) = file_name.split_once('/') { | |
500 | if dir == "Packages.diff" { | |
501 | // TODO re-evaluate? | |
502 | Ok(FileReferenceType::PDiff) | |
503 | } else { | |
504 | Ok(FileReferenceType::Unknown) | |
505 | } | |
506 | } else if file_name == "Release" { | |
507 | Ok(FileReferenceType::PseudoRelease(Some(arch.to_owned()))) | |
508 | } else { | |
509 | let comp = match file_name.strip_prefix("Packages") { | |
510 | None => { | |
511 | bail!("found unexpected non-Packages reference to '{path}'") | |
512 | } | |
513 | Some(ext) => FileReferenceType::match_compression(ext)?, | |
514 | }; | |
515 | //println!("compression: {comp:?}"); | |
516 | Ok(FileReferenceType::Packages(arch.to_owned(), comp)) | |
517 | } | |
518 | } | |
519 | ||
520 | fn merge_references( | |
521 | base_map: &mut HashMap<String, HashMap<String, FileReference>>, | |
522 | file_ref: FileReference, | |
523 | ) -> Result<(), Error> { | |
524 | let base = file_ref.basename()?; | |
525 | ||
526 | match base_map.get_mut(&base) { | |
527 | None => { | |
528 | let mut map = HashMap::new(); | |
529 | map.insert(file_ref.path.clone(), file_ref); | |
530 | base_map.insert(base, map); | |
531 | } | |
532 | Some(entries) => { | |
533 | match entries.get_mut(&file_ref.path) { | |
534 | Some(entry) => { | |
535 | if entry.size != file_ref.size { | |
536 | bail!( | |
537 | "Multiple entries for '{}' with size mismatch: {} / {}", | |
538 | entry.path, | |
539 | file_ref.size, | |
540 | entry.size | |
541 | ); | |
542 | } | |
543 | ||
544 | entry.checksums.merge(&file_ref.checksums).map_err(|err| { | |
545 | format_err!("Multiple checksums for '{}' - {err}", entry.path) | |
546 | })?; | |
547 | } | |
548 | None => { | |
549 | entries.insert(file_ref.path.clone(), file_ref); | |
550 | } | |
551 | }; | |
552 | } | |
553 | }; | |
554 | ||
555 | Ok(()) | |
556 | } | |
557 | ||
8cdd2311 FG |
558 | #[test] |
559 | pub fn test_deb_release_file() { | |
560 | let input = include_str!(concat!( | |
561 | env!("CARGO_MANIFEST_DIR"), | |
562 | "/tests/deb822/release/deb.debian.org_debian_dists_bullseye_Release" | |
563 | )); | |
564 | ||
565 | let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(input.as_bytes())).unwrap(); | |
566 | //println!("{:?}", deserialized); | |
567 | ||
568 | let parsed: ReleaseFile = deserialized.try_into().unwrap(); | |
569 | //println!("{:?}", parsed); | |
570 | ||
571 | assert_eq!(parsed.files.len(), 315); | |
572 | } | |
573 | ||
574 | #[test] | |
575 | pub fn test_deb_release_file_insecure() { | |
576 | let input = include_str!(concat!( | |
577 | env!("CARGO_MANIFEST_DIR"), | |
578 | "/tests/deb822/release/deb.debian.org_debian_dists_bullseye_Release_insecure" | |
579 | )); | |
580 | ||
581 | let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(input.as_bytes())).unwrap(); | |
582 | //println!("{:?}", deserialized); | |
583 | ||
584 | let parsed: Result<ReleaseFile, Error> = deserialized.try_into(); | |
585 | assert!(parsed.is_err()); | |
586 | ||
587 | println!("{:?}", parsed); | |
588 | } |