src/deb822/release_file.rs

   1 use std::collections::HashMap;
   2
   3 use anyhow::{bail, format_err, Error};
   4 use rfc822_like::de::Deserializer;
   5 use serde::Deserialize;
   6 use serde_json::Value;
   7
   8 use super::CheckSums;
   9
  10 #[derive(Debug, Deserialize)]
  11 #[serde(rename_all = "PascalCase")]
  12 pub struct ReleaseFileRaw {
  13     pub architectures: Option<String>,
  14     pub changelogs: Option<String>,
  15     pub codename: Option<String>,
  16     pub components: Option<String>,
  17     pub date: Option<String>,
  18     pub description: Option<String>,
  19     pub label: Option<String>,
  20     pub origin: Option<String>,
  21     pub suite: Option<String>,
  22     pub version: Option<String>,
  23
  24     #[serde(rename = "MD5Sum")]
  25     pub md5_sum: Option<String>,
  26     #[serde(rename = "SHA1")]
  27     pub sha1: Option<String>,
  28     #[serde(rename = "SHA256")]
  29     pub sha256: Option<String>,
  30     #[serde(rename = "SHA512")]
  31     pub sha512: Option<String>,
  32
  33     #[serde(flatten)]
  34     pub extra_fields: HashMap<String, Value>,
  35 }
  36
  37 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
  38 pub enum CompressionType {
  39     Bzip2,
  40     Gzip,
  41     Lzma,
  42     Xz,
  43 }
  44
  45 pub type Architecture = String;
  46 pub type Component = String;
  47
  48 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
  49 /// Type of file reference extraced from path.
  50 ///
  51 /// `Packages` and `Sources` will contain further reference to binary or source package files.
  52 ///  These are handled in `PackagesFile` and `SourcesFile` respectively.
  53 pub enum FileReferenceType {
  54     /// A `Commands` index listing command to package mappings
  55     Commands(Architecture, Option<CompressionType>),
  56     /// A `Contents` index listing contents of binary packages
  57     Contents(Architecture, Option<CompressionType>),
  58     /// A `Contents` index listing contents of binary udeb packages
  59     ContentsUdeb(Architecture, Option<CompressionType>),
  60     /// A DEP11 `Components` metadata file or `icons` archive
  61     Dep11(Option<CompressionType>),
  62     /// Referenced files which are not really part of the APT repository but only signed for trust-anchor reasons
  63     Ignored,
  64     /// PDiff indices
  65     PDiff,
  66     /// A `Packages` index listing binary package metadata and references
  67     Packages(Architecture, Option<CompressionType>),
  68     /// A compat `Release` file with no relevant content
  69     PseudoRelease(Option<Architecture>),
  70     /// A `Sources` index listing source package metadata and references
  71     Sources(Option<CompressionType>),
  72     /// A `Translation` file
  73     Translation(Option<CompressionType>),
  74     /// Unknown file reference
  75     Unknown,
  76 }
  77
  78 impl FileReferenceType {
  79     fn match_compression(value: &str) -> Result<Option<CompressionType>, Error> {
  80         if value.is_empty() {
  81             return Ok(None);
  82         }
  83
  84         let value = if let Some(stripped) = value.strip_prefix('.') {
  85             stripped
  86         } else {
  87             value
  88         };
  89
  90         match value {
  91             "bz2" => Ok(Some(CompressionType::Bzip2)),
  92             "gz" => Ok(Some(CompressionType::Gzip)),
  93             "lzma" => Ok(Some(CompressionType::Lzma)),
  94             "xz" => Ok(Some(CompressionType::Xz)),
  95             other => bail!("Unexpected file extension '{other}'."),
  96         }
  97     }
  98
  99     pub fn parse(component: &str, path: &str) -> Result<FileReferenceType, Error> {
 100         // everything referenced in a release file should be component-specific
 101         let rest = path
 102             .strip_prefix(&format!("{component}/"))
 103             .ok_or_else(|| format_err!("Doesn't start with component '{component}'"))?;
 104
 105         let parse_binary_dir =
 106             |file_name: &str, arch: &str| parse_binary_dir(file_name, arch, path);
 107
 108         if let Some((dir, rest)) = rest.split_once('/') {
 109             // reference into another subdir
 110             match dir {
 111                 "source" => {
 112                     // Sources or compat-Release
 113                     if let Some((dir, _rest)) = rest.split_once('/') {
 114                         if dir == "Sources.diff" {
 115                             Ok(FileReferenceType::PDiff)
 116                         } else {
 117                             Ok(FileReferenceType::Unknown)
 118                         }
 119                     } else if rest == "Release" {
 120                         Ok(FileReferenceType::PseudoRelease(None))
 121                     } else if let Some(ext) = rest.strip_prefix("Sources") {
 122                         let comp = FileReferenceType::match_compression(ext)?;
 123                         Ok(FileReferenceType::Sources(comp))
 124                     } else {
 125                         Ok(FileReferenceType::Unknown)
 126                     }
 127                 }
 128                 "cnf" => {
 129                     if let Some(rest) = rest.strip_prefix("Commands-") {
 130                         if let Some((arch, ext)) = rest.rsplit_once('.') {
 131                             Ok(FileReferenceType::Commands(
 132                                 arch.to_owned(),
 133                                 FileReferenceType::match_compression(ext).ok().flatten(),
 134                             ))
 135                         } else {
 136                             Ok(FileReferenceType::Commands(rest.to_owned(), None))
 137                         }
 138                     } else {
 139                         Ok(FileReferenceType::Unknown)
 140                     }
 141                 },
 142                 "dep11" => {
 143                     if let Some((_path, ext)) = rest.rsplit_once('.') {
 144                         Ok(FileReferenceType::Dep11(
 145                             FileReferenceType::match_compression(ext).ok().flatten(),
 146                         ))
 147                     } else {
 148                         Ok(FileReferenceType::Dep11(None))
 149                     }
 150                 }
 151                 "debian-installer" => {
 152                     // another layer, then like regular repo but pointing at udebs
 153                     if let Some((dir, rest)) = rest.split_once('/') {
 154                         if let Some(arch) = dir.strip_prefix("binary-") {
 155                             // Packages or compat-Release
 156                             return parse_binary_dir(rest, arch);
 157                         }
 158                     }
 159
 160                     // all the rest
 161                     Ok(FileReferenceType::Unknown)
 162                 }
 163                 "i18n" => {
 164                     if let Some((dir, _rest)) = rest.split_once('/') {
 165                         if dir.starts_with("Translation") && dir.ends_with(".diff") {
 166                             Ok(FileReferenceType::PDiff)
 167                         } else {
 168                             Ok(FileReferenceType::Unknown)
 169                         }
 170                     } else if let Some((_, ext)) = rest.split_once('.') {
 171                         Ok(FileReferenceType::Translation(
 172                             FileReferenceType::match_compression(ext)?,
 173                         ))
 174                     } else {
 175                         Ok(FileReferenceType::Translation(None))
 176                     }
 177                 }
 178                 _ => {
 179                     if let Some(arch) = dir.strip_prefix("binary-") {
 180                         // Packages or compat-Release
 181                         parse_binary_dir(rest, arch)
 182                     } else if let Some(_arch) = dir.strip_prefix("installer-") {
 183                         // netboot installer checksum files
 184                         Ok(FileReferenceType::Ignored)
 185                     } else {
 186                         // all the rest
 187                         Ok(FileReferenceType::Unknown)
 188                     }
 189                 }
 190             }
 191         } else if let Some(rest) = rest.strip_prefix("Contents-") {
 192             // reference to a top-level file - Contents-*
 193             let (rest, udeb) = if let Some(rest) = rest.strip_prefix("udeb-") {
 194                 (rest, true)
 195             } else {
 196                 (rest, false)
 197             };
 198             let (arch, comp) = match rest.split_once('.') {
 199                 Some((arch, comp_str)) => (
 200                     arch.to_owned(),
 201                     FileReferenceType::match_compression(comp_str)?,
 202                 ),
 203                 None => (rest.to_owned(), None),
 204             };
 205             if udeb {
 206                 Ok(FileReferenceType::ContentsUdeb(arch, comp))
 207             } else {
 208                 Ok(FileReferenceType::Contents(arch, comp))
 209             }
 210         } else {
 211             Ok(FileReferenceType::Unknown)
 212         }
 213     }
 214
 215     pub fn compression(&self) -> Option<CompressionType> {
 216         match *self {
 217             FileReferenceType::Commands(_, comp)
 218             | FileReferenceType::Contents(_, comp)
 219             | FileReferenceType::ContentsUdeb(_, comp)
 220             | FileReferenceType::Packages(_, comp)
 221             | FileReferenceType::Sources(comp)
 222             | FileReferenceType::Translation(comp)
 223             | FileReferenceType::Dep11(comp) => comp,
 224             FileReferenceType::Unknown
 225             | FileReferenceType::PDiff
 226             | FileReferenceType::PseudoRelease(_)
 227             | FileReferenceType::Ignored => None,
 228         }
 229     }
 230
 231     pub fn architecture(&self) -> Option<&Architecture> {
 232         match self {
 233             FileReferenceType::Commands(arch, _)
 234             | FileReferenceType::Contents(arch, _)
 235             | FileReferenceType::ContentsUdeb(arch, _)
 236             | FileReferenceType::Packages(arch, _) => Some(arch),
 237             FileReferenceType::PseudoRelease(arch) => arch.as_ref(),
 238             FileReferenceType::Unknown
 239             | FileReferenceType::PDiff
 240             | FileReferenceType::Sources(_)
 241             | FileReferenceType::Dep11(_)
 242             | FileReferenceType::Translation(_)
 243             | FileReferenceType::Ignored => None,
 244         }
 245     }
 246
 247     pub fn is_package_index(&self) -> bool {
 248         matches!(self, FileReferenceType::Packages(_, _))
 249     }
 250 }
 251
 252 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 253 pub struct FileReference {
 254     pub path: String,
 255     pub size: usize,
 256     pub checksums: CheckSums,
 257     pub component: Component,
 258     pub file_type: FileReferenceType,
 259 }
 260
 261 impl FileReference {
 262     pub fn basename(&self) -> Result<String, Error> {
 263         match self.file_type.compression() {
 264             Some(_) => {
 265                 let (base, _ext) = self
 266                     .path
 267                     .rsplit_once('.')
 268                     .ok_or_else(|| format_err!("compressed file without file extension"))?;
 269                 Ok(base.to_owned())
 270             }
 271             None => Ok(self.path.clone()),
 272         }
 273     }
 274 }
 275
 276 #[derive(Debug, Default, PartialEq, Eq)]
 277 /// A parsed representation of a Release file
 278 pub struct ReleaseFile {
 279     /// List of architectures, e.g., `amd64` or `all`.
 280     pub architectures: Vec<String>,
 281     // TODO No-Support-for-Architecture-all
 282     /// URL for changelog queries via `apt changelog`.
 283     pub changelogs: Option<String>,
 284     /// Release codename - single word, e.g., `bullseye`.
 285     pub codename: Option<String>,
 286     /// List of repository areas, e.g., `main`.
 287     pub components: Vec<String>,
 288     /// UTC timestamp of release file generation
 289     pub date: Option<u64>,
 290     /// UTC timestamp of release file expiration
 291     pub valid_until: Option<u64>,
 292     /// Repository description -
 293     // TODO exact format?
 294     pub description: Option<String>,
 295     /// Repository label - single line
 296     pub label: Option<String>,
 297     /// Repository origin - single line
 298     pub origin: Option<String>,
 299     /// Release suite - single word, e.g., `stable`.
 300     pub suite: Option<String>,
 301     /// Release version
 302     pub version: Option<String>,
 303
 304     /// Whether by-hash retrieval of referenced files is possible
 305     pub aquire_by_hash: bool,
 306
 307     /// Files referenced by this `Release` file, e.g., packages indices.
 308     ///
 309     /// Grouped by basename, since only the compressed version needs to actually exist on the repository server.
 310     pub files: HashMap<String, Vec<FileReference>>,
 311 }
 312
 313 impl TryFrom<ReleaseFileRaw> for ReleaseFile {
 314     type Error = Error;
 315
 316     fn try_from(value: ReleaseFileRaw) -> Result<Self, Self::Error> {
 317         let mut parsed = ReleaseFile::default();
 318
 319         parsed.architectures = whitespace_split_to_vec(
 320             &value
 321                 .architectures
 322                 .ok_or_else(|| format_err!("'Architectures' field missing."))?,
 323         );
 324         parsed.components = whitespace_split_to_vec(
 325             &value
 326                 .components
 327                 .ok_or_else(|| format_err!("'Components' field missing."))?,
 328         );
 329
 330         parsed.changelogs = value.changelogs;
 331         parsed.codename = value.codename;
 332
 333         parsed.date = value.date.as_deref().map(parse_date);
 334         parsed.valid_until = value
 335             .extra_fields
 336             .get("Valid-Until")
 337             .map(|val| parse_date(&val.to_string()));
 338
 339         parsed.description = value.description;
 340         parsed.label = value.label;
 341         parsed.origin = value.origin;
 342         parsed.suite = value.suite;
 343         parsed.version = value.version;
 344
 345         parsed.aquire_by_hash = match value.extra_fields.get("Acquire-By-Hash") {
 346             Some(val) => *val == "yes",
 347             None => false,
 348         };
 349
 350         // Fixup bullseye-security release files which have invalid components
 351         if parsed.label.as_deref() == Some("Debian-Security")
 352             && parsed.codename.as_deref() == Some("bullseye-security")
 353         {
 354             parsed.components = parsed
 355                 .components
 356                 .into_iter()
 357                 .map(|comp| {
 358                     if let Some(stripped) = comp.strip_prefix("updates/") {
 359                         stripped.to_owned()
 360                     } else {
 361                         comp
 362                     }
 363                 })
 364                 .collect();
 365         }
 366
 367         let mut references_map: HashMap<String, HashMap<String, FileReference>> = HashMap::new();
 368
 369         if let Some(md5) = value.md5_sum {
 370             for line in md5.lines() {
 371                 let (mut file_ref, checksum) =
 372                     parse_file_reference(line, 16, parsed.components.as_ref())?;
 373
 374                 let checksum = checksum
 375                     .try_into()
 376                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 377
 378                 file_ref.checksums.md5 = Some(checksum);
 379
 380                 merge_references(&mut references_map, file_ref)?;
 381             }
 382         }
 383
 384         if let Some(sha1) = value.sha1 {
 385             for line in sha1.lines() {
 386                 let (mut file_ref, checksum) =
 387                     parse_file_reference(line, 20, parsed.components.as_ref())?;
 388                 let checksum = checksum
 389                     .try_into()
 390                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 391
 392                 file_ref.checksums.sha1 = Some(checksum);
 393                 merge_references(&mut references_map, file_ref)?;
 394             }
 395         }
 396
 397         if let Some(sha256) = value.sha256 {
 398             for line in sha256.lines() {
 399                 let (mut file_ref, checksum) =
 400                     parse_file_reference(line, 32, parsed.components.as_ref())?;
 401                 let checksum = checksum
 402                     .try_into()
 403                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 404
 405                 file_ref.checksums.sha256 = Some(checksum);
 406                 merge_references(&mut references_map, file_ref)?;
 407             }
 408         }
 409
 410         if let Some(sha512) = value.sha512 {
 411             for line in sha512.lines() {
 412                 let (mut file_ref, checksum) =
 413                     parse_file_reference(line, 64, parsed.components.as_ref())?;
 414                 let checksum = checksum
 415                     .try_into()
 416                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 417
 418                 file_ref.checksums.sha512 = Some(checksum);
 419                 merge_references(&mut references_map, file_ref)?;
 420             }
 421         }
 422
 423         parsed.files =
 424             references_map
 425                 .into_iter()
 426                 .fold(HashMap::new(), |mut map, (base, inner_map)| {
 427                     map.insert(base, inner_map.into_values().collect());
 428                     map
 429                 });
 430
 431         if let Some(insecure) = parsed
 432             .files
 433             .values()
 434             .flatten()
 435             .find(|file| !file.checksums.is_secure())
 436         {
 437             bail!(
 438                 "found file reference without strong checksum: {}",
 439                 insecure.path
 440             );
 441         }
 442
 443         Ok(parsed)
 444     }
 445 }
 446
 447 impl TryFrom<String> for ReleaseFile {
 448     type Error = Error;
 449
 450     fn try_from(value: String) -> Result<Self, Self::Error> {
 451         value.as_bytes().try_into()
 452     }
 453 }
 454
 455 impl TryFrom<&[u8]> for ReleaseFile {
 456     type Error = Error;
 457
 458     fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
 459         let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(value))?;
 460         deserialized.try_into()
 461     }
 462 }
 463
 464 fn whitespace_split_to_vec(list_str: &str) -> Vec<String> {
 465     list_str
 466         .split_ascii_whitespace()
 467         .map(|arch| arch.to_owned())
 468         .collect()
 469 }
 470
 471 fn parse_file_reference(
 472     line: &str,
 473     csum_len: usize,
 474     components: &[String],
 475 ) -> Result<(FileReference, Vec<u8>), Error> {
 476     let mut split = line.split_ascii_whitespace();
 477     let checksum = split
 478         .next()
 479         .ok_or_else(|| format_err!("No 'checksum' field in the file reference line."))?;
 480     if checksum.len() > csum_len * 2 {
 481         bail!(
 482             "invalid checksum length: '{}', expected {} bytes",
 483             checksum,
 484             csum_len
 485         );
 486     }
 487
 488     let checksum = hex::decode(checksum)?;
 489
 490     let size = split
 491         .next()
 492         .ok_or_else(|| format_err!("No 'size' field in file reference line."))?
 493         .parse::<usize>()?;
 494
 495     let file = split
 496         .next()
 497         .ok_or_else(|| format_err!("No 'path' field in file reference line."))?
 498         .to_string();
 499
 500     let (component, file_type) = components
 501         .iter()
 502         .find_map(|component| {
 503             if !file.starts_with(&format!("{component}/")) {
 504                 return None;
 505             }
 506
 507             Some(
 508                 FileReferenceType::parse(component, &file)
 509                     .map(|file_type| (component.clone(), file_type)),
 510             )
 511         })
 512         .unwrap_or_else(|| Ok(("UNKNOWN".to_string(), FileReferenceType::Unknown)))?;
 513
 514     Ok((
 515         FileReference {
 516             path: file,
 517             size,
 518             checksums: CheckSums::default(),
 519             component,
 520             file_type,
 521         },
 522         checksum,
 523     ))
 524 }
 525
 526 fn parse_date(_date_str: &str) -> u64 {
 527     // TODO implement
 528     0
 529 }
 530
 531 fn parse_binary_dir(file_name: &str, arch: &str, path: &str) -> Result<FileReferenceType, Error> {
 532     if let Some((dir, _rest)) = file_name.split_once('/') {
 533         if dir == "Packages.diff" {
 534             // TODO re-evaluate?
 535             Ok(FileReferenceType::PDiff)
 536         } else {
 537             Ok(FileReferenceType::Unknown)
 538         }
 539     } else if file_name == "Release" {
 540         Ok(FileReferenceType::PseudoRelease(Some(arch.to_owned())))
 541     } else {
 542         let comp = match file_name.strip_prefix("Packages") {
 543             None => {
 544                 bail!("found unexpected non-Packages reference to '{path}'")
 545             }
 546             Some(ext) => FileReferenceType::match_compression(ext)?,
 547         };
 548         //println!("compression: {comp:?}");
 549         Ok(FileReferenceType::Packages(arch.to_owned(), comp))
 550     }
 551 }
 552
 553 fn merge_references(
 554     base_map: &mut HashMap<String, HashMap<String, FileReference>>,
 555     file_ref: FileReference,
 556 ) -> Result<(), Error> {
 557     let base = file_ref.basename()?;
 558
 559     match base_map.get_mut(&base) {
 560         None => {
 561             let mut map = HashMap::new();
 562             map.insert(file_ref.path.clone(), file_ref);
 563             base_map.insert(base, map);
 564         }
 565         Some(entries) => {
 566             match entries.get_mut(&file_ref.path) {
 567                 Some(entry) => {
 568                     if entry.size != file_ref.size {
 569                         bail!(
 570                             "Multiple entries for '{}' with size mismatch: {} / {}",
 571                             entry.path,
 572                             file_ref.size,
 573                             entry.size
 574                         );
 575                     }
 576
 577                     entry.checksums.merge(&file_ref.checksums).map_err(|err| {
 578                         format_err!("Multiple checksums for '{}' - {err}", entry.path)
 579                     })?;
 580                 }
 581                 None => {
 582                     entries.insert(file_ref.path.clone(), file_ref);
 583                 }
 584             };
 585         }
 586     };
 587
 588     Ok(())
 589 }
 590
 591 #[test]
 592 pub fn test_deb_release_file() {
 593     let input = include_str!(concat!(
 594         env!("CARGO_MANIFEST_DIR"),
 595         "/tests/deb822/release/deb.debian.org_debian_dists_bullseye_Release"
 596     ));
 597
 598     let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(input.as_bytes())).unwrap();
 599     //println!("{:?}", deserialized);
 600
 601     let parsed: ReleaseFile = deserialized.try_into().unwrap();
 602     //println!("{:?}", parsed);
 603
 604     assert_eq!(parsed.files.len(), 315);
 605 }
 606
 607 #[test]
 608 pub fn test_deb_release_file_insecure() {
 609     let input = include_str!(concat!(
 610         env!("CARGO_MANIFEST_DIR"),
 611         "/tests/deb822/release/deb.debian.org_debian_dists_bullseye_Release_insecure"
 612     ));
 613
 614     let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(input.as_bytes())).unwrap();
 615     //println!("{:?}", deserialized);
 616
 617     let parsed: Result<ReleaseFile, Error> = deserialized.try_into();
 618     assert!(parsed.is_err());
 619
 620     println!("{:?}", parsed);
 621 }