src/deb822/release_file.rs

   1 use std::collections::HashMap;
   2
   3 use anyhow::{bail, format_err, Error};
   4 use rfc822_like::de::Deserializer;
   5 use serde::Deserialize;
   6 use serde_json::Value;
   7
   8 use super::CheckSums;
   9
  10 #[derive(Debug, Deserialize)]
  11 #[serde(rename_all = "PascalCase")]
  12 pub struct ReleaseFileRaw {
  13     pub architectures: Option<String>,
  14     pub changelogs: Option<String>,
  15     pub codename: Option<String>,
  16     pub components: Option<String>,
  17     pub date: Option<String>,
  18     pub description: Option<String>,
  19     pub label: Option<String>,
  20     pub origin: Option<String>,
  21     pub suite: Option<String>,
  22     pub version: Option<String>,
  23
  24     #[serde(rename = "MD5Sum")]
  25     pub md5_sum: Option<String>,
  26     #[serde(rename = "SHA1")]
  27     pub sha1: Option<String>,
  28     #[serde(rename = "SHA256")]
  29     pub sha256: Option<String>,
  30     #[serde(rename = "SHA512")]
  31     pub sha512: Option<String>,
  32
  33     #[serde(flatten)]
  34     pub extra_fields: HashMap<String, Value>,
  35 }
  36
  37 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
  38 pub enum CompressionType {
  39     Bzip2,
  40     Gzip,
  41     Lzma,
  42     Xz,
  43 }
  44
  45 pub type Architecture = String;
  46 pub type Component = String;
  47
  48 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
  49 /// Type of file reference extraced from path.
  50 ///
  51 /// `Packages` and `Sources` will contain further reference to binary or source package files.
  52 ///  These are handled in `PackagesFile` and `SourcesFile` respectively.
  53 pub enum FileReferenceType {
  54     /// A `Commands` index listing command to package mappings
  55     Commands(Architecture, Option<CompressionType>),
  56     /// A `Contents` index listing contents of binary packages
  57     Contents(Architecture, Option<CompressionType>),
  58     /// A `Contents` index listing contents of binary udeb packages
  59     ContentsUdeb(Architecture, Option<CompressionType>),
  60     /// A DEP11 `Components` metadata file or `icons` archive
  61     Dep11(Option<CompressionType>),
  62     /// Referenced files which are not really part of the APT repository but only signed for trust-anchor reasons
  63     Ignored,
  64     /// PDiff indices
  65     PDiff,
  66     /// A `Packages` index listing binary package metadata and references
  67     Packages(Architecture, Option<CompressionType>),
  68     /// A compat `Release` file with no relevant content
  69     PseudoRelease(Option<Architecture>),
  70     /// A `Sources` index listing source package metadata and references
  71     Sources(Option<CompressionType>),
  72     /// A `Translation` file
  73     Translation(Option<CompressionType>),
  74     /// Unknown file reference
  75     Unknown,
  76 }
  77
  78 impl FileReferenceType {
  79     fn match_compression(value: &str) -> Result<Option<CompressionType>, Error> {
  80         if value.is_empty() {
  81             return Ok(None);
  82         }
  83
  84         let value = if let Some(stripped) = value.strip_prefix('.') {
  85             stripped
  86         } else {
  87             value
  88         };
  89
  90         match value {
  91             "bz2" => Ok(Some(CompressionType::Bzip2)),
  92             "gz" => Ok(Some(CompressionType::Gzip)),
  93             "lzma" => Ok(Some(CompressionType::Lzma)),
  94             "xz" => Ok(Some(CompressionType::Xz)),
  95             other => bail!("Unexpected file extension '{other}'."),
  96         }
  97     }
  98
  99     pub fn parse(component: &str, path: &str) -> Result<FileReferenceType, Error> {
 100         // everything referenced in a release file should be component-specific
 101         let rest = path
 102             .strip_prefix(&format!("{component}/"))
 103             .ok_or_else(|| format_err!("Doesn't start with component '{component}'"))?;
 104
 105         let parse_binary_dir =
 106             |file_name: &str, arch: &str| parse_binary_dir(file_name, arch, path);
 107
 108         if let Some((dir, rest)) = rest.split_once('/') {
 109             // reference into another subdir
 110             match dir {
 111                 "source" => {
 112                     // Sources or compat-Release
 113                     if let Some((dir, _rest)) = rest.split_once('/') {
 114                         if dir == "Sources.diff" {
 115                             Ok(FileReferenceType::PDiff)
 116                         } else {
 117                             Ok(FileReferenceType::Unknown)
 118                         }
 119                     } else if rest == "Release" {
 120                         Ok(FileReferenceType::PseudoRelease(None))
 121                     } else if let Some(ext) = rest.strip_prefix("Sources") {
 122                         let comp = FileReferenceType::match_compression(ext)?;
 123                         Ok(FileReferenceType::Sources(comp))
 124                     } else {
 125                         Ok(FileReferenceType::Unknown)
 126                     }
 127                 }
 128                 "cnf" => {
 129                     if let Some(rest) = rest.strip_prefix("Commands-") {
 130                         if let Some((arch, ext)) = rest.rsplit_once('.') {
 131                             Ok(FileReferenceType::Commands(
 132                                 arch.to_owned(),
 133                                 FileReferenceType::match_compression(ext).ok().flatten(),
 134                             ))
 135                         } else {
 136                             Ok(FileReferenceType::Commands(rest.to_owned(), None))
 137                         }
 138                     } else {
 139                         Ok(FileReferenceType::Unknown)
 140                     }
 141                 }
 142                 "dep11" => {
 143                     if let Some((_path, ext)) = rest.rsplit_once('.') {
 144                         Ok(FileReferenceType::Dep11(
 145                             FileReferenceType::match_compression(ext).ok().flatten(),
 146                         ))
 147                     } else {
 148                         Ok(FileReferenceType::Dep11(None))
 149                     }
 150                 }
 151                 "debian-installer" => {
 152                     // another layer, then like regular repo but pointing at udebs
 153                     if let Some((dir, rest)) = rest.split_once('/') {
 154                         if let Some(arch) = dir.strip_prefix("binary-") {
 155                             // Packages or compat-Release
 156                             return parse_binary_dir(rest, arch);
 157                         }
 158                     }
 159
 160                     // all the rest
 161                     Ok(FileReferenceType::Unknown)
 162                 }
 163                 "i18n" => {
 164                     if let Some((dir, _rest)) = rest.split_once('/') {
 165                         if dir.starts_with("Translation") && dir.ends_with(".diff") {
 166                             Ok(FileReferenceType::PDiff)
 167                         } else {
 168                             Ok(FileReferenceType::Unknown)
 169                         }
 170                     } else if let Some((_, ext)) = rest.split_once('.') {
 171                         Ok(FileReferenceType::Translation(
 172                             FileReferenceType::match_compression(ext)?,
 173                         ))
 174                     } else {
 175                         Ok(FileReferenceType::Translation(None))
 176                     }
 177                 }
 178                 _ => {
 179                     if let Some(arch) = dir.strip_prefix("binary-") {
 180                         // Packages or compat-Release
 181                         parse_binary_dir(rest, arch)
 182                     } else if let Some(_arch) = dir.strip_prefix("installer-") {
 183                         // netboot installer checksum files
 184                         Ok(FileReferenceType::Ignored)
 185                     } else {
 186                         // all the rest
 187                         Ok(FileReferenceType::Unknown)
 188                     }
 189                 }
 190             }
 191         } else if let Some(rest) = rest.strip_prefix("Contents-") {
 192             // reference to a top-level file - Contents-*
 193             let (rest, udeb) = if let Some(rest) = rest.strip_prefix("udeb-") {
 194                 (rest, true)
 195             } else {
 196                 (rest, false)
 197             };
 198             let (arch, comp) = match rest.split_once('.') {
 199                 Some((arch, comp_str)) => (
 200                     arch.to_owned(),
 201                     FileReferenceType::match_compression(comp_str)?,
 202                 ),
 203                 None => (rest.to_owned(), None),
 204             };
 205             if udeb {
 206                 Ok(FileReferenceType::ContentsUdeb(arch, comp))
 207             } else {
 208                 Ok(FileReferenceType::Contents(arch, comp))
 209             }
 210         } else {
 211             Ok(FileReferenceType::Unknown)
 212         }
 213     }
 214
 215     pub fn compression(&self) -> Option<CompressionType> {
 216         match *self {
 217             FileReferenceType::Commands(_, comp)
 218             | FileReferenceType::Contents(_, comp)
 219             | FileReferenceType::ContentsUdeb(_, comp)
 220             | FileReferenceType::Packages(_, comp)
 221             | FileReferenceType::Sources(comp)
 222             | FileReferenceType::Translation(comp)
 223             | FileReferenceType::Dep11(comp) => comp,
 224             FileReferenceType::Unknown
 225             | FileReferenceType::PDiff
 226             | FileReferenceType::PseudoRelease(_)
 227             | FileReferenceType::Ignored => None,
 228         }
 229     }
 230
 231     pub fn architecture(&self) -> Option<&Architecture> {
 232         match self {
 233             FileReferenceType::Commands(arch, _)
 234             | FileReferenceType::Contents(arch, _)
 235             | FileReferenceType::ContentsUdeb(arch, _)
 236             | FileReferenceType::Packages(arch, _) => Some(arch),
 237             FileReferenceType::PseudoRelease(arch) => arch.as_ref(),
 238             FileReferenceType::Unknown
 239             | FileReferenceType::PDiff
 240             | FileReferenceType::Sources(_)
 241             | FileReferenceType::Dep11(_)
 242             | FileReferenceType::Translation(_)
 243             | FileReferenceType::Ignored => None,
 244         }
 245     }
 246
 247     pub fn is_package_index(&self) -> bool {
 248         matches!(self, FileReferenceType::Packages(_, _) | FileReferenceType::Sources(_))
 249     }
 250 }
 251
 252 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 253 pub struct FileReference {
 254     pub path: String,
 255     pub size: usize,
 256     pub checksums: CheckSums,
 257     pub component: Component,
 258     pub file_type: FileReferenceType,
 259 }
 260
 261 impl FileReference {
 262     pub fn basename(&self) -> Result<String, Error> {
 263         match self.file_type.compression() {
 264             Some(_) => {
 265                 let (base, _ext) = self
 266                     .path
 267                     .rsplit_once('.')
 268                     .ok_or_else(|| format_err!("compressed file without file extension"))?;
 269                 Ok(base.to_owned())
 270             }
 271             None => Ok(self.path.clone()),
 272         }
 273     }
 274 }
 275
 276 #[derive(Debug, Default, PartialEq, Eq)]
 277 /// A parsed representation of a Release file
 278 pub struct ReleaseFile {
 279     /// List of architectures, e.g., `amd64` or `all`.
 280     pub architectures: Vec<String>,
 281     // TODO No-Support-for-Architecture-all
 282     /// URL for changelog queries via `apt changelog`.
 283     pub changelogs: Option<String>,
 284     /// Release codename - single word, e.g., `bullseye`.
 285     pub codename: Option<String>,
 286     /// List of repository areas, e.g., `main`.
 287     pub components: Vec<String>,
 288     /// UTC timestamp of release file generation
 289     pub date: Option<u64>,
 290     /// UTC timestamp of release file expiration
 291     pub valid_until: Option<u64>,
 292     /// Repository description -
 293     // TODO exact format?
 294     pub description: Option<String>,
 295     /// Repository label - single line
 296     pub label: Option<String>,
 297     /// Repository origin - single line
 298     pub origin: Option<String>,
 299     /// Release suite - single word, e.g., `stable`.
 300     pub suite: Option<String>,
 301     /// Release version
 302     pub version: Option<String>,
 303
 304     /// Whether by-hash retrieval of referenced files is possible
 305     pub aquire_by_hash: bool,
 306
 307     /// Files referenced by this `Release` file, e.g., packages indices.
 308     ///
 309     /// Grouped by basename, since only the compressed version needs to actually exist on the repository server.
 310     pub files: HashMap<String, Vec<FileReference>>,
 311 }
 312
 313 impl TryFrom<ReleaseFileRaw> for ReleaseFile {
 314     type Error = Error;
 315
 316     fn try_from(value: ReleaseFileRaw) -> Result<Self, Self::Error> {
 317         let mut parsed = ReleaseFile {
 318             architectures: whitespace_split_to_vec(
 319                 &value
 320                     .architectures
 321                     .ok_or_else(|| format_err!("'Architectures' field missing."))?,
 322             ),
 323             components: whitespace_split_to_vec(
 324                 &value
 325                     .components
 326                     .ok_or_else(|| format_err!("'Components' field missing."))?,
 327             ),
 328             changelogs: value.changelogs,
 329             codename: value.codename,
 330             date: value.date.as_deref().map(parse_date),
 331             valid_until: value
 332                 .extra_fields
 333                 .get("Valid-Until")
 334                 .map(|val| parse_date(&val.to_string())),
 335             description: value.description,
 336             label: value.label,
 337             origin: value.origin,
 338             suite: value.suite,
 339             files: HashMap::new(),
 340             aquire_by_hash: false,
 341             version: value.version,
 342         };
 343
 344         if let Some(val) = value.extra_fields.get("Acquire-By-Hash") {
 345             parsed.aquire_by_hash = *val == "yes";
 346         }
 347         // Fixup bullseye-security release files which have invalid components
 348         if parsed.label.as_deref() == Some("Debian-Security")
 349             && parsed.codename.as_deref() == Some("bullseye-security")
 350         {
 351             parsed.components = parsed
 352                 .components
 353                 .into_iter()
 354                 .map(|comp| {
 355                     if let Some(stripped) = comp.strip_prefix("updates/") {
 356                         stripped.to_owned()
 357                     } else {
 358                         comp
 359                     }
 360                 })
 361                 .collect();
 362         }
 363
 364         let mut references_map: HashMap<String, HashMap<String, FileReference>> = HashMap::new();
 365
 366         if let Some(md5) = value.md5_sum {
 367             for line in md5.lines() {
 368                 let (mut file_ref, checksum) =
 369                     parse_file_reference(line, 16, parsed.components.as_ref())?;
 370
 371                 let checksum = checksum
 372                     .try_into()
 373                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 374
 375                 file_ref.checksums.md5 = Some(checksum);
 376
 377                 merge_references(&mut references_map, file_ref)?;
 378             }
 379         }
 380
 381         if let Some(sha1) = value.sha1 {
 382             for line in sha1.lines() {
 383                 let (mut file_ref, checksum) =
 384                     parse_file_reference(line, 20, parsed.components.as_ref())?;
 385                 let checksum = checksum
 386                     .try_into()
 387                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 388
 389                 file_ref.checksums.sha1 = Some(checksum);
 390                 merge_references(&mut references_map, file_ref)?;
 391             }
 392         }
 393
 394         if let Some(sha256) = value.sha256 {
 395             for line in sha256.lines() {
 396                 let (mut file_ref, checksum) =
 397                     parse_file_reference(line, 32, parsed.components.as_ref())?;
 398                 let checksum = checksum
 399                     .try_into()
 400                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 401
 402                 file_ref.checksums.sha256 = Some(checksum);
 403                 merge_references(&mut references_map, file_ref)?;
 404             }
 405         }
 406
 407         if let Some(sha512) = value.sha512 {
 408             for line in sha512.lines() {
 409                 let (mut file_ref, checksum) =
 410                     parse_file_reference(line, 64, parsed.components.as_ref())?;
 411                 let checksum = checksum
 412                     .try_into()
 413                     .map_err(|_err| format_err!("unexpected checksum length"))?;
 414
 415                 file_ref.checksums.sha512 = Some(checksum);
 416                 merge_references(&mut references_map, file_ref)?;
 417             }
 418         }
 419
 420         parsed.files =
 421             references_map
 422                 .into_iter()
 423                 .fold(parsed.files, |mut map, (base, inner_map)| {
 424                     map.insert(base, inner_map.into_values().collect());
 425                     map
 426                 });
 427
 428         if let Some(insecure) = parsed
 429             .files
 430             .values()
 431             .flatten()
 432             .find(|file| !file.checksums.is_secure())
 433         {
 434             bail!(
 435                 "found file reference without strong checksum: {}",
 436                 insecure.path
 437             );
 438         }
 439
 440         Ok(parsed)
 441     }
 442 }
 443
 444 impl TryFrom<String> for ReleaseFile {
 445     type Error = Error;
 446
 447     fn try_from(value: String) -> Result<Self, Self::Error> {
 448         value.as_bytes().try_into()
 449     }
 450 }
 451
 452 impl TryFrom<&[u8]> for ReleaseFile {
 453     type Error = Error;
 454
 455     fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
 456         let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(value))?;
 457         deserialized.try_into()
 458     }
 459 }
 460
 461 fn whitespace_split_to_vec(list_str: &str) -> Vec<String> {
 462     list_str
 463         .split_ascii_whitespace()
 464         .map(|arch| arch.to_owned())
 465         .collect()
 466 }
 467
 468 fn parse_file_reference(
 469     line: &str,
 470     csum_len: usize,
 471     components: &[String],
 472 ) -> Result<(FileReference, Vec<u8>), Error> {
 473     let mut split = line.split_ascii_whitespace();
 474     let checksum = split
 475         .next()
 476         .ok_or_else(|| format_err!("No 'checksum' field in the file reference line."))?;
 477     if checksum.len() > csum_len * 2 {
 478         bail!(
 479             "invalid checksum length: '{}', expected {} bytes",
 480             checksum,
 481             csum_len
 482         );
 483     }
 484
 485     let checksum = hex::decode(checksum)?;
 486
 487     let size = split
 488         .next()
 489         .ok_or_else(|| format_err!("No 'size' field in file reference line."))?
 490         .parse::<usize>()?;
 491
 492     let file = split
 493         .next()
 494         .ok_or_else(|| format_err!("No 'path' field in file reference line."))?
 495         .to_string();
 496
 497     let (component, file_type) = components
 498         .iter()
 499         .find_map(|component| {
 500             if !file.starts_with(&format!("{component}/")) {
 501                 return None;
 502             }
 503
 504             Some(
 505                 FileReferenceType::parse(component, &file)
 506                     .map(|file_type| (component.clone(), file_type)),
 507             )
 508         })
 509         .unwrap_or_else(|| Ok(("UNKNOWN".to_string(), FileReferenceType::Unknown)))?;
 510
 511     Ok((
 512         FileReference {
 513             path: file,
 514             size,
 515             checksums: CheckSums::default(),
 516             component,
 517             file_type,
 518         },
 519         checksum,
 520     ))
 521 }
 522
 523 fn parse_date(_date_str: &str) -> u64 {
 524     // TODO implement
 525     0
 526 }
 527
 528 fn parse_binary_dir(file_name: &str, arch: &str, path: &str) -> Result<FileReferenceType, Error> {
 529     if let Some((dir, _rest)) = file_name.split_once('/') {
 530         if dir == "Packages.diff" {
 531             // TODO re-evaluate?
 532             Ok(FileReferenceType::PDiff)
 533         } else {
 534             Ok(FileReferenceType::Unknown)
 535         }
 536     } else if file_name == "Release" {
 537         Ok(FileReferenceType::PseudoRelease(Some(arch.to_owned())))
 538     } else {
 539         let comp = match file_name.strip_prefix("Packages") {
 540             None => {
 541                 bail!("found unexpected non-Packages reference to '{path}'")
 542             }
 543             Some(ext) => FileReferenceType::match_compression(ext)?,
 544         };
 545         //println!("compression: {comp:?}");
 546         Ok(FileReferenceType::Packages(arch.to_owned(), comp))
 547     }
 548 }
 549
 550 fn merge_references(
 551     base_map: &mut HashMap<String, HashMap<String, FileReference>>,
 552     file_ref: FileReference,
 553 ) -> Result<(), Error> {
 554     let base = file_ref.basename()?;
 555
 556     match base_map.get_mut(&base) {
 557         None => {
 558             let mut map = HashMap::new();
 559             map.insert(file_ref.path.clone(), file_ref);
 560             base_map.insert(base, map);
 561         }
 562         Some(entries) => {
 563             match entries.get_mut(&file_ref.path) {
 564                 Some(entry) => {
 565                     if entry.size != file_ref.size {
 566                         bail!(
 567                             "Multiple entries for '{}' with size mismatch: {} / {}",
 568                             entry.path,
 569                             file_ref.size,
 570                             entry.size
 571                         );
 572                     }
 573
 574                     entry.checksums.merge(&file_ref.checksums).map_err(|err| {
 575                         format_err!("Multiple checksums for '{}' - {err}", entry.path)
 576                     })?;
 577                 }
 578                 None => {
 579                     entries.insert(file_ref.path.clone(), file_ref);
 580                 }
 581             };
 582         }
 583     };
 584
 585     Ok(())
 586 }
 587
 588 #[test]
 589 pub fn test_deb_release_file() {
 590     let input = include_str!(concat!(
 591         env!("CARGO_MANIFEST_DIR"),
 592         "/tests/deb822/release/deb.debian.org_debian_dists_bullseye_Release"
 593     ));
 594
 595     let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(input.as_bytes())).unwrap();
 596     //println!("{:?}", deserialized);
 597
 598     let parsed: ReleaseFile = deserialized.try_into().unwrap();
 599     //println!("{:?}", parsed);
 600
 601     assert_eq!(parsed.files.len(), 315);
 602 }
 603
 604 #[test]
 605 pub fn test_deb_release_file_insecure() {
 606     let input = include_str!(concat!(
 607         env!("CARGO_MANIFEST_DIR"),
 608         "/tests/deb822/release/deb.debian.org_debian_dists_bullseye_Release_insecure"
 609     ));
 610
 611     let deserialized = ReleaseFileRaw::deserialize(Deserializer::new(input.as_bytes())).unwrap();
 612     //println!("{:?}", deserialized);
 613
 614     let parsed: Result<ReleaseFile, Error> = deserialized.try_into();
 615     assert!(parsed.is_err());
 616
 617     println!("{:?}", parsed);
 618 }