src/tools/build-manifest/src/main.rs

   1 //! Build a dist manifest, hash and sign everything.
   2 //! This gets called by `promote-release`
   3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
   4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
   5 //! by rustbuild (in `src/bootstrap/dist.rs`).
   6
   7 #![deny(warnings)]
   8
   9 use serde::Serialize;
  10 use toml;
  11
  12 use std::collections::BTreeMap;
  13 use std::collections::HashMap;
  14 use std::env;
  15 use std::fs::{self, File};
  16 use std::io::{self, Read, Write};
  17 use std::path::{Path, PathBuf};
  18 use std::process::{Command, Stdio};
  19
  20 static HOSTS: &[&str] = &[
  21     "aarch64-unknown-linux-gnu",
  22     "arm-unknown-linux-gnueabi",
  23     "arm-unknown-linux-gnueabihf",
  24     "armv7-unknown-linux-gnueabihf",
  25     "i686-apple-darwin",
  26     "i686-pc-windows-gnu",
  27     "i686-pc-windows-msvc",
  28     "i686-unknown-linux-gnu",
  29     "mips-unknown-linux-gnu",
  30     "mips64-unknown-linux-gnuabi64",
  31     "mips64el-unknown-linux-gnuabi64",
  32     "mipsel-unknown-linux-gnu",
  33     "mipsisa32r6-unknown-linux-gnu",
  34     "mipsisa32r6el-unknown-linux-gnu",
  35     "mipsisa64r6-unknown-linux-gnuabi64",
  36     "mipsisa64r6el-unknown-linux-gnuabi64",
  37     "powerpc-unknown-linux-gnu",
  38     "powerpc64-unknown-linux-gnu",
  39     "powerpc64le-unknown-linux-gnu",
  40     "s390x-unknown-linux-gnu",
  41     "x86_64-apple-darwin",
  42     "x86_64-pc-windows-gnu",
  43     "x86_64-pc-windows-msvc",
  44     "x86_64-unknown-freebsd",
  45     "x86_64-unknown-linux-gnu",
  46     "x86_64-unknown-linux-musl",
  47     "x86_64-unknown-netbsd",
  48 ];
  49
  50 static TARGETS: &[&str] = &[
  51     "aarch64-apple-ios",
  52     "aarch64-fuchsia",
  53     "aarch64-linux-android",
  54     "aarch64-pc-windows-msvc",
  55     "aarch64-unknown-cloudabi",
  56     "aarch64-unknown-hermit",
  57     "aarch64-unknown-linux-gnu",
  58     "aarch64-unknown-linux-musl",
  59     "aarch64-unknown-redox",
  60     "arm-linux-androideabi",
  61     "arm-unknown-linux-gnueabi",
  62     "arm-unknown-linux-gnueabihf",
  63     "arm-unknown-linux-musleabi",
  64     "arm-unknown-linux-musleabihf",
  65     "armv5te-unknown-linux-gnueabi",
  66     "armv5te-unknown-linux-musleabi",
  67     "armv7-apple-ios",
  68     "armv7-linux-androideabi",
  69     "thumbv7neon-linux-androideabi",
  70     "armv7-unknown-linux-gnueabi",
  71     "armv7-unknown-linux-gnueabihf",
  72     "armv7a-none-eabi",
  73     "thumbv7neon-unknown-linux-gnueabihf",
  74     "armv7-unknown-linux-musleabi",
  75     "armv7-unknown-linux-musleabihf",
  76     "armebv7r-none-eabi",
  77     "armebv7r-none-eabihf",
  78     "armv7r-none-eabi",
  79     "armv7r-none-eabihf",
  80     "armv7s-apple-ios",
  81     "asmjs-unknown-emscripten",
  82     "i386-apple-ios",
  83     "i586-pc-windows-msvc",
  84     "i586-unknown-linux-gnu",
  85     "i586-unknown-linux-musl",
  86     "i686-apple-darwin",
  87     "i686-linux-android",
  88     "i686-pc-windows-gnu",
  89     "i686-pc-windows-msvc",
  90     "i686-unknown-freebsd",
  91     "i686-unknown-linux-gnu",
  92     "i686-unknown-linux-musl",
  93     "mips-unknown-linux-gnu",
  94     "mips-unknown-linux-musl",
  95     "mips64-unknown-linux-gnuabi64",
  96     "mips64-unknown-linux-muslabi64",
  97     "mips64el-unknown-linux-gnuabi64",
  98     "mips64el-unknown-linux-muslabi64",
  99     "mipsisa32r6-unknown-linux-gnu",
 100     "mipsisa32r6el-unknown-linux-gnu",
 101     "mipsisa64r6-unknown-linux-gnuabi64",
 102     "mipsisa64r6el-unknown-linux-gnuabi64",
 103     "mipsel-unknown-linux-gnu",
 104     "mipsel-unknown-linux-musl",
 105     "nvptx64-nvidia-cuda",
 106     "powerpc-unknown-linux-gnu",
 107     "powerpc64-unknown-linux-gnu",
 108     "powerpc64le-unknown-linux-gnu",
 109     "riscv32i-unknown-none-elf",
 110     "riscv32imc-unknown-none-elf",
 111     "riscv32imac-unknown-none-elf",
 112     "riscv64imac-unknown-none-elf",
 113     "riscv64gc-unknown-none-elf",
 114     "riscv64gc-unknown-linux-gnu",
 115     "s390x-unknown-linux-gnu",
 116     "sparc64-unknown-linux-gnu",
 117     "sparcv9-sun-solaris",
 118     "thumbv6m-none-eabi",
 119     "thumbv7em-none-eabi",
 120     "thumbv7em-none-eabihf",
 121     "thumbv7m-none-eabi",
 122     "thumbv8m.base-none-eabi",
 123     "thumbv8m.main-none-eabi",
 124     "thumbv8m.main-none-eabihf",
 125     "wasm32-unknown-emscripten",
 126     "wasm32-unknown-unknown",
 127     "wasm32-wasi",
 128     "x86_64-apple-darwin",
 129     "x86_64-apple-ios",
 130     "x86_64-fortanix-unknown-sgx",
 131     "x86_64-fuchsia",
 132     "x86_64-linux-android",
 133     "x86_64-pc-windows-gnu",
 134     "x86_64-pc-windows-msvc",
 135     "x86_64-rumprun-netbsd",
 136     "x86_64-sun-solaris",
 137     "x86_64-pc-solaris",
 138     "x86_64-unknown-cloudabi",
 139     "x86_64-unknown-freebsd",
 140     "x86_64-unknown-linux-gnu",
 141     "x86_64-unknown-linux-gnux32",
 142     "x86_64-unknown-linux-musl",
 143     "x86_64-unknown-netbsd",
 144     "x86_64-unknown-redox",
 145     "x86_64-unknown-hermit",
 146 ];
 147
 148 static DOCS_TARGETS: &[&str] = &[
 149     "i686-apple-darwin",
 150     "i686-pc-windows-gnu",
 151     "i686-pc-windows-msvc",
 152     "i686-unknown-linux-gnu",
 153     "x86_64-apple-darwin",
 154     "x86_64-pc-windows-gnu",
 155     "x86_64-pc-windows-msvc",
 156     "x86_64-unknown-linux-gnu",
 157 ];
 158
 159 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 160
 161 #[derive(Serialize)]
 162 #[serde(rename_all = "kebab-case")]
 163 struct Manifest {
 164     manifest_version: String,
 165     date: String,
 166     pkg: BTreeMap<String, Package>,
 167     renames: BTreeMap<String, Rename>,
 168     profiles: BTreeMap<String, Vec<String>>,
 169 }
 170
 171 #[derive(Serialize)]
 172 struct Package {
 173     version: String,
 174     git_commit_hash: Option<String>,
 175     target: BTreeMap<String, Target>,
 176 }
 177
 178 #[derive(Serialize)]
 179 struct Rename {
 180     to: String,
 181 }
 182
 183 #[derive(Serialize, Default)]
 184 struct Target {
 185     available: bool,
 186     url: Option<String>,
 187     hash: Option<String>,
 188     xz_url: Option<String>,
 189     xz_hash: Option<String>,
 190     components: Option<Vec<Component>>,
 191     extensions: Option<Vec<Component>>,
 192 }
 193
 194 impl Target {
 195     fn unavailable() -> Self {
 196         Self::default()
 197     }
 198 }
 199
 200 #[derive(Serialize)]
 201 struct Component {
 202     pkg: String,
 203     target: String,
 204 }
 205
 206 impl Component {
 207     fn from_str(pkg: &str, target: &str) -> Self {
 208         Self { pkg: pkg.to_string(), target: target.to_string() }
 209     }
 210 }
 211
 212 macro_rules! t {
 213     ($e:expr) => {
 214         match $e {
 215             Ok(e) => e,
 216             Err(e) => panic!("{} failed with {}", stringify!($e), e),
 217         }
 218     };
 219 }
 220
 221 struct Builder {
 222     rust_release: String,
 223     cargo_release: String,
 224     rls_release: String,
 225     clippy_release: String,
 226     rustfmt_release: String,
 227     llvm_tools_release: String,
 228     lldb_release: String,
 229     miri_release: String,
 230
 231     input: PathBuf,
 232     output: PathBuf,
 233     gpg_passphrase: String,
 234     digests: BTreeMap<String, String>,
 235     s3_address: String,
 236     date: String,
 237
 238     rust_version: Option<String>,
 239     cargo_version: Option<String>,
 240     rls_version: Option<String>,
 241     clippy_version: Option<String>,
 242     rustfmt_version: Option<String>,
 243     llvm_tools_version: Option<String>,
 244     lldb_version: Option<String>,
 245     miri_version: Option<String>,
 246
 247     rust_git_commit_hash: Option<String>,
 248     cargo_git_commit_hash: Option<String>,
 249     rls_git_commit_hash: Option<String>,
 250     clippy_git_commit_hash: Option<String>,
 251     rustfmt_git_commit_hash: Option<String>,
 252     llvm_tools_git_commit_hash: Option<String>,
 253     lldb_git_commit_hash: Option<String>,
 254     miri_git_commit_hash: Option<String>,
 255
 256     should_sign: bool,
 257 }
 258
 259 fn main() {
 260     // Avoid signing packages while manually testing
 261     // Do NOT set this envvar in CI
 262     let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
 263
 264     // Safety check to ensure signing is always enabled on CI
 265     // The CI environment variable is set by both Travis and AppVeyor
 266     if !should_sign && env::var("CI").is_ok() {
 267         println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
 268         println!("If you're not running this on CI, unset the 'CI' env var.");
 269         panic!();
 270     }
 271
 272     let mut args = env::args().skip(1);
 273     let input = PathBuf::from(args.next().unwrap());
 274     let output = PathBuf::from(args.next().unwrap());
 275     let date = args.next().unwrap();
 276     let rust_release = args.next().unwrap();
 277     let s3_address = args.next().unwrap();
 278     let cargo_release = args.next().unwrap();
 279     let rls_release = args.next().unwrap();
 280     let clippy_release = args.next().unwrap();
 281     let miri_release = args.next().unwrap();
 282     let rustfmt_release = args.next().unwrap();
 283     let llvm_tools_release = args.next().unwrap();
 284     let lldb_release = args.next().unwrap();
 285
 286     // Do not ask for a passphrase while manually testing
 287     let mut passphrase = String::new();
 288     if should_sign {
 289         // `x.py` passes the passphrase via stdin.
 290         t!(io::stdin().read_to_string(&mut passphrase));
 291     }
 292
 293     Builder {
 294         rust_release,
 295         cargo_release,
 296         rls_release,
 297         clippy_release,
 298         rustfmt_release,
 299         llvm_tools_release,
 300         lldb_release,
 301         miri_release,
 302
 303         input,
 304         output,
 305         gpg_passphrase: passphrase,
 306         digests: BTreeMap::new(),
 307         s3_address,
 308         date,
 309
 310         rust_version: None,
 311         cargo_version: None,
 312         rls_version: None,
 313         clippy_version: None,
 314         rustfmt_version: None,
 315         llvm_tools_version: None,
 316         lldb_version: None,
 317         miri_version: None,
 318
 319         rust_git_commit_hash: None,
 320         cargo_git_commit_hash: None,
 321         rls_git_commit_hash: None,
 322         clippy_git_commit_hash: None,
 323         rustfmt_git_commit_hash: None,
 324         llvm_tools_git_commit_hash: None,
 325         lldb_git_commit_hash: None,
 326         miri_git_commit_hash: None,
 327
 328         should_sign,
 329     }
 330     .build();
 331 }
 332
 333 enum PkgType {
 334     RustSrc,
 335     Cargo,
 336     Rls,
 337     Clippy,
 338     Rustfmt,
 339     LlvmTools,
 340     Lldb,
 341     Miri,
 342     Other,
 343 }
 344
 345 impl PkgType {
 346     fn from_component(component: &str) -> Self {
 347         use PkgType::*;
 348         match component {
 349             "rust-src" => RustSrc,
 350             "cargo" => Cargo,
 351             "rls" | "rls-preview" => Rls,
 352             "clippy" | "clippy-preview" => Clippy,
 353             "rustfmt" | "rustfmt-preview" => Rustfmt,
 354             "llvm-tools" | "llvm-tools-preview" => LlvmTools,
 355             "lldb" | "lldb-preview" => Lldb,
 356             "miri" | "miri-preview" => Miri,
 357             _ => Other,
 358         }
 359     }
 360 }
 361
 362 impl Builder {
 363     fn build(&mut self) {
 364         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 365         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 366         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 367         self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
 368         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
 369         self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
 370         // lldb is only built for macOS.
 371         self.lldb_version = self.version("lldb", "x86_64-apple-darwin");
 372         self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu");
 373
 374         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
 375         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
 376         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
 377         self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
 378         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
 379         self.llvm_tools_git_commit_hash =
 380             self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu");
 381         self.lldb_git_commit_hash = self.git_commit_hash("lldb", "x86_64-unknown-linux-gnu");
 382         self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");
 383
 384         self.check_toolstate();
 385         self.digest_and_sign();
 386         let manifest = self.build_manifest();
 387         self.write_channel_files(&self.rust_release, &manifest);
 388
 389         if self.rust_release != "beta" && self.rust_release != "nightly" {
 390             self.write_channel_files("stable", &manifest);
 391         }
 392     }
 393
 394     /// If a tool does not pass its tests, don't ship it.
 395     /// Right now, we do this only for Miri.
 396     fn check_toolstate(&mut self) {
 397         let toolstates: Option<HashMap<String, String>> =
 398             File::open(self.input.join("toolstates-linux.json"))
 399                 .ok()
 400                 .and_then(|f| serde_json::from_reader(&f).ok());
 401         let toolstates = toolstates.unwrap_or_else(|| {
 402             println!(
 403                 "WARNING: `toolstates-linux.json` missing/malformed; \
 404                 assuming all tools failed"
 405             );
 406             HashMap::default() // Use empty map if anything went wrong.
 407         });
 408         // Mark some tools as missing based on toolstate.
 409         if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
 410             println!("Miri tests are not passing, removing component");
 411             self.miri_version = None;
 412             self.miri_git_commit_hash = None;
 413         }
 414     }
 415
 416     /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
 417     fn digest_and_sign(&mut self) {
 418         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 419             let filename = file.file_name().unwrap().to_str().unwrap();
 420             let digest = self.hash(&file);
 421             self.sign(&file);
 422             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 423         }
 424     }
 425
 426     fn build_manifest(&mut self) -> Manifest {
 427         let mut manifest = Manifest {
 428             manifest_version: "2".to_string(),
 429             date: self.date.to_string(),
 430             pkg: BTreeMap::new(),
 431             renames: BTreeMap::new(),
 432             profiles: BTreeMap::new(),
 433         };
 434         self.add_packages_to(&mut manifest);
 435         self.add_profiles_to(&mut manifest);
 436         self.add_renames_to(&mut manifest);
 437         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
 438         manifest
 439     }
 440
 441     fn add_packages_to(&mut self, manifest: &mut Manifest) {
 442         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
 443         package("rustc", HOSTS);
 444         package("rustc-dev", HOSTS);
 445         package("cargo", HOSTS);
 446         package("rust-mingw", MINGW);
 447         package("rust-std", TARGETS);
 448         package("rust-docs", DOCS_TARGETS);
 449         package("rust-src", &["*"]);
 450         package("rls-preview", HOSTS);
 451         package("clippy-preview", HOSTS);
 452         package("miri-preview", HOSTS);
 453         package("rustfmt-preview", HOSTS);
 454         package("rust-analysis", TARGETS);
 455         package("llvm-tools-preview", TARGETS);
 456         package("lldb-preview", TARGETS);
 457     }
 458
 459     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
 460         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
 461         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
 462         profile(
 463             "default",
 464             &[
 465                 "rustc",
 466                 "cargo",
 467                 "rust-std",
 468                 "rust-mingw",
 469                 "rust-docs",
 470                 "rustfmt-preview",
 471                 "clippy-preview",
 472             ],
 473         );
 474         profile(
 475             "complete",
 476             &[
 477                 "rustc",
 478                 "cargo",
 479                 "rust-std",
 480                 "rust-mingw",
 481                 "rust-docs",
 482                 "rustfmt-preview",
 483                 "clippy-preview",
 484                 "rls-preview",
 485                 "rust-src",
 486                 "llvm-tools-preview",
 487                 "lldb-preview",
 488                 "rust-analysis",
 489                 "miri-preview",
 490             ],
 491         );
 492
 493         // The compiler libraries are not stable for end users, and they're also huge, so we only
 494         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
 495         // for users to install the additional component manually, if needed.
 496         if self.rust_release == "nightly" {
 497             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
 498         }
 499     }
 500
 501     fn add_renames_to(&self, manifest: &mut Manifest) {
 502         let mut rename = |from: &str, to: &str| {
 503             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
 504         };
 505         rename("rls", "rls-preview");
 506         rename("rustfmt", "rustfmt-preview");
 507         rename("clippy", "clippy-preview");
 508         rename("miri", "miri-preview");
 509     }
 510
 511     fn rust_package(&mut self, manifest: &Manifest) -> Package {
 512         let mut pkg = Package {
 513             version: self
 514                 .cached_version("rust")
 515                 .as_ref()
 516                 .expect("Couldn't find Rust version")
 517                 .clone(),
 518             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
 519             target: BTreeMap::new(),
 520         };
 521         for host in HOSTS {
 522             if let Some(target) = self.target_host_combination(host, &manifest) {
 523                 pkg.target.insert(host.to_string(), target);
 524             } else {
 525                 pkg.target.insert(host.to_string(), Target::unavailable());
 526                 continue;
 527             }
 528         }
 529         pkg
 530     }
 531
 532     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
 533         let filename = self.filename("rust", host);
 534         let digest = self.digests.remove(&filename)?;
 535         let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 536         let xz_digest = self.digests.remove(&xz_filename);
 537         let mut components = Vec::new();
 538         let mut extensions = Vec::new();
 539
 540         let host_component = |pkg| Component::from_str(pkg, host);
 541
 542         // rustc/rust-std/cargo/docs are all required,
 543         // and so is rust-mingw if it's available for the target.
 544         components.extend(vec![
 545             host_component("rustc"),
 546             host_component("rust-std"),
 547             host_component("cargo"),
 548             host_component("rust-docs"),
 549         ]);
 550         if host.contains("pc-windows-gnu") {
 551             components.push(host_component("rust-mingw"));
 552         }
 553
 554         // Tools are always present in the manifest,
 555         // but might be marked as unavailable if they weren't built.
 556         extensions.extend(vec![
 557             host_component("clippy-preview"),
 558             host_component("miri-preview"),
 559             host_component("rls-preview"),
 560             host_component("rustfmt-preview"),
 561             host_component("llvm-tools-preview"),
 562             host_component("lldb-preview"),
 563             host_component("rust-analysis"),
 564         ]);
 565
 566         extensions.extend(
 567             TARGETS
 568                 .iter()
 569                 .filter(|&&target| target != host)
 570                 .map(|target| Component::from_str("rust-std", target)),
 571         );
 572         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
 573         extensions.push(Component::from_str("rust-src", "*"));
 574
 575         // If the components/extensions don't actually exist for this
 576         // particular host/target combination then nix it entirely from our
 577         // lists.
 578         let has_component = |c: &Component| {
 579             if c.target == "*" {
 580                 return true;
 581             }
 582             let pkg = match manifest.pkg.get(&c.pkg) {
 583                 Some(p) => p,
 584                 None => return false,
 585             };
 586             pkg.target.get(&c.target).is_some()
 587         };
 588         extensions.retain(&has_component);
 589         components.retain(&has_component);
 590
 591         Some(Target {
 592             available: true,
 593             url: Some(self.url(&filename)),
 594             hash: Some(digest),
 595             xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 596             xz_hash: xz_digest,
 597             components: Some(components),
 598             extensions: Some(extensions),
 599         })
 600     }
 601
 602     fn profile(
 603         &mut self,
 604         profile_name: &str,
 605         dst: &mut BTreeMap<String, Vec<String>>,
 606         pkgs: &[&str],
 607     ) {
 608         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
 609     }
 610
 611     fn extend_profile(
 612         &mut self,
 613         profile_name: &str,
 614         dst: &mut BTreeMap<String, Vec<String>>,
 615         pkgs: &[&str],
 616     ) {
 617         dst.get_mut(profile_name)
 618             .expect("existing profile")
 619             .extend(pkgs.iter().map(|s| (*s).to_owned()));
 620     }
 621
 622     fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
 623         let (version, mut is_present) = self
 624             .cached_version(pkgname)
 625             .as_ref()
 626             .cloned()
 627             .map(|version| (version, true))
 628             .unwrap_or_default(); // `is_present` defaults to `false` here.
 629
 630         // Miri is nightly-only; never ship it for other trains.
 631         if pkgname == "miri-preview" && self.rust_release != "nightly" {
 632             is_present = false; // Pretend the component is entirely missing.
 633         }
 634
 635         let targets = targets
 636             .iter()
 637             .map(|name| {
 638                 if is_present {
 639                     // The component generally exists, but it might still be missing for this target.
 640                     let filename = self.filename(pkgname, name);
 641                     let digest = match self.digests.remove(&filename) {
 642                         Some(digest) => digest,
 643                         // This component does not exist for this target -- skip it.
 644                         None => return (name.to_string(), Target::unavailable()),
 645                     };
 646                     let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 647                     let xz_digest = self.digests.remove(&xz_filename);
 648
 649                     (
 650                         name.to_string(),
 651                         Target {
 652                             available: true,
 653                             url: Some(self.url(&filename)),
 654                             hash: Some(digest),
 655                             xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 656                             xz_hash: xz_digest,
 657                             components: None,
 658                             extensions: None,
 659                         },
 660                     )
 661                 } else {
 662                     // If the component is not present for this build add it anyway but mark it as
 663                     // unavailable -- this way rustup won't allow upgrades without --force
 664                     (name.to_string(), Target::unavailable())
 665                 }
 666             })
 667             .collect();
 668
 669         dst.insert(
 670             pkgname.to_string(),
 671             Package {
 672                 version,
 673                 git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
 674                 target: targets,
 675             },
 676         );
 677     }
 678
 679     fn url(&self, filename: &str) -> String {
 680         format!("{}/{}/{}", self.s3_address, self.date, filename)
 681     }
 682
 683     fn filename(&self, component: &str, target: &str) -> String {
 684         use PkgType::*;
 685         match PkgType::from_component(component) {
 686             RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
 687             Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
 688             Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
 689             Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
 690             Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
 691             LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
 692             Lldb => format!("lldb-{}-{}.tar.gz", self.lldb_release, target),
 693             Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
 694             Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
 695         }
 696     }
 697
 698     fn cached_version(&self, component: &str) -> &Option<String> {
 699         use PkgType::*;
 700         match PkgType::from_component(component) {
 701             Cargo => &self.cargo_version,
 702             Rls => &self.rls_version,
 703             Clippy => &self.clippy_version,
 704             Rustfmt => &self.rustfmt_version,
 705             LlvmTools => &self.llvm_tools_version,
 706             Lldb => &self.lldb_version,
 707             Miri => &self.miri_version,
 708             _ => &self.rust_version,
 709         }
 710     }
 711
 712     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
 713         use PkgType::*;
 714         match PkgType::from_component(component) {
 715             Cargo => &self.cargo_git_commit_hash,
 716             Rls => &self.rls_git_commit_hash,
 717             Clippy => &self.clippy_git_commit_hash,
 718             Rustfmt => &self.rustfmt_git_commit_hash,
 719             LlvmTools => &self.llvm_tools_git_commit_hash,
 720             Lldb => &self.lldb_git_commit_hash,
 721             Miri => &self.miri_git_commit_hash,
 722             _ => &self.rust_git_commit_hash,
 723         }
 724     }
 725
 726     fn version(&self, component: &str, target: &str) -> Option<String> {
 727         self.untar(component, target, |filename| format!("{}/version", filename))
 728     }
 729
 730     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
 731         self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
 732     }
 733
 734     fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
 735     where
 736         F: FnOnce(String) -> String,
 737     {
 738         let mut cmd = Command::new("tar");
 739         let filename = self.filename(component, target);
 740         cmd.arg("xf")
 741             .arg(self.input.join(&filename))
 742             .arg(dir(filename.replace(".tar.gz", "")))
 743             .arg("-O");
 744         let output = t!(cmd.output());
 745         if output.status.success() {
 746             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 747         } else {
 748             None
 749         }
 750     }
 751
 752     fn hash(&self, path: &Path) -> String {
 753         let sha = t!(Command::new("shasum")
 754             .arg("-a")
 755             .arg("256")
 756             .arg(path.file_name().unwrap())
 757             .current_dir(path.parent().unwrap())
 758             .output());
 759         assert!(sha.status.success());
 760
 761         let filename = path.file_name().unwrap().to_str().unwrap();
 762         let sha256 = self.output.join(format!("{}.sha256", filename));
 763         t!(fs::write(&sha256, &sha.stdout));
 764
 765         let stdout = String::from_utf8_lossy(&sha.stdout);
 766         stdout.split_whitespace().next().unwrap().to_string()
 767     }
 768
 769     fn sign(&self, path: &Path) {
 770         if !self.should_sign {
 771             return;
 772         }
 773
 774         let filename = path.file_name().unwrap().to_str().unwrap();
 775         let asc = self.output.join(format!("{}.asc", filename));
 776         println!("signing: {:?}", path);
 777         let mut cmd = Command::new("gpg");
 778         cmd.arg("--pinentry-mode=loopback")
 779             .arg("--no-tty")
 780             .arg("--yes")
 781             .arg("--batch")
 782             .arg("--passphrase-fd")
 783             .arg("0")
 784             .arg("--personal-digest-preferences")
 785             .arg("SHA512")
 786             .arg("--armor")
 787             .arg("--output")
 788             .arg(&asc)
 789             .arg("--detach-sign")
 790             .arg(path)
 791             .stdin(Stdio::piped());
 792         let mut child = t!(cmd.spawn());
 793         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 794         assert!(t!(child.wait()).success());
 795     }
 796
 797     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
 798         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 799         self.write(&manifest.date, channel_name, "-date.txt");
 800         self.write(
 801             manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 802             channel_name,
 803             "-git-commit-hash.txt",
 804         );
 805     }
 806
 807     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
 808         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
 809         t!(fs::write(&dst, contents));
 810         self.hash(&dst);
 811         self.sign(&dst);
 812     }
 813 }