src/tools/build-manifest/src/main.rs

   1 //! Build a dist manifest, hash and sign everything.
   2 //! This gets called by `promote-release`
   3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
   4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
   5 //! by rustbuild (in `src/bootstrap/dist.rs`).
   6
   7 #![deny(warnings)]
   8
   9 use toml;
  10 use serde::Serialize;
  11
  12 use std::collections::BTreeMap;
  13 use std::env;
  14 use std::fs::{self, File};
  15 use std::io::{self, Read, Write};
  16 use std::path::{PathBuf, Path};
  17 use std::process::{Command, Stdio};
  18 use std::collections::HashMap;
  19
  20 static HOSTS: &[&str] = &[
  21     "aarch64-unknown-linux-gnu",
  22     "arm-unknown-linux-gnueabi",
  23     "arm-unknown-linux-gnueabihf",
  24     "armv7-unknown-linux-gnueabihf",
  25     "i686-apple-darwin",
  26     "i686-pc-windows-gnu",
  27     "i686-pc-windows-msvc",
  28     "i686-unknown-linux-gnu",
  29     "mips-unknown-linux-gnu",
  30     "mips64-unknown-linux-gnuabi64",
  31     "mips64el-unknown-linux-gnuabi64",
  32     "mipsel-unknown-linux-gnu",
  33     "mipsisa32r6-unknown-linux-gnu",
  34     "mipsisa32r6el-unknown-linux-gnu",
  35     "mipsisa64r6-unknown-linux-gnuabi64",
  36     "mipsisa64r6el-unknown-linux-gnuabi64",
  37     "powerpc-unknown-linux-gnu",
  38     "powerpc64-unknown-linux-gnu",
  39     "powerpc64le-unknown-linux-gnu",
  40     "s390x-unknown-linux-gnu",
  41     "x86_64-apple-darwin",
  42     "x86_64-pc-windows-gnu",
  43     "x86_64-pc-windows-msvc",
  44     "x86_64-unknown-freebsd",
  45     "x86_64-unknown-linux-gnu",
  46     "x86_64-unknown-linux-musl",
  47     "x86_64-unknown-netbsd",
  48 ];
  49
  50 static TARGETS: &[&str] = &[
  51     "aarch64-apple-ios",
  52     "aarch64-fuchsia",
  53     "aarch64-linux-android",
  54     "aarch64-pc-windows-msvc",
  55     "aarch64-unknown-cloudabi",
  56     "aarch64-unknown-hermit",
  57     "aarch64-unknown-linux-gnu",
  58     "aarch64-unknown-linux-musl",
  59     "aarch64-unknown-redox",
  60     "arm-linux-androideabi",
  61     "arm-unknown-linux-gnueabi",
  62     "arm-unknown-linux-gnueabihf",
  63     "arm-unknown-linux-musleabi",
  64     "arm-unknown-linux-musleabihf",
  65     "armv5te-unknown-linux-gnueabi",
  66     "armv5te-unknown-linux-musleabi",
  67     "armv7-apple-ios",
  68     "armv7-linux-androideabi",
  69     "thumbv7neon-linux-androideabi",
  70     "armv7-unknown-linux-gnueabi",
  71     "armv7-unknown-linux-gnueabihf",
  72     "thumbv7neon-unknown-linux-gnueabihf",
  73     "armv7-unknown-linux-musleabi",
  74     "armv7-unknown-linux-musleabihf",
  75     "armebv7r-none-eabi",
  76     "armebv7r-none-eabihf",
  77     "armv7r-none-eabi",
  78     "armv7r-none-eabihf",
  79     "armv7s-apple-ios",
  80     "asmjs-unknown-emscripten",
  81     "i386-apple-ios",
  82     "i586-pc-windows-msvc",
  83     "i586-unknown-linux-gnu",
  84     "i586-unknown-linux-musl",
  85     "i686-apple-darwin",
  86     "i686-linux-android",
  87     "i686-pc-windows-gnu",
  88     "i686-pc-windows-msvc",
  89     "i686-unknown-freebsd",
  90     "i686-unknown-linux-gnu",
  91     "i686-unknown-linux-musl",
  92     "mips-unknown-linux-gnu",
  93     "mips-unknown-linux-musl",
  94     "mips64-unknown-linux-gnuabi64",
  95     "mips64-unknown-linux-muslabi64",
  96     "mips64el-unknown-linux-gnuabi64",
  97     "mips64el-unknown-linux-muslabi64",
  98     "mipsisa32r6-unknown-linux-gnu",
  99     "mipsisa32r6el-unknown-linux-gnu",
 100     "mipsisa64r6-unknown-linux-gnuabi64",
 101     "mipsisa64r6el-unknown-linux-gnuabi64",
 102     "mipsel-unknown-linux-gnu",
 103     "mipsel-unknown-linux-musl",
 104     "nvptx64-nvidia-cuda",
 105     "powerpc-unknown-linux-gnu",
 106     "powerpc64-unknown-linux-gnu",
 107     "powerpc64le-unknown-linux-gnu",
 108     "riscv32i-unknown-none-elf",
 109     "riscv32imc-unknown-none-elf",
 110     "riscv32imac-unknown-none-elf",
 111     "riscv64imac-unknown-none-elf",
 112     "riscv64gc-unknown-none-elf",
 113     "s390x-unknown-linux-gnu",
 114     "sparc64-unknown-linux-gnu",
 115     "sparcv9-sun-solaris",
 116     "thumbv6m-none-eabi",
 117     "thumbv7em-none-eabi",
 118     "thumbv7em-none-eabihf",
 119     "thumbv7m-none-eabi",
 120     "thumbv8m.base-none-eabi",
 121     "thumbv8m.main-none-eabi",
 122     "thumbv8m.main-none-eabihf",
 123     "wasm32-unknown-emscripten",
 124     "wasm32-unknown-unknown",
 125     "wasm32-wasi",
 126     "x86_64-apple-darwin",
 127     "x86_64-apple-ios",
 128     "x86_64-fortanix-unknown-sgx",
 129     "x86_64-fuchsia",
 130     "x86_64-linux-android",
 131     "x86_64-pc-windows-gnu",
 132     "x86_64-pc-windows-msvc",
 133     "x86_64-rumprun-netbsd",
 134     "x86_64-sun-solaris",
 135     "x86_64-pc-solaris",
 136     "x86_64-unknown-cloudabi",
 137     "x86_64-unknown-freebsd",
 138     "x86_64-unknown-linux-gnu",
 139     "x86_64-unknown-linux-gnux32",
 140     "x86_64-unknown-linux-musl",
 141     "x86_64-unknown-netbsd",
 142     "x86_64-unknown-redox",
 143     "x86_64-unknown-hermit",
 144 ];
 145
 146 static DOCS_TARGETS: &[&str] = &[
 147     "i686-apple-darwin",
 148     "i686-pc-windows-gnu",
 149     "i686-pc-windows-msvc",
 150     "i686-unknown-linux-gnu",
 151     "x86_64-apple-darwin",
 152     "x86_64-pc-windows-gnu",
 153     "x86_64-pc-windows-msvc",
 154     "x86_64-unknown-linux-gnu",
 155 ];
 156
 157 static MINGW: &[&str] = &[
 158     "i686-pc-windows-gnu",
 159     "x86_64-pc-windows-gnu",
 160 ];
 161
 162 #[derive(Serialize)]
 163 #[serde(rename_all = "kebab-case")]
 164 struct Manifest {
 165     manifest_version: String,
 166     date: String,
 167     pkg: BTreeMap<String, Package>,
 168     renames: BTreeMap<String, Rename>,
 169     profiles: BTreeMap<String, Vec<String>>,
 170 }
 171
 172 #[derive(Serialize)]
 173 struct Package {
 174     version: String,
 175     git_commit_hash: Option<String>,
 176     target: BTreeMap<String, Target>,
 177 }
 178
 179 #[derive(Serialize)]
 180 struct Rename {
 181     to: String,
 182 }
 183
 184 #[derive(Serialize, Default)]
 185 struct Target {
 186     available: bool,
 187     url: Option<String>,
 188     hash: Option<String>,
 189     xz_url: Option<String>,
 190     xz_hash: Option<String>,
 191     components: Option<Vec<Component>>,
 192     extensions: Option<Vec<Component>>,
 193 }
 194
 195 impl Target {
 196     fn unavailable() -> Self { Self::default() }
 197 }
 198
 199 #[derive(Serialize)]
 200 struct Component {
 201     pkg: String,
 202     target: String,
 203 }
 204
 205 impl Component {
 206     fn from_str(pkg: &str, target: &str) -> Self {
 207         Self { pkg: pkg.to_string(), target: target.to_string() }
 208     }
 209 }
 210
 211 macro_rules! t {
 212     ($e:expr) => (match $e {
 213         Ok(e) => e,
 214         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 215     })
 216 }
 217
 218 struct Builder {
 219     rust_release: String,
 220     cargo_release: String,
 221     rls_release: String,
 222     clippy_release: String,
 223     rustfmt_release: String,
 224     llvm_tools_release: String,
 225     lldb_release: String,
 226     miri_release: String,
 227
 228     input: PathBuf,
 229     output: PathBuf,
 230     gpg_passphrase: String,
 231     digests: BTreeMap<String, String>,
 232     s3_address: String,
 233     date: String,
 234
 235     rust_version: Option<String>,
 236     cargo_version: Option<String>,
 237     rls_version: Option<String>,
 238     clippy_version: Option<String>,
 239     rustfmt_version: Option<String>,
 240     llvm_tools_version: Option<String>,
 241     lldb_version: Option<String>,
 242     miri_version: Option<String>,
 243
 244     rust_git_commit_hash: Option<String>,
 245     cargo_git_commit_hash: Option<String>,
 246     rls_git_commit_hash: Option<String>,
 247     clippy_git_commit_hash: Option<String>,
 248     rustfmt_git_commit_hash: Option<String>,
 249     llvm_tools_git_commit_hash: Option<String>,
 250     lldb_git_commit_hash: Option<String>,
 251     miri_git_commit_hash: Option<String>,
 252
 253     should_sign: bool,
 254 }
 255
 256 fn main() {
 257     // Avoid signing packages while manually testing
 258     // Do NOT set this envvar in CI
 259     let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
 260
 261     // Safety check to ensure signing is always enabled on CI
 262     // The CI environment variable is set by both Travis and AppVeyor
 263     if !should_sign && env::var("CI").is_ok() {
 264         println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
 265         println!("If you're not running this on CI, unset the 'CI' env var.");
 266         panic!();
 267     }
 268
 269     let mut args = env::args().skip(1);
 270     let input = PathBuf::from(args.next().unwrap());
 271     let output = PathBuf::from(args.next().unwrap());
 272     let date = args.next().unwrap();
 273     let rust_release = args.next().unwrap();
 274     let s3_address = args.next().unwrap();
 275     let cargo_release = args.next().unwrap();
 276     let rls_release = args.next().unwrap();
 277     let clippy_release = args.next().unwrap();
 278     let miri_release = args.next().unwrap();
 279     let rustfmt_release = args.next().unwrap();
 280     let llvm_tools_release = args.next().unwrap();
 281     let lldb_release = args.next().unwrap();
 282
 283     // Do not ask for a passphrase while manually testing
 284     let mut passphrase = String::new();
 285     if should_sign {
 286         // `x.py` passes the passphrase via stdin.
 287         t!(io::stdin().read_to_string(&mut passphrase));
 288     }
 289
 290     Builder {
 291         rust_release,
 292         cargo_release,
 293         rls_release,
 294         clippy_release,
 295         rustfmt_release,
 296         llvm_tools_release,
 297         lldb_release,
 298         miri_release,
 299
 300         input,
 301         output,
 302         gpg_passphrase: passphrase,
 303         digests: BTreeMap::new(),
 304         s3_address,
 305         date,
 306
 307         rust_version: None,
 308         cargo_version: None,
 309         rls_version: None,
 310         clippy_version: None,
 311         rustfmt_version: None,
 312         llvm_tools_version: None,
 313         lldb_version: None,
 314         miri_version: None,
 315
 316         rust_git_commit_hash: None,
 317         cargo_git_commit_hash: None,
 318         rls_git_commit_hash: None,
 319         clippy_git_commit_hash: None,
 320         rustfmt_git_commit_hash: None,
 321         llvm_tools_git_commit_hash: None,
 322         lldb_git_commit_hash: None,
 323         miri_git_commit_hash: None,
 324
 325         should_sign,
 326     }.build();
 327 }
 328
 329 enum PkgType { RustSrc, Cargo, Rls, Clippy, Rustfmt, LlvmTools, Lldb, Miri, Other }
 330
 331 impl PkgType {
 332     fn from_component(component: &str) -> Self {
 333         use PkgType::*;
 334         match component {
 335             "rust-src" => RustSrc,
 336             "cargo" => Cargo,
 337             "rls" | "rls-preview" => Rls,
 338             "clippy" | "clippy-preview" => Clippy,
 339             "rustfmt" | "rustfmt-preview" => Rustfmt,
 340             "llvm-tools" | "llvm-tools-preview" => LlvmTools,
 341             "lldb" | "lldb-preview" => Lldb,
 342             "miri" | "miri-preview" => Miri,
 343             _ => Other,
 344         }
 345     }
 346 }
 347
 348 impl Builder {
 349     fn build(&mut self) {
 350         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 351         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 352         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 353         self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
 354         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
 355         self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
 356         // lldb is only built for macOS.
 357         self.lldb_version = self.version("lldb", "x86_64-apple-darwin");
 358         self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu");
 359
 360         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
 361         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
 362         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
 363         self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
 364         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
 365         self.llvm_tools_git_commit_hash = self.git_commit_hash("llvm-tools",
 366                                                                "x86_64-unknown-linux-gnu");
 367         self.lldb_git_commit_hash = self.git_commit_hash("lldb", "x86_64-unknown-linux-gnu");
 368         self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");
 369
 370         self.check_toolstate();
 371         self.digest_and_sign();
 372         let manifest = self.build_manifest();
 373         self.write_channel_files(&self.rust_release, &manifest);
 374
 375         if self.rust_release != "beta" && self.rust_release != "nightly" {
 376             self.write_channel_files("stable", &manifest);
 377         }
 378     }
 379
 380     /// If a tool does not pass its tests, don't ship it.
 381     /// Right now, we do this only for Miri.
 382     fn check_toolstate(&mut self) {
 383         let toolstates: Option<HashMap<String, String>> =
 384             File::open(self.input.join("toolstates-linux.json")).ok()
 385                 .and_then(|f| serde_json::from_reader(&f).ok());
 386         let toolstates = toolstates.unwrap_or_else(|| {
 387             println!("WARNING: `toolstates-linux.json` missing/malformed; \
 388                 assuming all tools failed");
 389             HashMap::default() // Use empty map if anything went wrong.
 390         });
 391         // Mark some tools as missing based on toolstate.
 392         if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
 393             println!("Miri tests are not passing, removing component");
 394             self.miri_version = None;
 395             self.miri_git_commit_hash = None;
 396         }
 397     }
 398
 399     /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
 400     fn digest_and_sign(&mut self) {
 401         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 402             let filename = file.file_name().unwrap().to_str().unwrap();
 403             let digest = self.hash(&file);
 404             self.sign(&file);
 405             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 406         }
 407     }
 408
 409     fn build_manifest(&mut self) -> Manifest {
 410         let mut manifest = Manifest {
 411             manifest_version: "2".to_string(),
 412             date: self.date.to_string(),
 413             pkg: BTreeMap::new(),
 414             renames: BTreeMap::new(),
 415             profiles: BTreeMap::new(),
 416         };
 417         self.add_packages_to(&mut manifest);
 418         self.add_profiles_to(&mut manifest);
 419         self.add_renames_to(&mut manifest);
 420         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
 421         manifest
 422     }
 423
 424     fn add_packages_to(&mut self, manifest: &mut Manifest) {
 425         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
 426         package("rustc", HOSTS);
 427         package("rustc-dev", HOSTS);
 428         package("cargo", HOSTS);
 429         package("rust-mingw", MINGW);
 430         package("rust-std", TARGETS);
 431         package("rust-docs", DOCS_TARGETS);
 432         package("rust-src", &["*"]);
 433         package("rls-preview", HOSTS);
 434         package("clippy-preview", HOSTS);
 435         package("miri-preview", HOSTS);
 436         package("rustfmt-preview", HOSTS);
 437         package("rust-analysis", TARGETS);
 438         package("llvm-tools-preview", TARGETS);
 439         package("lldb-preview", TARGETS);
 440     }
 441
 442     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
 443         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
 444         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
 445         profile("default", &[
 446             "rustc", "cargo", "rust-std", "rust-mingw",
 447             "rust-docs", "rustfmt-preview", "clippy-preview"
 448         ]);
 449         profile("complete", &[
 450             "rustc", "cargo", "rust-std", "rust-mingw",
 451             "rust-docs", "rustfmt-preview", "clippy-preview",
 452             "rls-preview", "rust-src", "llvm-tools-preview",
 453             "lldb-preview", "rust-analysis", "miri-preview"
 454         ]);
 455
 456         // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
 457         // split out of `rust-std`. We'll include it by default as a transition for nightly users.
 458         if self.rust_release == "nightly" {
 459             self.extend_profile("default", &mut manifest.profiles, &["rustc-dev"]);
 460             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
 461         }
 462     }
 463
 464     fn add_renames_to(&self, manifest: &mut Manifest) {
 465         let mut rename = |from: &str, to: &str| manifest.renames.insert(
 466             from.to_owned(),
 467             Rename { to: to.to_owned() }
 468         );
 469         rename("rls", "rls-preview");
 470         rename("rustfmt", "rustfmt-preview");
 471         rename("clippy", "clippy-preview");
 472         rename("miri", "miri-preview");
 473     }
 474
 475     fn rust_package(&mut self, manifest: &Manifest) -> Package {
 476         let mut pkg = Package {
 477             version: self.cached_version("rust")
 478                          .as_ref()
 479                          .expect("Couldn't find Rust version")
 480                          .clone(),
 481             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
 482             target: BTreeMap::new(),
 483         };
 484         for host in HOSTS {
 485             if let Some(target) = self.target_host_combination(host, &manifest) {
 486                 pkg.target.insert(host.to_string(), target);
 487             } else {
 488                 pkg.target.insert(host.to_string(), Target::unavailable());
 489                 continue
 490             }
 491         }
 492         pkg
 493     }
 494
 495     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
 496         let filename = self.filename("rust", host);
 497         let digest = self.digests.remove(&filename)?;
 498         let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 499         let xz_digest = self.digests.remove(&xz_filename);
 500         let mut components = Vec::new();
 501         let mut extensions = Vec::new();
 502
 503         let host_component = |pkg| Component::from_str(pkg, host);
 504
 505         // rustc/rust-std/cargo/docs are all required,
 506         // and so is rust-mingw if it's available for the target.
 507         components.extend(vec![
 508             host_component("rustc"),
 509             host_component("rust-std"),
 510             host_component("cargo"),
 511             host_component("rust-docs"),
 512         ]);
 513         if host.contains("pc-windows-gnu") {
 514             components.push(host_component("rust-mingw"));
 515         }
 516
 517         // The compiler libraries are not stable for end users, but `rustc-dev` was only recently
 518         // split out of `rust-std`. We'll include it by default as a transition for nightly users,
 519         // but ship it as an optional component on the beta and stable channels.
 520         if self.rust_release == "nightly" {
 521             components.push(host_component("rustc-dev"));
 522         } else {
 523             extensions.push(host_component("rustc-dev"));
 524         }
 525
 526         // Tools are always present in the manifest,
 527         // but might be marked as unavailable if they weren't built.
 528         extensions.extend(vec![
 529             host_component("clippy-preview"),
 530             host_component("miri-preview"),
 531             host_component("rls-preview"),
 532             host_component("rustfmt-preview"),
 533             host_component("llvm-tools-preview"),
 534             host_component("lldb-preview"),
 535             host_component("rust-analysis"),
 536         ]);
 537
 538         extensions.extend(
 539             TARGETS.iter()
 540                 .filter(|&&target| target != host)
 541                 .map(|target| Component::from_str("rust-std", target))
 542         );
 543         extensions.extend(
 544             HOSTS.iter()
 545                 .filter(|&&target| target != host)
 546                 .map(|target| Component::from_str("rustc-dev", target))
 547         );
 548         extensions.push(Component::from_str("rust-src", "*"));
 549
 550         // If the components/extensions don't actually exist for this
 551         // particular host/target combination then nix it entirely from our
 552         // lists.
 553         let has_component = |c: &Component| {
 554             if c.target == "*" {
 555                 return true
 556             }
 557             let pkg = match manifest.pkg.get(&c.pkg) {
 558                 Some(p) => p,
 559                 None => return false,
 560             };
 561             pkg.target.get(&c.target).is_some()
 562         };
 563         extensions.retain(&has_component);
 564         components.retain(&has_component);
 565
 566         Some(Target {
 567             available: true,
 568             url: Some(self.url(&filename)),
 569             hash: Some(digest),
 570             xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 571             xz_hash: xz_digest,
 572             components: Some(components),
 573             extensions: Some(extensions),
 574         })
 575     }
 576
 577     fn profile(&mut self,
 578                profile_name: &str,
 579                dst: &mut BTreeMap<String, Vec<String>>,
 580                pkgs: &[&str]) {
 581         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
 582     }
 583
 584     fn extend_profile(&mut self,
 585                profile_name: &str,
 586                dst: &mut BTreeMap<String, Vec<String>>,
 587                pkgs: &[&str]) {
 588         dst.get_mut(profile_name).expect("existing profile")
 589             .extend(pkgs.iter().map(|s| (*s).to_owned()));
 590     }
 591
 592     fn package(&mut self,
 593                pkgname: &str,
 594                dst: &mut BTreeMap<String, Package>,
 595                targets: &[&str]) {
 596         let (version, mut is_present) = self.cached_version(pkgname)
 597             .as_ref()
 598             .cloned()
 599             .map(|version| (version, true))
 600             .unwrap_or_default(); // `is_present` defaults to `false` here.
 601
 602         // Miri is nightly-only; never ship it for other trains.
 603         if pkgname == "miri-preview" && self.rust_release != "nightly" {
 604             is_present = false; // Pretend the component is entirely missing.
 605         }
 606
 607         let targets = targets.iter().map(|name| {
 608             if is_present {
 609                 // The component generally exists, but it might still be missing for this target.
 610                 let filename = self.filename(pkgname, name);
 611                 let digest = match self.digests.remove(&filename) {
 612                     Some(digest) => digest,
 613                     // This component does not exist for this target -- skip it.
 614                     None => return (name.to_string(), Target::unavailable()),
 615                 };
 616                 let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 617                 let xz_digest = self.digests.remove(&xz_filename);
 618
 619                 (name.to_string(), Target {
 620                     available: true,
 621                     url: Some(self.url(&filename)),
 622                     hash: Some(digest),
 623                     xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 624                     xz_hash: xz_digest,
 625                     components: None,
 626                     extensions: None,
 627                 })
 628             } else {
 629                 // If the component is not present for this build add it anyway but mark it as
 630                 // unavailable -- this way rustup won't allow upgrades without --force
 631                 (name.to_string(), Target::unavailable())
 632             }
 633         }).collect();
 634
 635         dst.insert(pkgname.to_string(), Package {
 636             version,
 637             git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
 638             target: targets,
 639         });
 640     }
 641
 642     fn url(&self, filename: &str) -> String {
 643         format!("{}/{}/{}",
 644                 self.s3_address,
 645                 self.date,
 646                 filename)
 647     }
 648
 649     fn filename(&self, component: &str, target: &str) -> String {
 650         use PkgType::*;
 651         match PkgType::from_component(component) {
 652             RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
 653             Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
 654             Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
 655             Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
 656             Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
 657             LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
 658             Lldb => format!("lldb-{}-{}.tar.gz", self.lldb_release, target),
 659             Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
 660             Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
 661         }
 662     }
 663
 664     fn cached_version(&self, component: &str) -> &Option<String> {
 665         use PkgType::*;
 666         match PkgType::from_component(component) {
 667             Cargo => &self.cargo_version,
 668             Rls => &self.rls_version,
 669             Clippy => &self.clippy_version,
 670             Rustfmt => &self.rustfmt_version,
 671             LlvmTools => &self.llvm_tools_version,
 672             Lldb => &self.lldb_version,
 673             Miri => &self.miri_version,
 674             _ => &self.rust_version,
 675         }
 676     }
 677
 678     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
 679         use PkgType::*;
 680         match PkgType::from_component(component) {
 681             Cargo => &self.cargo_git_commit_hash,
 682             Rls => &self.rls_git_commit_hash,
 683             Clippy => &self.clippy_git_commit_hash,
 684             Rustfmt => &self.rustfmt_git_commit_hash,
 685             LlvmTools => &self.llvm_tools_git_commit_hash,
 686             Lldb => &self.lldb_git_commit_hash,
 687             Miri => &self.miri_git_commit_hash,
 688             _ => &self.rust_git_commit_hash,
 689         }
 690     }
 691
 692     fn version(&self, component: &str, target: &str) -> Option<String> {
 693         self.untar(component, target, |filename| format!("{}/version", filename))
 694     }
 695
 696     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
 697         self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
 698     }
 699
 700     fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
 701     where
 702         F: FnOnce(String) -> String
 703     {
 704         let mut cmd = Command::new("tar");
 705         let filename = self.filename(component, target);
 706         cmd.arg("xf")
 707            .arg(self.input.join(&filename))
 708            .arg(dir(filename.replace(".tar.gz", "")))
 709            .arg("-O");
 710         let output = t!(cmd.output());
 711         if output.status.success() {
 712             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 713         } else {
 714             None
 715         }
 716     }
 717
 718     fn hash(&self, path: &Path) -> String {
 719         let sha = t!(Command::new("shasum")
 720                         .arg("-a").arg("256")
 721                         .arg(path.file_name().unwrap())
 722                         .current_dir(path.parent().unwrap())
 723                         .output());
 724         assert!(sha.status.success());
 725
 726         let filename = path.file_name().unwrap().to_str().unwrap();
 727         let sha256 = self.output.join(format!("{}.sha256", filename));
 728         t!(fs::write(&sha256, &sha.stdout));
 729
 730         let stdout = String::from_utf8_lossy(&sha.stdout);
 731         stdout.split_whitespace().next().unwrap().to_string()
 732     }
 733
 734     fn sign(&self, path: &Path) {
 735         if !self.should_sign {
 736             return;
 737         }
 738
 739         let filename = path.file_name().unwrap().to_str().unwrap();
 740         let asc = self.output.join(format!("{}.asc", filename));
 741         println!("signing: {:?}", path);
 742         let mut cmd = Command::new("gpg");
 743         cmd.arg("--pinentry-mode=loopback")
 744             .arg("--no-tty")
 745             .arg("--yes")
 746             .arg("--batch")
 747             .arg("--passphrase-fd").arg("0")
 748             .arg("--personal-digest-preferences").arg("SHA512")
 749             .arg("--armor")
 750             .arg("--output").arg(&asc)
 751             .arg("--detach-sign").arg(path)
 752             .stdin(Stdio::piped());
 753         let mut child = t!(cmd.spawn());
 754         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 755         assert!(t!(child.wait()).success());
 756     }
 757
 758     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
 759         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 760         self.write(&manifest.date, channel_name, "-date.txt");
 761         self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 762                    channel_name, "-git-commit-hash.txt");
 763     }
 764
 765     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
 766         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
 767         t!(fs::write(&dst, contents));
 768         self.hash(&dst);
 769         self.sign(&dst);
 770     }
 771 }