src/tools/build-manifest/src/main.rs

   1 //! Build a dist manifest, hash and sign everything.
   2 //! This gets called by `promote-release`
   3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
   4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
   5 //! by rustbuild (in `src/bootstrap/dist.rs`).
   6
   7 mod checksum;
   8 mod manifest;
   9 mod versions;
  10
  11 use crate::checksum::Checksums;
  12 use crate::manifest::{Component, Manifest, Package, Rename, Target};
  13 use crate::versions::{PkgType, Versions};
  14 use std::collections::{BTreeMap, HashMap, HashSet};
  15 use std::env;
  16 use std::fs::{self, File};
  17 use std::path::{Path, PathBuf};
  18
  19 static HOSTS: &[&str] = &[
  20     "aarch64-apple-darwin",
  21     "aarch64-pc-windows-msvc",
  22     "aarch64-unknown-linux-gnu",
  23     "aarch64-unknown-linux-musl",
  24     "arm-unknown-linux-gnueabi",
  25     "arm-unknown-linux-gnueabihf",
  26     "armv7-unknown-linux-gnueabihf",
  27     "i686-apple-darwin",
  28     "i686-pc-windows-gnu",
  29     "i686-pc-windows-msvc",
  30     "i686-unknown-linux-gnu",
  31     "mips-unknown-linux-gnu",
  32     "mips64-unknown-linux-gnuabi64",
  33     "mips64el-unknown-linux-gnuabi64",
  34     "mipsel-unknown-linux-gnu",
  35     "mipsisa32r6-unknown-linux-gnu",
  36     "mipsisa32r6el-unknown-linux-gnu",
  37     "mipsisa64r6-unknown-linux-gnuabi64",
  38     "mipsisa64r6el-unknown-linux-gnuabi64",
  39     "powerpc-unknown-linux-gnu",
  40     "powerpc64-unknown-linux-gnu",
  41     "powerpc64le-unknown-linux-gnu",
  42     "riscv64gc-unknown-linux-gnu",
  43     "s390x-unknown-linux-gnu",
  44     "x86_64-apple-darwin",
  45     "x86_64-pc-windows-gnu",
  46     "x86_64-pc-windows-msvc",
  47     "x86_64-unknown-freebsd",
  48     "x86_64-unknown-illumos",
  49     "x86_64-unknown-linux-gnu",
  50     "x86_64-unknown-linux-musl",
  51     "x86_64-unknown-netbsd",
  52 ];
  53
  54 static TARGETS: &[&str] = &[
  55     "aarch64-apple-darwin",
  56     "aarch64-apple-ios",
  57     "aarch64-fuchsia",
  58     "aarch64-linux-android",
  59     "aarch64-pc-windows-msvc",
  60     "aarch64-unknown-hermit",
  61     "aarch64-unknown-linux-gnu",
  62     "aarch64-unknown-linux-musl",
  63     "aarch64-unknown-none",
  64     "aarch64-unknown-none-softfloat",
  65     "aarch64-unknown-redox",
  66     "arm-linux-androideabi",
  67     "arm-unknown-linux-gnueabi",
  68     "arm-unknown-linux-gnueabihf",
  69     "arm-unknown-linux-musleabi",
  70     "arm-unknown-linux-musleabihf",
  71     "armv5te-unknown-linux-gnueabi",
  72     "armv5te-unknown-linux-musleabi",
  73     "armv7-apple-ios",
  74     "armv7-linux-androideabi",
  75     "thumbv7neon-linux-androideabi",
  76     "armv7-unknown-linux-gnueabi",
  77     "armv7-unknown-linux-gnueabihf",
  78     "armv7a-none-eabi",
  79     "thumbv7neon-unknown-linux-gnueabihf",
  80     "armv7-unknown-linux-musleabi",
  81     "armv7-unknown-linux-musleabihf",
  82     "armebv7r-none-eabi",
  83     "armebv7r-none-eabihf",
  84     "armv7r-none-eabi",
  85     "armv7r-none-eabihf",
  86     "armv7s-apple-ios",
  87     "asmjs-unknown-emscripten",
  88     "i386-apple-ios",
  89     "i586-pc-windows-msvc",
  90     "i586-unknown-linux-gnu",
  91     "i586-unknown-linux-musl",
  92     "i686-apple-darwin",
  93     "i686-linux-android",
  94     "i686-pc-windows-gnu",
  95     "i686-pc-windows-msvc",
  96     "i686-unknown-freebsd",
  97     "i686-unknown-linux-gnu",
  98     "i686-unknown-linux-musl",
  99     "mips-unknown-linux-gnu",
 100     "mips-unknown-linux-musl",
 101     "mips64-unknown-linux-gnuabi64",
 102     "mips64-unknown-linux-muslabi64",
 103     "mips64el-unknown-linux-gnuabi64",
 104     "mips64el-unknown-linux-muslabi64",
 105     "mipsisa32r6-unknown-linux-gnu",
 106     "mipsisa32r6el-unknown-linux-gnu",
 107     "mipsisa64r6-unknown-linux-gnuabi64",
 108     "mipsisa64r6el-unknown-linux-gnuabi64",
 109     "mipsel-unknown-linux-gnu",
 110     "mipsel-unknown-linux-musl",
 111     "nvptx64-nvidia-cuda",
 112     "powerpc-unknown-linux-gnu",
 113     "powerpc64-unknown-linux-gnu",
 114     "powerpc64le-unknown-linux-gnu",
 115     "riscv32i-unknown-none-elf",
 116     "riscv32imc-unknown-none-elf",
 117     "riscv32imac-unknown-none-elf",
 118     "riscv32gc-unknown-linux-gnu",
 119     "riscv64imac-unknown-none-elf",
 120     "riscv64gc-unknown-none-elf",
 121     "riscv64gc-unknown-linux-gnu",
 122     "s390x-unknown-linux-gnu",
 123     "sparc64-unknown-linux-gnu",
 124     "sparcv9-sun-solaris",
 125     "thumbv6m-none-eabi",
 126     "thumbv7em-none-eabi",
 127     "thumbv7em-none-eabihf",
 128     "thumbv7m-none-eabi",
 129     "thumbv8m.base-none-eabi",
 130     "thumbv8m.main-none-eabi",
 131     "thumbv8m.main-none-eabihf",
 132     "wasm32-unknown-emscripten",
 133     "wasm32-unknown-unknown",
 134     "wasm32-wasi",
 135     "x86_64-apple-darwin",
 136     "x86_64-apple-ios",
 137     "x86_64-fortanix-unknown-sgx",
 138     "x86_64-fuchsia",
 139     "x86_64-linux-android",
 140     "x86_64-pc-windows-gnu",
 141     "x86_64-pc-windows-msvc",
 142     "x86_64-rumprun-netbsd",
 143     "x86_64-sun-solaris",
 144     "x86_64-pc-solaris",
 145     "x86_64-unknown-freebsd",
 146     "x86_64-unknown-illumos",
 147     "x86_64-unknown-linux-gnu",
 148     "x86_64-unknown-linux-gnux32",
 149     "x86_64-unknown-linux-musl",
 150     "x86_64-unknown-netbsd",
 151     "x86_64-unknown-redox",
 152     "x86_64-unknown-hermit",
 153 ];
 154
 155 static DOCS_TARGETS: &[&str] = &[
 156     "aarch64-unknown-linux-gnu",
 157     "i686-apple-darwin",
 158     "i686-pc-windows-gnu",
 159     "i686-pc-windows-msvc",
 160     "i686-unknown-linux-gnu",
 161     "x86_64-apple-darwin",
 162     "x86_64-pc-windows-gnu",
 163     "x86_64-pc-windows-msvc",
 164     "x86_64-unknown-linux-gnu",
 165     "x86_64-unknown-linux-musl",
 166 ];
 167
 168 static MSI_INSTALLERS: &[&str] = &[
 169     "aarch64-pc-windows-msvc",
 170     "i686-pc-windows-gnu",
 171     "i686-pc-windows-msvc",
 172     "x86_64-pc-windows-gnu",
 173     "x86_64-pc-windows-msvc",
 174 ];
 175
 176 static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
 177
 178 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 179
 180 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
 181
 182 macro_rules! t {
 183     ($e:expr) => {
 184         match $e {
 185             Ok(e) => e,
 186             Err(e) => panic!("{} failed with {}", stringify!($e), e),
 187         }
 188     };
 189 }
 190
 191 struct Builder {
 192     versions: Versions,
 193     checksums: Checksums,
 194     shipped_files: HashSet<String>,
 195
 196     input: PathBuf,
 197     output: PathBuf,
 198     s3_address: String,
 199     date: String,
 200 }
 201
 202 fn main() {
 203     let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
 204         num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
 205     } else {
 206         num_cpus::get()
 207     };
 208     rayon::ThreadPoolBuilder::new()
 209         .num_threads(num_threads)
 210         .build_global()
 211         .expect("failed to initialize Rayon");
 212
 213     let mut args = env::args().skip(1);
 214     let input = PathBuf::from(args.next().unwrap());
 215     let output = PathBuf::from(args.next().unwrap());
 216     let date = args.next().unwrap();
 217     let s3_address = args.next().unwrap();
 218     let channel = args.next().unwrap();
 219
 220     Builder {
 221         versions: Versions::new(&channel, &input).unwrap(),
 222         checksums: t!(Checksums::new()),
 223         shipped_files: HashSet::new(),
 224
 225         input,
 226         output,
 227         s3_address,
 228         date,
 229     }
 230     .build();
 231 }
 232
 233 impl Builder {
 234     fn build(&mut self) {
 235         self.check_toolstate();
 236         let manifest = self.build_manifest();
 237
 238         let channel = self.versions.channel().to_string();
 239         self.write_channel_files(&channel, &manifest);
 240         if channel == "stable" {
 241             // channel-rust-1.XX.YY.toml
 242             let rust_version = self.versions.rustc_version().to_string();
 243             self.write_channel_files(&rust_version, &manifest);
 244
 245             // channel-rust-1.XX.toml
 246             let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
 247             self.write_channel_files(&major_minor, &manifest);
 248         }
 249
 250         if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") {
 251             self.write_shipped_files(&Path::new(&path));
 252         }
 253
 254         t!(self.checksums.store_cache());
 255     }
 256
 257     /// If a tool does not pass its tests, don't ship it.
 258     /// Right now, we do this only for Miri.
 259     fn check_toolstate(&mut self) {
 260         let toolstates: Option<HashMap<String, String>> =
 261             File::open(self.input.join("toolstates-linux.json"))
 262                 .ok()
 263                 .and_then(|f| serde_json::from_reader(&f).ok());
 264         let toolstates = toolstates.unwrap_or_else(|| {
 265             println!(
 266                 "WARNING: `toolstates-linux.json` missing/malformed; \
 267                 assuming all tools failed"
 268             );
 269             HashMap::default() // Use empty map if anything went wrong.
 270         });
 271         // Mark some tools as missing based on toolstate.
 272         if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
 273             println!("Miri tests are not passing, removing component");
 274             self.versions.disable_version(&PkgType::Miri);
 275         }
 276     }
 277
 278     fn build_manifest(&mut self) -> Manifest {
 279         let mut manifest = Manifest {
 280             manifest_version: "2".to_string(),
 281             date: self.date.to_string(),
 282             pkg: BTreeMap::new(),
 283             artifacts: BTreeMap::new(),
 284             renames: BTreeMap::new(),
 285             profiles: BTreeMap::new(),
 286         };
 287         self.add_packages_to(&mut manifest);
 288         self.add_artifacts_to(&mut manifest);
 289         self.add_profiles_to(&mut manifest);
 290         self.add_renames_to(&mut manifest);
 291         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
 292
 293         self.checksums.fill_missing_checksums(&mut manifest);
 294
 295         manifest
 296     }
 297
 298     fn add_packages_to(&mut self, manifest: &mut Manifest) {
 299         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
 300         package("rustc", HOSTS);
 301         package("rustc-dev", HOSTS);
 302         package("reproducible-artifacts", HOSTS);
 303         package("rustc-docs", HOSTS);
 304         package("cargo", HOSTS);
 305         package("rust-mingw", MINGW);
 306         package("rust-std", TARGETS);
 307         package("rust-docs", DOCS_TARGETS);
 308         package("rust-src", &["*"]);
 309         package("rls-preview", HOSTS);
 310         package("rust-analyzer-preview", HOSTS);
 311         package("clippy-preview", HOSTS);
 312         package("miri-preview", HOSTS);
 313         package("rustfmt-preview", HOSTS);
 314         package("rust-analysis", TARGETS);
 315         package("llvm-tools-preview", TARGETS);
 316     }
 317
 318     fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
 319         manifest.add_artifact("source-code", |artifact| {
 320             let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
 321             artifact.add_tarball(self, "*", &tarball);
 322         });
 323
 324         manifest.add_artifact("installer-msi", |artifact| {
 325             for target in MSI_INSTALLERS {
 326                 let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
 327                 artifact.add_file(self, target, &msi);
 328             }
 329         });
 330
 331         manifest.add_artifact("installer-pkg", |artifact| {
 332             for target in PKG_INSTALLERS {
 333                 let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
 334                 artifact.add_file(self, target, &pkg);
 335             }
 336         });
 337     }
 338
 339     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
 340         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
 341         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
 342         profile(
 343             "default",
 344             &[
 345                 "rustc",
 346                 "cargo",
 347                 "rust-std",
 348                 "rust-mingw",
 349                 "rust-docs",
 350                 "rustfmt-preview",
 351                 "clippy-preview",
 352             ],
 353         );
 354         profile(
 355             "complete",
 356             &[
 357                 "rustc",
 358                 "cargo",
 359                 "rust-std",
 360                 "rust-mingw",
 361                 "rust-docs",
 362                 "rustfmt-preview",
 363                 "clippy-preview",
 364                 "rls-preview",
 365                 "rust-analyzer-preview",
 366                 "rust-src",
 367                 "llvm-tools-preview",
 368                 "rust-analysis",
 369                 "miri-preview",
 370             ],
 371         );
 372
 373         // The compiler libraries are not stable for end users, and they're also huge, so we only
 374         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
 375         // for users to install the additional component manually, if needed.
 376         if self.versions.channel() == "nightly" {
 377             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
 378             self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
 379         }
 380     }
 381
 382     fn add_renames_to(&self, manifest: &mut Manifest) {
 383         let mut rename = |from: &str, to: &str| {
 384             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
 385         };
 386         rename("rls", "rls-preview");
 387         rename("rustfmt", "rustfmt-preview");
 388         rename("clippy", "clippy-preview");
 389         rename("miri", "miri-preview");
 390     }
 391
 392     fn rust_package(&mut self, manifest: &Manifest) -> Package {
 393         let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
 394         let mut pkg = Package {
 395             version: version_info.version.expect("missing Rust version"),
 396             git_commit_hash: version_info.git_commit,
 397             target: BTreeMap::new(),
 398         };
 399         for host in HOSTS {
 400             if let Some(target) = self.target_host_combination(host, &manifest) {
 401                 pkg.target.insert(host.to_string(), target);
 402             } else {
 403                 pkg.target.insert(host.to_string(), Target::unavailable());
 404                 continue;
 405             }
 406         }
 407         pkg
 408     }
 409
 410     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
 411         let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
 412
 413         let mut target = Target::from_compressed_tar(self, &filename);
 414         if !target.available {
 415             return None;
 416         }
 417
 418         let mut components = Vec::new();
 419         let mut extensions = Vec::new();
 420
 421         let host_component = |pkg| Component::from_str(pkg, host);
 422
 423         // rustc/rust-std/cargo/docs are all required,
 424         // and so is rust-mingw if it's available for the target.
 425         components.extend(vec![
 426             host_component("rustc"),
 427             host_component("rust-std"),
 428             host_component("cargo"),
 429             host_component("rust-docs"),
 430         ]);
 431         if host.contains("pc-windows-gnu") {
 432             components.push(host_component("rust-mingw"));
 433         }
 434
 435         // Tools are always present in the manifest,
 436         // but might be marked as unavailable if they weren't built.
 437         extensions.extend(vec![
 438             host_component("clippy-preview"),
 439             host_component("miri-preview"),
 440             host_component("rls-preview"),
 441             host_component("rust-analyzer-preview"),
 442             host_component("rustfmt-preview"),
 443             host_component("llvm-tools-preview"),
 444             host_component("rust-analysis"),
 445         ]);
 446
 447         extensions.extend(
 448             TARGETS
 449                 .iter()
 450                 .filter(|&&target| target != host)
 451                 .map(|target| Component::from_str("rust-std", target)),
 452         );
 453         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
 454         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
 455         extensions.push(Component::from_str("rust-src", "*"));
 456
 457         // If the components/extensions don't actually exist for this
 458         // particular host/target combination then nix it entirely from our
 459         // lists.
 460         let has_component = |c: &Component| {
 461             if c.target == "*" {
 462                 return true;
 463             }
 464             let pkg = match manifest.pkg.get(&c.pkg) {
 465                 Some(p) => p,
 466                 None => return false,
 467             };
 468             pkg.target.get(&c.target).is_some()
 469         };
 470         extensions.retain(&has_component);
 471         components.retain(&has_component);
 472
 473         target.components = Some(components);
 474         target.extensions = Some(extensions);
 475         Some(target)
 476     }
 477
 478     fn profile(
 479         &mut self,
 480         profile_name: &str,
 481         dst: &mut BTreeMap<String, Vec<String>>,
 482         pkgs: &[&str],
 483     ) {
 484         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
 485     }
 486
 487     fn extend_profile(
 488         &mut self,
 489         profile_name: &str,
 490         dst: &mut BTreeMap<String, Vec<String>>,
 491         pkgs: &[&str],
 492     ) {
 493         dst.get_mut(profile_name)
 494             .expect("existing profile")
 495             .extend(pkgs.iter().map(|s| (*s).to_owned()));
 496     }
 497
 498     fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
 499         let version_info = self
 500             .versions
 501             .version(&PkgType::from_component(pkgname))
 502             .expect("failed to load package version");
 503         let mut is_present = version_info.present;
 504
 505         // Never ship nightly-only components for other trains.
 506         if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
 507             is_present = false; // Pretend the component is entirely missing.
 508         }
 509
 510         let targets = targets
 511             .iter()
 512             .map(|name| {
 513                 let target = if is_present {
 514                     let filename = self
 515                         .versions
 516                         .tarball_name(&PkgType::from_component(pkgname), name)
 517                         .unwrap();
 518
 519                     Target::from_compressed_tar(self, &filename)
 520                 } else {
 521                     // If the component is not present for this build add it anyway but mark it as
 522                     // unavailable -- this way rustup won't allow upgrades without --force
 523                     Target::unavailable()
 524                 };
 525                 (name.to_string(), target)
 526             })
 527             .collect();
 528
 529         dst.insert(
 530             pkgname.to_string(),
 531             Package {
 532                 version: version_info.version.unwrap_or_default(),
 533                 git_commit_hash: version_info.git_commit,
 534                 target: targets,
 535             },
 536         );
 537     }
 538
 539     fn url(&self, path: &Path) -> String {
 540         let file_name = path.file_name().unwrap().to_str().unwrap();
 541         format!("{}/{}/{}", self.s3_address, self.date, file_name)
 542     }
 543
 544     fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) {
 545         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 546         self.write(&manifest.date, channel_name, "-date.txt");
 547         self.write(
 548             manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 549             channel_name,
 550             "-git-commit-hash.txt",
 551         );
 552     }
 553
 554     fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) {
 555         let name = format!("channel-rust-{}{}", channel_name, suffix);
 556         self.shipped_files.insert(name.clone());
 557
 558         let dst = self.output.join(name);
 559         t!(fs::write(&dst, contents));
 560     }
 561
 562     fn write_shipped_files(&self, path: &Path) {
 563         let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::<Vec<_>>();
 564         files.sort();
 565         let content = format!("{}\n", files.join("\n"));
 566
 567         t!(std::fs::write(path, content.as_bytes()));
 568     }
 569 }