src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14 use std::process::Command;
  15
  16 use build_helper::{output, t};
  17
  18 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  49 pub struct Docs {
  50     pub host: TargetSelection,
  51 }
  52
  53 impl Step for Docs {
  54     type Output = Option<GeneratedTarball>;
  55     const DEFAULT: bool = true;
  56
  57     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  58         run.path("src/doc")
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         if !builder.config.docs {
  69             return None;
  70         }
  71         builder.default_doc(&[]);
  72
  73         let dest = "share/doc/rust/html";
  74
  75         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  76         tarball.set_product_name("Rust Documentation");
  77         tarball.add_dir(&builder.doc_out(host), dest);
  78         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  79         Some(tarball.generate())
  80     }
  81 }
  82
  83 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  84 pub struct RustcDocs {
  85     pub host: TargetSelection,
  86 }
  87
  88 impl Step for RustcDocs {
  89     type Output = Option<GeneratedTarball>;
  90     const DEFAULT: bool = true;
  91
  92     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  93         run.path("src/librustc")
  94     }
  95
  96     fn make_run(run: RunConfig<'_>) {
  97         run.builder.ensure(RustcDocs { host: run.target });
  98     }
  99
 100     /// Builds the `rustc-docs` installer component.
 101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 102         let host = self.host;
 103         if !builder.config.compiler_docs {
 104             return None;
 105         }
 106         builder.default_doc(&[]);
 107
 108         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 109         tarball.set_product_name("Rustc Documentation");
 110         tarball.add_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 111         Some(tarball.generate())
 112     }
 113 }
 114
 115 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 116     let mut found = Vec::with_capacity(files.len());
 117
 118     for file in files {
 119         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 120
 121         if let Some(file_path) = file_path {
 122             found.push(file_path);
 123         } else {
 124             panic!("Could not find '{}' in {:?}", file, path);
 125         }
 126     }
 127
 128     found
 129 }
 130
 131 fn make_win_dist(
 132     rust_root: &Path,
 133     plat_root: &Path,
 134     target: TargetSelection,
 135     builder: &Builder<'_>,
 136 ) {
 137     //Ask gcc where it keeps its stuff
 138     let mut cmd = Command::new(builder.cc(target));
 139     cmd.arg("-print-search-dirs");
 140     let gcc_out = output(&mut cmd);
 141
 142     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 143     let mut lib_path = Vec::new();
 144
 145     for line in gcc_out.lines() {
 146         let idx = line.find(':').unwrap();
 147         let key = &line[..idx];
 148         let trim_chars: &[_] = &[' ', '='];
 149         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 150
 151         if key == "programs" {
 152             bin_path.extend(value);
 153         } else if key == "libraries" {
 154             lib_path.extend(value);
 155         }
 156     }
 157
 158     let compiler = if target == "i686-pc-windows-gnu" {
 159         "i686-w64-mingw32-gcc.exe"
 160     } else if target == "x86_64-pc-windows-gnu" {
 161         "x86_64-w64-mingw32-gcc.exe"
 162     } else {
 163         "gcc.exe"
 164     };
 165     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 166     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 167     if target.starts_with("i686-") {
 168         rustc_dlls.push("libgcc_s_dw2-1.dll");
 169     } else {
 170         rustc_dlls.push("libgcc_s_seh-1.dll");
 171     }
 172
 173     let target_libs = [
 174         //MinGW libs
 175         "libgcc.a",
 176         "libgcc_eh.a",
 177         "libgcc_s.a",
 178         "libm.a",
 179         "libmingw32.a",
 180         "libmingwex.a",
 181         "libstdc++.a",
 182         "libiconv.a",
 183         "libmoldname.a",
 184         "libpthread.a",
 185         //Windows import libs
 186         "libadvapi32.a",
 187         "libbcrypt.a",
 188         "libcomctl32.a",
 189         "libcomdlg32.a",
 190         "libcredui.a",
 191         "libcrypt32.a",
 192         "libdbghelp.a",
 193         "libgdi32.a",
 194         "libimagehlp.a",
 195         "libiphlpapi.a",
 196         "libkernel32.a",
 197         "libmsimg32.a",
 198         "libmsvcrt.a",
 199         "libodbc32.a",
 200         "libole32.a",
 201         "liboleaut32.a",
 202         "libopengl32.a",
 203         "libpsapi.a",
 204         "librpcrt4.a",
 205         "libsecur32.a",
 206         "libsetupapi.a",
 207         "libshell32.a",
 208         "libsynchronization.a",
 209         "libuser32.a",
 210         "libuserenv.a",
 211         "libuuid.a",
 212         "libwinhttp.a",
 213         "libwinmm.a",
 214         "libwinspool.a",
 215         "libws2_32.a",
 216         "libwsock32.a",
 217     ];
 218
 219     //Find mingw artifacts we want to bundle
 220     let target_tools = find_files(&target_tools, &bin_path);
 221     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 222     let target_libs = find_files(&target_libs, &lib_path);
 223
 224     // Copy runtime dlls next to rustc.exe
 225     let dist_bin_dir = rust_root.join("bin/");
 226     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 227     for src in rustc_dlls {
 228         builder.copy_to_folder(&src, &dist_bin_dir);
 229     }
 230
 231     //Copy platform tools to platform-specific bin directory
 232     let target_bin_dir = plat_root
 233         .join("lib")
 234         .join("rustlib")
 235         .join(target.triple)
 236         .join("bin")
 237         .join("self-contained");
 238     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 239     for src in target_tools {
 240         builder.copy_to_folder(&src, &target_bin_dir);
 241     }
 242
 243     // Warn windows-gnu users that the bundled GCC cannot compile C files
 244     builder.create(
 245         &target_bin_dir.join("GCC-WARNING.txt"),
 246         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 247          used as a linker. In order to be able to compile projects containing C code use \
 248          the GCC provided by MinGW or Cygwin.",
 249     );
 250
 251     //Copy platform libs to platform-specific lib directory
 252     let target_lib_dir = plat_root
 253         .join("lib")
 254         .join("rustlib")
 255         .join(target.triple)
 256         .join("lib")
 257         .join("self-contained");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         builder.copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     pub host: TargetSelection,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<GeneratedTarball>;
 271     const DEFAULT: bool = true;
 272
 273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 274         run.never()
 275     }
 276
 277     fn make_run(run: RunConfig<'_>) {
 278         run.builder.ensure(Mingw { host: run.target });
 279     }
 280
 281     /// Builds the `rust-mingw` installer component.
 282     ///
 283     /// This contains all the bits and pieces to run the MinGW Windows targets
 284     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 285     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 286         let host = self.host;
 287         if !host.contains("pc-windows-gnu") {
 288             return None;
 289         }
 290
 291         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 292         tarball.set_product_name("Rust MinGW");
 293
 294         // The first argument is a "temporary directory" which is just
 295         // thrown away (this contains the runtime DLLs included in the rustc package
 296         // above) and the second argument is where to place all the MinGW components
 297         // (which is what we want).
 298         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 299
 300         Some(tarball.generate())
 301     }
 302 }
 303
 304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 305 pub struct Rustc {
 306     pub compiler: Compiler,
 307 }
 308
 309 impl Step for Rustc {
 310     type Output = GeneratedTarball;
 311     const DEFAULT: bool = true;
 312     const ONLY_HOSTS: bool = true;
 313
 314     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 315         run.path("src/librustc")
 316     }
 317
 318     fn make_run(run: RunConfig<'_>) {
 319         run.builder
 320             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 321     }
 322
 323     /// Creates the `rustc` installer component.
 324     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 325         let compiler = self.compiler;
 326         let host = self.compiler.host;
 327
 328         let tarball = Tarball::new(builder, "rustc", &host.triple);
 329
 330         // Prepare the rustc "image", what will actually end up getting installed
 331         prepare_image(builder, compiler, tarball.image_dir());
 332
 333         // On MinGW we've got a few runtime DLL dependencies that we need to
 334         // include. The first argument to this script is where to put these DLLs
 335         // (the image we're creating), and the second argument is a junk directory
 336         // to ignore all other MinGW stuff the script creates.
 337         //
 338         // On 32-bit MinGW we're always including a DLL which needs some extra
 339         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 340         // anything requiring us to distribute a license, but it's likely the
 341         // install will *also* include the rust-mingw package, which also needs
 342         // licenses, so to be safe we just include it here in all MinGW packages.
 343         if host.contains("pc-windows-gnu") {
 344             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 345             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 346         }
 347
 348         return tarball.generate();
 349
 350         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 351             let host = compiler.host;
 352             let src = builder.sysroot(compiler);
 353
 354             // Copy rustc/rustdoc binaries
 355             t!(fs::create_dir_all(image.join("bin")));
 356             builder.cp_r(&src.join("bin"), &image.join("bin"));
 357
 358             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 359
 360             let libdir_relative = builder.libdir_relative(compiler);
 361
 362             // Copy runtime DLLs needed by the compiler
 363             if libdir_relative.to_str() != Some("bin") {
 364                 let libdir = builder.rustc_libdir(compiler);
 365                 for entry in builder.read_dir(&libdir) {
 366                     let name = entry.file_name();
 367                     if let Some(s) = name.to_str() {
 368                         if is_dylib(s) {
 369                             // Don't use custom libdir here because ^lib/ will be resolved again
 370                             // with installer
 371                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 372                         }
 373                     }
 374                 }
 375             }
 376
 377             // Copy over the codegen backends
 378             let backends_src = builder.sysroot_codegen_backends(compiler);
 379             let backends_rel = backends_src
 380                 .strip_prefix(&src)
 381                 .unwrap()
 382                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 383                 .unwrap();
 384             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 385             let backends_dst = image.join("lib").join(&backends_rel);
 386
 387             t!(fs::create_dir_all(&backends_dst));
 388             builder.cp_r(&backends_src, &backends_dst);
 389
 390             // Copy libLLVM.so to the lib dir as well, if needed. While not
 391             // technically needed by rustc itself it's needed by lots of other
 392             // components like the llvm tools and LLD. LLD is included below and
 393             // tools/LLDB come later, so let's just throw it in the rustc
 394             // component for now.
 395             maybe_install_llvm_runtime(builder, host, image);
 396
 397             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let exe = exe("rust-lld", compiler.host);
 404                 builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 405             }
 406
 407             // Copy over llvm-dwp if it's there
 408             let exe = exe("rust-llvm-dwp", compiler.host);
 409             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 410
 411             // Man pages
 412             t!(fs::create_dir_all(image.join("share/man/man1")));
 413             let man_src = builder.src.join("src/doc/man");
 414             let man_dst = image.join("share/man/man1");
 415
 416             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 417             let time = env::var("SOURCE_DATE_EPOCH")
 418                 .map(|timestamp| {
 419                     let epoch = timestamp
 420                         .parse()
 421                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 422                         .unwrap();
 423
 424                     time::at(Timespec::new(epoch, 0))
 425                 })
 426                 .unwrap_or_else(|_| time::now());
 427
 428             let month_year = t!(time::strftime("%B %Y", &time));
 429             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 430             // to hardlink, and we don't want to edit the source templates
 431             for file_entry in builder.read_dir(&man_src) {
 432                 let page_src = file_entry.path();
 433                 let page_dst = man_dst.join(file_entry.file_name());
 434                 t!(fs::copy(&page_src, &page_dst));
 435                 // template in month/year and version number
 436                 builder.replace_in_file(
 437                     &page_dst,
 438                     &[
 439                         ("<INSERT DATE HERE>", &month_year),
 440                         ("<INSERT VERSION HERE>", &builder.version),
 441                     ],
 442                 );
 443             }
 444
 445             // Debugger scripts
 446             builder
 447                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 448
 449             // Misc license info
 450             let cp = |file: &str| {
 451                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 452             };
 453             cp("COPYRIGHT");
 454             cp("LICENSE-APACHE");
 455             cp("LICENSE-MIT");
 456             cp("README.md");
 457         }
 458     }
 459 }
 460
 461 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 462 pub struct DebuggerScripts {
 463     pub sysroot: Interned<PathBuf>,
 464     pub host: TargetSelection,
 465 }
 466
 467 impl Step for DebuggerScripts {
 468     type Output = ();
 469
 470     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 471         run.path("src/lldb_batchmode.py")
 472     }
 473
 474     fn make_run(run: RunConfig<'_>) {
 475         run.builder.ensure(DebuggerScripts {
 476             sysroot: run
 477                 .builder
 478                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 479             host: run.target,
 480         });
 481     }
 482
 483     /// Copies debugger scripts for `target` into the `sysroot` specified.
 484     fn run(self, builder: &Builder<'_>) {
 485         let host = self.host;
 486         let sysroot = self.sysroot;
 487         let dst = sysroot.join("lib/rustlib/etc");
 488         t!(fs::create_dir_all(&dst));
 489         let cp_debugger_script = |file: &str| {
 490             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 491         };
 492         if host.contains("windows-msvc") {
 493             // windbg debugger scripts
 494             builder.install(
 495                 &builder.src.join("src/etc/rust-windbg.cmd"),
 496                 &sysroot.join("bin"),
 497                 0o755,
 498             );
 499
 500             cp_debugger_script("natvis/intrinsic.natvis");
 501             cp_debugger_script("natvis/liballoc.natvis");
 502             cp_debugger_script("natvis/libcore.natvis");
 503             cp_debugger_script("natvis/libstd.natvis");
 504         } else {
 505             cp_debugger_script("rust_types.py");
 506
 507             // gdb debugger scripts
 508             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 509             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 510
 511             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 512             cp_debugger_script("gdb_lookup.py");
 513             cp_debugger_script("gdb_providers.py");
 514
 515             // lldb debugger scripts
 516             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 517
 518             cp_debugger_script("lldb_lookup.py");
 519             cp_debugger_script("lldb_providers.py");
 520             cp_debugger_script("lldb_commands")
 521         }
 522     }
 523 }
 524
 525 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 526     // The only true set of target libraries came from the build triple, so
 527     // let's reduce redundant work by only producing archives from that host.
 528     if compiler.host != builder.config.build {
 529         builder.info("\tskipping, not a build host");
 530         true
 531     } else {
 532         false
 533     }
 534 }
 535
 536 /// Copy stamped files into an image's `target/lib` directory.
 537 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 538     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 539     let self_contained_dst = dst.join("self-contained");
 540     t!(fs::create_dir_all(&dst));
 541     t!(fs::create_dir_all(&self_contained_dst));
 542     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 543         if dependency_type == DependencyType::TargetSelfContained {
 544             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 545         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 546             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 547         }
 548     }
 549 }
 550
 551 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 552 pub struct Std {
 553     pub compiler: Compiler,
 554     pub target: TargetSelection,
 555 }
 556
 557 impl Step for Std {
 558     type Output = Option<GeneratedTarball>;
 559     const DEFAULT: bool = true;
 560
 561     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 562         run.path("library/std")
 563     }
 564
 565     fn make_run(run: RunConfig<'_>) {
 566         run.builder.ensure(Std {
 567             compiler: run.builder.compiler_for(
 568                 run.builder.top_stage,
 569                 run.builder.config.build,
 570                 run.target,
 571             ),
 572             target: run.target,
 573         });
 574     }
 575
 576     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 577         let compiler = self.compiler;
 578         let target = self.target;
 579
 580         if skip_host_target_lib(builder, compiler) {
 581             return None;
 582         }
 583
 584         builder.ensure(compile::Std { compiler, target });
 585
 586         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 587         tarball.include_target_in_component_name(true);
 588
 589         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 590         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 591         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 592
 593         Some(tarball.generate())
 594     }
 595 }
 596
 597 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 598 pub struct RustcDev {
 599     pub compiler: Compiler,
 600     pub target: TargetSelection,
 601 }
 602
 603 impl Step for RustcDev {
 604     type Output = Option<GeneratedTarball>;
 605     const DEFAULT: bool = true;
 606     const ONLY_HOSTS: bool = true;
 607
 608     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 609         run.path("rustc-dev")
 610     }
 611
 612     fn make_run(run: RunConfig<'_>) {
 613         run.builder.ensure(RustcDev {
 614             compiler: run.builder.compiler_for(
 615                 run.builder.top_stage,
 616                 run.builder.config.build,
 617                 run.target,
 618             ),
 619             target: run.target,
 620         });
 621     }
 622
 623     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 624         let compiler = self.compiler;
 625         let target = self.target;
 626         if skip_host_target_lib(builder, compiler) {
 627             return None;
 628         }
 629
 630         builder.ensure(compile::Rustc { compiler, target });
 631
 632         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 633
 634         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 635         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 636         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 637
 638         let src_files = &["Cargo.lock"];
 639         // This is the reduced set of paths which will become the rustc-dev component
 640         // (essentially the compiler crates and all of their path dependencies).
 641         copy_src_dirs(
 642             builder,
 643             &builder.src,
 644             &["compiler"],
 645             &[],
 646             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 647         );
 648         // This particular crate is used as a build dependency of the above.
 649         copy_src_dirs(
 650             builder,
 651             &builder.src,
 652             &["src/build_helper"],
 653             &[],
 654             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 655         );
 656         for file in src_files {
 657             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 658         }
 659
 660         Some(tarball.generate())
 661     }
 662 }
 663
 664 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 665 pub struct Analysis {
 666     pub compiler: Compiler,
 667     pub target: TargetSelection,
 668 }
 669
 670 impl Step for Analysis {
 671     type Output = Option<GeneratedTarball>;
 672     const DEFAULT: bool = true;
 673
 674     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 675         let builder = run.builder;
 676         run.path("analysis").default_condition(builder.config.extended)
 677     }
 678
 679     fn make_run(run: RunConfig<'_>) {
 680         run.builder.ensure(Analysis {
 681             // Find the actual compiler (handling the full bootstrap option) which
 682             // produced the save-analysis data because that data isn't copied
 683             // through the sysroot uplifting.
 684             compiler: run.builder.compiler_for(
 685                 run.builder.top_stage,
 686                 run.builder.config.build,
 687                 run.target,
 688             ),
 689             target: run.target,
 690         });
 691     }
 692
 693     /// Creates a tarball of save-analysis metadata, if available.
 694     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 695         let compiler = self.compiler;
 696         let target = self.target;
 697         assert!(builder.config.extended);
 698         if compiler.host != builder.config.build {
 699             return None;
 700         }
 701
 702         builder.ensure(compile::Std { compiler, target });
 703         let src = builder
 704             .stage_out(compiler, Mode::Std)
 705             .join(target.triple)
 706             .join(builder.cargo_dir())
 707             .join("deps")
 708             .join("save-analysis");
 709
 710         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 711         tarball.include_target_in_component_name(true);
 712         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 713         Some(tarball.generate())
 714     }
 715 }
 716
 717 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 718 /// `dst_dir`.
 719 fn copy_src_dirs(
 720     builder: &Builder<'_>,
 721     base: &Path,
 722     src_dirs: &[&str],
 723     exclude_dirs: &[&str],
 724     dst_dir: &Path,
 725 ) {
 726     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 727         let spath = match path.to_str() {
 728             Some(path) => path,
 729             None => return false,
 730         };
 731         if spath.ends_with('~') || spath.ends_with(".pyc") {
 732             return false;
 733         }
 734
 735         const LLVM_PROJECTS: &[&str] = &[
 736             "llvm-project/clang",
 737             "llvm-project\\clang",
 738             "llvm-project/libunwind",
 739             "llvm-project\\libunwind",
 740             "llvm-project/lld",
 741             "llvm-project\\lld",
 742             "llvm-project/lldb",
 743             "llvm-project\\lldb",
 744             "llvm-project/llvm",
 745             "llvm-project\\llvm",
 746             "llvm-project/compiler-rt",
 747             "llvm-project\\compiler-rt",
 748         ];
 749         if spath.contains("llvm-project")
 750             && !spath.ends_with("llvm-project")
 751             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 752         {
 753             return false;
 754         }
 755
 756         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 757         if LLVM_TEST.iter().any(|path| spath.contains(path))
 758             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 759         {
 760             return false;
 761         }
 762
 763         let full_path = Path::new(dir).join(path);
 764         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 765             return false;
 766         }
 767
 768         let excludes = [
 769             "CVS",
 770             "RCS",
 771             "SCCS",
 772             ".git",
 773             ".gitignore",
 774             ".gitmodules",
 775             ".gitattributes",
 776             ".cvsignore",
 777             ".svn",
 778             ".arch-ids",
 779             "{arch}",
 780             "=RELEASE-ID",
 781             "=meta-update",
 782             "=update",
 783             ".bzr",
 784             ".bzrignore",
 785             ".bzrtags",
 786             ".hg",
 787             ".hgignore",
 788             ".hgrags",
 789             "_darcs",
 790         ];
 791         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 792     }
 793
 794     // Copy the directories using our filter
 795     for item in src_dirs {
 796         let dst = &dst_dir.join(item);
 797         t!(fs::create_dir_all(dst));
 798         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 799     }
 800 }
 801
 802 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 803 pub struct Src;
 804
 805 impl Step for Src {
 806     /// The output path of the src installer tarball
 807     type Output = GeneratedTarball;
 808     const DEFAULT: bool = true;
 809     const ONLY_HOSTS: bool = true;
 810
 811     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 812         run.path("src")
 813     }
 814
 815     fn make_run(run: RunConfig<'_>) {
 816         run.builder.ensure(Src);
 817     }
 818
 819     /// Creates the `rust-src` installer component
 820     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 821         let tarball = Tarball::new_targetless(builder, "rust-src");
 822
 823         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 824         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 825         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 826         // and fix them...
 827         //
 828         // NOTE: if you update the paths here, you also should update the "virtual" path
 829         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 830         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 831
 832         let src_files = ["Cargo.lock"];
 833         // This is the reduced set of paths which will become the rust-src component
 834         // (essentially libstd and all of its path dependencies).
 835         copy_src_dirs(
 836             builder,
 837             &builder.src,
 838             &["library", "src/llvm-project/libunwind"],
 839             &[
 840                 // not needed and contains symlinks which rustup currently
 841                 // chokes on when unpacking.
 842                 "library/backtrace/crates",
 843             ],
 844             &dst_src,
 845         );
 846         for file in src_files.iter() {
 847             builder.copy(&builder.src.join(file), &dst_src.join(file));
 848         }
 849
 850         tarball.generate()
 851     }
 852 }
 853
 854 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 855 pub struct PlainSourceTarball;
 856
 857 impl Step for PlainSourceTarball {
 858     /// Produces the location of the tarball generated
 859     type Output = GeneratedTarball;
 860     const DEFAULT: bool = true;
 861     const ONLY_HOSTS: bool = true;
 862
 863     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 864         let builder = run.builder;
 865         run.path("src").default_condition(builder.config.rust_dist_src)
 866     }
 867
 868     fn make_run(run: RunConfig<'_>) {
 869         run.builder.ensure(PlainSourceTarball);
 870     }
 871
 872     /// Creates the plain source tarball
 873     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 874         let tarball = Tarball::new(builder, "rustc", "src");
 875         let plain_dst_src = tarball.image_dir();
 876
 877         // This is the set of root paths which will become part of the source package
 878         let src_files = [
 879             "COPYRIGHT",
 880             "LICENSE-APACHE",
 881             "LICENSE-MIT",
 882             "CONTRIBUTING.md",
 883             "README.md",
 884             "RELEASES.md",
 885             "configure",
 886             "x.py",
 887             "config.toml.example",
 888             "Cargo.toml",
 889             "Cargo.lock",
 890         ];
 891         let src_dirs = ["src", "compiler", "library"];
 892
 893         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 894
 895         // Copy the files normally
 896         for item in &src_files {
 897             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 898         }
 899
 900         // Create the version file
 901         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 902         if let Some(sha) = builder.rust_sha() {
 903             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 904         }
 905
 906         // If we're building from git sources, we need to vendor a complete distribution.
 907         if builder.rust_info.is_git() {
 908             // Vendor all Cargo dependencies
 909             let mut cmd = Command::new(&builder.initial_cargo);
 910             cmd.arg("vendor")
 911                 .arg("--sync")
 912                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 913                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 914                 .current_dir(&plain_dst_src);
 915             builder.run(&mut cmd);
 916         }
 917
 918         tarball.bare()
 919     }
 920 }
 921
 922 // We have to run a few shell scripts, which choke quite a bit on both `\`
 923 // characters and on `C:\` paths, so normalize both of them away.
 924 pub fn sanitize_sh(path: &Path) -> String {
 925     let path = path.to_str().unwrap().replace("\\", "/");
 926     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 927
 928     fn unc_to_lfs(s: &str) -> &str {
 929         s.strip_prefix("//?/").unwrap_or(s)
 930     }
 931
 932     fn change_drive(s: &str) -> Option<String> {
 933         let mut ch = s.chars();
 934         let drive = ch.next().unwrap_or('C');
 935         if ch.next() != Some(':') {
 936             return None;
 937         }
 938         if ch.next() != Some('/') {
 939             return None;
 940         }
 941         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 942     }
 943 }
 944
 945 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 946 pub struct Cargo {
 947     pub compiler: Compiler,
 948     pub target: TargetSelection,
 949 }
 950
 951 impl Step for Cargo {
 952     type Output = GeneratedTarball;
 953     const ONLY_HOSTS: bool = true;
 954
 955     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 956         run.path("cargo")
 957     }
 958
 959     fn make_run(run: RunConfig<'_>) {
 960         run.builder.ensure(Cargo {
 961             compiler: run.builder.compiler_for(
 962                 run.builder.top_stage,
 963                 run.builder.config.build,
 964                 run.target,
 965             ),
 966             target: run.target,
 967         });
 968     }
 969
 970     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 971         let compiler = self.compiler;
 972         let target = self.target;
 973
 974         let cargo = builder.ensure(tool::Cargo { compiler, target });
 975         let src = builder.src.join("src/tools/cargo");
 976         let etc = src.join("src/etc");
 977
 978         // Prepare the image directory
 979         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 980         tarball.set_overlay(OverlayKind::Cargo);
 981
 982         tarball.add_file(&cargo, "bin", 0o755);
 983         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 984         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 985         tarball.add_dir(etc.join("man"), "share/man/man1");
 986         tarball.add_legal_and_readme_to("share/doc/cargo");
 987
 988         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 989             let dirent = dirent.expect("read dir entry");
 990             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 991                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 992             }
 993         }
 994
 995         tarball.generate()
 996     }
 997 }
 998
 999 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1000 pub struct Rls {
1001     pub compiler: Compiler,
1002     pub target: TargetSelection,
1003 }
1004
1005 impl Step for Rls {
1006     type Output = Option<GeneratedTarball>;
1007     const ONLY_HOSTS: bool = true;
1008
1009     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1010         run.path("rls")
1011     }
1012
1013     fn make_run(run: RunConfig<'_>) {
1014         run.builder.ensure(Rls {
1015             compiler: run.builder.compiler_for(
1016                 run.builder.top_stage,
1017                 run.builder.config.build,
1018                 run.target,
1019             ),
1020             target: run.target,
1021         });
1022     }
1023
1024     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1025         let compiler = self.compiler;
1026         let target = self.target;
1027         assert!(builder.config.extended);
1028
1029         let rls = builder
1030             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1031             .or_else(|| {
1032                 // We ignore failure on aarch64 Windows because RLS currently
1033                 // fails to build, due to winapi 0.2 not supporting aarch64.
1034                 missing_tool(
1035                     "RLS",
1036                     builder.build.config.missing_tools
1037                         || (target.triple.contains("aarch64") && target.triple.contains("windows")),
1038                 );
1039                 None
1040             })?;
1041
1042         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1043         tarball.set_overlay(OverlayKind::RLS);
1044         tarball.is_preview(true);
1045         tarball.add_file(rls, "bin", 0o755);
1046         tarball.add_legal_and_readme_to("share/doc/rls");
1047         Some(tarball.generate())
1048     }
1049 }
1050
1051 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1052 pub struct RustAnalyzer {
1053     pub compiler: Compiler,
1054     pub target: TargetSelection,
1055 }
1056
1057 impl Step for RustAnalyzer {
1058     type Output = Option<GeneratedTarball>;
1059     const ONLY_HOSTS: bool = true;
1060
1061     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1062         run.path("rust-analyzer")
1063     }
1064
1065     fn make_run(run: RunConfig<'_>) {
1066         run.builder.ensure(RustAnalyzer {
1067             compiler: run.builder.compiler_for(
1068                 run.builder.top_stage,
1069                 run.builder.config.build,
1070                 run.target,
1071             ),
1072             target: run.target,
1073         });
1074     }
1075
1076     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1077         let compiler = self.compiler;
1078         let target = self.target;
1079         assert!(builder.config.extended);
1080
1081         if target.contains("riscv64") {
1082             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1083             // to build. See #74813 for details.
1084             return None;
1085         }
1086
1087         let rust_analyzer = builder
1088             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1089             .expect("rust-analyzer always builds");
1090
1091         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1092         tarball.set_overlay(OverlayKind::RustAnalyzer);
1093         tarball.is_preview(true);
1094         tarball.add_file(rust_analyzer, "bin", 0o755);
1095         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1096         Some(tarball.generate())
1097     }
1098 }
1099
1100 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1101 pub struct Clippy {
1102     pub compiler: Compiler,
1103     pub target: TargetSelection,
1104 }
1105
1106 impl Step for Clippy {
1107     type Output = GeneratedTarball;
1108     const ONLY_HOSTS: bool = true;
1109
1110     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1111         run.path("clippy")
1112     }
1113
1114     fn make_run(run: RunConfig<'_>) {
1115         run.builder.ensure(Clippy {
1116             compiler: run.builder.compiler_for(
1117                 run.builder.top_stage,
1118                 run.builder.config.build,
1119                 run.target,
1120             ),
1121             target: run.target,
1122         });
1123     }
1124
1125     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
1126         let compiler = self.compiler;
1127         let target = self.target;
1128         assert!(builder.config.extended);
1129
1130         // Prepare the image directory
1131         // We expect clippy to build, because we've exited this step above if tool
1132         // state for clippy isn't testing.
1133         let clippy = builder
1134             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1135             .expect("clippy expected to build - essential tool");
1136         let cargoclippy = builder
1137             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1138             .expect("clippy expected to build - essential tool");
1139
1140         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1141         tarball.set_overlay(OverlayKind::Clippy);
1142         tarball.is_preview(true);
1143         tarball.add_file(clippy, "bin", 0o755);
1144         tarball.add_file(cargoclippy, "bin", 0o755);
1145         tarball.add_legal_and_readme_to("share/doc/clippy");
1146         tarball.generate()
1147     }
1148 }
1149
1150 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1151 pub struct Miri {
1152     pub compiler: Compiler,
1153     pub target: TargetSelection,
1154 }
1155
1156 impl Step for Miri {
1157     type Output = Option<GeneratedTarball>;
1158     const ONLY_HOSTS: bool = true;
1159
1160     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1161         run.path("miri")
1162     }
1163
1164     fn make_run(run: RunConfig<'_>) {
1165         run.builder.ensure(Miri {
1166             compiler: run.builder.compiler_for(
1167                 run.builder.top_stage,
1168                 run.builder.config.build,
1169                 run.target,
1170             ),
1171             target: run.target,
1172         });
1173     }
1174
1175     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1176         let compiler = self.compiler;
1177         let target = self.target;
1178         assert!(builder.config.extended);
1179
1180         let miri = builder
1181             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1182             .or_else(|| {
1183                 missing_tool("miri", builder.build.config.missing_tools);
1184                 None
1185             })?;
1186         let cargomiri = builder
1187             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1188             .or_else(|| {
1189                 missing_tool("cargo miri", builder.build.config.missing_tools);
1190                 None
1191             })?;
1192
1193         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1194         tarball.set_overlay(OverlayKind::Miri);
1195         tarball.is_preview(true);
1196         tarball.add_file(miri, "bin", 0o755);
1197         tarball.add_file(cargomiri, "bin", 0o755);
1198         tarball.add_legal_and_readme_to("share/doc/miri");
1199         Some(tarball.generate())
1200     }
1201 }
1202
1203 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1204 pub struct Rustfmt {
1205     pub compiler: Compiler,
1206     pub target: TargetSelection,
1207 }
1208
1209 impl Step for Rustfmt {
1210     type Output = Option<GeneratedTarball>;
1211     const ONLY_HOSTS: bool = true;
1212
1213     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1214         run.path("rustfmt")
1215     }
1216
1217     fn make_run(run: RunConfig<'_>) {
1218         run.builder.ensure(Rustfmt {
1219             compiler: run.builder.compiler_for(
1220                 run.builder.top_stage,
1221                 run.builder.config.build,
1222                 run.target,
1223             ),
1224             target: run.target,
1225         });
1226     }
1227
1228     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1229         let compiler = self.compiler;
1230         let target = self.target;
1231
1232         let rustfmt = builder
1233             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1234             .or_else(|| {
1235                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1236                 None
1237             })?;
1238         let cargofmt = builder
1239             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1240             .or_else(|| {
1241                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1242                 None
1243             })?;
1244
1245         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1246         tarball.set_overlay(OverlayKind::Rustfmt);
1247         tarball.is_preview(true);
1248         tarball.add_file(rustfmt, "bin", 0o755);
1249         tarball.add_file(cargofmt, "bin", 0o755);
1250         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1251         Some(tarball.generate())
1252     }
1253 }
1254
1255 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1256 pub struct Extended {
1257     stage: u32,
1258     host: TargetSelection,
1259     target: TargetSelection,
1260 }
1261
1262 impl Step for Extended {
1263     type Output = ();
1264     const DEFAULT: bool = true;
1265     const ONLY_HOSTS: bool = true;
1266
1267     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1268         let builder = run.builder;
1269         run.path("extended").default_condition(builder.config.extended)
1270     }
1271
1272     fn make_run(run: RunConfig<'_>) {
1273         run.builder.ensure(Extended {
1274             stage: run.builder.top_stage,
1275             host: run.builder.config.build,
1276             target: run.target,
1277         });
1278     }
1279
1280     /// Creates a combined installer for the specified target in the provided stage.
1281     fn run(self, builder: &Builder<'_>) {
1282         let target = self.target;
1283         let stage = self.stage;
1284         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1285
1286         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1287
1288         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
1289         let cargo_installer = builder.ensure(Cargo { compiler, target });
1290         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1291         let rls_installer = builder.ensure(Rls { compiler, target });
1292         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
1293         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1294         let clippy_installer = builder.ensure(Clippy { compiler, target });
1295         let miri_installer = builder.ensure(Miri { compiler, target });
1296         let mingw_installer = builder.ensure(Mingw { host: target });
1297         let analysis_installer = builder.ensure(Analysis { compiler, target });
1298
1299         let docs_installer = builder.ensure(Docs { host: target });
1300         let std_installer = builder.ensure(Std { compiler, target });
1301
1302         let etc = builder.src.join("src/etc/installer");
1303
1304         // Avoid producing tarballs during a dry run.
1305         if builder.config.dry_run {
1306             return;
1307         }
1308
1309         // When rust-std package split from rustc, we needed to ensure that during
1310         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1311         // the std files during uninstall. To do this ensure that rustc comes
1312         // before rust-std in the list below.
1313         let mut tarballs = Vec::new();
1314         tarballs.push(rustc_installer);
1315         tarballs.push(cargo_installer);
1316         tarballs.extend(rls_installer.clone());
1317         tarballs.extend(rust_analyzer_installer.clone());
1318         tarballs.push(clippy_installer);
1319         tarballs.extend(miri_installer.clone());
1320         tarballs.extend(rustfmt_installer.clone());
1321         tarballs.extend(llvm_tools_installer);
1322         if let Some(analysis_installer) = analysis_installer {
1323             tarballs.push(analysis_installer);
1324         }
1325         tarballs.push(std_installer.expect("missing std"));
1326         if let Some(docs_installer) = docs_installer {
1327             tarballs.push(docs_installer);
1328         }
1329         if target.contains("pc-windows-gnu") {
1330             tarballs.push(mingw_installer.unwrap());
1331         }
1332
1333         let tarball = Tarball::new(builder, "rust", &target.triple);
1334         let generated = tarball.combine(&tarballs);
1335
1336         let tmp = tmpdir(builder).join("combined-tarball");
1337         let work = generated.work_dir();
1338
1339         let mut license = String::new();
1340         license += &builder.read(&builder.src.join("COPYRIGHT"));
1341         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1342         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1343         license.push('\n');
1344         license.push('\n');
1345
1346         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1347         let mut rtf = rtf.to_string();
1348         rtf.push('\n');
1349         for line in license.lines() {
1350             rtf.push_str(line);
1351             rtf.push_str("\\line ");
1352         }
1353         rtf.push('}');
1354
1355         fn filter(contents: &str, marker: &str) -> String {
1356             let start = format!("tool-{}-start", marker);
1357             let end = format!("tool-{}-end", marker);
1358             let mut lines = Vec::new();
1359             let mut omitted = false;
1360             for line in contents.lines() {
1361                 if line.contains(&start) {
1362                     omitted = true;
1363                 } else if line.contains(&end) {
1364                     omitted = false;
1365                 } else if !omitted {
1366                     lines.push(line);
1367                 }
1368             }
1369
1370             lines.join("\n")
1371         }
1372
1373         let xform = |p: &Path| {
1374             let mut contents = t!(fs::read_to_string(p));
1375             if rls_installer.is_none() {
1376                 contents = filter(&contents, "rls");
1377             }
1378             if rust_analyzer_installer.is_none() {
1379                 contents = filter(&contents, "rust-analyzer");
1380             }
1381             if miri_installer.is_none() {
1382                 contents = filter(&contents, "miri");
1383             }
1384             if rustfmt_installer.is_none() {
1385                 contents = filter(&contents, "rustfmt");
1386             }
1387             let ret = tmp.join(p.file_name().unwrap());
1388             t!(fs::write(&ret, &contents));
1389             ret
1390         };
1391
1392         if target.contains("apple-darwin") {
1393             builder.info("building pkg installer");
1394             let pkg = tmp.join("pkg");
1395             let _ = fs::remove_dir_all(&pkg);
1396
1397             let pkgbuild = |component: &str| {
1398                 let mut cmd = Command::new("pkgbuild");
1399                 cmd.arg("--identifier")
1400                     .arg(format!("org.rust-lang.{}", component))
1401                     .arg("--scripts")
1402                     .arg(pkg.join(component))
1403                     .arg("--nopayload")
1404                     .arg(pkg.join(component).with_extension("pkg"));
1405                 builder.run(&mut cmd);
1406             };
1407
1408             let prepare = |name: &str| {
1409                 builder.create_dir(&pkg.join(name));
1410                 builder.cp_r(
1411                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1412                     &pkg.join(name),
1413                 );
1414                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1415                 pkgbuild(name);
1416             };
1417             prepare("rustc");
1418             prepare("cargo");
1419             prepare("rust-docs");
1420             prepare("rust-std");
1421             prepare("rust-analysis");
1422             prepare("clippy");
1423
1424             if rls_installer.is_some() {
1425                 prepare("rls");
1426             }
1427             if rust_analyzer_installer.is_some() {
1428                 prepare("rust-analyzer");
1429             }
1430             if miri_installer.is_some() {
1431                 prepare("miri");
1432             }
1433
1434             // create an 'uninstall' package
1435             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1436             pkgbuild("uninstall");
1437
1438             builder.create_dir(&pkg.join("res"));
1439             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1440             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1441             let mut cmd = Command::new("productbuild");
1442             cmd.arg("--distribution")
1443                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1444                 .arg("--resources")
1445                 .arg(pkg.join("res"))
1446                 .arg(distdir(builder).join(format!(
1447                     "{}-{}.pkg",
1448                     pkgname(builder, "rust"),
1449                     target.triple
1450                 )))
1451                 .arg("--package-path")
1452                 .arg(&pkg);
1453             let _time = timeit(builder);
1454             builder.run(&mut cmd);
1455         }
1456
1457         if target.contains("windows") {
1458             let exe = tmp.join("exe");
1459             let _ = fs::remove_dir_all(&exe);
1460
1461             let prepare = |name: &str| {
1462                 builder.create_dir(&exe.join(name));
1463                 let dir = if name == "rust-std" || name == "rust-analysis" {
1464                     format!("{}-{}", name, target.triple)
1465                 } else if name == "rls" {
1466                     "rls-preview".to_string()
1467                 } else if name == "rust-analyzer" {
1468                     "rust-analyzer-preview".to_string()
1469                 } else if name == "clippy" {
1470                     "clippy-preview".to_string()
1471                 } else if name == "miri" {
1472                     "miri-preview".to_string()
1473                 } else {
1474                     name.to_string()
1475                 };
1476                 builder.cp_r(
1477                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1478                     &exe.join(name),
1479                 );
1480                 builder.remove(&exe.join(name).join("manifest.in"));
1481             };
1482             prepare("rustc");
1483             prepare("cargo");
1484             prepare("rust-analysis");
1485             prepare("rust-docs");
1486             prepare("rust-std");
1487             prepare("clippy");
1488             if rls_installer.is_some() {
1489                 prepare("rls");
1490             }
1491             if rust_analyzer_installer.is_some() {
1492                 prepare("rust-analyzer");
1493             }
1494             if miri_installer.is_some() {
1495                 prepare("miri");
1496             }
1497             if target.contains("windows-gnu") {
1498                 prepare("rust-mingw");
1499             }
1500
1501             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1502
1503             // Generate msi installer
1504             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1505             let heat = wix.join("bin/heat.exe");
1506             let candle = wix.join("bin/candle.exe");
1507             let light = wix.join("bin/light.exe");
1508
1509             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1510             builder.run(
1511                 Command::new(&heat)
1512                     .current_dir(&exe)
1513                     .arg("dir")
1514                     .arg("rustc")
1515                     .args(&heat_flags)
1516                     .arg("-cg")
1517                     .arg("RustcGroup")
1518                     .arg("-dr")
1519                     .arg("Rustc")
1520                     .arg("-var")
1521                     .arg("var.RustcDir")
1522                     .arg("-out")
1523                     .arg(exe.join("RustcGroup.wxs")),
1524             );
1525             builder.run(
1526                 Command::new(&heat)
1527                     .current_dir(&exe)
1528                     .arg("dir")
1529                     .arg("rust-docs")
1530                     .args(&heat_flags)
1531                     .arg("-cg")
1532                     .arg("DocsGroup")
1533                     .arg("-dr")
1534                     .arg("Docs")
1535                     .arg("-var")
1536                     .arg("var.DocsDir")
1537                     .arg("-out")
1538                     .arg(exe.join("DocsGroup.wxs"))
1539                     .arg("-t")
1540                     .arg(etc.join("msi/squash-components.xsl")),
1541             );
1542             builder.run(
1543                 Command::new(&heat)
1544                     .current_dir(&exe)
1545                     .arg("dir")
1546                     .arg("cargo")
1547                     .args(&heat_flags)
1548                     .arg("-cg")
1549                     .arg("CargoGroup")
1550                     .arg("-dr")
1551                     .arg("Cargo")
1552                     .arg("-var")
1553                     .arg("var.CargoDir")
1554                     .arg("-out")
1555                     .arg(exe.join("CargoGroup.wxs"))
1556                     .arg("-t")
1557                     .arg(etc.join("msi/remove-duplicates.xsl")),
1558             );
1559             builder.run(
1560                 Command::new(&heat)
1561                     .current_dir(&exe)
1562                     .arg("dir")
1563                     .arg("rust-std")
1564                     .args(&heat_flags)
1565                     .arg("-cg")
1566                     .arg("StdGroup")
1567                     .arg("-dr")
1568                     .arg("Std")
1569                     .arg("-var")
1570                     .arg("var.StdDir")
1571                     .arg("-out")
1572                     .arg(exe.join("StdGroup.wxs")),
1573             );
1574             if rls_installer.is_some() {
1575                 builder.run(
1576                     Command::new(&heat)
1577                         .current_dir(&exe)
1578                         .arg("dir")
1579                         .arg("rls")
1580                         .args(&heat_flags)
1581                         .arg("-cg")
1582                         .arg("RlsGroup")
1583                         .arg("-dr")
1584                         .arg("Rls")
1585                         .arg("-var")
1586                         .arg("var.RlsDir")
1587                         .arg("-out")
1588                         .arg(exe.join("RlsGroup.wxs"))
1589                         .arg("-t")
1590                         .arg(etc.join("msi/remove-duplicates.xsl")),
1591                 );
1592             }
1593             if rust_analyzer_installer.is_some() {
1594                 builder.run(
1595                     Command::new(&heat)
1596                         .current_dir(&exe)
1597                         .arg("dir")
1598                         .arg("rust-analyzer")
1599                         .args(&heat_flags)
1600                         .arg("-cg")
1601                         .arg("RustAnalyzerGroup")
1602                         .arg("-dr")
1603                         .arg("RustAnalyzer")
1604                         .arg("-var")
1605                         .arg("var.RustAnalyzerDir")
1606                         .arg("-out")
1607                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1608                         .arg("-t")
1609                         .arg(etc.join("msi/remove-duplicates.xsl")),
1610                 );
1611             }
1612             builder.run(
1613                 Command::new(&heat)
1614                     .current_dir(&exe)
1615                     .arg("dir")
1616                     .arg("clippy")
1617                     .args(&heat_flags)
1618                     .arg("-cg")
1619                     .arg("ClippyGroup")
1620                     .arg("-dr")
1621                     .arg("Clippy")
1622                     .arg("-var")
1623                     .arg("var.ClippyDir")
1624                     .arg("-out")
1625                     .arg(exe.join("ClippyGroup.wxs"))
1626                     .arg("-t")
1627                     .arg(etc.join("msi/remove-duplicates.xsl")),
1628             );
1629             if miri_installer.is_some() {
1630                 builder.run(
1631                     Command::new(&heat)
1632                         .current_dir(&exe)
1633                         .arg("dir")
1634                         .arg("miri")
1635                         .args(&heat_flags)
1636                         .arg("-cg")
1637                         .arg("MiriGroup")
1638                         .arg("-dr")
1639                         .arg("Miri")
1640                         .arg("-var")
1641                         .arg("var.MiriDir")
1642                         .arg("-out")
1643                         .arg(exe.join("MiriGroup.wxs"))
1644                         .arg("-t")
1645                         .arg(etc.join("msi/remove-duplicates.xsl")),
1646                 );
1647             }
1648             builder.run(
1649                 Command::new(&heat)
1650                     .current_dir(&exe)
1651                     .arg("dir")
1652                     .arg("rust-analysis")
1653                     .args(&heat_flags)
1654                     .arg("-cg")
1655                     .arg("AnalysisGroup")
1656                     .arg("-dr")
1657                     .arg("Analysis")
1658                     .arg("-var")
1659                     .arg("var.AnalysisDir")
1660                     .arg("-out")
1661                     .arg(exe.join("AnalysisGroup.wxs"))
1662                     .arg("-t")
1663                     .arg(etc.join("msi/remove-duplicates.xsl")),
1664             );
1665             if target.contains("windows-gnu") {
1666                 builder.run(
1667                     Command::new(&heat)
1668                         .current_dir(&exe)
1669                         .arg("dir")
1670                         .arg("rust-mingw")
1671                         .args(&heat_flags)
1672                         .arg("-cg")
1673                         .arg("GccGroup")
1674                         .arg("-dr")
1675                         .arg("Gcc")
1676                         .arg("-var")
1677                         .arg("var.GccDir")
1678                         .arg("-out")
1679                         .arg(exe.join("GccGroup.wxs")),
1680                 );
1681             }
1682
1683             let candle = |input: &Path| {
1684                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1685                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1686                 let mut cmd = Command::new(&candle);
1687                 cmd.current_dir(&exe)
1688                     .arg("-nologo")
1689                     .arg("-dRustcDir=rustc")
1690                     .arg("-dDocsDir=rust-docs")
1691                     .arg("-dCargoDir=cargo")
1692                     .arg("-dStdDir=rust-std")
1693                     .arg("-dAnalysisDir=rust-analysis")
1694                     .arg("-dClippyDir=clippy")
1695                     .arg("-arch")
1696                     .arg(&arch)
1697                     .arg("-out")
1698                     .arg(&output)
1699                     .arg(&input);
1700                 add_env(builder, &mut cmd, target);
1701
1702                 if rls_installer.is_some() {
1703                     cmd.arg("-dRlsDir=rls");
1704                 }
1705                 if rust_analyzer_installer.is_some() {
1706                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1707                 }
1708                 if miri_installer.is_some() {
1709                     cmd.arg("-dMiriDir=miri");
1710                 }
1711                 if target.contains("windows-gnu") {
1712                     cmd.arg("-dGccDir=rust-mingw");
1713                 }
1714                 builder.run(&mut cmd);
1715             };
1716             candle(&xform(&etc.join("msi/rust.wxs")));
1717             candle(&etc.join("msi/ui.wxs"));
1718             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1719             candle("RustcGroup.wxs".as_ref());
1720             candle("DocsGroup.wxs".as_ref());
1721             candle("CargoGroup.wxs".as_ref());
1722             candle("StdGroup.wxs".as_ref());
1723             candle("ClippyGroup.wxs".as_ref());
1724             if rls_installer.is_some() {
1725                 candle("RlsGroup.wxs".as_ref());
1726             }
1727             if rust_analyzer_installer.is_some() {
1728                 candle("RustAnalyzerGroup.wxs".as_ref());
1729             }
1730             if miri_installer.is_some() {
1731                 candle("MiriGroup.wxs".as_ref());
1732             }
1733             candle("AnalysisGroup.wxs".as_ref());
1734
1735             if target.contains("windows-gnu") {
1736                 candle("GccGroup.wxs".as_ref());
1737             }
1738
1739             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1740             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1741             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1742
1743             builder.info(&format!("building `msi` installer with {:?}", light));
1744             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1745             let mut cmd = Command::new(&light);
1746             cmd.arg("-nologo")
1747                 .arg("-ext")
1748                 .arg("WixUIExtension")
1749                 .arg("-ext")
1750                 .arg("WixUtilExtension")
1751                 .arg("-out")
1752                 .arg(exe.join(&filename))
1753                 .arg("rust.wixobj")
1754                 .arg("ui.wixobj")
1755                 .arg("rustwelcomedlg.wixobj")
1756                 .arg("RustcGroup.wixobj")
1757                 .arg("DocsGroup.wixobj")
1758                 .arg("CargoGroup.wixobj")
1759                 .arg("StdGroup.wixobj")
1760                 .arg("AnalysisGroup.wixobj")
1761                 .arg("ClippyGroup.wixobj")
1762                 .current_dir(&exe);
1763
1764             if rls_installer.is_some() {
1765                 cmd.arg("RlsGroup.wixobj");
1766             }
1767             if rust_analyzer_installer.is_some() {
1768                 cmd.arg("RustAnalyzerGroup.wixobj");
1769             }
1770             if miri_installer.is_some() {
1771                 cmd.arg("MiriGroup.wixobj");
1772             }
1773
1774             if target.contains("windows-gnu") {
1775                 cmd.arg("GccGroup.wixobj");
1776             }
1777             // ICE57 wrongly complains about the shortcuts
1778             cmd.arg("-sice:ICE57");
1779
1780             let _time = timeit(builder);
1781             builder.run(&mut cmd);
1782
1783             if !builder.config.dry_run {
1784                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1785             }
1786         }
1787     }
1788 }
1789
1790 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1791     let mut parts = builder.version.split('.');
1792     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1793         .env("CFG_RELEASE_NUM", &builder.version)
1794         .env("CFG_RELEASE", builder.rust_release())
1795         .env("CFG_VER_MAJOR", parts.next().unwrap())
1796         .env("CFG_VER_MINOR", parts.next().unwrap())
1797         .env("CFG_VER_PATCH", parts.next().unwrap())
1798         .env("CFG_VER_BUILD", "0") // just needed to build
1799         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1800         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1801         .env("CFG_BUILD", target.triple)
1802         .env("CFG_CHANNEL", &builder.config.channel);
1803
1804     if target.contains("windows-gnu") {
1805         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1806     } else {
1807         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1808     }
1809
1810     if target.contains("x86_64") {
1811         cmd.env("CFG_PLATFORM", "x64");
1812     } else {
1813         cmd.env("CFG_PLATFORM", "x86");
1814     }
1815 }
1816
1817 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1818 ///
1819
1820 /// Returns whether the files were actually copied.
1821 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1822     if let Some(config) = builder.config.target_config.get(&target) {
1823         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1824             // If the LLVM was externally provided, then we don't currently copy
1825             // artifacts into the sysroot. This is not necessarily the right
1826             // choice (in particular, it will require the LLVM dylib to be in
1827             // the linker's load path at runtime), but the common use case for
1828             // external LLVMs is distribution provided LLVMs, and in that case
1829             // they're usually in the standard search path (e.g., /usr/lib) and
1830             // copying them here is going to cause problems as we may end up
1831             // with the wrong files and isn't what distributions want.
1832             //
1833             // This behavior may be revisited in the future though.
1834             //
1835             // If the LLVM is coming from ourselves (just from CI) though, we
1836             // still want to install it, as it otherwise won't be available.
1837             return false;
1838         }
1839     }
1840
1841     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1842     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1843     // clear why this is the case, though. llvm-config will emit the versioned
1844     // paths and we don't want those in the sysroot (as we're expecting
1845     // unversioned paths).
1846     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1847         let src_libdir = builder.llvm_out(target).join("lib");
1848         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1849         if llvm_dylib_path.exists() {
1850             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1851         }
1852         !builder.config.dry_run
1853     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1854         let mut cmd = Command::new(llvm_config);
1855         cmd.arg("--libfiles");
1856         builder.verbose(&format!("running {:?}", cmd));
1857         let files = output(&mut cmd);
1858         for file in files.trim_end().split(' ') {
1859             builder.install(Path::new(file), dst_libdir, 0o644);
1860         }
1861         !builder.config.dry_run
1862     } else {
1863         false
1864     }
1865 }
1866
1867 /// Maybe add libLLVM.so to the target lib-dir for linking.
1868 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1869     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1870     // We do not need to copy LLVM files into the sysroot if it is not
1871     // dynamically linked; it is already included into librustc_llvm
1872     // statically.
1873     if builder.config.llvm_link_shared {
1874         maybe_install_llvm(builder, target, &dst_libdir);
1875     }
1876 }
1877
1878 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1879 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1880     let dst_libdir =
1881         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1882     // We do not need to copy LLVM files into the sysroot if it is not
1883     // dynamically linked; it is already included into librustc_llvm
1884     // statically.
1885     if builder.config.llvm_link_shared {
1886         maybe_install_llvm(builder, target, &dst_libdir);
1887     }
1888 }
1889
1890 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1891 pub struct LlvmTools {
1892     pub target: TargetSelection,
1893 }
1894
1895 impl Step for LlvmTools {
1896     type Output = Option<GeneratedTarball>;
1897     const ONLY_HOSTS: bool = true;
1898
1899     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1900         run.path("llvm-tools")
1901     }
1902
1903     fn make_run(run: RunConfig<'_>) {
1904         run.builder.ensure(LlvmTools { target: run.target });
1905     }
1906
1907     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1908         let target = self.target;
1909         assert!(builder.config.extended);
1910
1911         /* run only if llvm-config isn't used */
1912         if let Some(config) = builder.config.target_config.get(&target) {
1913             if let Some(ref _s) = config.llvm_config {
1914                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1915                 return None;
1916             }
1917         }
1918
1919         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1920         tarball.set_overlay(OverlayKind::LLVM);
1921         tarball.is_preview(true);
1922
1923         // Prepare the image directory
1924         let src_bindir = builder.llvm_out(target).join("bin");
1925         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1926         for tool in LLVM_TOOLS {
1927             let exe = src_bindir.join(exe(tool, target));
1928             tarball.add_file(&exe, &dst_bindir, 0o755);
1929         }
1930
1931         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1932         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1933         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1934         // compiler libraries.
1935         maybe_install_llvm_target(builder, target, tarball.image_dir());
1936
1937         Some(tarball.generate())
1938     }
1939 }
1940
1941 // Tarball intended for internal consumption to ease rustc/std development.
1942 //
1943 // Should not be considered stable by end users.
1944 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1945 pub struct RustDev {
1946     pub target: TargetSelection,
1947 }
1948
1949 impl Step for RustDev {
1950     type Output = Option<GeneratedTarball>;
1951     const DEFAULT: bool = true;
1952     const ONLY_HOSTS: bool = true;
1953
1954     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1955         run.path("rust-dev")
1956     }
1957
1958     fn make_run(run: RunConfig<'_>) {
1959         run.builder.ensure(RustDev { target: run.target });
1960     }
1961
1962     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1963         let target = self.target;
1964
1965         /* run only if llvm-config isn't used */
1966         if let Some(config) = builder.config.target_config.get(&target) {
1967             if let Some(ref _s) = config.llvm_config {
1968                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
1969                 return None;
1970             }
1971         }
1972
1973         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
1974         tarball.set_overlay(OverlayKind::LLVM);
1975
1976         let src_bindir = builder.llvm_out(target).join("bin");
1977         for bin in &[
1978             "llvm-config",
1979             "llvm-ar",
1980             "llvm-objdump",
1981             "llvm-profdata",
1982             "llvm-bcanalyzer",
1983             "llvm-cov",
1984             "llvm-dwp",
1985         ] {
1986             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
1987         }
1988         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
1989
1990         // Copy the include directory as well; needed mostly to build
1991         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
1992         // just broadly useful to be able to link against the bundled LLVM.
1993         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
1994
1995         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1996         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1997         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1998         // compiler libraries.
1999         let dst_libdir = tarball.image_dir().join("lib");
2000         maybe_install_llvm(builder, target, &dst_libdir);
2001         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2002         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2003
2004         Some(tarball.generate())
2005     }
2006 }
2007
2008 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2009 /// release process to avoid cloning the monorepo and building stuff.
2010 ///
2011 /// Should not be considered stable by end users.
2012 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2013 pub struct BuildManifest {
2014     pub target: TargetSelection,
2015 }
2016
2017 impl Step for BuildManifest {
2018     type Output = GeneratedTarball;
2019     const DEFAULT: bool = false;
2020     const ONLY_HOSTS: bool = true;
2021
2022     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2023         run.path("src/tools/build-manifest")
2024     }
2025
2026     fn make_run(run: RunConfig<'_>) {
2027         run.builder.ensure(BuildManifest { target: run.target });
2028     }
2029
2030     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2031         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2032
2033         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2034         tarball.add_file(&build_manifest, "bin", 0o755);
2035         tarball.generate()
2036     }
2037 }
2038
2039 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2040 ///
2041 /// Currently this is the PGO profile data.
2042 ///
2043 /// Should not be considered stable by end users.
2044 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2045 pub struct ReproducibleArtifacts {
2046     pub target: TargetSelection,
2047 }
2048
2049 impl Step for ReproducibleArtifacts {
2050     type Output = Option<GeneratedTarball>;
2051     const DEFAULT: bool = true;
2052     const ONLY_HOSTS: bool = true;
2053
2054     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2055         run.path("reproducible")
2056     }
2057
2058     fn make_run(run: RunConfig<'_>) {
2059         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2060     }
2061
2062     fn run(self, builder: &Builder<'_>) -> Self::Output {
2063         let path = builder.config.rust_profile_use.as_ref()?;
2064
2065         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2066         tarball.add_file(path, ".", 0o644);
2067         Some(tarball.generate())
2068     }
2069 }