src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14 use std::process::Command;
  15
  16 use build_helper::{output, t};
  17
  18 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  49 pub struct Docs {
  50     pub host: TargetSelection,
  51 }
  52
  53 impl Step for Docs {
  54     type Output = Option<GeneratedTarball>;
  55     const DEFAULT: bool = true;
  56
  57     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  58         run.path("src/doc")
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         if !builder.config.docs {
  69             return None;
  70         }
  71         builder.default_doc(&[]);
  72
  73         let dest = "share/doc/rust/html";
  74
  75         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  76         tarball.set_product_name("Rust Documentation");
  77         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  78         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  79         Some(tarball.generate())
  80     }
  81 }
  82
  83 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  84 pub struct RustcDocs {
  85     pub host: TargetSelection,
  86 }
  87
  88 impl Step for RustcDocs {
  89     type Output = Option<GeneratedTarball>;
  90     const DEFAULT: bool = true;
  91
  92     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  93         run.path("src/librustc")
  94     }
  95
  96     fn make_run(run: RunConfig<'_>) {
  97         run.builder.ensure(RustcDocs { host: run.target });
  98     }
  99
 100     /// Builds the `rustc-docs` installer component.
 101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 102         let host = self.host;
 103         if !builder.config.compiler_docs {
 104             return None;
 105         }
 106         builder.default_doc(&[]);
 107
 108         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 109         tarball.set_product_name("Rustc Documentation");
 110         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 111         Some(tarball.generate())
 112     }
 113 }
 114
 115 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 116     let mut found = Vec::with_capacity(files.len());
 117
 118     for file in files {
 119         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 120
 121         if let Some(file_path) = file_path {
 122             found.push(file_path);
 123         } else {
 124             panic!("Could not find '{}' in {:?}", file, path);
 125         }
 126     }
 127
 128     found
 129 }
 130
 131 fn make_win_dist(
 132     rust_root: &Path,
 133     plat_root: &Path,
 134     target: TargetSelection,
 135     builder: &Builder<'_>,
 136 ) {
 137     //Ask gcc where it keeps its stuff
 138     let mut cmd = Command::new(builder.cc(target));
 139     cmd.arg("-print-search-dirs");
 140     let gcc_out = output(&mut cmd);
 141
 142     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 143     let mut lib_path = Vec::new();
 144
 145     for line in gcc_out.lines() {
 146         let idx = line.find(':').unwrap();
 147         let key = &line[..idx];
 148         let trim_chars: &[_] = &[' ', '='];
 149         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 150
 151         if key == "programs" {
 152             bin_path.extend(value);
 153         } else if key == "libraries" {
 154             lib_path.extend(value);
 155         }
 156     }
 157
 158     let compiler = if target == "i686-pc-windows-gnu" {
 159         "i686-w64-mingw32-gcc.exe"
 160     } else if target == "x86_64-pc-windows-gnu" {
 161         "x86_64-w64-mingw32-gcc.exe"
 162     } else {
 163         "gcc.exe"
 164     };
 165     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 166     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 167     if target.starts_with("i686-") {
 168         rustc_dlls.push("libgcc_s_dw2-1.dll");
 169     } else {
 170         rustc_dlls.push("libgcc_s_seh-1.dll");
 171     }
 172
 173     let target_libs = [
 174         //MinGW libs
 175         "libgcc.a",
 176         "libgcc_eh.a",
 177         "libgcc_s.a",
 178         "libm.a",
 179         "libmingw32.a",
 180         "libmingwex.a",
 181         "libstdc++.a",
 182         "libiconv.a",
 183         "libmoldname.a",
 184         "libpthread.a",
 185         //Windows import libs
 186         "libadvapi32.a",
 187         "libbcrypt.a",
 188         "libcomctl32.a",
 189         "libcomdlg32.a",
 190         "libcredui.a",
 191         "libcrypt32.a",
 192         "libdbghelp.a",
 193         "libgdi32.a",
 194         "libimagehlp.a",
 195         "libiphlpapi.a",
 196         "libkernel32.a",
 197         "libmsimg32.a",
 198         "libmsvcrt.a",
 199         "libodbc32.a",
 200         "libole32.a",
 201         "liboleaut32.a",
 202         "libopengl32.a",
 203         "libpsapi.a",
 204         "librpcrt4.a",
 205         "libsecur32.a",
 206         "libsetupapi.a",
 207         "libshell32.a",
 208         "libsynchronization.a",
 209         "libuser32.a",
 210         "libuserenv.a",
 211         "libuuid.a",
 212         "libwinhttp.a",
 213         "libwinmm.a",
 214         "libwinspool.a",
 215         "libws2_32.a",
 216         "libwsock32.a",
 217     ];
 218
 219     //Find mingw artifacts we want to bundle
 220     let target_tools = find_files(&target_tools, &bin_path);
 221     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 222     let target_libs = find_files(&target_libs, &lib_path);
 223
 224     // Copy runtime dlls next to rustc.exe
 225     let dist_bin_dir = rust_root.join("bin/");
 226     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 227     for src in rustc_dlls {
 228         builder.copy_to_folder(&src, &dist_bin_dir);
 229     }
 230
 231     //Copy platform tools to platform-specific bin directory
 232     let target_bin_dir = plat_root
 233         .join("lib")
 234         .join("rustlib")
 235         .join(target.triple)
 236         .join("bin")
 237         .join("self-contained");
 238     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 239     for src in target_tools {
 240         builder.copy_to_folder(&src, &target_bin_dir);
 241     }
 242
 243     // Warn windows-gnu users that the bundled GCC cannot compile C files
 244     builder.create(
 245         &target_bin_dir.join("GCC-WARNING.txt"),
 246         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 247          used as a linker. In order to be able to compile projects containing C code use \
 248          the GCC provided by MinGW or Cygwin.",
 249     );
 250
 251     //Copy platform libs to platform-specific lib directory
 252     let target_lib_dir = plat_root
 253         .join("lib")
 254         .join("rustlib")
 255         .join(target.triple)
 256         .join("lib")
 257         .join("self-contained");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         builder.copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     pub host: TargetSelection,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<GeneratedTarball>;
 271     const DEFAULT: bool = true;
 272
 273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 274         run.never()
 275     }
 276
 277     fn make_run(run: RunConfig<'_>) {
 278         run.builder.ensure(Mingw { host: run.target });
 279     }
 280
 281     /// Builds the `rust-mingw` installer component.
 282     ///
 283     /// This contains all the bits and pieces to run the MinGW Windows targets
 284     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 285     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 286         let host = self.host;
 287         if !host.contains("pc-windows-gnu") {
 288             return None;
 289         }
 290
 291         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 292         tarball.set_product_name("Rust MinGW");
 293
 294         // The first argument is a "temporary directory" which is just
 295         // thrown away (this contains the runtime DLLs included in the rustc package
 296         // above) and the second argument is where to place all the MinGW components
 297         // (which is what we want).
 298         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 299
 300         Some(tarball.generate())
 301     }
 302 }
 303
 304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 305 pub struct Rustc {
 306     pub compiler: Compiler,
 307 }
 308
 309 impl Step for Rustc {
 310     type Output = GeneratedTarball;
 311     const DEFAULT: bool = true;
 312     const ONLY_HOSTS: bool = true;
 313
 314     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 315         run.path("src/librustc")
 316     }
 317
 318     fn make_run(run: RunConfig<'_>) {
 319         run.builder
 320             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 321     }
 322
 323     /// Creates the `rustc` installer component.
 324     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 325         let compiler = self.compiler;
 326         let host = self.compiler.host;
 327
 328         let tarball = Tarball::new(builder, "rustc", &host.triple);
 329
 330         // Prepare the rustc "image", what will actually end up getting installed
 331         prepare_image(builder, compiler, tarball.image_dir());
 332
 333         // On MinGW we've got a few runtime DLL dependencies that we need to
 334         // include. The first argument to this script is where to put these DLLs
 335         // (the image we're creating), and the second argument is a junk directory
 336         // to ignore all other MinGW stuff the script creates.
 337         //
 338         // On 32-bit MinGW we're always including a DLL which needs some extra
 339         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 340         // anything requiring us to distribute a license, but it's likely the
 341         // install will *also* include the rust-mingw package, which also needs
 342         // licenses, so to be safe we just include it here in all MinGW packages.
 343         if host.contains("pc-windows-gnu") {
 344             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 345             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 346         }
 347
 348         return tarball.generate();
 349
 350         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 351             let host = compiler.host;
 352             let src = builder.sysroot(compiler);
 353
 354             // Copy rustc/rustdoc binaries
 355             t!(fs::create_dir_all(image.join("bin")));
 356             builder.cp_r(&src.join("bin"), &image.join("bin"));
 357
 358             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 359
 360             let libdir_relative = builder.libdir_relative(compiler);
 361
 362             // Copy runtime DLLs needed by the compiler
 363             if libdir_relative.to_str() != Some("bin") {
 364                 let libdir = builder.rustc_libdir(compiler);
 365                 for entry in builder.read_dir(&libdir) {
 366                     let name = entry.file_name();
 367                     if let Some(s) = name.to_str() {
 368                         if is_dylib(s) {
 369                             // Don't use custom libdir here because ^lib/ will be resolved again
 370                             // with installer
 371                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 372                         }
 373                     }
 374                 }
 375             }
 376
 377             // Copy over the codegen backends
 378             let backends_src = builder.sysroot_codegen_backends(compiler);
 379             let backends_rel = backends_src
 380                 .strip_prefix(&src)
 381                 .unwrap()
 382                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 383                 .unwrap();
 384             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 385             let backends_dst = image.join("lib").join(&backends_rel);
 386
 387             t!(fs::create_dir_all(&backends_dst));
 388             builder.cp_r(&backends_src, &backends_dst);
 389
 390             // Copy libLLVM.so to the lib dir as well, if needed. While not
 391             // technically needed by rustc itself it's needed by lots of other
 392             // components like the llvm tools and LLD. LLD is included below and
 393             // tools/LLDB come later, so let's just throw it in the rustc
 394             // component for now.
 395             maybe_install_llvm_runtime(builder, host, image);
 396
 397             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let exe = exe("rust-lld", compiler.host);
 404                 builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 405             }
 406
 407             // Copy over llvm-dwp if it's there
 408             let exe = exe("rust-llvm-dwp", compiler.host);
 409             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 410
 411             // Man pages
 412             t!(fs::create_dir_all(image.join("share/man/man1")));
 413             let man_src = builder.src.join("src/doc/man");
 414             let man_dst = image.join("share/man/man1");
 415
 416             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 417             let time = env::var("SOURCE_DATE_EPOCH")
 418                 .map(|timestamp| {
 419                     let epoch = timestamp
 420                         .parse()
 421                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 422                         .unwrap();
 423
 424                     time::at(Timespec::new(epoch, 0))
 425                 })
 426                 .unwrap_or_else(|_| time::now());
 427
 428             let month_year = t!(time::strftime("%B %Y", &time));
 429             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 430             // to hardlink, and we don't want to edit the source templates
 431             for file_entry in builder.read_dir(&man_src) {
 432                 let page_src = file_entry.path();
 433                 let page_dst = man_dst.join(file_entry.file_name());
 434                 t!(fs::copy(&page_src, &page_dst));
 435                 // template in month/year and version number
 436                 builder.replace_in_file(
 437                     &page_dst,
 438                     &[
 439                         ("<INSERT DATE HERE>", &month_year),
 440                         ("<INSERT VERSION HERE>", &builder.version),
 441                     ],
 442                 );
 443             }
 444
 445             // Debugger scripts
 446             builder
 447                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 448
 449             // Misc license info
 450             let cp = |file: &str| {
 451                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 452             };
 453             cp("COPYRIGHT");
 454             cp("LICENSE-APACHE");
 455             cp("LICENSE-MIT");
 456             cp("README.md");
 457         }
 458     }
 459 }
 460
 461 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 462 pub struct DebuggerScripts {
 463     pub sysroot: Interned<PathBuf>,
 464     pub host: TargetSelection,
 465 }
 466
 467 impl Step for DebuggerScripts {
 468     type Output = ();
 469
 470     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 471         run.path("src/lldb_batchmode.py")
 472     }
 473
 474     fn make_run(run: RunConfig<'_>) {
 475         run.builder.ensure(DebuggerScripts {
 476             sysroot: run
 477                 .builder
 478                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 479             host: run.target,
 480         });
 481     }
 482
 483     /// Copies debugger scripts for `target` into the `sysroot` specified.
 484     fn run(self, builder: &Builder<'_>) {
 485         let host = self.host;
 486         let sysroot = self.sysroot;
 487         let dst = sysroot.join("lib/rustlib/etc");
 488         t!(fs::create_dir_all(&dst));
 489         let cp_debugger_script = |file: &str| {
 490             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 491         };
 492         if host.contains("windows-msvc") {
 493             // windbg debugger scripts
 494             builder.install(
 495                 &builder.src.join("src/etc/rust-windbg.cmd"),
 496                 &sysroot.join("bin"),
 497                 0o755,
 498             );
 499
 500             cp_debugger_script("natvis/intrinsic.natvis");
 501             cp_debugger_script("natvis/liballoc.natvis");
 502             cp_debugger_script("natvis/libcore.natvis");
 503             cp_debugger_script("natvis/libstd.natvis");
 504         } else {
 505             cp_debugger_script("rust_types.py");
 506
 507             // gdb debugger scripts
 508             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 509             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 510
 511             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 512             cp_debugger_script("gdb_lookup.py");
 513             cp_debugger_script("gdb_providers.py");
 514
 515             // lldb debugger scripts
 516             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 517
 518             cp_debugger_script("lldb_lookup.py");
 519             cp_debugger_script("lldb_providers.py");
 520             cp_debugger_script("lldb_commands")
 521         }
 522     }
 523 }
 524
 525 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 526     // The only true set of target libraries came from the build triple, so
 527     // let's reduce redundant work by only producing archives from that host.
 528     if compiler.host != builder.config.build {
 529         builder.info("\tskipping, not a build host");
 530         true
 531     } else {
 532         false
 533     }
 534 }
 535
 536 /// Copy stamped files into an image's `target/lib` directory.
 537 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 538     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 539     let self_contained_dst = dst.join("self-contained");
 540     t!(fs::create_dir_all(&dst));
 541     t!(fs::create_dir_all(&self_contained_dst));
 542     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 543         if dependency_type == DependencyType::TargetSelfContained {
 544             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 545         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 546             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 547         }
 548     }
 549 }
 550
 551 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 552 pub struct Std {
 553     pub compiler: Compiler,
 554     pub target: TargetSelection,
 555 }
 556
 557 impl Step for Std {
 558     type Output = Option<GeneratedTarball>;
 559     const DEFAULT: bool = true;
 560
 561     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 562         run.path("library/std")
 563     }
 564
 565     fn make_run(run: RunConfig<'_>) {
 566         run.builder.ensure(Std {
 567             compiler: run.builder.compiler_for(
 568                 run.builder.top_stage,
 569                 run.builder.config.build,
 570                 run.target,
 571             ),
 572             target: run.target,
 573         });
 574     }
 575
 576     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 577         let compiler = self.compiler;
 578         let target = self.target;
 579
 580         if skip_host_target_lib(builder, compiler) {
 581             return None;
 582         }
 583
 584         builder.ensure(compile::Std { compiler, target });
 585
 586         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 587         tarball.include_target_in_component_name(true);
 588
 589         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 590         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 591         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 592
 593         Some(tarball.generate())
 594     }
 595 }
 596
 597 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 598 pub struct RustcDev {
 599     pub compiler: Compiler,
 600     pub target: TargetSelection,
 601 }
 602
 603 impl Step for RustcDev {
 604     type Output = Option<GeneratedTarball>;
 605     const DEFAULT: bool = true;
 606     const ONLY_HOSTS: bool = true;
 607
 608     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 609         run.path("rustc-dev")
 610     }
 611
 612     fn make_run(run: RunConfig<'_>) {
 613         run.builder.ensure(RustcDev {
 614             compiler: run.builder.compiler_for(
 615                 run.builder.top_stage,
 616                 run.builder.config.build,
 617                 run.target,
 618             ),
 619             target: run.target,
 620         });
 621     }
 622
 623     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 624         let compiler = self.compiler;
 625         let target = self.target;
 626         if skip_host_target_lib(builder, compiler) {
 627             return None;
 628         }
 629
 630         builder.ensure(compile::Rustc { compiler, target });
 631
 632         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 633
 634         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 635         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 636         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 637
 638         let src_files = &["Cargo.lock"];
 639         // This is the reduced set of paths which will become the rustc-dev component
 640         // (essentially the compiler crates and all of their path dependencies).
 641         copy_src_dirs(
 642             builder,
 643             &builder.src,
 644             &["compiler"],
 645             &[],
 646             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 647         );
 648         // This particular crate is used as a build dependency of the above.
 649         copy_src_dirs(
 650             builder,
 651             &builder.src,
 652             &["src/build_helper"],
 653             &[],
 654             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 655         );
 656         for file in src_files {
 657             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 658         }
 659
 660         Some(tarball.generate())
 661     }
 662 }
 663
 664 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 665 pub struct Analysis {
 666     pub compiler: Compiler,
 667     pub target: TargetSelection,
 668 }
 669
 670 impl Step for Analysis {
 671     type Output = Option<GeneratedTarball>;
 672     const DEFAULT: bool = true;
 673
 674     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 675         let builder = run.builder;
 676         run.path("analysis").default_condition(builder.config.extended)
 677     }
 678
 679     fn make_run(run: RunConfig<'_>) {
 680         run.builder.ensure(Analysis {
 681             // Find the actual compiler (handling the full bootstrap option) which
 682             // produced the save-analysis data because that data isn't copied
 683             // through the sysroot uplifting.
 684             compiler: run.builder.compiler_for(
 685                 run.builder.top_stage,
 686                 run.builder.config.build,
 687                 run.target,
 688             ),
 689             target: run.target,
 690         });
 691     }
 692
 693     /// Creates a tarball of save-analysis metadata, if available.
 694     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 695         let compiler = self.compiler;
 696         let target = self.target;
 697         assert!(builder.config.extended);
 698         if compiler.host != builder.config.build {
 699             return None;
 700         }
 701
 702         builder.ensure(compile::Std { compiler, target });
 703         let src = builder
 704             .stage_out(compiler, Mode::Std)
 705             .join(target.triple)
 706             .join(builder.cargo_dir())
 707             .join("deps")
 708             .join("save-analysis");
 709
 710         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 711         tarball.include_target_in_component_name(true);
 712         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 713         Some(tarball.generate())
 714     }
 715 }
 716
 717 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 718 /// `dst_dir`.
 719 fn copy_src_dirs(
 720     builder: &Builder<'_>,
 721     base: &Path,
 722     src_dirs: &[&str],
 723     exclude_dirs: &[&str],
 724     dst_dir: &Path,
 725 ) {
 726     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 727         let spath = match path.to_str() {
 728             Some(path) => path,
 729             None => return false,
 730         };
 731         if spath.ends_with('~') || spath.ends_with(".pyc") {
 732             return false;
 733         }
 734
 735         const LLVM_PROJECTS: &[&str] = &[
 736             "llvm-project/clang",
 737             "llvm-project\\clang",
 738             "llvm-project/libunwind",
 739             "llvm-project\\libunwind",
 740             "llvm-project/lld",
 741             "llvm-project\\lld",
 742             "llvm-project/lldb",
 743             "llvm-project\\lldb",
 744             "llvm-project/llvm",
 745             "llvm-project\\llvm",
 746             "llvm-project/compiler-rt",
 747             "llvm-project\\compiler-rt",
 748         ];
 749         if spath.contains("llvm-project")
 750             && !spath.ends_with("llvm-project")
 751             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 752         {
 753             return false;
 754         }
 755
 756         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 757         if LLVM_TEST.iter().any(|path| spath.contains(path))
 758             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 759         {
 760             return false;
 761         }
 762
 763         let full_path = Path::new(dir).join(path);
 764         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 765             return false;
 766         }
 767
 768         let excludes = [
 769             "CVS",
 770             "RCS",
 771             "SCCS",
 772             ".git",
 773             ".gitignore",
 774             ".gitmodules",
 775             ".gitattributes",
 776             ".cvsignore",
 777             ".svn",
 778             ".arch-ids",
 779             "{arch}",
 780             "=RELEASE-ID",
 781             "=meta-update",
 782             "=update",
 783             ".bzr",
 784             ".bzrignore",
 785             ".bzrtags",
 786             ".hg",
 787             ".hgignore",
 788             ".hgrags",
 789             "_darcs",
 790         ];
 791         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 792     }
 793
 794     // Copy the directories using our filter
 795     for item in src_dirs {
 796         let dst = &dst_dir.join(item);
 797         t!(fs::create_dir_all(dst));
 798         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 799     }
 800 }
 801
 802 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 803 pub struct Src;
 804
 805 impl Step for Src {
 806     /// The output path of the src installer tarball
 807     type Output = GeneratedTarball;
 808     const DEFAULT: bool = true;
 809     const ONLY_HOSTS: bool = true;
 810
 811     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 812         run.path("src")
 813     }
 814
 815     fn make_run(run: RunConfig<'_>) {
 816         run.builder.ensure(Src);
 817     }
 818
 819     /// Creates the `rust-src` installer component
 820     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 821         let tarball = Tarball::new_targetless(builder, "rust-src");
 822
 823         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 824         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 825         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 826         // and fix them...
 827         //
 828         // NOTE: if you update the paths here, you also should update the "virtual" path
 829         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 830         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 831
 832         let src_files = ["Cargo.lock"];
 833         // This is the reduced set of paths which will become the rust-src component
 834         // (essentially libstd and all of its path dependencies).
 835         copy_src_dirs(
 836             builder,
 837             &builder.src,
 838             &["library", "src/llvm-project/libunwind"],
 839             &[
 840                 // not needed and contains symlinks which rustup currently
 841                 // chokes on when unpacking.
 842                 "library/backtrace/crates",
 843             ],
 844             &dst_src,
 845         );
 846         for file in src_files.iter() {
 847             builder.copy(&builder.src.join(file), &dst_src.join(file));
 848         }
 849
 850         tarball.generate()
 851     }
 852 }
 853
 854 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 855 pub struct PlainSourceTarball;
 856
 857 impl Step for PlainSourceTarball {
 858     /// Produces the location of the tarball generated
 859     type Output = GeneratedTarball;
 860     const DEFAULT: bool = true;
 861     const ONLY_HOSTS: bool = true;
 862
 863     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 864         let builder = run.builder;
 865         run.path("src").default_condition(builder.config.rust_dist_src)
 866     }
 867
 868     fn make_run(run: RunConfig<'_>) {
 869         run.builder.ensure(PlainSourceTarball);
 870     }
 871
 872     /// Creates the plain source tarball
 873     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 874         let tarball = Tarball::new(builder, "rustc", "src");
 875         let plain_dst_src = tarball.image_dir();
 876
 877         // This is the set of root paths which will become part of the source package
 878         let src_files = [
 879             "COPYRIGHT",
 880             "LICENSE-APACHE",
 881             "LICENSE-MIT",
 882             "CONTRIBUTING.md",
 883             "README.md",
 884             "RELEASES.md",
 885             "configure",
 886             "x.py",
 887             "config.toml.example",
 888             "Cargo.toml",
 889             "Cargo.lock",
 890         ];
 891         let src_dirs = ["src", "compiler", "library"];
 892
 893         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 894
 895         // Copy the files normally
 896         for item in &src_files {
 897             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 898         }
 899
 900         // Create the version file
 901         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 902         if let Some(sha) = builder.rust_sha() {
 903             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 904         }
 905
 906         // If we're building from git sources, we need to vendor a complete distribution.
 907         if builder.rust_info.is_git() {
 908             // Vendor all Cargo dependencies
 909             let mut cmd = Command::new(&builder.initial_cargo);
 910             cmd.arg("vendor")
 911                 .arg("--sync")
 912                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 913                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 914                 .current_dir(&plain_dst_src);
 915             builder.run(&mut cmd);
 916         }
 917
 918         tarball.bare()
 919     }
 920 }
 921
 922 // We have to run a few shell scripts, which choke quite a bit on both `\`
 923 // characters and on `C:\` paths, so normalize both of them away.
 924 pub fn sanitize_sh(path: &Path) -> String {
 925     let path = path.to_str().unwrap().replace("\\", "/");
 926     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 927
 928     fn unc_to_lfs(s: &str) -> &str {
 929         s.strip_prefix("//?/").unwrap_or(s)
 930     }
 931
 932     fn change_drive(s: &str) -> Option<String> {
 933         let mut ch = s.chars();
 934         let drive = ch.next().unwrap_or('C');
 935         if ch.next() != Some(':') {
 936             return None;
 937         }
 938         if ch.next() != Some('/') {
 939             return None;
 940         }
 941         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 942     }
 943 }
 944
 945 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 946 pub struct Cargo {
 947     pub compiler: Compiler,
 948     pub target: TargetSelection,
 949 }
 950
 951 impl Step for Cargo {
 952     type Output = GeneratedTarball;
 953     const ONLY_HOSTS: bool = true;
 954
 955     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 956         run.path("cargo")
 957     }
 958
 959     fn make_run(run: RunConfig<'_>) {
 960         run.builder.ensure(Cargo {
 961             compiler: run.builder.compiler_for(
 962                 run.builder.top_stage,
 963                 run.builder.config.build,
 964                 run.target,
 965             ),
 966             target: run.target,
 967         });
 968     }
 969
 970     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 971         let compiler = self.compiler;
 972         let target = self.target;
 973
 974         let cargo = builder.ensure(tool::Cargo { compiler, target });
 975         let src = builder.src.join("src/tools/cargo");
 976         let etc = src.join("src/etc");
 977
 978         // Prepare the image directory
 979         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 980         tarball.set_overlay(OverlayKind::Cargo);
 981
 982         tarball.add_file(&cargo, "bin", 0o755);
 983         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 984         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 985         tarball.add_dir(etc.join("man"), "share/man/man1");
 986         tarball.add_legal_and_readme_to("share/doc/cargo");
 987
 988         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 989             let dirent = dirent.expect("read dir entry");
 990             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 991                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 992             }
 993         }
 994
 995         tarball.generate()
 996     }
 997 }
 998
 999 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1000 pub struct Rls {
1001     pub compiler: Compiler,
1002     pub target: TargetSelection,
1003 }
1004
1005 impl Step for Rls {
1006     type Output = Option<GeneratedTarball>;
1007     const ONLY_HOSTS: bool = true;
1008
1009     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1010         run.path("rls")
1011     }
1012
1013     fn make_run(run: RunConfig<'_>) {
1014         run.builder.ensure(Rls {
1015             compiler: run.builder.compiler_for(
1016                 run.builder.top_stage,
1017                 run.builder.config.build,
1018                 run.target,
1019             ),
1020             target: run.target,
1021         });
1022     }
1023
1024     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1025         let compiler = self.compiler;
1026         let target = self.target;
1027         assert!(builder.config.extended);
1028
1029         let rls = builder
1030             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1031             .or_else(|| {
1032                 // We ignore failure on aarch64 Windows because RLS currently
1033                 // fails to build, due to winapi 0.2 not supporting aarch64.
1034                 missing_tool(
1035                     "RLS",
1036                     builder.build.config.missing_tools
1037                         || (target.triple.contains("aarch64") && target.triple.contains("windows")),
1038                 );
1039                 None
1040             })?;
1041
1042         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1043         tarball.set_overlay(OverlayKind::RLS);
1044         tarball.is_preview(true);
1045         tarball.add_file(rls, "bin", 0o755);
1046         tarball.add_legal_and_readme_to("share/doc/rls");
1047         Some(tarball.generate())
1048     }
1049 }
1050
1051 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1052 pub struct RustAnalyzer {
1053     pub compiler: Compiler,
1054     pub target: TargetSelection,
1055 }
1056
1057 impl Step for RustAnalyzer {
1058     type Output = Option<GeneratedTarball>;
1059     const ONLY_HOSTS: bool = true;
1060
1061     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1062         run.path("rust-analyzer")
1063     }
1064
1065     fn make_run(run: RunConfig<'_>) {
1066         run.builder.ensure(RustAnalyzer {
1067             compiler: run.builder.compiler_for(
1068                 run.builder.top_stage,
1069                 run.builder.config.build,
1070                 run.target,
1071             ),
1072             target: run.target,
1073         });
1074     }
1075
1076     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1077         let compiler = self.compiler;
1078         let target = self.target;
1079         assert!(builder.config.extended);
1080
1081         if target.contains("riscv64") {
1082             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1083             // to build. See #74813 for details.
1084             return None;
1085         }
1086
1087         let rust_analyzer = builder
1088             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1089             .expect("rust-analyzer always builds");
1090
1091         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1092         tarball.set_overlay(OverlayKind::RustAnalyzer);
1093         tarball.is_preview(true);
1094         tarball.add_file(rust_analyzer, "bin", 0o755);
1095         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1096         Some(tarball.generate())
1097     }
1098 }
1099
1100 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1101 pub struct Clippy {
1102     pub compiler: Compiler,
1103     pub target: TargetSelection,
1104 }
1105
1106 impl Step for Clippy {
1107     type Output = GeneratedTarball;
1108     const ONLY_HOSTS: bool = true;
1109
1110     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1111         run.path("clippy")
1112     }
1113
1114     fn make_run(run: RunConfig<'_>) {
1115         run.builder.ensure(Clippy {
1116             compiler: run.builder.compiler_for(
1117                 run.builder.top_stage,
1118                 run.builder.config.build,
1119                 run.target,
1120             ),
1121             target: run.target,
1122         });
1123     }
1124
1125     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
1126         let compiler = self.compiler;
1127         let target = self.target;
1128         assert!(builder.config.extended);
1129
1130         // Prepare the image directory
1131         // We expect clippy to build, because we've exited this step above if tool
1132         // state for clippy isn't testing.
1133         let clippy = builder
1134             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1135             .expect("clippy expected to build - essential tool");
1136         let cargoclippy = builder
1137             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1138             .expect("clippy expected to build - essential tool");
1139
1140         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1141         tarball.set_overlay(OverlayKind::Clippy);
1142         tarball.is_preview(true);
1143         tarball.add_file(clippy, "bin", 0o755);
1144         tarball.add_file(cargoclippy, "bin", 0o755);
1145         tarball.add_legal_and_readme_to("share/doc/clippy");
1146         tarball.generate()
1147     }
1148 }
1149
1150 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1151 pub struct Miri {
1152     pub compiler: Compiler,
1153     pub target: TargetSelection,
1154 }
1155
1156 impl Step for Miri {
1157     type Output = Option<GeneratedTarball>;
1158     const ONLY_HOSTS: bool = true;
1159
1160     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1161         run.path("miri")
1162     }
1163
1164     fn make_run(run: RunConfig<'_>) {
1165         run.builder.ensure(Miri {
1166             compiler: run.builder.compiler_for(
1167                 run.builder.top_stage,
1168                 run.builder.config.build,
1169                 run.target,
1170             ),
1171             target: run.target,
1172         });
1173     }
1174
1175     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1176         let compiler = self.compiler;
1177         let target = self.target;
1178         assert!(builder.config.extended);
1179
1180         let miri = builder
1181             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1182             .or_else(|| {
1183                 missing_tool("miri", builder.build.config.missing_tools);
1184                 None
1185             })?;
1186         let cargomiri = builder
1187             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1188             .or_else(|| {
1189                 missing_tool("cargo miri", builder.build.config.missing_tools);
1190                 None
1191             })?;
1192
1193         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1194         tarball.set_overlay(OverlayKind::Miri);
1195         tarball.is_preview(true);
1196         tarball.add_file(miri, "bin", 0o755);
1197         tarball.add_file(cargomiri, "bin", 0o755);
1198         tarball.add_legal_and_readme_to("share/doc/miri");
1199         Some(tarball.generate())
1200     }
1201 }
1202
1203 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1204 pub struct Rustfmt {
1205     pub compiler: Compiler,
1206     pub target: TargetSelection,
1207 }
1208
1209 impl Step for Rustfmt {
1210     type Output = Option<GeneratedTarball>;
1211     const ONLY_HOSTS: bool = true;
1212
1213     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1214         run.path("rustfmt")
1215     }
1216
1217     fn make_run(run: RunConfig<'_>) {
1218         run.builder.ensure(Rustfmt {
1219             compiler: run.builder.compiler_for(
1220                 run.builder.top_stage,
1221                 run.builder.config.build,
1222                 run.target,
1223             ),
1224             target: run.target,
1225         });
1226     }
1227
1228     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1229         let compiler = self.compiler;
1230         let target = self.target;
1231
1232         let rustfmt = builder
1233             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1234             .or_else(|| {
1235                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1236                 None
1237             })?;
1238         let cargofmt = builder
1239             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1240             .or_else(|| {
1241                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1242                 None
1243             })?;
1244
1245         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1246         tarball.set_overlay(OverlayKind::Rustfmt);
1247         tarball.is_preview(true);
1248         tarball.add_file(rustfmt, "bin", 0o755);
1249         tarball.add_file(cargofmt, "bin", 0o755);
1250         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1251         Some(tarball.generate())
1252     }
1253 }
1254
1255 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1256 pub struct RustDemangler {
1257     pub compiler: Compiler,
1258     pub target: TargetSelection,
1259 }
1260
1261 impl Step for RustDemangler {
1262     type Output = Option<GeneratedTarball>;
1263     const ONLY_HOSTS: bool = true;
1264
1265     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1266         run.path("rust-demangler")
1267     }
1268
1269     fn make_run(run: RunConfig<'_>) {
1270         run.builder.ensure(RustDemangler {
1271             compiler: run.builder.compiler_for(
1272                 run.builder.top_stage,
1273                 run.builder.config.build,
1274                 run.target,
1275             ),
1276             target: run.target,
1277         });
1278     }
1279
1280     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1281         let compiler = self.compiler;
1282         let target = self.target;
1283         assert!(builder.config.extended);
1284
1285         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1286         let profiler = builder.config.profiler_enabled(target);
1287         if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
1288             return None;
1289         }
1290
1291         let rust_demangler = builder
1292             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1293             .expect("rust-demangler expected to build - in-tree tool");
1294
1295         // Prepare the image directory
1296         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1297         tarball.set_overlay(OverlayKind::RustDemangler);
1298         tarball.is_preview(true);
1299         tarball.add_file(&rust_demangler, "bin", 0o755);
1300         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1301         Some(tarball.generate())
1302     }
1303 }
1304
1305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1306 pub struct Extended {
1307     stage: u32,
1308     host: TargetSelection,
1309     target: TargetSelection,
1310 }
1311
1312 impl Step for Extended {
1313     type Output = ();
1314     const DEFAULT: bool = true;
1315     const ONLY_HOSTS: bool = true;
1316
1317     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1318         let builder = run.builder;
1319         run.path("extended").default_condition(builder.config.extended)
1320     }
1321
1322     fn make_run(run: RunConfig<'_>) {
1323         run.builder.ensure(Extended {
1324             stage: run.builder.top_stage,
1325             host: run.builder.config.build,
1326             target: run.target,
1327         });
1328     }
1329
1330     /// Creates a combined installer for the specified target in the provided stage.
1331     fn run(self, builder: &Builder<'_>) {
1332         let target = self.target;
1333         let stage = self.stage;
1334         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1335
1336         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1337
1338         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
1339         let cargo_installer = builder.ensure(Cargo { compiler, target });
1340         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1341         let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
1342         let rls_installer = builder.ensure(Rls { compiler, target });
1343         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
1344         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1345         let clippy_installer = builder.ensure(Clippy { compiler, target });
1346         let miri_installer = builder.ensure(Miri { compiler, target });
1347         let mingw_installer = builder.ensure(Mingw { host: target });
1348         let analysis_installer = builder.ensure(Analysis { compiler, target });
1349
1350         let docs_installer = builder.ensure(Docs { host: target });
1351         let std_installer = builder.ensure(Std { compiler, target });
1352
1353         let etc = builder.src.join("src/etc/installer");
1354
1355         // Avoid producing tarballs during a dry run.
1356         if builder.config.dry_run {
1357             return;
1358         }
1359
1360         // When rust-std package split from rustc, we needed to ensure that during
1361         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1362         // the std files during uninstall. To do this ensure that rustc comes
1363         // before rust-std in the list below.
1364         let mut tarballs = Vec::new();
1365         tarballs.push(rustc_installer);
1366         tarballs.push(cargo_installer);
1367         tarballs.push(clippy_installer);
1368         tarballs.extend(rust_demangler_installer.clone());
1369         tarballs.extend(rls_installer.clone());
1370         tarballs.extend(rust_analyzer_installer.clone());
1371         tarballs.extend(miri_installer.clone());
1372         tarballs.extend(rustfmt_installer.clone());
1373         tarballs.extend(llvm_tools_installer);
1374         if let Some(analysis_installer) = analysis_installer {
1375             tarballs.push(analysis_installer);
1376         }
1377         tarballs.push(std_installer.expect("missing std"));
1378         if let Some(docs_installer) = docs_installer {
1379             tarballs.push(docs_installer);
1380         }
1381         if target.contains("pc-windows-gnu") {
1382             tarballs.push(mingw_installer.unwrap());
1383         }
1384
1385         let tarball = Tarball::new(builder, "rust", &target.triple);
1386         let generated = tarball.combine(&tarballs);
1387
1388         let tmp = tmpdir(builder).join("combined-tarball");
1389         let work = generated.work_dir();
1390
1391         let mut license = String::new();
1392         license += &builder.read(&builder.src.join("COPYRIGHT"));
1393         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1394         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1395         license.push('\n');
1396         license.push('\n');
1397
1398         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1399         let mut rtf = rtf.to_string();
1400         rtf.push('\n');
1401         for line in license.lines() {
1402             rtf.push_str(line);
1403             rtf.push_str("\\line ");
1404         }
1405         rtf.push('}');
1406
1407         fn filter(contents: &str, marker: &str) -> String {
1408             let start = format!("tool-{}-start", marker);
1409             let end = format!("tool-{}-end", marker);
1410             let mut lines = Vec::new();
1411             let mut omitted = false;
1412             for line in contents.lines() {
1413                 if line.contains(&start) {
1414                     omitted = true;
1415                 } else if line.contains(&end) {
1416                     omitted = false;
1417                 } else if !omitted {
1418                     lines.push(line);
1419                 }
1420             }
1421
1422             lines.join("\n")
1423         }
1424
1425         let xform = |p: &Path| {
1426             let mut contents = t!(fs::read_to_string(p));
1427             if rust_demangler_installer.is_none() {
1428                 contents = filter(&contents, "rust-demangler");
1429             }
1430             if rls_installer.is_none() {
1431                 contents = filter(&contents, "rls");
1432             }
1433             if rust_analyzer_installer.is_none() {
1434                 contents = filter(&contents, "rust-analyzer");
1435             }
1436             if miri_installer.is_none() {
1437                 contents = filter(&contents, "miri");
1438             }
1439             if rustfmt_installer.is_none() {
1440                 contents = filter(&contents, "rustfmt");
1441             }
1442             let ret = tmp.join(p.file_name().unwrap());
1443             t!(fs::write(&ret, &contents));
1444             ret
1445         };
1446
1447         if target.contains("apple-darwin") {
1448             builder.info("building pkg installer");
1449             let pkg = tmp.join("pkg");
1450             let _ = fs::remove_dir_all(&pkg);
1451
1452             let pkgbuild = |component: &str| {
1453                 let mut cmd = Command::new("pkgbuild");
1454                 cmd.arg("--identifier")
1455                     .arg(format!("org.rust-lang.{}", component))
1456                     .arg("--scripts")
1457                     .arg(pkg.join(component))
1458                     .arg("--nopayload")
1459                     .arg(pkg.join(component).with_extension("pkg"));
1460                 builder.run(&mut cmd);
1461             };
1462
1463             let prepare = |name: &str| {
1464                 builder.create_dir(&pkg.join(name));
1465                 builder.cp_r(
1466                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1467                     &pkg.join(name),
1468                 );
1469                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1470                 pkgbuild(name);
1471             };
1472             prepare("rustc");
1473             prepare("cargo");
1474             prepare("rust-docs");
1475             prepare("rust-std");
1476             prepare("rust-analysis");
1477             prepare("clippy");
1478             if rust_demangler_installer.is_some() {
1479                 prepare("rust-demangler");
1480             }
1481             if rls_installer.is_some() {
1482                 prepare("rls");
1483             }
1484             if rust_analyzer_installer.is_some() {
1485                 prepare("rust-analyzer");
1486             }
1487             if miri_installer.is_some() {
1488                 prepare("miri");
1489             }
1490
1491             // create an 'uninstall' package
1492             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1493             pkgbuild("uninstall");
1494
1495             builder.create_dir(&pkg.join("res"));
1496             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1497             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1498             let mut cmd = Command::new("productbuild");
1499             cmd.arg("--distribution")
1500                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1501                 .arg("--resources")
1502                 .arg(pkg.join("res"))
1503                 .arg(distdir(builder).join(format!(
1504                     "{}-{}.pkg",
1505                     pkgname(builder, "rust"),
1506                     target.triple
1507                 )))
1508                 .arg("--package-path")
1509                 .arg(&pkg);
1510             let _time = timeit(builder);
1511             builder.run(&mut cmd);
1512         }
1513
1514         if target.contains("windows") {
1515             let exe = tmp.join("exe");
1516             let _ = fs::remove_dir_all(&exe);
1517
1518             let prepare = |name: &str| {
1519                 builder.create_dir(&exe.join(name));
1520                 let dir = if name == "rust-std" || name == "rust-analysis" {
1521                     format!("{}-{}", name, target.triple)
1522                 } else if name == "rls" {
1523                     "rls-preview".to_string()
1524                 } else if name == "rust-analyzer" {
1525                     "rust-analyzer-preview".to_string()
1526                 } else if name == "clippy" {
1527                     "clippy-preview".to_string()
1528                 } else if name == "rust-demangler" {
1529                     "rust-demangler-preview".to_string()
1530                 } else if name == "miri" {
1531                     "miri-preview".to_string()
1532                 } else {
1533                     name.to_string()
1534                 };
1535                 builder.cp_r(
1536                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1537                     &exe.join(name),
1538                 );
1539                 builder.remove(&exe.join(name).join("manifest.in"));
1540             };
1541             prepare("rustc");
1542             prepare("cargo");
1543             prepare("rust-analysis");
1544             prepare("rust-docs");
1545             prepare("rust-std");
1546             prepare("clippy");
1547             if rust_demangler_installer.is_some() {
1548                 prepare("rust-demangler");
1549             }
1550             if rls_installer.is_some() {
1551                 prepare("rls");
1552             }
1553             if rust_analyzer_installer.is_some() {
1554                 prepare("rust-analyzer");
1555             }
1556             if miri_installer.is_some() {
1557                 prepare("miri");
1558             }
1559             if target.contains("windows-gnu") {
1560                 prepare("rust-mingw");
1561             }
1562
1563             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1564
1565             // Generate msi installer
1566             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1567             let heat = wix.join("bin/heat.exe");
1568             let candle = wix.join("bin/candle.exe");
1569             let light = wix.join("bin/light.exe");
1570
1571             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1572             builder.run(
1573                 Command::new(&heat)
1574                     .current_dir(&exe)
1575                     .arg("dir")
1576                     .arg("rustc")
1577                     .args(&heat_flags)
1578                     .arg("-cg")
1579                     .arg("RustcGroup")
1580                     .arg("-dr")
1581                     .arg("Rustc")
1582                     .arg("-var")
1583                     .arg("var.RustcDir")
1584                     .arg("-out")
1585                     .arg(exe.join("RustcGroup.wxs")),
1586             );
1587             builder.run(
1588                 Command::new(&heat)
1589                     .current_dir(&exe)
1590                     .arg("dir")
1591                     .arg("rust-docs")
1592                     .args(&heat_flags)
1593                     .arg("-cg")
1594                     .arg("DocsGroup")
1595                     .arg("-dr")
1596                     .arg("Docs")
1597                     .arg("-var")
1598                     .arg("var.DocsDir")
1599                     .arg("-out")
1600                     .arg(exe.join("DocsGroup.wxs"))
1601                     .arg("-t")
1602                     .arg(etc.join("msi/squash-components.xsl")),
1603             );
1604             builder.run(
1605                 Command::new(&heat)
1606                     .current_dir(&exe)
1607                     .arg("dir")
1608                     .arg("cargo")
1609                     .args(&heat_flags)
1610                     .arg("-cg")
1611                     .arg("CargoGroup")
1612                     .arg("-dr")
1613                     .arg("Cargo")
1614                     .arg("-var")
1615                     .arg("var.CargoDir")
1616                     .arg("-out")
1617                     .arg(exe.join("CargoGroup.wxs"))
1618                     .arg("-t")
1619                     .arg(etc.join("msi/remove-duplicates.xsl")),
1620             );
1621             builder.run(
1622                 Command::new(&heat)
1623                     .current_dir(&exe)
1624                     .arg("dir")
1625                     .arg("rust-std")
1626                     .args(&heat_flags)
1627                     .arg("-cg")
1628                     .arg("StdGroup")
1629                     .arg("-dr")
1630                     .arg("Std")
1631                     .arg("-var")
1632                     .arg("var.StdDir")
1633                     .arg("-out")
1634                     .arg(exe.join("StdGroup.wxs")),
1635             );
1636             if rls_installer.is_some() {
1637                 builder.run(
1638                     Command::new(&heat)
1639                         .current_dir(&exe)
1640                         .arg("dir")
1641                         .arg("rls")
1642                         .args(&heat_flags)
1643                         .arg("-cg")
1644                         .arg("RlsGroup")
1645                         .arg("-dr")
1646                         .arg("Rls")
1647                         .arg("-var")
1648                         .arg("var.RlsDir")
1649                         .arg("-out")
1650                         .arg(exe.join("RlsGroup.wxs"))
1651                         .arg("-t")
1652                         .arg(etc.join("msi/remove-duplicates.xsl")),
1653                 );
1654             }
1655             if rust_analyzer_installer.is_some() {
1656                 builder.run(
1657                     Command::new(&heat)
1658                         .current_dir(&exe)
1659                         .arg("dir")
1660                         .arg("rust-analyzer")
1661                         .args(&heat_flags)
1662                         .arg("-cg")
1663                         .arg("RustAnalyzerGroup")
1664                         .arg("-dr")
1665                         .arg("RustAnalyzer")
1666                         .arg("-var")
1667                         .arg("var.RustAnalyzerDir")
1668                         .arg("-out")
1669                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1670                         .arg("-t")
1671                         .arg(etc.join("msi/remove-duplicates.xsl")),
1672                 );
1673             }
1674             builder.run(
1675                 Command::new(&heat)
1676                     .current_dir(&exe)
1677                     .arg("dir")
1678                     .arg("clippy")
1679                     .args(&heat_flags)
1680                     .arg("-cg")
1681                     .arg("ClippyGroup")
1682                     .arg("-dr")
1683                     .arg("Clippy")
1684                     .arg("-var")
1685                     .arg("var.ClippyDir")
1686                     .arg("-out")
1687                     .arg(exe.join("ClippyGroup.wxs"))
1688                     .arg("-t")
1689                     .arg(etc.join("msi/remove-duplicates.xsl")),
1690             );
1691             if rust_demangler_installer.is_some() {
1692                 builder.run(
1693                     Command::new(&heat)
1694                         .current_dir(&exe)
1695                         .arg("dir")
1696                         .arg("rust-demangler")
1697                         .args(&heat_flags)
1698                         .arg("-cg")
1699                         .arg("RustDemanglerGroup")
1700                         .arg("-dr")
1701                         .arg("RustDemangler")
1702                         .arg("-var")
1703                         .arg("var.RustDemanglerDir")
1704                         .arg("-out")
1705                         .arg(exe.join("RustDemanglerGroup.wxs"))
1706                         .arg("-t")
1707                         .arg(etc.join("msi/remove-duplicates.xsl")),
1708                 );
1709             }
1710             if miri_installer.is_some() {
1711                 builder.run(
1712                     Command::new(&heat)
1713                         .current_dir(&exe)
1714                         .arg("dir")
1715                         .arg("miri")
1716                         .args(&heat_flags)
1717                         .arg("-cg")
1718                         .arg("MiriGroup")
1719                         .arg("-dr")
1720                         .arg("Miri")
1721                         .arg("-var")
1722                         .arg("var.MiriDir")
1723                         .arg("-out")
1724                         .arg(exe.join("MiriGroup.wxs"))
1725                         .arg("-t")
1726                         .arg(etc.join("msi/remove-duplicates.xsl")),
1727                 );
1728             }
1729             builder.run(
1730                 Command::new(&heat)
1731                     .current_dir(&exe)
1732                     .arg("dir")
1733                     .arg("rust-analysis")
1734                     .args(&heat_flags)
1735                     .arg("-cg")
1736                     .arg("AnalysisGroup")
1737                     .arg("-dr")
1738                     .arg("Analysis")
1739                     .arg("-var")
1740                     .arg("var.AnalysisDir")
1741                     .arg("-out")
1742                     .arg(exe.join("AnalysisGroup.wxs"))
1743                     .arg("-t")
1744                     .arg(etc.join("msi/remove-duplicates.xsl")),
1745             );
1746             if target.contains("windows-gnu") {
1747                 builder.run(
1748                     Command::new(&heat)
1749                         .current_dir(&exe)
1750                         .arg("dir")
1751                         .arg("rust-mingw")
1752                         .args(&heat_flags)
1753                         .arg("-cg")
1754                         .arg("GccGroup")
1755                         .arg("-dr")
1756                         .arg("Gcc")
1757                         .arg("-var")
1758                         .arg("var.GccDir")
1759                         .arg("-out")
1760                         .arg(exe.join("GccGroup.wxs")),
1761                 );
1762             }
1763
1764             let candle = |input: &Path| {
1765                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1766                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1767                 let mut cmd = Command::new(&candle);
1768                 cmd.current_dir(&exe)
1769                     .arg("-nologo")
1770                     .arg("-dRustcDir=rustc")
1771                     .arg("-dDocsDir=rust-docs")
1772                     .arg("-dCargoDir=cargo")
1773                     .arg("-dStdDir=rust-std")
1774                     .arg("-dAnalysisDir=rust-analysis")
1775                     .arg("-dClippyDir=clippy")
1776                     .arg("-arch")
1777                     .arg(&arch)
1778                     .arg("-out")
1779                     .arg(&output)
1780                     .arg(&input);
1781                 add_env(builder, &mut cmd, target);
1782
1783                 if rust_demangler_installer.is_some() {
1784                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1785                 }
1786                 if rls_installer.is_some() {
1787                     cmd.arg("-dRlsDir=rls");
1788                 }
1789                 if rust_analyzer_installer.is_some() {
1790                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1791                 }
1792                 if miri_installer.is_some() {
1793                     cmd.arg("-dMiriDir=miri");
1794                 }
1795                 if target.contains("windows-gnu") {
1796                     cmd.arg("-dGccDir=rust-mingw");
1797                 }
1798                 builder.run(&mut cmd);
1799             };
1800             candle(&xform(&etc.join("msi/rust.wxs")));
1801             candle(&etc.join("msi/ui.wxs"));
1802             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1803             candle("RustcGroup.wxs".as_ref());
1804             candle("DocsGroup.wxs".as_ref());
1805             candle("CargoGroup.wxs".as_ref());
1806             candle("StdGroup.wxs".as_ref());
1807             candle("ClippyGroup.wxs".as_ref());
1808             if rust_demangler_installer.is_some() {
1809                 candle("RustDemanglerGroup.wxs".as_ref());
1810             }
1811             if rls_installer.is_some() {
1812                 candle("RlsGroup.wxs".as_ref());
1813             }
1814             if rust_analyzer_installer.is_some() {
1815                 candle("RustAnalyzerGroup.wxs".as_ref());
1816             }
1817             if miri_installer.is_some() {
1818                 candle("MiriGroup.wxs".as_ref());
1819             }
1820             candle("AnalysisGroup.wxs".as_ref());
1821
1822             if target.contains("windows-gnu") {
1823                 candle("GccGroup.wxs".as_ref());
1824             }
1825
1826             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1827             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1828             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1829
1830             builder.info(&format!("building `msi` installer with {:?}", light));
1831             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1832             let mut cmd = Command::new(&light);
1833             cmd.arg("-nologo")
1834                 .arg("-ext")
1835                 .arg("WixUIExtension")
1836                 .arg("-ext")
1837                 .arg("WixUtilExtension")
1838                 .arg("-out")
1839                 .arg(exe.join(&filename))
1840                 .arg("rust.wixobj")
1841                 .arg("ui.wixobj")
1842                 .arg("rustwelcomedlg.wixobj")
1843                 .arg("RustcGroup.wixobj")
1844                 .arg("DocsGroup.wixobj")
1845                 .arg("CargoGroup.wixobj")
1846                 .arg("StdGroup.wixobj")
1847                 .arg("AnalysisGroup.wixobj")
1848                 .arg("ClippyGroup.wixobj")
1849                 .current_dir(&exe);
1850
1851             if rls_installer.is_some() {
1852                 cmd.arg("RlsGroup.wixobj");
1853             }
1854             if rust_analyzer_installer.is_some() {
1855                 cmd.arg("RustAnalyzerGroup.wixobj");
1856             }
1857             if rust_demangler_installer.is_some() {
1858                 cmd.arg("RustDemanglerGroup.wixobj");
1859             }
1860             if miri_installer.is_some() {
1861                 cmd.arg("MiriGroup.wixobj");
1862             }
1863
1864             if target.contains("windows-gnu") {
1865                 cmd.arg("GccGroup.wixobj");
1866             }
1867             // ICE57 wrongly complains about the shortcuts
1868             cmd.arg("-sice:ICE57");
1869
1870             let _time = timeit(builder);
1871             builder.run(&mut cmd);
1872
1873             if !builder.config.dry_run {
1874                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1875             }
1876         }
1877     }
1878 }
1879
1880 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1881     let mut parts = builder.version.split('.');
1882     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1883         .env("CFG_RELEASE_NUM", &builder.version)
1884         .env("CFG_RELEASE", builder.rust_release())
1885         .env("CFG_VER_MAJOR", parts.next().unwrap())
1886         .env("CFG_VER_MINOR", parts.next().unwrap())
1887         .env("CFG_VER_PATCH", parts.next().unwrap())
1888         .env("CFG_VER_BUILD", "0") // just needed to build
1889         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1890         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1891         .env("CFG_BUILD", target.triple)
1892         .env("CFG_CHANNEL", &builder.config.channel);
1893
1894     if target.contains("windows-gnu") {
1895         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1896     } else {
1897         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1898     }
1899
1900     if target.contains("x86_64") {
1901         cmd.env("CFG_PLATFORM", "x64");
1902     } else {
1903         cmd.env("CFG_PLATFORM", "x86");
1904     }
1905 }
1906
1907 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1908 ///
1909
1910 /// Returns whether the files were actually copied.
1911 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1912     if let Some(config) = builder.config.target_config.get(&target) {
1913         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1914             // If the LLVM was externally provided, then we don't currently copy
1915             // artifacts into the sysroot. This is not necessarily the right
1916             // choice (in particular, it will require the LLVM dylib to be in
1917             // the linker's load path at runtime), but the common use case for
1918             // external LLVMs is distribution provided LLVMs, and in that case
1919             // they're usually in the standard search path (e.g., /usr/lib) and
1920             // copying them here is going to cause problems as we may end up
1921             // with the wrong files and isn't what distributions want.
1922             //
1923             // This behavior may be revisited in the future though.
1924             //
1925             // If the LLVM is coming from ourselves (just from CI) though, we
1926             // still want to install it, as it otherwise won't be available.
1927             return false;
1928         }
1929     }
1930
1931     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1932     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1933     // clear why this is the case, though. llvm-config will emit the versioned
1934     // paths and we don't want those in the sysroot (as we're expecting
1935     // unversioned paths).
1936     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1937         let src_libdir = builder.llvm_out(target).join("lib");
1938         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1939         if llvm_dylib_path.exists() {
1940             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1941         }
1942         !builder.config.dry_run
1943     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1944         let mut cmd = Command::new(llvm_config);
1945         cmd.arg("--libfiles");
1946         builder.verbose(&format!("running {:?}", cmd));
1947         let files = output(&mut cmd);
1948         for file in files.trim_end().split(' ') {
1949             builder.install(Path::new(file), dst_libdir, 0o644);
1950         }
1951         !builder.config.dry_run
1952     } else {
1953         false
1954     }
1955 }
1956
1957 /// Maybe add libLLVM.so to the target lib-dir for linking.
1958 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1959     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1960     // We do not need to copy LLVM files into the sysroot if it is not
1961     // dynamically linked; it is already included into librustc_llvm
1962     // statically.
1963     if builder.config.llvm_link_shared {
1964         maybe_install_llvm(builder, target, &dst_libdir);
1965     }
1966 }
1967
1968 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1969 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1970     let dst_libdir =
1971         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1972     // We do not need to copy LLVM files into the sysroot if it is not
1973     // dynamically linked; it is already included into librustc_llvm
1974     // statically.
1975     if builder.config.llvm_link_shared {
1976         maybe_install_llvm(builder, target, &dst_libdir);
1977     }
1978 }
1979
1980 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1981 pub struct LlvmTools {
1982     pub target: TargetSelection,
1983 }
1984
1985 impl Step for LlvmTools {
1986     type Output = Option<GeneratedTarball>;
1987     const ONLY_HOSTS: bool = true;
1988
1989     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1990         run.path("llvm-tools")
1991     }
1992
1993     fn make_run(run: RunConfig<'_>) {
1994         run.builder.ensure(LlvmTools { target: run.target });
1995     }
1996
1997     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1998         let target = self.target;
1999         assert!(builder.config.extended);
2000
2001         /* run only if llvm-config isn't used */
2002         if let Some(config) = builder.config.target_config.get(&target) {
2003             if let Some(ref _s) = config.llvm_config {
2004                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2005                 return None;
2006             }
2007         }
2008
2009         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2010         tarball.set_overlay(OverlayKind::LLVM);
2011         tarball.is_preview(true);
2012
2013         // Prepare the image directory
2014         let src_bindir = builder.llvm_out(target).join("bin");
2015         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2016         for tool in LLVM_TOOLS {
2017             let exe = src_bindir.join(exe(tool, target));
2018             tarball.add_file(&exe, &dst_bindir, 0o755);
2019         }
2020
2021         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2022         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2023         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2024         // compiler libraries.
2025         maybe_install_llvm_target(builder, target, tarball.image_dir());
2026
2027         Some(tarball.generate())
2028     }
2029 }
2030
2031 // Tarball intended for internal consumption to ease rustc/std development.
2032 //
2033 // Should not be considered stable by end users.
2034 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2035 pub struct RustDev {
2036     pub target: TargetSelection,
2037 }
2038
2039 impl Step for RustDev {
2040     type Output = Option<GeneratedTarball>;
2041     const DEFAULT: bool = true;
2042     const ONLY_HOSTS: bool = true;
2043
2044     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2045         run.path("rust-dev")
2046     }
2047
2048     fn make_run(run: RunConfig<'_>) {
2049         run.builder.ensure(RustDev { target: run.target });
2050     }
2051
2052     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2053         let target = self.target;
2054
2055         /* run only if llvm-config isn't used */
2056         if let Some(config) = builder.config.target_config.get(&target) {
2057             if let Some(ref _s) = config.llvm_config {
2058                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2059                 return None;
2060             }
2061         }
2062
2063         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2064         tarball.set_overlay(OverlayKind::LLVM);
2065
2066         let src_bindir = builder.llvm_out(target).join("bin");
2067         for bin in &[
2068             "llvm-config",
2069             "llvm-ar",
2070             "llvm-objdump",
2071             "llvm-profdata",
2072             "llvm-bcanalyzer",
2073             "llvm-cov",
2074             "llvm-dwp",
2075         ] {
2076             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2077         }
2078         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2079
2080         // Copy the include directory as well; needed mostly to build
2081         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2082         // just broadly useful to be able to link against the bundled LLVM.
2083         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2084
2085         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2086         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2087         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2088         // compiler libraries.
2089         let dst_libdir = tarball.image_dir().join("lib");
2090         maybe_install_llvm(builder, target, &dst_libdir);
2091         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2092         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2093
2094         Some(tarball.generate())
2095     }
2096 }
2097
2098 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2099 /// release process to avoid cloning the monorepo and building stuff.
2100 ///
2101 /// Should not be considered stable by end users.
2102 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2103 pub struct BuildManifest {
2104     pub target: TargetSelection,
2105 }
2106
2107 impl Step for BuildManifest {
2108     type Output = GeneratedTarball;
2109     const DEFAULT: bool = false;
2110     const ONLY_HOSTS: bool = true;
2111
2112     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2113         run.path("src/tools/build-manifest")
2114     }
2115
2116     fn make_run(run: RunConfig<'_>) {
2117         run.builder.ensure(BuildManifest { target: run.target });
2118     }
2119
2120     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2121         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2122
2123         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2124         tarball.add_file(&build_manifest, "bin", 0o755);
2125         tarball.generate()
2126     }
2127 }
2128
2129 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2130 ///
2131 /// Currently this is the PGO profile data.
2132 ///
2133 /// Should not be considered stable by end users.
2134 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2135 pub struct ReproducibleArtifacts {
2136     pub target: TargetSelection,
2137 }
2138
2139 impl Step for ReproducibleArtifacts {
2140     type Output = Option<GeneratedTarball>;
2141     const DEFAULT: bool = true;
2142     const ONLY_HOSTS: bool = true;
2143
2144     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2145         run.path("reproducible")
2146     }
2147
2148     fn make_run(run: RunConfig<'_>) {
2149         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2150     }
2151
2152     fn run(self, builder: &Builder<'_>) -> Self::Output {
2153         let path = builder.config.rust_profile_use.as_ref()?;
2154
2155         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2156         tarball.add_file(path, ".", 0o644);
2157         Some(tarball.generate())
2158     }
2159 }