1 use std
::collections
::HashSet
;
4 use std
::path
::PathBuf
;
5 use std
::process
::{exit, Command}
;
9 use crate::builder
::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}
;
10 use crate::cache
::Interned
;
12 use crate::channel
::GitInfo
;
14 use crate::toolstate
::ToolState
;
15 use crate::util
::{add_dylib_path, exe, CiEnv}
;
19 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
25 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
28 target
: Interned
<String
>,
32 is_optional_tool
: bool
,
33 source_type
: SourceType
,
34 extra_features
: Vec
<String
>,
37 impl Step
for ToolBuild
{
38 type Output
= Option
<PathBuf
>;
40 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
44 /// Builds a tool in `src/tools`
46 /// This will build the specified tool with the specified `host` compiler in
47 /// `stage` into the normal cargo output directory.
48 fn run(self, builder
: &Builder
<'_
>) -> Option
<PathBuf
> {
49 let compiler
= self.compiler
;
50 let target
= self.target
;
53 let is_optional_tool
= self.is_optional_tool
;
56 Mode
::ToolRustc
=> builder
.ensure(compile
::Rustc { compiler, target }
),
57 Mode
::ToolStd
=> builder
.ensure(compile
::Std { compiler, target }
),
58 Mode
::ToolBootstrap
=> {}
// uses downloaded stage0 compiler libs
59 _
=> panic
!("unexpected Mode for tool build"),
62 let cargo
= prepare_tool_cargo(
73 builder
.info(&format
!("Building stage{} tool {} ({})", compiler
.stage
, tool
, target
));
74 let mut duplicates
= Vec
::new();
75 let is_expected
= compile
::stream_cargo(builder
, cargo
, vec
![], &mut |msg
| {
76 // Only care about big things like the RLS/Cargo for now
78 "rls" | "cargo" | "clippy-driver" | "miri" | "rustfmt" => {}
82 let (id
, features
, filenames
) = match msg
{
83 compile
::CargoMessage
::CompilerArtifact
{
88 } => (package_id
, features
, filenames
),
91 let features
= features
.iter().map(|s
| s
.to_string()).collect
::<Vec
<_
>>();
93 for path
in filenames
{
94 let val
= (tool
, PathBuf
::from(&*path
), features
.clone());
95 // we're only interested in deduplicating rlibs for now
96 if val
.1.extension().and_then(|s
| s
.to_str()) != Some("rlib") {
100 // Don't worry about compiles that turn out to be host
101 // dependencies or build scripts. To skip these we look for
102 // anything that goes in `.../release/deps` but *doesn't* go in
103 // `$target/release/deps`. This ensure that outputs in
104 // `$target/release` are still considered candidates for
106 if let Some(parent
) = val
.1.parent() {
107 if parent
.ends_with("release/deps") {
108 let maybe_target
= parent
110 .and_then(|p
| p
.parent())
111 .and_then(|p
| p
.file_name())
112 .and_then(|p
| p
.to_str())
114 if maybe_target
!= &*target
{
120 // Record that we've built an artifact for `id`, and if one was
121 // already listed then we need to see if we reused the same
122 // artifact or produced a duplicate.
123 let mut artifacts
= builder
.tool_artifacts
.borrow_mut();
124 let prev_artifacts
= artifacts
.entry(target
).or_default();
125 let prev
= match prev_artifacts
.get(&*id
) {
128 prev_artifacts
.insert(id
.to_string(), val
);
133 return; // same path, same artifact
136 // If the paths are different and one of them *isn't* inside of
137 // `release/deps`, then it means it's probably in
138 // `$target/release`, or it's some final artifact like
139 // `libcargo.rlib`. In these situations Cargo probably just
140 // copied it up from `$target/release/deps/libcargo-xxxx.rlib`,
141 // so if the features are equal we can just skip it.
142 let prev_no_hash
= prev
.1.parent().unwrap().ends_with("release/deps");
143 let val_no_hash
= val
.1.parent().unwrap().ends_with("release/deps");
144 if prev
.2 == val
.2 || !prev_no_hash
|| !val_no_hash
{
148 // ... and otherwise this looks like we duplicated some sort of
149 // compilation, so record it to generate an error later.
150 duplicates
.push((id
.to_string(), val
, prev
.clone()));
154 if is_expected
&& !duplicates
.is_empty() {
156 "duplicate artifacts found when compiling a tool, this \
157 typically means that something was recompiled because \
158 a transitive dependency has different features activated \
159 than in a previous build:\n"
162 "the following dependencies are duplicated although they \
163 have the same features enabled:"
165 for (id
, cur
, prev
) in duplicates
.drain_filter(|(_
, cur
, prev
)| cur
.2 == prev
.2) {
168 println
!(" `{}` ({:?})\n `{}` ({:?})", cur
.0, cur
.1, prev
.0, prev
.1);
170 println
!("the following dependencies have different features:");
171 for (id
, cur
, prev
) in duplicates
{
173 let cur_features
: HashSet
<_
> = cur
.2.into_iter
().collect();
174 let prev_features
: HashSet
<_
> = prev
.2.into_iter
().collect();
176 " `{}` additionally enabled features {:?} at {:?}",
178 &cur_features
- &prev_features
,
182 " `{}` additionally enabled features {:?} at {:?}",
184 &prev_features
- &cur_features
,
190 "to fix this you will probably want to edit the local \
191 src/tools/rustc-workspace-hack/Cargo.toml crate, as \
192 that will update the dependency graph to ensure that \
193 these crates all share the same feature set"
195 panic
!("tools should not compile multiple copies of the same crate");
198 builder
.save_toolstate(
200 if is_expected { ToolState::TestFail }
else { ToolState::BuildFail }
,
204 if !is_optional_tool
{
211 builder
.cargo_out(compiler
, self.mode
, target
).join(exe(tool
, &compiler
.host
));
212 let bin
= builder
.tools_dir(compiler
).join(exe(tool
, &compiler
.host
));
213 builder
.copy(&cargo_out
, &bin
);
219 pub fn prepare_tool_cargo(
220 builder
: &Builder
<'_
>,
223 target
: Interned
<String
>,
224 command
: &'
static str,
226 source_type
: SourceType
,
227 extra_features
: &[String
],
229 let mut cargo
= builder
.cargo(compiler
, mode
, target
, command
);
230 let dir
= builder
.src
.join(path
);
231 cargo
.arg("--manifest-path").arg(dir
.join("Cargo.toml"));
233 if source_type
== SourceType
::Submodule
{
234 cargo
.env("RUSTC_EXTERNAL_TOOL", "1");
237 let mut features
= extra_features
.to_vec();
238 if builder
.build
.config
.cargo_native_static
{
239 if path
.ends_with("cargo")
240 || path
.ends_with("rls")
241 || path
.ends_with("clippy")
242 || path
.ends_with("miri")
243 || path
.ends_with("rustbook")
244 || path
.ends_with("rustfmt")
246 cargo
.env("LIBZ_SYS_STATIC", "1");
247 features
.push("rustc-workspace-hack/all-static".to_string());
251 // if tools are using lzma we want to force the build script to build its
253 cargo
.env("LZMA_API_STATIC", "1");
255 cargo
.env("CFG_RELEASE_CHANNEL", &builder
.config
.channel
);
256 cargo
.env("CFG_VERSION", builder
.rust_version());
257 cargo
.env("CFG_RELEASE_NUM", channel
::CFG_RELEASE_NUM
);
259 let info
= GitInfo
::new(builder
.config
.ignore_git
, &dir
);
260 if let Some(sha
) = info
.sha() {
261 cargo
.env("CFG_COMMIT_HASH", sha
);
263 if let Some(sha_short
) = info
.sha_short() {
264 cargo
.env("CFG_SHORT_COMMIT_HASH", sha_short
);
266 if let Some(date
) = info
.commit_date() {
267 cargo
.env("CFG_COMMIT_DATE", date
);
269 if !features
.is_empty() {
270 cargo
.arg("--features").arg(&features
.join(", "));
275 fn rustbook_features() -> Vec
<String
> {
276 let mut features
= Vec
::new();
278 // Due to CI budged and risk of spurious failures we want to limit jobs running this check.
279 // At same time local builds should run it regardless of the platform.
280 // `CiEnv::None` means it's local build and `CHECK_LINKS` is defined in x86_64-gnu-tools to
281 // explicitly enable it on single job
282 if CiEnv
::current() == CiEnv
::None
|| env
::var("CHECK_LINKS").is_ok() {
283 features
.push("linkcheck".to_string());
289 macro_rules
! bootstrap_tool
{
291 $name
:ident
, $path
:expr
, $tool_name
:expr
292 $
(,is_external_tool
= $external
:expr
)*
293 $
(,is_unstable_tool
= $unstable
:expr
)*
294 $
(,features
= $features
:expr
)*
297 #[derive(Copy, PartialEq, Eq, Clone)]
304 impl<'a
> Builder
<'a
> {
305 pub fn tool_exe(&self, tool
: Tool
) -> PathBuf
{
309 compiler
: self.compiler(0, self.config
.build
),
310 target
: self.config
.build
,
318 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
320 pub compiler
: Compiler
,
321 pub target
: Interned
<String
>,
324 impl Step
for $name
{
325 type Output
= PathBuf
;
327 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
331 fn make_run(run
: RunConfig
<'_
>) {
332 run
.builder
.ensure($name
{
334 compiler
: run
.builder
.compiler(0, run
.builder
.config
.build
),
339 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
340 builder
.ensure(ToolBuild
{
341 compiler
: self.compiler
,
344 mode
: if false $
(|| $unstable
)* {
345 // use in-tree libraries for unstable features
351 is_optional_tool
: false,
352 source_type
: if false $
(|| $external
)* {
353 SourceType
::Submodule
358 // FIXME(#60643): avoid this lint by using `_`
359 let mut _tmp
= Vec
::new();
360 $
(_tmp
.extend($features
);)*
363 }).expect("expected to build -- essential tool")
371 Rustbook
, "src/tools/rustbook", "rustbook", features
= rustbook_features();
372 UnstableBookGen
, "src/tools/unstable-book-gen", "unstable-book-gen";
373 Tidy
, "src/tools/tidy", "tidy";
374 Linkchecker
, "src/tools/linkchecker", "linkchecker";
375 CargoTest
, "src/tools/cargotest", "cargotest";
376 Compiletest
, "src/tools/compiletest", "compiletest", is_unstable_tool
= true;
377 BuildManifest
, "src/tools/build-manifest", "build-manifest";
378 RemoteTestClient
, "src/tools/remote-test-client", "remote-test-client";
379 RustInstaller
, "src/tools/rust-installer", "fabricate", is_external_tool
= true;
380 RustdocTheme
, "src/tools/rustdoc-themes", "rustdoc-themes";
381 ExpandYamlAnchors
, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
384 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
385 pub struct ErrorIndex
{
386 pub compiler
: Compiler
,
390 pub fn command(builder
: &Builder
<'_
>, compiler
: Compiler
) -> Command
{
391 let mut cmd
= Command
::new(builder
.ensure(ErrorIndex { compiler }
));
393 vec
![PathBuf
::from(&builder
.sysroot_libdir(compiler
, compiler
.host
))],
400 impl Step
for ErrorIndex
{
401 type Output
= PathBuf
;
403 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
404 run
.path("src/tools/error_index_generator")
407 fn make_run(run
: RunConfig
<'_
>) {
408 // Compile the error-index in the same stage as rustdoc to avoid
409 // recompiling rustdoc twice if we can.
410 let stage
= if run
.builder
.top_stage
>= 2 { run.builder.top_stage }
else { 0 }
;
412 .ensure(ErrorIndex { compiler: run.builder.compiler(stage, run.builder.config.build) }
);
415 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
418 compiler
: self.compiler
,
419 target
: self.compiler
.host
,
420 tool
: "error_index_generator",
421 mode
: Mode
::ToolRustc
,
422 path
: "src/tools/error_index_generator",
423 is_optional_tool
: false,
424 source_type
: SourceType
::InTree
,
425 extra_features
: Vec
::new(),
427 .expect("expected to build -- essential tool")
431 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
432 pub struct RemoteTestServer
{
433 pub compiler
: Compiler
,
434 pub target
: Interned
<String
>,
437 impl Step
for RemoteTestServer
{
438 type Output
= PathBuf
;
440 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
441 run
.path("src/tools/remote-test-server")
444 fn make_run(run
: RunConfig
<'_
>) {
445 run
.builder
.ensure(RemoteTestServer
{
446 compiler
: run
.builder
.compiler(run
.builder
.top_stage
, run
.builder
.config
.build
),
451 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
454 compiler
: self.compiler
,
456 tool
: "remote-test-server",
458 path
: "src/tools/remote-test-server",
459 is_optional_tool
: false,
460 source_type
: SourceType
::InTree
,
461 extra_features
: Vec
::new(),
463 .expect("expected to build -- essential tool")
467 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
469 /// This should only ever be 0 or 2.
470 /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.
471 pub compiler
: Compiler
,
474 impl Step
for Rustdoc
{
475 type Output
= PathBuf
;
476 const DEFAULT
: bool
= true;
477 const ONLY_HOSTS
: bool
= true;
479 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
480 run
.path("src/tools/rustdoc")
483 fn make_run(run
: RunConfig
<'_
>) {
485 .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.host) }
);
488 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
489 let target_compiler
= self.compiler
;
490 if target_compiler
.stage
== 0 {
491 if !target_compiler
.is_snapshot(builder
) {
492 panic
!("rustdoc in stage 0 must be snapshot rustdoc");
494 return builder
.initial_rustc
.with_file_name(exe("rustdoc", &target_compiler
.host
));
496 let target
= target_compiler
.host
;
497 // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
498 // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
499 // compilers, which isn't what we want. Rustdoc should be linked in the same way as the
500 // rustc compiler it's paired with, so it must be built with the previous stage compiler.
501 let build_compiler
= builder
.compiler(target_compiler
.stage
- 1, builder
.config
.build
);
503 // The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
504 // compiler libraries, ...) are built. Rustdoc does not require the presence of any
505 // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since
506 // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional
507 // libraries here. The intuition here is that If we've built a compiler, we should be able
510 let cargo
= prepare_tool_cargo(
521 builder
.info(&format
!(
522 "Building rustdoc for stage{} ({})",
523 target_compiler
.stage
, target_compiler
.host
525 builder
.run(&mut cargo
.into());
527 // Cargo adds a number of paths to the dylib search path on windows, which results in
528 // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
529 // rustdoc a different name.
530 let tool_rustdoc
= builder
531 .cargo_out(build_compiler
, Mode
::ToolRustc
, target
)
532 .join(exe("rustdoc_tool_binary", &target_compiler
.host
));
534 // don't create a stage0-sysroot/bin directory.
535 if target_compiler
.stage
> 0 {
536 let sysroot
= builder
.sysroot(target_compiler
);
537 let bindir
= sysroot
.join("bin");
538 t
!(fs
::create_dir_all(&bindir
));
539 let bin_rustdoc
= bindir
.join(exe("rustdoc", &*target_compiler
.host
));
540 let _
= fs
::remove_file(&bin_rustdoc
);
541 builder
.copy(&tool_rustdoc
, &bin_rustdoc
);
549 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
551 pub compiler
: Compiler
,
552 pub target
: Interned
<String
>,
555 impl Step
for Cargo
{
556 type Output
= PathBuf
;
557 const DEFAULT
: bool
= true;
558 const ONLY_HOSTS
: bool
= true;
560 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
561 let builder
= run
.builder
;
562 run
.path("src/tools/cargo").default_condition(builder
.config
.extended
)
565 fn make_run(run
: RunConfig
<'_
>) {
566 run
.builder
.ensure(Cargo
{
567 compiler
: run
.builder
.compiler(run
.builder
.top_stage
, run
.builder
.config
.build
),
572 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
575 compiler
: self.compiler
,
578 mode
: Mode
::ToolRustc
,
579 path
: "src/tools/cargo",
580 is_optional_tool
: false,
581 source_type
: SourceType
::Submodule
,
582 extra_features
: Vec
::new(),
584 .expect("expected to build -- essential tool")
588 macro_rules
! tool_extended
{
589 (($sel
:ident
, $builder
:ident
),
594 $extra_deps
:block
;)+) => {
596 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
598 pub compiler
: Compiler
,
599 pub target
: Interned
<String
>,
600 pub extra_features
: Vec
<String
>,
603 impl Step
for $name
{
604 type Output
= Option
<PathBuf
>;
605 const DEFAULT
: bool
= true;
606 const ONLY_HOSTS
: bool
= true;
608 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
609 let builder
= run
.builder
;
610 run
.path($path
).default_condition(
611 builder
.config
.extended
612 && builder
.config
.tools
.as_ref().map_or(true, |tools
| {
613 tools
.iter().any(|tool
| match tool
.as_ref() {
614 "clippy" => $tool_name
== "clippy-driver",
615 x
=> $tool_name
== x
,
621 fn make_run(run
: RunConfig
<'_
>) {
622 run
.builder
.ensure($name
{
623 compiler
: run
.builder
.compiler(run
.builder
.top_stage
, run
.builder
.config
.build
),
625 extra_features
: Vec
::new(),
630 fn run(mut $sel
, $builder
: &Builder
<'_
>) -> Option
<PathBuf
> {
632 $builder
.ensure(ToolBuild
{
633 compiler
: $sel
.compiler
,
636 mode
: Mode
::ToolRustc
,
638 extra_features
: $sel
.extra_features
,
639 is_optional_tool
: true,
640 source_type
: SourceType
::Submodule
,
648 tool_extended
!((self, builder
),
649 Cargofmt
, rustfmt
, "src/tools/rustfmt", "cargo-fmt", {}
;
650 CargoClippy
, clippy
, "src/tools/clippy", "cargo-clippy", {}
;
651 Clippy
, clippy
, "src/tools/clippy", "clippy-driver", {}
;
652 Miri
, miri
, "src/tools/miri", "miri", {}
;
653 CargoMiri
, miri
, "src/tools/miri", "cargo-miri", {}
;
654 Rls
, rls
, "src/tools/rls", "rls", {
655 let clippy
= builder
.ensure(Clippy
{
656 compiler
: self.compiler
,
658 extra_features
: Vec
::new(),
660 if clippy
.is_some() {
661 self.extra_features
.push("clippy".to_owned());
664 Rustfmt
, rustfmt
, "src/tools/rustfmt", "rustfmt", {}
;
667 impl<'a
> Builder
<'a
> {
668 /// Gets a `Command` which is ready to run `tool` in `stage` built for
670 pub fn tool_cmd(&self, tool
: Tool
) -> Command
{
671 let mut cmd
= Command
::new(self.tool_exe(tool
));
672 let compiler
= self.compiler(0, self.config
.build
);
673 let host
= &compiler
.host
;
674 // Prepares the `cmd` provided to be able to run the `compiler` provided.
676 // Notably this munges the dynamic library lookup path to point to the
677 // right location to run `compiler`.
678 let mut lib_paths
: Vec
<PathBuf
> = vec
![
679 self.build
.rustc_snapshot_libdir(),
680 self.cargo_out(compiler
, Mode
::ToolBootstrap
, *host
).join("deps"),
683 // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make
684 // mode) and that C compiler may need some extra PATH modification. Do
686 if compiler
.host
.contains("msvc") {
687 let curpaths
= env
::var_os("PATH").unwrap_or_default();
688 let curpaths
= env
::split_paths(&curpaths
).collect
::<Vec
<_
>>();
689 for &(ref k
, ref v
) in self.cc
[&compiler
.host
].env() {
693 for path
in env
::split_paths(v
) {
694 if !curpaths
.contains(&path
) {
695 lib_paths
.push(path
);
701 add_dylib_path(lib_paths
, &mut cmd
);