1 use std
::collections
::HashSet
;
4 use std
::path
::PathBuf
;
5 use std
::process
::{exit, Command}
;
9 use crate::builder
::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}
;
10 use crate::cache
::Interned
;
12 use crate::channel
::GitInfo
;
14 use crate::toolstate
::ToolState
;
15 use crate::util
::{add_dylib_path, exe, CiEnv}
;
19 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
25 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
28 target
: Interned
<String
>,
32 is_optional_tool
: bool
,
33 source_type
: SourceType
,
34 extra_features
: Vec
<String
>,
37 impl Step
for ToolBuild
{
38 type Output
= Option
<PathBuf
>;
40 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
44 /// Builds a tool in `src/tools`
46 /// This will build the specified tool with the specified `host` compiler in
47 /// `stage` into the normal cargo output directory.
48 fn run(self, builder
: &Builder
<'_
>) -> Option
<PathBuf
> {
49 let compiler
= self.compiler
;
50 let target
= self.target
;
53 let is_optional_tool
= self.is_optional_tool
;
56 Mode
::ToolRustc
=> builder
.ensure(compile
::Rustc { compiler, target }
),
57 Mode
::ToolStd
=> builder
.ensure(compile
::Std { compiler, target }
),
58 Mode
::ToolBootstrap
=> {}
// uses downloaded stage0 compiler libs
59 _
=> panic
!("unexpected Mode for tool build"),
62 let cargo
= prepare_tool_cargo(
73 builder
.info(&format
!("Building stage{} tool {} ({})", compiler
.stage
, tool
, target
));
74 let mut duplicates
= Vec
::new();
75 let is_expected
= compile
::stream_cargo(builder
, cargo
, vec
![], &mut |msg
| {
76 // Only care about big things like the RLS/Cargo for now
78 "rls" | "cargo" | "clippy-driver" | "miri" | "rustfmt" => {}
82 let (id
, features
, filenames
) = match msg
{
83 compile
::CargoMessage
::CompilerArtifact
{
88 } => (package_id
, features
, filenames
),
91 let features
= features
.iter().map(|s
| s
.to_string()).collect
::<Vec
<_
>>();
93 for path
in filenames
{
94 let val
= (tool
, PathBuf
::from(&*path
), features
.clone());
95 // we're only interested in deduplicating rlibs for now
96 if val
.1.extension().and_then(|s
| s
.to_str()) != Some("rlib") {
100 // Don't worry about compiles that turn out to be host
101 // dependencies or build scripts. To skip these we look for
102 // anything that goes in `.../release/deps` but *doesn't* go in
103 // `$target/release/deps`. This ensure that outputs in
104 // `$target/release` are still considered candidates for
106 if let Some(parent
) = val
.1.parent() {
107 if parent
.ends_with("release/deps") {
108 let maybe_target
= parent
110 .and_then(|p
| p
.parent())
111 .and_then(|p
| p
.file_name())
112 .and_then(|p
| p
.to_str())
114 if maybe_target
!= &*target
{
120 // Record that we've built an artifact for `id`, and if one was
121 // already listed then we need to see if we reused the same
122 // artifact or produced a duplicate.
123 let mut artifacts
= builder
.tool_artifacts
.borrow_mut();
124 let prev_artifacts
= artifacts
.entry(target
).or_default();
125 let prev
= match prev_artifacts
.get(&*id
) {
128 prev_artifacts
.insert(id
.to_string(), val
);
133 return; // same path, same artifact
136 // If the paths are different and one of them *isn't* inside of
137 // `release/deps`, then it means it's probably in
138 // `$target/release`, or it's some final artifact like
139 // `libcargo.rlib`. In these situations Cargo probably just
140 // copied it up from `$target/release/deps/libcargo-xxxx.rlib`,
141 // so if the features are equal we can just skip it.
142 let prev_no_hash
= prev
.1.parent().unwrap().ends_with("release/deps");
143 let val_no_hash
= val
.1.parent().unwrap().ends_with("release/deps");
144 if prev
.2 == val
.2 || !prev_no_hash
|| !val_no_hash
{
148 // ... and otherwise this looks like we duplicated some sort of
149 // compilation, so record it to generate an error later.
150 duplicates
.push((id
.to_string(), val
, prev
.clone()));
154 if is_expected
&& !duplicates
.is_empty() {
156 "duplicate artifacts found when compiling a tool, this \
157 typically means that something was recompiled because \
158 a transitive dependency has different features activated \
159 than in a previous build:\n"
162 "the following dependencies are duplicated although they \
163 have the same features enabled:"
165 for (id
, cur
, prev
) in duplicates
.drain_filter(|(_
, cur
, prev
)| cur
.2 == prev
.2) {
168 println
!(" `{}` ({:?})\n `{}` ({:?})", cur
.0, cur
.1, prev
.0, prev
.1);
170 println
!("the following dependencies have different features:");
171 for (id
, cur
, prev
) in duplicates
{
173 let cur_features
: HashSet
<_
> = cur
.2.into_iter
().collect();
174 let prev_features
: HashSet
<_
> = prev
.2.into_iter
().collect();
176 " `{}` additionally enabled features {:?} at {:?}",
178 &cur_features
- &prev_features
,
182 " `{}` additionally enabled features {:?} at {:?}",
184 &prev_features
- &cur_features
,
190 "to fix this you will probably want to edit the local \
191 src/tools/rustc-workspace-hack/Cargo.toml crate, as \
192 that will update the dependency graph to ensure that \
193 these crates all share the same feature set"
195 panic
!("tools should not compile multiple copies of the same crate");
198 builder
.save_toolstate(
200 if is_expected { ToolState::TestFail }
else { ToolState::BuildFail }
,
204 if !is_optional_tool
{
211 builder
.cargo_out(compiler
, self.mode
, target
).join(exe(tool
, &compiler
.host
));
212 let bin
= builder
.tools_dir(compiler
).join(exe(tool
, &compiler
.host
));
213 builder
.copy(&cargo_out
, &bin
);
219 pub fn prepare_tool_cargo(
220 builder
: &Builder
<'_
>,
223 target
: Interned
<String
>,
224 command
: &'
static str,
226 source_type
: SourceType
,
227 extra_features
: &[String
],
229 let mut cargo
= builder
.cargo(compiler
, mode
, target
, command
);
230 let dir
= builder
.src
.join(path
);
231 cargo
.arg("--manifest-path").arg(dir
.join("Cargo.toml"));
233 if source_type
== SourceType
::Submodule
{
234 cargo
.env("RUSTC_EXTERNAL_TOOL", "1");
237 let mut features
= extra_features
.to_vec();
238 if builder
.build
.config
.cargo_native_static
{
239 if path
.ends_with("cargo")
240 || path
.ends_with("rls")
241 || path
.ends_with("clippy")
242 || path
.ends_with("miri")
243 || path
.ends_with("rustbook")
244 || path
.ends_with("rustfmt")
246 cargo
.env("LIBZ_SYS_STATIC", "1");
247 features
.push("rustc-workspace-hack/all-static".to_string());
251 // if tools are using lzma we want to force the build script to build its
253 cargo
.env("LZMA_API_STATIC", "1");
255 // CFG_RELEASE is needed by rustfmt (and possibly other tools) which
256 // import rustc-ap-rustc_attr which requires this to be set for the
257 // `#[cfg(version(...))]` attribute.
258 cargo
.env("CFG_RELEASE", builder
.rust_release());
259 cargo
.env("CFG_RELEASE_CHANNEL", &builder
.config
.channel
);
260 cargo
.env("CFG_VERSION", builder
.rust_version());
261 cargo
.env("CFG_RELEASE_NUM", channel
::CFG_RELEASE_NUM
);
263 let info
= GitInfo
::new(builder
.config
.ignore_git
, &dir
);
264 if let Some(sha
) = info
.sha() {
265 cargo
.env("CFG_COMMIT_HASH", sha
);
267 if let Some(sha_short
) = info
.sha_short() {
268 cargo
.env("CFG_SHORT_COMMIT_HASH", sha_short
);
270 if let Some(date
) = info
.commit_date() {
271 cargo
.env("CFG_COMMIT_DATE", date
);
273 if !features
.is_empty() {
274 cargo
.arg("--features").arg(&features
.join(", "));
279 fn rustbook_features() -> Vec
<String
> {
280 let mut features
= Vec
::new();
282 // Due to CI budged and risk of spurious failures we want to limit jobs running this check.
283 // At same time local builds should run it regardless of the platform.
284 // `CiEnv::None` means it's local build and `CHECK_LINKS` is defined in x86_64-gnu-tools to
285 // explicitly enable it on single job
286 if CiEnv
::current() == CiEnv
::None
|| env
::var("CHECK_LINKS").is_ok() {
287 features
.push("linkcheck".to_string());
293 macro_rules
! bootstrap_tool
{
295 $name
:ident
, $path
:expr
, $tool_name
:expr
296 $
(,is_external_tool
= $external
:expr
)*
297 $
(,is_unstable_tool
= $unstable
:expr
)*
298 $
(,features
= $features
:expr
)*
301 #[derive(Copy, PartialEq, Eq, Clone)]
308 impl<'a
> Builder
<'a
> {
309 pub fn tool_exe(&self, tool
: Tool
) -> PathBuf
{
313 compiler
: self.compiler(0, self.config
.build
),
314 target
: self.config
.build
,
322 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
324 pub compiler
: Compiler
,
325 pub target
: Interned
<String
>,
328 impl Step
for $name
{
329 type Output
= PathBuf
;
331 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
335 fn make_run(run
: RunConfig
<'_
>) {
336 run
.builder
.ensure($name
{
338 compiler
: run
.builder
.compiler(0, run
.builder
.config
.build
),
343 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
344 builder
.ensure(ToolBuild
{
345 compiler
: self.compiler
,
348 mode
: if false $
(|| $unstable
)* {
349 // use in-tree libraries for unstable features
355 is_optional_tool
: false,
356 source_type
: if false $
(|| $external
)* {
357 SourceType
::Submodule
362 // FIXME(#60643): avoid this lint by using `_`
363 let mut _tmp
= Vec
::new();
364 $
(_tmp
.extend($features
);)*
367 }).expect("expected to build -- essential tool")
375 Rustbook
, "src/tools/rustbook", "rustbook", features
= rustbook_features();
376 UnstableBookGen
, "src/tools/unstable-book-gen", "unstable-book-gen";
377 Tidy
, "src/tools/tidy", "tidy";
378 Linkchecker
, "src/tools/linkchecker", "linkchecker";
379 CargoTest
, "src/tools/cargotest", "cargotest";
380 Compiletest
, "src/tools/compiletest", "compiletest", is_unstable_tool
= true;
381 BuildManifest
, "src/tools/build-manifest", "build-manifest";
382 RemoteTestClient
, "src/tools/remote-test-client", "remote-test-client";
383 RustInstaller
, "src/tools/rust-installer", "fabricate", is_external_tool
= true;
384 RustdocTheme
, "src/tools/rustdoc-themes", "rustdoc-themes";
385 ExpandYamlAnchors
, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
388 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
389 pub struct ErrorIndex
{
390 pub compiler
: Compiler
,
394 pub fn command(builder
: &Builder
<'_
>, compiler
: Compiler
) -> Command
{
395 let mut cmd
= Command
::new(builder
.ensure(ErrorIndex { compiler }
));
397 vec
![PathBuf
::from(&builder
.sysroot_libdir(compiler
, compiler
.host
))],
404 impl Step
for ErrorIndex
{
405 type Output
= PathBuf
;
407 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
408 run
.path("src/tools/error_index_generator")
411 fn make_run(run
: RunConfig
<'_
>) {
412 // Compile the error-index in the same stage as rustdoc to avoid
413 // recompiling rustdoc twice if we can.
414 let stage
= if run
.builder
.top_stage
>= 2 { run.builder.top_stage }
else { 0 }
;
416 .ensure(ErrorIndex { compiler: run.builder.compiler(stage, run.builder.config.build) }
);
419 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
422 compiler
: self.compiler
,
423 target
: self.compiler
.host
,
424 tool
: "error_index_generator",
425 mode
: Mode
::ToolRustc
,
426 path
: "src/tools/error_index_generator",
427 is_optional_tool
: false,
428 source_type
: SourceType
::InTree
,
429 extra_features
: Vec
::new(),
431 .expect("expected to build -- essential tool")
435 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
436 pub struct RemoteTestServer
{
437 pub compiler
: Compiler
,
438 pub target
: Interned
<String
>,
441 impl Step
for RemoteTestServer
{
442 type Output
= PathBuf
;
444 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
445 run
.path("src/tools/remote-test-server")
448 fn make_run(run
: RunConfig
<'_
>) {
449 run
.builder
.ensure(RemoteTestServer
{
450 compiler
: run
.builder
.compiler(run
.builder
.top_stage
, run
.builder
.config
.build
),
455 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
458 compiler
: self.compiler
,
460 tool
: "remote-test-server",
462 path
: "src/tools/remote-test-server",
463 is_optional_tool
: false,
464 source_type
: SourceType
::InTree
,
465 extra_features
: Vec
::new(),
467 .expect("expected to build -- essential tool")
471 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
473 /// This should only ever be 0 or 2.
474 /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.
475 pub compiler
: Compiler
,
478 impl Step
for Rustdoc
{
479 type Output
= PathBuf
;
480 const DEFAULT
: bool
= true;
481 const ONLY_HOSTS
: bool
= true;
483 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
484 run
.path("src/tools/rustdoc")
487 fn make_run(run
: RunConfig
<'_
>) {
489 .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.host) }
);
492 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
493 let target_compiler
= self.compiler
;
494 if target_compiler
.stage
== 0 {
495 if !target_compiler
.is_snapshot(builder
) {
496 panic
!("rustdoc in stage 0 must be snapshot rustdoc");
498 return builder
.initial_rustc
.with_file_name(exe("rustdoc", &target_compiler
.host
));
500 let target
= target_compiler
.host
;
501 // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
502 // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
503 // compilers, which isn't what we want. Rustdoc should be linked in the same way as the
504 // rustc compiler it's paired with, so it must be built with the previous stage compiler.
505 let build_compiler
= builder
.compiler(target_compiler
.stage
- 1, builder
.config
.build
);
507 // The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
508 // compiler libraries, ...) are built. Rustdoc does not require the presence of any
509 // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since
510 // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional
511 // libraries here. The intuition here is that If we've built a compiler, we should be able
514 let cargo
= prepare_tool_cargo(
525 builder
.info(&format
!(
526 "Building rustdoc for stage{} ({})",
527 target_compiler
.stage
, target_compiler
.host
529 builder
.run(&mut cargo
.into());
531 // Cargo adds a number of paths to the dylib search path on windows, which results in
532 // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
533 // rustdoc a different name.
534 let tool_rustdoc
= builder
535 .cargo_out(build_compiler
, Mode
::ToolRustc
, target
)
536 .join(exe("rustdoc_tool_binary", &target_compiler
.host
));
538 // don't create a stage0-sysroot/bin directory.
539 if target_compiler
.stage
> 0 {
540 let sysroot
= builder
.sysroot(target_compiler
);
541 let bindir
= sysroot
.join("bin");
542 t
!(fs
::create_dir_all(&bindir
));
543 let bin_rustdoc
= bindir
.join(exe("rustdoc", &*target_compiler
.host
));
544 let _
= fs
::remove_file(&bin_rustdoc
);
545 builder
.copy(&tool_rustdoc
, &bin_rustdoc
);
553 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
555 pub compiler
: Compiler
,
556 pub target
: Interned
<String
>,
559 impl Step
for Cargo
{
560 type Output
= PathBuf
;
561 const DEFAULT
: bool
= true;
562 const ONLY_HOSTS
: bool
= true;
564 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
565 let builder
= run
.builder
;
566 run
.path("src/tools/cargo").default_condition(builder
.config
.extended
)
569 fn make_run(run
: RunConfig
<'_
>) {
570 run
.builder
.ensure(Cargo
{
571 compiler
: run
.builder
.compiler(run
.builder
.top_stage
, run
.builder
.config
.build
),
576 fn run(self, builder
: &Builder
<'_
>) -> PathBuf
{
579 compiler
: self.compiler
,
582 mode
: Mode
::ToolRustc
,
583 path
: "src/tools/cargo",
584 is_optional_tool
: false,
585 source_type
: SourceType
::Submodule
,
586 extra_features
: Vec
::new(),
588 .expect("expected to build -- essential tool")
592 macro_rules
! tool_extended
{
593 (($sel
:ident
, $builder
:ident
),
598 $extra_deps
:block
;)+) => {
600 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
602 pub compiler
: Compiler
,
603 pub target
: Interned
<String
>,
604 pub extra_features
: Vec
<String
>,
607 impl Step
for $name
{
608 type Output
= Option
<PathBuf
>;
609 const DEFAULT
: bool
= true;
610 const ONLY_HOSTS
: bool
= true;
612 fn should_run(run
: ShouldRun
<'_
>) -> ShouldRun
<'_
> {
613 let builder
= run
.builder
;
614 run
.path($path
).default_condition(
615 builder
.config
.extended
616 && builder
.config
.tools
.as_ref().map_or(true, |tools
| {
617 tools
.iter().any(|tool
| match tool
.as_ref() {
618 "clippy" => $tool_name
== "clippy-driver",
619 x
=> $tool_name
== x
,
625 fn make_run(run
: RunConfig
<'_
>) {
626 run
.builder
.ensure($name
{
627 compiler
: run
.builder
.compiler(run
.builder
.top_stage
, run
.builder
.config
.build
),
629 extra_features
: Vec
::new(),
634 fn run(mut $sel
, $builder
: &Builder
<'_
>) -> Option
<PathBuf
> {
636 $builder
.ensure(ToolBuild
{
637 compiler
: $sel
.compiler
,
640 mode
: Mode
::ToolRustc
,
642 extra_features
: $sel
.extra_features
,
643 is_optional_tool
: true,
644 source_type
: SourceType
::Submodule
,
652 // Note: tools need to be also added to `Builder::get_step_descriptions` in `build.rs`
653 // to make `./x.py build <tool>` work.
654 tool_extended
!((self, builder
),
655 Cargofmt
, rustfmt
, "src/tools/rustfmt", "cargo-fmt", {}
;
656 CargoClippy
, clippy
, "src/tools/clippy", "cargo-clippy", {}
;
657 Clippy
, clippy
, "src/tools/clippy", "clippy-driver", {}
;
658 Miri
, miri
, "src/tools/miri", "miri", {}
;
659 CargoMiri
, miri
, "src/tools/miri/cargo-miri", "cargo-miri", {}
;
660 Rls
, rls
, "src/tools/rls", "rls", {
661 builder
.ensure(Clippy
{
662 compiler
: self.compiler
,
664 extra_features
: Vec
::new(),
666 self.extra_features
.push("clippy".to_owned());
668 Rustfmt
, rustfmt
, "src/tools/rustfmt", "rustfmt", {}
;
671 impl<'a
> Builder
<'a
> {
672 /// Gets a `Command` which is ready to run `tool` in `stage` built for
674 pub fn tool_cmd(&self, tool
: Tool
) -> Command
{
675 let mut cmd
= Command
::new(self.tool_exe(tool
));
676 let compiler
= self.compiler(0, self.config
.build
);
677 let host
= &compiler
.host
;
678 // Prepares the `cmd` provided to be able to run the `compiler` provided.
680 // Notably this munges the dynamic library lookup path to point to the
681 // right location to run `compiler`.
682 let mut lib_paths
: Vec
<PathBuf
> = vec
![
683 self.build
.rustc_snapshot_libdir(),
684 self.cargo_out(compiler
, Mode
::ToolBootstrap
, *host
).join("deps"),
687 // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make
688 // mode) and that C compiler may need some extra PATH modification. Do
690 if compiler
.host
.contains("msvc") {
691 let curpaths
= env
::var_os("PATH").unwrap_or_default();
692 let curpaths
= env
::split_paths(&curpaths
).collect
::<Vec
<_
>>();
693 for &(ref k
, ref v
) in self.cc
[&compiler
.host
].env() {
697 for path
in env
::split_paths(v
) {
698 if !curpaths
.contains(&path
) {
699 lib_paths
.push(path
);
705 add_dylib_path(lib_paths
, &mut cmd
);