1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
12 use std
::cell
::RefCell
;
13 use std
::collections
::BTreeSet
;
19 use std
::path
::{Path, PathBuf}
;
20 use std
::process
::Command
;
25 use util
::{exe, libdir, add_lib_path}
;
27 use cache
::{INTERNER, Interned, Cache}
;
29 use flags
::Subcommand
;
36 pub struct Builder
<'a
> {
41 stack
: RefCell
<Vec
<Box
<Any
>>>,
44 impl<'a
> Deref
for Builder
<'a
> {
47 fn deref(&self) -> &Self::Target
{
52 pub trait Step
: '
static + Clone
+ Debug
+ PartialEq
+ Eq
+ Hash
{
53 /// `PathBuf` when directories are created or to return a `Compiler` once
54 /// it's been assembled.
57 const DEFAULT
: bool
= false;
59 /// Run this rule for all hosts without cross compiling.
60 const ONLY_HOSTS
: bool
= false;
62 /// Run this rule for all targets, but only with the native host.
63 const ONLY_BUILD_TARGETS
: bool
= false;
65 /// Only run this step with the build triple as host and target.
66 const ONLY_BUILD
: bool
= false;
68 /// Primary function to execute this rule. Can call `builder.ensure(...)`
69 /// with other steps to run those.
70 fn run(self, builder
: &Builder
) -> Self::Output
;
72 /// When bootstrap is passed a set of paths, this controls whether this rule
73 /// will execute. However, it does not get called in a "default" context
74 /// when we are not passed any paths; in that case, make_run is called
76 fn should_run(run
: ShouldRun
) -> ShouldRun
;
78 /// Build up a "root" rule, either as a default rule or from a path passed
81 /// When path is `None`, we are executing in a context where no paths were
82 /// passed. When `./x.py build` is run, for example, this rule could get
83 /// called if it is in the correct list below with a path of `None`.
84 fn make_run(_run
: RunConfig
) {
85 // It is reasonable to not have an implementation of make_run for rules
86 // who do not want to get called from the root context. This means that
87 // they are likely dependencies (e.g., sysroot creation) or similar, and
88 // as such calling them from ./x.py isn't logical.
93 pub struct RunConfig
<'a
> {
94 pub builder
: &'a Builder
<'a
>,
95 pub host
: Interned
<String
>,
96 pub target
: Interned
<String
>,
97 pub path
: Option
<&'a Path
>,
100 struct StepDescription
{
103 only_build_targets
: bool
,
105 should_run
: fn(ShouldRun
) -> ShouldRun
,
106 make_run
: fn(RunConfig
),
109 impl StepDescription
{
110 fn from
<S
: Step
>() -> StepDescription
{
113 only_hosts
: S
::ONLY_HOSTS
,
114 only_build_targets
: S
::ONLY_BUILD_TARGETS
,
115 only_build
: S
::ONLY_BUILD
,
116 should_run
: S
::should_run
,
117 make_run
: S
::make_run
,
121 fn maybe_run(&self, builder
: &Builder
, path
: Option
<&Path
>) {
122 let build
= builder
.build
;
123 let hosts
= if self.only_build_targets
|| self.only_build
{
129 // Determine the targets participating in this rule.
130 let targets
= if self.only_hosts
{
131 if build
.config
.run_host_only
{
133 } else if self.only_build
{
143 for target
in targets
{
144 let run
= RunConfig
{
150 (self.make_run
)(run
);
155 fn run(v
: &[StepDescription
], builder
: &Builder
, paths
: &[PathBuf
]) {
156 let should_runs
= v
.iter().map(|desc
| {
157 (desc
.should_run
)(ShouldRun
::new(builder
))
158 }).collect
::<Vec
<_
>>();
159 if paths
.is_empty() {
160 for (desc
, should_run
) in v
.iter().zip(should_runs
) {
161 if desc
.default && should_run
.is_really_default
{
162 desc
.maybe_run(builder
, None
);
167 let mut attempted_run
= false;
168 for (desc
, should_run
) in v
.iter().zip(&should_runs
) {
169 if should_run
.run(path
) {
170 attempted_run
= true;
171 desc
.maybe_run(builder
, Some(path
));
176 eprintln
!("Warning: no rules matched {}.", path
.display());
184 pub struct ShouldRun
<'a
> {
185 pub builder
: &'a Builder
<'a
>,
186 // use a BTreeSet to maintain sort order
187 paths
: BTreeSet
<PathBuf
>,
189 // If this is a default rule, this is an additional constraint placed on
190 // it's run. Generally something like compiler docs being enabled.
191 is_really_default
: bool
,
194 impl<'a
> ShouldRun
<'a
> {
195 fn new(builder
: &'a Builder
) -> ShouldRun
<'a
> {
198 paths
: BTreeSet
::new(),
199 is_really_default
: true, // by default no additional conditions
203 pub fn default_condition(mut self, cond
: bool
) -> Self {
204 self.is_really_default
= cond
;
208 pub fn krate(mut self, name
: &str) -> Self {
209 for (_
, krate_path
) in self.builder
.crates(name
) {
210 self.paths
.insert(PathBuf
::from(krate_path
));
215 pub fn path(mut self, path
: &str) -> Self {
216 self.paths
.insert(PathBuf
::from(path
));
220 // allows being more explicit about why should_run in Step returns the value passed to it
221 pub fn never(self) -> ShouldRun
<'a
> {
225 fn run(&self, path
: &Path
) -> bool
{
226 self.paths
.iter().any(|p
| path
.ends_with(p
))
230 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
240 impl<'a
> Builder
<'a
> {
241 fn get_step_descriptions(kind
: Kind
) -> Vec
<StepDescription
> {
242 macro_rules
! describe
{
243 ($
($rule
:ty
),+ $
(,)*) => {{
244 vec
![$
(StepDescription
::from
::<$rule
>()),+]
248 Kind
::Build
=> describe
!(compile
::Std
, compile
::Test
, compile
::Rustc
,
249 compile
::StartupObjects
, tool
::BuildManifest
, tool
::Rustbook
, tool
::ErrorIndex
,
250 tool
::UnstableBookGen
, tool
::Tidy
, tool
::Linkchecker
, tool
::CargoTest
,
251 tool
::Compiletest
, tool
::RemoteTestServer
, tool
::RemoteTestClient
,
252 tool
::RustInstaller
, tool
::Cargo
, tool
::Rls
, tool
::Rustdoc
, tool
::Clippy
,
253 native
::Llvm
, tool
::Rustfmt
, tool
::Miri
),
254 Kind
::Test
=> describe
!(check
::Tidy
, check
::Bootstrap
, check
::DefaultCompiletest
,
255 check
::HostCompiletest
, check
::Crate
, check
::CrateLibrustc
, check
::Rustdoc
,
256 check
::Linkcheck
, check
::Cargotest
, check
::Cargo
, check
::Rls
, check
::Docs
,
257 check
::ErrorIndex
, check
::Distcheck
, check
::Rustfmt
, check
::Miri
, check
::Clippy
),
258 Kind
::Bench
=> describe
!(check
::Crate
, check
::CrateLibrustc
),
259 Kind
::Doc
=> describe
!(doc
::UnstableBook
, doc
::UnstableBookGen
, doc
::TheBook
,
260 doc
::Standalone
, doc
::Std
, doc
::Test
, doc
::Rustc
, doc
::ErrorIndex
, doc
::Nomicon
,
261 doc
::Reference
, doc
::Rustdoc
, doc
::CargoBook
),
262 Kind
::Dist
=> describe
!(dist
::Docs
, dist
::Mingw
, dist
::Rustc
, dist
::DebuggerScripts
,
263 dist
::Std
, dist
::Analysis
, dist
::Src
, dist
::PlainSourceTarball
, dist
::Cargo
,
264 dist
::Rls
, dist
::Extended
, dist
::HashSign
, dist
::DontDistWithMiriEnabled
),
265 Kind
::Install
=> describe
!(install
::Docs
, install
::Std
, install
::Cargo
, install
::Rls
,
266 install
::Analysis
, install
::Src
, install
::Rustc
),
270 pub fn get_help(build
: &Build
, subcommand
: &str) -> Option
<String
> {
271 let kind
= match subcommand
{
272 "build" => Kind
::Build
,
274 "test" => Kind
::Test
,
275 "bench" => Kind
::Bench
,
276 "dist" => Kind
::Dist
,
277 "install" => Kind
::Install
,
281 let builder
= Builder
{
283 top_stage
: build
.config
.stage
.unwrap_or(2),
286 stack
: RefCell
::new(Vec
::new()),
289 let builder
= &builder
;
290 let mut should_run
= ShouldRun
::new(builder
);
291 for desc
in Builder
::get_step_descriptions(builder
.kind
) {
292 should_run
= (desc
.should_run
)(should_run
);
294 let mut help
= String
::from("Available paths:\n");
295 for path
in should_run
.paths
{
296 help
.push_str(format
!(" ./x.py {} {}\n", subcommand
, path
.display()).as_str());
301 pub fn run(build
: &Build
) {
302 let (kind
, paths
) = match build
.config
.cmd
{
303 Subcommand
::Build { ref paths }
=> (Kind
::Build
, &paths
[..]),
304 Subcommand
::Doc { ref paths }
=> (Kind
::Doc
, &paths
[..]),
305 Subcommand
::Test { ref paths, .. }
=> (Kind
::Test
, &paths
[..]),
306 Subcommand
::Bench { ref paths, .. }
=> (Kind
::Bench
, &paths
[..]),
307 Subcommand
::Dist { ref paths }
=> (Kind
::Dist
, &paths
[..]),
308 Subcommand
::Install { ref paths }
=> (Kind
::Install
, &paths
[..]),
309 Subcommand
::Clean { .. }
=> panic
!(),
312 let builder
= Builder
{
314 top_stage
: build
.config
.stage
.unwrap_or(2),
317 stack
: RefCell
::new(Vec
::new()),
320 StepDescription
::run(&Builder
::get_step_descriptions(builder
.kind
), &builder
, paths
);
323 pub fn default_doc(&self, paths
: Option
<&[PathBuf
]>) {
324 let paths
= paths
.unwrap_or(&[]);
325 StepDescription
::run(&Builder
::get_step_descriptions(Kind
::Doc
), self, paths
);
328 /// Obtain a compiler at a given stage and for a given host. Explicitly does
329 /// not take `Compiler` since all `Compiler` instances are meant to be
330 /// obtained through this function, since it ensures that they are valid
331 /// (i.e., built and assembled).
332 pub fn compiler(&self, stage
: u32, host
: Interned
<String
>) -> Compiler
{
333 self.ensure(compile
::Assemble { target_compiler: Compiler { stage, host }
})
336 pub fn sysroot(&self, compiler
: Compiler
) -> Interned
<PathBuf
> {
337 self.ensure(compile
::Sysroot { compiler }
)
340 /// Returns the libdir where the standard library and other artifacts are
341 /// found for a compiler's sysroot.
342 pub fn sysroot_libdir(
343 &self, compiler
: Compiler
, target
: Interned
<String
>
344 ) -> Interned
<PathBuf
> {
345 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
348 target
: Interned
<String
>,
350 impl Step
for Libdir
{
351 type Output
= Interned
<PathBuf
>;
353 fn should_run(run
: ShouldRun
) -> ShouldRun
{
357 fn run(self, builder
: &Builder
) -> Interned
<PathBuf
> {
358 let compiler
= self.compiler
;
359 let lib
= if compiler
.stage
>= 2 && builder
.build
.config
.libdir_relative
.is_some() {
360 builder
.build
.config
.libdir_relative
.clone().unwrap()
364 let sysroot
= builder
.sysroot(self.compiler
).join(lib
)
365 .join("rustlib").join(self.target
).join("lib");
366 let _
= fs
::remove_dir_all(&sysroot
);
367 t
!(fs
::create_dir_all(&sysroot
));
368 INTERNER
.intern_path(sysroot
)
371 self.ensure(Libdir { compiler, target }
)
374 /// Returns the compiler's libdir where it stores the dynamic libraries that
375 /// it itself links against.
377 /// For example this returns `<sysroot>/lib` on Unix and `<sysroot>/bin` on
379 pub fn rustc_libdir(&self, compiler
: Compiler
) -> PathBuf
{
380 if compiler
.is_snapshot(self) {
381 self.build
.rustc_snapshot_libdir()
383 self.sysroot(compiler
).join(libdir(&compiler
.host
))
387 /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic
388 /// library lookup path.
389 pub fn add_rustc_lib_path(&self, compiler
: Compiler
, cmd
: &mut Command
) {
390 // Windows doesn't need dylib path munging because the dlls for the
391 // compiler live next to the compiler and the system will find them
397 add_lib_path(vec
![self.rustc_libdir(compiler
)], cmd
);
400 /// Get a path to the compiler specified.
401 pub fn rustc(&self, compiler
: Compiler
) -> PathBuf
{
402 if compiler
.is_snapshot(self) {
403 self.initial_rustc
.clone()
405 self.sysroot(compiler
).join("bin").join(exe("rustc", &compiler
.host
))
409 pub fn rustdoc(&self, host
: Interned
<String
>) -> PathBuf
{
410 self.ensure(tool
::Rustdoc { host }
)
413 pub fn rustdoc_cmd(&self, host
: Interned
<String
>) -> Command
{
414 let mut cmd
= Command
::new(&self.out
.join("bootstrap/debug/rustdoc"));
415 let compiler
= self.compiler(self.top_stage
, host
);
417 .env("RUSTC_STAGE", compiler
.stage
.to_string())
418 .env("RUSTC_SYSROOT", self.sysroot(compiler
))
419 .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler
, self.build
.build
))
420 .env("CFG_RELEASE_CHANNEL", &self.build
.config
.channel
)
421 .env("RUSTDOC_REAL", self.rustdoc(host
));
425 /// Prepares an invocation of `cargo` to be run.
427 /// This will create a `Command` that represents a pending execution of
428 /// Cargo. This cargo will be configured to use `compiler` as the actual
429 /// rustc compiler, its output will be scoped by `mode`'s output directory,
430 /// it will pass the `--target` flag for the specified `target`, and will be
431 /// executing the Cargo command `cmd`.
435 target
: Interned
<String
>,
436 cmd
: &str) -> Command
{
437 let mut cargo
= Command
::new(&self.initial_cargo
);
438 let out_dir
= self.stage_out(compiler
, mode
);
439 cargo
.env("CARGO_TARGET_DIR", out_dir
)
441 .arg("--target").arg(target
);
443 // If we were invoked from `make` then that's already got a jobserver
444 // set up for us so no need to tell Cargo about jobs all over again.
445 if env
::var_os("MAKEFLAGS").is_none() && env
::var_os("MFLAGS").is_none() {
446 cargo
.arg("-j").arg(self.jobs().to_string());
449 // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005
450 // Force cargo to output binaries with disambiguating hashes in the name
451 cargo
.env("__CARGO_DEFAULT_LIB_METADATA", &self.config
.channel
);
454 if compiler
.stage
== 0 && self.local_rebuild
{
455 // Assume the local-rebuild rustc already has stage1 features.
458 stage
= compiler
.stage
;
461 // Customize the compiler we're running. Specify the compiler to cargo
462 // as our shim and then pass it some various options used to configure
463 // how the actual compiler itself is called.
465 // These variables are primarily all read by
466 // src/bootstrap/bin/{rustc.rs,rustdoc.rs}
467 cargo
.env("RUSTBUILD_NATIVE_DIR", self.native_dir(target
))
468 .env("RUSTC", self.out
.join("bootstrap/debug/rustc"))
469 .env("RUSTC_REAL", self.rustc(compiler
))
470 .env("RUSTC_STAGE", stage
.to_string())
471 .env("RUSTC_CODEGEN_UNITS",
472 self.config
.rust_codegen_units
.to_string())
473 .env("RUSTC_DEBUG_ASSERTIONS",
474 self.config
.rust_debug_assertions
.to_string())
475 .env("RUSTC_SYSROOT", self.sysroot(compiler
))
476 .env("RUSTC_LIBDIR", self.rustc_libdir(compiler
))
477 .env("RUSTC_RPATH", self.config
.rust_rpath
.to_string())
478 .env("RUSTDOC", self.out
.join("bootstrap/debug/rustdoc"))
479 .env("RUSTDOC_REAL", if cmd
== "doc" || cmd
== "test" {
480 self.rustdoc(compiler
.host
)
482 PathBuf
::from("/path/to/nowhere/rustdoc/not/required")
484 .env("TEST_MIRI", self.config
.test_miri
.to_string())
485 .env("RUSTC_FLAGS", self.rustc_flags(target
).join(" "));
487 if mode
!= Mode
::Tool
{
488 // Tools don't get debuginfo right now, e.g. cargo and rls don't
489 // get compiled with debuginfo.
490 cargo
.env("RUSTC_DEBUGINFO", self.config
.rust_debuginfo
.to_string())
491 .env("RUSTC_DEBUGINFO_LINES", self.config
.rust_debuginfo_lines
.to_string())
492 .env("RUSTC_FORCE_UNSTABLE", "1");
494 // Currently the compiler depends on crates from crates.io, and
495 // then other crates can depend on the compiler (e.g. proc-macro
496 // crates). Let's say, for example that rustc itself depends on the
497 // bitflags crate. If an external crate then depends on the
498 // bitflags crate as well, we need to make sure they don't
499 // conflict, even if they pick the same version of bitflags. We'll
500 // want to make sure that e.g. a plugin and rustc each get their
501 // own copy of bitflags.
503 // Cargo ensures that this works in general through the -C metadata
504 // flag. This flag will frob the symbols in the binary to make sure
505 // they're different, even though the source code is the exact
506 // same. To solve this problem for the compiler we extend Cargo's
507 // already-passed -C metadata flag with our own. Our rustc.rs
508 // wrapper around the actual rustc will detect -C metadata being
509 // passed and frob it with this extra string we're passing in.
510 cargo
.env("RUSTC_METADATA_SUFFIX", "rustc");
513 if let Some(x
) = self.crt_static(target
) {
514 cargo
.env("RUSTC_CRT_STATIC", x
.to_string());
517 // Enable usage of unstable features
518 cargo
.env("RUSTC_BOOTSTRAP", "1");
519 self.add_rust_test_threads(&mut cargo
);
521 // Almost all of the crates that we compile as part of the bootstrap may
522 // have a build script, including the standard library. To compile a
523 // build script, however, it itself needs a standard library! This
524 // introduces a bit of a pickle when we're compiling the standard
527 // To work around this we actually end up using the snapshot compiler
528 // (stage0) for compiling build scripts of the standard library itself.
529 // The stage0 compiler is guaranteed to have a libstd available for use.
531 // For other crates, however, we know that we've already got a standard
532 // library up and running, so we can use the normal compiler to compile
533 // build scripts in that situation.
535 // If LLVM support is disabled we need to use the snapshot compiler to compile
536 // build scripts, as the new compiler doesnt support executables.
537 if mode
== Mode
::Libstd
|| !self.build
.config
.llvm_enabled
{
538 cargo
.env("RUSTC_SNAPSHOT", &self.initial_rustc
)
539 .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir());
541 cargo
.env("RUSTC_SNAPSHOT", self.rustc(compiler
))
542 .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler
));
545 // Ignore incremental modes except for stage0, since we're
546 // not guaranteeing correctness across builds if the compiler
547 // is changing under your feet.`
548 if self.config
.incremental
&& compiler
.stage
== 0 {
549 let incr_dir
= self.incremental_dir(compiler
);
550 cargo
.env("RUSTC_INCREMENTAL", incr_dir
);
553 if let Some(ref on_fail
) = self.config
.on_fail
{
554 cargo
.env("RUSTC_ON_FAIL", on_fail
);
557 cargo
.env("RUSTC_VERBOSE", format
!("{}", self.verbosity
));
559 // Specify some various options for build scripts used throughout
562 // FIXME: the guard against msvc shouldn't need to be here
563 if !target
.contains("msvc") {
564 cargo
.env(format
!("CC_{}", target
), self.cc(target
))
565 .env(format
!("AR_{}", target
), self.ar(target
).unwrap()) // only msvc is None
566 .env(format
!("CFLAGS_{}", target
), self.cflags(target
).join(" "));
568 if let Ok(cxx
) = self.cxx(target
) {
569 cargo
.env(format
!("CXX_{}", target
), cxx
);
573 if mode
== Mode
::Libstd
&& self.config
.extended
&& compiler
.is_final_stage(self) {
574 cargo
.env("RUSTC_SAVE_ANALYSIS", "api".to_string());
577 // Environment variables *required* throughout the build
579 // FIXME: should update code to not require this env var
580 cargo
.env("CFG_COMPILER_HOST_TRIPLE", target
);
582 // Set this for all builds to make sure doc builds also get it.
583 cargo
.env("CFG_RELEASE_CHANNEL", &self.build
.config
.channel
);
585 if self.is_verbose() {
588 // FIXME: cargo bench does not accept `--release`
589 if self.config
.rust_optimize
&& cmd
!= "bench" {
590 cargo
.arg("--release");
592 if self.config
.locked_deps
{
593 cargo
.arg("--locked");
595 if self.config
.vendor
|| self.is_sudo
{
596 cargo
.arg("--frozen");
599 self.ci_env
.force_coloring_in_ci(&mut cargo
);
604 /// Ensure that a given step is built, returning it's output. This will
605 /// cache the step, so it is safe (and good!) to call this as often as
606 /// needed to ensure that all dependencies are built.
607 pub fn ensure
<S
: Step
>(&'a
self, step
: S
) -> S
::Output
{
609 let mut stack
= self.stack
.borrow_mut();
610 for stack_step
in stack
.iter() {
612 if stack_step
.downcast_ref
::<S
>().map_or(true, |stack_step
| *stack_step
!= step
) {
615 let mut out
= String
::new();
616 out
+= &format
!("\n\nCycle in build detected when adding {:?}\n", step
);
617 for el
in stack
.iter().rev() {
618 out
+= &format
!("\t{:?}\n", el
);
622 if let Some(out
) = self.cache
.get(&step
) {
623 self.build
.verbose(&format
!("{}c {:?}", " ".repeat(stack
.len()), step
));
627 self.build
.verbose(&format
!("{}> {:?}", " ".repeat(stack
.len()), step
));
628 stack
.push(Box
::new(step
.clone()));
630 let out
= step
.clone().run(self);
632 let mut stack
= self.stack
.borrow_mut();
633 let cur_step
= stack
.pop().expect("step stack empty");
634 assert_eq
!(cur_step
.downcast_ref(), Some(&step
));
636 self.build
.verbose(&format
!("{}< {:?}", " ".repeat(self.stack
.borrow().len()), step
));
637 self.cache
.put(step
, out
.clone());