1 //! Constructs the dependency graph for compilation.
3 //! Rust code is typically organized as a set of Cargo packages. The
4 //! dependencies between the packages themselves are stored in the
5 //! `Resolve` struct. However, we can't use that information as is for
6 //! compilation! A package typically contains several targets, or crates,
7 //! and these targets has inter-dependencies. For example, you need to
8 //! compile the `lib` target before the `bin` one, and you need to compile
9 //! `build.rs` before either of those.
11 //! So, we need to lower the `Resolve`, which specifies dependencies between
12 //! *packages*, to a graph of dependencies between their *targets*, and this
13 //! is exactly what this module is doing! Well, almost exactly: another
14 //! complication is that we might want to compile the same target several times
15 //! (for example, with and without tests), so we actually build a dependency
16 //! graph of `Unit`s, which capture these properties.
18 use crate::core
::compiler
::Unit
;
19 use crate::core
::compiler
::{BuildContext, CompileKind, CompileMode}
;
20 use crate::core
::dependency
::DepKind
;
21 use crate::core
::package
::Downloads
;
22 use crate::core
::profiles
::{Profile, UnitFor}
;
23 use crate::core
::resolver
::features
::{FeaturesFor, ResolvedFeatures}
;
24 use crate::core
::resolver
::Resolve
;
25 use crate::core
::{InternedString, Package, PackageId, Target}
;
26 use crate::CargoResult
;
28 use std
::collections
::{HashMap, HashSet}
;
30 /// The dependency graph of Units.
31 pub type UnitGraph
<'a
> = HashMap
<Unit
<'a
>, Vec
<UnitDep
<'a
>>>;
33 /// A unit dependency.
34 #[derive(Debug, Clone, Hash, Eq, PartialEq, PartialOrd, Ord)]
35 pub struct UnitDep
<'a
> {
36 /// The dependency unit.
38 /// The purpose of this dependency (a dependency for a test, or a build
40 pub unit_for
: UnitFor
,
41 /// The name the parent uses to refer to this dependency.
42 pub extern_crate_name
: InternedString
,
43 /// Whether or not this is a public dependency.
45 /// If `true`, the dependency should not be added to Rust's prelude.
49 /// Collection of stuff used while creating the `UnitGraph`.
50 struct State
<'a
, 'cfg
> {
51 bcx
: &'a BuildContext
<'a
, 'cfg
>,
52 waiting_on_download
: HashSet
<PackageId
>,
53 downloads
: Downloads
<'a
, 'cfg
>,
54 unit_dependencies
: UnitGraph
<'a
>,
55 package_cache
: HashMap
<PackageId
, &'a Package
>,
56 usr_resolve
: &'a Resolve
,
57 usr_features
: &'a ResolvedFeatures
,
58 std_resolve
: Option
<&'a Resolve
>,
59 std_features
: Option
<&'a ResolvedFeatures
>,
60 /// This flag is `true` while generating the dependencies for the standard
65 pub fn build_unit_dependencies
<'a
, 'cfg
>(
66 bcx
: &'a BuildContext
<'a
, 'cfg
>,
68 features
: &'a ResolvedFeatures
,
69 std_resolve
: Option
<&'
a (Resolve
, ResolvedFeatures
)>,
71 std_roots
: &[Unit
<'a
>],
72 ) -> CargoResult
<UnitGraph
<'a
>> {
73 let (std_resolve
, std_features
) = match std_resolve
{
74 Some((r
, f
)) => (Some(r
), Some(f
)),
77 let mut state
= State
{
79 downloads
: bcx
.packages
.enable_download()?
,
80 waiting_on_download
: HashSet
::new(),
81 unit_dependencies
: HashMap
::new(),
82 package_cache
: HashMap
::new(),
84 usr_features
: features
,
90 let std_unit_deps
= calc_deps_of_std(&mut state
, std_roots
)?
;
92 deps_of_roots(roots
, &mut state
)?
;
93 super::links
::validate_links(state
.resolve(), &state
.unit_dependencies
)?
;
94 // Hopefully there aren't any links conflicts with the standard library?
96 if let Some(std_unit_deps
) = std_unit_deps
{
97 attach_std_deps(&mut state
, std_roots
, std_unit_deps
);
100 connect_run_custom_build_deps(&mut state
.unit_dependencies
);
102 // Dependencies are used in tons of places throughout the backend, many of
103 // which affect the determinism of the build itself. As a result be sure
104 // that dependency lists are always sorted to ensure we've always got a
105 // deterministic output.
106 for list
in state
.unit_dependencies
.values_mut() {
109 trace
!("ALL UNIT DEPENDENCIES {:#?}", state
.unit_dependencies
);
111 Ok(state
.unit_dependencies
)
114 /// Compute all the dependencies for the standard library.
115 fn calc_deps_of_std
<'a
, 'cfg
>(
116 mut state
: &mut State
<'a
, 'cfg
>,
117 std_roots
: &[Unit
<'a
>],
118 ) -> CargoResult
<Option
<UnitGraph
<'a
>>> {
119 if std_roots
.is_empty() {
122 // Compute dependencies for the standard library.
124 deps_of_roots(std_roots
, &mut state
)?
;
125 state
.is_std
= false;
126 Ok(Some(std
::mem
::replace(
127 &mut state
.unit_dependencies
,
132 /// Add the standard library units to the `unit_dependencies`.
133 fn attach_std_deps
<'a
, 'cfg
>(
134 state
: &mut State
<'a
, 'cfg
>,
135 std_roots
: &[Unit
<'a
>],
136 std_unit_deps
: UnitGraph
<'a
>,
138 // Attach the standard library as a dependency of every target unit.
139 for (unit
, deps
) in state
.unit_dependencies
.iter_mut() {
140 if !unit
.kind
.is_host() && !unit
.mode
.is_run_custom_build() {
141 deps
.extend(std_roots
.iter().map(|unit
| UnitDep
{
143 unit_for
: UnitFor
::new_normal(),
144 extern_crate_name
: unit
.pkg
.name(),
145 // TODO: Does this `public` make sense?
151 // And also include the dependencies of the standard library itself.
152 for (unit
, deps
) in std_unit_deps
.into_iter() {
153 if let Some(other_unit
) = state
.unit_dependencies
.insert(unit
, deps
) {
154 panic
!("std unit collision with existing unit: {:?}", other_unit
);
159 /// Compute all the dependencies of the given root units.
160 /// The result is stored in state.unit_dependencies.
161 fn deps_of_roots
<'a
, 'cfg
>(roots
: &[Unit
<'a
>], mut state
: &mut State
<'a
, 'cfg
>) -> CargoResult
<()> {
162 // Loop because we are downloading while building the dependency graph.
163 // The partially-built unit graph is discarded through each pass of the
164 // loop because it is incomplete because not all required Packages have
167 for unit
in roots
.iter() {
168 state
.get(unit
.pkg
.package_id())?
;
170 // Dependencies of tests/benches should not have `panic` set.
171 // We check the global test mode to see if we are running in `cargo
172 // test` in which case we ensure all dependencies have `panic`
173 // cleared, and avoid building the lib thrice (once with `panic`, once
174 // without, once for `--test`). In particular, the lib included for
175 // Doc tests and examples are `Build` mode here.
176 let unit_for
= if unit
.mode
.is_any_test() || state
.bcx
.build_config
.test() {
177 UnitFor
::new_test(state
.bcx
.config
)
178 } else if unit
.target
.is_custom_build() {
179 // This normally doesn't happen, except `clean` aggressively
180 // generates all units.
181 UnitFor
::new_build(false)
182 } else if unit
.target
.for_host() {
183 // Proc macro / plugin should never have panic set.
184 UnitFor
::new_compiler()
186 UnitFor
::new_normal()
188 deps_of(unit
, &mut state
, unit_for
)?
;
191 if !state
.waiting_on_download
.is_empty() {
192 state
.finish_some_downloads()?
;
193 state
.unit_dependencies
.clear();
201 /// Compute the dependencies of a single unit.
202 fn deps_of
<'a
, 'cfg
>(
204 state
: &mut State
<'a
, 'cfg
>,
206 ) -> CargoResult
<()> {
207 // Currently the `unit_dependencies` map does not include `unit_for`. This should
208 // be safe for now. `TestDependency` only exists to clear the `panic`
209 // flag, and you'll never ask for a `unit` with `panic` set as a
210 // `TestDependency`. `CustomBuild` should also be fine since if the
211 // requested unit's settings are the same as `Any`, `CustomBuild` can't
212 // affect anything else in the hierarchy.
213 if !state
.unit_dependencies
.contains_key(unit
) {
214 let unit_deps
= compute_deps(unit
, state
, unit_for
)?
;
215 state
.unit_dependencies
.insert(*unit
, unit_deps
.clone());
216 for unit_dep
in unit_deps
{
217 deps_of(&unit_dep
.unit
, state
, unit_dep
.unit_for
)?
;
223 /// For a package, returns all targets that are registered as dependencies
224 /// for that package.
225 /// This returns a `Vec` of `(Unit, UnitFor)` pairs. The `UnitFor`
226 /// is the profile type that should be used for dependencies of the unit.
227 fn compute_deps
<'a
, 'cfg
>(
229 state
: &mut State
<'a
, 'cfg
>,
231 ) -> CargoResult
<Vec
<UnitDep
<'a
>>> {
232 if unit
.mode
.is_run_custom_build() {
233 return compute_deps_custom_build(unit
, unit_for
, state
);
234 } else if unit
.mode
.is_doc() {
235 // Note: this does not include doc test.
236 return compute_deps_doc(unit
, state
);
240 let id
= unit
.pkg
.package_id();
241 let filtered_deps
= state
.resolve().deps(id
).filter(|&(_id
, deps
)| {
242 assert
!(!deps
.is_empty());
243 deps
.iter().any(|dep
| {
244 // If this target is a build command, then we only want build
245 // dependencies, otherwise we want everything *other than* build
247 if unit
.target
.is_custom_build() != dep
.is_build() {
251 // If this dependency is **not** a transitive dependency, then it
252 // only applies to test/example targets.
253 if !dep
.is_transitive()
254 && !unit
.target
.is_test()
255 && !unit
.target
.is_example()
256 && !unit
.mode
.is_any_test()
261 // If this dependency is only available for certain platforms,
262 // make sure we're only enabling it for that platform.
263 if !bcx
.target_data
.dep_platform_activated(dep
, unit
.kind
) {
267 // If we've gotten past all that, then this dependency is
273 let mut ret
= Vec
::new();
274 for (id
, _
) in filtered_deps
{
275 let pkg
= match state
.get(id
)?
{
279 let lib
= match pkg
.targets().iter().find(|t
| t
.is_lib()) {
283 let mode
= check_or_build_mode(unit
.mode
, lib
);
284 let dep_unit_for
= unit_for
285 .with_for_host(lib
.for_host())
286 // If it is a custom build script, then it *only* has build dependencies.
287 .with_build_dep(unit
.target
.is_custom_build());
289 if bcx
.config
.cli_unstable().dual_proc_macros
&& lib
.proc_macro() && !unit
.kind
.is_host() {
290 let unit_dep
= new_unit_dep(state
, unit
, pkg
, lib
, dep_unit_for
, unit
.kind
, mode
)?
;
293 new_unit_dep(state
, unit
, pkg
, lib
, dep_unit_for
, CompileKind
::Host
, mode
)?
;
296 let unit_dep
= new_unit_dep(
302 unit
.kind
.for_target(lib
),
309 // If this target is a build script, then what we've collected so far is
310 // all we need. If this isn't a build script, then it depends on the
311 // build script if there is one.
312 if unit
.target
.is_custom_build() {
315 ret
.extend(dep_build_script(unit
, unit_for
, state
)?
);
317 // If this target is a binary, test, example, etc, then it depends on
318 // the library of the same package. The call to `resolve.deps` above
319 // didn't include `pkg` in the return values, so we need to special case
320 // it here and see if we need to push `(pkg, pkg_lib_target)`.
321 if unit
.target
.is_lib() && unit
.mode
!= CompileMode
::Doctest
{
324 ret
.extend(maybe_lib(unit
, state
, unit_for
)?
);
326 // If any integration tests/benches are being run, make sure that
327 // binaries are built as well.
328 if !unit
.mode
.is_check()
329 && unit
.mode
.is_any_test()
330 && (unit
.target
.is_test() || unit
.target
.is_bench())
337 let no_required_features
= Vec
::new();
340 // Skip binaries with required features that have not been selected.
341 t
.required_features().unwrap_or(&no_required_features
).iter().all(|f
| {
342 unit
.features
.contains(&InternedString
::new(f
.as_str()))
351 UnitFor
::new_normal(),
352 unit
.kind
.for_target(t
),
356 .collect
::<CargoResult
<Vec
<UnitDep
<'a
>>>>()?
,
363 /// Returns the dependencies needed to run a build script.
365 /// The `unit` provided must represent an execution of a build script, and
366 /// the returned set of units must all be run before `unit` is run.
367 fn compute_deps_custom_build
<'a
, 'cfg
>(
370 state
: &mut State
<'a
, 'cfg
>,
371 ) -> CargoResult
<Vec
<UnitDep
<'a
>>> {
372 if let Some(links
) = unit
.pkg
.manifest().links() {
373 if state
.bcx
.script_override(links
, unit
.kind
).is_some() {
374 // Overridden build scripts don't have any dependencies.
375 return Ok(Vec
::new());
378 // All dependencies of this unit should use profiles for custom
380 let script_unit_for
= UnitFor
::new_build(unit_for
.is_for_build_dep());
381 // When not overridden, then the dependencies to run a build script are:
383 // 1. Compiling the build script itself.
384 // 2. For each immediate dependency of our package which has a `links`
385 // key, the execution of that build script.
387 // We don't have a great way of handling (2) here right now so this is
388 // deferred until after the graph of all unit dependencies has been
390 let unit_dep
= new_unit_dep(
396 // Build scripts always compiled for the host.
403 /// Returns the dependencies necessary to document a package.
404 fn compute_deps_doc
<'a
, 'cfg
>(
406 state
: &mut State
<'a
, 'cfg
>,
407 ) -> CargoResult
<Vec
<UnitDep
<'a
>>> {
411 .deps(unit
.pkg
.package_id())
412 .filter(|&(_id
, deps
)| {
413 deps
.iter().any(|dep
| match dep
.kind() {
414 DepKind
::Normal
=> bcx
.target_data
.dep_platform_activated(dep
, unit
.kind
),
419 // To document a library, we depend on dependencies actually being
420 // built. If we're documenting *all* libraries, then we also depend on
421 // the documentation of the library being built.
422 let mut ret
= Vec
::new();
423 for (id
, _deps
) in deps
{
424 let dep
= match state
.get(id
)?
{
428 let lib
= match dep
.targets().iter().find(|t
| t
.is_lib()) {
432 // Rustdoc only needs rmeta files for regular dependencies.
433 // However, for plugins/proc macros, deps should be built like normal.
434 let mode
= check_or_build_mode(unit
.mode
, lib
);
435 let dep_unit_for
= UnitFor
::new_normal().with_for_host(lib
.for_host());
436 let lib_unit_dep
= new_unit_dep(
442 unit
.kind
.for_target(lib
),
445 ret
.push(lib_unit_dep
);
446 if let CompileMode
::Doc { deps: true }
= unit
.mode
{
447 // Document this lib as well.
448 let doc_unit_dep
= new_unit_dep(
454 unit
.kind
.for_target(lib
),
457 ret
.push(doc_unit_dep
);
461 // Be sure to build/run the build script for documented libraries.
462 ret
.extend(dep_build_script(unit
, UnitFor
::new_normal(), state
)?
);
464 // If we document a binary/example, we need the library available.
465 if unit
.target
.is_bin() || unit
.target
.is_example() {
466 ret
.extend(maybe_lib(unit
, state
, UnitFor
::new_normal())?
);
473 state
: &mut State
<'a
, '_
>,
475 ) -> CargoResult
<Option
<UnitDep
<'a
>>> {
479 .find(|t
| t
.linkable())
481 let mode
= check_or_build_mode(unit
.mode
, t
);
488 unit
.kind
.for_target(t
),
495 /// If a build script is scheduled to be run for the package specified by
496 /// `unit`, this function will return the unit to run that build script.
498 /// Overriding a build script simply means that the running of the build
499 /// script itself doesn't have any dependencies, so even in that case a unit
500 /// of work is still returned. `None` is only returned if the package has no
502 fn dep_build_script
<'a
>(
505 state
: &State
<'a
, '_
>,
506 ) -> CargoResult
<Option
<UnitDep
<'a
>>> {
510 .find(|t
| t
.is_custom_build())
512 // The profile stored in the Unit is the profile for the thing
513 // the custom build script is running for.
517 .get_profile_run_custom_build(&unit
.profile
);
518 // UnitFor::new_build is used because we want the `host` flag set
519 // for all of our build dependencies (so they all get
520 // build-override profiles), including compiling the build.rs
523 // If `is_for_build_dep` here is `false`, that means we are a
524 // build.rs script for a normal dependency and we want to set the
525 // CARGO_FEATURE_* environment variables to the features as a
528 // If `is_for_build_dep` here is `true`, that means that this
529 // package is being used as a build dependency, and so we only
530 // want to set CARGO_FEATURE_* variables for the build-dependency
531 // side of the graph.
533 // Keep in mind that the RunCustomBuild unit and the Compile
534 // build.rs unit use the same features. This is because some
535 // people use `cfg!` and `#[cfg]` expressions to check for enabled
536 // features instead of just checking `CARGO_FEATURE_*` at runtime.
537 // In the case with `-Zfeatures=build_dep`, and a shared
538 // dependency has different features enabled for normal vs. build,
539 // then the build.rs script will get compiled twice. I believe it
540 // is not feasible to only build it once because it would break a
541 // large number of scripts (they would think they have the wrong
542 // set of features enabled).
543 let script_unit_for
= UnitFor
::new_build(unit_for
.is_for_build_dep());
544 new_unit_dep_with_profile(
551 CompileMode
::RunCustomBuild
,
558 /// Choose the correct mode for dependencies.
559 fn check_or_build_mode(mode
: CompileMode
, target
: &Target
) -> CompileMode
{
561 CompileMode
::Check { .. }
| CompileMode
::Doc { .. }
=> {
562 if target
.for_host() {
563 // Plugin and proc macro targets should be compiled like
567 // Regular dependencies should not be checked with --test.
568 // Regular dependencies of doc targets should emit rmeta only.
569 CompileMode
::Check { test: false }
572 _
=> CompileMode
::Build
,
576 /// Create a new Unit for a dependency from `parent` to `pkg` and `target`.
578 state
: &State
<'a
, '_
>,
585 ) -> CargoResult
<UnitDep
<'a
>> {
586 let profile
= state
.bcx
.profiles
.get_profile(
588 state
.bcx
.ws
.is_member(pkg
),
592 new_unit_dep_with_profile(state
, parent
, pkg
, target
, unit_for
, kind
, mode
, profile
)
595 fn new_unit_dep_with_profile
<'a
>(
596 state
: &State
<'a
, '_
>,
604 ) -> CargoResult
<UnitDep
<'a
>> {
605 // TODO: consider making extern_crate_name return InternedString?
606 let extern_crate_name
= InternedString
::new(&state
.resolve().extern_crate_name(
607 parent
.pkg
.package_id(),
613 .is_public_dep(parent
.pkg
.package_id(), pkg
.package_id());
614 let features_for
= match unit_for
.is_for_build_dep() {
615 true => FeaturesFor
::BuildDep
,
616 false => FeaturesFor
::NormalOrDev
,
618 let features
= state
.activated_features(pkg
.package_id(), features_for
);
622 .intern(pkg
, target
, profile
, kind
, mode
, features
, state
.is_std
);
632 /// Fill in missing dependencies for units of the `RunCustomBuild`
634 /// As mentioned above in `compute_deps_custom_build` each build script
635 /// execution has two dependencies. The first is compiling the build script
636 /// itself (already added) and the second is that all crates the package of the
637 /// build script depends on with `links` keys, their build script execution. (a
638 /// bit confusing eh?)
640 /// Here we take the entire `deps` map and add more dependencies from execution
641 /// of one build script to execution of another build script.
642 fn connect_run_custom_build_deps(unit_dependencies
: &mut UnitGraph
<'_
>) {
643 let mut new_deps
= Vec
::new();
646 // First up build a reverse dependency map. This is a mapping of all
647 // `RunCustomBuild` known steps to the unit which depends on them. For
648 // example a library might depend on a build script, so this map will
649 // have the build script as the key and the library would be in the
651 let mut reverse_deps_map
= HashMap
::new();
652 for (unit
, deps
) in unit_dependencies
.iter() {
654 if dep
.unit
.mode
== CompileMode
::RunCustomBuild
{
657 .or_insert_with(HashSet
::new
)
663 // Next, we take a look at all build scripts executions listed in the
664 // dependency map. Our job here is to take everything that depends on
665 // this build script (from our reverse map above) and look at the other
666 // package dependencies of these parents.
668 // If we depend on a linkable target and the build script mentions
669 // `links`, then we depend on that package's build script! Here we use
670 // `dep_build_script` to manufacture an appropriate build script unit to
672 for unit
in unit_dependencies
674 .filter(|k
| k
.mode
== CompileMode
::RunCustomBuild
)
676 // This is the lib that runs this custom build.
677 let reverse_deps
= match reverse_deps_map
.get(unit
) {
682 let to_add
= reverse_deps
684 // Get all deps for lib.
685 .flat_map(|reverse_dep
| unit_dependencies
[reverse_dep
].iter())
686 // Only deps with `links`.
688 other
.unit
.pkg
!= unit
.pkg
689 && other
.unit
.target
.linkable()
690 && other
.unit
.pkg
.manifest().links().is_some()
692 // Get the RunCustomBuild for other lib.
693 .filter_map(|other
| {
694 unit_dependencies
[&other
.unit
]
696 .find(|other_dep
| other_dep
.unit
.mode
== CompileMode
::RunCustomBuild
)
699 .collect
::<HashSet
<_
>>();
701 if !to_add
.is_empty() {
702 // (RunCustomBuild, set(other RunCustomBuild))
703 new_deps
.push((*unit
, to_add
));
708 // And finally, add in all the missing dependencies!
709 for (unit
, new_deps
) in new_deps
{
710 unit_dependencies
.get_mut(&unit
).unwrap().extend(new_deps
);
714 impl<'a
, 'cfg
> State
<'a
, 'cfg
> {
715 fn resolve(&self) -> &'a Resolve
{
717 self.std_resolve
.unwrap()
723 fn activated_features(
726 features_for
: FeaturesFor
,
727 ) -> Vec
<InternedString
> {
728 let features
= if self.is_std
{
729 self.std_features
.unwrap()
733 features
.activated_features(pkg_id
, features_for
)
736 fn get(&mut self, id
: PackageId
) -> CargoResult
<Option
<&'a Package
>> {
737 if let Some(pkg
) = self.package_cache
.get(&id
) {
738 return Ok(Some(pkg
));
740 if !self.waiting_on_download
.insert(id
) {
743 if let Some(pkg
) = self.downloads
.start(id
)?
{
744 self.package_cache
.insert(id
, pkg
);
745 self.waiting_on_download
.remove(&id
);
746 return Ok(Some(pkg
));
751 /// Completes at least one downloading, maybe waiting for more to complete.
753 /// This function will block the current thread waiting for at least one
754 /// crate to finish downloading. The function may continue to download more
755 /// crates if it looks like there's a long enough queue of crates to keep
756 /// downloading. When only a handful of packages remain this function
757 /// returns, and it's hoped that by returning we'll be able to push more
758 /// packages to download into the queue.
759 fn finish_some_downloads(&mut self) -> CargoResult
<()> {
760 assert
!(self.downloads
.remaining() > 0);
762 let pkg
= self.downloads
.wait()?
;
763 self.waiting_on_download
.remove(&pkg
.package_id());
764 self.package_cache
.insert(pkg
.package_id(), pkg
);
766 // Arbitrarily choose that 5 or more packages concurrently download
767 // is a good enough number to "fill the network pipe". If we have
768 // less than this let's recompute the whole unit dependency graph
769 // again and try to find some more packages to download.
770 if self.downloads
.remaining() < 5 {