1 use std
::collections
::hash_map
::Entry
;
3 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
4 use rustc_hir
::def
::DefKind
;
5 use rustc_hir
::def_id
::{DefId, CRATE_DEF_INDEX, LOCAL_CRATE}
;
6 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
7 use rustc_middle
::middle
::exported_symbols
::SymbolExportLevel
;
8 use rustc_middle
::mir
::mono
::{CodegenUnit, CodegenUnitNameBuilder, Linkage, Visibility}
;
9 use rustc_middle
::mir
::mono
::{InstantiationMode, MonoItem}
;
10 use rustc_middle
::ty
::print
::characteristic_def_id_of_type
;
11 use rustc_middle
::ty
::{self, DefIdTree, InstanceDef, TyCtxt}
;
12 use rustc_span
::symbol
::Symbol
;
14 use crate::monomorphize
::collector
::InliningMap
;
15 use crate::monomorphize
::partitioning
::merging
;
16 use crate::monomorphize
::partitioning
::{
17 MonoItemPlacement
, Partitioner
, PostInliningPartitioning
, PreInliningPartitioning
,
20 pub struct DefaultPartitioning
;
22 impl<'tcx
> Partitioner
<'tcx
> for DefaultPartitioning
{
23 fn place_root_mono_items(
26 mono_items
: &mut dyn Iterator
<Item
= MonoItem
<'tcx
>>,
27 ) -> PreInliningPartitioning
<'tcx
> {
28 let mut roots
= FxHashSet
::default();
29 let mut codegen_units
= FxHashMap
::default();
30 let is_incremental_build
= tcx
.sess
.opts
.incremental
.is_some();
31 let mut internalization_candidates
= FxHashSet
::default();
33 // Determine if monomorphizations instantiated in this crate will be made
34 // available to downstream crates. This depends on whether we are in
35 // share-generics mode and whether the current crate can even have
37 let export_generics
= tcx
.sess
.opts
.share_generics() && tcx
.local_crate_exports_generics();
39 let cgu_name_builder
= &mut CodegenUnitNameBuilder
::new(tcx
);
40 let cgu_name_cache
= &mut FxHashMap
::default();
42 for mono_item
in mono_items
{
43 match mono_item
.instantiation_mode(tcx
) {
44 InstantiationMode
::GloballyShared { .. }
=> {}
45 InstantiationMode
::LocalCopy
=> continue,
48 let characteristic_def_id
= characteristic_def_id_of_mono_item(tcx
, mono_item
);
49 let is_volatile
= is_incremental_build
&& mono_item
.is_generic_fn();
51 let codegen_unit_name
= match characteristic_def_id
{
52 Some(def_id
) => compute_codegen_unit_name(
59 None
=> fallback_cgu_name(cgu_name_builder
),
62 let codegen_unit
= codegen_units
63 .entry(codegen_unit_name
)
64 .or_insert_with(|| CodegenUnit
::new(codegen_unit_name
));
66 let mut can_be_internalized
= true;
67 let (linkage
, visibility
) = mono_item_linkage_and_visibility(
70 &mut can_be_internalized
,
73 if visibility
== Visibility
::Hidden
&& can_be_internalized
{
74 internalization_candidates
.insert(mono_item
);
77 codegen_unit
.items_mut().insert(mono_item
, (linkage
, visibility
));
78 roots
.insert(mono_item
);
81 // Always ensure we have at least one CGU; otherwise, if we have a
82 // crate with just types (for example), we could wind up with no CGU.
83 if codegen_units
.is_empty() {
84 let codegen_unit_name
= fallback_cgu_name(cgu_name_builder
);
85 codegen_units
.insert(codegen_unit_name
, CodegenUnit
::new(codegen_unit_name
));
88 PreInliningPartitioning
{
89 codegen_units
: codegen_units
91 .map(|(_
, codegen_unit
)| codegen_unit
)
94 internalization_candidates
,
98 fn merge_codegen_units(
101 initial_partitioning
: &mut PreInliningPartitioning
<'tcx
>,
102 target_cgu_count
: usize,
104 merging
::merge_codegen_units(tcx
, initial_partitioning
, target_cgu_count
);
107 fn place_inlined_mono_items(
109 initial_partitioning
: PreInliningPartitioning
<'tcx
>,
110 inlining_map
: &InliningMap
<'tcx
>,
111 ) -> PostInliningPartitioning
<'tcx
> {
112 let mut new_partitioning
= Vec
::new();
113 let mut mono_item_placements
= FxHashMap
::default();
115 let PreInliningPartitioning
{
116 codegen_units
: initial_cgus
,
118 internalization_candidates
,
119 } = initial_partitioning
;
121 let single_codegen_unit
= initial_cgus
.len() == 1;
123 for old_codegen_unit
in initial_cgus
{
124 // Collect all items that need to be available in this codegen unit.
125 let mut reachable
= FxHashSet
::default();
126 for root
in old_codegen_unit
.items().keys() {
127 follow_inlining(*root
, inlining_map
, &mut reachable
);
130 let mut new_codegen_unit
= CodegenUnit
::new(old_codegen_unit
.name());
132 // Add all monomorphizations that are not already there.
133 for mono_item
in reachable
{
134 if let Some(linkage
) = old_codegen_unit
.items().get(&mono_item
) {
135 // This is a root, just copy it over.
136 new_codegen_unit
.items_mut().insert(mono_item
, *linkage
);
138 if roots
.contains(&mono_item
) {
140 "GloballyShared mono-item inlined into other CGU: \
146 // This is a CGU-private copy.
149 .insert(mono_item
, (Linkage
::Internal
, Visibility
::Default
));
152 if !single_codegen_unit
{
153 // If there is more than one codegen unit, we need to keep track
154 // in which codegen units each monomorphization is placed.
155 match mono_item_placements
.entry(mono_item
) {
156 Entry
::Occupied(e
) => {
157 let placement
= e
.into_mut();
158 debug_assert
!(match *placement
{
159 MonoItemPlacement
::SingleCgu { cgu_name }
=> {
160 cgu_name
!= new_codegen_unit
.name()
162 MonoItemPlacement
::MultipleCgus
=> true,
164 *placement
= MonoItemPlacement
::MultipleCgus
;
166 Entry
::Vacant(e
) => {
167 e
.insert(MonoItemPlacement
::SingleCgu
{
168 cgu_name
: new_codegen_unit
.name(),
175 new_partitioning
.push(new_codegen_unit
);
178 return PostInliningPartitioning
{
179 codegen_units
: new_partitioning
,
180 mono_item_placements
,
181 internalization_candidates
,
184 fn follow_inlining
<'tcx
>(
185 mono_item
: MonoItem
<'tcx
>,
186 inlining_map
: &InliningMap
<'tcx
>,
187 visited
: &mut FxHashSet
<MonoItem
<'tcx
>>,
189 if !visited
.insert(mono_item
) {
193 inlining_map
.with_inlining_candidates(mono_item
, |target
| {
194 follow_inlining(target
, inlining_map
, visited
);
199 fn internalize_symbols(
202 partitioning
: &mut PostInliningPartitioning
<'tcx
>,
203 inlining_map
: &InliningMap
<'tcx
>,
205 if partitioning
.codegen_units
.len() == 1 {
206 // Fast path for when there is only one codegen unit. In this case we
207 // can internalize all candidates, since there is nowhere else they
208 // could be accessed from.
209 for cgu
in &mut partitioning
.codegen_units
{
210 for candidate
in &partitioning
.internalization_candidates
{
211 cgu
.items_mut().insert(*candidate
, (Linkage
::Internal
, Visibility
::Default
));
218 // Build a map from every monomorphization to all the monomorphizations that
220 let mut accessor_map
: FxHashMap
<MonoItem
<'tcx
>, Vec
<MonoItem
<'tcx
>>> = Default
::default();
221 inlining_map
.iter_accesses(|accessor
, accessees
| {
222 for accessee
in accessees
{
223 accessor_map
.entry(*accessee
).or_default().push(accessor
);
227 let mono_item_placements
= &partitioning
.mono_item_placements
;
229 // For each internalization candidates in each codegen unit, check if it is
230 // accessed from outside its defining codegen unit.
231 for cgu
in &mut partitioning
.codegen_units
{
232 let home_cgu
= MonoItemPlacement
::SingleCgu { cgu_name: cgu.name() }
;
234 for (accessee
, linkage_and_visibility
) in cgu
.items_mut() {
235 if !partitioning
.internalization_candidates
.contains(accessee
) {
236 // This item is no candidate for internalizing, so skip it.
239 debug_assert_eq
!(mono_item_placements
[accessee
], home_cgu
);
241 if let Some(accessors
) = accessor_map
.get(accessee
) {
244 .filter_map(|accessor
| {
245 // Some accessors might not have been
246 // instantiated. We can safely ignore those.
247 mono_item_placements
.get(accessor
)
249 .any(|placement
| *placement
!= home_cgu
)
251 // Found an accessor from another CGU, so skip to the next
252 // item without marking this one as internal.
257 // If we got here, we did not find any accesses from other CGUs,
258 // so it's fine to make this monomorphization internal.
259 *linkage_and_visibility
= (Linkage
::Internal
, Visibility
::Default
);
265 fn characteristic_def_id_of_mono_item
<'tcx
>(
267 mono_item
: MonoItem
<'tcx
>,
270 MonoItem
::Fn(instance
) => {
271 let def_id
= match instance
.def
{
272 ty
::InstanceDef
::Item(def
) => def
.did
,
273 ty
::InstanceDef
::VtableShim(..)
274 | ty
::InstanceDef
::ReifyShim(..)
275 | ty
::InstanceDef
::FnPtrShim(..)
276 | ty
::InstanceDef
::ClosureOnceShim { .. }
277 | ty
::InstanceDef
::Intrinsic(..)
278 | ty
::InstanceDef
::DropGlue(..)
279 | ty
::InstanceDef
::Virtual(..)
280 | ty
::InstanceDef
::CloneShim(..) => return None
,
283 // If this is a method, we want to put it into the same module as
284 // its self-type. If the self-type does not provide a characteristic
285 // DefId, we use the location of the impl after all.
287 if tcx
.trait_of_item(def_id
).is_some() {
288 let self_ty
= instance
.substs
.type_at(0);
289 // This is a default implementation of a trait method.
290 return characteristic_def_id_of_type(self_ty
).or(Some(def_id
));
293 if let Some(impl_def_id
) = tcx
.impl_of_method(def_id
) {
294 if tcx
.sess
.opts
.incremental
.is_some()
295 && tcx
.trait_id_of_impl(impl_def_id
) == tcx
.lang_items().drop_trait()
297 // Put `Drop::drop` into the same cgu as `drop_in_place`
298 // since `drop_in_place` is the only thing that can
302 // This is a method within an impl, find out what the self-type is:
303 let impl_self_ty
= tcx
.subst_and_normalize_erasing_regions(
305 ty
::ParamEnv
::reveal_all(),
306 &tcx
.type_of(impl_def_id
),
308 if let Some(def_id
) = characteristic_def_id_of_type(impl_self_ty
) {
315 MonoItem
::Static(def_id
) => Some(def_id
),
316 MonoItem
::GlobalAsm(hir_id
) => Some(tcx
.hir().local_def_id(hir_id
).to_def_id()),
320 fn compute_codegen_unit_name(
322 name_builder
: &mut CodegenUnitNameBuilder
<'_
>,
325 cache
: &mut CguNameCache
,
327 // Find the innermost module that is not nested within a function.
328 let mut current_def_id
= def_id
;
329 let mut cgu_def_id
= None
;
330 // Walk backwards from the item we want to find the module for.
332 if current_def_id
.index
== CRATE_DEF_INDEX
{
333 if cgu_def_id
.is_none() {
334 // If we have not found a module yet, take the crate root.
335 cgu_def_id
= Some(DefId { krate: def_id.krate, index: CRATE_DEF_INDEX }
);
338 } else if tcx
.def_kind(current_def_id
) == DefKind
::Mod
{
339 if cgu_def_id
.is_none() {
340 cgu_def_id
= Some(current_def_id
);
343 // If we encounter something that is not a module, throw away
344 // any module that we've found so far because we now know that
345 // it is nested within something else.
349 current_def_id
= tcx
.parent(current_def_id
).unwrap();
352 let cgu_def_id
= cgu_def_id
.unwrap();
354 *cache
.entry((cgu_def_id
, volatile
)).or_insert_with(|| {
355 let def_path
= tcx
.def_path(cgu_def_id
);
357 let components
= def_path
.data
.iter().map(|part
| part
.data
.as_symbol());
359 let volatile_suffix
= volatile
.then_some("volatile");
361 name_builder
.build_cgu_name(def_path
.krate
, components
, volatile_suffix
)
365 // Anything we can't find a proper codegen unit for goes into this.
366 fn fallback_cgu_name(name_builder
: &mut CodegenUnitNameBuilder
<'_
>) -> Symbol
{
367 name_builder
.build_cgu_name(LOCAL_CRATE
, &["fallback"], Some("cgu"))
370 fn mono_item_linkage_and_visibility(
372 mono_item
: &MonoItem
<'tcx
>,
373 can_be_internalized
: &mut bool
,
374 export_generics
: bool
,
375 ) -> (Linkage
, Visibility
) {
376 if let Some(explicit_linkage
) = mono_item
.explicit_linkage(tcx
) {
377 return (explicit_linkage
, Visibility
::Default
);
379 let vis
= mono_item_visibility(tcx
, mono_item
, can_be_internalized
, export_generics
);
380 (Linkage
::External
, vis
)
383 type CguNameCache
= FxHashMap
<(DefId
, bool
), Symbol
>;
385 fn mono_item_visibility(
387 mono_item
: &MonoItem
<'tcx
>,
388 can_be_internalized
: &mut bool
,
389 export_generics
: bool
,
391 let instance
= match mono_item
{
392 // This is pretty complicated; see below.
393 MonoItem
::Fn(instance
) => instance
,
395 // Misc handling for generics and such, but otherwise:
396 MonoItem
::Static(def_id
) => {
397 return if tcx
.is_reachable_non_generic(*def_id
) {
398 *can_be_internalized
= false;
399 default_visibility(tcx
, *def_id
, false)
404 MonoItem
::GlobalAsm(hir_id
) => {
405 let def_id
= tcx
.hir().local_def_id(*hir_id
);
406 return if tcx
.is_reachable_non_generic(def_id
) {
407 *can_be_internalized
= false;
408 default_visibility(tcx
, def_id
.to_def_id(), false)
415 let def_id
= match instance
.def
{
416 InstanceDef
::Item(def
) => def
.did
,
417 InstanceDef
::DropGlue(def_id
, Some(_
)) => def_id
,
419 // These are all compiler glue and such, never exported, always hidden.
420 InstanceDef
::VtableShim(..)
421 | InstanceDef
::ReifyShim(..)
422 | InstanceDef
::FnPtrShim(..)
423 | InstanceDef
::Virtual(..)
424 | InstanceDef
::Intrinsic(..)
425 | InstanceDef
::ClosureOnceShim { .. }
426 | InstanceDef
::DropGlue(..)
427 | InstanceDef
::CloneShim(..) => return Visibility
::Hidden
,
430 // The `start_fn` lang item is actually a monomorphized instance of a
431 // function in the standard library, used for the `main` function. We don't
432 // want to export it so we tag it with `Hidden` visibility but this symbol
433 // is only referenced from the actual `main` symbol which we unfortunately
434 // don't know anything about during partitioning/collection. As a result we
435 // forcibly keep this symbol out of the `internalization_candidates` set.
437 // FIXME: eventually we don't want to always force this symbol to have
438 // hidden visibility, it should indeed be a candidate for
439 // internalization, but we have to understand that it's referenced
440 // from the `main` symbol we'll generate later.
442 // This may be fixable with a new `InstanceDef` perhaps? Unsure!
443 if tcx
.lang_items().start_fn() == Some(def_id
) {
444 *can_be_internalized
= false;
445 return Visibility
::Hidden
;
448 let is_generic
= instance
.substs
.non_erasable_generics().next().is_some();
450 // Upstream `DefId` instances get different handling than local ones.
451 if !def_id
.is_local() {
452 return if export_generics
&& is_generic
{
453 // If it is a upstream monomorphization and we export generics, we must make
454 // it available to downstream crates.
455 *can_be_internalized
= false;
456 default_visibility(tcx
, def_id
, true)
464 if tcx
.is_unreachable_local_definition(def_id
) {
465 // This instance cannot be used from another crate.
468 // This instance might be useful in a downstream crate.
469 *can_be_internalized
= false;
470 default_visibility(tcx
, def_id
, true)
473 // We are not exporting generics or the definition is not reachable
474 // for downstream crates, we can internalize its instantiations.
478 // If this isn't a generic function then we mark this a `Default` if
479 // this is a reachable item, meaning that it's a symbol other crates may
480 // access when they link to us.
481 if tcx
.is_reachable_non_generic(def_id
) {
482 *can_be_internalized
= false;
483 debug_assert
!(!is_generic
);
484 return default_visibility(tcx
, def_id
, false);
487 // If this isn't reachable then we're gonna tag this with `Hidden`
488 // visibility. In some situations though we'll want to prevent this
489 // symbol from being internalized.
491 // There's two categories of items here:
493 // * First is weak lang items. These are basically mechanisms for
494 // libcore to forward-reference symbols defined later in crates like
495 // the standard library or `#[panic_handler]` definitions. The
496 // definition of these weak lang items needs to be referenceable by
497 // libcore, so we're no longer a candidate for internalization.
498 // Removal of these functions can't be done by LLVM but rather must be
499 // done by the linker as it's a non-local decision.
501 // * Second is "std internal symbols". Currently this is primarily used
502 // for allocator symbols. Allocators are a little weird in their
503 // implementation, but the idea is that the compiler, at the last
504 // minute, defines an allocator with an injected object file. The
505 // `alloc` crate references these symbols (`__rust_alloc`) and the
506 // definition doesn't get hooked up until a linked crate artifact is
509 // The symbols synthesized by the compiler (`__rust_alloc`) are thin
510 // veneers around the actual implementation, some other symbol which
511 // implements the same ABI. These symbols (things like `__rg_alloc`,
512 // `__rdl_alloc`, `__rde_alloc`, etc), are all tagged with "std
513 // internal symbols".
515 // The std-internal symbols here **should not show up in a dll as an
516 // exported interface**, so they return `false` from
517 // `is_reachable_non_generic` above and we'll give them `Hidden`
518 // visibility below. Like the weak lang items, though, we can't let
519 // LLVM internalize them as this decision is left up to the linker to
520 // omit them, so prevent them from being internalized.
521 let attrs
= tcx
.codegen_fn_attrs(def_id
);
522 if attrs
.flags
.contains(CodegenFnAttrFlags
::RUSTC_STD_INTERNAL_SYMBOL
) {
523 *can_be_internalized
= false;
530 fn default_visibility(tcx
: TyCtxt
<'_
>, id
: DefId
, is_generic
: bool
) -> Visibility
{
531 if !tcx
.sess
.target
.target
.options
.default_hidden_visibility
{
532 return Visibility
::Default
;
535 // Generic functions never have export-level C.
537 return Visibility
::Hidden
;
540 // Things with export level C don't get instantiated in
541 // downstream crates.
543 return Visibility
::Hidden
;
546 // C-export level items remain at `Default`, all other internal
547 // items become `Hidden`.
548 match tcx
.reachable_non_generics(id
.krate
).get(&id
) {
549 Some(SymbolExportLevel
::C
) => Visibility
::Default
,
550 _
=> Visibility
::Hidden
,