1 use std
::collections
::hash_map
::Entry
;
3 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
4 use rustc_hir
::def
::DefKind
;
5 use rustc_hir
::def_id
::{DefId, CRATE_DEF_INDEX, LOCAL_CRATE}
;
6 use rustc_hir
::definitions
::DefPathDataName
;
7 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
8 use rustc_middle
::middle
::exported_symbols
::SymbolExportLevel
;
9 use rustc_middle
::mir
::mono
::{CodegenUnit, CodegenUnitNameBuilder, Linkage, Visibility}
;
10 use rustc_middle
::mir
::mono
::{InstantiationMode, MonoItem}
;
11 use rustc_middle
::ty
::print
::characteristic_def_id_of_type
;
12 use rustc_middle
::ty
::{self, fold::TypeFoldable, DefIdTree, InstanceDef, TyCtxt}
;
13 use rustc_span
::symbol
::Symbol
;
15 use super::PartitioningCx
;
16 use crate::collector
::InliningMap
;
17 use crate::partitioning
::merging
;
18 use crate::partitioning
::{
19 MonoItemPlacement
, Partitioner
, PostInliningPartitioning
, PreInliningPartitioning
,
22 pub struct DefaultPartitioning
;
24 impl<'tcx
> Partitioner
<'tcx
> for DefaultPartitioning
{
25 fn place_root_mono_items(
27 cx
: &PartitioningCx
<'_
, 'tcx
>,
28 mono_items
: &mut dyn Iterator
<Item
= MonoItem
<'tcx
>>,
29 ) -> PreInliningPartitioning
<'tcx
> {
30 let mut roots
= FxHashSet
::default();
31 let mut codegen_units
= FxHashMap
::default();
32 let is_incremental_build
= cx
.tcx
.sess
.opts
.incremental
.is_some();
33 let mut internalization_candidates
= FxHashSet
::default();
35 // Determine if monomorphizations instantiated in this crate will be made
36 // available to downstream crates. This depends on whether we are in
37 // share-generics mode and whether the current crate can even have
40 cx
.tcx
.sess
.opts
.share_generics() && cx
.tcx
.local_crate_exports_generics();
42 let cgu_name_builder
= &mut CodegenUnitNameBuilder
::new(cx
.tcx
);
43 let cgu_name_cache
= &mut FxHashMap
::default();
45 for mono_item
in mono_items
{
46 match mono_item
.instantiation_mode(cx
.tcx
) {
47 InstantiationMode
::GloballyShared { .. }
=> {}
48 InstantiationMode
::LocalCopy
=> continue,
51 let characteristic_def_id
= characteristic_def_id_of_mono_item(cx
.tcx
, mono_item
);
52 let is_volatile
= is_incremental_build
&& mono_item
.is_generic_fn();
54 let codegen_unit_name
= match characteristic_def_id
{
55 Some(def_id
) => compute_codegen_unit_name(
62 None
=> fallback_cgu_name(cgu_name_builder
),
65 let codegen_unit
= codegen_units
66 .entry(codegen_unit_name
)
67 .or_insert_with(|| CodegenUnit
::new(codegen_unit_name
));
69 let mut can_be_internalized
= true;
70 let (linkage
, visibility
) = mono_item_linkage_and_visibility(
73 &mut can_be_internalized
,
76 if visibility
== Visibility
::Hidden
&& can_be_internalized
{
77 internalization_candidates
.insert(mono_item
);
80 codegen_unit
.items_mut().insert(mono_item
, (linkage
, visibility
));
81 roots
.insert(mono_item
);
84 // Always ensure we have at least one CGU; otherwise, if we have a
85 // crate with just types (for example), we could wind up with no CGU.
86 if codegen_units
.is_empty() {
87 let codegen_unit_name
= fallback_cgu_name(cgu_name_builder
);
88 codegen_units
.insert(codegen_unit_name
, CodegenUnit
::new(codegen_unit_name
));
91 PreInliningPartitioning
{
92 codegen_units
: codegen_units
94 .map(|(_
, codegen_unit
)| codegen_unit
)
97 internalization_candidates
,
101 fn merge_codegen_units(
103 cx
: &PartitioningCx
<'_
, 'tcx
>,
104 initial_partitioning
: &mut PreInliningPartitioning
<'tcx
>,
106 merging
::merge_codegen_units(cx
, initial_partitioning
);
109 fn place_inlined_mono_items(
111 cx
: &PartitioningCx
<'_
, 'tcx
>,
112 initial_partitioning
: PreInliningPartitioning
<'tcx
>,
113 ) -> PostInliningPartitioning
<'tcx
> {
114 let mut new_partitioning
= Vec
::new();
115 let mut mono_item_placements
= FxHashMap
::default();
117 let PreInliningPartitioning
{
118 codegen_units
: initial_cgus
,
120 internalization_candidates
,
121 } = initial_partitioning
;
123 let single_codegen_unit
= initial_cgus
.len() == 1;
125 for old_codegen_unit
in initial_cgus
{
126 // Collect all items that need to be available in this codegen unit.
127 let mut reachable
= FxHashSet
::default();
128 for root
in old_codegen_unit
.items().keys() {
129 follow_inlining(*root
, cx
.inlining_map
, &mut reachable
);
132 let mut new_codegen_unit
= CodegenUnit
::new(old_codegen_unit
.name());
134 // Add all monomorphizations that are not already there.
135 for mono_item
in reachable
{
136 if let Some(linkage
) = old_codegen_unit
.items().get(&mono_item
) {
137 // This is a root, just copy it over.
138 new_codegen_unit
.items_mut().insert(mono_item
, *linkage
);
140 if roots
.contains(&mono_item
) {
142 "GloballyShared mono-item inlined into other CGU: \
148 // This is a CGU-private copy.
151 .insert(mono_item
, (Linkage
::Internal
, Visibility
::Default
));
154 if !single_codegen_unit
{
155 // If there is more than one codegen unit, we need to keep track
156 // in which codegen units each monomorphization is placed.
157 match mono_item_placements
.entry(mono_item
) {
158 Entry
::Occupied(e
) => {
159 let placement
= e
.into_mut();
160 debug_assert
!(match *placement
{
161 MonoItemPlacement
::SingleCgu { cgu_name }
=> {
162 cgu_name
!= new_codegen_unit
.name()
164 MonoItemPlacement
::MultipleCgus
=> true,
166 *placement
= MonoItemPlacement
::MultipleCgus
;
168 Entry
::Vacant(e
) => {
169 e
.insert(MonoItemPlacement
::SingleCgu
{
170 cgu_name
: new_codegen_unit
.name(),
177 new_partitioning
.push(new_codegen_unit
);
180 return PostInliningPartitioning
{
181 codegen_units
: new_partitioning
,
182 mono_item_placements
,
183 internalization_candidates
,
186 fn follow_inlining
<'tcx
>(
187 mono_item
: MonoItem
<'tcx
>,
188 inlining_map
: &InliningMap
<'tcx
>,
189 visited
: &mut FxHashSet
<MonoItem
<'tcx
>>,
191 if !visited
.insert(mono_item
) {
195 inlining_map
.with_inlining_candidates(mono_item
, |target
| {
196 follow_inlining(target
, inlining_map
, visited
);
201 fn internalize_symbols(
203 cx
: &PartitioningCx
<'_
, 'tcx
>,
204 partitioning
: &mut PostInliningPartitioning
<'tcx
>,
206 if partitioning
.codegen_units
.len() == 1 {
207 // Fast path for when there is only one codegen unit. In this case we
208 // can internalize all candidates, since there is nowhere else they
209 // could be accessed from.
210 for cgu
in &mut partitioning
.codegen_units
{
211 for candidate
in &partitioning
.internalization_candidates
{
212 cgu
.items_mut().insert(*candidate
, (Linkage
::Internal
, Visibility
::Default
));
219 // Build a map from every monomorphization to all the monomorphizations that
221 let mut accessor_map
: FxHashMap
<MonoItem
<'tcx
>, Vec
<MonoItem
<'tcx
>>> = Default
::default();
222 cx
.inlining_map
.iter_accesses(|accessor
, accessees
| {
223 for accessee
in accessees
{
224 accessor_map
.entry(*accessee
).or_default().push(accessor
);
228 let mono_item_placements
= &partitioning
.mono_item_placements
;
230 // For each internalization candidates in each codegen unit, check if it is
231 // accessed from outside its defining codegen unit.
232 for cgu
in &mut partitioning
.codegen_units
{
233 let home_cgu
= MonoItemPlacement
::SingleCgu { cgu_name: cgu.name() }
;
235 for (accessee
, linkage_and_visibility
) in cgu
.items_mut() {
236 if !partitioning
.internalization_candidates
.contains(accessee
) {
237 // This item is no candidate for internalizing, so skip it.
240 debug_assert_eq
!(mono_item_placements
[accessee
], home_cgu
);
242 if let Some(accessors
) = accessor_map
.get(accessee
) {
245 .filter_map(|accessor
| {
246 // Some accessors might not have been
247 // instantiated. We can safely ignore those.
248 mono_item_placements
.get(accessor
)
250 .any(|placement
| *placement
!= home_cgu
)
252 // Found an accessor from another CGU, so skip to the next
253 // item without marking this one as internal.
258 // If we got here, we did not find any accesses from other CGUs,
259 // so it's fine to make this monomorphization internal.
260 *linkage_and_visibility
= (Linkage
::Internal
, Visibility
::Default
);
266 fn characteristic_def_id_of_mono_item
<'tcx
>(
268 mono_item
: MonoItem
<'tcx
>,
271 MonoItem
::Fn(instance
) => {
272 let def_id
= match instance
.def
{
273 ty
::InstanceDef
::Item(def
) => def
.did
,
274 ty
::InstanceDef
::VtableShim(..)
275 | ty
::InstanceDef
::ReifyShim(..)
276 | ty
::InstanceDef
::FnPtrShim(..)
277 | ty
::InstanceDef
::ClosureOnceShim { .. }
278 | ty
::InstanceDef
::Intrinsic(..)
279 | ty
::InstanceDef
::DropGlue(..)
280 | ty
::InstanceDef
::Virtual(..)
281 | ty
::InstanceDef
::CloneShim(..) => return None
,
284 // If this is a method, we want to put it into the same module as
285 // its self-type. If the self-type does not provide a characteristic
286 // DefId, we use the location of the impl after all.
288 if tcx
.trait_of_item(def_id
).is_some() {
289 let self_ty
= instance
.substs
.type_at(0);
290 // This is a default implementation of a trait method.
291 return characteristic_def_id_of_type(self_ty
).or(Some(def_id
));
294 if let Some(impl_def_id
) = tcx
.impl_of_method(def_id
) {
295 if tcx
.sess
.opts
.incremental
.is_some()
296 && tcx
.trait_id_of_impl(impl_def_id
) == tcx
.lang_items().drop_trait()
298 // Put `Drop::drop` into the same cgu as `drop_in_place`
299 // since `drop_in_place` is the only thing that can
304 // When polymorphization is enabled, methods which do not depend on their generic
305 // parameters, but the self-type of their impl block do will fail to normalize.
306 if !tcx
.sess
.opts
.debugging_opts
.polymorphize
|| !instance
.needs_subst() {
307 // This is a method within an impl, find out what the self-type is:
308 let impl_self_ty
= tcx
.subst_and_normalize_erasing_regions(
310 ty
::ParamEnv
::reveal_all(),
311 tcx
.type_of(impl_def_id
),
313 if let Some(def_id
) = characteristic_def_id_of_type(impl_self_ty
) {
321 MonoItem
::Static(def_id
) => Some(def_id
),
322 MonoItem
::GlobalAsm(item_id
) => Some(item_id
.def_id
.to_def_id()),
326 fn compute_codegen_unit_name(
328 name_builder
: &mut CodegenUnitNameBuilder
<'_
>,
331 cache
: &mut CguNameCache
,
333 // Find the innermost module that is not nested within a function.
334 let mut current_def_id
= def_id
;
335 let mut cgu_def_id
= None
;
336 // Walk backwards from the item we want to find the module for.
338 if current_def_id
.index
== CRATE_DEF_INDEX
{
339 if cgu_def_id
.is_none() {
340 // If we have not found a module yet, take the crate root.
341 cgu_def_id
= Some(DefId { krate: def_id.krate, index: CRATE_DEF_INDEX }
);
344 } else if tcx
.def_kind(current_def_id
) == DefKind
::Mod
{
345 if cgu_def_id
.is_none() {
346 cgu_def_id
= Some(current_def_id
);
349 // If we encounter something that is not a module, throw away
350 // any module that we've found so far because we now know that
351 // it is nested within something else.
355 current_def_id
= tcx
.parent(current_def_id
).unwrap();
358 let cgu_def_id
= cgu_def_id
.unwrap();
360 *cache
.entry((cgu_def_id
, volatile
)).or_insert_with(|| {
361 let def_path
= tcx
.def_path(cgu_def_id
);
363 let components
= def_path
.data
.iter().map(|part
| match part
.data
.name() {
364 DefPathDataName
::Named(name
) => name
,
365 DefPathDataName
::Anon { .. }
=> unreachable
!(),
368 let volatile_suffix
= volatile
.then_some("volatile");
370 name_builder
.build_cgu_name(def_path
.krate
, components
, volatile_suffix
)
374 // Anything we can't find a proper codegen unit for goes into this.
375 fn fallback_cgu_name(name_builder
: &mut CodegenUnitNameBuilder
<'_
>) -> Symbol
{
376 name_builder
.build_cgu_name(LOCAL_CRATE
, &["fallback"], Some("cgu"))
379 fn mono_item_linkage_and_visibility
<'tcx
>(
381 mono_item
: &MonoItem
<'tcx
>,
382 can_be_internalized
: &mut bool
,
383 export_generics
: bool
,
384 ) -> (Linkage
, Visibility
) {
385 if let Some(explicit_linkage
) = mono_item
.explicit_linkage(tcx
) {
386 return (explicit_linkage
, Visibility
::Default
);
388 let vis
= mono_item_visibility(tcx
, mono_item
, can_be_internalized
, export_generics
);
389 (Linkage
::External
, vis
)
392 type CguNameCache
= FxHashMap
<(DefId
, bool
), Symbol
>;
394 fn mono_item_visibility
<'tcx
>(
396 mono_item
: &MonoItem
<'tcx
>,
397 can_be_internalized
: &mut bool
,
398 export_generics
: bool
,
400 let instance
= match mono_item
{
401 // This is pretty complicated; see below.
402 MonoItem
::Fn(instance
) => instance
,
404 // Misc handling for generics and such, but otherwise:
405 MonoItem
::Static(def_id
) => {
406 return if tcx
.is_reachable_non_generic(*def_id
) {
407 *can_be_internalized
= false;
408 default_visibility(tcx
, *def_id
, false)
413 MonoItem
::GlobalAsm(item_id
) => {
414 return if tcx
.is_reachable_non_generic(item_id
.def_id
) {
415 *can_be_internalized
= false;
416 default_visibility(tcx
, item_id
.def_id
.to_def_id(), false)
423 let def_id
= match instance
.def
{
424 InstanceDef
::Item(def
) => def
.did
,
425 InstanceDef
::DropGlue(def_id
, Some(_
)) => def_id
,
427 // These are all compiler glue and such, never exported, always hidden.
428 InstanceDef
::VtableShim(..)
429 | InstanceDef
::ReifyShim(..)
430 | InstanceDef
::FnPtrShim(..)
431 | InstanceDef
::Virtual(..)
432 | InstanceDef
::Intrinsic(..)
433 | InstanceDef
::ClosureOnceShim { .. }
434 | InstanceDef
::DropGlue(..)
435 | InstanceDef
::CloneShim(..) => return Visibility
::Hidden
,
438 // The `start_fn` lang item is actually a monomorphized instance of a
439 // function in the standard library, used for the `main` function. We don't
440 // want to export it so we tag it with `Hidden` visibility but this symbol
441 // is only referenced from the actual `main` symbol which we unfortunately
442 // don't know anything about during partitioning/collection. As a result we
443 // forcibly keep this symbol out of the `internalization_candidates` set.
445 // FIXME: eventually we don't want to always force this symbol to have
446 // hidden visibility, it should indeed be a candidate for
447 // internalization, but we have to understand that it's referenced
448 // from the `main` symbol we'll generate later.
450 // This may be fixable with a new `InstanceDef` perhaps? Unsure!
451 if tcx
.lang_items().start_fn() == Some(def_id
) {
452 *can_be_internalized
= false;
453 return Visibility
::Hidden
;
456 let is_generic
= instance
.substs
.non_erasable_generics().next().is_some();
458 // Upstream `DefId` instances get different handling than local ones.
459 let Some(def_id
) = def_id
.as_local() else {
460 return if export_generics
&& is_generic
{
461 // If it is an upstream monomorphization and we export generics, we must make
462 // it available to downstream crates.
463 *can_be_internalized
= false;
464 default_visibility(tcx
, def_id
, true)
472 if tcx
.is_unreachable_local_definition(def_id
) {
473 // This instance cannot be used from another crate.
476 // This instance might be useful in a downstream crate.
477 *can_be_internalized
= false;
478 default_visibility(tcx
, def_id
.to_def_id(), true)
481 // We are not exporting generics or the definition is not reachable
482 // for downstream crates, we can internalize its instantiations.
486 // If this isn't a generic function then we mark this a `Default` if
487 // this is a reachable item, meaning that it's a symbol other crates may
488 // access when they link to us.
489 if tcx
.is_reachable_non_generic(def_id
.to_def_id()) {
490 *can_be_internalized
= false;
491 debug_assert
!(!is_generic
);
492 return default_visibility(tcx
, def_id
.to_def_id(), false);
495 // If this isn't reachable then we're gonna tag this with `Hidden`
496 // visibility. In some situations though we'll want to prevent this
497 // symbol from being internalized.
499 // There's two categories of items here:
501 // * First is weak lang items. These are basically mechanisms for
502 // libcore to forward-reference symbols defined later in crates like
503 // the standard library or `#[panic_handler]` definitions. The
504 // definition of these weak lang items needs to be referencable by
505 // libcore, so we're no longer a candidate for internalization.
506 // Removal of these functions can't be done by LLVM but rather must be
507 // done by the linker as it's a non-local decision.
509 // * Second is "std internal symbols". Currently this is primarily used
510 // for allocator symbols. Allocators are a little weird in their
511 // implementation, but the idea is that the compiler, at the last
512 // minute, defines an allocator with an injected object file. The
513 // `alloc` crate references these symbols (`__rust_alloc`) and the
514 // definition doesn't get hooked up until a linked crate artifact is
517 // The symbols synthesized by the compiler (`__rust_alloc`) are thin
518 // veneers around the actual implementation, some other symbol which
519 // implements the same ABI. These symbols (things like `__rg_alloc`,
520 // `__rdl_alloc`, `__rde_alloc`, etc), are all tagged with "std
521 // internal symbols".
523 // The std-internal symbols here **should not show up in a dll as an
524 // exported interface**, so they return `false` from
525 // `is_reachable_non_generic` above and we'll give them `Hidden`
526 // visibility below. Like the weak lang items, though, we can't let
527 // LLVM internalize them as this decision is left up to the linker to
528 // omit them, so prevent them from being internalized.
529 let attrs
= tcx
.codegen_fn_attrs(def_id
);
530 if attrs
.flags
.contains(CodegenFnAttrFlags
::RUSTC_STD_INTERNAL_SYMBOL
) {
531 *can_be_internalized
= false;
538 fn default_visibility(tcx
: TyCtxt
<'_
>, id
: DefId
, is_generic
: bool
) -> Visibility
{
539 if !tcx
.sess
.target
.default_hidden_visibility
{
540 return Visibility
::Default
;
543 // Generic functions never have export-level C.
545 return Visibility
::Hidden
;
548 // Things with export level C don't get instantiated in
549 // downstream crates.
551 return Visibility
::Hidden
;
554 // C-export level items remain at `Default`, all other internal
555 // items become `Hidden`.
556 match tcx
.reachable_non_generics(id
.krate
).get(&id
) {
557 Some(SymbolExportLevel
::C
) => Visibility
::Default
,
558 _
=> Visibility
::Hidden
,