1 //! Codegen the completed AST to the LLVM IR.
3 //! Some functions here, such as `codegen_block` and `codegen_expr`, return a value --
4 //! the result of the codegen to LLVM -- while others, such as `codegen_fn`
5 //! and `mono_item`, are called only for the side effect of adding a
6 //! particular definition to the LLVM IR output we're producing.
8 //! Hopefully useful general knowledge about codegen:
10 //! * There's no way to find out the `Ty` type of a `Value`. Doing so
11 //! would be "trying to get the eggs out of an omelette" (credit:
12 //! pcwalton). You can, instead, find out its `llvm::Type` by calling `val_ty`,
13 //! but one `llvm::Type` corresponds to many `Ty`s; for instance, `tup(int, int,
14 //! int)` and `rec(x=int, y=int, z=int)` will have the same `llvm::Type`.
16 use crate::back
::write
::{
17 start_async_codegen
, submit_codegened_module_to_llvm
, submit_post_lto_module_to_llvm
,
18 submit_pre_lto_module_to_llvm
, OngoingCodegen
,
20 use crate::common
::{IntPredicate, RealPredicate, TypeKind}
;
23 use crate::mir
::operand
::OperandValue
;
24 use crate::mir
::place
::PlaceRef
;
26 use crate::{CachedModuleCodegen, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}
;
28 use rustc
::middle
::codegen_fn_attrs
::CodegenFnAttrs
;
29 use rustc
::middle
::cstore
::EncodedMetadata
;
30 use rustc
::middle
::cstore
::{self, LinkagePreference}
;
31 use rustc
::middle
::lang_items
::StartFnLangItem
;
32 use rustc
::middle
::weak_lang_items
;
33 use rustc
::mir
::mono
::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}
;
34 use rustc
::session
::config
::{self, EntryFnType, Lto}
;
35 use rustc
::session
::Session
;
36 use rustc
::ty
::layout
::{self, Align, HasTyCtxt, LayoutOf, TyLayout, VariantIdx}
;
37 use rustc
::ty
::layout
::{FAT_PTR_ADDR, FAT_PTR_EXTRA}
;
38 use rustc
::ty
::query
::Providers
;
39 use rustc
::ty
::{self, Instance, Ty, TyCtxt}
;
40 use rustc_codegen_utils
::{check_for_rustc_errors_attr, symbol_names_test}
;
41 use rustc_data_structures
::fx
::FxHashMap
;
42 use rustc_data_structures
::profiling
::print_time_passes_entry
;
43 use rustc_data_structures
::sync
::{par_iter, Lock, ParallelIterator}
;
45 use rustc_hir
::def_id
::{DefId, LOCAL_CRATE}
;
46 use rustc_index
::vec
::Idx
;
47 use rustc_session
::cgu_reuse_tracker
::CguReuse
;
52 use std
::ops
::{Deref, DerefMut}
;
53 use std
::time
::{Duration, Instant}
;
55 pub fn bin_op_to_icmp_predicate(op
: hir
::BinOpKind
, signed
: bool
) -> IntPredicate
{
57 hir
::BinOpKind
::Eq
=> IntPredicate
::IntEQ
,
58 hir
::BinOpKind
::Ne
=> IntPredicate
::IntNE
,
59 hir
::BinOpKind
::Lt
=> {
66 hir
::BinOpKind
::Le
=> {
73 hir
::BinOpKind
::Gt
=> {
80 hir
::BinOpKind
::Ge
=> {
88 "comparison_op_to_icmp_predicate: expected comparison operator, \
95 pub fn bin_op_to_fcmp_predicate(op
: hir
::BinOpKind
) -> RealPredicate
{
97 hir
::BinOpKind
::Eq
=> RealPredicate
::RealOEQ
,
98 hir
::BinOpKind
::Ne
=> RealPredicate
::RealUNE
,
99 hir
::BinOpKind
::Lt
=> RealPredicate
::RealOLT
,
100 hir
::BinOpKind
::Le
=> RealPredicate
::RealOLE
,
101 hir
::BinOpKind
::Gt
=> RealPredicate
::RealOGT
,
102 hir
::BinOpKind
::Ge
=> RealPredicate
::RealOGE
,
105 "comparison_op_to_fcmp_predicate: expected comparison operator, \
113 pub fn compare_simd_types
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
121 let signed
= match t
.kind
{
123 let cmp
= bin_op_to_fcmp_predicate(op
);
124 let cmp
= bx
.fcmp(cmp
, lhs
, rhs
);
125 return bx
.sext(cmp
, ret_ty
);
127 ty
::Uint(_
) => false,
129 _
=> bug
!("compare_simd_types: invalid SIMD type"),
132 let cmp
= bin_op_to_icmp_predicate(op
, signed
);
133 let cmp
= bx
.icmp(cmp
, lhs
, rhs
);
134 // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension
135 // to get the correctly sized type. This will compile to a single instruction
136 // once the IR is converted to assembly if the SIMD instruction is supported
137 // by the target architecture.
141 /// Retrieves the information we are losing (making dynamic) in an unsizing
144 /// The `old_info` argument is a bit odd. It is intended for use in an upcast,
145 /// where the new vtable for an object will be derived from the old one.
146 pub fn unsized_info
<'tcx
, Cx
: CodegenMethods
<'tcx
>>(
150 old_info
: Option
<Cx
::Value
>,
152 let (source
, target
) =
153 cx
.tcx().struct_lockstep_tails_erasing_lifetimes(source
, target
, cx
.param_env());
154 match (&source
.kind
, &target
.kind
) {
155 (&ty
::Array(_
, len
), &ty
::Slice(_
)) => {
156 cx
.const_usize(len
.eval_usize(cx
.tcx(), ty
::ParamEnv
::reveal_all()))
158 (&ty
::Dynamic(..), &ty
::Dynamic(..)) => {
159 // For now, upcasts are limited to changes in marker
160 // traits, and hence never actually require an actual
161 // change to the vtable.
162 old_info
.expect("unsized_info: missing old info for trait upcast")
164 (_
, &ty
::Dynamic(ref data
, ..)) => {
165 let vtable_ptr
= cx
.layout_of(cx
.tcx().mk_mut_ptr(target
)).field(cx
, FAT_PTR_EXTRA
);
167 meth
::get_vtable(cx
, source
, data
.principal()),
168 cx
.backend_type(vtable_ptr
),
171 _
=> bug
!("unsized_info: invalid unsizing {:?} -> {:?}", source
, target
),
175 /// Coerces `src` to `dst_ty`. `src_ty` must be a thin pointer.
176 pub fn unsize_thin_ptr
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
181 ) -> (Bx
::Value
, Bx
::Value
) {
182 debug
!("unsize_thin_ptr: {:?} => {:?}", src_ty
, dst_ty
);
183 match (&src_ty
.kind
, &dst_ty
.kind
) {
184 (&ty
::Ref(_
, a
, _
), &ty
::Ref(_
, b
, _
))
185 | (&ty
::Ref(_
, a
, _
), &ty
::RawPtr(ty
::TypeAndMut { ty: b, .. }
))
186 | (&ty
::RawPtr(ty
::TypeAndMut { ty: a, .. }
), &ty
::RawPtr(ty
::TypeAndMut { ty: b, .. }
)) => {
187 assert
!(bx
.cx().type_is_sized(a
));
188 let ptr_ty
= bx
.cx().type_ptr_to(bx
.cx().backend_type(bx
.cx().layout_of(b
)));
189 (bx
.pointercast(src
, ptr_ty
), unsized_info(bx
.cx(), a
, b
, None
))
191 (&ty
::Adt(def_a
, _
), &ty
::Adt(def_b
, _
)) => {
192 assert_eq
!(def_a
, def_b
);
194 let src_layout
= bx
.cx().layout_of(src_ty
);
195 let dst_layout
= bx
.cx().layout_of(dst_ty
);
196 let mut result
= None
;
197 for i
in 0..src_layout
.fields
.count() {
198 let src_f
= src_layout
.field(bx
.cx(), i
);
199 assert_eq
!(src_layout
.fields
.offset(i
).bytes(), 0);
200 assert_eq
!(dst_layout
.fields
.offset(i
).bytes(), 0);
204 assert_eq
!(src_layout
.size
, src_f
.size
);
206 let dst_f
= dst_layout
.field(bx
.cx(), i
);
207 assert_ne
!(src_f
.ty
, dst_f
.ty
);
208 assert_eq
!(result
, None
);
209 result
= Some(unsize_thin_ptr(bx
, src
, src_f
.ty
, dst_f
.ty
));
211 let (lldata
, llextra
) = result
.unwrap();
212 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
213 // FIXME(eddyb) move these out of this `match` arm, so they're always
214 // applied, uniformly, no matter the source/destination types.
216 bx
.bitcast(lldata
, bx
.cx().scalar_pair_element_backend_type(dst_layout
, 0, true)),
217 bx
.bitcast(llextra
, bx
.cx().scalar_pair_element_backend_type(dst_layout
, 1, true)),
220 _
=> bug
!("unsize_thin_ptr: called on bad types"),
224 /// Coerces `src`, which is a reference to a value of type `src_ty`,
225 /// to a value of type `dst_ty`, and stores the result in `dst`.
226 pub fn coerce_unsized_into
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
228 src
: PlaceRef
<'tcx
, Bx
::Value
>,
229 dst
: PlaceRef
<'tcx
, Bx
::Value
>,
231 let src_ty
= src
.layout
.ty
;
232 let dst_ty
= dst
.layout
.ty
;
233 match (&src_ty
.kind
, &dst_ty
.kind
) {
234 (&ty
::Ref(..), &ty
::Ref(..))
235 | (&ty
::Ref(..), &ty
::RawPtr(..))
236 | (&ty
::RawPtr(..), &ty
::RawPtr(..)) => {
237 let (base
, info
) = match bx
.load_operand(src
).val
{
238 OperandValue
::Pair(base
, info
) => {
239 // fat-ptr to fat-ptr unsize preserves the vtable
240 // i.e., &'a fmt::Debug+Send => &'a fmt::Debug
241 // So we need to pointercast the base to ensure
242 // the types match up.
243 // FIXME(eddyb) use `scalar_pair_element_backend_type` here,
244 // like `unsize_thin_ptr` does.
245 let thin_ptr
= dst
.layout
.field(bx
.cx(), FAT_PTR_ADDR
);
246 (bx
.pointercast(base
, bx
.cx().backend_type(thin_ptr
)), info
)
248 OperandValue
::Immediate(base
) => unsize_thin_ptr(bx
, base
, src_ty
, dst_ty
),
249 OperandValue
::Ref(..) => bug
!(),
251 OperandValue
::Pair(base
, info
).store(bx
, dst
);
254 (&ty
::Adt(def_a
, _
), &ty
::Adt(def_b
, _
)) => {
255 assert_eq
!(def_a
, def_b
);
257 for i
in 0..def_a
.variants
[VariantIdx
::new(0)].fields
.len() {
258 let src_f
= src
.project_field(bx
, i
);
259 let dst_f
= dst
.project_field(bx
, i
);
261 if dst_f
.layout
.is_zst() {
265 if src_f
.layout
.ty
== dst_f
.layout
.ty
{
276 coerce_unsized_into(bx
, src_f
, dst_f
);
280 _
=> bug
!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty
, dst_ty
,),
284 pub fn cast_shift_expr_rhs
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
290 cast_shift_rhs(bx
, op
, lhs
, rhs
)
293 fn cast_shift_rhs
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
299 // Shifts may have any size int on the rhs
301 let mut rhs_llty
= bx
.cx().val_ty(rhs
);
302 let mut lhs_llty
= bx
.cx().val_ty(lhs
);
303 if bx
.cx().type_kind(rhs_llty
) == TypeKind
::Vector
{
304 rhs_llty
= bx
.cx().element_type(rhs_llty
)
306 if bx
.cx().type_kind(lhs_llty
) == TypeKind
::Vector
{
307 lhs_llty
= bx
.cx().element_type(lhs_llty
)
309 let rhs_sz
= bx
.cx().int_width(rhs_llty
);
310 let lhs_sz
= bx
.cx().int_width(lhs_llty
);
312 bx
.trunc(rhs
, lhs_llty
)
313 } else if lhs_sz
> rhs_sz
{
314 // FIXME (#1877: If in the future shifting by negative
315 // values is no longer undefined then this is wrong.
316 bx
.zext(rhs
, lhs_llty
)
325 /// Returns `true` if this session's target will use SEH-based unwinding.
327 /// This is only true for MSVC targets, and even then the 64-bit MSVC target
328 /// currently uses SEH-ish unwinding with DWARF info tables to the side (same as
329 /// 64-bit MinGW) instead of "full SEH".
330 pub fn wants_msvc_seh(sess
: &Session
) -> bool
{
331 sess
.target
.target
.options
.is_like_msvc
334 pub fn from_immediate
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
338 if bx
.cx().val_ty(val
) == bx
.cx().type_i1() { bx.zext(val, bx.cx().type_i8()) }
else { val }
341 pub fn to_immediate
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
344 layout
: layout
::TyLayout
<'_
>,
346 if let layout
::Abi
::Scalar(ref scalar
) = layout
.abi
{
347 return to_immediate_scalar(bx
, val
, scalar
);
352 pub fn to_immediate_scalar
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
355 scalar
: &layout
::Scalar
,
357 if scalar
.is_bool() {
358 return bx
.trunc(val
, bx
.cx().type_i1());
363 pub fn memcpy_ty
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
369 layout
: TyLayout
<'tcx
>,
372 let size
= layout
.size
.bytes();
377 bx
.memcpy(dst
, dst_align
, src
, src_align
, bx
.cx().const_usize(size
), flags
);
380 pub fn codegen_instance
<'a
, 'tcx
: 'a
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
381 cx
: &'a Bx
::CodegenCx
,
382 instance
: Instance
<'tcx
>,
384 // this is an info! to allow collecting monomorphization statistics
385 // and to allow finding the last function before LLVM aborts from
387 info
!("codegen_instance({})", instance
);
389 mir
::codegen_mir
::<Bx
>(cx
, instance
);
392 /// Creates the `main` function which will initialize the rust runtime and call
393 /// users main function.
394 pub fn maybe_create_entry_wrapper
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(cx
: &'a Bx
::CodegenCx
) {
395 let (main_def_id
, span
) = match cx
.tcx().entry_fn(LOCAL_CRATE
) {
396 Some((def_id
, _
)) => (def_id
, cx
.tcx().def_span(def_id
)),
400 let instance
= Instance
::mono(cx
.tcx(), main_def_id
);
402 if !cx
.codegen_unit().contains_item(&MonoItem
::Fn(instance
)) {
403 // We want to create the wrapper in the same codegen unit as Rust's main
408 let main_llfn
= cx
.get_fn_addr(instance
);
410 let et
= cx
.tcx().entry_fn(LOCAL_CRATE
).map(|e
| e
.1);
412 Some(EntryFnType
::Main
) => create_entry_fn
::<Bx
>(cx
, span
, main_llfn
, main_def_id
, true),
413 Some(EntryFnType
::Start
) => create_entry_fn
::<Bx
>(cx
, span
, main_llfn
, main_def_id
, false),
414 None
=> {}
// Do nothing.
417 fn create_entry_fn
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
418 cx
: &'a Bx
::CodegenCx
,
420 rust_main
: Bx
::Value
,
421 rust_main_def_id
: DefId
,
422 use_start_lang_item
: bool
,
424 // The entry function is either `int main(void)` or `int main(int argc, char **argv)`,
425 // depending on whether the target needs `argc` and `argv` to be passed in.
426 let llfty
= if cx
.sess().target
.target
.options
.main_needs_argc_argv
{
427 cx
.type_func(&[cx
.type_int(), cx
.type_ptr_to(cx
.type_i8p())], cx
.type_int())
429 cx
.type_func(&[], cx
.type_int())
432 let main_ret_ty
= cx
.tcx().fn_sig(rust_main_def_id
).output();
433 // Given that `main()` has no arguments,
434 // then its return type cannot have
435 // late-bound regions, since late-bound
436 // regions must appear in the argument
438 let main_ret_ty
= cx
.tcx().erase_regions(&main_ret_ty
.no_bound_vars().unwrap());
440 if cx
.get_defined_value("main").is_some() {
441 // FIXME: We should be smart and show a better diagnostic here.
443 .struct_span_err(sp
, "entry symbol `main` defined multiple times")
444 .help("did you use `#[no_mangle]` on `fn main`? Use `#[start]` instead")
446 cx
.sess().abort_if_errors();
449 let llfn
= cx
.declare_cfn("main", llfty
);
451 // `main` should respect same config for frame pointer elimination as rest of code
452 cx
.set_frame_pointer_elimination(llfn
);
453 cx
.apply_target_cpu_attr(llfn
);
455 let mut bx
= Bx
::new_block(&cx
, llfn
, "top");
457 bx
.insert_reference_to_gdb_debug_scripts_section_global();
459 let (arg_argc
, arg_argv
) = get_argc_argv(cx
, &mut bx
);
461 let (start_fn
, args
) = if use_start_lang_item
{
462 let start_def_id
= cx
.tcx().require_lang_item(StartFnLangItem
, None
);
463 let start_fn
= cx
.get_fn_addr(
464 ty
::Instance
::resolve(
466 ty
::ParamEnv
::reveal_all(),
468 cx
.tcx().intern_substs(&[main_ret_ty
.into()]),
474 vec
![bx
.pointercast(rust_main
, cx
.type_ptr_to(cx
.type_i8p())), arg_argc
, arg_argv
],
477 debug
!("using user-defined start fn");
478 (rust_main
, vec
![arg_argc
, arg_argv
])
481 let result
= bx
.call(start_fn
, &args
, None
);
482 let cast
= bx
.intcast(result
, cx
.type_int(), true);
487 /// Obtain the `argc` and `argv` values to pass to the rust start function.
488 fn get_argc_argv
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
489 cx
: &'a Bx
::CodegenCx
,
491 ) -> (Bx
::Value
, Bx
::Value
) {
492 if cx
.sess().target
.target
.options
.main_needs_argc_argv
{
493 // Params from native `main()` used as args for rust start function
494 let param_argc
= bx
.get_param(0);
495 let param_argv
= bx
.get_param(1);
496 let arg_argc
= bx
.intcast(param_argc
, cx
.type_isize(), true);
497 let arg_argv
= param_argv
;
500 // The Rust start function doesn't need `argc` and `argv`, so just pass zeros.
501 let arg_argc
= bx
.const_int(cx
.type_int(), 0);
502 let arg_argv
= bx
.const_null(cx
.type_ptr_to(cx
.type_i8p()));
507 pub const CODEGEN_WORKER_ID
: usize = ::std
::usize::MAX
;
509 pub fn codegen_crate
<B
: ExtraBackendMethods
>(
512 metadata
: EncodedMetadata
,
513 need_metadata_module
: bool
,
514 ) -> OngoingCodegen
<B
> {
515 check_for_rustc_errors_attr(tcx
);
517 // Skip crate items and just output metadata in -Z no-codegen mode.
518 if tcx
.sess
.opts
.debugging_opts
.no_codegen
|| !tcx
.sess
.opts
.output_types
.should_codegen() {
519 let ongoing_codegen
= start_async_codegen(backend
, tcx
, metadata
, 1);
521 ongoing_codegen
.codegen_finished(tcx
);
525 ongoing_codegen
.check_for_errors(tcx
.sess
);
527 return ongoing_codegen
;
530 let cgu_name_builder
= &mut CodegenUnitNameBuilder
::new(tcx
);
532 // Run the monomorphization collector and partition the collected items into
534 let codegen_units
= tcx
.collect_and_partition_mono_items(LOCAL_CRATE
).1;
535 let codegen_units
= (*codegen_units
).clone();
537 // Force all codegen_unit queries so they are already either red or green
538 // when compile_codegen_unit accesses them. We are not able to re-execute
539 // the codegen_unit query from just the DepNode, so an unknown color would
540 // lead to having to re-execute compile_codegen_unit, possibly
542 if tcx
.dep_graph
.is_fully_enabled() {
543 for cgu
in &codegen_units
{
544 tcx
.codegen_unit(cgu
.name());
548 let ongoing_codegen
= start_async_codegen(backend
.clone(), tcx
, metadata
, codegen_units
.len());
549 let ongoing_codegen
= AbortCodegenOnDrop
::<B
>(Some(ongoing_codegen
));
551 // Codegen an allocator shim, if necessary.
553 // If the crate doesn't have an `allocator_kind` set then there's definitely
554 // no shim to generate. Otherwise we also check our dependency graph for all
555 // our output crate types. If anything there looks like its a `Dynamic`
556 // linkage, then it's already got an allocator shim and we'll be using that
557 // one instead. If nothing exists then it's our job to generate the
559 let any_dynamic_crate
= tcx
.dependency_formats(LOCAL_CRATE
).iter().any(|(_
, list
)| {
560 use rustc
::middle
::dependency_format
::Linkage
;
561 list
.iter().any(|&linkage
| linkage
== Linkage
::Dynamic
)
563 let allocator_module
= if any_dynamic_crate
{
565 } else if let Some(kind
) = tcx
.allocator_kind() {
567 cgu_name_builder
.build_cgu_name(LOCAL_CRATE
, &["crate"], Some("allocator")).to_string();
568 let mut modules
= backend
.new_metadata(tcx
, &llmod_id
);
570 .time("write_allocator_module", || backend
.codegen_allocator(tcx
, &mut modules
, kind
));
572 Some(ModuleCodegen { name: llmod_id, module_llvm: modules, kind: ModuleKind::Allocator }
)
577 if let Some(allocator_module
) = allocator_module
{
578 ongoing_codegen
.submit_pre_codegened_module_to_llvm(tcx
, allocator_module
);
581 if need_metadata_module
{
582 // Codegen the encoded metadata.
583 let metadata_cgu_name
=
584 cgu_name_builder
.build_cgu_name(LOCAL_CRATE
, &["crate"], Some("metadata")).to_string();
585 let mut metadata_llvm_module
= backend
.new_metadata(tcx
, &metadata_cgu_name
);
586 tcx
.sess
.time("write_compressed_metadata", || {
587 backend
.write_compressed_metadata(
589 &ongoing_codegen
.metadata
,
590 &mut metadata_llvm_module
,
594 let metadata_module
= ModuleCodegen
{
595 name
: metadata_cgu_name
,
596 module_llvm
: metadata_llvm_module
,
597 kind
: ModuleKind
::Metadata
,
599 ongoing_codegen
.submit_pre_codegened_module_to_llvm(tcx
, metadata_module
);
602 // We sort the codegen units by size. This way we can schedule work for LLVM
603 // a bit more efficiently.
604 let codegen_units
= {
605 let mut codegen_units
= codegen_units
;
606 codegen_units
.sort_by_cached_key(|cgu
| cmp
::Reverse(cgu
.size_estimate()));
610 let total_codegen_time
= Lock
::new(Duration
::new(0, 0));
612 // The non-parallel compiler can only translate codegen units to LLVM IR
613 // on a single thread, leading to a staircase effect where the N LLVM
614 // threads have to wait on the single codegen threads to generate work
615 // for them. The parallel compiler does not have this restriction, so
616 // we can pre-load the LLVM queue in parallel before handing off
617 // coordination to the OnGoingCodegen scheduler.
619 // This likely is a temporary measure. Once we don't have to support the
620 // non-parallel compiler anymore, we can compile CGUs end-to-end in
621 // parallel and get rid of the complicated scheduling logic.
622 let pre_compile_cgus
= |cgu_reuse
: &[CguReuse
]| {
623 if cfg
!(parallel_compiler
) {
624 tcx
.sess
.time("compile_first_CGU_batch", || {
625 // Try to find one CGU to compile per thread.
626 let cgus
: Vec
<_
> = cgu_reuse
629 .filter(|&(_
, reuse
)| reuse
== &CguReuse
::No
)
630 .take(tcx
.sess
.threads())
633 // Compile the found CGUs in parallel.
636 let start_time
= Instant
::now();
637 let module
= backend
.compile_codegen_unit(tcx
, codegen_units
[i
].name());
638 let mut time
= total_codegen_time
.lock();
639 *time
+= start_time
.elapsed();
649 let mut cgu_reuse
= Vec
::new();
650 let mut pre_compiled_cgus
: Option
<FxHashMap
<usize, _
>> = None
;
652 for (i
, cgu
) in codegen_units
.iter().enumerate() {
653 ongoing_codegen
.wait_for_signal_to_codegen_item();
654 ongoing_codegen
.check_for_errors(tcx
.sess
);
656 // Do some setup work in the first iteration
657 if pre_compiled_cgus
.is_none() {
658 // Calculate the CGU reuse
659 cgu_reuse
= tcx
.sess
.time("find_cgu_reuse", || {
660 codegen_units
.iter().map(|cgu
| determine_cgu_reuse(tcx
, &cgu
)).collect()
662 // Pre compile some CGUs
663 pre_compiled_cgus
= Some(pre_compile_cgus(&cgu_reuse
));
666 let cgu_reuse
= cgu_reuse
[i
];
667 tcx
.sess
.cgu_reuse_tracker
.set_actual_reuse(&cgu
.name().as_str(), cgu_reuse
);
672 if let Some(cgu
) = pre_compiled_cgus
.as_mut().unwrap().remove(&i
) {
675 let start_time
= Instant
::now();
676 let module
= backend
.compile_codegen_unit(tcx
, cgu
.name());
677 let mut time
= total_codegen_time
.lock();
678 *time
+= start_time
.elapsed();
681 submit_codegened_module_to_llvm(
683 &ongoing_codegen
.coordinator_send
,
689 CguReuse
::PreLto
=> {
690 submit_pre_lto_module_to_llvm(
693 &ongoing_codegen
.coordinator_send
,
694 CachedModuleCodegen
{
695 name
: cgu
.name().to_string(),
696 source
: cgu
.work_product(tcx
),
701 CguReuse
::PostLto
=> {
702 submit_post_lto_module_to_llvm(
704 &ongoing_codegen
.coordinator_send
,
705 CachedModuleCodegen
{
706 name
: cgu
.name().to_string(),
707 source
: cgu
.work_product(tcx
),
715 ongoing_codegen
.codegen_finished(tcx
);
717 // Since the main thread is sometimes blocked during codegen, we keep track
718 // -Ztime-passes output manually.
719 print_time_passes_entry(
720 tcx
.sess
.time_passes(),
721 "codegen_to_LLVM_IR",
722 total_codegen_time
.into_inner(),
725 ::rustc_incremental
::assert_module_sources
::assert_module_sources(tcx
);
727 symbol_names_test
::report_symbol_names(tcx
);
729 ongoing_codegen
.check_for_errors(tcx
.sess
);
733 ongoing_codegen
.into_inner()
736 /// A curious wrapper structure whose only purpose is to call `codegen_aborted`
737 /// when it's dropped abnormally.
739 /// In the process of working on rust-lang/rust#55238 a mysterious segfault was
740 /// stumbled upon. The segfault was never reproduced locally, but it was
741 /// suspected to be related to the fact that codegen worker threads were
742 /// sticking around by the time the main thread was exiting, causing issues.
744 /// This structure is an attempt to fix that issue where the `codegen_aborted`
745 /// message will block until all workers have finished. This should ensure that
746 /// even if the main codegen thread panics we'll wait for pending work to
747 /// complete before returning from the main thread, hopefully avoiding
750 /// If you see this comment in the code, then it means that this workaround
751 /// worked! We may yet one day track down the mysterious cause of that
753 struct AbortCodegenOnDrop
<B
: ExtraBackendMethods
>(Option
<OngoingCodegen
<B
>>);
755 impl<B
: ExtraBackendMethods
> AbortCodegenOnDrop
<B
> {
756 fn into_inner(mut self) -> OngoingCodegen
<B
> {
757 self.0.take().unwrap()
761 impl<B
: ExtraBackendMethods
> Deref
for AbortCodegenOnDrop
<B
> {
762 type Target
= OngoingCodegen
<B
>;
764 fn deref(&self) -> &OngoingCodegen
<B
> {
765 self.0.as_ref().unwrap()
769 impl<B
: ExtraBackendMethods
> DerefMut
for AbortCodegenOnDrop
<B
> {
770 fn deref_mut(&mut self) -> &mut OngoingCodegen
<B
> {
771 self.0.as_mut().unwrap()
775 impl<B
: ExtraBackendMethods
> Drop
for AbortCodegenOnDrop
<B
> {
777 if let Some(codegen
) = self.0.take() {
778 codegen
.codegen_aborted();
783 fn finalize_tcx(tcx
: TyCtxt
<'_
>) {
784 tcx
.sess
.time("assert_dep_graph", || ::rustc_incremental
::assert_dep_graph(tcx
));
785 tcx
.sess
.time("serialize_dep_graph", || ::rustc_incremental
::save_dep_graph(tcx
));
787 // We assume that no queries are run past here. If there are new queries
788 // after this point, they'll show up as "<unknown>" in self-profiling data.
790 let _prof_timer
= tcx
.prof
.generic_activity("self_profile_alloc_query_strings");
791 tcx
.alloc_self_profile_query_strings();
796 pub fn new(tcx
: TyCtxt
<'_
>) -> CrateInfo
{
797 let mut info
= CrateInfo
{
799 compiler_builtins
: None
,
800 profiler_runtime
: None
,
801 is_no_builtins
: Default
::default(),
802 native_libraries
: Default
::default(),
803 used_libraries
: tcx
.native_libraries(LOCAL_CRATE
),
804 link_args
: tcx
.link_args(LOCAL_CRATE
),
805 crate_name
: Default
::default(),
806 used_crates_dynamic
: cstore
::used_crates(tcx
, LinkagePreference
::RequireDynamic
),
807 used_crates_static
: cstore
::used_crates(tcx
, LinkagePreference
::RequireStatic
),
808 used_crate_source
: Default
::default(),
809 lang_item_to_crate
: Default
::default(),
810 missing_lang_items
: Default
::default(),
811 dependency_formats
: tcx
.dependency_formats(LOCAL_CRATE
),
813 let lang_items
= tcx
.lang_items();
815 let crates
= tcx
.crates();
817 let n_crates
= crates
.len();
818 info
.native_libraries
.reserve(n_crates
);
819 info
.crate_name
.reserve(n_crates
);
820 info
.used_crate_source
.reserve(n_crates
);
821 info
.missing_lang_items
.reserve(n_crates
);
823 for &cnum
in crates
.iter() {
824 info
.native_libraries
.insert(cnum
, tcx
.native_libraries(cnum
));
825 info
.crate_name
.insert(cnum
, tcx
.crate_name(cnum
).to_string());
826 info
.used_crate_source
.insert(cnum
, tcx
.used_crate_source(cnum
));
827 if tcx
.is_panic_runtime(cnum
) {
828 info
.panic_runtime
= Some(cnum
);
830 if tcx
.is_compiler_builtins(cnum
) {
831 info
.compiler_builtins
= Some(cnum
);
833 if tcx
.is_profiler_runtime(cnum
) {
834 info
.profiler_runtime
= Some(cnum
);
836 if tcx
.is_no_builtins(cnum
) {
837 info
.is_no_builtins
.insert(cnum
);
839 let missing
= tcx
.missing_lang_items(cnum
);
840 for &item
in missing
.iter() {
841 if let Ok(id
) = lang_items
.require(item
) {
842 info
.lang_item_to_crate
.insert(item
, id
.krate
);
846 // No need to look for lang items that are whitelisted and don't
847 // actually need to exist.
848 let missing
= missing
851 .filter(|&l
| !weak_lang_items
::whitelisted(tcx
, l
))
853 info
.missing_lang_items
.insert(cnum
, missing
);
860 pub fn provide_both(providers
: &mut Providers
<'_
>) {
861 providers
.backend_optimization_level
= |tcx
, cratenum
| {
862 let for_speed
= match tcx
.sess
.opts
.optimize
{
863 // If globally no optimisation is done, #[optimize] has no effect.
865 // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the
866 // pass manager and it is likely that some module-wide passes (such as inliner or
867 // cross-function constant propagation) would ignore the `optnone` annotation we put
868 // on the functions, thus necessarily involving these functions into optimisations.
869 config
::OptLevel
::No
=> return config
::OptLevel
::No
,
870 // If globally optimise-speed is already specified, just use that level.
871 config
::OptLevel
::Less
=> return config
::OptLevel
::Less
,
872 config
::OptLevel
::Default
=> return config
::OptLevel
::Default
,
873 config
::OptLevel
::Aggressive
=> return config
::OptLevel
::Aggressive
,
874 // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size)
876 config
::OptLevel
::Size
=> config
::OptLevel
::Default
,
877 config
::OptLevel
::SizeMin
=> config
::OptLevel
::Default
,
880 let (defids
, _
) = tcx
.collect_and_partition_mono_items(cratenum
);
882 let CodegenFnAttrs { optimize, .. }
= tcx
.codegen_fn_attrs(*id
);
884 attr
::OptimizeAttr
::None
=> continue,
885 attr
::OptimizeAttr
::Size
=> continue,
886 attr
::OptimizeAttr
::Speed
=> {
891 return tcx
.sess
.opts
.optimize
;
894 providers
.dllimport_foreign_items
= |tcx
, krate
| {
895 let module_map
= tcx
.foreign_modules(krate
);
897 module_map
.iter().map(|lib
| (lib
.def_id
, lib
)).collect
::<FxHashMap
<_
, _
>>();
900 .native_libraries(krate
)
903 if lib
.kind
!= cstore
::NativeLibraryKind
::NativeUnknown
{
906 let cfg
= match lib
.cfg
{
907 Some(ref cfg
) => cfg
,
910 attr
::cfg_matches(cfg
, &tcx
.sess
.parse_sess
, None
)
912 .filter_map(|lib
| lib
.foreign_module
)
913 .map(|id
| &module_map
[&id
])
914 .flat_map(|module
| module
.foreign_items
.iter().cloned())
916 tcx
.arena
.alloc(dllimports
)
919 providers
.is_dllimport_foreign_item
=
920 |tcx
, def_id
| tcx
.dllimport_foreign_items(def_id
.krate
).contains(&def_id
);
923 fn determine_cgu_reuse
<'tcx
>(tcx
: TyCtxt
<'tcx
>, cgu
: &CodegenUnit
<'tcx
>) -> CguReuse
{
924 if !tcx
.dep_graph
.is_fully_enabled() {
928 let work_product_id
= &cgu
.work_product_id();
929 if tcx
.dep_graph
.previous_work_product(work_product_id
).is_none() {
930 // We don't have anything cached for this CGU. This can happen
931 // if the CGU did not exist in the previous session.
935 // Try to mark the CGU as green. If it we can do so, it means that nothing
936 // affecting the LLVM module has changed and we can re-use a cached version.
937 // If we compile with any kind of LTO, this means we can re-use the bitcode
938 // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only
939 // know that later). If we are not doing LTO, there is only one optimized
940 // version of each module, so we re-use that.
941 let dep_node
= cgu
.codegen_dep_node(tcx
);
943 !tcx
.dep_graph
.dep_node_exists(&dep_node
),
944 "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",
948 if tcx
.dep_graph
.try_mark_green(tcx
, &dep_node
).is_some() {
949 // We can re-use either the pre- or the post-thinlto state
950 if tcx
.sess
.lto() != Lto
::No { CguReuse::PreLto }
else { CguReuse::PostLto }