1 use crate::assert_module_sources
::CguReuse
;
2 use crate::back
::link
::are_upstream_rust_objects_already_included
;
3 use crate::back
::metadata
::create_compressed_metadata_file
;
4 use crate::back
::write
::{
5 compute_per_cgu_lto_type
, start_async_codegen
, submit_codegened_module_to_llvm
,
6 submit_post_lto_module_to_llvm
, submit_pre_lto_module_to_llvm
, ComputedLtoType
, OngoingCodegen
,
8 use crate::common
::{IntPredicate, RealPredicate, TypeKind}
;
12 use crate::mir
::operand
::OperandValue
;
13 use crate::mir
::place
::PlaceRef
;
15 use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}
;
17 use rustc_ast
::expand
::allocator
::{global_fn_name, AllocatorKind, ALLOCATOR_METHODS}
;
18 use rustc_attr
as attr
;
19 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
20 use rustc_data_structures
::profiling
::{get_resident_set_size, print_time_passes_entry}
;
21 use rustc_data_structures
::sync
::par_map
;
23 use rustc_hir
::def_id
::{DefId, LOCAL_CRATE}
;
24 use rustc_hir
::lang_items
::LangItem
;
25 use rustc_metadata
::EncodedMetadata
;
26 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrs
;
27 use rustc_middle
::middle
::debugger_visualizer
::{DebuggerVisualizerFile, DebuggerVisualizerType}
;
28 use rustc_middle
::middle
::exported_symbols
;
29 use rustc_middle
::middle
::exported_symbols
::SymbolExportKind
;
30 use rustc_middle
::middle
::lang_items
;
31 use rustc_middle
::mir
::mono
::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}
;
32 use rustc_middle
::query
::Providers
;
33 use rustc_middle
::ty
::layout
::{HasTyCtxt, LayoutOf, TyAndLayout}
;
34 use rustc_middle
::ty
::{self, Instance, Ty, TyCtxt}
;
35 use rustc_session
::config
::{self, CrateType, EntryFnType, OutputType}
;
36 use rustc_session
::Session
;
37 use rustc_span
::symbol
::sym
;
38 use rustc_span
::Symbol
;
39 use rustc_target
::abi
::{Align, FIRST_VARIANT}
;
42 use std
::collections
::BTreeSet
;
43 use std
::time
::{Duration, Instant}
;
45 use itertools
::Itertools
;
47 pub fn bin_op_to_icmp_predicate(op
: hir
::BinOpKind
, signed
: bool
) -> IntPredicate
{
49 hir
::BinOpKind
::Eq
=> IntPredicate
::IntEQ
,
50 hir
::BinOpKind
::Ne
=> IntPredicate
::IntNE
,
51 hir
::BinOpKind
::Lt
=> {
58 hir
::BinOpKind
::Le
=> {
65 hir
::BinOpKind
::Gt
=> {
72 hir
::BinOpKind
::Ge
=> {
80 "comparison_op_to_icmp_predicate: expected comparison operator, \
87 pub fn bin_op_to_fcmp_predicate(op
: hir
::BinOpKind
) -> RealPredicate
{
89 hir
::BinOpKind
::Eq
=> RealPredicate
::RealOEQ
,
90 hir
::BinOpKind
::Ne
=> RealPredicate
::RealUNE
,
91 hir
::BinOpKind
::Lt
=> RealPredicate
::RealOLT
,
92 hir
::BinOpKind
::Le
=> RealPredicate
::RealOLE
,
93 hir
::BinOpKind
::Gt
=> RealPredicate
::RealOGT
,
94 hir
::BinOpKind
::Ge
=> RealPredicate
::RealOGE
,
97 "comparison_op_to_fcmp_predicate: expected comparison operator, \
105 pub fn compare_simd_types
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
113 let signed
= match t
.kind() {
115 let cmp
= bin_op_to_fcmp_predicate(op
);
116 let cmp
= bx
.fcmp(cmp
, lhs
, rhs
);
117 return bx
.sext(cmp
, ret_ty
);
119 ty
::Uint(_
) => false,
121 _
=> bug
!("compare_simd_types: invalid SIMD type"),
124 let cmp
= bin_op_to_icmp_predicate(op
, signed
);
125 let cmp
= bx
.icmp(cmp
, lhs
, rhs
);
126 // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension
127 // to get the correctly sized type. This will compile to a single instruction
128 // once the IR is converted to assembly if the SIMD instruction is supported
129 // by the target architecture.
133 /// Retrieves the information we are losing (making dynamic) in an unsizing
136 /// The `old_info` argument is a bit odd. It is intended for use in an upcast,
137 /// where the new vtable for an object will be derived from the old one.
138 pub fn unsized_info
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
142 old_info
: Option
<Bx
::Value
>,
145 let (source
, target
) =
146 cx
.tcx().struct_lockstep_tails_erasing_lifetimes(source
, target
, bx
.param_env());
147 match (source
.kind(), target
.kind()) {
148 (&ty
::Array(_
, len
), &ty
::Slice(_
)) => {
149 cx
.const_usize(len
.eval_target_usize(cx
.tcx(), ty
::ParamEnv
::reveal_all()))
151 (&ty
::Dynamic(data_a
, _
, src_dyn_kind
), &ty
::Dynamic(data_b
, _
, target_dyn_kind
))
152 if src_dyn_kind
== target_dyn_kind
=>
155 old_info
.expect("unsized_info: missing old info for trait upcasting coercion");
156 if data_a
.principal_def_id() == data_b
.principal_def_id() {
157 // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
161 // trait upcasting coercion
164 cx
.tcx().vtable_trait_upcasting_coercion_new_vptr_slot((source
, target
));
166 if let Some(entry_idx
) = vptr_entry_idx
{
167 let ptr_ty
= cx
.type_ptr();
168 let ptr_align
= cx
.tcx().data_layout
.pointer_align
.abi
;
169 let gep
= bx
.inbounds_gep(
172 &[bx
.const_usize(u64::try_from(entry_idx
).unwrap())],
174 let new_vptr
= bx
.load(ptr_ty
, gep
, ptr_align
);
175 bx
.nonnull_metadata(new_vptr
);
176 // VTable loads are invariant.
177 bx
.set_invariant_load(new_vptr
);
183 (_
, ty
::Dynamic(data
, _
, _
)) => meth
::get_vtable(cx
, source
, data
.principal()),
184 _
=> bug
!("unsized_info: invalid unsizing {:?} -> {:?}", source
, target
),
188 /// Coerces `src` to `dst_ty`. `src_ty` must be a pointer.
189 pub fn unsize_ptr
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
194 old_info
: Option
<Bx
::Value
>,
195 ) -> (Bx
::Value
, Bx
::Value
) {
196 debug
!("unsize_ptr: {:?} => {:?}", src_ty
, dst_ty
);
197 match (src_ty
.kind(), dst_ty
.kind()) {
198 (&ty
::Ref(_
, a
, _
), &ty
::Ref(_
, b
, _
) | &ty
::RawPtr(ty
::TypeAndMut { ty: b, .. }
))
199 | (&ty
::RawPtr(ty
::TypeAndMut { ty: a, .. }
), &ty
::RawPtr(ty
::TypeAndMut { ty: b, .. }
)) => {
200 assert_eq
!(bx
.cx().type_is_sized(a
), old_info
.is_none());
201 (src
, unsized_info(bx
, a
, b
, old_info
))
203 (&ty
::Adt(def_a
, _
), &ty
::Adt(def_b
, _
)) => {
204 assert_eq
!(def_a
, def_b
); // implies same number of fields
205 let src_layout
= bx
.cx().layout_of(src_ty
);
206 let dst_layout
= bx
.cx().layout_of(dst_ty
);
207 if src_ty
== dst_ty
{
208 return (src
, old_info
.unwrap());
210 let mut result
= None
;
211 for i
in 0..src_layout
.fields
.count() {
212 let src_f
= src_layout
.field(bx
.cx(), i
);
214 // We are looking for the one non-1-ZST field; this is not it.
218 assert_eq
!(src_layout
.fields
.offset(i
).bytes(), 0);
219 assert_eq
!(dst_layout
.fields
.offset(i
).bytes(), 0);
220 assert_eq
!(src_layout
.size
, src_f
.size
);
222 let dst_f
= dst_layout
.field(bx
.cx(), i
);
223 assert_ne
!(src_f
.ty
, dst_f
.ty
);
224 assert_eq
!(result
, None
);
225 result
= Some(unsize_ptr(bx
, src
, src_f
.ty
, dst_f
.ty
, old_info
));
229 _
=> bug
!("unsize_ptr: called on bad types"),
233 /// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type.
234 pub fn cast_to_dyn_star
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
237 src_ty_and_layout
: TyAndLayout
<'tcx
>,
239 old_info
: Option
<Bx
::Value
>,
240 ) -> (Bx
::Value
, Bx
::Value
) {
241 debug
!("cast_to_dyn_star: {:?} => {:?}", src_ty_and_layout
.ty
, dst_ty
);
243 matches
!(dst_ty
.kind(), ty
::Dynamic(_
, _
, ty
::DynStar
)),
244 "destination type must be a dyn*"
246 let src
= match bx
.cx().type_kind(bx
.cx().backend_type(src_ty_and_layout
)) {
247 TypeKind
::Pointer
=> src
,
248 TypeKind
::Integer
=> bx
.inttoptr(src
, bx
.type_ptr()),
249 // FIXME(dyn-star): We probably have to do a bitcast first, then inttoptr.
250 kind
=> bug
!("unexpected TypeKind for left-hand side of `dyn*` cast: {kind:?}"),
252 (src
, unsized_info(bx
, src_ty_and_layout
.ty
, dst_ty
, old_info
))
255 /// Coerces `src`, which is a reference to a value of type `src_ty`,
256 /// to a value of type `dst_ty`, and stores the result in `dst`.
257 pub fn coerce_unsized_into
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
259 src
: PlaceRef
<'tcx
, Bx
::Value
>,
260 dst
: PlaceRef
<'tcx
, Bx
::Value
>,
262 let src_ty
= src
.layout
.ty
;
263 let dst_ty
= dst
.layout
.ty
;
264 match (src_ty
.kind(), dst_ty
.kind()) {
265 (&ty
::Ref(..), &ty
::Ref(..) | &ty
::RawPtr(..)) | (&ty
::RawPtr(..), &ty
::RawPtr(..)) => {
266 let (base
, info
) = match bx
.load_operand(src
).val
{
267 OperandValue
::Pair(base
, info
) => unsize_ptr(bx
, base
, src_ty
, dst_ty
, Some(info
)),
268 OperandValue
::Immediate(base
) => unsize_ptr(bx
, base
, src_ty
, dst_ty
, None
),
269 OperandValue
::Ref(..) | OperandValue
::ZeroSized
=> bug
!(),
271 OperandValue
::Pair(base
, info
).store(bx
, dst
);
274 (&ty
::Adt(def_a
, _
), &ty
::Adt(def_b
, _
)) => {
275 assert_eq
!(def_a
, def_b
); // implies same number of fields
277 for i
in def_a
.variant(FIRST_VARIANT
).fields
.indices() {
278 let src_f
= src
.project_field(bx
, i
.as_usize());
279 let dst_f
= dst
.project_field(bx
, i
.as_usize());
281 if dst_f
.layout
.is_zst() {
282 // No data here, nothing to copy/coerce.
286 if src_f
.layout
.ty
== dst_f
.layout
.ty
{
297 coerce_unsized_into(bx
, src_f
, dst_f
);
301 _
=> bug
!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty
, dst_ty
,),
305 pub fn cast_shift_expr_rhs
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
310 // Shifts may have any size int on the rhs
311 let mut rhs_llty
= bx
.cx().val_ty(rhs
);
312 let mut lhs_llty
= bx
.cx().val_ty(lhs
);
313 if bx
.cx().type_kind(rhs_llty
) == TypeKind
::Vector
{
314 rhs_llty
= bx
.cx().element_type(rhs_llty
)
316 if bx
.cx().type_kind(lhs_llty
) == TypeKind
::Vector
{
317 lhs_llty
= bx
.cx().element_type(lhs_llty
)
319 let rhs_sz
= bx
.cx().int_width(rhs_llty
);
320 let lhs_sz
= bx
.cx().int_width(lhs_llty
);
322 bx
.trunc(rhs
, lhs_llty
)
323 } else if lhs_sz
> rhs_sz
{
324 // We zero-extend even if the RHS is signed. So e.g. `(x: i32) << -1i8` will zero-extend the
325 // RHS to `255i32`. But then we mask the shift amount to be within the size of the LHS
326 // anyway so the result is `31` as it should be. All the extra bits introduced by zext
327 // are masked off so their value does not matter.
328 // FIXME: if we ever support 512bit integers, this will be wrong! For such large integers,
329 // the extra bits introduced by zext are *not* all masked away any more.
330 assert
!(lhs_sz
<= 256);
331 bx
.zext(rhs
, lhs_llty
)
337 // Returns `true` if this session's target will use native wasm
338 // exceptions. This means that the VM does the unwinding for
340 pub fn wants_wasm_eh(sess
: &Session
) -> bool
{
341 sess
.target
.is_like_wasm
&& sess
.target
.os
!= "emscripten"
344 /// Returns `true` if this session's target will use SEH-based unwinding.
346 /// This is only true for MSVC targets, and even then the 64-bit MSVC target
347 /// currently uses SEH-ish unwinding with DWARF info tables to the side (same as
348 /// 64-bit MinGW) instead of "full SEH".
349 pub fn wants_msvc_seh(sess
: &Session
) -> bool
{
350 sess
.target
.is_like_msvc
353 /// Returns `true` if this session's target requires the new exception
354 /// handling LLVM IR instructions (catchpad / cleanuppad / ... instead
356 pub fn wants_new_eh_instructions(sess
: &Session
) -> bool
{
357 wants_wasm_eh(sess
) || wants_msvc_seh(sess
)
360 pub fn memcpy_ty
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
366 layout
: TyAndLayout
<'tcx
>,
369 let size
= layout
.size
.bytes();
374 if flags
== MemFlags
::empty()
375 && let Some(bty
) = bx
.cx().scalar_copy_backend_type(layout
)
377 let temp
= bx
.load(bty
, src
, src_align
);
378 bx
.store(temp
, dst
, dst_align
);
380 bx
.memcpy(dst
, dst_align
, src
, src_align
, bx
.cx().const_usize(size
), flags
);
384 pub fn codegen_instance
<'a
, 'tcx
: 'a
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
385 cx
: &'a Bx
::CodegenCx
,
386 instance
: Instance
<'tcx
>,
388 // this is an info! to allow collecting monomorphization statistics
389 // and to allow finding the last function before LLVM aborts from
391 info
!("codegen_instance({})", instance
);
393 mir
::codegen_mir
::<Bx
>(cx
, instance
);
396 /// Creates the `main` function which will initialize the rust runtime and call
397 /// users main function.
398 pub fn maybe_create_entry_wrapper
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
399 cx
: &'a Bx
::CodegenCx
,
400 ) -> Option
<Bx
::Function
> {
401 let (main_def_id
, entry_type
) = cx
.tcx().entry_fn(())?
;
402 let main_is_local
= main_def_id
.is_local();
403 let instance
= Instance
::mono(cx
.tcx(), main_def_id
);
406 // We want to create the wrapper in the same codegen unit as Rust's main
408 if !cx
.codegen_unit().contains_item(&MonoItem
::Fn(instance
)) {
411 } else if !cx
.codegen_unit().is_primary() {
412 // We want to create the wrapper only when the codegen unit is the primary one
416 let main_llfn
= cx
.get_fn_addr(instance
);
418 let entry_fn
= create_entry_fn
::<Bx
>(cx
, main_llfn
, main_def_id
, entry_type
);
419 return Some(entry_fn
);
421 fn create_entry_fn
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
422 cx
: &'a Bx
::CodegenCx
,
423 rust_main
: Bx
::Value
,
424 rust_main_def_id
: DefId
,
425 entry_type
: EntryFnType
,
427 // The entry function is either `int main(void)` or `int main(int argc, char **argv)`, or
428 // `usize efi_main(void *handle, void *system_table)` depending on the target.
429 let llfty
= if cx
.sess().target
.os
.contains("uefi") {
430 cx
.type_func(&[cx
.type_ptr(), cx
.type_ptr()], cx
.type_isize())
431 } else if cx
.sess().target
.main_needs_argc_argv
{
432 cx
.type_func(&[cx
.type_int(), cx
.type_ptr()], cx
.type_int())
434 cx
.type_func(&[], cx
.type_int())
437 let main_ret_ty
= cx
.tcx().fn_sig(rust_main_def_id
).no_bound_vars().unwrap().output();
438 // Given that `main()` has no arguments,
439 // then its return type cannot have
440 // late-bound regions, since late-bound
441 // regions must appear in the argument
443 let main_ret_ty
= cx
.tcx().normalize_erasing_regions(
444 ty
::ParamEnv
::reveal_all(),
445 main_ret_ty
.no_bound_vars().unwrap(),
448 let Some(llfn
) = cx
.declare_c_main(llfty
) else {
449 // FIXME: We should be smart and show a better diagnostic here.
450 let span
= cx
.tcx().def_span(rust_main_def_id
);
451 let dcx
= cx
.tcx().dcx();
452 dcx
.emit_err(errors
::MultipleMainFunctions { span }
);
453 dcx
.abort_if_errors();
457 // `main` should respect same config for frame pointer elimination as rest of code
458 cx
.set_frame_pointer_type(llfn
);
459 cx
.apply_target_cpu_attr(llfn
);
461 let llbb
= Bx
::append_block(cx
, llfn
, "top");
462 let mut bx
= Bx
::build(cx
, llbb
);
464 bx
.insert_reference_to_gdb_debug_scripts_section_global();
466 let isize_ty
= cx
.type_isize();
467 let ptr_ty
= cx
.type_ptr();
468 let (arg_argc
, arg_argv
) = get_argc_argv(cx
, &mut bx
);
470 let (start_fn
, start_ty
, args
) = if let EntryFnType
::Main { sigpipe }
= entry_type
{
471 let start_def_id
= cx
.tcx().require_lang_item(LangItem
::Start
, None
);
472 let start_fn
= cx
.get_fn_addr(
473 ty
::Instance
::resolve(
475 ty
::ParamEnv
::reveal_all(),
477 cx
.tcx().mk_args(&[main_ret_ty
.into()]),
483 let i8_ty
= cx
.type_i8();
484 let arg_sigpipe
= bx
.const_u8(sigpipe
);
486 let start_ty
= cx
.type_func(&[cx
.val_ty(rust_main
), isize_ty
, ptr_ty
, i8_ty
], isize_ty
);
487 (start_fn
, start_ty
, vec
![rust_main
, arg_argc
, arg_argv
, arg_sigpipe
])
489 debug
!("using user-defined start fn");
490 let start_ty
= cx
.type_func(&[isize_ty
, ptr_ty
], isize_ty
);
491 (rust_main
, start_ty
, vec
![arg_argc
, arg_argv
])
494 let result
= bx
.call(start_ty
, None
, None
, start_fn
, &args
, None
);
495 if cx
.sess().target
.os
.contains("uefi") {
498 let cast
= bx
.intcast(result
, cx
.type_int(), true);
506 /// Obtain the `argc` and `argv` values to pass to the rust start function.
507 fn get_argc_argv
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
508 cx
: &'a Bx
::CodegenCx
,
510 ) -> (Bx
::Value
, Bx
::Value
) {
511 if cx
.sess().target
.os
.contains("uefi") {
513 let param_handle
= bx
.get_param(0);
514 let param_system_table
= bx
.get_param(1);
515 let arg_argc
= bx
.const_int(cx
.type_isize(), 2);
516 let arg_argv
= bx
.alloca(cx
.type_array(cx
.type_ptr(), 2), Align
::ONE
);
517 bx
.store(param_handle
, arg_argv
, Align
::ONE
);
518 let arg_argv_el1
= bx
.gep(cx
.type_ptr(), arg_argv
, &[bx
.const_int(cx
.type_int(), 1)]);
519 bx
.store(param_system_table
, arg_argv_el1
, Align
::ONE
);
521 } else if cx
.sess().target
.main_needs_argc_argv
{
522 // Params from native `main()` used as args for rust start function
523 let param_argc
= bx
.get_param(0);
524 let param_argv
= bx
.get_param(1);
525 let arg_argc
= bx
.intcast(param_argc
, cx
.type_isize(), true);
526 let arg_argv
= param_argv
;
529 // The Rust start function doesn't need `argc` and `argv`, so just pass zeros.
530 let arg_argc
= bx
.const_int(cx
.type_int(), 0);
531 let arg_argv
= bx
.const_null(cx
.type_ptr());
536 /// This function returns all of the debugger visualizers specified for the
537 /// current crate as well as all upstream crates transitively that match the
538 /// `visualizer_type` specified.
539 pub fn collect_debugger_visualizers_transitive(
541 visualizer_type
: DebuggerVisualizerType
,
542 ) -> BTreeSet
<DebuggerVisualizerFile
> {
543 tcx
.debugger_visualizers(LOCAL_CRATE
)
549 let used_crate_source
= tcx
.used_crate_source(*cnum
);
550 used_crate_source
.rlib
.is_some() || used_crate_source
.rmeta
.is_some()
552 .flat_map(|&cnum
| tcx
.debugger_visualizers(cnum
)),
554 .filter(|visualizer
| visualizer
.visualizer_type
== visualizer_type
)
556 .collect
::<BTreeSet
<_
>>()
559 /// Decide allocator kind to codegen. If `Some(_)` this will be the same as
560 /// `tcx.allocator_kind`, but it may be `None` in more cases (e.g. if using
561 /// allocator definitions from a dylib dependency).
562 pub fn allocator_kind_for_codegen(tcx
: TyCtxt
<'_
>) -> Option
<AllocatorKind
> {
563 // If the crate doesn't have an `allocator_kind` set then there's definitely
564 // no shim to generate. Otherwise we also check our dependency graph for all
565 // our output crate types. If anything there looks like its a `Dynamic`
566 // linkage, then it's already got an allocator shim and we'll be using that
567 // one instead. If nothing exists then it's our job to generate the
569 let any_dynamic_crate
= tcx
.dependency_formats(()).iter().any(|(_
, list
)| {
570 use rustc_middle
::middle
::dependency_format
::Linkage
;
571 list
.iter().any(|&linkage
| linkage
== Linkage
::Dynamic
)
573 if any_dynamic_crate { None }
else { tcx.allocator_kind(()) }
576 pub fn codegen_crate
<B
: ExtraBackendMethods
>(
580 metadata
: EncodedMetadata
,
581 need_metadata_module
: bool
,
582 ) -> OngoingCodegen
<B
> {
583 // Skip crate items and just output metadata in -Z no-codegen mode.
584 if tcx
.sess
.opts
.unstable_opts
.no_codegen
|| !tcx
.sess
.opts
.output_types
.should_codegen() {
585 let ongoing_codegen
= start_async_codegen(backend
, tcx
, target_cpu
, metadata
, None
);
587 ongoing_codegen
.codegen_finished(tcx
);
589 ongoing_codegen
.check_for_errors(tcx
.sess
);
591 return ongoing_codegen
;
594 let cgu_name_builder
= &mut CodegenUnitNameBuilder
::new(tcx
);
596 // Run the monomorphization collector and partition the collected items into
598 let codegen_units
= tcx
.collect_and_partition_mono_items(()).1;
600 // Force all codegen_unit queries so they are already either red or green
601 // when compile_codegen_unit accesses them. We are not able to re-execute
602 // the codegen_unit query from just the DepNode, so an unknown color would
603 // lead to having to re-execute compile_codegen_unit, possibly
605 if tcx
.dep_graph
.is_fully_enabled() {
606 for cgu
in codegen_units
{
607 tcx
.ensure().codegen_unit(cgu
.name());
611 let metadata_module
= need_metadata_module
.then(|| {
612 // Emit compressed metadata object.
613 let metadata_cgu_name
=
614 cgu_name_builder
.build_cgu_name(LOCAL_CRATE
, &["crate"], Some("metadata")).to_string();
615 tcx
.sess
.time("write_compressed_metadata", || {
617 tcx
.output_filenames(()).temp_path(OutputType
::Metadata
, Some(&metadata_cgu_name
));
618 let data
= create_compressed_metadata_file(
621 &exported_symbols
::metadata_symbol_name(tcx
),
623 if let Err(error
) = std
::fs
::write(&file_name
, data
) {
624 tcx
.dcx().emit_fatal(errors
::MetadataObjectFileWrite { error }
);
627 name
: metadata_cgu_name
,
628 kind
: ModuleKind
::Metadata
,
629 object
: Some(file_name
),
636 let ongoing_codegen
=
637 start_async_codegen(backend
.clone(), tcx
, target_cpu
, metadata
, metadata_module
);
639 // Codegen an allocator shim, if necessary.
640 if let Some(kind
) = allocator_kind_for_codegen(tcx
) {
642 cgu_name_builder
.build_cgu_name(LOCAL_CRATE
, &["crate"], Some("allocator")).to_string();
643 let module_llvm
= tcx
.sess
.time("write_allocator_module", || {
644 backend
.codegen_allocator(
648 // If allocator_kind is Some then alloc_error_handler_kind must
650 tcx
.alloc_error_handler_kind(()).unwrap(),
654 ongoing_codegen
.wait_for_signal_to_codegen_item();
655 ongoing_codegen
.check_for_errors(tcx
.sess
);
657 // These modules are generally cheap and won't throw off scheduling.
659 submit_codegened_module_to_llvm(
661 &ongoing_codegen
.coordinator
.sender
,
662 ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator }
,
667 // For better throughput during parallel processing by LLVM, we used to sort
668 // CGUs largest to smallest. This would lead to better thread utilization
669 // by, for example, preventing a large CGU from being processed last and
670 // having only one LLVM thread working while the rest remained idle.
672 // However, this strategy would lead to high memory usage, as it meant the
673 // LLVM-IR for all of the largest CGUs would be resident in memory at once.
675 // Instead, we can compromise by ordering CGUs such that the largest and
676 // smallest are first, second largest and smallest are next, etc. If there
677 // are large size variations, this can reduce memory usage significantly.
678 let codegen_units
: Vec
<_
> = {
679 let mut sorted_cgus
= codegen_units
.iter().collect
::<Vec
<_
>>();
680 sorted_cgus
.sort_by_key(|cgu
| cmp
::Reverse(cgu
.size_estimate()));
682 let (first_half
, second_half
) = sorted_cgus
.split_at(sorted_cgus
.len() / 2);
683 first_half
.iter().interleave(second_half
.iter().rev()).copied().collect()
686 // Calculate the CGU reuse
687 let cgu_reuse
= tcx
.sess
.time("find_cgu_reuse", || {
688 codegen_units
.iter().map(|cgu
| determine_cgu_reuse(tcx
, cgu
)).collect
::<Vec
<_
>>()
691 crate::assert_module_sources
::assert_module_sources(tcx
, &|cgu_reuse_tracker
| {
692 for (i
, cgu
) in codegen_units
.iter().enumerate() {
693 let cgu_reuse
= cgu_reuse
[i
];
694 cgu_reuse_tracker
.set_actual_reuse(cgu
.name().as_str(), cgu_reuse
);
698 let mut total_codegen_time
= Duration
::new(0, 0);
699 let start_rss
= tcx
.sess
.opts
.unstable_opts
.time_passes
.then(|| get_resident_set_size());
701 // The non-parallel compiler can only translate codegen units to LLVM IR
702 // on a single thread, leading to a staircase effect where the N LLVM
703 // threads have to wait on the single codegen threads to generate work
704 // for them. The parallel compiler does not have this restriction, so
705 // we can pre-load the LLVM queue in parallel before handing off
706 // coordination to the OnGoingCodegen scheduler.
708 // This likely is a temporary measure. Once we don't have to support the
709 // non-parallel compiler anymore, we can compile CGUs end-to-end in
710 // parallel and get rid of the complicated scheduling logic.
711 let mut pre_compiled_cgus
= if tcx
.sess
.threads() > 1 {
712 tcx
.sess
.time("compile_first_CGU_batch", || {
713 // Try to find one CGU to compile per thread.
714 let cgus
: Vec
<_
> = cgu_reuse
717 .filter(|&(_
, reuse
)| reuse
== &CguReuse
::No
)
718 .take(tcx
.sess
.threads())
721 // Compile the found CGUs in parallel.
722 let start_time
= Instant
::now();
724 let pre_compiled_cgus
= par_map(cgus
, |(i
, _
)| {
725 let module
= backend
.compile_codegen_unit(tcx
, codegen_units
[i
].name());
729 total_codegen_time
+= start_time
.elapsed();
737 for (i
, cgu
) in codegen_units
.iter().enumerate() {
738 ongoing_codegen
.wait_for_signal_to_codegen_item();
739 ongoing_codegen
.check_for_errors(tcx
.sess
);
741 let cgu_reuse
= cgu_reuse
[i
];
745 let (module
, cost
) = if let Some(cgu
) = pre_compiled_cgus
.remove(&i
) {
748 let start_time
= Instant
::now();
749 let module
= backend
.compile_codegen_unit(tcx
, cgu
.name());
750 total_codegen_time
+= start_time
.elapsed();
753 // This will unwind if there are errors, which triggers our `AbortCodegenOnDrop`
754 // guard. Unfortunately, just skipping the `submit_codegened_module_to_llvm` makes
755 // compilation hang on post-monomorphization errors.
756 tcx
.dcx().abort_if_errors();
758 submit_codegened_module_to_llvm(
760 &ongoing_codegen
.coordinator
.sender
,
765 CguReuse
::PreLto
=> {
766 submit_pre_lto_module_to_llvm(
769 &ongoing_codegen
.coordinator
.sender
,
770 CachedModuleCodegen
{
771 name
: cgu
.name().to_string(),
772 source
: cgu
.previous_work_product(tcx
),
776 CguReuse
::PostLto
=> {
777 submit_post_lto_module_to_llvm(
779 &ongoing_codegen
.coordinator
.sender
,
780 CachedModuleCodegen
{
781 name
: cgu
.name().to_string(),
782 source
: cgu
.previous_work_product(tcx
),
789 ongoing_codegen
.codegen_finished(tcx
);
791 // Since the main thread is sometimes blocked during codegen, we keep track
792 // -Ztime-passes output manually.
793 if tcx
.sess
.opts
.unstable_opts
.time_passes
{
794 let end_rss
= get_resident_set_size();
796 print_time_passes_entry(
797 "codegen_to_LLVM_IR",
801 tcx
.sess
.opts
.unstable_opts
.time_passes_format
,
805 ongoing_codegen
.check_for_errors(tcx
.sess
);
810 pub fn new(tcx
: TyCtxt
<'_
>, target_cpu
: String
) -> CrateInfo
{
811 let crate_types
= tcx
.crate_types().to_vec();
812 let exported_symbols
= crate_types
814 .map(|&c
| (c
, crate::back
::linker
::exported_symbols(tcx
, c
)))
817 crate_types
.iter().map(|&c
| (c
, crate::back
::linker
::linked_symbols(tcx
, c
))).collect();
818 let local_crate_name
= tcx
.crate_name(LOCAL_CRATE
);
819 let crate_attrs
= tcx
.hir().attrs(rustc_hir
::CRATE_HIR_ID
);
820 let subsystem
= attr
::first_attr_value_str_by_name(crate_attrs
, sym
::windows_subsystem
);
821 let windows_subsystem
= subsystem
.map(|subsystem
| {
822 if subsystem
!= sym
::windows
&& subsystem
!= sym
::console
{
823 tcx
.dcx().emit_fatal(errors
::InvalidWindowsSubsystem { subsystem }
);
825 subsystem
.to_string()
828 // This list is used when generating the command line to pass through to
829 // system linker. The linker expects undefined symbols on the left of the
830 // command line to be defined in libraries on the right, not the other way
831 // around. For more info, see some comments in the add_used_library function
834 // In order to get this left-to-right dependency ordering, we use the reverse
835 // postorder of all crates putting the leaves at the right-most positions.
836 let mut compiler_builtins
= None
;
837 let mut used_crates
: Vec
<_
> = tcx
843 let link
= !tcx
.dep_kind(cnum
).macros_only();
844 if link
&& tcx
.is_compiler_builtins(cnum
) {
845 compiler_builtins
= Some(cnum
);
851 // `compiler_builtins` are always placed last to ensure that they're linked correctly.
852 used_crates
.extend(compiler_builtins
);
854 let mut info
= CrateInfo
{
861 profiler_runtime
: None
,
862 is_no_builtins
: Default
::default(),
863 native_libraries
: Default
::default(),
864 used_libraries
: tcx
.native_libraries(LOCAL_CRATE
).iter().map(Into
::into
).collect(),
865 crate_name
: Default
::default(),
867 used_crate_source
: Default
::default(),
868 dependency_formats
: tcx
.dependency_formats(()).clone(),
870 natvis_debugger_visualizers
: Default
::default(),
872 let crates
= tcx
.crates(());
874 let n_crates
= crates
.len();
875 info
.native_libraries
.reserve(n_crates
);
876 info
.crate_name
.reserve(n_crates
);
877 info
.used_crate_source
.reserve(n_crates
);
879 for &cnum
in crates
.iter() {
880 info
.native_libraries
881 .insert(cnum
, tcx
.native_libraries(cnum
).iter().map(Into
::into
).collect());
882 info
.crate_name
.insert(cnum
, tcx
.crate_name(cnum
));
884 let used_crate_source
= tcx
.used_crate_source(cnum
);
885 info
.used_crate_source
.insert(cnum
, used_crate_source
.clone());
886 if tcx
.is_profiler_runtime(cnum
) {
887 info
.profiler_runtime
= Some(cnum
);
889 if tcx
.is_no_builtins(cnum
) {
890 info
.is_no_builtins
.insert(cnum
);
894 // Handle circular dependencies in the standard library.
895 // See comment before `add_linked_symbol_object` function for the details.
896 // If global LTO is enabled then almost everything (*) is glued into a single object file,
897 // so this logic is not necessary and can cause issues on some targets (due to weak lang
898 // item symbols being "privatized" to that object file), so we disable it.
899 // (*) Native libs, and `#[compiler_builtins]` and `#[no_builtins]` crates are not glued,
900 // and we assume that they cannot define weak lang items. This is not currently enforced
901 // by the compiler, but that's ok because all this stuff is unstable anyway.
902 let target
= &tcx
.sess
.target
;
903 if !are_upstream_rust_objects_already_included(tcx
.sess
) {
904 let missing_weak_lang_items
: FxHashSet
<Symbol
> = info
907 .flat_map(|&cnum
| tcx
.missing_lang_items(cnum
))
908 .filter(|l
| l
.is_weak())
910 let name
= l
.link_name()?
;
911 lang_items
::required(tcx
, l
).then_some(name
)
914 let prefix
= if target
.is_like_windows
&& target
.arch
== "x86" { "_" }
else { "" }
;
916 // This loop only adds new items to values of the hash map, so the order in which we
917 // iterate over the values is not important.
918 #[allow(rustc::potential_query_instability)]
921 .filter(|(crate_type
, _
)| {
922 !matches
!(crate_type
, CrateType
::Rlib
| CrateType
::Staticlib
)
924 .for_each(|(_
, linked_symbols
)| {
925 let mut symbols
= missing_weak_lang_items
927 .map(|item
| (format
!("{prefix}{item}"), SymbolExportKind
::Text
))
928 .collect
::<Vec
<_
>>();
929 symbols
.sort_unstable_by(|a
, b
| a
.0.cmp(&b
.0));
930 linked_symbols
.extend(symbols
);
931 if tcx
.allocator_kind(()).is_some() {
932 // At least one crate needs a global allocator. This crate may be placed
933 // after the crate that defines it in the linker order, in which case some
934 // linkers return an error. By adding the global allocator shim methods to
935 // the linked_symbols list, linking the generated symbols.o will ensure that
936 // circular dependencies involving the global allocator don't lead to linker
938 linked_symbols
.extend(ALLOCATOR_METHODS
.iter().map(|method
| {
940 format
!("{prefix}{}", global_fn_name(method
.name
).as_str()),
941 SymbolExportKind
::Text
,
948 let embed_visualizers
= tcx
.crate_types().iter().any(|&crate_type
| match crate_type
{
949 CrateType
::Executable
| CrateType
::Dylib
| CrateType
::Cdylib
=> {
950 // These are crate types for which we invoke the linker and can embed
951 // NatVis visualizers.
954 CrateType
::ProcMacro
=> {
955 // We could embed NatVis for proc macro crates too (to improve the debugging
956 // experience for them) but it does not seem like a good default, since
957 // this is a rare use case and we don't want to slow down the common case.
960 CrateType
::Staticlib
| CrateType
::Rlib
=> {
961 // We don't invoke the linker for these, so we don't need to collect the NatVis for them.
966 if target
.is_like_msvc
&& embed_visualizers
{
967 info
.natvis_debugger_visualizers
=
968 collect_debugger_visualizers_transitive(tcx
, DebuggerVisualizerType
::Natvis
);
975 pub fn provide(providers
: &mut Providers
) {
976 providers
.backend_optimization_level
= |tcx
, cratenum
| {
977 let for_speed
= match tcx
.sess
.opts
.optimize
{
978 // If globally no optimisation is done, #[optimize] has no effect.
980 // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the
981 // pass manager and it is likely that some module-wide passes (such as inliner or
982 // cross-function constant propagation) would ignore the `optnone` annotation we put
983 // on the functions, thus necessarily involving these functions into optimisations.
984 config
::OptLevel
::No
=> return config
::OptLevel
::No
,
985 // If globally optimise-speed is already specified, just use that level.
986 config
::OptLevel
::Less
=> return config
::OptLevel
::Less
,
987 config
::OptLevel
::Default
=> return config
::OptLevel
::Default
,
988 config
::OptLevel
::Aggressive
=> return config
::OptLevel
::Aggressive
,
989 // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size)
991 config
::OptLevel
::Size
=> config
::OptLevel
::Default
,
992 config
::OptLevel
::SizeMin
=> config
::OptLevel
::Default
,
995 let (defids
, _
) = tcx
.collect_and_partition_mono_items(cratenum
);
997 let any_for_speed
= defids
.items().any(|id
| {
998 let CodegenFnAttrs { optimize, .. }
= tcx
.codegen_fn_attrs(*id
);
1000 attr
::OptimizeAttr
::None
| attr
::OptimizeAttr
::Size
=> false,
1001 attr
::OptimizeAttr
::Speed
=> true,
1009 tcx
.sess
.opts
.optimize
1013 pub fn determine_cgu_reuse
<'tcx
>(tcx
: TyCtxt
<'tcx
>, cgu
: &CodegenUnit
<'tcx
>) -> CguReuse
{
1014 if !tcx
.dep_graph
.is_fully_enabled() {
1015 return CguReuse
::No
;
1018 let work_product_id
= &cgu
.work_product_id();
1019 if tcx
.dep_graph
.previous_work_product(work_product_id
).is_none() {
1020 // We don't have anything cached for this CGU. This can happen
1021 // if the CGU did not exist in the previous session.
1022 return CguReuse
::No
;
1025 // Try to mark the CGU as green. If it we can do so, it means that nothing
1026 // affecting the LLVM module has changed and we can re-use a cached version.
1027 // If we compile with any kind of LTO, this means we can re-use the bitcode
1028 // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only
1029 // know that later). If we are not doing LTO, there is only one optimized
1030 // version of each module, so we re-use that.
1031 let dep_node
= cgu
.codegen_dep_node(tcx
);
1033 !tcx
.dep_graph
.dep_node_exists(&dep_node
),
1034 "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",
1038 if tcx
.try_mark_green(&dep_node
) {
1039 // We can re-use either the pre- or the post-thinlto state. If no LTO is
1040 // being performed then we can use post-LTO artifacts, otherwise we must
1041 // reuse pre-LTO artifacts
1042 match compute_per_cgu_lto_type(
1046 ModuleKind
::Regular
,
1048 ComputedLtoType
::No
=> CguReuse
::PostLto
,
1049 _
=> CguReuse
::PreLto
,