1 use crate::back
::link
::are_upstream_rust_objects_already_included
;
2 use crate::back
::metadata
::create_compressed_metadata_file
;
3 use crate::back
::write
::{
4 compute_per_cgu_lto_type
, start_async_codegen
, submit_codegened_module_to_llvm
,
5 submit_post_lto_module_to_llvm
, submit_pre_lto_module_to_llvm
, ComputedLtoType
, OngoingCodegen
,
7 use crate::common
::{IntPredicate, RealPredicate, TypeKind}
;
10 use crate::mir
::operand
::OperandValue
;
11 use crate::mir
::place
::PlaceRef
;
13 use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}
;
15 use rustc_attr
as attr
;
16 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
17 use rustc_data_structures
::profiling
::{get_resident_set_size, print_time_passes_entry}
;
19 use rustc_data_structures
::sync
::par_iter
;
20 #[cfg(parallel_compiler)]
21 use rustc_data_structures
::sync
::ParallelIterator
;
23 use rustc_hir
::def_id
::{DefId, LOCAL_CRATE}
;
24 use rustc_hir
::lang_items
::LangItem
;
25 use rustc_index
::vec
::Idx
;
26 use rustc_metadata
::EncodedMetadata
;
27 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrs
;
28 use rustc_middle
::middle
::exported_symbols
;
29 use rustc_middle
::middle
::exported_symbols
::SymbolExportKind
;
30 use rustc_middle
::middle
::lang_items
;
31 use rustc_middle
::mir
::mono
::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}
;
32 use rustc_middle
::ty
::layout
::{HasTyCtxt, LayoutOf, TyAndLayout}
;
33 use rustc_middle
::ty
::query
::Providers
;
34 use rustc_middle
::ty
::{self, Instance, Ty, TyCtxt}
;
35 use rustc_session
::cgu_reuse_tracker
::CguReuse
;
36 use rustc_session
::config
::{self, CrateType, EntryFnType, OutputType}
;
37 use rustc_session
::Session
;
38 use rustc_span
::symbol
::sym
;
39 use rustc_span
::Symbol
;
40 use rustc_span
::{DebuggerVisualizerFile, DebuggerVisualizerType}
;
41 use rustc_target
::abi
::{Align, Size, VariantIdx}
;
43 use std
::collections
::BTreeSet
;
44 use std
::convert
::TryFrom
;
45 use std
::time
::{Duration, Instant}
;
47 use itertools
::Itertools
;
49 pub fn bin_op_to_icmp_predicate(op
: hir
::BinOpKind
, signed
: bool
) -> IntPredicate
{
51 hir
::BinOpKind
::Eq
=> IntPredicate
::IntEQ
,
52 hir
::BinOpKind
::Ne
=> IntPredicate
::IntNE
,
53 hir
::BinOpKind
::Lt
=> {
60 hir
::BinOpKind
::Le
=> {
67 hir
::BinOpKind
::Gt
=> {
74 hir
::BinOpKind
::Ge
=> {
82 "comparison_op_to_icmp_predicate: expected comparison operator, \
89 pub fn bin_op_to_fcmp_predicate(op
: hir
::BinOpKind
) -> RealPredicate
{
91 hir
::BinOpKind
::Eq
=> RealPredicate
::RealOEQ
,
92 hir
::BinOpKind
::Ne
=> RealPredicate
::RealUNE
,
93 hir
::BinOpKind
::Lt
=> RealPredicate
::RealOLT
,
94 hir
::BinOpKind
::Le
=> RealPredicate
::RealOLE
,
95 hir
::BinOpKind
::Gt
=> RealPredicate
::RealOGT
,
96 hir
::BinOpKind
::Ge
=> RealPredicate
::RealOGE
,
99 "comparison_op_to_fcmp_predicate: expected comparison operator, \
107 pub fn compare_simd_types
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
115 let signed
= match t
.kind() {
117 let cmp
= bin_op_to_fcmp_predicate(op
);
118 let cmp
= bx
.fcmp(cmp
, lhs
, rhs
);
119 return bx
.sext(cmp
, ret_ty
);
121 ty
::Uint(_
) => false,
123 _
=> bug
!("compare_simd_types: invalid SIMD type"),
126 let cmp
= bin_op_to_icmp_predicate(op
, signed
);
127 let cmp
= bx
.icmp(cmp
, lhs
, rhs
);
128 // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension
129 // to get the correctly sized type. This will compile to a single instruction
130 // once the IR is converted to assembly if the SIMD instruction is supported
131 // by the target architecture.
135 /// Retrieves the information we are losing (making dynamic) in an unsizing
138 /// The `old_info` argument is a bit odd. It is intended for use in an upcast,
139 /// where the new vtable for an object will be derived from the old one.
140 pub fn unsized_info
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
144 old_info
: Option
<Bx
::Value
>,
147 let (source
, target
) =
148 cx
.tcx().struct_lockstep_tails_erasing_lifetimes(source
, target
, bx
.param_env());
149 match (source
.kind(), target
.kind()) {
150 (&ty
::Array(_
, len
), &ty
::Slice(_
)) => {
151 cx
.const_usize(len
.eval_usize(cx
.tcx(), ty
::ParamEnv
::reveal_all()))
154 &ty
::Dynamic(ref data_a
, _
, src_dyn_kind
),
155 &ty
::Dynamic(ref data_b
, _
, target_dyn_kind
),
157 assert_eq
!(src_dyn_kind
, target_dyn_kind
);
160 old_info
.expect("unsized_info: missing old info for trait upcasting coercion");
161 if data_a
.principal_def_id() == data_b
.principal_def_id() {
162 // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
166 // trait upcasting coercion
169 cx
.tcx().vtable_trait_upcasting_coercion_new_vptr_slot((source
, target
));
171 if let Some(entry_idx
) = vptr_entry_idx
{
172 let ptr_ty
= cx
.type_i8p();
173 let ptr_align
= cx
.tcx().data_layout
.pointer_align
.abi
;
174 let vtable_ptr_ty
= vtable_ptr_ty(cx
, target
, target_dyn_kind
);
175 let llvtable
= bx
.pointercast(old_info
, bx
.type_ptr_to(ptr_ty
));
176 let gep
= bx
.inbounds_gep(
179 &[bx
.const_usize(u64::try_from(entry_idx
).unwrap())],
181 let new_vptr
= bx
.load(ptr_ty
, gep
, ptr_align
);
182 bx
.nonnull_metadata(new_vptr
);
183 // VTable loads are invariant.
184 bx
.set_invariant_load(new_vptr
);
185 bx
.pointercast(new_vptr
, vtable_ptr_ty
)
190 (_
, &ty
::Dynamic(ref data
, _
, target_dyn_kind
)) => {
191 let vtable_ptr_ty
= vtable_ptr_ty(cx
, target
, target_dyn_kind
);
192 cx
.const_ptrcast(meth
::get_vtable(cx
, source
, data
.principal()), vtable_ptr_ty
)
194 _
=> bug
!("unsized_info: invalid unsizing {:?} -> {:?}", source
, target
),
198 // Returns the vtable pointer type of a `dyn` or `dyn*` type
199 fn vtable_ptr_ty
<'tcx
, Cx
: CodegenMethods
<'tcx
>>(
203 ) -> <Cx
as BackendTypes
>::Type
{
204 cx
.scalar_pair_element_backend_type(
205 cx
.layout_of(match kind
{
206 // vtable is the second field of `*mut dyn Trait`
207 ty
::Dyn
=> cx
.tcx().mk_mut_ptr(target
),
208 // vtable is the second field of `dyn* Trait`
209 ty
::DynStar
=> target
,
216 /// Coerces `src` to `dst_ty`. `src_ty` must be a pointer.
217 pub fn unsize_ptr
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
222 old_info
: Option
<Bx
::Value
>,
223 ) -> (Bx
::Value
, Bx
::Value
) {
224 debug
!("unsize_ptr: {:?} => {:?}", src_ty
, dst_ty
);
225 match (src_ty
.kind(), dst_ty
.kind()) {
226 (&ty
::Ref(_
, a
, _
), &ty
::Ref(_
, b
, _
) | &ty
::RawPtr(ty
::TypeAndMut { ty: b, .. }
))
227 | (&ty
::RawPtr(ty
::TypeAndMut { ty: a, .. }
), &ty
::RawPtr(ty
::TypeAndMut { ty: b, .. }
)) => {
228 assert_eq
!(bx
.cx().type_is_sized(a
), old_info
.is_none());
229 let ptr_ty
= bx
.cx().type_ptr_to(bx
.cx().backend_type(bx
.cx().layout_of(b
)));
230 (bx
.pointercast(src
, ptr_ty
), unsized_info(bx
, a
, b
, old_info
))
232 (&ty
::Adt(def_a
, _
), &ty
::Adt(def_b
, _
)) => {
233 assert_eq
!(def_a
, def_b
);
234 let src_layout
= bx
.cx().layout_of(src_ty
);
235 let dst_layout
= bx
.cx().layout_of(dst_ty
);
236 if src_ty
== dst_ty
{
237 return (src
, old_info
.unwrap());
239 let mut result
= None
;
240 for i
in 0..src_layout
.fields
.count() {
241 let src_f
= src_layout
.field(bx
.cx(), i
);
246 assert_eq
!(src_layout
.fields
.offset(i
).bytes(), 0);
247 assert_eq
!(dst_layout
.fields
.offset(i
).bytes(), 0);
248 assert_eq
!(src_layout
.size
, src_f
.size
);
250 let dst_f
= dst_layout
.field(bx
.cx(), i
);
251 assert_ne
!(src_f
.ty
, dst_f
.ty
);
252 assert_eq
!(result
, None
);
253 result
= Some(unsize_ptr(bx
, src
, src_f
.ty
, dst_f
.ty
, old_info
));
255 let (lldata
, llextra
) = result
.unwrap();
256 let lldata_ty
= bx
.cx().scalar_pair_element_backend_type(dst_layout
, 0, true);
257 let llextra_ty
= bx
.cx().scalar_pair_element_backend_type(dst_layout
, 1, true);
258 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
259 (bx
.bitcast(lldata
, lldata_ty
), bx
.bitcast(llextra
, llextra_ty
))
261 _
=> bug
!("unsize_ptr: called on bad types"),
265 /// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type.
266 pub fn cast_to_dyn_star
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
269 src_ty_and_layout
: TyAndLayout
<'tcx
>,
271 old_info
: Option
<Bx
::Value
>,
272 ) -> (Bx
::Value
, Bx
::Value
) {
273 debug
!("cast_to_dyn_star: {:?} => {:?}", src_ty_and_layout
.ty
, dst_ty
);
275 matches
!(dst_ty
.kind(), ty
::Dynamic(_
, _
, ty
::DynStar
)),
276 "destination type must be a dyn*"
278 // FIXME(dyn-star): this is probably not the best way to check if this is
279 // a pointer, and really we should ensure that the value is a suitable
280 // pointer earlier in the compilation process.
281 let src
= match src_ty_and_layout
.pointee_info_at(bx
.cx(), Size
::ZERO
) {
282 Some(_
) => bx
.ptrtoint(src
, bx
.cx().type_isize()),
283 None
=> bx
.bitcast(src
, bx
.type_isize()),
285 (src
, unsized_info(bx
, src_ty_and_layout
.ty
, dst_ty
, old_info
))
288 /// Coerces `src`, which is a reference to a value of type `src_ty`,
289 /// to a value of type `dst_ty`, and stores the result in `dst`.
290 pub fn coerce_unsized_into
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
292 src
: PlaceRef
<'tcx
, Bx
::Value
>,
293 dst
: PlaceRef
<'tcx
, Bx
::Value
>,
295 let src_ty
= src
.layout
.ty
;
296 let dst_ty
= dst
.layout
.ty
;
297 match (src_ty
.kind(), dst_ty
.kind()) {
298 (&ty
::Ref(..), &ty
::Ref(..) | &ty
::RawPtr(..)) | (&ty
::RawPtr(..), &ty
::RawPtr(..)) => {
299 let (base
, info
) = match bx
.load_operand(src
).val
{
300 OperandValue
::Pair(base
, info
) => unsize_ptr(bx
, base
, src_ty
, dst_ty
, Some(info
)),
301 OperandValue
::Immediate(base
) => unsize_ptr(bx
, base
, src_ty
, dst_ty
, None
),
302 OperandValue
::Ref(..) => bug
!(),
304 OperandValue
::Pair(base
, info
).store(bx
, dst
);
307 (&ty
::Adt(def_a
, _
), &ty
::Adt(def_b
, _
)) => {
308 assert_eq
!(def_a
, def_b
);
310 for i
in 0..def_a
.variant(VariantIdx
::new(0)).fields
.len() {
311 let src_f
= src
.project_field(bx
, i
);
312 let dst_f
= dst
.project_field(bx
, i
);
314 if dst_f
.layout
.is_zst() {
318 if src_f
.layout
.ty
== dst_f
.layout
.ty
{
329 coerce_unsized_into(bx
, src_f
, dst_f
);
333 _
=> bug
!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty
, dst_ty
,),
337 pub fn cast_shift_expr_rhs
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
342 // Shifts may have any size int on the rhs
343 let mut rhs_llty
= bx
.cx().val_ty(rhs
);
344 let mut lhs_llty
= bx
.cx().val_ty(lhs
);
345 if bx
.cx().type_kind(rhs_llty
) == TypeKind
::Vector
{
346 rhs_llty
= bx
.cx().element_type(rhs_llty
)
348 if bx
.cx().type_kind(lhs_llty
) == TypeKind
::Vector
{
349 lhs_llty
= bx
.cx().element_type(lhs_llty
)
351 let rhs_sz
= bx
.cx().int_width(rhs_llty
);
352 let lhs_sz
= bx
.cx().int_width(lhs_llty
);
354 bx
.trunc(rhs
, lhs_llty
)
355 } else if lhs_sz
> rhs_sz
{
356 // FIXME (#1877: If in the future shifting by negative
357 // values is no longer undefined then this is wrong.
358 bx
.zext(rhs
, lhs_llty
)
364 /// Returns `true` if this session's target will use SEH-based unwinding.
366 /// This is only true for MSVC targets, and even then the 64-bit MSVC target
367 /// currently uses SEH-ish unwinding with DWARF info tables to the side (same as
368 /// 64-bit MinGW) instead of "full SEH".
369 pub fn wants_msvc_seh(sess
: &Session
) -> bool
{
370 sess
.target
.is_like_msvc
373 pub fn memcpy_ty
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
379 layout
: TyAndLayout
<'tcx
>,
382 let size
= layout
.size
.bytes();
387 bx
.memcpy(dst
, dst_align
, src
, src_align
, bx
.cx().const_usize(size
), flags
);
390 pub fn codegen_instance
<'a
, 'tcx
: 'a
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
391 cx
: &'a Bx
::CodegenCx
,
392 instance
: Instance
<'tcx
>,
394 // this is an info! to allow collecting monomorphization statistics
395 // and to allow finding the last function before LLVM aborts from
397 info
!("codegen_instance({})", instance
);
399 mir
::codegen_mir
::<Bx
>(cx
, instance
);
402 /// Creates the `main` function which will initialize the rust runtime and call
403 /// users main function.
404 pub fn maybe_create_entry_wrapper
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
405 cx
: &'a Bx
::CodegenCx
,
406 ) -> Option
<Bx
::Function
> {
407 let (main_def_id
, entry_type
) = cx
.tcx().entry_fn(())?
;
408 let main_is_local
= main_def_id
.is_local();
409 let instance
= Instance
::mono(cx
.tcx(), main_def_id
);
412 // We want to create the wrapper in the same codegen unit as Rust's main
414 if !cx
.codegen_unit().contains_item(&MonoItem
::Fn(instance
)) {
417 } else if !cx
.codegen_unit().is_primary() {
418 // We want to create the wrapper only when the codegen unit is the primary one
422 let main_llfn
= cx
.get_fn_addr(instance
);
424 let entry_fn
= create_entry_fn
::<Bx
>(cx
, main_llfn
, main_def_id
, entry_type
);
425 return Some(entry_fn
);
427 fn create_entry_fn
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
428 cx
: &'a Bx
::CodegenCx
,
429 rust_main
: Bx
::Value
,
430 rust_main_def_id
: DefId
,
431 entry_type
: EntryFnType
,
433 // The entry function is either `int main(void)` or `int main(int argc, char **argv)`,
434 // depending on whether the target needs `argc` and `argv` to be passed in.
435 let llfty
= if cx
.sess().target
.main_needs_argc_argv
{
436 cx
.type_func(&[cx
.type_int(), cx
.type_ptr_to(cx
.type_i8p())], cx
.type_int())
438 cx
.type_func(&[], cx
.type_int())
441 let main_ret_ty
= cx
.tcx().fn_sig(rust_main_def_id
).output();
442 // Given that `main()` has no arguments,
443 // then its return type cannot have
444 // late-bound regions, since late-bound
445 // regions must appear in the argument
447 let main_ret_ty
= cx
.tcx().normalize_erasing_regions(
448 ty
::ParamEnv
::reveal_all(),
449 main_ret_ty
.no_bound_vars().unwrap(),
452 let Some(llfn
) = cx
.declare_c_main(llfty
) else {
453 // FIXME: We should be smart and show a better diagnostic here.
454 let span
= cx
.tcx().def_span(rust_main_def_id
);
456 .struct_span_err(span
, "entry symbol `main` declared multiple times")
457 .help("did you use `#[no_mangle]` on `fn main`? Use `#[start]` instead")
459 cx
.sess().abort_if_errors();
463 // `main` should respect same config for frame pointer elimination as rest of code
464 cx
.set_frame_pointer_type(llfn
);
465 cx
.apply_target_cpu_attr(llfn
);
467 let llbb
= Bx
::append_block(&cx
, llfn
, "top");
468 let mut bx
= Bx
::build(&cx
, llbb
);
470 bx
.insert_reference_to_gdb_debug_scripts_section_global();
472 let isize_ty
= cx
.type_isize();
473 let i8pp_ty
= cx
.type_ptr_to(cx
.type_i8p());
474 let (arg_argc
, arg_argv
) = get_argc_argv(cx
, &mut bx
);
476 let (start_fn
, start_ty
, args
) = if let EntryFnType
::Main { sigpipe }
= entry_type
{
477 let start_def_id
= cx
.tcx().require_lang_item(LangItem
::Start
, None
);
478 let start_fn
= cx
.get_fn_addr(
479 ty
::Instance
::resolve(
481 ty
::ParamEnv
::reveal_all(),
483 cx
.tcx().intern_substs(&[main_ret_ty
.into()]),
489 let i8_ty
= cx
.type_i8();
490 let arg_sigpipe
= bx
.const_u8(sigpipe
);
493 cx
.type_func(&[cx
.val_ty(rust_main
), isize_ty
, i8pp_ty
, i8_ty
], isize_ty
);
494 (start_fn
, start_ty
, vec
![rust_main
, arg_argc
, arg_argv
, arg_sigpipe
])
496 debug
!("using user-defined start fn");
497 let start_ty
= cx
.type_func(&[isize_ty
, i8pp_ty
], isize_ty
);
498 (rust_main
, start_ty
, vec
![arg_argc
, arg_argv
])
501 let result
= bx
.call(start_ty
, None
, start_fn
, &args
, None
);
502 let cast
= bx
.intcast(result
, cx
.type_int(), true);
509 /// Obtain the `argc` and `argv` values to pass to the rust start function.
510 fn get_argc_argv
<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
511 cx
: &'a Bx
::CodegenCx
,
513 ) -> (Bx
::Value
, Bx
::Value
) {
514 if cx
.sess().target
.main_needs_argc_argv
{
515 // Params from native `main()` used as args for rust start function
516 let param_argc
= bx
.get_param(0);
517 let param_argv
= bx
.get_param(1);
518 let arg_argc
= bx
.intcast(param_argc
, cx
.type_isize(), true);
519 let arg_argv
= param_argv
;
522 // The Rust start function doesn't need `argc` and `argv`, so just pass zeros.
523 let arg_argc
= bx
.const_int(cx
.type_int(), 0);
524 let arg_argv
= bx
.const_null(cx
.type_ptr_to(cx
.type_i8p()));
529 /// This function returns all of the debugger visualizers specified for the
530 /// current crate as well as all upstream crates transitively that match the
531 /// `visualizer_type` specified.
532 pub fn collect_debugger_visualizers_transitive(
534 visualizer_type
: DebuggerVisualizerType
,
535 ) -> BTreeSet
<DebuggerVisualizerFile
> {
536 tcx
.debugger_visualizers(LOCAL_CRATE
)
542 let used_crate_source
= tcx
.used_crate_source(*cnum
);
543 used_crate_source
.rlib
.is_some() || used_crate_source
.rmeta
.is_some()
545 .flat_map(|&cnum
| tcx
.debugger_visualizers(cnum
)),
547 .filter(|visualizer
| visualizer
.visualizer_type
== visualizer_type
)
549 .collect
::<BTreeSet
<_
>>()
552 pub fn codegen_crate
<B
: ExtraBackendMethods
>(
556 metadata
: EncodedMetadata
,
557 need_metadata_module
: bool
,
558 ) -> OngoingCodegen
<B
> {
559 // Skip crate items and just output metadata in -Z no-codegen mode.
560 if tcx
.sess
.opts
.unstable_opts
.no_codegen
|| !tcx
.sess
.opts
.output_types
.should_codegen() {
561 let ongoing_codegen
= start_async_codegen(backend
, tcx
, target_cpu
, metadata
, None
, 1);
563 ongoing_codegen
.codegen_finished(tcx
);
565 ongoing_codegen
.check_for_errors(tcx
.sess
);
567 return ongoing_codegen
;
570 let cgu_name_builder
= &mut CodegenUnitNameBuilder
::new(tcx
);
572 // Run the monomorphization collector and partition the collected items into
574 let codegen_units
= tcx
.collect_and_partition_mono_items(()).1;
576 // Force all codegen_unit queries so they are already either red or green
577 // when compile_codegen_unit accesses them. We are not able to re-execute
578 // the codegen_unit query from just the DepNode, so an unknown color would
579 // lead to having to re-execute compile_codegen_unit, possibly
581 if tcx
.dep_graph
.is_fully_enabled() {
582 for cgu
in codegen_units
{
583 tcx
.ensure().codegen_unit(cgu
.name());
587 let metadata_module
= if need_metadata_module
{
588 // Emit compressed metadata object.
589 let metadata_cgu_name
=
590 cgu_name_builder
.build_cgu_name(LOCAL_CRATE
, &["crate"], Some("metadata")).to_string();
591 tcx
.sess
.time("write_compressed_metadata", || {
593 tcx
.output_filenames(()).temp_path(OutputType
::Metadata
, Some(&metadata_cgu_name
));
594 let data
= create_compressed_metadata_file(
597 &exported_symbols
::metadata_symbol_name(tcx
),
599 if let Err(err
) = std
::fs
::write(&file_name
, data
) {
600 tcx
.sess
.fatal(&format
!("error writing metadata object file: {}", err
));
602 Some(CompiledModule
{
603 name
: metadata_cgu_name
,
604 kind
: ModuleKind
::Metadata
,
605 object
: Some(file_name
),
614 let ongoing_codegen
= start_async_codegen(
623 // Codegen an allocator shim, if necessary.
625 // If the crate doesn't have an `allocator_kind` set then there's definitely
626 // no shim to generate. Otherwise we also check our dependency graph for all
627 // our output crate types. If anything there looks like its a `Dynamic`
628 // linkage, then it's already got an allocator shim and we'll be using that
629 // one instead. If nothing exists then it's our job to generate the
631 let any_dynamic_crate
= tcx
.dependency_formats(()).iter().any(|(_
, list
)| {
632 use rustc_middle
::middle
::dependency_format
::Linkage
;
633 list
.iter().any(|&linkage
| linkage
== Linkage
::Dynamic
)
635 let allocator_module
= if any_dynamic_crate
{
637 } else if let Some(kind
) = tcx
.allocator_kind(()) {
639 cgu_name_builder
.build_cgu_name(LOCAL_CRATE
, &["crate"], Some("allocator")).to_string();
640 let module_llvm
= tcx
.sess
.time("write_allocator_module", || {
641 backend
.codegen_allocator(
645 // If allocator_kind is Some then alloc_error_handler_kind must
647 tcx
.alloc_error_handler_kind(()).unwrap(),
651 Some(ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator }
)
656 if let Some(allocator_module
) = allocator_module
{
657 ongoing_codegen
.submit_pre_codegened_module_to_llvm(tcx
, allocator_module
);
660 // For better throughput during parallel processing by LLVM, we used to sort
661 // CGUs largest to smallest. This would lead to better thread utilization
662 // by, for example, preventing a large CGU from being processed last and
663 // having only one LLVM thread working while the rest remained idle.
665 // However, this strategy would lead to high memory usage, as it meant the
666 // LLVM-IR for all of the largest CGUs would be resident in memory at once.
668 // Instead, we can compromise by ordering CGUs such that the largest and
669 // smallest are first, second largest and smallest are next, etc. If there
670 // are large size variations, this can reduce memory usage significantly.
671 let codegen_units
: Vec
<_
> = {
672 let mut sorted_cgus
= codegen_units
.iter().collect
::<Vec
<_
>>();
673 sorted_cgus
.sort_by_cached_key(|cgu
| cgu
.size_estimate());
675 let (first_half
, second_half
) = sorted_cgus
.split_at(sorted_cgus
.len() / 2);
676 second_half
.iter().rev().interleave(first_half
).copied().collect()
679 // Calculate the CGU reuse
680 let cgu_reuse
= tcx
.sess
.time("find_cgu_reuse", || {
681 codegen_units
.iter().map(|cgu
| determine_cgu_reuse(tcx
, &cgu
)).collect
::<Vec
<_
>>()
684 let mut total_codegen_time
= Duration
::new(0, 0);
685 let start_rss
= tcx
.sess
.time_passes().then(|| get_resident_set_size());
687 // The non-parallel compiler can only translate codegen units to LLVM IR
688 // on a single thread, leading to a staircase effect where the N LLVM
689 // threads have to wait on the single codegen threads to generate work
690 // for them. The parallel compiler does not have this restriction, so
691 // we can pre-load the LLVM queue in parallel before handing off
692 // coordination to the OnGoingCodegen scheduler.
694 // This likely is a temporary measure. Once we don't have to support the
695 // non-parallel compiler anymore, we can compile CGUs end-to-end in
696 // parallel and get rid of the complicated scheduling logic.
697 let mut pre_compiled_cgus
= if cfg
!(parallel_compiler
) {
698 tcx
.sess
.time("compile_first_CGU_batch", || {
699 // Try to find one CGU to compile per thread.
700 let cgus
: Vec
<_
> = cgu_reuse
703 .filter(|&(_
, reuse
)| reuse
== &CguReuse
::No
)
704 .take(tcx
.sess
.threads())
707 // Compile the found CGUs in parallel.
708 let start_time
= Instant
::now();
710 let pre_compiled_cgus
= par_iter(cgus
)
712 let module
= backend
.compile_codegen_unit(tcx
, codegen_units
[i
].name());
717 total_codegen_time
+= start_time
.elapsed();
725 for (i
, cgu
) in codegen_units
.iter().enumerate() {
726 ongoing_codegen
.wait_for_signal_to_codegen_item();
727 ongoing_codegen
.check_for_errors(tcx
.sess
);
729 let cgu_reuse
= cgu_reuse
[i
];
730 tcx
.sess
.cgu_reuse_tracker
.set_actual_reuse(cgu
.name().as_str(), cgu_reuse
);
734 let (module
, cost
) = if let Some(cgu
) = pre_compiled_cgus
.remove(&i
) {
737 let start_time
= Instant
::now();
738 let module
= backend
.compile_codegen_unit(tcx
, cgu
.name());
739 total_codegen_time
+= start_time
.elapsed();
742 // This will unwind if there are errors, which triggers our `AbortCodegenOnDrop`
743 // guard. Unfortunately, just skipping the `submit_codegened_module_to_llvm` makes
744 // compilation hang on post-monomorphization errors.
745 tcx
.sess
.abort_if_errors();
747 submit_codegened_module_to_llvm(
749 &ongoing_codegen
.coordinator
.sender
,
755 CguReuse
::PreLto
=> {
756 submit_pre_lto_module_to_llvm(
759 &ongoing_codegen
.coordinator
.sender
,
760 CachedModuleCodegen
{
761 name
: cgu
.name().to_string(),
762 source
: cgu
.previous_work_product(tcx
),
767 CguReuse
::PostLto
=> {
768 submit_post_lto_module_to_llvm(
770 &ongoing_codegen
.coordinator
.sender
,
771 CachedModuleCodegen
{
772 name
: cgu
.name().to_string(),
773 source
: cgu
.previous_work_product(tcx
),
781 ongoing_codegen
.codegen_finished(tcx
);
783 // Since the main thread is sometimes blocked during codegen, we keep track
784 // -Ztime-passes output manually.
785 if tcx
.sess
.time_passes() {
786 let end_rss
= get_resident_set_size();
788 print_time_passes_entry(
789 "codegen_to_LLVM_IR",
796 ongoing_codegen
.check_for_errors(tcx
.sess
);
801 pub fn new(tcx
: TyCtxt
<'_
>, target_cpu
: String
) -> CrateInfo
{
802 let exported_symbols
= tcx
806 .map(|&c
| (c
, crate::back
::linker
::exported_symbols(tcx
, c
)))
808 let linked_symbols
= tcx
812 .map(|&c
| (c
, crate::back
::linker
::linked_symbols(tcx
, c
)))
814 let local_crate_name
= tcx
.crate_name(LOCAL_CRATE
);
815 let crate_attrs
= tcx
.hir().attrs(rustc_hir
::CRATE_HIR_ID
);
816 let subsystem
= tcx
.sess
.first_attr_value_str_by_name(crate_attrs
, sym
::windows_subsystem
);
817 let windows_subsystem
= subsystem
.map(|subsystem
| {
818 if subsystem
!= sym
::windows
&& subsystem
!= sym
::console
{
819 tcx
.sess
.fatal(&format
!(
820 "invalid windows subsystem `{}`, only \
821 `windows` and `console` are allowed",
825 subsystem
.to_string()
828 // This list is used when generating the command line to pass through to
829 // system linker. The linker expects undefined symbols on the left of the
830 // command line to be defined in libraries on the right, not the other way
831 // around. For more info, see some comments in the add_used_library function
834 // In order to get this left-to-right dependency ordering, we use the reverse
835 // postorder of all crates putting the leaves at the right-most positions.
836 let mut compiler_builtins
= None
;
837 let mut used_crates
: Vec
<_
> = tcx
843 let link
= !tcx
.dep_kind(cnum
).macros_only();
844 if link
&& tcx
.is_compiler_builtins(cnum
) {
845 compiler_builtins
= Some(cnum
);
851 // `compiler_builtins` are always placed last to ensure that they're linked correctly.
852 used_crates
.extend(compiler_builtins
);
854 let mut info
= CrateInfo
{
860 profiler_runtime
: None
,
861 is_no_builtins
: Default
::default(),
862 native_libraries
: Default
::default(),
863 used_libraries
: tcx
.native_libraries(LOCAL_CRATE
).iter().map(Into
::into
).collect(),
864 crate_name
: Default
::default(),
866 used_crate_source
: Default
::default(),
867 dependency_formats
: tcx
.dependency_formats(()).clone(),
869 natvis_debugger_visualizers
: Default
::default(),
871 let crates
= tcx
.crates(());
873 let n_crates
= crates
.len();
874 info
.native_libraries
.reserve(n_crates
);
875 info
.crate_name
.reserve(n_crates
);
876 info
.used_crate_source
.reserve(n_crates
);
878 for &cnum
in crates
.iter() {
879 info
.native_libraries
880 .insert(cnum
, tcx
.native_libraries(cnum
).iter().map(Into
::into
).collect());
881 info
.crate_name
.insert(cnum
, tcx
.crate_name(cnum
));
883 let used_crate_source
= tcx
.used_crate_source(cnum
);
884 info
.used_crate_source
.insert(cnum
, used_crate_source
.clone());
885 if tcx
.is_profiler_runtime(cnum
) {
886 info
.profiler_runtime
= Some(cnum
);
888 if tcx
.is_no_builtins(cnum
) {
889 info
.is_no_builtins
.insert(cnum
);
893 // Handle circular dependencies in the standard library.
894 // See comment before `add_linked_symbol_object` function for the details.
895 // If global LTO is enabled then almost everything (*) is glued into a single object file,
896 // so this logic is not necessary and can cause issues on some targets (due to weak lang
897 // item symbols being "privatized" to that object file), so we disable it.
898 // (*) Native libs, and `#[compiler_builtins]` and `#[no_builtins]` crates are not glued,
899 // and we assume that they cannot define weak lang items. This is not currently enforced
900 // by the compiler, but that's ok because all this stuff is unstable anyway.
901 let target
= &tcx
.sess
.target
;
902 if !are_upstream_rust_objects_already_included(tcx
.sess
) {
903 let missing_weak_lang_items
: FxHashSet
<Symbol
> = info
906 .flat_map(|&cnum
| tcx
.missing_lang_items(cnum
))
907 .filter(|l
| l
.is_weak())
909 let name
= l
.link_name()?
;
910 lang_items
::required(tcx
, l
).then_some(name
)
913 let prefix
= if target
.is_like_windows
&& target
.arch
== "x86" { "_" }
else { "" }
;
916 .filter(|(crate_type
, _
)| {
917 !matches
!(crate_type
, CrateType
::Rlib
| CrateType
::Staticlib
)
919 .for_each(|(_
, linked_symbols
)| {
920 linked_symbols
.extend(
921 missing_weak_lang_items
923 .map(|item
| (format
!("{prefix}{item}"), SymbolExportKind
::Text
)),
928 let embed_visualizers
= tcx
.sess
.crate_types().iter().any(|&crate_type
| match crate_type
{
929 CrateType
::Executable
| CrateType
::Dylib
| CrateType
::Cdylib
=> {
930 // These are crate types for which we invoke the linker and can embed
931 // NatVis visualizers.
934 CrateType
::ProcMacro
=> {
935 // We could embed NatVis for proc macro crates too (to improve the debugging
936 // experience for them) but it does not seem like a good default, since
937 // this is a rare use case and we don't want to slow down the common case.
940 CrateType
::Staticlib
| CrateType
::Rlib
=> {
941 // We don't invoke the linker for these, so we don't need to collect the NatVis for them.
946 if target
.is_like_msvc
&& embed_visualizers
{
947 info
.natvis_debugger_visualizers
=
948 collect_debugger_visualizers_transitive(tcx
, DebuggerVisualizerType
::Natvis
);
955 pub fn provide(providers
: &mut Providers
) {
956 providers
.backend_optimization_level
= |tcx
, cratenum
| {
957 let for_speed
= match tcx
.sess
.opts
.optimize
{
958 // If globally no optimisation is done, #[optimize] has no effect.
960 // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the
961 // pass manager and it is likely that some module-wide passes (such as inliner or
962 // cross-function constant propagation) would ignore the `optnone` annotation we put
963 // on the functions, thus necessarily involving these functions into optimisations.
964 config
::OptLevel
::No
=> return config
::OptLevel
::No
,
965 // If globally optimise-speed is already specified, just use that level.
966 config
::OptLevel
::Less
=> return config
::OptLevel
::Less
,
967 config
::OptLevel
::Default
=> return config
::OptLevel
::Default
,
968 config
::OptLevel
::Aggressive
=> return config
::OptLevel
::Aggressive
,
969 // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size)
971 config
::OptLevel
::Size
=> config
::OptLevel
::Default
,
972 config
::OptLevel
::SizeMin
=> config
::OptLevel
::Default
,
975 let (defids
, _
) = tcx
.collect_and_partition_mono_items(cratenum
);
977 let CodegenFnAttrs { optimize, .. }
= tcx
.codegen_fn_attrs(*id
);
979 attr
::OptimizeAttr
::None
=> continue,
980 attr
::OptimizeAttr
::Size
=> continue,
981 attr
::OptimizeAttr
::Speed
=> {
986 tcx
.sess
.opts
.optimize
990 fn determine_cgu_reuse
<'tcx
>(tcx
: TyCtxt
<'tcx
>, cgu
: &CodegenUnit
<'tcx
>) -> CguReuse
{
991 if !tcx
.dep_graph
.is_fully_enabled() {
995 let work_product_id
= &cgu
.work_product_id();
996 if tcx
.dep_graph
.previous_work_product(work_product_id
).is_none() {
997 // We don't have anything cached for this CGU. This can happen
998 // if the CGU did not exist in the previous session.
1002 // Try to mark the CGU as green. If it we can do so, it means that nothing
1003 // affecting the LLVM module has changed and we can re-use a cached version.
1004 // If we compile with any kind of LTO, this means we can re-use the bitcode
1005 // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only
1006 // know that later). If we are not doing LTO, there is only one optimized
1007 // version of each module, so we re-use that.
1008 let dep_node
= cgu
.codegen_dep_node(tcx
);
1010 !tcx
.dep_graph
.dep_node_exists(&dep_node
),
1011 "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",
1015 if tcx
.try_mark_green(&dep_node
) {
1016 // We can re-use either the pre- or the post-thinlto state. If no LTO is
1017 // being performed then we can use post-LTO artifacts, otherwise we must
1018 // reuse pre-LTO artifacts
1019 match compute_per_cgu_lto_type(
1022 &tcx
.sess
.crate_types(),
1023 ModuleKind
::Regular
,
1025 ComputedLtoType
::No
=> CguReuse
::PostLto
,
1026 _
=> CguReuse
::PreLto
,