1 use crate::back
::lto
::ThinBuffer
;
2 use crate::back
::profiling
::{
3 selfprofile_after_pass_callback
, selfprofile_before_pass_callback
, LlvmSelfProfiler
,
8 use crate::llvm
::{self, DiagnosticInfo, PassManager, SMDiagnostic}
;
10 use crate::type_
::Type
;
11 use crate::LlvmCodegenBackend
;
12 use crate::ModuleLlvm
;
13 use rustc_codegen_ssa
::back
::link
::ensure_removed
;
14 use rustc_codegen_ssa
::back
::write
::{
15 BitcodeSection
, CodegenContext
, EmitObj
, ModuleConfig
, TargetMachineFactoryConfig
,
16 TargetMachineFactoryFn
,
18 use rustc_codegen_ssa
::traits
::*;
19 use rustc_codegen_ssa
::{CompiledModule, ModuleCodegen}
;
20 use rustc_data_structures
::profiling
::SelfProfilerRef
;
21 use rustc_data_structures
::small_c_str
::SmallCStr
;
22 use rustc_errors
::{FatalError, Handler, Level}
;
23 use rustc_fs_util
::{link_or_copy, path_to_c_string}
;
24 use rustc_middle
::bug
;
25 use rustc_middle
::ty
::TyCtxt
;
26 use rustc_session
::config
::{self, Lto, OutputType, Passes, SwitchWithOptPath}
;
27 use rustc_session
::Session
;
28 use rustc_span
::symbol
::sym
;
29 use rustc_span
::InnerSpan
;
30 use rustc_target
::spec
::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}
;
33 use libc
::{c_char, c_int, c_uint, c_void, size_t}
;
34 use std
::ffi
::CString
;
36 use std
::io
::{self, Write}
;
37 use std
::path
::{Path, PathBuf}
;
42 pub fn llvm_err(handler
: &rustc_errors
::Handler
, msg
: &str) -> FatalError
{
43 match llvm
::last_error() {
44 Some(err
) => handler
.fatal(&format
!("{}: {}", msg
, err
)),
45 None
=> handler
.fatal(msg
),
49 pub fn write_output_file(
50 handler
: &rustc_errors
::Handler
,
51 target
: &'ll llvm
::TargetMachine
,
52 pm
: &llvm
::PassManager
<'ll
>,
55 dwo_output
: Option
<&Path
>,
56 file_type
: llvm
::FileType
,
57 self_profiler_ref
: &SelfProfilerRef
,
58 ) -> Result
<(), FatalError
> {
60 let output_c
= path_to_c_string(output
);
61 let result
= if let Some(dwo_output
) = dwo_output
{
62 let dwo_output_c
= path_to_c_string(dwo_output
);
63 llvm
::LLVMRustWriteOutputFile(
68 dwo_output_c
.as_ptr(),
72 llvm
::LLVMRustWriteOutputFile(
82 // Record artifact sizes for self-profiling
83 if result
== llvm
::LLVMRustResult
::Success
{
84 let artifact_kind
= match file_type
{
85 llvm
::FileType
::ObjectFile
=> "object_file",
86 llvm
::FileType
::AssemblyFile
=> "assembly_file",
88 record_artifact_size(self_profiler_ref
, artifact_kind
, output
);
89 if let Some(dwo_file
) = dwo_output
{
90 record_artifact_size(self_profiler_ref
, "dwo_file", dwo_file
);
94 result
.into_result().map_err(|()| {
95 let msg
= format
!("could not write output to {}", output
.display());
96 llvm_err(handler
, &msg
)
101 pub fn create_informational_target_machine(sess
: &Session
) -> &'
static mut llvm
::TargetMachine
{
102 let config
= TargetMachineFactoryConfig { split_dwarf_file: None }
;
103 target_machine_factory(sess
, config
::OptLevel
::No
)(config
)
104 .unwrap_or_else(|err
| llvm_err(sess
.diagnostic(), &err
).raise())
107 pub fn create_target_machine(tcx
: TyCtxt
<'_
>, mod_name
: &str) -> &'
static mut llvm
::TargetMachine
{
108 let split_dwarf_file
= if tcx
.sess
.target_can_use_split_dwarf() {
109 tcx
.output_filenames(()).split_dwarf_path(tcx
.sess
.split_debuginfo(), Some(mod_name
))
113 let config
= TargetMachineFactoryConfig { split_dwarf_file }
;
114 target_machine_factory(tcx
.sess
, tcx
.backend_optimization_level(()))(config
)
115 .unwrap_or_else(|err
| llvm_err(tcx
.sess
.diagnostic(), &err
).raise())
118 pub fn to_llvm_opt_settings(
119 cfg
: config
::OptLevel
,
120 ) -> (llvm
::CodeGenOptLevel
, llvm
::CodeGenOptSize
) {
121 use self::config
::OptLevel
::*;
123 No
=> (llvm
::CodeGenOptLevel
::None
, llvm
::CodeGenOptSizeNone
),
124 Less
=> (llvm
::CodeGenOptLevel
::Less
, llvm
::CodeGenOptSizeNone
),
125 Default
=> (llvm
::CodeGenOptLevel
::Default
, llvm
::CodeGenOptSizeNone
),
126 Aggressive
=> (llvm
::CodeGenOptLevel
::Aggressive
, llvm
::CodeGenOptSizeNone
),
127 Size
=> (llvm
::CodeGenOptLevel
::Default
, llvm
::CodeGenOptSizeDefault
),
128 SizeMin
=> (llvm
::CodeGenOptLevel
::Default
, llvm
::CodeGenOptSizeAggressive
),
132 fn to_pass_builder_opt_level(cfg
: config
::OptLevel
) -> llvm
::PassBuilderOptLevel
{
133 use config
::OptLevel
::*;
135 No
=> llvm
::PassBuilderOptLevel
::O0
,
136 Less
=> llvm
::PassBuilderOptLevel
::O1
,
137 Default
=> llvm
::PassBuilderOptLevel
::O2
,
138 Aggressive
=> llvm
::PassBuilderOptLevel
::O3
,
139 Size
=> llvm
::PassBuilderOptLevel
::Os
,
140 SizeMin
=> llvm
::PassBuilderOptLevel
::Oz
,
144 fn to_llvm_relocation_model(relocation_model
: RelocModel
) -> llvm
::RelocModel
{
145 match relocation_model
{
146 RelocModel
::Static
=> llvm
::RelocModel
::Static
,
147 // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute.
148 RelocModel
::Pic
| RelocModel
::Pie
=> llvm
::RelocModel
::PIC
,
149 RelocModel
::DynamicNoPic
=> llvm
::RelocModel
::DynamicNoPic
,
150 RelocModel
::Ropi
=> llvm
::RelocModel
::ROPI
,
151 RelocModel
::Rwpi
=> llvm
::RelocModel
::RWPI
,
152 RelocModel
::RopiRwpi
=> llvm
::RelocModel
::ROPI_RWPI
,
156 pub(crate) fn to_llvm_code_model(code_model
: Option
<CodeModel
>) -> llvm
::CodeModel
{
158 Some(CodeModel
::Tiny
) => llvm
::CodeModel
::Tiny
,
159 Some(CodeModel
::Small
) => llvm
::CodeModel
::Small
,
160 Some(CodeModel
::Kernel
) => llvm
::CodeModel
::Kernel
,
161 Some(CodeModel
::Medium
) => llvm
::CodeModel
::Medium
,
162 Some(CodeModel
::Large
) => llvm
::CodeModel
::Large
,
163 None
=> llvm
::CodeModel
::None
,
167 pub fn target_machine_factory(
169 optlvl
: config
::OptLevel
,
170 ) -> TargetMachineFactoryFn
<LlvmCodegenBackend
> {
171 let reloc_model
= to_llvm_relocation_model(sess
.relocation_model());
173 let (opt_level
, _
) = to_llvm_opt_settings(optlvl
);
174 let use_softfp
= sess
.opts
.cg
.soft_float
;
176 let ffunction_sections
=
177 sess
.opts
.debugging_opts
.function_sections
.unwrap_or(sess
.target
.function_sections
);
178 let fdata_sections
= ffunction_sections
;
179 let funique_section_names
= !sess
.opts
.debugging_opts
.no_unique_section_names
;
181 let code_model
= to_llvm_code_model(sess
.code_model());
183 let mut singlethread
= sess
.target
.singlethread
;
185 // On the wasm target once the `atomics` feature is enabled that means that
186 // we're no longer single-threaded, or otherwise we don't want LLVM to
187 // lower atomic operations to single-threaded operations.
188 if singlethread
&& sess
.target
.is_like_wasm
&& sess
.target_features
.contains(&sym
::atomics
) {
189 singlethread
= false;
192 let triple
= SmallCStr
::new(&sess
.target
.llvm_target
);
193 let cpu
= SmallCStr
::new(llvm_util
::target_cpu(sess
));
194 let features
= llvm_util
::llvm_global_features(sess
).join(",");
195 let features
= CString
::new(features
).unwrap();
196 let abi
= SmallCStr
::new(&sess
.target
.llvm_abiname
);
197 let trap_unreachable
=
198 sess
.opts
.debugging_opts
.trap_unreachable
.unwrap_or(sess
.target
.trap_unreachable
);
199 let emit_stack_size_section
= sess
.opts
.debugging_opts
.emit_stack_sizes
;
201 let asm_comments
= sess
.asm_comments();
202 let relax_elf_relocations
=
203 sess
.opts
.debugging_opts
.relax_elf_relocations
.unwrap_or(sess
.target
.relax_elf_relocations
);
206 !sess
.opts
.debugging_opts
.use_ctors_section
.unwrap_or(sess
.target
.use_ctors_section
);
208 Arc
::new(move |config
: TargetMachineFactoryConfig
| {
209 let split_dwarf_file
= config
.split_dwarf_file
.unwrap_or_default();
210 let split_dwarf_file
= CString
::new(split_dwarf_file
.to_str().unwrap()).unwrap();
213 llvm
::LLVMRustCreateTargetMachine(
224 funique_section_names
,
228 emit_stack_size_section
,
229 relax_elf_relocations
,
231 split_dwarf_file
.as_ptr(),
236 format
!("Could not create LLVM TargetMachine for triple: {}", triple
.to_str().unwrap())
241 pub(crate) fn save_temp_bitcode(
242 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
243 module
: &ModuleCodegen
<ModuleLlvm
>,
246 if !cgcx
.save_temps
{
250 let ext
= format
!("{}.bc", name
);
251 let cgu
= Some(&module
.name
[..]);
252 let path
= cgcx
.output_filenames
.temp_path_ext(&ext
, cgu
);
253 let cstr
= path_to_c_string(&path
);
254 let llmod
= module
.module_llvm
.llmod();
255 llvm
::LLVMWriteBitcodeToFile(llmod
, cstr
.as_ptr());
259 pub struct DiagnosticHandlers
<'a
> {
260 data
: *mut (&'a CodegenContext
<LlvmCodegenBackend
>, &'a Handler
),
261 llcx
: &'a llvm
::Context
,
264 impl<'a
> DiagnosticHandlers
<'a
> {
266 cgcx
: &'a CodegenContext
<LlvmCodegenBackend
>,
267 handler
: &'a Handler
,
268 llcx
: &'a llvm
::Context
,
270 let data
= Box
::into_raw(Box
::new((cgcx
, handler
)));
272 llvm
::LLVMRustSetInlineAsmDiagnosticHandler(llcx
, inline_asm_handler
, data
.cast());
273 llvm
::LLVMContextSetDiagnosticHandler(llcx
, diagnostic_handler
, data
.cast());
275 DiagnosticHandlers { data, llcx }
279 impl<'a
> Drop
for DiagnosticHandlers
<'a
> {
281 use std
::ptr
::null_mut
;
283 llvm
::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx
, inline_asm_handler
, null_mut());
284 llvm
::LLVMContextSetDiagnosticHandler(self.llcx
, diagnostic_handler
, null_mut());
285 drop(Box
::from_raw(self.data
));
290 fn report_inline_asm(
291 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
293 level
: llvm
::DiagnosticLevel
,
295 source
: Option
<(String
, Vec
<InnerSpan
>)>,
297 // In LTO build we may get srcloc values from other crates which are invalid
298 // since they use a different source map. To be safe we just suppress these
300 if matches
!(cgcx
.lto
, Lto
::Fat
| Lto
::Thin
) {
303 let level
= match level
{
304 llvm
::DiagnosticLevel
::Error
=> Level
::Error { lint: false }
,
305 llvm
::DiagnosticLevel
::Warning
=> Level
::Warning
,
306 llvm
::DiagnosticLevel
::Note
| llvm
::DiagnosticLevel
::Remark
=> Level
::Note
,
308 cgcx
.diag_emitter
.inline_asm_error(cookie
as u32, msg
, level
, source
);
311 unsafe extern "C" fn inline_asm_handler(diag
: &SMDiagnostic
, user
: *const c_void
, cookie
: c_uint
) {
315 let (cgcx
, _
) = *(user
as *const (&CodegenContext
<LlvmCodegenBackend
>, &Handler
));
317 let smdiag
= llvm
::diagnostic
::SrcMgrDiagnostic
::unpack(diag
);
318 report_inline_asm(cgcx
, smdiag
.message
, smdiag
.level
, cookie
, smdiag
.source
);
321 unsafe extern "C" fn diagnostic_handler(info
: &DiagnosticInfo
, user
: *mut c_void
) {
325 let (cgcx
, diag_handler
) = *(user
as *const (&CodegenContext
<LlvmCodegenBackend
>, &Handler
));
327 match llvm
::diagnostic
::Diagnostic
::unpack(info
) {
328 llvm
::diagnostic
::InlineAsm(inline
) => {
329 report_inline_asm(cgcx
, inline
.message
, inline
.level
, inline
.cookie
, inline
.source
);
332 llvm
::diagnostic
::Optimization(opt
) => {
333 let enabled
= match cgcx
.remark
{
335 Passes
::Some(ref v
) => v
.iter().any(|s
| *s
== opt
.pass_name
),
339 diag_handler
.note_without_error(&format
!(
340 "optimization {} for {} at {}:{}:{}: {}",
350 llvm
::diagnostic
::PGO(diagnostic_ref
) | llvm
::diagnostic
::Linker(diagnostic_ref
) => {
351 let msg
= llvm
::build_string(|s
| {
352 llvm
::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref
, s
)
354 .expect("non-UTF8 diagnostic");
355 diag_handler
.warn(&msg
);
357 llvm
::diagnostic
::Unsupported(diagnostic_ref
) => {
358 let msg
= llvm
::build_string(|s
| {
359 llvm
::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref
, s
)
361 .expect("non-UTF8 diagnostic");
362 diag_handler
.err(&msg
);
364 llvm
::diagnostic
::UnknownDiagnostic(..) => {}
368 fn get_pgo_gen_path(config
: &ModuleConfig
) -> Option
<CString
> {
369 match config
.pgo_gen
{
370 SwitchWithOptPath
::Enabled(ref opt_dir_path
) => {
371 let path
= if let Some(dir_path
) = opt_dir_path
{
372 dir_path
.join("default_%m.profraw")
374 PathBuf
::from("default_%m.profraw")
377 Some(CString
::new(format
!("{}", path
.display())).unwrap())
379 SwitchWithOptPath
::Disabled
=> None
,
383 fn get_pgo_use_path(config
: &ModuleConfig
) -> Option
<CString
> {
387 .map(|path_buf
| CString
::new(path_buf
.to_string_lossy().as_bytes()).unwrap())
390 fn get_pgo_sample_use_path(config
: &ModuleConfig
) -> Option
<CString
> {
394 .map(|path_buf
| CString
::new(path_buf
.to_string_lossy().as_bytes()).unwrap())
397 pub(crate) fn should_use_new_llvm_pass_manager(
398 _cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
399 config
: &ModuleConfig
,
401 // The new pass manager is causing significant performance issues such as #91128, and is
402 // therefore disabled in stable versions of rustc by default.
404 .new_llvm_pass_manager
408 pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
409 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
410 diag_handler
: &Handler
,
411 module
: &ModuleCodegen
<ModuleLlvm
>,
412 config
: &ModuleConfig
,
413 opt_level
: config
::OptLevel
,
414 opt_stage
: llvm
::OptStage
,
415 ) -> Result
<(), FatalError
> {
417 opt_level
!= config
::OptLevel
::Size
&& opt_level
!= config
::OptLevel
::SizeMin
;
418 let using_thin_buffers
= opt_stage
== llvm
::OptStage
::PreLinkThinLTO
|| config
.bitcode_needed();
419 let pgo_gen_path
= get_pgo_gen_path(config
);
420 let pgo_use_path
= get_pgo_use_path(config
);
421 let pgo_sample_use_path
= get_pgo_sample_use_path(config
);
422 let is_lto
= opt_stage
== llvm
::OptStage
::ThinLTO
|| opt_stage
== llvm
::OptStage
::FatLTO
;
423 // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
424 let sanitizer_options
= if !is_lto
{
425 Some(llvm
::SanitizerOptions
{
426 sanitize_address
: config
.sanitizer
.contains(SanitizerSet
::ADDRESS
),
427 sanitize_address_recover
: config
.sanitizer_recover
.contains(SanitizerSet
::ADDRESS
),
428 sanitize_memory
: config
.sanitizer
.contains(SanitizerSet
::MEMORY
),
429 sanitize_memory_recover
: config
.sanitizer_recover
.contains(SanitizerSet
::MEMORY
),
430 sanitize_memory_track_origins
: config
.sanitizer_memory_track_origins
as c_int
,
431 sanitize_thread
: config
.sanitizer
.contains(SanitizerSet
::THREAD
),
432 sanitize_hwaddress
: config
.sanitizer
.contains(SanitizerSet
::HWADDRESS
),
433 sanitize_hwaddress_recover
: config
.sanitizer_recover
.contains(SanitizerSet
::HWADDRESS
),
439 let mut llvm_profiler
= if cgcx
.prof
.llvm_recording_enabled() {
440 Some(LlvmSelfProfiler
::new(cgcx
.prof
.get_self_profiler().unwrap()))
445 let llvm_selfprofiler
=
446 llvm_profiler
.as_mut().map(|s
| s
as *mut _
as *mut c_void
).unwrap_or(std
::ptr
::null_mut());
448 let extra_passes
= config
.passes
.join(",");
450 // FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
451 // We would have to add upstream support for this first, before we can support
452 // config.inline_threshold and our more aggressive default thresholds.
453 let result
= llvm
::LLVMRustOptimizeWithNewPassManager(
454 module
.module_llvm
.llmod(),
455 &*module
.module_llvm
.tm
,
456 to_pass_builder_opt_level(opt_level
),
458 config
.no_prepopulate_passes
,
459 config
.verify_llvm_ir
,
461 config
.merge_functions
,
463 config
.vectorize_slp
,
464 config
.vectorize_loop
,
466 config
.emit_lifetime_markers
,
467 sanitizer_options
.as_ref(),
468 pgo_gen_path
.as_ref().map_or(std
::ptr
::null(), |s
| s
.as_ptr()),
469 pgo_use_path
.as_ref().map_or(std
::ptr
::null(), |s
| s
.as_ptr()),
470 config
.instrument_coverage
,
471 config
.instrument_gcov
,
472 pgo_sample_use_path
.as_ref().map_or(std
::ptr
::null(), |s
| s
.as_ptr()),
473 config
.debug_info_for_profiling
,
475 selfprofile_before_pass_callback
,
476 selfprofile_after_pass_callback
,
477 extra_passes
.as_ptr().cast(),
480 result
.into_result().map_err(|()| llvm_err(diag_handler
, "failed to run LLVM passes"))
483 // Unsafe due to LLVM calls.
484 pub(crate) unsafe fn optimize(
485 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
486 diag_handler
: &Handler
,
487 module
: &ModuleCodegen
<ModuleLlvm
>,
488 config
: &ModuleConfig
,
489 ) -> Result
<(), FatalError
> {
490 let _timer
= cgcx
.prof
.generic_activity_with_arg("LLVM_module_optimize", &module
.name
[..]);
492 let llmod
= module
.module_llvm
.llmod();
493 let llcx
= &*module
.module_llvm
.llcx
;
494 let tm
= &*module
.module_llvm
.tm
;
495 let _handlers
= DiagnosticHandlers
::new(cgcx
, diag_handler
, llcx
);
497 let module_name
= module
.name
.clone();
498 let module_name
= Some(&module_name
[..]);
500 if config
.emit_no_opt_bc
{
501 let out
= cgcx
.output_filenames
.temp_path_ext("no-opt.bc", module_name
);
502 let out
= path_to_c_string(&out
);
503 llvm
::LLVMWriteBitcodeToFile(llmod
, out
.as_ptr());
506 if let Some(opt_level
) = config
.opt_level
{
507 if should_use_new_llvm_pass_manager(cgcx
, config
) {
508 let opt_stage
= match cgcx
.lto
{
509 Lto
::Fat
=> llvm
::OptStage
::PreLinkFatLTO
,
510 Lto
::Thin
| Lto
::ThinLocal
=> llvm
::OptStage
::PreLinkThinLTO
,
511 _
if cgcx
.opts
.cg
.linker_plugin_lto
.enabled() => llvm
::OptStage
::PreLinkThinLTO
,
512 _
=> llvm
::OptStage
::PreLinkNoLTO
,
514 return optimize_with_new_llvm_pass_manager(
524 if cgcx
.prof
.llvm_recording_enabled() {
526 .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`");
529 // Create the two optimizing pass managers. These mirror what clang
530 // does, and are by populated by LLVM's default PassManagerBuilder.
531 // Each manager has a different set of passes, but they also share
532 // some common passes.
533 let fpm
= llvm
::LLVMCreateFunctionPassManagerForModule(llmod
);
534 let mpm
= llvm
::LLVMCreatePassManager();
537 let find_pass
= |pass_name
: &str| {
538 let pass_name
= SmallCStr
::new(pass_name
);
539 llvm
::LLVMRustFindAndCreatePass(pass_name
.as_ptr())
542 if config
.verify_llvm_ir
{
543 // Verification should run as the very first pass.
544 llvm
::LLVMRustAddPass(fpm
, find_pass("verify").unwrap());
547 let mut extra_passes
= Vec
::new();
548 let mut have_name_anon_globals_pass
= false;
550 for pass_name
in &config
.passes
{
551 if pass_name
== "lint" {
552 // Linting should also be performed early, directly on the generated IR.
553 llvm
::LLVMRustAddPass(fpm
, find_pass("lint").unwrap());
557 if let Some(pass
) = find_pass(pass_name
) {
558 extra_passes
.push(pass
);
560 diag_handler
.warn(&format
!("unknown pass `{}`, ignoring", pass_name
));
563 if pass_name
== "name-anon-globals" {
564 have_name_anon_globals_pass
= true;
568 // Instrumentation must be inserted before optimization,
569 // otherwise LLVM may optimize some functions away which
572 // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp.
573 if config
.instrument_gcov
{
574 llvm
::LLVMRustAddPass(mpm
, find_pass("insert-gcov-profiling").unwrap());
576 if config
.instrument_coverage
{
577 llvm
::LLVMRustAddPass(mpm
, find_pass("instrprof").unwrap());
579 if config
.debug_info_for_profiling
{
580 llvm
::LLVMRustAddPass(mpm
, find_pass("add-discriminators").unwrap());
583 add_sanitizer_passes(config
, &mut extra_passes
);
585 // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need
586 // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise
587 // we'll get errors in LLVM.
588 let using_thin_buffers
= config
.bitcode_needed();
589 if !config
.no_prepopulate_passes
{
590 llvm
::LLVMAddAnalysisPasses(tm
, fpm
);
591 llvm
::LLVMAddAnalysisPasses(tm
, mpm
);
592 let opt_level
= to_llvm_opt_settings(opt_level
).0;
593 let prepare_for_thin_lto
= cgcx
.lto
== Lto
::Thin
594 || cgcx
.lto
== Lto
::ThinLocal
595 || (cgcx
.lto
!= Lto
::Fat
&& cgcx
.opts
.cg
.linker_plugin_lto
.enabled());
596 with_llvm_pmb(llmod
, config
, opt_level
, prepare_for_thin_lto
, &mut |b
| {
597 llvm
::LLVMRustAddLastExtensionPasses(
599 extra_passes
.as_ptr(),
600 extra_passes
.len() as size_t
,
602 llvm
::LLVMPassManagerBuilderPopulateFunctionPassManager(b
, fpm
);
603 llvm
::LLVMPassManagerBuilderPopulateModulePassManager(b
, mpm
);
606 have_name_anon_globals_pass
= have_name_anon_globals_pass
|| prepare_for_thin_lto
;
607 if using_thin_buffers
&& !prepare_for_thin_lto
{
608 llvm
::LLVMRustAddPass(mpm
, find_pass("name-anon-globals").unwrap());
609 have_name_anon_globals_pass
= true;
612 // If we don't use the standard pipeline, directly populate the MPM
613 // with the extra passes.
614 for pass
in extra_passes
{
615 llvm
::LLVMRustAddPass(mpm
, pass
);
619 if using_thin_buffers
&& !have_name_anon_globals_pass
{
620 // As described above, this will probably cause an error in LLVM
621 if config
.no_prepopulate_passes
{
623 "The current compilation is going to use thin LTO buffers \
624 without running LLVM's NameAnonGlobals pass. \
625 This will likely cause errors in LLVM. Consider adding \
626 -C passes=name-anon-globals to the compiler command line.",
630 "We are using thin LTO buffers without running the NameAnonGlobals pass. \
631 This will likely cause errors in LLVM and should never happen."
637 diag_handler
.abort_if_errors();
639 // Finally, run the actual optimization passes
641 let _timer
= cgcx
.prof
.extra_verbose_generic_activity(
642 "LLVM_module_optimize_function_passes",
645 llvm
::LLVMRustRunFunctionPassManager(fpm
, llmod
);
648 let _timer
= cgcx
.prof
.extra_verbose_generic_activity(
649 "LLVM_module_optimize_module_passes",
652 llvm
::LLVMRunPassManager(mpm
, llmod
);
655 // Deallocate managers that we're now done with
656 llvm
::LLVMDisposePassManager(fpm
);
657 llvm
::LLVMDisposePassManager(mpm
);
662 unsafe fn add_sanitizer_passes(config
: &ModuleConfig
, passes
: &mut Vec
<&'
static mut llvm
::Pass
>) {
663 if config
.sanitizer
.contains(SanitizerSet
::ADDRESS
) {
664 let recover
= config
.sanitizer_recover
.contains(SanitizerSet
::ADDRESS
);
665 passes
.push(llvm
::LLVMRustCreateAddressSanitizerFunctionPass(recover
));
666 passes
.push(llvm
::LLVMRustCreateModuleAddressSanitizerPass(recover
));
668 if config
.sanitizer
.contains(SanitizerSet
::MEMORY
) {
669 let track_origins
= config
.sanitizer_memory_track_origins
as c_int
;
670 let recover
= config
.sanitizer_recover
.contains(SanitizerSet
::MEMORY
);
671 passes
.push(llvm
::LLVMRustCreateMemorySanitizerPass(track_origins
, recover
));
673 if config
.sanitizer
.contains(SanitizerSet
::THREAD
) {
674 passes
.push(llvm
::LLVMRustCreateThreadSanitizerPass());
676 if config
.sanitizer
.contains(SanitizerSet
::HWADDRESS
) {
677 let recover
= config
.sanitizer_recover
.contains(SanitizerSet
::HWADDRESS
);
678 passes
.push(llvm
::LLVMRustCreateHWAddressSanitizerPass(recover
));
683 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
684 diag_handler
: &Handler
,
685 mut modules
: Vec
<ModuleCodegen
<ModuleLlvm
>>,
686 ) -> Result
<ModuleCodegen
<ModuleLlvm
>, FatalError
> {
687 use super::lto
::{Linker, ModuleBuffer}
;
688 // Sort the modules by name to ensure to ensure deterministic behavior.
689 modules
.sort_by(|a
, b
| a
.name
.cmp(&b
.name
));
690 let (first
, elements
) =
691 modules
.split_first().expect("Bug! modules must contain at least one module.");
693 let mut linker
= Linker
::new(first
.module_llvm
.llmod());
694 for module
in elements
{
696 cgcx
.prof
.generic_activity_with_arg("LLVM_link_module", format
!("{:?}", module
.name
));
697 let buffer
= ModuleBuffer
::new(module
.module_llvm
.llmod());
698 linker
.add(buffer
.data()).map_err(|()| {
699 let msg
= format
!("failed to serialize module {:?}", module
.name
);
700 llvm_err(diag_handler
, &msg
)
704 Ok(modules
.remove(0))
707 pub(crate) unsafe fn codegen(
708 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
709 diag_handler
: &Handler
,
710 module
: ModuleCodegen
<ModuleLlvm
>,
711 config
: &ModuleConfig
,
712 ) -> Result
<CompiledModule
, FatalError
> {
713 let _timer
= cgcx
.prof
.generic_activity_with_arg("LLVM_module_codegen", &module
.name
[..]);
715 let llmod
= module
.module_llvm
.llmod();
716 let llcx
= &*module
.module_llvm
.llcx
;
717 let tm
= &*module
.module_llvm
.tm
;
718 let module_name
= module
.name
.clone();
719 let module_name
= Some(&module_name
[..]);
720 let handlers
= DiagnosticHandlers
::new(cgcx
, diag_handler
, llcx
);
722 if cgcx
.msvc_imps_needed
{
723 create_msvc_imps(cgcx
, llcx
, llmod
);
726 // A codegen-specific pass manager is used to generate object
727 // files for an LLVM module.
729 // Apparently each of these pass managers is a one-shot kind of
730 // thing, so we create a new one for each type of output. The
731 // pass manager passed to the closure should be ensured to not
732 // escape the closure itself, and the manager should only be
734 unsafe fn with_codegen
<'ll
, F
, R
>(
735 tm
: &'ll llvm
::TargetMachine
,
736 llmod
: &'ll llvm
::Module
,
741 F
: FnOnce(&'ll
mut PassManager
<'ll
>) -> R
,
743 let cpm
= llvm
::LLVMCreatePassManager();
744 llvm
::LLVMAddAnalysisPasses(tm
, cpm
);
745 llvm
::LLVMRustAddLibraryInfo(cpm
, llmod
, no_builtins
);
749 // Two things to note:
750 // - If object files are just LLVM bitcode we write bitcode, copy it to
751 // the .o file, and delete the bitcode if it wasn't otherwise
753 // - If we don't have the integrated assembler then we need to emit
754 // asm from LLVM and use `gcc` to create the object file.
756 let bc_out
= cgcx
.output_filenames
.temp_path(OutputType
::Bitcode
, module_name
);
757 let obj_out
= cgcx
.output_filenames
.temp_path(OutputType
::Object
, module_name
);
759 if config
.bitcode_needed() {
762 .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &module
.name
[..]);
763 let thin
= ThinBuffer
::new(llmod
);
764 let data
= thin
.data();
766 if let Some(bitcode_filename
) = bc_out
.file_name() {
767 cgcx
.prof
.artifact_size(
769 bitcode_filename
.to_string_lossy(),
774 if config
.emit_bc
|| config
.emit_obj
== EmitObj
::Bitcode
{
775 let _timer
= cgcx
.prof
.generic_activity_with_arg(
776 "LLVM_module_codegen_emit_bitcode",
779 if let Err(e
) = fs
::write(&bc_out
, data
) {
780 let msg
= format
!("failed to write bytecode to {}: {}", bc_out
.display(), e
);
781 diag_handler
.err(&msg
);
785 if config
.emit_obj
== EmitObj
::ObjectCode(BitcodeSection
::Full
) {
786 let _timer
= cgcx
.prof
.generic_activity_with_arg(
787 "LLVM_module_codegen_embed_bitcode",
790 embed_bitcode(cgcx
, llcx
, llmod
, &config
.bc_cmdline
, data
);
797 .generic_activity_with_arg("LLVM_module_codegen_emit_ir", &module
.name
[..]);
798 let out
= cgcx
.output_filenames
.temp_path(OutputType
::LlvmAssembly
, module_name
);
799 let out_c
= path_to_c_string(&out
);
801 extern "C" fn demangle_callback(
802 input_ptr
: *const c_char
,
804 output_ptr
: *mut c_char
,
808 unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) }
;
810 let input
= match str::from_utf8(input
) {
815 let output
= unsafe {
816 slice
::from_raw_parts_mut(output_ptr
as *mut u8, output_len
as usize)
818 let mut cursor
= io
::Cursor
::new(output
);
820 let demangled
= match rustc_demangle
::try_demangle(input
) {
825 if write
!(cursor
, "{:#}", demangled
).is_err() {
826 // Possible only if provided buffer is not big enough
830 cursor
.position() as size_t
833 let result
= llvm
::LLVMRustPrintModule(llmod
, out_c
.as_ptr(), demangle_callback
);
835 if result
== llvm
::LLVMRustResult
::Success
{
836 record_artifact_size(&cgcx
.prof
, "llvm_ir", &out
);
839 result
.into_result().map_err(|()| {
840 let msg
= format
!("failed to write LLVM IR to {}", out
.display());
841 llvm_err(diag_handler
, &msg
)
848 .generic_activity_with_arg("LLVM_module_codegen_emit_asm", &module
.name
[..]);
849 let path
= cgcx
.output_filenames
.temp_path(OutputType
::Assembly
, module_name
);
851 // We can't use the same module for asm and object code output,
852 // because that triggers various errors like invalid IR or broken
853 // binaries. So we must clone the module to produce the asm output
854 // if we are also producing object code.
855 let llmod
= if let EmitObj
::ObjectCode(_
) = config
.emit_obj
{
856 llvm
::LLVMCloneModule(llmod
)
860 with_codegen(tm
, llmod
, config
.no_builtins
, |cpm
| {
868 llvm
::FileType
::AssemblyFile
,
874 match config
.emit_obj
{
875 EmitObj
::ObjectCode(_
) => {
878 .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &module
.name
[..]);
880 let dwo_out
= cgcx
.output_filenames
.temp_path_dwo(module_name
);
881 let dwo_out
= match cgcx
.split_debuginfo
{
882 // Don't change how DWARF is emitted in single mode (or when disabled).
883 SplitDebuginfo
::Off
| SplitDebuginfo
::Packed
=> None
,
884 // Emit (a subset of the) DWARF into a separate file in split mode.
885 SplitDebuginfo
::Unpacked
=> {
886 if cgcx
.target_can_use_split_dwarf
{
887 Some(dwo_out
.as_path())
894 with_codegen(tm
, llmod
, config
.no_builtins
, |cpm
| {
902 llvm
::FileType
::ObjectFile
,
908 EmitObj
::Bitcode
=> {
909 debug
!("copying bitcode {:?} to obj {:?}", bc_out
, obj_out
);
910 if let Err(e
) = link_or_copy(&bc_out
, &obj_out
) {
911 diag_handler
.err(&format
!("failed to copy bitcode to object file: {}", e
));
915 debug
!("removing_bitcode {:?}", bc_out
);
916 ensure_removed(diag_handler
, &bc_out
);
926 Ok(module
.into_compiled_module(
927 config
.emit_obj
!= EmitObj
::None
,
928 cgcx
.target_can_use_split_dwarf
&& cgcx
.split_debuginfo
== SplitDebuginfo
::Unpacked
,
930 &cgcx
.output_filenames
,
934 /// Embed the bitcode of an LLVM module in the LLVM module itself.
936 /// This is done primarily for iOS where it appears to be standard to compile C
937 /// code at least with `-fembed-bitcode` which creates two sections in the
940 /// * __LLVM,__bitcode
941 /// * __LLVM,__cmdline
943 /// It appears *both* of these sections are necessary to get the linker to
944 /// recognize what's going on. A suitable cmdline value is taken from the
947 /// Furthermore debug/O1 builds don't actually embed bitcode but rather just
948 /// embed an empty section.
950 /// Basically all of this is us attempting to follow in the footsteps of clang
951 /// on iOS. See #35968 for lots more info.
952 unsafe fn embed_bitcode(
953 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
954 llcx
: &llvm
::Context
,
955 llmod
: &llvm
::Module
,
959 let llconst
= common
::bytes_in_context(llcx
, bitcode
);
960 let llglobal
= llvm
::LLVMAddGlobal(
962 common
::val_ty(llconst
),
963 "rustc.embedded.module\0".as_ptr().cast(),
965 llvm
::LLVMSetInitializer(llglobal
, llconst
);
967 let is_apple
= cgcx
.opts
.target_triple
.triple().contains("-ios")
968 || cgcx
.opts
.target_triple
.triple().contains("-darwin")
969 || cgcx
.opts
.target_triple
.triple().contains("-tvos");
971 let section
= if is_apple { "__LLVM,__bitcode\0" }
else { ".llvmbc\0" }
;
972 llvm
::LLVMSetSection(llglobal
, section
.as_ptr().cast());
973 llvm
::LLVMRustSetLinkage(llglobal
, llvm
::Linkage
::PrivateLinkage
);
974 llvm
::LLVMSetGlobalConstant(llglobal
, llvm
::True
);
976 let llconst
= common
::bytes_in_context(llcx
, cmdline
.as_bytes());
977 let llglobal
= llvm
::LLVMAddGlobal(
979 common
::val_ty(llconst
),
980 "rustc.embedded.cmdline\0".as_ptr().cast(),
982 llvm
::LLVMSetInitializer(llglobal
, llconst
);
983 let section
= if is_apple { "__LLVM,__cmdline\0" }
else { ".llvmcmd\0" }
;
984 llvm
::LLVMSetSection(llglobal
, section
.as_ptr().cast());
985 llvm
::LLVMRustSetLinkage(llglobal
, llvm
::Linkage
::PrivateLinkage
);
987 // We're adding custom sections to the output object file, but we definitely
988 // do not want these custom sections to make their way into the final linked
989 // executable. The purpose of these custom sections is for tooling
990 // surrounding object files to work with the LLVM IR, if necessary. For
991 // example rustc's own LTO will look for LLVM IR inside of the object file
992 // in these sections by default.
994 // To handle this is a bit different depending on the object file format
995 // used by the backend, broken down into a few different categories:
997 // * Mach-O - this is for macOS. Inspecting the source code for the native
998 // linker here shows that the `.llvmbc` and `.llvmcmd` sections are
999 // automatically skipped by the linker. In that case there's nothing extra
1000 // that we need to do here.
1002 // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and
1003 // `.llvmcmd` sections, so there's nothing extra we need to do.
1005 // * COFF - if we don't do anything the linker will by default copy all
1006 // these sections to the output artifact, not what we want! To subvert
1007 // this we want to flag the sections we inserted here as
1008 // `IMAGE_SCN_LNK_REMOVE`. Unfortunately though LLVM has no native way to
1009 // do this. Thankfully though we can do this with some inline assembly,
1010 // which is easy enough to add via module-level global inline asm.
1012 // * ELF - this is very similar to COFF above. One difference is that these
1013 // sections are removed from the output linked artifact when
1014 // `--gc-sections` is passed, which we pass by default. If that flag isn't
1015 // passed though then these sections will show up in the final output.
1016 // Additionally the flag that we need to set here is `SHF_EXCLUDE`.
1018 || cgcx
.opts
.target_triple
.triple().starts_with("wasm")
1019 || cgcx
.opts
.target_triple
.triple().starts_with("asmjs")
1021 // nothing to do here
1022 } else if cgcx
.is_pe_coff
{
1024 .section .llvmbc,\"n\"
1025 .section .llvmcmd,\"n\"
1027 llvm
::LLVMRustAppendModuleInlineAsm(llmod
, asm
.as_ptr().cast(), asm
.len());
1030 .section .llvmbc,\"e\"
1031 .section .llvmcmd,\"e\"
1033 llvm
::LLVMRustAppendModuleInlineAsm(llmod
, asm
.as_ptr().cast(), asm
.len());
1037 pub unsafe fn with_llvm_pmb(
1038 llmod
: &llvm
::Module
,
1039 config
: &ModuleConfig
,
1040 opt_level
: llvm
::CodeGenOptLevel
,
1041 prepare_for_thin_lto
: bool
,
1042 f
: &mut dyn FnMut(&llvm
::PassManagerBuilder
),
1046 // Create the PassManagerBuilder for LLVM. We configure it with
1047 // reasonable defaults and prepare it to actually populate the pass
1049 let builder
= llvm
::LLVMPassManagerBuilderCreate();
1050 let opt_size
= config
.opt_size
.map_or(llvm
::CodeGenOptSizeNone
, |x
| to_llvm_opt_settings(x
).1);
1051 let inline_threshold
= config
.inline_threshold
;
1052 let pgo_gen_path
= get_pgo_gen_path(config
);
1053 let pgo_use_path
= get_pgo_use_path(config
);
1054 let pgo_sample_use_path
= get_pgo_sample_use_path(config
);
1056 llvm
::LLVMRustConfigurePassManagerBuilder(
1059 config
.merge_functions
,
1060 config
.vectorize_slp
,
1061 config
.vectorize_loop
,
1062 prepare_for_thin_lto
,
1063 pgo_gen_path
.as_ref().map_or(ptr
::null(), |s
| s
.as_ptr()),
1064 pgo_use_path
.as_ref().map_or(ptr
::null(), |s
| s
.as_ptr()),
1065 pgo_sample_use_path
.as_ref().map_or(ptr
::null(), |s
| s
.as_ptr()),
1068 llvm
::LLVMPassManagerBuilderSetSizeLevel(builder
, opt_size
as u32);
1070 if opt_size
!= llvm
::CodeGenOptSizeNone
{
1071 llvm
::LLVMPassManagerBuilderSetDisableUnrollLoops(builder
, 1);
1074 llvm
::LLVMRustAddBuilderLibraryInfo(builder
, llmod
, config
.no_builtins
);
1076 // Here we match what clang does (kinda). For O0 we only inline
1077 // always-inline functions (but don't add lifetime intrinsics), at O1 we
1078 // inline with lifetime intrinsics, and O2+ we add an inliner with a
1079 // thresholds copied from clang.
1080 match (opt_level
, opt_size
, inline_threshold
) {
1082 llvm
::LLVMPassManagerBuilderUseInlinerWithThreshold(builder
, t
);
1084 (llvm
::CodeGenOptLevel
::Aggressive
, ..) => {
1085 llvm
::LLVMPassManagerBuilderUseInlinerWithThreshold(builder
, 275);
1087 (_
, llvm
::CodeGenOptSizeDefault
, _
) => {
1088 llvm
::LLVMPassManagerBuilderUseInlinerWithThreshold(builder
, 75);
1090 (_
, llvm
::CodeGenOptSizeAggressive
, _
) => {
1091 llvm
::LLVMPassManagerBuilderUseInlinerWithThreshold(builder
, 25);
1093 (llvm
::CodeGenOptLevel
::None
, ..) => {
1094 llvm
::LLVMRustAddAlwaysInlinePass(builder
, config
.emit_lifetime_markers
);
1096 (llvm
::CodeGenOptLevel
::Less
, ..) => {
1097 llvm
::LLVMRustAddAlwaysInlinePass(builder
, config
.emit_lifetime_markers
);
1099 (llvm
::CodeGenOptLevel
::Default
, ..) => {
1100 llvm
::LLVMPassManagerBuilderUseInlinerWithThreshold(builder
, 225);
1105 llvm
::LLVMPassManagerBuilderDispose(builder
);
1108 // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`.
1109 // This is required to satisfy `dllimport` references to static data in .rlibs
1110 // when using MSVC linker. We do this only for data, as linker can fix up
1111 // code references on its own.
1112 // See #26591, #27438
1113 fn create_msvc_imps(
1114 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
1115 llcx
: &llvm
::Context
,
1116 llmod
: &llvm
::Module
,
1118 if !cgcx
.msvc_imps_needed
{
1121 // The x86 ABI seems to require that leading underscores are added to symbol
1122 // names, so we need an extra underscore on x86. There's also a leading
1123 // '\x01' here which disables LLVM's symbol mangling (e.g., no extra
1124 // underscores added in front).
1125 let prefix
= if cgcx
.target_arch
== "x86" { "\x01__imp__" }
else { "\x01__imp_" }
;
1128 let i8p_ty
= Type
::i8p_llcx(llcx
);
1129 let globals
= base
::iter_globals(llmod
)
1131 llvm
::LLVMRustGetLinkage(val
) == llvm
::Linkage
::ExternalLinkage
1132 && llvm
::LLVMIsDeclaration(val
) == 0
1135 // Exclude some symbols that we know are not Rust symbols.
1136 let name
= llvm
::get_value_name(val
);
1137 if ignored(name
) { None }
else { Some((val, name)) }
1139 .map(move |(val
, name
)| {
1140 let mut imp_name
= prefix
.as_bytes().to_vec();
1141 imp_name
.extend(name
);
1142 let imp_name
= CString
::new(imp_name
).unwrap();
1145 .collect
::<Vec
<_
>>();
1147 for (imp_name
, val
) in globals
{
1148 let imp
= llvm
::LLVMAddGlobal(llmod
, i8p_ty
, imp_name
.as_ptr().cast());
1149 llvm
::LLVMSetInitializer(imp
, consts
::ptrcast(val
, i8p_ty
));
1150 llvm
::LLVMRustSetLinkage(imp
, llvm
::Linkage
::ExternalLinkage
);
1154 // Use this function to exclude certain symbols from `__imp` generation.
1155 fn ignored(symbol_name
: &[u8]) -> bool
{
1156 // These are symbols generated by LLVM's profiling instrumentation
1157 symbol_name
.starts_with(b
"__llvm_profile_")
1161 fn record_artifact_size(
1162 self_profiler_ref
: &SelfProfilerRef
,
1163 artifact_kind
: &'
static str,
1166 // Don't stat the file if we are not going to record its size.
1167 if !self_profiler_ref
.enabled() {
1171 if let Some(artifact_name
) = path
.file_name() {
1172 let file_size
= std
::fs
::metadata(path
).map(|m
| m
.len()).unwrap_or(0);
1173 self_profiler_ref
.artifact_size(artifact_kind
, artifact_name
.to_string_lossy(), file_size
);