1 use crate::back
::write
::{
2 self, save_temp_bitcode
, to_llvm_opt_settings
, with_llvm_pmb
, DiagnosticHandlers
,
4 use crate::llvm
::archive_ro
::ArchiveRO
;
5 use crate::llvm
::{self, False, True}
;
6 use crate::{LlvmCodegenBackend, ModuleLlvm}
;
7 use log
::{debug, info}
;
8 use rustc_codegen_ssa
::back
::lto
::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}
;
9 use rustc_codegen_ssa
::back
::symbol_export
;
10 use rustc_codegen_ssa
::back
::write
::{CodegenContext, FatLTOInput, ModuleConfig}
;
11 use rustc_codegen_ssa
::traits
::*;
12 use rustc_codegen_ssa
::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}
;
13 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
14 use rustc_errors
::{FatalError, Handler}
;
15 use rustc_hir
::def_id
::LOCAL_CRATE
;
16 use rustc_middle
::bug
;
17 use rustc_middle
::dep_graph
::WorkProduct
;
18 use rustc_middle
::middle
::exported_symbols
::SymbolExportLevel
;
19 use rustc_session
::cgu_reuse_tracker
::CguReuse
;
20 use rustc_session
::config
::{self, CrateType, Lto}
;
22 use std
::ffi
::{CStr, CString}
;
31 /// We keep track of past LTO imports that were used to produce the current set
32 /// of compiled object files that we might choose to reuse during this
33 /// compilation session.
34 pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME
: &str = "thin-lto-past-imports.bin";
36 pub fn crate_type_allows_lto(crate_type
: CrateType
) -> bool
{
38 CrateType
::Executable
| CrateType
::Staticlib
| CrateType
::Cdylib
=> true,
39 CrateType
::Dylib
| CrateType
::Rlib
| CrateType
::ProcMacro
=> false,
44 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
45 diag_handler
: &Handler
,
46 ) -> Result
<(Vec
<CString
>, Vec
<(SerializedModule
<ModuleBuffer
>, CString
)>), FatalError
> {
47 let export_threshold
= match cgcx
.lto
{
48 // We're just doing LTO for our one crate
49 Lto
::ThinLocal
=> SymbolExportLevel
::Rust
,
51 // We're doing LTO for the entire crate graph
52 Lto
::Fat
| Lto
::Thin
=> symbol_export
::crates_export_threshold(&cgcx
.crate_types
),
54 Lto
::No
=> panic
!("didn't request LTO but we're doing LTO"),
57 let symbol_filter
= &|&(ref name
, level
): &(String
, SymbolExportLevel
)| {
58 if level
.is_below_threshold(export_threshold
) {
59 Some(CString
::new(name
.as_str()).unwrap())
64 let exported_symbols
= cgcx
.exported_symbols
.as_ref().expect("needs exported symbols for LTO");
65 let mut symbol_white_list
= {
66 let _timer
= cgcx
.prof
.generic_activity("LLVM_lto_generate_symbol_white_list");
67 exported_symbols
[&LOCAL_CRATE
].iter().filter_map(symbol_filter
).collect
::<Vec
<CString
>>()
69 info
!("{} symbols to preserve in this crate", symbol_white_list
.len());
71 // If we're performing LTO for the entire crate graph, then for each of our
72 // upstream dependencies, find the corresponding rlib and load the bitcode
75 // We save off all the bytecode and LLVM module ids for later processing
76 // with either fat or thin LTO
77 let mut upstream_modules
= Vec
::new();
78 if cgcx
.lto
!= Lto
::ThinLocal
{
79 if cgcx
.opts
.cg
.prefer_dynamic
{
81 .struct_err("cannot prefer dynamic linking when performing LTO")
83 "only 'staticlib', 'bin', and 'cdylib' outputs are \
87 return Err(FatalError
);
90 // Make sure we actually can run LTO
91 for crate_type
in cgcx
.crate_types
.iter() {
92 if !crate_type_allows_lto(*crate_type
) {
93 let e
= diag_handler
.fatal(
94 "lto can only be run for executables, cdylibs and \
95 static library outputs",
101 for &(cnum
, ref path
) in cgcx
.each_linked_rlib_for_lto
.iter() {
102 let exported_symbols
=
103 cgcx
.exported_symbols
.as_ref().expect("needs exported symbols for LTO");
105 let _timer
= cgcx
.prof
.generic_activity("LLVM_lto_generate_symbol_white_list");
106 symbol_white_list
.extend(exported_symbols
[&cnum
].iter().filter_map(symbol_filter
));
109 let archive
= ArchiveRO
::open(&path
).expect("wanted an rlib");
110 let obj_files
= archive
112 .filter_map(|child
| child
.ok().and_then(|c
| c
.name().map(|name
| (name
, c
))))
113 .filter(|&(name
, _
)| looks_like_rust_object_file(name
));
114 for (name
, child
) in obj_files
{
115 info
!("adding bitcode from {}", name
);
116 match get_bitcode_slice_from_object_data(child
.data()) {
118 let module
= SerializedModule
::FromRlib(data
.to_vec());
119 upstream_modules
.push((module
, CString
::new(name
).unwrap()));
121 Err(msg
) => return Err(diag_handler
.fatal(&msg
)),
127 Ok((symbol_white_list
, upstream_modules
))
130 fn get_bitcode_slice_from_object_data(obj
: &[u8]) -> Result
<&[u8], String
> {
133 unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }
;
136 let bc
= unsafe { slice::from_raw_parts(data, len) }
;
138 // `bc` must be a sub-slice of `obj`.
139 assert
!(obj
.as_ptr() <= bc
.as_ptr());
140 assert
!(bc
[bc
.len()..bc
.len()].as_ptr() <= obj
[obj
.len()..obj
.len()].as_ptr());
145 let msg
= llvm
::last_error().unwrap_or_else(|| "unknown LLVM error".to_string());
146 Err(format
!("failed to get bitcode from object file for LTO ({})", msg
))
150 /// Performs fat LTO by merging all modules into a single one and returning it
151 /// for further optimization.
152 pub(crate) fn run_fat(
153 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
154 modules
: Vec
<FatLTOInput
<LlvmCodegenBackend
>>,
155 cached_modules
: Vec
<(SerializedModule
<ModuleBuffer
>, WorkProduct
)>,
156 ) -> Result
<LtoModuleCodegen
<LlvmCodegenBackend
>, FatalError
> {
157 let diag_handler
= cgcx
.create_diag_handler();
158 let (symbol_white_list
, upstream_modules
) = prepare_lto(cgcx
, &diag_handler
)?
;
159 let symbol_white_list
= symbol_white_list
.iter().map(|c
| c
.as_ptr()).collect
::<Vec
<_
>>();
160 fat_lto(cgcx
, &diag_handler
, modules
, cached_modules
, upstream_modules
, &symbol_white_list
)
163 /// Performs thin LTO by performing necessary global analysis and returning two
164 /// lists, one of the modules that need optimization and another for modules that
165 /// can simply be copied over from the incr. comp. cache.
166 pub(crate) fn run_thin(
167 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
168 modules
: Vec
<(String
, ThinBuffer
)>,
169 cached_modules
: Vec
<(SerializedModule
<ModuleBuffer
>, WorkProduct
)>,
170 ) -> Result
<(Vec
<LtoModuleCodegen
<LlvmCodegenBackend
>>, Vec
<WorkProduct
>), FatalError
> {
171 let diag_handler
= cgcx
.create_diag_handler();
172 let (symbol_white_list
, upstream_modules
) = prepare_lto(cgcx
, &diag_handler
)?
;
173 let symbol_white_list
= symbol_white_list
.iter().map(|c
| c
.as_ptr()).collect
::<Vec
<_
>>();
174 if cgcx
.opts
.cg
.linker_plugin_lto
.enabled() {
176 "We should never reach this case if the LTO step \
177 is deferred to the linker"
180 thin_lto(cgcx
, &diag_handler
, modules
, upstream_modules
, cached_modules
, &symbol_white_list
)
183 pub(crate) fn prepare_thin(module
: ModuleCodegen
<ModuleLlvm
>) -> (String
, ThinBuffer
) {
184 let name
= module
.name
.clone();
185 let buffer
= ThinBuffer
::new(module
.module_llvm
.llmod());
190 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
191 diag_handler
: &Handler
,
192 modules
: Vec
<FatLTOInput
<LlvmCodegenBackend
>>,
193 cached_modules
: Vec
<(SerializedModule
<ModuleBuffer
>, WorkProduct
)>,
194 mut serialized_modules
: Vec
<(SerializedModule
<ModuleBuffer
>, CString
)>,
195 symbol_white_list
: &[*const libc
::c_char
],
196 ) -> Result
<LtoModuleCodegen
<LlvmCodegenBackend
>, FatalError
> {
197 let _timer
= cgcx
.prof
.generic_activity("LLVM_fat_lto_build_monolithic_module");
198 info
!("going for a fat lto");
200 // Sort out all our lists of incoming modules into two lists.
202 // * `serialized_modules` (also and argument to this function) contains all
203 // modules that are serialized in-memory.
204 // * `in_memory` contains modules which are already parsed and in-memory,
205 // such as from multi-CGU builds.
207 // All of `cached_modules` (cached from previous incremental builds) can
208 // immediately go onto the `serialized_modules` modules list and then we can
209 // split the `modules` array into these two lists.
210 let mut in_memory
= Vec
::new();
211 serialized_modules
.extend(cached_modules
.into_iter().map(|(buffer
, wp
)| {
212 info
!("pushing cached module {:?}", wp
.cgu_name
);
213 (buffer
, CString
::new(wp
.cgu_name
).unwrap())
215 for module
in modules
{
217 FatLTOInput
::InMemory(m
) => in_memory
.push(m
),
218 FatLTOInput
::Serialized { name, buffer }
=> {
219 info
!("pushing serialized module {:?}", name
);
220 let buffer
= SerializedModule
::Local(buffer
);
221 serialized_modules
.push((buffer
, CString
::new(name
).unwrap()));
226 // Find the "costliest" module and merge everything into that codegen unit.
227 // All the other modules will be serialized and reparsed into the new
228 // context, so this hopefully avoids serializing and parsing the largest
231 // Additionally use a regular module as the base here to ensure that various
232 // file copy operations in the backend work correctly. The only other kind
233 // of module here should be an allocator one, and if your crate is smaller
234 // than the allocator module then the size doesn't really matter anyway.
235 let costliest_module
= in_memory
238 .filter(|&(_
, module
)| module
.kind
== ModuleKind
::Regular
)
240 let cost
= unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }
;
245 // If we found a costliest module, we're good to go. Otherwise all our
246 // inputs were serialized which could happen in the case, for example, that
247 // all our inputs were incrementally reread from the cache and we're just
248 // re-executing the LTO passes. If that's the case deserialize the first
249 // module and create a linker with it.
250 let module
: ModuleCodegen
<ModuleLlvm
> = match costliest_module
{
251 Some((_cost
, i
)) => in_memory
.remove(i
),
253 assert
!(!serialized_modules
.is_empty(), "must have at least one serialized module");
254 let (buffer
, name
) = serialized_modules
.remove(0);
255 info
!("no in-memory regular modules to choose from, parsing {:?}", name
);
257 module_llvm
: ModuleLlvm
::parse(cgcx
, &name
, buffer
.data(), diag_handler
)?
,
258 name
: name
.into_string().unwrap(),
259 kind
: ModuleKind
::Regular
,
263 let mut serialized_bitcode
= Vec
::new();
265 let (llcx
, llmod
) = {
266 let llvm
= &module
.module_llvm
;
267 (&llvm
.llcx
, llvm
.llmod())
269 info
!("using {:?} as a base module", module
.name
);
271 // The linking steps below may produce errors and diagnostics within LLVM
272 // which we'd like to handle and print, so set up our diagnostic handlers
273 // (which get unregistered when they go out of scope below).
274 let _handler
= DiagnosticHandlers
::new(cgcx
, diag_handler
, llcx
);
276 // For all other modules we codegened we'll need to link them into our own
277 // bitcode. All modules were codegened in their own LLVM context, however,
278 // and we want to move everything to the same LLVM context. Currently the
279 // way we know of to do that is to serialize them to a string and them parse
280 // them later. Not great but hey, that's why it's "fat" LTO, right?
281 for module
in in_memory
{
282 let buffer
= ModuleBuffer
::new(module
.module_llvm
.llmod());
283 let llmod_id
= CString
::new(&module
.name
[..]).unwrap();
284 serialized_modules
.push((SerializedModule
::Local(buffer
), llmod_id
));
286 // Sort the modules to ensure we produce deterministic results.
287 serialized_modules
.sort_by(|module1
, module2
| module1
.1
.cmp(&module2
.1
));
289 // For all serialized bitcode files we parse them and link them in as we did
290 // above, this is all mostly handled in C++. Like above, though, we don't
291 // know much about the memory management here so we err on the side of being
292 // save and persist everything with the original module.
293 let mut linker
= Linker
::new(llmod
);
294 for (bc_decoded
, name
) in serialized_modules
{
297 .generic_activity_with_arg("LLVM_fat_lto_link_module", format
!("{:?}", name
));
298 info
!("linking {:?}", name
);
299 let data
= bc_decoded
.data();
300 linker
.add(&data
).map_err(|()| {
301 let msg
= format
!("failed to load bc of {:?}", name
);
302 write
::llvm_err(&diag_handler
, &msg
)
304 serialized_bitcode
.push(bc_decoded
);
307 save_temp_bitcode(&cgcx
, &module
, "lto.input");
309 // Internalize everything that *isn't* in our whitelist to help strip out
310 // more modules and such
312 let ptr
= symbol_white_list
.as_ptr();
313 llvm
::LLVMRustRunRestrictionPass(
315 ptr
as *const *const libc
::c_char
,
316 symbol_white_list
.len() as libc
::size_t
,
318 save_temp_bitcode(&cgcx
, &module
, "lto.after-restriction");
321 if cgcx
.no_landing_pads
{
323 llvm
::LLVMRustMarkAllFunctionsNounwind(llmod
);
325 save_temp_bitcode(&cgcx
, &module
, "lto.after-nounwind");
329 Ok(LtoModuleCodegen
::Fat { module: Some(module), _serialized_bitcode: serialized_bitcode }
)
332 struct Linker
<'a
>(&'a
mut llvm
::Linker
<'a
>);
335 fn new(llmod
: &'a llvm
::Module
) -> Self {
336 unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) }
339 fn add(&mut self, bytecode
: &[u8]) -> Result
<(), ()> {
341 if llvm
::LLVMRustLinkerAdd(
343 bytecode
.as_ptr() as *const libc
::c_char
,
354 impl Drop
for Linker
<'a
> {
357 llvm
::LLVMRustLinkerFree(&mut *(self.0 as *mut _
));
362 /// Prepare "thin" LTO to get run on these modules.
364 /// The general structure of ThinLTO is quite different from the structure of
365 /// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into
366 /// one giant LLVM module, and then we run more optimization passes over this
367 /// big module after internalizing most symbols. Thin LTO, on the other hand,
368 /// avoid this large bottleneck through more targeted optimization.
370 /// At a high level Thin LTO looks like:
372 /// 1. Prepare a "summary" of each LLVM module in question which describes
373 /// the values inside, cost of the values, etc.
374 /// 2. Merge the summaries of all modules in question into one "index"
375 /// 3. Perform some global analysis on this index
376 /// 4. For each module, use the index and analysis calculated previously to
377 /// perform local transformations on the module, for example inlining
378 /// small functions from other modules.
379 /// 5. Run thin-specific optimization passes over each module, and then code
380 /// generate everything at the end.
382 /// The summary for each module is intended to be quite cheap, and the global
383 /// index is relatively quite cheap to create as well. As a result, the goal of
384 /// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more
385 /// situations. For example one cheap optimization is that we can parallelize
386 /// all codegen modules, easily making use of all the cores on a machine.
388 /// With all that in mind, the function here is designed at specifically just
389 /// calculating the *index* for ThinLTO. This index will then be shared amongst
390 /// all of the `LtoModuleCodegen` units returned below and destroyed once
391 /// they all go out of scope.
393 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
394 diag_handler
: &Handler
,
395 modules
: Vec
<(String
, ThinBuffer
)>,
396 serialized_modules
: Vec
<(SerializedModule
<ModuleBuffer
>, CString
)>,
397 cached_modules
: Vec
<(SerializedModule
<ModuleBuffer
>, WorkProduct
)>,
398 symbol_white_list
: &[*const libc
::c_char
],
399 ) -> Result
<(Vec
<LtoModuleCodegen
<LlvmCodegenBackend
>>, Vec
<WorkProduct
>), FatalError
> {
400 let _timer
= cgcx
.prof
.generic_activity("LLVM_thin_lto_global_analysis");
402 info
!("going for that thin, thin LTO");
404 let green_modules
: FxHashMap
<_
, _
> =
405 cached_modules
.iter().map(|&(_
, ref wp
)| (wp
.cgu_name
.clone(), wp
.clone())).collect();
407 let full_scope_len
= modules
.len() + serialized_modules
.len() + cached_modules
.len();
408 let mut thin_buffers
= Vec
::with_capacity(modules
.len());
409 let mut module_names
= Vec
::with_capacity(full_scope_len
);
410 let mut thin_modules
= Vec
::with_capacity(full_scope_len
);
412 for (i
, (name
, buffer
)) in modules
.into_iter().enumerate() {
413 info
!("local module: {} - {}", i
, name
);
414 let cname
= CString
::new(name
.clone()).unwrap();
415 thin_modules
.push(llvm
::ThinLTOModule
{
416 identifier
: cname
.as_ptr(),
417 data
: buffer
.data().as_ptr(),
418 len
: buffer
.data().len(),
420 thin_buffers
.push(buffer
);
421 module_names
.push(cname
);
424 // FIXME: All upstream crates are deserialized internally in the
425 // function below to extract their summary and modules. Note that
426 // unlike the loop above we *must* decode and/or read something
427 // here as these are all just serialized files on disk. An
428 // improvement, however, to make here would be to store the
429 // module summary separately from the actual module itself. Right
430 // now this is store in one large bitcode file, and the entire
431 // file is deflate-compressed. We could try to bypass some of the
432 // decompression by storing the index uncompressed and only
433 // lazily decompressing the bytecode if necessary.
435 // Note that truly taking advantage of this optimization will
436 // likely be further down the road. We'd have to implement
437 // incremental ThinLTO first where we could actually avoid
438 // looking at upstream modules entirely sometimes (the contents,
439 // we must always unconditionally look at the index).
440 let mut serialized
= Vec
::with_capacity(serialized_modules
.len() + cached_modules
.len());
443 cached_modules
.into_iter().map(|(sm
, wp
)| (sm
, CString
::new(wp
.cgu_name
).unwrap()));
445 for (module
, name
) in serialized_modules
.into_iter().chain(cached_modules
) {
446 info
!("upstream or cached module {:?}", name
);
447 thin_modules
.push(llvm
::ThinLTOModule
{
448 identifier
: name
.as_ptr(),
449 data
: module
.data().as_ptr(),
450 len
: module
.data().len(),
452 serialized
.push(module
);
453 module_names
.push(name
);
457 assert_eq
!(thin_modules
.len(), module_names
.len());
459 // Delegate to the C++ bindings to create some data here. Once this is a
460 // tried-and-true interface we may wish to try to upstream some of this
461 // to LLVM itself, right now we reimplement a lot of what they do
463 let data
= llvm
::LLVMRustCreateThinLTOData(
464 thin_modules
.as_ptr(),
465 thin_modules
.len() as u32,
466 symbol_white_list
.as_ptr(),
467 symbol_white_list
.len() as u32,
469 .ok_or_else(|| write
::llvm_err(&diag_handler
, "failed to prepare thin LTO context"))?
;
471 info
!("thin LTO data created");
473 let (import_map_path
, prev_import_map
, curr_import_map
) =
474 if let Some(ref incr_comp_session_dir
) = cgcx
.incr_comp_session_dir
{
475 let path
= incr_comp_session_dir
.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME
);
476 // If previous imports have been deleted, or we get an IO error
477 // reading the file storing them, then we'll just use `None` as the
478 // prev_import_map, which will force the code to be recompiled.
479 let prev
= if path
.exists() {
480 ThinLTOImportMaps
::load_from_file(&path
).ok()
484 let curr
= ThinLTOImportMaps
::from_thin_lto_data(data
);
485 (Some(path
), prev
, curr
)
487 // If we don't compile incrementally, we don't need to load the
488 // import data from LLVM.
489 assert
!(green_modules
.is_empty());
490 let curr
= ThinLTOImportMaps
::default();
493 info
!("thin LTO import map loaded");
495 let data
= ThinData(data
);
497 // Throw our data in an `Arc` as we'll be sharing it across threads. We
498 // also put all memory referenced by the C++ data (buffers, ids, etc)
499 // into the arc as well. After this we'll create a thin module
500 // codegen per module in this data.
501 let shared
= Arc
::new(ThinShared
{
504 serialized_modules
: serialized
,
508 let mut copy_jobs
= vec
![];
509 let mut opt_jobs
= vec
![];
511 info
!("checking which modules can be-reused and which have to be re-optimized.");
512 for (module_index
, module_name
) in shared
.module_names
.iter().enumerate() {
513 let module_name
= module_name_to_str(module_name
);
515 // If (1.) the module hasn't changed, and (2.) none of the modules
516 // it imports from have changed, *and* (3.) the import and export
517 // sets themselves have not changed from the previous compile when
518 // it was last ThinLTO'ed, then we can re-use the post-ThinLTO
519 // version of the module. Otherwise, freshly perform LTO
522 // (Note that globally, the export set is just the inverse of the
525 // For further justification of why the above is necessary and sufficient,
526 // see the LLVM blog post on ThinLTO:
528 // http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html
530 // which states the following:
533 // any particular ThinLTO backend must be redone iff:
535 // 1. The corresponding (primary) module’s bitcode changed
536 // 2. The list of imports into or exports from the module changed
537 // 3. The bitcode for any module being imported from has changed
538 // 4. Any global analysis result affecting either the primary module
539 // or anything it imports has changed.
542 // This strategy means we can always save the computed imports as
543 // canon: when we reuse the post-ThinLTO version, condition (3.)
544 // ensures that the current import set is the same as the previous
545 // one. (And of course, when we don't reuse the post-ThinLTO
546 // version, the current import set *is* the correct one, since we
547 // are doing the ThinLTO in this current compilation cycle.)
549 // For more discussion, see rust-lang/rust#59535 (where the import
550 // issue was discovered) and rust-lang/rust#69798 (where the
551 // analogous export issue was discovered).
552 if let (Some(prev_import_map
), true) =
553 (prev_import_map
.as_ref(), green_modules
.contains_key(module_name
))
555 assert
!(cgcx
.incr_comp_session_dir
.is_some());
557 let prev_imports
= prev_import_map
.imports_of(module_name
);
558 let curr_imports
= curr_import_map
.imports_of(module_name
);
559 let prev_exports
= prev_import_map
.exports_of(module_name
);
560 let curr_exports
= curr_import_map
.exports_of(module_name
);
561 let imports_all_green
= curr_imports
563 .all(|imported_module
| green_modules
.contains_key(imported_module
));
565 && equivalent_as_sets(prev_imports
, curr_imports
)
566 && equivalent_as_sets(prev_exports
, curr_exports
)
568 let work_product
= green_modules
[module_name
].clone();
569 copy_jobs
.push(work_product
);
570 info
!(" - {}: re-used", module_name
);
571 assert
!(cgcx
.incr_comp_session_dir
.is_some());
572 cgcx
.cgu_reuse_tracker
.set_actual_reuse(module_name
, CguReuse
::PostLto
);
577 info
!(" - {}: re-compiled", module_name
);
578 opt_jobs
.push(LtoModuleCodegen
::Thin(ThinModule
{
579 shared
: shared
.clone(),
584 // Save the current ThinLTO import information for the next compilation
585 // session, overwriting the previous serialized imports (if any).
586 if let Some(path
) = import_map_path
{
587 if let Err(err
) = curr_import_map
.save_to_file(&path
) {
588 let msg
= format
!("Error while writing ThinLTO import data: {}", err
);
589 return Err(write
::llvm_err(&diag_handler
, &msg
));
593 Ok((opt_jobs
, copy_jobs
))
597 /// Given two slices, each with no repeat elements. returns true if and only if
598 /// the two slices have the same contents when considered as sets (i.e. when
599 /// element order is disregarded).
600 fn equivalent_as_sets(a
: &[String
], b
: &[String
]) -> bool
{
601 // cheap path: unequal lengths means cannot possibly be set equivalent.
602 if a
.len() != b
.len() {
605 // fast path: before building new things, check if inputs are equivalent as is.
609 // slow path: general set comparison.
610 let a
: FxHashSet
<&str> = a
.iter().map(|s
| s
.as_str()).collect();
611 let b
: FxHashSet
<&str> = b
.iter().map(|s
| s
.as_str()).collect();
615 pub(crate) fn run_pass_manager(
616 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
617 module
: &ModuleCodegen
<ModuleLlvm
>,
618 config
: &ModuleConfig
,
621 let _timer
= cgcx
.prof
.extra_verbose_generic_activity("LLVM_lto_optimize", &module
.name
[..]);
623 // Now we have one massive module inside of llmod. Time to run the
624 // LTO-specific optimization passes that LLVM provides.
626 // This code is based off the code found in llvm's LTO code generator:
627 // tools/lto/LTOCodeGenerator.cpp
628 debug
!("running the pass manager");
630 if write
::should_use_new_llvm_pass_manager(config
) {
631 let opt_stage
= if thin { llvm::OptStage::ThinLTO }
else { llvm::OptStage::FatLTO }
;
632 let opt_level
= config
.opt_level
.unwrap_or(config
::OptLevel
::No
);
633 // See comment below for why this is necessary.
634 let opt_level
= if let config
::OptLevel
::No
= opt_level
{
635 config
::OptLevel
::Less
639 write
::optimize_with_new_llvm_pass_manager(cgcx
, module
, config
, opt_level
, opt_stage
);
644 let pm
= llvm
::LLVMCreatePassManager();
645 llvm
::LLVMAddAnalysisPasses(module
.module_llvm
.tm
, pm
);
647 if config
.verify_llvm_ir
{
648 let pass
= llvm
::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast());
649 llvm
::LLVMRustAddPass(pm
, pass
.unwrap());
652 // When optimizing for LTO we don't actually pass in `-O0`, but we force
653 // it to always happen at least with `-O1`.
655 // With ThinLTO we mess around a lot with symbol visibility in a way
656 // that will actually cause linking failures if we optimize at O0 which
657 // notable is lacking in dead code elimination. To ensure we at least
658 // get some optimizations and correctly link we forcibly switch to `-O1`
659 // to get dead code elimination.
661 // Note that in general this shouldn't matter too much as you typically
662 // only turn on ThinLTO when you're compiling with optimizations
664 let opt_level
= config
666 .map(|x
| to_llvm_opt_settings(x
).0)
667 .unwrap_or(llvm
::CodeGenOptLevel
::None
);
668 let opt_level
= match opt_level
{
669 llvm
::CodeGenOptLevel
::None
=> llvm
::CodeGenOptLevel
::Less
,
672 with_llvm_pmb(module
.module_llvm
.llmod(), config
, opt_level
, false, &mut |b
| {
674 llvm
::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b
, pm
);
676 llvm
::LLVMPassManagerBuilderPopulateLTOPassManager(
677 b
, pm
, /* Internalize = */ False
, /* RunInliner = */ True
,
682 // We always generate bitcode through ThinLTOBuffers,
683 // which do not support anonymous globals
684 if config
.bitcode_needed() {
685 let pass
= llvm
::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast());
686 llvm
::LLVMRustAddPass(pm
, pass
.unwrap());
689 if config
.verify_llvm_ir
{
690 let pass
= llvm
::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast());
691 llvm
::LLVMRustAddPass(pm
, pass
.unwrap());
694 llvm
::LLVMRunPassManager(pm
, module
.module_llvm
.llmod());
696 llvm
::LLVMDisposePassManager(pm
);
701 pub struct ModuleBuffer(&'
static mut llvm
::ModuleBuffer
);
703 unsafe impl Send
for ModuleBuffer {}
704 unsafe impl Sync
for ModuleBuffer {}
707 pub fn new(m
: &llvm
::Module
) -> ModuleBuffer
{
708 ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }
)
712 impl ModuleBufferMethods
for ModuleBuffer
{
713 fn data(&self) -> &[u8] {
715 let ptr
= llvm
::LLVMRustModuleBufferPtr(self.0);
716 let len
= llvm
::LLVMRustModuleBufferLen(self.0);
717 slice
::from_raw_parts(ptr
, len
)
722 impl Drop
for ModuleBuffer
{
725 llvm
::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _
));
730 pub struct ThinData(&'
static mut llvm
::ThinLTOData
);
732 unsafe impl Send
for ThinData {}
733 unsafe impl Sync
for ThinData {}
735 impl Drop
for ThinData
{
738 llvm
::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _
));
743 pub struct ThinBuffer(&'
static mut llvm
::ThinLTOBuffer
);
745 unsafe impl Send
for ThinBuffer {}
746 unsafe impl Sync
for ThinBuffer {}
749 pub fn new(m
: &llvm
::Module
) -> ThinBuffer
{
751 let buffer
= llvm
::LLVMRustThinLTOBufferCreate(m
);
757 impl ThinBufferMethods
for ThinBuffer
{
758 fn data(&self) -> &[u8] {
760 let ptr
= llvm
::LLVMRustThinLTOBufferPtr(self.0) as *const _
;
761 let len
= llvm
::LLVMRustThinLTOBufferLen(self.0);
762 slice
::from_raw_parts(ptr
, len
)
767 impl Drop
for ThinBuffer
{
770 llvm
::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _
));
775 pub unsafe fn optimize_thin_module(
776 thin_module
: &mut ThinModule
<LlvmCodegenBackend
>,
777 cgcx
: &CodegenContext
<LlvmCodegenBackend
>,
778 ) -> Result
<ModuleCodegen
<ModuleLlvm
>, FatalError
> {
779 let diag_handler
= cgcx
.create_diag_handler();
780 let tm
= (cgcx
.tm_factory
.0)().map_err(|e
| write
::llvm_err(&diag_handler
, &e
))?
;
782 // Right now the implementation we've got only works over serialized
783 // modules, so we create a fresh new LLVM context and parse the module
784 // into that context. One day, however, we may do this for upstream
785 // crates but for locally codegened modules we may be able to reuse
786 // that LLVM Context and Module.
787 let llcx
= llvm
::LLVMRustContextCreate(cgcx
.fewer_names
);
788 let llmod_raw
= parse_module(
790 &thin_module
.shared
.module_names
[thin_module
.idx
],
794 let module
= ModuleCodegen
{
795 module_llvm
: ModuleLlvm { llmod_raw, llcx, tm }
,
796 name
: thin_module
.name().to_string(),
797 kind
: ModuleKind
::Regular
,
800 let llmod
= module
.module_llvm
.llmod();
801 save_temp_bitcode(&cgcx
, &module
, "thin-lto-input");
803 // Before we do much else find the "main" `DICompileUnit` that we'll be
804 // using below. If we find more than one though then rustc has changed
805 // in a way we're not ready for, so generate an ICE by returning
807 let mut cu1
= ptr
::null_mut();
808 let mut cu2
= ptr
::null_mut();
809 llvm
::LLVMRustThinLTOGetDICompileUnit(llmod
, &mut cu1
, &mut cu2
);
811 let msg
= "multiple source DICompileUnits found";
812 return Err(write
::llvm_err(&diag_handler
, msg
));
815 // Like with "fat" LTO, get some better optimizations if landing pads
816 // are disabled by removing all landing pads.
817 if cgcx
.no_landing_pads
{
820 .generic_activity_with_arg("LLVM_thin_lto_remove_landing_pads", thin_module
.name());
821 llvm
::LLVMRustMarkAllFunctionsNounwind(llmod
);
822 save_temp_bitcode(&cgcx
, &module
, "thin-lto-after-nounwind");
825 // Up next comes the per-module local analyses that we do for Thin LTO.
826 // Each of these functions is basically copied from the LLVM
827 // implementation and then tailored to suit this implementation. Ideally
828 // each of these would be supported by upstream LLVM but that's perhaps
829 // a patch for another day!
831 // You can find some more comments about these functions in the LLVM
832 // bindings we've got (currently `PassWrapper.cpp`)
835 cgcx
.prof
.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module
.name());
836 if !llvm
::LLVMRustPrepareThinLTORename(thin_module
.shared
.data
.0, llmod
) {
837 let msg
= "failed to prepare thin LTO module";
838 return Err(write
::llvm_err(&diag_handler
, msg
));
840 save_temp_bitcode(cgcx
, &module
, "thin-lto-after-rename");
846 .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module
.name());
847 if !llvm
::LLVMRustPrepareThinLTOResolveWeak(thin_module
.shared
.data
.0, llmod
) {
848 let msg
= "failed to prepare thin LTO module";
849 return Err(write
::llvm_err(&diag_handler
, msg
));
851 save_temp_bitcode(cgcx
, &module
, "thin-lto-after-resolve");
857 .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module
.name());
858 if !llvm
::LLVMRustPrepareThinLTOInternalize(thin_module
.shared
.data
.0, llmod
) {
859 let msg
= "failed to prepare thin LTO module";
860 return Err(write
::llvm_err(&diag_handler
, msg
));
862 save_temp_bitcode(cgcx
, &module
, "thin-lto-after-internalize");
867 cgcx
.prof
.generic_activity_with_arg("LLVM_thin_lto_import", thin_module
.name());
868 if !llvm
::LLVMRustPrepareThinLTOImport(thin_module
.shared
.data
.0, llmod
) {
869 let msg
= "failed to prepare thin LTO module";
870 return Err(write
::llvm_err(&diag_handler
, msg
));
872 save_temp_bitcode(cgcx
, &module
, "thin-lto-after-import");
875 // Ok now this is a bit unfortunate. This is also something you won't
876 // find upstream in LLVM's ThinLTO passes! This is a hack for now to
877 // work around bugs in LLVM.
879 // First discovered in #45511 it was found that as part of ThinLTO
880 // importing passes LLVM will import `DICompileUnit` metadata
881 // information across modules. This means that we'll be working with one
882 // LLVM module that has multiple `DICompileUnit` instances in it (a
883 // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of
884 // bugs in LLVM's backend which generates invalid DWARF in a situation
887 // https://bugs.llvm.org/show_bug.cgi?id=35212
888 // https://bugs.llvm.org/show_bug.cgi?id=35562
890 // While the first bug there is fixed the second ended up causing #46346
891 // which was basically a resurgence of #45511 after LLVM's bug 35212 was
894 // This function below is a huge hack around this problem. The function
895 // below is defined in `PassWrapper.cpp` and will basically "merge"
896 // all `DICompileUnit` instances in a module. Basically it'll take all
897 // the objects, rewrite all pointers of `DISubprogram` to point to the
898 // first `DICompileUnit`, and then delete all the other units.
900 // This is probably mangling to the debug info slightly (but hopefully
901 // not too much) but for now at least gets LLVM to emit valid DWARF (or
902 // so it appears). Hopefully we can remove this once upstream bugs are
907 .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module
.name());
908 llvm
::LLVMRustThinLTOPatchDICompileUnit(llmod
, cu1
);
909 save_temp_bitcode(cgcx
, &module
, "thin-lto-after-patch");
912 // Alright now that we've done everything related to the ThinLTO
913 // analysis it's time to run some optimizations! Here we use the same
914 // `run_pass_manager` as the "fat" LTO above except that we tell it to
915 // populate a thin-specific pass manager, which presumably LLVM treats a
916 // little differently.
918 info
!("running thin lto passes over {}", module
.name
);
919 let config
= cgcx
.config(module
.kind
);
920 run_pass_manager(cgcx
, &module
, config
, true);
921 save_temp_bitcode(cgcx
, &module
, "thin-lto-after-pm");
927 /// Summarizes module import/export relationships used by LLVM's ThinLTO pass.
929 /// Note that we tend to have two such instances of `ThinLTOImportMaps` in use:
930 /// one loaded from a file that represents the relationships used during the
931 /// compilation associated with the incremetnal build artifacts we are
932 /// attempting to reuse, and another constructed via `from_thin_lto_data`, which
933 /// captures the relationships of ThinLTO in the current compilation.
934 #[derive(Debug, Default)]
935 pub struct ThinLTOImportMaps
{
936 // key = llvm name of importing module, value = list of modules it imports from
937 imports
: FxHashMap
<String
, Vec
<String
>>,
938 // key = llvm name of exporting module, value = list of modules it exports to
939 exports
: FxHashMap
<String
, Vec
<String
>>,
942 impl ThinLTOImportMaps
{
943 /// Returns modules imported by `llvm_module_name` during some ThinLTO pass.
944 fn imports_of(&self, llvm_module_name
: &str) -> &[String
] {
945 self.imports
.get(llvm_module_name
).map(|v
| &v
[..]).unwrap_or(&[])
948 /// Returns modules exported by `llvm_module_name` during some ThinLTO pass.
949 fn exports_of(&self, llvm_module_name
: &str) -> &[String
] {
950 self.exports
.get(llvm_module_name
).map(|v
| &v
[..]).unwrap_or(&[])
953 fn save_to_file(&self, path
: &Path
) -> io
::Result
<()> {
955 let file
= File
::create(path
)?
;
956 let mut writer
= io
::BufWriter
::new(file
);
957 for (importing_module_name
, imported_modules
) in &self.imports
{
958 writeln
!(writer
, "{}", importing_module_name
)?
;
959 for imported_module
in imported_modules
{
960 writeln
!(writer
, " {}", imported_module
)?
;
967 fn load_from_file(path
: &Path
) -> io
::Result
<ThinLTOImportMaps
> {
968 use std
::io
::BufRead
;
969 let mut imports
= FxHashMap
::default();
970 let mut exports
: FxHashMap
<_
, Vec
<_
>> = FxHashMap
::default();
971 let mut current_module
: Option
<String
> = None
;
972 let mut current_imports
: Vec
<String
> = vec
![];
973 let file
= File
::open(path
)?
;
974 for line
in io
::BufReader
::new(file
).lines() {
977 let importing_module
= current_module
.take().expect("Importing module not set");
978 for imported
in ¤t_imports
{
979 exports
.entry(imported
.clone()).or_default().push(importing_module
.clone());
981 imports
.insert(importing_module
, mem
::replace(&mut current_imports
, vec
![]));
982 } else if line
.starts_with(' '
) {
983 // Space marks an imported module
984 assert_ne
!(current_module
, None
);
985 current_imports
.push(line
.trim().to_string());
987 // Otherwise, beginning of a new module (must be start or follow empty line)
988 assert_eq
!(current_module
, None
);
989 current_module
= Some(line
.trim().to_string());
992 Ok(ThinLTOImportMaps { imports, exports }
)
995 /// Loads the ThinLTO import map from ThinLTOData.
996 unsafe fn from_thin_lto_data(data
: *const llvm
::ThinLTOData
) -> ThinLTOImportMaps
{
997 unsafe extern "C" fn imported_module_callback(
998 payload
: *mut libc
::c_void
,
999 importing_module_name
: *const libc
::c_char
,
1000 imported_module_name
: *const libc
::c_char
,
1002 let map
= &mut *(payload
as *mut ThinLTOImportMaps
);
1003 let importing_module_name
= CStr
::from_ptr(importing_module_name
);
1004 let importing_module_name
= module_name_to_str(&importing_module_name
);
1005 let imported_module_name
= CStr
::from_ptr(imported_module_name
);
1006 let imported_module_name
= module_name_to_str(&imported_module_name
);
1008 if !map
.imports
.contains_key(importing_module_name
) {
1009 map
.imports
.insert(importing_module_name
.to_owned(), vec
![]);
1013 .get_mut(importing_module_name
)
1015 .push(imported_module_name
.to_owned());
1017 if !map
.exports
.contains_key(imported_module_name
) {
1018 map
.exports
.insert(imported_module_name
.to_owned(), vec
![]);
1022 .get_mut(imported_module_name
)
1024 .push(importing_module_name
.to_owned());
1027 let mut map
= ThinLTOImportMaps
::default();
1028 llvm
::LLVMRustGetThinLTOModuleImports(
1030 imported_module_callback
,
1031 &mut map
as *mut _
as *mut libc
::c_void
,
1037 fn module_name_to_str(c_str
: &CStr
) -> &str {
1038 c_str
.to_str().unwrap_or_else(|e
| {
1039 bug
!("Encountered non-utf8 LLVM module name `{}`: {}", c_str
.to_string_lossy(), e
)
1043 pub fn parse_module
<'a
>(
1044 cx
: &'a llvm
::Context
,
1047 diag_handler
: &Handler
,
1048 ) -> Result
<&'a llvm
::Module
, FatalError
> {
1050 llvm
::LLVMRustParseBitcodeForLTO(cx
, data
.as_ptr(), data
.len(), name
.as_ptr()).ok_or_else(
1052 let msg
= "failed to parse bitcode for LTO module";
1053 write
::llvm_err(&diag_handler
, msg
)