]>
Commit | Line | Data |
---|---|---|
dfeec247 XL |
1 | use crate::back::write::{ |
2 | self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, | |
3 | }; | |
9fa01778 | 4 | use crate::llvm::archive_ro::ArchiveRO; |
dfeec247 XL |
5 | use crate::llvm::{self, False, True}; |
6 | use crate::{LlvmCodegenBackend, ModuleLlvm}; | |
7 | use log::{debug, info}; | |
dfeec247 | 8 | use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; |
a1dfa0c6 | 9 | use rustc_codegen_ssa::back::symbol_export; |
dfeec247 | 10 | use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, ModuleConfig}; |
a1dfa0c6 | 11 | use rustc_codegen_ssa::traits::*; |
f9f354fc | 12 | use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; |
dfeec247 | 13 | use rustc_data_structures::fx::{FxHashMap, FxHashSet}; |
60c5eb7d | 14 | use rustc_errors::{FatalError, Handler}; |
dfeec247 | 15 | use rustc_hir::def_id::LOCAL_CRATE; |
ba9703b0 XL |
16 | use rustc_middle::bug; |
17 | use rustc_middle::dep_graph::WorkProduct; | |
18 | use rustc_middle::middle::exported_symbols::SymbolExportLevel; | |
60c5eb7d | 19 | use rustc_session::cgu_reuse_tracker::CguReuse; |
f9f354fc | 20 | use rustc_session::config::{self, CrateType, Lto}; |
1a4d82fc | 21 | |
b7449926 | 22 | use std::ffi::{CStr, CString}; |
dfeec247 XL |
23 | use std::fs::File; |
24 | use std::io; | |
25 | use std::mem; | |
26 | use std::path::Path; | |
ff7c6d11 | 27 | use std::ptr; |
ea8adc8c XL |
28 | use std::slice; |
29 | use std::sync::Arc; | |
1a4d82fc | 30 | |
dfeec247 XL |
31 | /// We keep track of past LTO imports that were used to produce the current set |
32 | /// of compiled object files that we might choose to reuse during this | |
33 | /// compilation session. | |
34 | pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-imports.bin"; | |
35 | ||
f9f354fc | 36 | pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { |
476ff2be | 37 | match crate_type { |
f9f354fc XL |
38 | CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, |
39 | CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, | |
476ff2be SL |
40 | } |
41 | } | |
42 | ||
dfeec247 XL |
43 | fn prepare_lto( |
44 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
45 | diag_handler: &Handler, | |
46 | ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> { | |
2c00a5a8 XL |
47 | let export_threshold = match cgcx.lto { |
48 | // We're just doing LTO for our one crate | |
49 | Lto::ThinLocal => SymbolExportLevel::Rust, | |
50 | ||
51 | // We're doing LTO for the entire crate graph | |
dfeec247 | 52 | Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types), |
2c00a5a8 XL |
53 | |
54 | Lto::No => panic!("didn't request LTO but we're doing LTO"), | |
ea8adc8c XL |
55 | }; |
56 | ||
0531ce1d | 57 | let symbol_filter = &|&(ref name, level): &(String, SymbolExportLevel)| { |
ea8adc8c | 58 | if level.is_below_threshold(export_threshold) { |
e74abb32 | 59 | Some(CString::new(name.as_str()).unwrap()) |
476ff2be SL |
60 | } else { |
61 | None | |
62 | } | |
63 | }; | |
dfeec247 | 64 | let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); |
f035d41b XL |
65 | let mut symbols_below_threshold = { |
66 | let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); | |
dfeec247 | 67 | exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>() |
e74abb32 | 68 | }; |
f035d41b | 69 | info!("{} symbols to preserve in this crate", symbols_below_threshold.len()); |
ea8adc8c XL |
70 | |
71 | // If we're performing LTO for the entire crate graph, then for each of our | |
72 | // upstream dependencies, find the corresponding rlib and load the bitcode | |
73 | // from the archive. | |
74 | // | |
75 | // We save off all the bytecode and LLVM module ids for later processing | |
76 | // with either fat or thin LTO | |
77 | let mut upstream_modules = Vec::new(); | |
2c00a5a8 | 78 | if cgcx.lto != Lto::ThinLocal { |
ea8adc8c | 79 | if cgcx.opts.cg.prefer_dynamic { |
dfeec247 XL |
80 | diag_handler |
81 | .struct_err("cannot prefer dynamic linking when performing LTO") | |
82 | .note( | |
83 | "only 'staticlib', 'bin', and 'cdylib' outputs are \ | |
84 | supported with LTO", | |
85 | ) | |
86 | .emit(); | |
87 | return Err(FatalError); | |
ea8adc8c XL |
88 | } |
89 | ||
90 | // Make sure we actually can run LTO | |
91 | for crate_type in cgcx.crate_types.iter() { | |
92 | if !crate_type_allows_lto(*crate_type) { | |
dfeec247 XL |
93 | let e = diag_handler.fatal( |
94 | "lto can only be run for executables, cdylibs and \ | |
95 | static library outputs", | |
96 | ); | |
97 | return Err(e); | |
ea8adc8c XL |
98 | } |
99 | } | |
100 | ||
101 | for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { | |
dfeec247 XL |
102 | let exported_symbols = |
103 | cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); | |
e74abb32 | 104 | { |
f035d41b XL |
105 | let _timer = |
106 | cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); | |
107 | symbols_below_threshold | |
108 | .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); | |
e74abb32 | 109 | } |
ea8adc8c XL |
110 | |
111 | let archive = ArchiveRO::open(&path).expect("wanted an rlib"); | |
f9f354fc | 112 | let obj_files = archive |
dfeec247 XL |
113 | .iter() |
114 | .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) | |
f9f354fc XL |
115 | .filter(|&(name, _)| looks_like_rust_object_file(name)); |
116 | for (name, child) in obj_files { | |
117 | info!("adding bitcode from {}", name); | |
118 | match get_bitcode_slice_from_object_data(child.data()) { | |
119 | Ok(data) => { | |
120 | let module = SerializedModule::FromRlib(data.to_vec()); | |
121 | upstream_modules.push((module, CString::new(name).unwrap())); | |
122 | } | |
123 | Err(msg) => return Err(diag_handler.fatal(&msg)), | |
124 | } | |
ea8adc8c | 125 | } |
ea8adc8c XL |
126 | } |
127 | } | |
1a4d82fc | 128 | |
f035d41b | 129 | Ok((symbols_below_threshold, upstream_modules)) |
0731742a XL |
130 | } |
131 | ||
f9f354fc XL |
132 | fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { |
133 | let mut len = 0; | |
134 | let data = | |
135 | unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; | |
136 | if !data.is_null() { | |
137 | assert!(len != 0); | |
138 | let bc = unsafe { slice::from_raw_parts(data, len) }; | |
139 | ||
140 | // `bc` must be a sub-slice of `obj`. | |
141 | assert!(obj.as_ptr() <= bc.as_ptr()); | |
142 | assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr()); | |
143 | ||
144 | Ok(bc) | |
145 | } else { | |
146 | assert!(len == 0); | |
147 | let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); | |
148 | Err(format!("failed to get bitcode from object file for LTO ({})", msg)) | |
149 | } | |
150 | } | |
151 | ||
0731742a XL |
152 | /// Performs fat LTO by merging all modules into a single one and returning it |
153 | /// for further optimization. | |
dfeec247 XL |
154 | pub(crate) fn run_fat( |
155 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
156 | modules: Vec<FatLTOInput<LlvmCodegenBackend>>, | |
157 | cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, | |
158 | ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> { | |
0731742a | 159 | let diag_handler = cgcx.create_diag_handler(); |
f035d41b XL |
160 | let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; |
161 | let symbols_below_threshold = | |
162 | symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); | |
163 | fat_lto( | |
164 | cgcx, | |
165 | &diag_handler, | |
166 | modules, | |
167 | cached_modules, | |
168 | upstream_modules, | |
169 | &symbols_below_threshold, | |
170 | ) | |
0731742a XL |
171 | } |
172 | ||
173 | /// Performs thin LTO by performing necessary global analysis and returning two | |
174 | /// lists, one of the modules that need optimization and another for modules that | |
175 | /// can simply be copied over from the incr. comp. cache. | |
dfeec247 XL |
176 | pub(crate) fn run_thin( |
177 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
178 | modules: Vec<(String, ThinBuffer)>, | |
179 | cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, | |
180 | ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { | |
0731742a | 181 | let diag_handler = cgcx.create_diag_handler(); |
f035d41b XL |
182 | let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; |
183 | let symbols_below_threshold = | |
184 | symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); | |
9fa01778 | 185 | if cgcx.opts.cg.linker_plugin_lto.enabled() { |
dfeec247 XL |
186 | unreachable!( |
187 | "We should never reach this case if the LTO step \ | |
188 | is deferred to the linker" | |
189 | ); | |
0731742a | 190 | } |
f035d41b XL |
191 | thin_lto( |
192 | cgcx, | |
193 | &diag_handler, | |
194 | modules, | |
195 | upstream_modules, | |
196 | cached_modules, | |
197 | &symbols_below_threshold, | |
198 | ) | |
0731742a XL |
199 | } |
200 | ||
dfeec247 | 201 | pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { |
0731742a XL |
202 | let name = module.name.clone(); |
203 | let buffer = ThinBuffer::new(module.module_llvm.llmod()); | |
0731742a | 204 | (name, buffer) |
ea8adc8c XL |
205 | } |
206 | ||
dfeec247 XL |
207 | fn fat_lto( |
208 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
209 | diag_handler: &Handler, | |
210 | modules: Vec<FatLTOInput<LlvmCodegenBackend>>, | |
211 | cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, | |
212 | mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, | |
f035d41b | 213 | symbols_below_threshold: &[*const libc::c_char], |
dfeec247 | 214 | ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> { |
e74abb32 | 215 | let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); |
ea8adc8c XL |
216 | info!("going for a fat lto"); |
217 | ||
e1599b0c XL |
218 | // Sort out all our lists of incoming modules into two lists. |
219 | // | |
220 | // * `serialized_modules` (also and argument to this function) contains all | |
221 | // modules that are serialized in-memory. | |
222 | // * `in_memory` contains modules which are already parsed and in-memory, | |
223 | // such as from multi-CGU builds. | |
224 | // | |
225 | // All of `cached_modules` (cached from previous incremental builds) can | |
226 | // immediately go onto the `serialized_modules` modules list and then we can | |
227 | // split the `modules` array into these two lists. | |
228 | let mut in_memory = Vec::new(); | |
229 | serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { | |
230 | info!("pushing cached module {:?}", wp.cgu_name); | |
231 | (buffer, CString::new(wp.cgu_name).unwrap()) | |
232 | })); | |
233 | for module in modules { | |
234 | match module { | |
235 | FatLTOInput::InMemory(m) => in_memory.push(m), | |
236 | FatLTOInput::Serialized { name, buffer } => { | |
237 | info!("pushing serialized module {:?}", name); | |
238 | let buffer = SerializedModule::Local(buffer); | |
239 | serialized_modules.push((buffer, CString::new(name).unwrap())); | |
240 | } | |
241 | } | |
242 | } | |
243 | ||
ea8adc8c XL |
244 | // Find the "costliest" module and merge everything into that codegen unit. |
245 | // All the other modules will be serialized and reparsed into the new | |
246 | // context, so this hopefully avoids serializing and parsing the largest | |
247 | // codegen unit. | |
248 | // | |
249 | // Additionally use a regular module as the base here to ensure that various | |
250 | // file copy operations in the backend work correctly. The only other kind | |
251 | // of module here should be an allocator one, and if your crate is smaller | |
252 | // than the allocator module then the size doesn't really matter anyway. | |
dfeec247 XL |
253 | let costliest_module = in_memory |
254 | .iter() | |
ea8adc8c XL |
255 | .enumerate() |
256 | .filter(|&(_, module)| module.kind == ModuleKind::Regular) | |
257 | .map(|(i, module)| { | |
dfeec247 | 258 | let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; |
ea8adc8c XL |
259 | (cost, i) |
260 | }) | |
9fa01778 XL |
261 | .max(); |
262 | ||
263 | // If we found a costliest module, we're good to go. Otherwise all our | |
264 | // inputs were serialized which could happen in the case, for example, that | |
265 | // all our inputs were incrementally reread from the cache and we're just | |
266 | // re-executing the LTO passes. If that's the case deserialize the first | |
267 | // module and create a linker with it. | |
268 | let module: ModuleCodegen<ModuleLlvm> = match costliest_module { | |
e1599b0c | 269 | Some((_cost, i)) => in_memory.remove(i), |
9fa01778 | 270 | None => { |
74b04a01 | 271 | assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); |
e1599b0c XL |
272 | let (buffer, name) = serialized_modules.remove(0); |
273 | info!("no in-memory regular modules to choose from, parsing {:?}", name); | |
9fa01778 | 274 | ModuleCodegen { |
e1599b0c XL |
275 | module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?, |
276 | name: name.into_string().unwrap(), | |
9fa01778 XL |
277 | kind: ModuleKind::Regular, |
278 | } | |
279 | } | |
280 | }; | |
ea8adc8c | 281 | let mut serialized_bitcode = Vec::new(); |
b7449926 XL |
282 | { |
283 | let (llcx, llmod) = { | |
284 | let llvm = &module.module_llvm; | |
285 | (&llvm.llcx, llvm.llmod()) | |
286 | }; | |
287 | info!("using {:?} as a base module", module.name); | |
288 | ||
289 | // The linking steps below may produce errors and diagnostics within LLVM | |
290 | // which we'd like to handle and print, so set up our diagnostic handlers | |
291 | // (which get unregistered when they go out of scope below). | |
292 | let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx); | |
293 | ||
294 | // For all other modules we codegened we'll need to link them into our own | |
295 | // bitcode. All modules were codegened in their own LLVM context, however, | |
296 | // and we want to move everything to the same LLVM context. Currently the | |
297 | // way we know of to do that is to serialize them to a string and them parse | |
298 | // them later. Not great but hey, that's why it's "fat" LTO, right? | |
e1599b0c XL |
299 | for module in in_memory { |
300 | let buffer = ModuleBuffer::new(module.module_llvm.llmod()); | |
301 | let llmod_id = CString::new(&module.name[..]).unwrap(); | |
302 | serialized_modules.push((SerializedModule::Local(buffer), llmod_id)); | |
303 | } | |
416331ca | 304 | // Sort the modules to ensure we produce deterministic results. |
e1599b0c | 305 | serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1)); |
1a4d82fc | 306 | |
b7449926 XL |
307 | // For all serialized bitcode files we parse them and link them in as we did |
308 | // above, this is all mostly handled in C++. Like above, though, we don't | |
309 | // know much about the memory management here so we err on the side of being | |
310 | // save and persist everything with the original module. | |
311 | let mut linker = Linker::new(llmod); | |
312 | for (bc_decoded, name) in serialized_modules { | |
74b04a01 XL |
313 | let _timer = cgcx |
314 | .prof | |
315 | .generic_activity_with_arg("LLVM_fat_lto_link_module", format!("{:?}", name)); | |
b7449926 | 316 | info!("linking {:?}", name); |
74b04a01 XL |
317 | let data = bc_decoded.data(); |
318 | linker.add(&data).map_err(|()| { | |
319 | let msg = format!("failed to load bc of {:?}", name); | |
320 | write::llvm_err(&diag_handler, &msg) | |
b7449926 | 321 | })?; |
b7449926 XL |
322 | serialized_bitcode.push(bc_decoded); |
323 | } | |
324 | drop(linker); | |
a1dfa0c6 | 325 | save_temp_bitcode(&cgcx, &module, "lto.input"); |
1a4d82fc | 326 | |
f035d41b | 327 | // Internalize everything below threshold to help strip out more modules and such. |
1a4d82fc | 328 | unsafe { |
f035d41b | 329 | let ptr = symbols_below_threshold.as_ptr(); |
dfeec247 XL |
330 | llvm::LLVMRustRunRestrictionPass( |
331 | llmod, | |
332 | ptr as *const *const libc::c_char, | |
f035d41b | 333 | symbols_below_threshold.len() as libc::size_t, |
dfeec247 | 334 | ); |
a1dfa0c6 | 335 | save_temp_bitcode(&cgcx, &module, "lto.after-restriction"); |
b7449926 XL |
336 | } |
337 | ||
338 | if cgcx.no_landing_pads { | |
339 | unsafe { | |
340 | llvm::LLVMRustMarkAllFunctionsNounwind(llmod); | |
341 | } | |
a1dfa0c6 | 342 | save_temp_bitcode(&cgcx, &module, "lto.after-nounwind"); |
1a4d82fc JJ |
343 | } |
344 | } | |
345 | ||
dfeec247 | 346 | Ok(LtoModuleCodegen::Fat { module: Some(module), _serialized_bitcode: serialized_bitcode }) |
ea8adc8c XL |
347 | } |
348 | ||
b7449926 | 349 | struct Linker<'a>(&'a mut llvm::Linker<'a>); |
0531ce1d | 350 | |
b7449926 XL |
351 | impl Linker<'a> { |
352 | fn new(llmod: &'a llvm::Module) -> Self { | |
0531ce1d XL |
353 | unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } |
354 | } | |
355 | ||
356 | fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { | |
357 | unsafe { | |
dfeec247 XL |
358 | if llvm::LLVMRustLinkerAdd( |
359 | self.0, | |
360 | bytecode.as_ptr() as *const libc::c_char, | |
361 | bytecode.len(), | |
362 | ) { | |
0531ce1d XL |
363 | Ok(()) |
364 | } else { | |
365 | Err(()) | |
366 | } | |
367 | } | |
368 | } | |
369 | } | |
370 | ||
b7449926 | 371 | impl Drop for Linker<'a> { |
0531ce1d | 372 | fn drop(&mut self) { |
dfeec247 XL |
373 | unsafe { |
374 | llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _)); | |
375 | } | |
0531ce1d XL |
376 | } |
377 | } | |
378 | ||
ea8adc8c XL |
379 | /// Prepare "thin" LTO to get run on these modules. |
380 | /// | |
381 | /// The general structure of ThinLTO is quite different from the structure of | |
382 | /// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into | |
383 | /// one giant LLVM module, and then we run more optimization passes over this | |
384 | /// big module after internalizing most symbols. Thin LTO, on the other hand, | |
385 | /// avoid this large bottleneck through more targeted optimization. | |
386 | /// | |
387 | /// At a high level Thin LTO looks like: | |
388 | /// | |
389 | /// 1. Prepare a "summary" of each LLVM module in question which describes | |
390 | /// the values inside, cost of the values, etc. | |
391 | /// 2. Merge the summaries of all modules in question into one "index" | |
392 | /// 3. Perform some global analysis on this index | |
393 | /// 4. For each module, use the index and analysis calculated previously to | |
394 | /// perform local transformations on the module, for example inlining | |
395 | /// small functions from other modules. | |
396 | /// 5. Run thin-specific optimization passes over each module, and then code | |
397 | /// generate everything at the end. | |
398 | /// | |
399 | /// The summary for each module is intended to be quite cheap, and the global | |
400 | /// index is relatively quite cheap to create as well. As a result, the goal of | |
401 | /// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more | |
402 | /// situations. For example one cheap optimization is that we can parallelize | |
403 | /// all codegen modules, easily making use of all the cores on a machine. | |
404 | /// | |
405 | /// With all that in mind, the function here is designed at specifically just | |
406 | /// calculating the *index* for ThinLTO. This index will then be shared amongst | |
94b46f34 | 407 | /// all of the `LtoModuleCodegen` units returned below and destroyed once |
ea8adc8c | 408 | /// they all go out of scope. |
dfeec247 XL |
409 | fn thin_lto( |
410 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
411 | diag_handler: &Handler, | |
412 | modules: Vec<(String, ThinBuffer)>, | |
413 | serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, | |
414 | cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, | |
f035d41b | 415 | symbols_below_threshold: &[*const libc::c_char], |
dfeec247 | 416 | ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { |
e74abb32 | 417 | let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); |
ea8adc8c XL |
418 | unsafe { |
419 | info!("going for that thin, thin LTO"); | |
420 | ||
dfeec247 XL |
421 | let green_modules: FxHashMap<_, _> = |
422 | cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect(); | |
b7449926 | 423 | |
a1dfa0c6 XL |
424 | let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len(); |
425 | let mut thin_buffers = Vec::with_capacity(modules.len()); | |
426 | let mut module_names = Vec::with_capacity(full_scope_len); | |
427 | let mut thin_modules = Vec::with_capacity(full_scope_len); | |
ea8adc8c | 428 | |
0731742a XL |
429 | for (i, (name, buffer)) in modules.into_iter().enumerate() { |
430 | info!("local module: {} - {}", i, name); | |
431 | let cname = CString::new(name.clone()).unwrap(); | |
ea8adc8c | 432 | thin_modules.push(llvm::ThinLTOModule { |
0731742a | 433 | identifier: cname.as_ptr(), |
ea8adc8c XL |
434 | data: buffer.data().as_ptr(), |
435 | len: buffer.data().len(), | |
436 | }); | |
437 | thin_buffers.push(buffer); | |
0731742a | 438 | module_names.push(cname); |
b039eaaf | 439 | } |
ea8adc8c XL |
440 | |
441 | // FIXME: All upstream crates are deserialized internally in the | |
442 | // function below to extract their summary and modules. Note that | |
443 | // unlike the loop above we *must* decode and/or read something | |
444 | // here as these are all just serialized files on disk. An | |
445 | // improvement, however, to make here would be to store the | |
446 | // module summary separately from the actual module itself. Right | |
447 | // now this is store in one large bitcode file, and the entire | |
448 | // file is deflate-compressed. We could try to bypass some of the | |
449 | // decompression by storing the index uncompressed and only | |
450 | // lazily decompressing the bytecode if necessary. | |
451 | // | |
452 | // Note that truly taking advantage of this optimization will | |
453 | // likely be further down the road. We'd have to implement | |
454 | // incremental ThinLTO first where we could actually avoid | |
455 | // looking at upstream modules entirely sometimes (the contents, | |
456 | // we must always unconditionally look at the index). | |
a1dfa0c6 | 457 | let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len()); |
b7449926 | 458 | |
dfeec247 XL |
459 | let cached_modules = |
460 | cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap())); | |
b7449926 XL |
461 | |
462 | for (module, name) in serialized_modules.into_iter().chain(cached_modules) { | |
463 | info!("upstream or cached module {:?}", name); | |
ea8adc8c XL |
464 | thin_modules.push(llvm::ThinLTOModule { |
465 | identifier: name.as_ptr(), | |
466 | data: module.data().as_ptr(), | |
467 | len: module.data().len(), | |
468 | }); | |
469 | serialized.push(module); | |
470 | module_names.push(name); | |
471 | } | |
472 | ||
b7449926 XL |
473 | // Sanity check |
474 | assert_eq!(thin_modules.len(), module_names.len()); | |
475 | ||
ea8adc8c XL |
476 | // Delegate to the C++ bindings to create some data here. Once this is a |
477 | // tried-and-true interface we may wish to try to upstream some of this | |
478 | // to LLVM itself, right now we reimplement a lot of what they do | |
479 | // upstream... | |
480 | let data = llvm::LLVMRustCreateThinLTOData( | |
481 | thin_modules.as_ptr(), | |
482 | thin_modules.len() as u32, | |
f035d41b XL |
483 | symbols_below_threshold.as_ptr(), |
484 | symbols_below_threshold.len() as u32, | |
dfeec247 XL |
485 | ) |
486 | .ok_or_else(|| write::llvm_err(&diag_handler, "failed to prepare thin LTO context"))?; | |
b7449926 | 487 | |
ea8adc8c | 488 | info!("thin LTO data created"); |
ea8adc8c | 489 | |
dfeec247 XL |
490 | let (import_map_path, prev_import_map, curr_import_map) = |
491 | if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { | |
492 | let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); | |
493 | // If previous imports have been deleted, or we get an IO error | |
494 | // reading the file storing them, then we'll just use `None` as the | |
495 | // prev_import_map, which will force the code to be recompiled. | |
74b04a01 XL |
496 | let prev = if path.exists() { |
497 | ThinLTOImportMaps::load_from_file(&path).ok() | |
498 | } else { | |
499 | None | |
500 | }; | |
501 | let curr = ThinLTOImportMaps::from_thin_lto_data(data); | |
dfeec247 XL |
502 | (Some(path), prev, curr) |
503 | } else { | |
504 | // If we don't compile incrementally, we don't need to load the | |
505 | // import data from LLVM. | |
506 | assert!(green_modules.is_empty()); | |
74b04a01 | 507 | let curr = ThinLTOImportMaps::default(); |
dfeec247 XL |
508 | (None, None, curr) |
509 | }; | |
b7449926 | 510 | info!("thin LTO import map loaded"); |
b7449926 XL |
511 | |
512 | let data = ThinData(data); | |
513 | ||
ea8adc8c XL |
514 | // Throw our data in an `Arc` as we'll be sharing it across threads. We |
515 | // also put all memory referenced by the C++ data (buffers, ids, etc) | |
516 | // into the arc as well. After this we'll create a thin module | |
94b46f34 | 517 | // codegen per module in this data. |
ea8adc8c XL |
518 | let shared = Arc::new(ThinShared { |
519 | data, | |
520 | thin_buffers, | |
521 | serialized_modules: serialized, | |
522 | module_names, | |
523 | }); | |
b7449926 XL |
524 | |
525 | let mut copy_jobs = vec![]; | |
526 | let mut opt_jobs = vec![]; | |
527 | ||
528 | info!("checking which modules can be-reused and which have to be re-optimized."); | |
529 | for (module_index, module_name) in shared.module_names.iter().enumerate() { | |
530 | let module_name = module_name_to_str(module_name); | |
531 | ||
dfeec247 | 532 | // If (1.) the module hasn't changed, and (2.) none of the modules |
74b04a01 XL |
533 | // it imports from have changed, *and* (3.) the import and export |
534 | // sets themselves have not changed from the previous compile when | |
535 | // it was last ThinLTO'ed, then we can re-use the post-ThinLTO | |
536 | // version of the module. Otherwise, freshly perform LTO | |
537 | // optimization. | |
538 | // | |
539 | // (Note that globally, the export set is just the inverse of the | |
540 | // import set.) | |
541 | // | |
542 | // For further justification of why the above is necessary and sufficient, | |
543 | // see the LLVM blog post on ThinLTO: | |
544 | // | |
545 | // http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html | |
546 | // | |
547 | // which states the following: | |
548 | // | |
549 | // ```quote | |
550 | // any particular ThinLTO backend must be redone iff: | |
551 | // | |
552 | // 1. The corresponding (primary) module’s bitcode changed | |
553 | // 2. The list of imports into or exports from the module changed | |
554 | // 3. The bitcode for any module being imported from has changed | |
555 | // 4. Any global analysis result affecting either the primary module | |
556 | // or anything it imports has changed. | |
557 | // ``` | |
dfeec247 XL |
558 | // |
559 | // This strategy means we can always save the computed imports as | |
560 | // canon: when we reuse the post-ThinLTO version, condition (3.) | |
74b04a01 | 561 | // ensures that the current import set is the same as the previous |
dfeec247 XL |
562 | // one. (And of course, when we don't reuse the post-ThinLTO |
563 | // version, the current import set *is* the correct one, since we | |
564 | // are doing the ThinLTO in this current compilation cycle.) | |
565 | // | |
74b04a01 XL |
566 | // For more discussion, see rust-lang/rust#59535 (where the import |
567 | // issue was discovered) and rust-lang/rust#69798 (where the | |
568 | // analogous export issue was discovered). | |
dfeec247 XL |
569 | if let (Some(prev_import_map), true) = |
570 | (prev_import_map.as_ref(), green_modules.contains_key(module_name)) | |
571 | { | |
572 | assert!(cgcx.incr_comp_session_dir.is_some()); | |
573 | ||
74b04a01 XL |
574 | let prev_imports = prev_import_map.imports_of(module_name); |
575 | let curr_imports = curr_import_map.imports_of(module_name); | |
576 | let prev_exports = prev_import_map.exports_of(module_name); | |
577 | let curr_exports = curr_import_map.exports_of(module_name); | |
dfeec247 | 578 | let imports_all_green = curr_imports |
b7449926 XL |
579 | .iter() |
580 | .all(|imported_module| green_modules.contains_key(imported_module)); | |
74b04a01 XL |
581 | if imports_all_green |
582 | && equivalent_as_sets(prev_imports, curr_imports) | |
583 | && equivalent_as_sets(prev_exports, curr_exports) | |
584 | { | |
b7449926 XL |
585 | let work_product = green_modules[module_name].clone(); |
586 | copy_jobs.push(work_product); | |
587 | info!(" - {}: re-used", module_name); | |
dfeec247 XL |
588 | assert!(cgcx.incr_comp_session_dir.is_some()); |
589 | cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto); | |
590 | continue; | |
b7449926 XL |
591 | } |
592 | } | |
593 | ||
594 | info!(" - {}: re-compiled", module_name); | |
595 | opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { | |
ea8adc8c | 596 | shared: shared.clone(), |
b7449926 XL |
597 | idx: module_index, |
598 | })); | |
599 | } | |
600 | ||
74b04a01 | 601 | // Save the current ThinLTO import information for the next compilation |
dfeec247 XL |
602 | // session, overwriting the previous serialized imports (if any). |
603 | if let Some(path) = import_map_path { | |
604 | if let Err(err) = curr_import_map.save_to_file(&path) { | |
605 | let msg = format!("Error while writing ThinLTO import data: {}", err); | |
606 | return Err(write::llvm_err(&diag_handler, &msg)); | |
607 | } | |
608 | } | |
609 | ||
b7449926 | 610 | Ok((opt_jobs, copy_jobs)) |
b039eaaf | 611 | } |
ea8adc8c | 612 | } |
b039eaaf | 613 | |
dfeec247 XL |
614 | /// Given two slices, each with no repeat elements. returns true if and only if |
615 | /// the two slices have the same contents when considered as sets (i.e. when | |
616 | /// element order is disregarded). | |
617 | fn equivalent_as_sets(a: &[String], b: &[String]) -> bool { | |
618 | // cheap path: unequal lengths means cannot possibly be set equivalent. | |
619 | if a.len() != b.len() { | |
620 | return false; | |
621 | } | |
622 | // fast path: before building new things, check if inputs are equivalent as is. | |
623 | if a == b { | |
624 | return true; | |
625 | } | |
626 | // slow path: general set comparison. | |
627 | let a: FxHashSet<&str> = a.iter().map(|s| s.as_str()).collect(); | |
628 | let b: FxHashSet<&str> = b.iter().map(|s| s.as_str()).collect(); | |
629 | a == b | |
630 | } | |
631 | ||
632 | pub(crate) fn run_pass_manager( | |
633 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
634 | module: &ModuleCodegen<ModuleLlvm>, | |
635 | config: &ModuleConfig, | |
636 | thin: bool, | |
637 | ) { | |
74b04a01 XL |
638 | let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &module.name[..]); |
639 | ||
1a4d82fc JJ |
640 | // Now we have one massive module inside of llmod. Time to run the |
641 | // LTO-specific optimization passes that LLVM provides. | |
642 | // | |
643 | // This code is based off the code found in llvm's LTO code generator: | |
644 | // tools/lto/LTOCodeGenerator.cpp | |
645 | debug!("running the pass manager"); | |
646 | unsafe { | |
74b04a01 XL |
647 | if write::should_use_new_llvm_pass_manager(config) { |
648 | let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; | |
649 | let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); | |
650 | // See comment below for why this is necessary. | |
651 | let opt_level = if let config::OptLevel::No = opt_level { | |
652 | config::OptLevel::Less | |
653 | } else { | |
654 | opt_level | |
655 | }; | |
656 | write::optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage); | |
657 | debug!("lto done"); | |
658 | return; | |
659 | } | |
660 | ||
1a4d82fc | 661 | let pm = llvm::LLVMCreatePassManager(); |
60c5eb7d | 662 | llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); |
8faf50e0 XL |
663 | |
664 | if config.verify_llvm_ir { | |
e74abb32 | 665 | let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); |
b7449926 | 666 | llvm::LLVMRustAddPass(pm, pass.unwrap()); |
8faf50e0 | 667 | } |
1a4d82fc | 668 | |
abe05a73 XL |
669 | // When optimizing for LTO we don't actually pass in `-O0`, but we force |
670 | // it to always happen at least with `-O1`. | |
671 | // | |
672 | // With ThinLTO we mess around a lot with symbol visibility in a way | |
673 | // that will actually cause linking failures if we optimize at O0 which | |
674 | // notable is lacking in dead code elimination. To ensure we at least | |
675 | // get some optimizations and correctly link we forcibly switch to `-O1` | |
676 | // to get dead code elimination. | |
677 | // | |
678 | // Note that in general this shouldn't matter too much as you typically | |
679 | // only turn on ThinLTO when you're compiling with optimizations | |
680 | // otherwise. | |
dfeec247 XL |
681 | let opt_level = config |
682 | .opt_level | |
683 | .map(|x| to_llvm_opt_settings(x).0) | |
a1dfa0c6 | 684 | .unwrap_or(llvm::CodeGenOptLevel::None); |
abe05a73 XL |
685 | let opt_level = match opt_level { |
686 | llvm::CodeGenOptLevel::None => llvm::CodeGenOptLevel::Less, | |
687 | level => level, | |
688 | }; | |
a1dfa0c6 | 689 | with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { |
ea8adc8c | 690 | if thin { |
a1dfa0c6 | 691 | llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); |
ea8adc8c | 692 | } else { |
dfeec247 XL |
693 | llvm::LLVMPassManagerBuilderPopulateLTOPassManager( |
694 | b, pm, /* Internalize = */ False, /* RunInliner = */ True, | |
695 | ); | |
ea8adc8c | 696 | } |
c1a9b12d | 697 | }); |
1a4d82fc | 698 | |
a1dfa0c6 XL |
699 | // We always generate bitcode through ThinLTOBuffers, |
700 | // which do not support anonymous globals | |
701 | if config.bitcode_needed() { | |
e74abb32 | 702 | let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast()); |
a1dfa0c6 XL |
703 | llvm::LLVMRustAddPass(pm, pass.unwrap()); |
704 | } | |
705 | ||
8faf50e0 | 706 | if config.verify_llvm_ir { |
e74abb32 | 707 | let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); |
b7449926 | 708 | llvm::LLVMRustAddPass(pm, pass.unwrap()); |
8faf50e0 | 709 | } |
1a4d82fc | 710 | |
74b04a01 | 711 | llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()); |
1a4d82fc JJ |
712 | |
713 | llvm::LLVMDisposePassManager(pm); | |
714 | } | |
715 | debug!("lto done"); | |
716 | } | |
717 | ||
b7449926 | 718 | pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); |
ea8adc8c XL |
719 | |
720 | unsafe impl Send for ModuleBuffer {} | |
721 | unsafe impl Sync for ModuleBuffer {} | |
722 | ||
723 | impl ModuleBuffer { | |
b7449926 | 724 | pub fn new(m: &llvm::Module) -> ModuleBuffer { |
dfeec247 | 725 | ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) |
ea8adc8c | 726 | } |
a1dfa0c6 | 727 | } |
ea8adc8c | 728 | |
a1dfa0c6 XL |
729 | impl ModuleBufferMethods for ModuleBuffer { |
730 | fn data(&self) -> &[u8] { | |
ea8adc8c XL |
731 | unsafe { |
732 | let ptr = llvm::LLVMRustModuleBufferPtr(self.0); | |
733 | let len = llvm::LLVMRustModuleBufferLen(self.0); | |
734 | slice::from_raw_parts(ptr, len) | |
735 | } | |
736 | } | |
737 | } | |
738 | ||
739 | impl Drop for ModuleBuffer { | |
740 | fn drop(&mut self) { | |
dfeec247 XL |
741 | unsafe { |
742 | llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _)); | |
743 | } | |
ea8adc8c XL |
744 | } |
745 | } | |
746 | ||
a1dfa0c6 | 747 | pub struct ThinData(&'static mut llvm::ThinLTOData); |
ea8adc8c XL |
748 | |
749 | unsafe impl Send for ThinData {} | |
750 | unsafe impl Sync for ThinData {} | |
751 | ||
752 | impl Drop for ThinData { | |
753 | fn drop(&mut self) { | |
754 | unsafe { | |
b7449926 | 755 | llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _)); |
ea8adc8c XL |
756 | } |
757 | } | |
758 | } | |
759 | ||
b7449926 | 760 | pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); |
ea8adc8c XL |
761 | |
762 | unsafe impl Send for ThinBuffer {} | |
763 | unsafe impl Sync for ThinBuffer {} | |
764 | ||
765 | impl ThinBuffer { | |
b7449926 | 766 | pub fn new(m: &llvm::Module) -> ThinBuffer { |
abe05a73 XL |
767 | unsafe { |
768 | let buffer = llvm::LLVMRustThinLTOBufferCreate(m); | |
769 | ThinBuffer(buffer) | |
770 | } | |
771 | } | |
a1dfa0c6 | 772 | } |
abe05a73 | 773 | |
a1dfa0c6 XL |
774 | impl ThinBufferMethods for ThinBuffer { |
775 | fn data(&self) -> &[u8] { | |
ea8adc8c XL |
776 | unsafe { |
777 | let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; | |
778 | let len = llvm::LLVMRustThinLTOBufferLen(self.0); | |
779 | slice::from_raw_parts(ptr, len) | |
780 | } | |
781 | } | |
1a4d82fc JJ |
782 | } |
783 | ||
ea8adc8c XL |
784 | impl Drop for ThinBuffer { |
785 | fn drop(&mut self) { | |
786 | unsafe { | |
b7449926 | 787 | llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); |
ea8adc8c XL |
788 | } |
789 | } | |
1a4d82fc JJ |
790 | } |
791 | ||
a1dfa0c6 XL |
792 | pub unsafe fn optimize_thin_module( |
793 | thin_module: &mut ThinModule<LlvmCodegenBackend>, | |
794 | cgcx: &CodegenContext<LlvmCodegenBackend>, | |
a1dfa0c6 XL |
795 | ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { |
796 | let diag_handler = cgcx.create_diag_handler(); | |
dfeec247 | 797 | let tm = (cgcx.tm_factory.0)().map_err(|e| write::llvm_err(&diag_handler, &e))?; |
a1dfa0c6 XL |
798 | |
799 | // Right now the implementation we've got only works over serialized | |
800 | // modules, so we create a fresh new LLVM context and parse the module | |
801 | // into that context. One day, however, we may do this for upstream | |
802 | // crates but for locally codegened modules we may be able to reuse | |
803 | // that LLVM Context and Module. | |
804 | let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); | |
9fa01778 | 805 | let llmod_raw = parse_module( |
a1dfa0c6 | 806 | llcx, |
9fa01778 XL |
807 | &thin_module.shared.module_names[thin_module.idx], |
808 | thin_module.data(), | |
809 | &diag_handler, | |
810 | )? as *const _; | |
a1dfa0c6 | 811 | let module = ModuleCodegen { |
dfeec247 | 812 | module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, |
a1dfa0c6 XL |
813 | name: thin_module.name().to_string(), |
814 | kind: ModuleKind::Regular, | |
815 | }; | |
816 | { | |
f035d41b | 817 | let target = &*module.module_llvm.tm; |
a1dfa0c6 XL |
818 | let llmod = module.module_llvm.llmod(); |
819 | save_temp_bitcode(&cgcx, &module, "thin-lto-input"); | |
820 | ||
821 | // Before we do much else find the "main" `DICompileUnit` that we'll be | |
822 | // using below. If we find more than one though then rustc has changed | |
823 | // in a way we're not ready for, so generate an ICE by returning | |
824 | // an error. | |
825 | let mut cu1 = ptr::null_mut(); | |
826 | let mut cu2 = ptr::null_mut(); | |
827 | llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); | |
828 | if !cu2.is_null() { | |
829 | let msg = "multiple source DICompileUnits found"; | |
dfeec247 | 830 | return Err(write::llvm_err(&diag_handler, msg)); |
a1dfa0c6 | 831 | } |
ff7c6d11 | 832 | |
a1dfa0c6 XL |
833 | // Like with "fat" LTO, get some better optimizations if landing pads |
834 | // are disabled by removing all landing pads. | |
835 | if cgcx.no_landing_pads { | |
74b04a01 XL |
836 | let _timer = cgcx |
837 | .prof | |
838 | .generic_activity_with_arg("LLVM_thin_lto_remove_landing_pads", thin_module.name()); | |
a1dfa0c6 XL |
839 | llvm::LLVMRustMarkAllFunctionsNounwind(llmod); |
840 | save_temp_bitcode(&cgcx, &module, "thin-lto-after-nounwind"); | |
a1dfa0c6 | 841 | } |
ea8adc8c | 842 | |
a1dfa0c6 XL |
843 | // Up next comes the per-module local analyses that we do for Thin LTO. |
844 | // Each of these functions is basically copied from the LLVM | |
845 | // implementation and then tailored to suit this implementation. Ideally | |
846 | // each of these would be supported by upstream LLVM but that's perhaps | |
847 | // a patch for another day! | |
848 | // | |
849 | // You can find some more comments about these functions in the LLVM | |
850 | // bindings we've got (currently `PassWrapper.cpp`) | |
e74abb32 | 851 | { |
74b04a01 XL |
852 | let _timer = |
853 | cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); | |
f035d41b | 854 | if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { |
e74abb32 | 855 | let msg = "failed to prepare thin LTO module"; |
dfeec247 | 856 | return Err(write::llvm_err(&diag_handler, msg)); |
e74abb32 XL |
857 | } |
858 | save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); | |
a1dfa0c6 | 859 | } |
e74abb32 XL |
860 | |
861 | { | |
74b04a01 XL |
862 | let _timer = cgcx |
863 | .prof | |
864 | .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); | |
e74abb32 XL |
865 | if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { |
866 | let msg = "failed to prepare thin LTO module"; | |
dfeec247 | 867 | return Err(write::llvm_err(&diag_handler, msg)); |
e74abb32 XL |
868 | } |
869 | save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); | |
ea8adc8c | 870 | } |
e74abb32 XL |
871 | |
872 | { | |
74b04a01 XL |
873 | let _timer = cgcx |
874 | .prof | |
875 | .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); | |
e74abb32 XL |
876 | if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { |
877 | let msg = "failed to prepare thin LTO module"; | |
dfeec247 | 878 | return Err(write::llvm_err(&diag_handler, msg)); |
e74abb32 XL |
879 | } |
880 | save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); | |
a1dfa0c6 | 881 | } |
e74abb32 XL |
882 | |
883 | { | |
74b04a01 XL |
884 | let _timer = |
885 | cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); | |
f035d41b | 886 | if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { |
e74abb32 | 887 | let msg = "failed to prepare thin LTO module"; |
dfeec247 | 888 | return Err(write::llvm_err(&diag_handler, msg)); |
e74abb32 XL |
889 | } |
890 | save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); | |
a1dfa0c6 | 891 | } |
b7449926 | 892 | |
a1dfa0c6 XL |
893 | // Ok now this is a bit unfortunate. This is also something you won't |
894 | // find upstream in LLVM's ThinLTO passes! This is a hack for now to | |
895 | // work around bugs in LLVM. | |
896 | // | |
897 | // First discovered in #45511 it was found that as part of ThinLTO | |
898 | // importing passes LLVM will import `DICompileUnit` metadata | |
899 | // information across modules. This means that we'll be working with one | |
900 | // LLVM module that has multiple `DICompileUnit` instances in it (a | |
901 | // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of | |
902 | // bugs in LLVM's backend which generates invalid DWARF in a situation | |
903 | // like this: | |
904 | // | |
905 | // https://bugs.llvm.org/show_bug.cgi?id=35212 | |
906 | // https://bugs.llvm.org/show_bug.cgi?id=35562 | |
907 | // | |
908 | // While the first bug there is fixed the second ended up causing #46346 | |
909 | // which was basically a resurgence of #45511 after LLVM's bug 35212 was | |
910 | // fixed. | |
911 | // | |
912 | // This function below is a huge hack around this problem. The function | |
913 | // below is defined in `PassWrapper.cpp` and will basically "merge" | |
914 | // all `DICompileUnit` instances in a module. Basically it'll take all | |
915 | // the objects, rewrite all pointers of `DISubprogram` to point to the | |
916 | // first `DICompileUnit`, and then delete all the other units. | |
917 | // | |
918 | // This is probably mangling to the debug info slightly (but hopefully | |
919 | // not too much) but for now at least gets LLVM to emit valid DWARF (or | |
920 | // so it appears). Hopefully we can remove this once upstream bugs are | |
921 | // fixed in LLVM. | |
e74abb32 | 922 | { |
74b04a01 XL |
923 | let _timer = cgcx |
924 | .prof | |
925 | .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); | |
e74abb32 XL |
926 | llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); |
927 | save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); | |
928 | } | |
a1dfa0c6 XL |
929 | |
930 | // Alright now that we've done everything related to the ThinLTO | |
931 | // analysis it's time to run some optimizations! Here we use the same | |
932 | // `run_pass_manager` as the "fat" LTO above except that we tell it to | |
933 | // populate a thin-specific pass manager, which presumably LLVM treats a | |
934 | // little differently. | |
e74abb32 | 935 | { |
e74abb32 XL |
936 | info!("running thin lto passes over {}", module.name); |
937 | let config = cgcx.config(module.kind); | |
938 | run_pass_manager(cgcx, &module, config, true); | |
939 | save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); | |
940 | } | |
b7449926 | 941 | } |
a1dfa0c6 | 942 | Ok(module) |
b7449926 XL |
943 | } |
944 | ||
74b04a01 XL |
945 | /// Summarizes module import/export relationships used by LLVM's ThinLTO pass. |
946 | /// | |
947 | /// Note that we tend to have two such instances of `ThinLTOImportMaps` in use: | |
948 | /// one loaded from a file that represents the relationships used during the | |
949 | /// compilation associated with the incremetnal build artifacts we are | |
950 | /// attempting to reuse, and another constructed via `from_thin_lto_data`, which | |
951 | /// captures the relationships of ThinLTO in the current compilation. | |
0bf4aa26 | 952 | #[derive(Debug, Default)] |
74b04a01 | 953 | pub struct ThinLTOImportMaps { |
b7449926 XL |
954 | // key = llvm name of importing module, value = list of modules it imports from |
955 | imports: FxHashMap<String, Vec<String>>, | |
74b04a01 XL |
956 | // key = llvm name of exporting module, value = list of modules it exports to |
957 | exports: FxHashMap<String, Vec<String>>, | |
b7449926 XL |
958 | } |
959 | ||
74b04a01 XL |
960 | impl ThinLTOImportMaps { |
961 | /// Returns modules imported by `llvm_module_name` during some ThinLTO pass. | |
962 | fn imports_of(&self, llvm_module_name: &str) -> &[String] { | |
b7449926 XL |
963 | self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) |
964 | } | |
ff7c6d11 | 965 | |
74b04a01 XL |
966 | /// Returns modules exported by `llvm_module_name` during some ThinLTO pass. |
967 | fn exports_of(&self, llvm_module_name: &str) -> &[String] { | |
968 | self.exports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) | |
969 | } | |
970 | ||
dfeec247 XL |
971 | fn save_to_file(&self, path: &Path) -> io::Result<()> { |
972 | use std::io::Write; | |
973 | let file = File::create(path)?; | |
974 | let mut writer = io::BufWriter::new(file); | |
975 | for (importing_module_name, imported_modules) in &self.imports { | |
976 | writeln!(writer, "{}", importing_module_name)?; | |
977 | for imported_module in imported_modules { | |
978 | writeln!(writer, " {}", imported_module)?; | |
979 | } | |
980 | writeln!(writer)?; | |
981 | } | |
982 | Ok(()) | |
983 | } | |
984 | ||
74b04a01 | 985 | fn load_from_file(path: &Path) -> io::Result<ThinLTOImportMaps> { |
dfeec247 XL |
986 | use std::io::BufRead; |
987 | let mut imports = FxHashMap::default(); | |
74b04a01 XL |
988 | let mut exports: FxHashMap<_, Vec<_>> = FxHashMap::default(); |
989 | let mut current_module: Option<String> = None; | |
990 | let mut current_imports: Vec<String> = vec![]; | |
dfeec247 XL |
991 | let file = File::open(path)?; |
992 | for line in io::BufReader::new(file).lines() { | |
993 | let line = line?; | |
994 | if line.is_empty() { | |
995 | let importing_module = current_module.take().expect("Importing module not set"); | |
74b04a01 XL |
996 | for imported in ¤t_imports { |
997 | exports.entry(imported.clone()).or_default().push(importing_module.clone()); | |
998 | } | |
dfeec247 | 999 | imports.insert(importing_module, mem::replace(&mut current_imports, vec![])); |
74b04a01 | 1000 | } else if line.starts_with(' ') { |
dfeec247 XL |
1001 | // Space marks an imported module |
1002 | assert_ne!(current_module, None); | |
1003 | current_imports.push(line.trim().to_string()); | |
1004 | } else { | |
1005 | // Otherwise, beginning of a new module (must be start or follow empty line) | |
1006 | assert_eq!(current_module, None); | |
1007 | current_module = Some(line.trim().to_string()); | |
1008 | } | |
1009 | } | |
74b04a01 | 1010 | Ok(ThinLTOImportMaps { imports, exports }) |
dfeec247 XL |
1011 | } |
1012 | ||
9fa01778 | 1013 | /// Loads the ThinLTO import map from ThinLTOData. |
74b04a01 | 1014 | unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImportMaps { |
dfeec247 XL |
1015 | unsafe extern "C" fn imported_module_callback( |
1016 | payload: *mut libc::c_void, | |
1017 | importing_module_name: *const libc::c_char, | |
1018 | imported_module_name: *const libc::c_char, | |
1019 | ) { | |
74b04a01 | 1020 | let map = &mut *(payload as *mut ThinLTOImportMaps); |
b7449926 XL |
1021 | let importing_module_name = CStr::from_ptr(importing_module_name); |
1022 | let importing_module_name = module_name_to_str(&importing_module_name); | |
1023 | let imported_module_name = CStr::from_ptr(imported_module_name); | |
1024 | let imported_module_name = module_name_to_str(&imported_module_name); | |
1025 | ||
1026 | if !map.imports.contains_key(importing_module_name) { | |
1027 | map.imports.insert(importing_module_name.to_owned(), vec![]); | |
1028 | } | |
1029 | ||
1030 | map.imports | |
dfeec247 XL |
1031 | .get_mut(importing_module_name) |
1032 | .unwrap() | |
1033 | .push(imported_module_name.to_owned()); | |
74b04a01 XL |
1034 | |
1035 | if !map.exports.contains_key(imported_module_name) { | |
1036 | map.exports.insert(imported_module_name.to_owned(), vec![]); | |
1037 | } | |
1038 | ||
1039 | map.exports | |
1040 | .get_mut(imported_module_name) | |
1041 | .unwrap() | |
1042 | .push(importing_module_name.to_owned()); | |
b7449926 | 1043 | } |
74b04a01 XL |
1044 | |
1045 | let mut map = ThinLTOImportMaps::default(); | |
dfeec247 XL |
1046 | llvm::LLVMRustGetThinLTOModuleImports( |
1047 | data, | |
1048 | imported_module_callback, | |
1049 | &mut map as *mut _ as *mut libc::c_void, | |
1050 | ); | |
b7449926 XL |
1051 | map |
1052 | } | |
1053 | } | |
1054 | ||
1055 | fn module_name_to_str(c_str: &CStr) -> &str { | |
dfeec247 XL |
1056 | c_str.to_str().unwrap_or_else(|e| { |
1057 | bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e) | |
1058 | }) | |
1a4d82fc | 1059 | } |
9fa01778 | 1060 | |
e1599b0c | 1061 | pub fn parse_module<'a>( |
9fa01778 XL |
1062 | cx: &'a llvm::Context, |
1063 | name: &CStr, | |
1064 | data: &[u8], | |
1065 | diag_handler: &Handler, | |
1066 | ) -> Result<&'a llvm::Module, FatalError> { | |
1067 | unsafe { | |
dfeec247 XL |
1068 | llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( |
1069 | || { | |
1070 | let msg = "failed to parse bitcode for LTO module"; | |
1071 | write::llvm_err(&diag_handler, msg) | |
1072 | }, | |
1073 | ) | |
9fa01778 XL |
1074 | } |
1075 | } |