]>
Commit | Line | Data |
---|---|---|
3dfed10e XL |
1 | use crate::llvm; |
2 | ||
c295e0f8 | 3 | use crate::abi::Abi; |
f035d41b XL |
4 | use crate::builder::Builder; |
5 | use crate::common::CodegenCx; | |
add651ee FG |
6 | use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; |
7 | use crate::coverageinfo::map_data::FunctionCoverage; | |
3dfed10e XL |
8 | |
9 | use libc::c_uint; | |
3dfed10e | 10 | use rustc_codegen_ssa::traits::{ |
fe692bf9 FG |
11 | BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, |
12 | StaticMethods, | |
3dfed10e | 13 | }; |
f035d41b | 14 | use rustc_data_structures::fx::FxHashMap; |
cdc7bbd5 XL |
15 | use rustc_hir as hir; |
16 | use rustc_hir::def_id::DefId; | |
3dfed10e | 17 | use rustc_llvm::RustString; |
cdc7bbd5 | 18 | use rustc_middle::bug; |
781aab86 | 19 | use rustc_middle::mir::coverage::{CounterId, CoverageKind}; |
fe692bf9 | 20 | use rustc_middle::mir::Coverage; |
cdc7bbd5 | 21 | use rustc_middle::ty; |
fe692bf9 | 22 | use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; |
add651ee | 23 | use rustc_middle::ty::GenericArgs; |
f035d41b | 24 | use rustc_middle::ty::Instance; |
fe692bf9 | 25 | use rustc_middle::ty::Ty; |
f035d41b XL |
26 | |
27 | use std::cell::RefCell; | |
3dfed10e | 28 | |
add651ee | 29 | pub(crate) mod ffi; |
fe692bf9 | 30 | pub(crate) mod map_data; |
3dfed10e XL |
31 | pub mod mapgen; |
32 | ||
add651ee | 33 | const UNUSED_FUNCTION_COUNTER_ID: CounterId = CounterId::START; |
cdc7bbd5 | 34 | |
fc512014 | 35 | const VAR_ALIGN_BYTES: usize = 8; |
f035d41b XL |
36 | |
37 | /// A context object for maintaining all state needed by the coverageinfo module. | |
cdc7bbd5 | 38 | pub struct CrateCoverageContext<'ll, 'tcx> { |
487cf647 | 39 | /// Coverage data for each instrumented function identified by DefId. |
29967ef6 | 40 | pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, |
cdc7bbd5 | 41 | pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, |
f035d41b XL |
42 | } |
43 | ||
cdc7bbd5 | 44 | impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { |
f035d41b | 45 | pub fn new() -> Self { |
cdc7bbd5 XL |
46 | Self { |
47 | function_coverage_map: Default::default(), | |
48 | pgo_func_name_var_map: Default::default(), | |
49 | } | |
f035d41b | 50 | } |
f035d41b | 51 | |
29967ef6 | 52 | pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { |
3dfed10e | 53 | self.function_coverage_map.replace(FxHashMap::default()) |
f035d41b XL |
54 | } |
55 | } | |
56 | ||
fe692bf9 FG |
57 | // These methods used to be part of trait `CoverageInfoMethods`, which no longer |
58 | // exists after most coverage code was moved out of SSA. | |
59 | impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { | |
60 | pub(crate) fn coverageinfo_finalize(&self) { | |
3dfed10e | 61 | mapgen::finalize(self) |
f035d41b | 62 | } |
f035d41b | 63 | |
fe692bf9 FG |
64 | /// For LLVM codegen, returns a function-specific `Value` for a global |
65 | /// string, to hold the function name passed to LLVM intrinsic | |
66 | /// `instrprof.increment()`. The `Value` is only created once per instance. | |
67 | /// Multiple invocations with the same instance return the same `Value`. | |
cdc7bbd5 XL |
68 | fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { |
69 | if let Some(coverage_context) = self.coverage_context() { | |
70 | debug!("getting pgo_func_name_var for instance={:?}", instance); | |
71 | let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); | |
72 | pgo_func_name_var_map | |
73 | .entry(instance) | |
74 | .or_insert_with(|| create_pgo_func_name_var(self, instance)) | |
75 | } else { | |
76 | bug!("Could not get the `coverage_context`"); | |
77 | } | |
3dfed10e XL |
78 | } |
79 | ||
cdc7bbd5 XL |
80 | /// Functions with MIR-based coverage are normally codegenned _only_ if |
81 | /// called. LLVM coverage tools typically expect every function to be | |
82 | /// defined (even if unused), with at least one call to LLVM intrinsic | |
83 | /// `instrprof.increment`. | |
84 | /// | |
85 | /// Codegen a small function that will never be called, with one counter | |
86 | /// that will never be incremented. | |
87 | /// | |
88 | /// For used/called functions, the coverageinfo was already added to the | |
89 | /// `function_coverage_map` (keyed by function `Instance`) during codegen. | |
90 | /// But in this case, since the unused function was _not_ previously | |
91 | /// codegenned, collect the coverage `CodeRegion`s from the MIR and add | |
92 | /// them. The first `CodeRegion` is used to add a single counter, with the | |
93 | /// same counter ID used in the injected `instrprof.increment` intrinsic | |
94 | /// call. Since the function is never called, all other `CodeRegion`s can be | |
95 | /// added as `unreachable_region`s. | |
96 | fn define_unused_fn(&self, def_id: DefId) { | |
5e7ed085 | 97 | let instance = declare_unused_fn(self, def_id); |
cdc7bbd5 XL |
98 | codegen_unused_fn_and_counter(self, instance); |
99 | add_unused_function_coverage(self, instance, def_id); | |
100 | } | |
101 | } | |
102 | ||
a2a8927a | 103 | impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { |
fe692bf9 FG |
104 | fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { |
105 | let bx = self; | |
106 | ||
781aab86 FG |
107 | let Some(coverage_context) = bx.coverage_context() else { return }; |
108 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
109 | let func_coverage = coverage_map | |
110 | .entry(instance) | |
111 | .or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance)); | |
112 | ||
fe692bf9 FG |
113 | let Coverage { kind, code_region } = coverage.clone(); |
114 | match kind { | |
115 | CoverageKind::Counter { function_source_hash, id } => { | |
781aab86 FG |
116 | debug!( |
117 | "ensuring function source hash is set for instance={:?}; function_source_hash={}", | |
118 | instance, function_source_hash, | |
119 | ); | |
120 | func_coverage.set_function_source_hash(function_source_hash); | |
121 | ||
122 | if let Some(code_region) = code_region { | |
123 | // Note: Some counters do not have code regions, but may still be referenced | |
124 | // from expressions. In that case, don't add the counter to the coverage map, | |
125 | // but do inject the counter intrinsic. | |
fe692bf9 | 126 | debug!( |
781aab86 FG |
127 | "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", |
128 | instance, id, code_region, | |
fe692bf9 | 129 | ); |
781aab86 | 130 | func_coverage.add_counter(id, code_region); |
fe692bf9 | 131 | } |
781aab86 FG |
132 | // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, |
133 | // as that needs an exclusive borrow. | |
134 | drop(coverage_map); | |
135 | ||
136 | let coverageinfo = bx.tcx().coverageinfo(instance.def); | |
137 | ||
138 | let fn_name = bx.get_pgo_func_name_var(instance); | |
139 | let hash = bx.const_u64(function_source_hash); | |
140 | let num_counters = bx.const_u32(coverageinfo.num_counters); | |
141 | let index = bx.const_u32(id.as_u32()); | |
142 | debug!( | |
143 | "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", | |
144 | fn_name, hash, num_counters, index, | |
145 | ); | |
146 | bx.instrprof_increment(fn_name, hash, num_counters, index); | |
fe692bf9 FG |
147 | } |
148 | CoverageKind::Expression { id, lhs, op, rhs } => { | |
781aab86 FG |
149 | debug!( |
150 | "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; region: {:?}", | |
151 | instance, id, lhs, op, rhs, code_region, | |
152 | ); | |
153 | func_coverage.add_counter_expression(id, lhs, op, rhs, code_region); | |
fe692bf9 FG |
154 | } |
155 | CoverageKind::Unreachable => { | |
781aab86 FG |
156 | let code_region = |
157 | code_region.expect("unreachable regions always have code regions"); | |
158 | debug!( | |
159 | "adding unreachable code to coverage_map: instance={:?}, at {:?}", | |
160 | instance, code_region, | |
fe692bf9 | 161 | ); |
781aab86 | 162 | func_coverage.add_unreachable_region(code_region); |
fe692bf9 FG |
163 | } |
164 | } | |
165 | } | |
166 | } | |
167 | ||
5e7ed085 | 168 | fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { |
cdc7bbd5 XL |
169 | let tcx = cx.tcx; |
170 | ||
171 | let instance = Instance::new( | |
5e7ed085 | 172 | def_id, |
add651ee | 173 | GenericArgs::for_item(tcx, def_id, |param, _| { |
cdc7bbd5 XL |
174 | if let ty::GenericParamDefKind::Lifetime = param.kind { |
175 | tcx.lifetimes.re_erased.into() | |
176 | } else { | |
177 | tcx.mk_param_from_def(param) | |
178 | } | |
179 | }), | |
180 | ); | |
181 | ||
182 | let llfn = cx.declare_fn( | |
c295e0f8 XL |
183 | tcx.symbol_name(instance).name, |
184 | cx.fn_abi_of_fn_ptr( | |
cdc7bbd5 | 185 | ty::Binder::dummy(tcx.mk_fn_sig( |
fe692bf9 FG |
186 | [Ty::new_unit(tcx)], |
187 | Ty::new_unit(tcx), | |
cdc7bbd5 XL |
188 | false, |
189 | hir::Unsafety::Unsafe, | |
190 | Abi::Rust, | |
191 | )), | |
c295e0f8 | 192 | ty::List::empty(), |
cdc7bbd5 | 193 | ), |
49aad941 | 194 | None, |
cdc7bbd5 XL |
195 | ); |
196 | ||
a2a8927a XL |
197 | llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); |
198 | llvm::set_visibility(llfn, llvm::Visibility::Default); | |
cdc7bbd5 XL |
199 | |
200 | assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); | |
201 | ||
202 | instance | |
203 | } | |
204 | ||
a2a8927a | 205 | fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { |
cdc7bbd5 | 206 | let llfn = cx.get_fn(instance); |
17df50a5 XL |
207 | let llbb = Builder::append_block(cx, llfn, "unused_function"); |
208 | let mut bx = Builder::build(cx, llbb); | |
cdc7bbd5 XL |
209 | let fn_name = bx.get_pgo_func_name_var(instance); |
210 | let hash = bx.const_u64(0); | |
211 | let num_counters = bx.const_u32(1); | |
212 | let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); | |
213 | debug!( | |
214 | "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, | |
215 | index={:?}) for unused function: {:?}", | |
216 | fn_name, hash, num_counters, index, instance | |
217 | ); | |
218 | bx.instrprof_increment(fn_name, hash, num_counters, index); | |
219 | bx.ret_void(); | |
220 | } | |
221 | ||
a2a8927a XL |
222 | fn add_unused_function_coverage<'tcx>( |
223 | cx: &CodegenCx<'_, 'tcx>, | |
cdc7bbd5 XL |
224 | instance: Instance<'tcx>, |
225 | def_id: DefId, | |
226 | ) { | |
227 | let tcx = cx.tcx; | |
228 | ||
229 | let mut function_coverage = FunctionCoverage::unused(tcx, instance); | |
230 | for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { | |
231 | if index == 0 { | |
232 | // Insert at least one real counter so the LLVM CoverageMappingReader will find expected | |
233 | // definitions. | |
234 | function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); | |
235 | } else { | |
236 | function_coverage.add_unreachable_region(code_region.clone()); | |
237 | } | |
238 | } | |
239 | ||
240 | if let Some(coverage_context) = cx.coverage_context() { | |
241 | coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); | |
242 | } else { | |
243 | bug!("Could not get the `coverage_context`"); | |
244 | } | |
245 | } | |
246 | ||
247 | /// Calls llvm::createPGOFuncNameVar() with the given function instance's | |
248 | /// mangled function name. The LLVM API returns an llvm::GlobalVariable | |
249 | /// containing the function name, with the specific variable name and linkage | |
250 | /// required by LLVM InstrProf source-based coverage instrumentation. Use | |
251 | /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per | |
252 | /// `Instance`. | |
a2a8927a | 253 | fn create_pgo_func_name_var<'ll, 'tcx>( |
cdc7bbd5 XL |
254 | cx: &CodegenCx<'ll, 'tcx>, |
255 | instance: Instance<'tcx>, | |
256 | ) -> &'ll llvm::Value { | |
add651ee | 257 | let mangled_fn_name: &str = cx.tcx.symbol_name(instance).name; |
cdc7bbd5 | 258 | let llfn = cx.get_fn(instance); |
add651ee FG |
259 | unsafe { |
260 | llvm::LLVMRustCoverageCreatePGOFuncNameVar( | |
261 | llfn, | |
262 | mangled_fn_name.as_ptr().cast(), | |
263 | mangled_fn_name.len(), | |
264 | ) | |
265 | } | |
cdc7bbd5 XL |
266 | } |
267 | ||
3dfed10e | 268 | pub(crate) fn write_filenames_section_to_buffer<'a>( |
add651ee | 269 | filenames: impl IntoIterator<Item = &'a str>, |
3dfed10e XL |
270 | buffer: &RustString, |
271 | ) { | |
add651ee FG |
272 | let (pointers, lengths) = filenames |
273 | .into_iter() | |
274 | .map(|s: &str| (s.as_ptr().cast(), s.len())) | |
275 | .unzip::<_, _, Vec<_>, Vec<_>>(); | |
276 | ||
3dfed10e XL |
277 | unsafe { |
278 | llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( | |
add651ee FG |
279 | pointers.as_ptr(), |
280 | pointers.len(), | |
281 | lengths.as_ptr(), | |
282 | lengths.len(), | |
3dfed10e XL |
283 | buffer, |
284 | ); | |
285 | } | |
286 | } | |
287 | ||
288 | pub(crate) fn write_mapping_to_buffer( | |
289 | virtual_file_mapping: Vec<u32>, | |
290 | expressions: Vec<CounterExpression>, | |
6a06907d | 291 | mapping_regions: Vec<CounterMappingRegion>, |
3dfed10e XL |
292 | buffer: &RustString, |
293 | ) { | |
294 | unsafe { | |
295 | llvm::LLVMRustCoverageWriteMappingToBuffer( | |
296 | virtual_file_mapping.as_ptr(), | |
297 | virtual_file_mapping.len() as c_uint, | |
298 | expressions.as_ptr(), | |
299 | expressions.len() as c_uint, | |
6a06907d | 300 | mapping_regions.as_ptr(), |
3dfed10e XL |
301 | mapping_regions.len() as c_uint, |
302 | buffer, | |
303 | ); | |
304 | } | |
305 | } | |
cdc7bbd5 | 306 | |
add651ee | 307 | pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { |
fc512014 | 308 | unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } |
3dfed10e XL |
309 | } |
310 | ||
311 | pub(crate) fn mapping_version() -> u32 { | |
312 | unsafe { llvm::LLVMRustCoverageMappingVersion() } | |
313 | } | |
314 | ||
fc512014 | 315 | pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( |
3dfed10e XL |
316 | cx: &CodegenCx<'ll, 'tcx>, |
317 | cov_data_val: &'ll llvm::Value, | |
318 | ) { | |
319 | let covmap_var_name = llvm::build_string(|s| unsafe { | |
320 | llvm::LLVMRustCoverageWriteMappingVarNameToString(s); | |
321 | }) | |
322 | .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); | |
323 | debug!("covmap var name: {:?}", covmap_var_name); | |
324 | ||
325 | let covmap_section_name = llvm::build_string(|s| unsafe { | |
fc512014 | 326 | llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); |
3dfed10e XL |
327 | }) |
328 | .expect("Rust Coverage section name failed UTF-8 conversion"); | |
329 | debug!("covmap section name: {:?}", covmap_section_name); | |
330 | ||
331 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); | |
332 | llvm::set_initializer(llglobal, cov_data_val); | |
333 | llvm::set_global_constant(llglobal, true); | |
fc512014 | 334 | llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); |
3dfed10e | 335 | llvm::set_section(llglobal, &covmap_section_name); |
fc512014 XL |
336 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); |
337 | cx.add_used_global(llglobal); | |
338 | } | |
339 | ||
340 | pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( | |
341 | cx: &CodegenCx<'ll, 'tcx>, | |
add651ee | 342 | covfun_section_name: &str, |
fc512014 XL |
343 | func_name_hash: u64, |
344 | func_record_val: &'ll llvm::Value, | |
345 | is_used: bool, | |
346 | ) { | |
347 | // Assign a name to the function record. This is used to merge duplicates. | |
348 | // | |
349 | // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that | |
350 | // are included-but-not-used. If (or when) Rust generates functions that are | |
351 | // included-but-not-used, note that a dummy description for a function included-but-not-used | |
352 | // in a Crate can be replaced by full description provided by a different Crate. The two kinds | |
353 | // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by | |
354 | // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. | |
355 | let func_record_var_name = | |
356 | format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); | |
357 | debug!("function record var name: {:?}", func_record_var_name); | |
add651ee | 358 | debug!("function record section name: {:?}", covfun_section_name); |
fc512014 XL |
359 | |
360 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); | |
361 | llvm::set_initializer(llglobal, func_record_val); | |
362 | llvm::set_global_constant(llglobal, true); | |
363 | llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); | |
364 | llvm::set_visibility(llglobal, llvm::Visibility::Hidden); | |
add651ee | 365 | llvm::set_section(llglobal, covfun_section_name); |
fc512014 XL |
366 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); |
367 | llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); | |
3dfed10e XL |
368 | cx.add_used_global(llglobal); |
369 | } | |
add651ee FG |
370 | |
371 | /// Returns the section name string to pass through to the linker when embedding | |
372 | /// per-function coverage information in the object file, according to the target | |
373 | /// platform's object file format. | |
374 | /// | |
375 | /// LLVM's coverage tools read coverage mapping details from this section when | |
376 | /// producing coverage reports. | |
377 | /// | |
378 | /// Typical values are: | |
379 | /// - `__llvm_covfun` on Linux | |
380 | /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) | |
381 | /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) | |
382 | pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String { | |
383 | llvm::build_string(|s| unsafe { | |
384 | llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); | |
385 | }) | |
386 | .expect("Rust Coverage function record section name failed UTF-8 conversion") | |
387 | } |