]>
Commit | Line | Data |
---|---|---|
3dfed10e XL |
1 | use crate::llvm; |
2 | ||
c295e0f8 | 3 | use crate::abi::Abi; |
f035d41b XL |
4 | use crate::builder::Builder; |
5 | use crate::common::CodegenCx; | |
3dfed10e XL |
6 | |
7 | use libc::c_uint; | |
8 | use llvm::coverageinfo::CounterMappingRegion; | |
9 | use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; | |
10 | use rustc_codegen_ssa::traits::{ | |
cdc7bbd5 XL |
11 | BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, |
12 | MiscMethods, StaticMethods, | |
3dfed10e | 13 | }; |
f035d41b | 14 | use rustc_data_structures::fx::FxHashMap; |
cdc7bbd5 XL |
15 | use rustc_hir as hir; |
16 | use rustc_hir::def_id::DefId; | |
3dfed10e | 17 | use rustc_llvm::RustString; |
cdc7bbd5 | 18 | use rustc_middle::bug; |
3dfed10e | 19 | use rustc_middle::mir::coverage::{ |
29967ef6 | 20 | CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, |
3dfed10e | 21 | }; |
cdc7bbd5 | 22 | use rustc_middle::ty; |
c295e0f8 | 23 | use rustc_middle::ty::layout::FnAbiOf; |
cdc7bbd5 | 24 | use rustc_middle::ty::subst::InternalSubsts; |
f035d41b XL |
25 | use rustc_middle::ty::Instance; |
26 | ||
27 | use std::cell::RefCell; | |
3dfed10e XL |
28 | use std::ffi::CString; |
29 | ||
cdc7bbd5 | 30 | use std::iter; |
3dfed10e XL |
31 | |
32 | pub mod mapgen; | |
33 | ||
cdc7bbd5 XL |
34 | const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; |
35 | ||
fc512014 | 36 | const VAR_ALIGN_BYTES: usize = 8; |
f035d41b XL |
37 | |
38 | /// A context object for maintaining all state needed by the coverageinfo module. | |
cdc7bbd5 | 39 | pub struct CrateCoverageContext<'ll, 'tcx> { |
29967ef6 XL |
40 | // Coverage data for each instrumented function identified by DefId. |
41 | pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, | |
cdc7bbd5 | 42 | pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, |
f035d41b XL |
43 | } |
44 | ||
cdc7bbd5 | 45 | impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { |
f035d41b | 46 | pub fn new() -> Self { |
cdc7bbd5 XL |
47 | Self { |
48 | function_coverage_map: Default::default(), | |
49 | pgo_func_name_var_map: Default::default(), | |
50 | } | |
f035d41b | 51 | } |
f035d41b | 52 | |
29967ef6 | 53 | pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { |
3dfed10e | 54 | self.function_coverage_map.replace(FxHashMap::default()) |
f035d41b XL |
55 | } |
56 | } | |
57 | ||
a2a8927a | 58 | impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { |
f035d41b | 59 | fn coverageinfo_finalize(&self) { |
3dfed10e | 60 | mapgen::finalize(self) |
f035d41b | 61 | } |
f035d41b | 62 | |
cdc7bbd5 XL |
63 | fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { |
64 | if let Some(coverage_context) = self.coverage_context() { | |
65 | debug!("getting pgo_func_name_var for instance={:?}", instance); | |
66 | let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); | |
67 | pgo_func_name_var_map | |
68 | .entry(instance) | |
69 | .or_insert_with(|| create_pgo_func_name_var(self, instance)) | |
70 | } else { | |
71 | bug!("Could not get the `coverage_context`"); | |
72 | } | |
3dfed10e XL |
73 | } |
74 | ||
cdc7bbd5 XL |
75 | /// Functions with MIR-based coverage are normally codegenned _only_ if |
76 | /// called. LLVM coverage tools typically expect every function to be | |
77 | /// defined (even if unused), with at least one call to LLVM intrinsic | |
78 | /// `instrprof.increment`. | |
79 | /// | |
80 | /// Codegen a small function that will never be called, with one counter | |
81 | /// that will never be incremented. | |
82 | /// | |
83 | /// For used/called functions, the coverageinfo was already added to the | |
84 | /// `function_coverage_map` (keyed by function `Instance`) during codegen. | |
85 | /// But in this case, since the unused function was _not_ previously | |
86 | /// codegenned, collect the coverage `CodeRegion`s from the MIR and add | |
87 | /// them. The first `CodeRegion` is used to add a single counter, with the | |
88 | /// same counter ID used in the injected `instrprof.increment` intrinsic | |
89 | /// call. Since the function is never called, all other `CodeRegion`s can be | |
90 | /// added as `unreachable_region`s. | |
91 | fn define_unused_fn(&self, def_id: DefId) { | |
5e7ed085 | 92 | let instance = declare_unused_fn(self, def_id); |
cdc7bbd5 XL |
93 | codegen_unused_fn_and_counter(self, instance); |
94 | add_unused_function_coverage(self, instance, def_id); | |
95 | } | |
96 | } | |
97 | ||
a2a8927a | 98 | impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { |
29967ef6 | 99 | fn set_function_source_hash( |
f035d41b XL |
100 | &mut self, |
101 | instance: Instance<'tcx>, | |
3dfed10e | 102 | function_source_hash: u64, |
29967ef6 XL |
103 | ) -> bool { |
104 | if let Some(coverage_context) = self.coverage_context() { | |
105 | debug!( | |
106 | "ensuring function source hash is set for instance={:?}; function_source_hash={}", | |
107 | instance, function_source_hash, | |
108 | ); | |
109 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
110 | coverage_map | |
111 | .entry(instance) | |
112 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
113 | .set_function_source_hash(function_source_hash); | |
114 | true | |
115 | } else { | |
116 | false | |
117 | } | |
118 | } | |
119 | ||
120 | fn add_coverage_counter( | |
121 | &mut self, | |
122 | instance: Instance<'tcx>, | |
3dfed10e XL |
123 | id: CounterValueReference, |
124 | region: CodeRegion, | |
29967ef6 XL |
125 | ) -> bool { |
126 | if let Some(coverage_context) = self.coverage_context() { | |
127 | debug!( | |
128 | "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", | |
129 | instance, id, region, | |
130 | ); | |
131 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
132 | coverage_map | |
133 | .entry(instance) | |
134 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
135 | .add_counter(id, region); | |
136 | true | |
137 | } else { | |
138 | false | |
139 | } | |
f035d41b XL |
140 | } |
141 | ||
29967ef6 | 142 | fn add_coverage_counter_expression( |
f035d41b XL |
143 | &mut self, |
144 | instance: Instance<'tcx>, | |
29967ef6 | 145 | id: InjectedExpressionId, |
3dfed10e XL |
146 | lhs: ExpressionOperandId, |
147 | op: Op, | |
148 | rhs: ExpressionOperandId, | |
29967ef6 XL |
149 | region: Option<CodeRegion>, |
150 | ) -> bool { | |
151 | if let Some(coverage_context) = self.coverage_context() { | |
152 | debug!( | |
153 | "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ | |
154 | region: {:?}", | |
155 | instance, id, lhs, op, rhs, region, | |
156 | ); | |
157 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
158 | coverage_map | |
159 | .entry(instance) | |
160 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
161 | .add_counter_expression(id, lhs, op, rhs, region); | |
162 | true | |
163 | } else { | |
164 | false | |
165 | } | |
f035d41b XL |
166 | } |
167 | ||
29967ef6 XL |
168 | fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { |
169 | if let Some(coverage_context) = self.coverage_context() { | |
170 | debug!( | |
171 | "adding unreachable code to coverage_map: instance={:?}, at {:?}", | |
172 | instance, region, | |
173 | ); | |
174 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
175 | coverage_map | |
176 | .entry(instance) | |
177 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
178 | .add_unreachable_region(region); | |
179 | true | |
180 | } else { | |
181 | false | |
182 | } | |
f035d41b XL |
183 | } |
184 | } | |
3dfed10e | 185 | |
5e7ed085 | 186 | fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { |
cdc7bbd5 XL |
187 | let tcx = cx.tcx; |
188 | ||
189 | let instance = Instance::new( | |
5e7ed085 FG |
190 | def_id, |
191 | InternalSubsts::for_item(tcx, def_id, |param, _| { | |
cdc7bbd5 XL |
192 | if let ty::GenericParamDefKind::Lifetime = param.kind { |
193 | tcx.lifetimes.re_erased.into() | |
194 | } else { | |
195 | tcx.mk_param_from_def(param) | |
196 | } | |
197 | }), | |
198 | ); | |
199 | ||
200 | let llfn = cx.declare_fn( | |
c295e0f8 XL |
201 | tcx.symbol_name(instance).name, |
202 | cx.fn_abi_of_fn_ptr( | |
cdc7bbd5 XL |
203 | ty::Binder::dummy(tcx.mk_fn_sig( |
204 | iter::once(tcx.mk_unit()), | |
205 | tcx.mk_unit(), | |
206 | false, | |
207 | hir::Unsafety::Unsafe, | |
208 | Abi::Rust, | |
209 | )), | |
c295e0f8 | 210 | ty::List::empty(), |
cdc7bbd5 XL |
211 | ), |
212 | ); | |
213 | ||
a2a8927a XL |
214 | llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); |
215 | llvm::set_visibility(llfn, llvm::Visibility::Default); | |
cdc7bbd5 XL |
216 | |
217 | assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); | |
218 | ||
219 | instance | |
220 | } | |
221 | ||
a2a8927a | 222 | fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { |
cdc7bbd5 | 223 | let llfn = cx.get_fn(instance); |
17df50a5 XL |
224 | let llbb = Builder::append_block(cx, llfn, "unused_function"); |
225 | let mut bx = Builder::build(cx, llbb); | |
cdc7bbd5 XL |
226 | let fn_name = bx.get_pgo_func_name_var(instance); |
227 | let hash = bx.const_u64(0); | |
228 | let num_counters = bx.const_u32(1); | |
229 | let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); | |
230 | debug!( | |
231 | "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, | |
232 | index={:?}) for unused function: {:?}", | |
233 | fn_name, hash, num_counters, index, instance | |
234 | ); | |
235 | bx.instrprof_increment(fn_name, hash, num_counters, index); | |
236 | bx.ret_void(); | |
237 | } | |
238 | ||
a2a8927a XL |
239 | fn add_unused_function_coverage<'tcx>( |
240 | cx: &CodegenCx<'_, 'tcx>, | |
cdc7bbd5 XL |
241 | instance: Instance<'tcx>, |
242 | def_id: DefId, | |
243 | ) { | |
244 | let tcx = cx.tcx; | |
245 | ||
246 | let mut function_coverage = FunctionCoverage::unused(tcx, instance); | |
247 | for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { | |
248 | if index == 0 { | |
249 | // Insert at least one real counter so the LLVM CoverageMappingReader will find expected | |
250 | // definitions. | |
251 | function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); | |
252 | } else { | |
253 | function_coverage.add_unreachable_region(code_region.clone()); | |
254 | } | |
255 | } | |
256 | ||
257 | if let Some(coverage_context) = cx.coverage_context() { | |
258 | coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); | |
259 | } else { | |
260 | bug!("Could not get the `coverage_context`"); | |
261 | } | |
262 | } | |
263 | ||
264 | /// Calls llvm::createPGOFuncNameVar() with the given function instance's | |
265 | /// mangled function name. The LLVM API returns an llvm::GlobalVariable | |
266 | /// containing the function name, with the specific variable name and linkage | |
267 | /// required by LLVM InstrProf source-based coverage instrumentation. Use | |
268 | /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per | |
269 | /// `Instance`. | |
a2a8927a | 270 | fn create_pgo_func_name_var<'ll, 'tcx>( |
cdc7bbd5 XL |
271 | cx: &CodegenCx<'ll, 'tcx>, |
272 | instance: Instance<'tcx>, | |
273 | ) -> &'ll llvm::Value { | |
274 | let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) | |
275 | .expect("error converting function name to C string"); | |
276 | let llfn = cx.get_fn(instance); | |
277 | unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } | |
278 | } | |
279 | ||
3dfed10e XL |
280 | pub(crate) fn write_filenames_section_to_buffer<'a>( |
281 | filenames: impl IntoIterator<Item = &'a CString>, | |
282 | buffer: &RustString, | |
283 | ) { | |
284 | let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>(); | |
285 | unsafe { | |
286 | llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( | |
287 | c_str_vec.as_ptr(), | |
288 | c_str_vec.len(), | |
289 | buffer, | |
290 | ); | |
291 | } | |
292 | } | |
293 | ||
294 | pub(crate) fn write_mapping_to_buffer( | |
295 | virtual_file_mapping: Vec<u32>, | |
296 | expressions: Vec<CounterExpression>, | |
6a06907d | 297 | mapping_regions: Vec<CounterMappingRegion>, |
3dfed10e XL |
298 | buffer: &RustString, |
299 | ) { | |
300 | unsafe { | |
301 | llvm::LLVMRustCoverageWriteMappingToBuffer( | |
302 | virtual_file_mapping.as_ptr(), | |
303 | virtual_file_mapping.len() as c_uint, | |
304 | expressions.as_ptr(), | |
305 | expressions.len() as c_uint, | |
6a06907d | 306 | mapping_regions.as_ptr(), |
3dfed10e XL |
307 | mapping_regions.len() as c_uint, |
308 | buffer, | |
309 | ); | |
310 | } | |
311 | } | |
cdc7bbd5 | 312 | |
fc512014 XL |
313 | pub(crate) fn hash_str(strval: &str) -> u64 { |
314 | let strval = CString::new(strval).expect("null error converting hashable str to C string"); | |
315 | unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } | |
316 | } | |
3dfed10e | 317 | |
fc512014 XL |
318 | pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 { |
319 | unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } | |
3dfed10e XL |
320 | } |
321 | ||
322 | pub(crate) fn mapping_version() -> u32 { | |
323 | unsafe { llvm::LLVMRustCoverageMappingVersion() } | |
324 | } | |
325 | ||
fc512014 | 326 | pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( |
3dfed10e XL |
327 | cx: &CodegenCx<'ll, 'tcx>, |
328 | cov_data_val: &'ll llvm::Value, | |
329 | ) { | |
330 | let covmap_var_name = llvm::build_string(|s| unsafe { | |
331 | llvm::LLVMRustCoverageWriteMappingVarNameToString(s); | |
332 | }) | |
333 | .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); | |
334 | debug!("covmap var name: {:?}", covmap_var_name); | |
335 | ||
336 | let covmap_section_name = llvm::build_string(|s| unsafe { | |
fc512014 | 337 | llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); |
3dfed10e XL |
338 | }) |
339 | .expect("Rust Coverage section name failed UTF-8 conversion"); | |
340 | debug!("covmap section name: {:?}", covmap_section_name); | |
341 | ||
342 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); | |
343 | llvm::set_initializer(llglobal, cov_data_val); | |
344 | llvm::set_global_constant(llglobal, true); | |
fc512014 | 345 | llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); |
3dfed10e | 346 | llvm::set_section(llglobal, &covmap_section_name); |
fc512014 XL |
347 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); |
348 | cx.add_used_global(llglobal); | |
349 | } | |
350 | ||
351 | pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( | |
352 | cx: &CodegenCx<'ll, 'tcx>, | |
353 | func_name_hash: u64, | |
354 | func_record_val: &'ll llvm::Value, | |
355 | is_used: bool, | |
356 | ) { | |
357 | // Assign a name to the function record. This is used to merge duplicates. | |
358 | // | |
359 | // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that | |
360 | // are included-but-not-used. If (or when) Rust generates functions that are | |
361 | // included-but-not-used, note that a dummy description for a function included-but-not-used | |
362 | // in a Crate can be replaced by full description provided by a different Crate. The two kinds | |
363 | // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by | |
364 | // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. | |
365 | let func_record_var_name = | |
366 | format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); | |
367 | debug!("function record var name: {:?}", func_record_var_name); | |
368 | ||
369 | let func_record_section_name = llvm::build_string(|s| unsafe { | |
370 | llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); | |
371 | }) | |
372 | .expect("Rust Coverage function record section name failed UTF-8 conversion"); | |
373 | debug!("function record section name: {:?}", func_record_section_name); | |
374 | ||
375 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); | |
376 | llvm::set_initializer(llglobal, func_record_val); | |
377 | llvm::set_global_constant(llglobal, true); | |
378 | llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); | |
379 | llvm::set_visibility(llglobal, llvm::Visibility::Hidden); | |
380 | llvm::set_section(llglobal, &func_record_section_name); | |
381 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); | |
382 | llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); | |
3dfed10e XL |
383 | cx.add_used_global(llglobal); |
384 | } |