]>
Commit | Line | Data |
---|---|---|
3dfed10e XL |
1 | use crate::llvm; |
2 | ||
c295e0f8 | 3 | use crate::abi::Abi; |
f035d41b XL |
4 | use crate::builder::Builder; |
5 | use crate::common::CodegenCx; | |
3dfed10e XL |
6 | |
7 | use libc::c_uint; | |
8 | use llvm::coverageinfo::CounterMappingRegion; | |
9 | use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; | |
10 | use rustc_codegen_ssa::traits::{ | |
cdc7bbd5 XL |
11 | BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, |
12 | MiscMethods, StaticMethods, | |
3dfed10e | 13 | }; |
f035d41b | 14 | use rustc_data_structures::fx::FxHashMap; |
cdc7bbd5 XL |
15 | use rustc_hir as hir; |
16 | use rustc_hir::def_id::DefId; | |
3dfed10e | 17 | use rustc_llvm::RustString; |
cdc7bbd5 | 18 | use rustc_middle::bug; |
3dfed10e | 19 | use rustc_middle::mir::coverage::{ |
29967ef6 | 20 | CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, |
3dfed10e | 21 | }; |
cdc7bbd5 | 22 | use rustc_middle::ty; |
c295e0f8 | 23 | use rustc_middle::ty::layout::FnAbiOf; |
cdc7bbd5 | 24 | use rustc_middle::ty::subst::InternalSubsts; |
f035d41b XL |
25 | use rustc_middle::ty::Instance; |
26 | ||
27 | use std::cell::RefCell; | |
3dfed10e XL |
28 | use std::ffi::CString; |
29 | ||
cdc7bbd5 | 30 | use std::iter; |
3dfed10e XL |
31 | use tracing::debug; |
32 | ||
33 | pub mod mapgen; | |
34 | ||
cdc7bbd5 XL |
35 | const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; |
36 | ||
fc512014 | 37 | const VAR_ALIGN_BYTES: usize = 8; |
f035d41b XL |
38 | |
39 | /// A context object for maintaining all state needed by the coverageinfo module. | |
cdc7bbd5 | 40 | pub struct CrateCoverageContext<'ll, 'tcx> { |
29967ef6 XL |
41 | // Coverage data for each instrumented function identified by DefId. |
42 | pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, | |
cdc7bbd5 | 43 | pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, |
f035d41b XL |
44 | } |
45 | ||
cdc7bbd5 | 46 | impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { |
f035d41b | 47 | pub fn new() -> Self { |
cdc7bbd5 XL |
48 | Self { |
49 | function_coverage_map: Default::default(), | |
50 | pgo_func_name_var_map: Default::default(), | |
51 | } | |
f035d41b | 52 | } |
f035d41b | 53 | |
29967ef6 | 54 | pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { |
3dfed10e | 55 | self.function_coverage_map.replace(FxHashMap::default()) |
f035d41b XL |
56 | } |
57 | } | |
58 | ||
cdc7bbd5 | 59 | impl CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { |
f035d41b | 60 | fn coverageinfo_finalize(&self) { |
3dfed10e | 61 | mapgen::finalize(self) |
f035d41b | 62 | } |
f035d41b | 63 | |
cdc7bbd5 XL |
64 | fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { |
65 | if let Some(coverage_context) = self.coverage_context() { | |
66 | debug!("getting pgo_func_name_var for instance={:?}", instance); | |
67 | let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); | |
68 | pgo_func_name_var_map | |
69 | .entry(instance) | |
70 | .or_insert_with(|| create_pgo_func_name_var(self, instance)) | |
71 | } else { | |
72 | bug!("Could not get the `coverage_context`"); | |
73 | } | |
3dfed10e XL |
74 | } |
75 | ||
cdc7bbd5 XL |
76 | /// Functions with MIR-based coverage are normally codegenned _only_ if |
77 | /// called. LLVM coverage tools typically expect every function to be | |
78 | /// defined (even if unused), with at least one call to LLVM intrinsic | |
79 | /// `instrprof.increment`. | |
80 | /// | |
81 | /// Codegen a small function that will never be called, with one counter | |
82 | /// that will never be incremented. | |
83 | /// | |
84 | /// For used/called functions, the coverageinfo was already added to the | |
85 | /// `function_coverage_map` (keyed by function `Instance`) during codegen. | |
86 | /// But in this case, since the unused function was _not_ previously | |
87 | /// codegenned, collect the coverage `CodeRegion`s from the MIR and add | |
88 | /// them. The first `CodeRegion` is used to add a single counter, with the | |
89 | /// same counter ID used in the injected `instrprof.increment` intrinsic | |
90 | /// call. Since the function is never called, all other `CodeRegion`s can be | |
91 | /// added as `unreachable_region`s. | |
92 | fn define_unused_fn(&self, def_id: DefId) { | |
93 | let instance = declare_unused_fn(self, &def_id); | |
94 | codegen_unused_fn_and_counter(self, instance); | |
95 | add_unused_function_coverage(self, instance, def_id); | |
96 | } | |
97 | } | |
98 | ||
99 | impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { | |
29967ef6 | 100 | fn set_function_source_hash( |
f035d41b XL |
101 | &mut self, |
102 | instance: Instance<'tcx>, | |
3dfed10e | 103 | function_source_hash: u64, |
29967ef6 XL |
104 | ) -> bool { |
105 | if let Some(coverage_context) = self.coverage_context() { | |
106 | debug!( | |
107 | "ensuring function source hash is set for instance={:?}; function_source_hash={}", | |
108 | instance, function_source_hash, | |
109 | ); | |
110 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
111 | coverage_map | |
112 | .entry(instance) | |
113 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
114 | .set_function_source_hash(function_source_hash); | |
115 | true | |
116 | } else { | |
117 | false | |
118 | } | |
119 | } | |
120 | ||
121 | fn add_coverage_counter( | |
122 | &mut self, | |
123 | instance: Instance<'tcx>, | |
3dfed10e XL |
124 | id: CounterValueReference, |
125 | region: CodeRegion, | |
29967ef6 XL |
126 | ) -> bool { |
127 | if let Some(coverage_context) = self.coverage_context() { | |
128 | debug!( | |
129 | "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", | |
130 | instance, id, region, | |
131 | ); | |
132 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
133 | coverage_map | |
134 | .entry(instance) | |
135 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
136 | .add_counter(id, region); | |
137 | true | |
138 | } else { | |
139 | false | |
140 | } | |
f035d41b XL |
141 | } |
142 | ||
29967ef6 | 143 | fn add_coverage_counter_expression( |
f035d41b XL |
144 | &mut self, |
145 | instance: Instance<'tcx>, | |
29967ef6 | 146 | id: InjectedExpressionId, |
3dfed10e XL |
147 | lhs: ExpressionOperandId, |
148 | op: Op, | |
149 | rhs: ExpressionOperandId, | |
29967ef6 XL |
150 | region: Option<CodeRegion>, |
151 | ) -> bool { | |
152 | if let Some(coverage_context) = self.coverage_context() { | |
153 | debug!( | |
154 | "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ | |
155 | region: {:?}", | |
156 | instance, id, lhs, op, rhs, region, | |
157 | ); | |
158 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
159 | coverage_map | |
160 | .entry(instance) | |
161 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
162 | .add_counter_expression(id, lhs, op, rhs, region); | |
163 | true | |
164 | } else { | |
165 | false | |
166 | } | |
f035d41b XL |
167 | } |
168 | ||
29967ef6 XL |
169 | fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { |
170 | if let Some(coverage_context) = self.coverage_context() { | |
171 | debug!( | |
172 | "adding unreachable code to coverage_map: instance={:?}, at {:?}", | |
173 | instance, region, | |
174 | ); | |
175 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
176 | coverage_map | |
177 | .entry(instance) | |
178 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
179 | .add_unreachable_region(region); | |
180 | true | |
181 | } else { | |
182 | false | |
183 | } | |
f035d41b XL |
184 | } |
185 | } | |
3dfed10e | 186 | |
cdc7bbd5 XL |
187 | fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx> { |
188 | let tcx = cx.tcx; | |
189 | ||
190 | let instance = Instance::new( | |
191 | *def_id, | |
192 | InternalSubsts::for_item(tcx, *def_id, |param, _| { | |
193 | if let ty::GenericParamDefKind::Lifetime = param.kind { | |
194 | tcx.lifetimes.re_erased.into() | |
195 | } else { | |
196 | tcx.mk_param_from_def(param) | |
197 | } | |
198 | }), | |
199 | ); | |
200 | ||
201 | let llfn = cx.declare_fn( | |
c295e0f8 XL |
202 | tcx.symbol_name(instance).name, |
203 | cx.fn_abi_of_fn_ptr( | |
cdc7bbd5 XL |
204 | ty::Binder::dummy(tcx.mk_fn_sig( |
205 | iter::once(tcx.mk_unit()), | |
206 | tcx.mk_unit(), | |
207 | false, | |
208 | hir::Unsafety::Unsafe, | |
209 | Abi::Rust, | |
210 | )), | |
c295e0f8 | 211 | ty::List::empty(), |
cdc7bbd5 XL |
212 | ), |
213 | ); | |
214 | ||
215 | llvm::set_linkage(llfn, llvm::Linkage::WeakAnyLinkage); | |
216 | llvm::set_visibility(llfn, llvm::Visibility::Hidden); | |
217 | ||
218 | assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); | |
219 | ||
220 | instance | |
221 | } | |
222 | ||
223 | fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) { | |
224 | let llfn = cx.get_fn(instance); | |
17df50a5 XL |
225 | let llbb = Builder::append_block(cx, llfn, "unused_function"); |
226 | let mut bx = Builder::build(cx, llbb); | |
cdc7bbd5 XL |
227 | let fn_name = bx.get_pgo_func_name_var(instance); |
228 | let hash = bx.const_u64(0); | |
229 | let num_counters = bx.const_u32(1); | |
230 | let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); | |
231 | debug!( | |
232 | "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, | |
233 | index={:?}) for unused function: {:?}", | |
234 | fn_name, hash, num_counters, index, instance | |
235 | ); | |
236 | bx.instrprof_increment(fn_name, hash, num_counters, index); | |
237 | bx.ret_void(); | |
238 | } | |
239 | ||
240 | fn add_unused_function_coverage( | |
241 | cx: &CodegenCx<'ll, 'tcx>, | |
242 | instance: Instance<'tcx>, | |
243 | def_id: DefId, | |
244 | ) { | |
245 | let tcx = cx.tcx; | |
246 | ||
247 | let mut function_coverage = FunctionCoverage::unused(tcx, instance); | |
248 | for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { | |
249 | if index == 0 { | |
250 | // Insert at least one real counter so the LLVM CoverageMappingReader will find expected | |
251 | // definitions. | |
252 | function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); | |
253 | } else { | |
254 | function_coverage.add_unreachable_region(code_region.clone()); | |
255 | } | |
256 | } | |
257 | ||
258 | if let Some(coverage_context) = cx.coverage_context() { | |
259 | coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); | |
260 | } else { | |
261 | bug!("Could not get the `coverage_context`"); | |
262 | } | |
263 | } | |
264 | ||
265 | /// Calls llvm::createPGOFuncNameVar() with the given function instance's | |
266 | /// mangled function name. The LLVM API returns an llvm::GlobalVariable | |
267 | /// containing the function name, with the specific variable name and linkage | |
268 | /// required by LLVM InstrProf source-based coverage instrumentation. Use | |
269 | /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per | |
270 | /// `Instance`. | |
271 | fn create_pgo_func_name_var( | |
272 | cx: &CodegenCx<'ll, 'tcx>, | |
273 | instance: Instance<'tcx>, | |
274 | ) -> &'ll llvm::Value { | |
275 | let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) | |
276 | .expect("error converting function name to C string"); | |
277 | let llfn = cx.get_fn(instance); | |
278 | unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } | |
279 | } | |
280 | ||
3dfed10e XL |
281 | pub(crate) fn write_filenames_section_to_buffer<'a>( |
282 | filenames: impl IntoIterator<Item = &'a CString>, | |
283 | buffer: &RustString, | |
284 | ) { | |
285 | let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>(); | |
286 | unsafe { | |
287 | llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( | |
288 | c_str_vec.as_ptr(), | |
289 | c_str_vec.len(), | |
290 | buffer, | |
291 | ); | |
292 | } | |
293 | } | |
294 | ||
295 | pub(crate) fn write_mapping_to_buffer( | |
296 | virtual_file_mapping: Vec<u32>, | |
297 | expressions: Vec<CounterExpression>, | |
6a06907d | 298 | mapping_regions: Vec<CounterMappingRegion>, |
3dfed10e XL |
299 | buffer: &RustString, |
300 | ) { | |
301 | unsafe { | |
302 | llvm::LLVMRustCoverageWriteMappingToBuffer( | |
303 | virtual_file_mapping.as_ptr(), | |
304 | virtual_file_mapping.len() as c_uint, | |
305 | expressions.as_ptr(), | |
306 | expressions.len() as c_uint, | |
6a06907d | 307 | mapping_regions.as_ptr(), |
3dfed10e XL |
308 | mapping_regions.len() as c_uint, |
309 | buffer, | |
310 | ); | |
311 | } | |
312 | } | |
cdc7bbd5 | 313 | |
fc512014 XL |
314 | pub(crate) fn hash_str(strval: &str) -> u64 { |
315 | let strval = CString::new(strval).expect("null error converting hashable str to C string"); | |
316 | unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } | |
317 | } | |
3dfed10e | 318 | |
fc512014 XL |
319 | pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 { |
320 | unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } | |
3dfed10e XL |
321 | } |
322 | ||
323 | pub(crate) fn mapping_version() -> u32 { | |
324 | unsafe { llvm::LLVMRustCoverageMappingVersion() } | |
325 | } | |
326 | ||
fc512014 | 327 | pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( |
3dfed10e XL |
328 | cx: &CodegenCx<'ll, 'tcx>, |
329 | cov_data_val: &'ll llvm::Value, | |
330 | ) { | |
331 | let covmap_var_name = llvm::build_string(|s| unsafe { | |
332 | llvm::LLVMRustCoverageWriteMappingVarNameToString(s); | |
333 | }) | |
334 | .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); | |
335 | debug!("covmap var name: {:?}", covmap_var_name); | |
336 | ||
337 | let covmap_section_name = llvm::build_string(|s| unsafe { | |
fc512014 | 338 | llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); |
3dfed10e XL |
339 | }) |
340 | .expect("Rust Coverage section name failed UTF-8 conversion"); | |
341 | debug!("covmap section name: {:?}", covmap_section_name); | |
342 | ||
343 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); | |
344 | llvm::set_initializer(llglobal, cov_data_val); | |
345 | llvm::set_global_constant(llglobal, true); | |
fc512014 | 346 | llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); |
3dfed10e | 347 | llvm::set_section(llglobal, &covmap_section_name); |
fc512014 XL |
348 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); |
349 | cx.add_used_global(llglobal); | |
350 | } | |
351 | ||
352 | pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( | |
353 | cx: &CodegenCx<'ll, 'tcx>, | |
354 | func_name_hash: u64, | |
355 | func_record_val: &'ll llvm::Value, | |
356 | is_used: bool, | |
357 | ) { | |
358 | // Assign a name to the function record. This is used to merge duplicates. | |
359 | // | |
360 | // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that | |
361 | // are included-but-not-used. If (or when) Rust generates functions that are | |
362 | // included-but-not-used, note that a dummy description for a function included-but-not-used | |
363 | // in a Crate can be replaced by full description provided by a different Crate. The two kinds | |
364 | // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by | |
365 | // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. | |
366 | let func_record_var_name = | |
367 | format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); | |
368 | debug!("function record var name: {:?}", func_record_var_name); | |
369 | ||
370 | let func_record_section_name = llvm::build_string(|s| unsafe { | |
371 | llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); | |
372 | }) | |
373 | .expect("Rust Coverage function record section name failed UTF-8 conversion"); | |
374 | debug!("function record section name: {:?}", func_record_section_name); | |
375 | ||
376 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); | |
377 | llvm::set_initializer(llglobal, func_record_val); | |
378 | llvm::set_global_constant(llglobal, true); | |
379 | llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); | |
380 | llvm::set_visibility(llglobal, llvm::Visibility::Hidden); | |
381 | llvm::set_section(llglobal, &func_record_section_name); | |
382 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); | |
383 | llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); | |
3dfed10e XL |
384 | cx.add_used_global(llglobal); |
385 | } |