]>
Commit | Line | Data |
---|---|---|
3dfed10e XL |
1 | use crate::llvm; |
2 | ||
cdc7bbd5 | 3 | use crate::abi::{Abi, FnAbi}; |
f035d41b XL |
4 | use crate::builder::Builder; |
5 | use crate::common::CodegenCx; | |
3dfed10e XL |
6 | |
7 | use libc::c_uint; | |
8 | use llvm::coverageinfo::CounterMappingRegion; | |
9 | use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; | |
10 | use rustc_codegen_ssa::traits::{ | |
cdc7bbd5 XL |
11 | BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, |
12 | MiscMethods, StaticMethods, | |
3dfed10e | 13 | }; |
f035d41b | 14 | use rustc_data_structures::fx::FxHashMap; |
cdc7bbd5 XL |
15 | use rustc_hir as hir; |
16 | use rustc_hir::def_id::DefId; | |
3dfed10e | 17 | use rustc_llvm::RustString; |
cdc7bbd5 | 18 | use rustc_middle::bug; |
3dfed10e | 19 | use rustc_middle::mir::coverage::{ |
29967ef6 | 20 | CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, |
3dfed10e | 21 | }; |
cdc7bbd5 XL |
22 | use rustc_middle::ty; |
23 | use rustc_middle::ty::layout::FnAbiExt; | |
24 | use rustc_middle::ty::subst::InternalSubsts; | |
f035d41b XL |
25 | use rustc_middle::ty::Instance; |
26 | ||
27 | use std::cell::RefCell; | |
3dfed10e XL |
28 | use std::ffi::CString; |
29 | ||
cdc7bbd5 | 30 | use std::iter; |
3dfed10e XL |
31 | use tracing::debug; |
32 | ||
33 | pub mod mapgen; | |
34 | ||
cdc7bbd5 XL |
35 | const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; |
36 | ||
fc512014 | 37 | const VAR_ALIGN_BYTES: usize = 8; |
f035d41b XL |
38 | |
39 | /// A context object for maintaining all state needed by the coverageinfo module. | |
cdc7bbd5 | 40 | pub struct CrateCoverageContext<'ll, 'tcx> { |
29967ef6 XL |
41 | // Coverage data for each instrumented function identified by DefId. |
42 | pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, | |
cdc7bbd5 | 43 | pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, |
f035d41b XL |
44 | } |
45 | ||
cdc7bbd5 | 46 | impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { |
f035d41b | 47 | pub fn new() -> Self { |
cdc7bbd5 XL |
48 | Self { |
49 | function_coverage_map: Default::default(), | |
50 | pgo_func_name_var_map: Default::default(), | |
51 | } | |
f035d41b | 52 | } |
f035d41b | 53 | |
29967ef6 | 54 | pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { |
3dfed10e | 55 | self.function_coverage_map.replace(FxHashMap::default()) |
f035d41b XL |
56 | } |
57 | } | |
58 | ||
cdc7bbd5 | 59 | impl CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { |
f035d41b | 60 | fn coverageinfo_finalize(&self) { |
3dfed10e | 61 | mapgen::finalize(self) |
f035d41b | 62 | } |
f035d41b | 63 | |
cdc7bbd5 XL |
64 | fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { |
65 | if let Some(coverage_context) = self.coverage_context() { | |
66 | debug!("getting pgo_func_name_var for instance={:?}", instance); | |
67 | let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); | |
68 | pgo_func_name_var_map | |
69 | .entry(instance) | |
70 | .or_insert_with(|| create_pgo_func_name_var(self, instance)) | |
71 | } else { | |
72 | bug!("Could not get the `coverage_context`"); | |
73 | } | |
3dfed10e XL |
74 | } |
75 | ||
cdc7bbd5 XL |
76 | /// Functions with MIR-based coverage are normally codegenned _only_ if |
77 | /// called. LLVM coverage tools typically expect every function to be | |
78 | /// defined (even if unused), with at least one call to LLVM intrinsic | |
79 | /// `instrprof.increment`. | |
80 | /// | |
81 | /// Codegen a small function that will never be called, with one counter | |
82 | /// that will never be incremented. | |
83 | /// | |
84 | /// For used/called functions, the coverageinfo was already added to the | |
85 | /// `function_coverage_map` (keyed by function `Instance`) during codegen. | |
86 | /// But in this case, since the unused function was _not_ previously | |
87 | /// codegenned, collect the coverage `CodeRegion`s from the MIR and add | |
88 | /// them. The first `CodeRegion` is used to add a single counter, with the | |
89 | /// same counter ID used in the injected `instrprof.increment` intrinsic | |
90 | /// call. Since the function is never called, all other `CodeRegion`s can be | |
91 | /// added as `unreachable_region`s. | |
92 | fn define_unused_fn(&self, def_id: DefId) { | |
93 | let instance = declare_unused_fn(self, &def_id); | |
94 | codegen_unused_fn_and_counter(self, instance); | |
95 | add_unused_function_coverage(self, instance, def_id); | |
96 | } | |
97 | } | |
98 | ||
99 | impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { | |
29967ef6 | 100 | fn set_function_source_hash( |
f035d41b XL |
101 | &mut self, |
102 | instance: Instance<'tcx>, | |
3dfed10e | 103 | function_source_hash: u64, |
29967ef6 XL |
104 | ) -> bool { |
105 | if let Some(coverage_context) = self.coverage_context() { | |
106 | debug!( | |
107 | "ensuring function source hash is set for instance={:?}; function_source_hash={}", | |
108 | instance, function_source_hash, | |
109 | ); | |
110 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
111 | coverage_map | |
112 | .entry(instance) | |
113 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
114 | .set_function_source_hash(function_source_hash); | |
115 | true | |
116 | } else { | |
117 | false | |
118 | } | |
119 | } | |
120 | ||
121 | fn add_coverage_counter( | |
122 | &mut self, | |
123 | instance: Instance<'tcx>, | |
3dfed10e XL |
124 | id: CounterValueReference, |
125 | region: CodeRegion, | |
29967ef6 XL |
126 | ) -> bool { |
127 | if let Some(coverage_context) = self.coverage_context() { | |
128 | debug!( | |
129 | "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", | |
130 | instance, id, region, | |
131 | ); | |
132 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
133 | coverage_map | |
134 | .entry(instance) | |
135 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
136 | .add_counter(id, region); | |
137 | true | |
138 | } else { | |
139 | false | |
140 | } | |
f035d41b XL |
141 | } |
142 | ||
29967ef6 | 143 | fn add_coverage_counter_expression( |
f035d41b XL |
144 | &mut self, |
145 | instance: Instance<'tcx>, | |
29967ef6 | 146 | id: InjectedExpressionId, |
3dfed10e XL |
147 | lhs: ExpressionOperandId, |
148 | op: Op, | |
149 | rhs: ExpressionOperandId, | |
29967ef6 XL |
150 | region: Option<CodeRegion>, |
151 | ) -> bool { | |
152 | if let Some(coverage_context) = self.coverage_context() { | |
153 | debug!( | |
154 | "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ | |
155 | region: {:?}", | |
156 | instance, id, lhs, op, rhs, region, | |
157 | ); | |
158 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
159 | coverage_map | |
160 | .entry(instance) | |
161 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
162 | .add_counter_expression(id, lhs, op, rhs, region); | |
163 | true | |
164 | } else { | |
165 | false | |
166 | } | |
f035d41b XL |
167 | } |
168 | ||
29967ef6 XL |
169 | fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { |
170 | if let Some(coverage_context) = self.coverage_context() { | |
171 | debug!( | |
172 | "adding unreachable code to coverage_map: instance={:?}, at {:?}", | |
173 | instance, region, | |
174 | ); | |
175 | let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); | |
176 | coverage_map | |
177 | .entry(instance) | |
178 | .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) | |
179 | .add_unreachable_region(region); | |
180 | true | |
181 | } else { | |
182 | false | |
183 | } | |
f035d41b XL |
184 | } |
185 | } | |
3dfed10e | 186 | |
cdc7bbd5 XL |
187 | fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx> { |
188 | let tcx = cx.tcx; | |
189 | ||
190 | let instance = Instance::new( | |
191 | *def_id, | |
192 | InternalSubsts::for_item(tcx, *def_id, |param, _| { | |
193 | if let ty::GenericParamDefKind::Lifetime = param.kind { | |
194 | tcx.lifetimes.re_erased.into() | |
195 | } else { | |
196 | tcx.mk_param_from_def(param) | |
197 | } | |
198 | }), | |
199 | ); | |
200 | ||
201 | let llfn = cx.declare_fn( | |
202 | &tcx.symbol_name(instance).name, | |
203 | &FnAbi::of_fn_ptr( | |
204 | cx, | |
205 | ty::Binder::dummy(tcx.mk_fn_sig( | |
206 | iter::once(tcx.mk_unit()), | |
207 | tcx.mk_unit(), | |
208 | false, | |
209 | hir::Unsafety::Unsafe, | |
210 | Abi::Rust, | |
211 | )), | |
212 | &[], | |
213 | ), | |
214 | ); | |
215 | ||
216 | llvm::set_linkage(llfn, llvm::Linkage::WeakAnyLinkage); | |
217 | llvm::set_visibility(llfn, llvm::Visibility::Hidden); | |
218 | ||
219 | assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); | |
220 | ||
221 | instance | |
222 | } | |
223 | ||
224 | fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) { | |
225 | let llfn = cx.get_fn(instance); | |
17df50a5 XL |
226 | let llbb = Builder::append_block(cx, llfn, "unused_function"); |
227 | let mut bx = Builder::build(cx, llbb); | |
cdc7bbd5 XL |
228 | let fn_name = bx.get_pgo_func_name_var(instance); |
229 | let hash = bx.const_u64(0); | |
230 | let num_counters = bx.const_u32(1); | |
231 | let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); | |
232 | debug!( | |
233 | "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, | |
234 | index={:?}) for unused function: {:?}", | |
235 | fn_name, hash, num_counters, index, instance | |
236 | ); | |
237 | bx.instrprof_increment(fn_name, hash, num_counters, index); | |
238 | bx.ret_void(); | |
239 | } | |
240 | ||
241 | fn add_unused_function_coverage( | |
242 | cx: &CodegenCx<'ll, 'tcx>, | |
243 | instance: Instance<'tcx>, | |
244 | def_id: DefId, | |
245 | ) { | |
246 | let tcx = cx.tcx; | |
247 | ||
248 | let mut function_coverage = FunctionCoverage::unused(tcx, instance); | |
249 | for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { | |
250 | if index == 0 { | |
251 | // Insert at least one real counter so the LLVM CoverageMappingReader will find expected | |
252 | // definitions. | |
253 | function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); | |
254 | } else { | |
255 | function_coverage.add_unreachable_region(code_region.clone()); | |
256 | } | |
257 | } | |
258 | ||
259 | if let Some(coverage_context) = cx.coverage_context() { | |
260 | coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); | |
261 | } else { | |
262 | bug!("Could not get the `coverage_context`"); | |
263 | } | |
264 | } | |
265 | ||
266 | /// Calls llvm::createPGOFuncNameVar() with the given function instance's | |
267 | /// mangled function name. The LLVM API returns an llvm::GlobalVariable | |
268 | /// containing the function name, with the specific variable name and linkage | |
269 | /// required by LLVM InstrProf source-based coverage instrumentation. Use | |
270 | /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per | |
271 | /// `Instance`. | |
272 | fn create_pgo_func_name_var( | |
273 | cx: &CodegenCx<'ll, 'tcx>, | |
274 | instance: Instance<'tcx>, | |
275 | ) -> &'ll llvm::Value { | |
276 | let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) | |
277 | .expect("error converting function name to C string"); | |
278 | let llfn = cx.get_fn(instance); | |
279 | unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } | |
280 | } | |
281 | ||
3dfed10e XL |
282 | pub(crate) fn write_filenames_section_to_buffer<'a>( |
283 | filenames: impl IntoIterator<Item = &'a CString>, | |
284 | buffer: &RustString, | |
285 | ) { | |
286 | let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>(); | |
287 | unsafe { | |
288 | llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( | |
289 | c_str_vec.as_ptr(), | |
290 | c_str_vec.len(), | |
291 | buffer, | |
292 | ); | |
293 | } | |
294 | } | |
295 | ||
296 | pub(crate) fn write_mapping_to_buffer( | |
297 | virtual_file_mapping: Vec<u32>, | |
298 | expressions: Vec<CounterExpression>, | |
6a06907d | 299 | mapping_regions: Vec<CounterMappingRegion>, |
3dfed10e XL |
300 | buffer: &RustString, |
301 | ) { | |
302 | unsafe { | |
303 | llvm::LLVMRustCoverageWriteMappingToBuffer( | |
304 | virtual_file_mapping.as_ptr(), | |
305 | virtual_file_mapping.len() as c_uint, | |
306 | expressions.as_ptr(), | |
307 | expressions.len() as c_uint, | |
6a06907d | 308 | mapping_regions.as_ptr(), |
3dfed10e XL |
309 | mapping_regions.len() as c_uint, |
310 | buffer, | |
311 | ); | |
312 | } | |
313 | } | |
cdc7bbd5 | 314 | |
fc512014 XL |
315 | pub(crate) fn hash_str(strval: &str) -> u64 { |
316 | let strval = CString::new(strval).expect("null error converting hashable str to C string"); | |
317 | unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } | |
318 | } | |
3dfed10e | 319 | |
fc512014 XL |
320 | pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 { |
321 | unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } | |
3dfed10e XL |
322 | } |
323 | ||
324 | pub(crate) fn mapping_version() -> u32 { | |
325 | unsafe { llvm::LLVMRustCoverageMappingVersion() } | |
326 | } | |
327 | ||
fc512014 | 328 | pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( |
3dfed10e XL |
329 | cx: &CodegenCx<'ll, 'tcx>, |
330 | cov_data_val: &'ll llvm::Value, | |
331 | ) { | |
332 | let covmap_var_name = llvm::build_string(|s| unsafe { | |
333 | llvm::LLVMRustCoverageWriteMappingVarNameToString(s); | |
334 | }) | |
335 | .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); | |
336 | debug!("covmap var name: {:?}", covmap_var_name); | |
337 | ||
338 | let covmap_section_name = llvm::build_string(|s| unsafe { | |
fc512014 | 339 | llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); |
3dfed10e XL |
340 | }) |
341 | .expect("Rust Coverage section name failed UTF-8 conversion"); | |
342 | debug!("covmap section name: {:?}", covmap_section_name); | |
343 | ||
344 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); | |
345 | llvm::set_initializer(llglobal, cov_data_val); | |
346 | llvm::set_global_constant(llglobal, true); | |
fc512014 | 347 | llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); |
3dfed10e | 348 | llvm::set_section(llglobal, &covmap_section_name); |
fc512014 XL |
349 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); |
350 | cx.add_used_global(llglobal); | |
351 | } | |
352 | ||
353 | pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( | |
354 | cx: &CodegenCx<'ll, 'tcx>, | |
355 | func_name_hash: u64, | |
356 | func_record_val: &'ll llvm::Value, | |
357 | is_used: bool, | |
358 | ) { | |
359 | // Assign a name to the function record. This is used to merge duplicates. | |
360 | // | |
361 | // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that | |
362 | // are included-but-not-used. If (or when) Rust generates functions that are | |
363 | // included-but-not-used, note that a dummy description for a function included-but-not-used | |
364 | // in a Crate can be replaced by full description provided by a different Crate. The two kinds | |
365 | // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by | |
366 | // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. | |
367 | let func_record_var_name = | |
368 | format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); | |
369 | debug!("function record var name: {:?}", func_record_var_name); | |
370 | ||
371 | let func_record_section_name = llvm::build_string(|s| unsafe { | |
372 | llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); | |
373 | }) | |
374 | .expect("Rust Coverage function record section name failed UTF-8 conversion"); | |
375 | debug!("function record section name: {:?}", func_record_section_name); | |
376 | ||
377 | let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); | |
378 | llvm::set_initializer(llglobal, func_record_val); | |
379 | llvm::set_global_constant(llglobal, true); | |
380 | llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); | |
381 | llvm::set_visibility(llglobal, llvm::Visibility::Hidden); | |
382 | llvm::set_section(llglobal, &func_record_section_name); | |
383 | llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); | |
384 | llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); | |
3dfed10e XL |
385 | cx.add_used_global(llglobal); |
386 | } |