11 use counters
::CoverageCounters
;
12 use graph
::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph}
;
13 use spans
::{CoverageSpan, CoverageSpans}
;
17 use rustc_data_structures
::graph
::WithNumNodes
;
18 use rustc_data_structures
::sync
::Lrc
;
19 use rustc_index
::vec
::IndexVec
;
20 use rustc_middle
::hir
;
21 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
22 use rustc_middle
::mir
::coverage
::*;
23 use rustc_middle
::mir
::dump_enabled
;
24 use rustc_middle
::mir
::{
25 self, BasicBlock
, BasicBlockData
, Coverage
, SourceInfo
, Statement
, StatementKind
, Terminator
,
28 use rustc_middle
::ty
::TyCtxt
;
29 use rustc_span
::def_id
::DefId
;
30 use rustc_span
::source_map
::SourceMap
;
31 use rustc_span
::{CharPos, ExpnKind, Pos, SourceFile, Span, Symbol}
;
33 /// A simple error message wrapper for `coverage::Error`s.
40 pub fn from_string
<T
>(message
: String
) -> Result
<T
, Error
> {
45 /// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected
46 /// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen
47 /// to construct the coverage map.
48 pub struct InstrumentCoverage
;
50 impl<'tcx
> MirPass
<'tcx
> for InstrumentCoverage
{
51 fn is_enabled(&self, sess
: &rustc_session
::Session
) -> bool
{
52 sess
.instrument_coverage()
55 fn run_pass(&self, tcx
: TyCtxt
<'tcx
>, mir_body
: &mut mir
::Body
<'tcx
>) {
56 let mir_source
= mir_body
.source
;
58 // If the InstrumentCoverage pass is called on promoted MIRs, skip them.
59 // See: https://github.com/rust-lang/rust/pull/73011#discussion_r438317601
60 if mir_source
.promoted
.is_some() {
62 "InstrumentCoverage skipped for {:?} (already promoted for Miri evaluation)",
69 tcx
.hir().get_by_def_id(mir_source
.def_id().expect_local()).fn_kind().is_some();
71 // Only instrument functions, methods, and closures (not constants since they are evaluated
72 // at compile time by Miri).
73 // FIXME(#73156): Handle source code coverage in const eval, but note, if and when const
74 // expressions get coverage spans, we will probably have to "carve out" space for const
75 // expressions from coverage spans in enclosing MIR's, like we do for closures. (That might
76 // be tricky if const expressions have no corresponding statements in the enclosing MIR.
77 // Closures are carved out by their initial `Assign` statement.)
79 trace
!("InstrumentCoverage skipped for {:?} (not an fn-like)", mir_source
.def_id());
83 match mir_body
.basic_blocks
[mir
::START_BLOCK
].terminator().kind
{
84 TerminatorKind
::Unreachable
=> {
85 trace
!("InstrumentCoverage skipped for unreachable `START_BLOCK`");
91 let codegen_fn_attrs
= tcx
.codegen_fn_attrs(mir_source
.def_id());
92 if codegen_fn_attrs
.flags
.contains(CodegenFnAttrFlags
::NO_COVERAGE
) {
96 trace
!("InstrumentCoverage starting for {:?}", mir_source
.def_id());
97 Instrumentor
::new(&self.name(), tcx
, mir_body
).inject_counters();
98 trace
!("InstrumentCoverage done for {:?}", mir_source
.def_id());
102 struct Instrumentor
<'a
, 'tcx
> {
105 mir_body
: &'a
mut mir
::Body
<'tcx
>,
106 source_file
: Lrc
<SourceFile
>,
109 basic_coverage_blocks
: CoverageGraph
,
110 coverage_counters
: CoverageCounters
,
113 impl<'a
, 'tcx
> Instrumentor
<'a
, 'tcx
> {
114 fn new(pass_name
: &'a
str, tcx
: TyCtxt
<'tcx
>, mir_body
: &'a
mut mir
::Body
<'tcx
>) -> Self {
115 let source_map
= tcx
.sess
.source_map();
116 let def_id
= mir_body
.source
.def_id();
117 let (some_fn_sig
, hir_body
) = fn_sig_and_body(tcx
, def_id
);
119 let body_span
= get_body_span(tcx
, hir_body
, mir_body
);
121 let source_file
= source_map
.lookup_source_file(body_span
.lo());
122 let fn_sig_span
= match some_fn_sig
.filter(|fn_sig
| {
123 fn_sig
.span
.eq_ctxt(body_span
)
124 && Lrc
::ptr_eq(&source_file
, &source_map
.lookup_source_file(fn_sig
.span
.lo()))
126 Some(fn_sig
) => fn_sig
.span
.with_hi(body_span
.lo()),
127 None
=> body_span
.shrink_to_lo(),
131 "instrumenting {}: {:?}, fn sig span: {:?}, body span: {:?}",
132 if tcx
.is_closure(def_id
) { "closure" }
else { "function" }
,
138 let function_source_hash
= hash_mir_source(tcx
, hir_body
);
139 let basic_coverage_blocks
= CoverageGraph
::from_mir(mir_body
);
147 basic_coverage_blocks
,
148 coverage_counters
: CoverageCounters
::new(function_source_hash
),
152 fn inject_counters(&'a
mut self) {
154 let mir_source
= self.mir_body
.source
;
155 let def_id
= mir_source
.def_id();
156 let fn_sig_span
= self.fn_sig_span
;
157 let body_span
= self.body_span
;
159 let mut graphviz_data
= debug
::GraphvizData
::new();
160 let mut debug_used_expressions
= debug
::UsedExpressions
::new();
162 let dump_mir
= dump_enabled(tcx
, self.pass_name
, def_id
);
163 let dump_graphviz
= dump_mir
&& tcx
.sess
.opts
.unstable_opts
.dump_mir_graphviz
;
164 let dump_spanview
= dump_mir
&& tcx
.sess
.opts
.unstable_opts
.dump_mir_spanview
.is_some();
167 graphviz_data
.enable();
168 self.coverage_counters
.enable_debug();
171 if dump_graphviz
|| level_enabled
!(tracing
::Level
::DEBUG
) {
172 debug_used_expressions
.enable();
175 ////////////////////////////////////////////////////
176 // Compute `CoverageSpan`s from the `CoverageGraph`.
177 let coverage_spans
= CoverageSpans
::generate_coverage_spans(
181 &self.basic_coverage_blocks
,
185 debug
::dump_coverage_spanview(
188 &self.basic_coverage_blocks
,
195 ////////////////////////////////////////////////////
196 // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure
197 // every `CoverageSpan` has a `Counter` or `Expression` assigned to its `BasicCoverageBlock`
198 // and all `Expression` dependencies (operands) are also generated, for any other
199 // `BasicCoverageBlock`s not already associated with a `CoverageSpan`.
201 // Intermediate expressions (used to compute other `Expression` values), which have no
202 // direct associate to any `BasicCoverageBlock`, are returned in the method `Result`.
203 let intermediate_expressions_or_error
= self
205 .make_bcb_counters(&mut self.basic_coverage_blocks
, &coverage_spans
);
207 let (result
, intermediate_expressions
) = match intermediate_expressions_or_error
{
208 Ok(intermediate_expressions
) => {
209 // If debugging, add any intermediate expressions (which are not associated with any
210 // BCB) to the `debug_used_expressions` map.
211 if debug_used_expressions
.is_enabled() {
212 for intermediate_expression
in &intermediate_expressions
{
213 debug_used_expressions
.add_expression_operands(intermediate_expression
);
217 ////////////////////////////////////////////////////
218 // Remove the counter or edge counter from of each `CoverageSpan`s associated
219 // `BasicCoverageBlock`, and inject a `Coverage` statement into the MIR.
221 // `Coverage` statements injected from `CoverageSpan`s will include the code regions
222 // (source code start and end positions) to be counted by the associated counter.
224 // These `CoverageSpan`-associated counters are removed from their associated
225 // `BasicCoverageBlock`s so that the only remaining counters in the `CoverageGraph`
226 // are indirect counters (to be injected next, without associated code regions).
227 self.inject_coverage_span_counters(
230 &mut debug_used_expressions
,
233 ////////////////////////////////////////////////////
234 // For any remaining `BasicCoverageBlock` counters (that were not associated with
235 // any `CoverageSpan`), inject `Coverage` statements (_without_ code region `Span`s)
236 // to ensure `BasicCoverageBlock` counters that other `Expression`s may depend on
237 // are in fact counted, even though they don't directly contribute to counting
238 // their own independent code region's coverage.
239 self.inject_indirect_counters(&mut graphviz_data
, &mut debug_used_expressions
);
241 // Intermediate expressions will be injected as the final step, after generating
242 // debug output, if any.
243 ////////////////////////////////////////////////////
245 (Ok(()), intermediate_expressions
)
247 Err(e
) => (Err(e
), Vec
::new()),
250 if graphviz_data
.is_enabled() {
251 // Even if there was an error, a partial CoverageGraph can still generate a useful
253 debug
::dump_coverage_graphviz(
257 &self.basic_coverage_blocks
,
258 &self.coverage_counters
.debug_counters
,
260 &intermediate_expressions
,
261 &debug_used_expressions
,
265 if let Err(e
) = result
{
266 bug
!("Error processing: {:?}: {:?}", self.mir_body
.source
.def_id(), e
.message
)
269 // Depending on current `debug_options()`, `alert_on_unused_expressions()` could panic, so
270 // this check is performed as late as possible, to allow other debug output (logs and dump
271 // files), which might be helpful in analyzing unused expressions, to still be generated.
272 debug_used_expressions
.alert_on_unused_expressions(&self.coverage_counters
.debug_counters
);
274 ////////////////////////////////////////////////////
275 // Finally, inject the intermediate expressions collected along the way.
276 for intermediate_expression
in intermediate_expressions
{
277 inject_intermediate_expression(self.mir_body
, intermediate_expression
);
281 /// Inject a counter for each `CoverageSpan`. There can be multiple `CoverageSpan`s for a given
282 /// BCB, but only one actual counter needs to be incremented per BCB. `bb_counters` maps each
283 /// `bcb` to its `Counter`, when injected. Subsequent `CoverageSpan`s for a BCB that already has
284 /// a `Counter` will inject an `Expression` instead, and compute its value by adding `ZERO` to
285 /// the BCB `Counter` value.
287 /// If debugging, add every BCB `Expression` associated with a `CoverageSpan`s to the
288 /// `used_expression_operands` map.
289 fn inject_coverage_span_counters(
291 coverage_spans
: Vec
<CoverageSpan
>,
292 graphviz_data
: &mut debug
::GraphvizData
,
293 debug_used_expressions
: &mut debug
::UsedExpressions
,
296 let source_map
= tcx
.sess
.source_map();
297 let body_span
= self.body_span
;
298 let file_name
= Symbol
::intern(&self.source_file
.name
.prefer_remapped().to_string_lossy());
300 let mut bcb_counters
= IndexVec
::from_elem_n(None
, self.basic_coverage_blocks
.num_nodes());
301 for covspan
in coverage_spans
{
302 let bcb
= covspan
.bcb
;
303 let span
= covspan
.span
;
304 let counter_kind
= if let Some(&counter_operand
) = bcb_counters
[bcb
].as_ref() {
305 self.coverage_counters
.make_identity_counter(counter_operand
)
306 } else if let Some(counter_kind
) = self.bcb_data_mut(bcb
).take_counter() {
307 bcb_counters
[bcb
] = Some(counter_kind
.as_operand_id());
308 debug_used_expressions
.add_expression_operands(&counter_kind
);
311 bug
!("Every BasicCoverageBlock should have a Counter or Expression");
313 graphviz_data
.add_bcb_coverage_span_with_counter(bcb
, &covspan
, &counter_kind
);
316 "Calling make_code_region(file_name={}, source_file={:?}, span={}, body_span={})",
319 source_map
.span_to_diagnostic_string(span
),
320 source_map
.span_to_diagnostic_string(body_span
)
326 self.bcb_leader_bb(bcb
),
327 Some(make_code_region(source_map
, file_name
, &self.source_file
, span
, body_span
)),
332 /// `inject_coverage_span_counters()` looped through the `CoverageSpan`s and injected the
333 /// counter from the `CoverageSpan`s `BasicCoverageBlock`, removing it from the BCB in the
334 /// process (via `take_counter()`).
336 /// Any other counter associated with a `BasicCoverageBlock`, or its incoming edge, but not
337 /// associated with a `CoverageSpan`, should only exist if the counter is an `Expression`
338 /// dependency (one of the expression operands). Collect them, and inject the additional
339 /// counters into the MIR, without a reportable coverage span.
340 fn inject_indirect_counters(
342 graphviz_data
: &mut debug
::GraphvizData
,
343 debug_used_expressions
: &mut debug
::UsedExpressions
,
345 let mut bcb_counters_without_direct_coverage_spans
= Vec
::new();
346 for (target_bcb
, target_bcb_data
) in self.basic_coverage_blocks
.iter_enumerated_mut() {
347 if let Some(counter_kind
) = target_bcb_data
.take_counter() {
348 bcb_counters_without_direct_coverage_spans
.push((None
, target_bcb
, counter_kind
));
350 if let Some(edge_counters
) = target_bcb_data
.take_edge_counters() {
351 for (from_bcb
, counter_kind
) in edge_counters
{
352 bcb_counters_without_direct_coverage_spans
.push((
361 // If debug is enabled, validate that every BCB or edge counter not directly associated
362 // with a coverage span is at least indirectly associated (it is a dependency of a BCB
363 // counter that _is_ associated with a coverage span).
364 debug_used_expressions
.validate(&bcb_counters_without_direct_coverage_spans
);
366 for (edge_from_bcb
, target_bcb
, counter_kind
) in bcb_counters_without_direct_coverage_spans
368 debug_used_expressions
.add_unused_expression_if_not_found(
375 CoverageKind
::Counter { .. }
=> {
376 let inject_to_bb
= if let Some(from_bcb
) = edge_from_bcb
{
377 // The MIR edge starts `from_bb` (the outgoing / last BasicBlock in
378 // `from_bcb`) and ends at `to_bb` (the incoming / first BasicBlock in the
379 // `target_bcb`; also called the `leader_bb`).
380 let from_bb
= self.bcb_last_bb(from_bcb
);
381 let to_bb
= self.bcb_leader_bb(target_bcb
);
383 let new_bb
= inject_edge_counter_basic_block(self.mir_body
, from_bb
, to_bb
);
384 graphviz_data
.set_edge_counter(from_bcb
, new_bb
, &counter_kind
);
386 "Edge {:?} (last {:?}) -> {:?} (leader {:?}) requires a new MIR \
387 BasicBlock {:?}, for unclaimed edge counter {}",
393 self.format_counter(&counter_kind
),
397 let target_bb
= self.bcb_last_bb(target_bcb
);
398 graphviz_data
.add_bcb_dependency_counter(target_bcb
, &counter_kind
);
400 "{:?} ({:?}) gets a new Coverage statement for unclaimed counter {}",
403 self.format_counter(&counter_kind
),
408 inject_statement(self.mir_body
, counter_kind
, inject_to_bb
, None
);
410 CoverageKind
::Expression { .. }
=> {
411 inject_intermediate_expression(self.mir_body
, counter_kind
)
413 _
=> bug
!("CoverageKind should be a counter"),
419 fn bcb_leader_bb(&self, bcb
: BasicCoverageBlock
) -> BasicBlock
{
420 self.bcb_data(bcb
).leader_bb()
424 fn bcb_last_bb(&self, bcb
: BasicCoverageBlock
) -> BasicBlock
{
425 self.bcb_data(bcb
).last_bb()
429 fn bcb_data(&self, bcb
: BasicCoverageBlock
) -> &BasicCoverageBlockData
{
430 &self.basic_coverage_blocks
[bcb
]
434 fn bcb_data_mut(&mut self, bcb
: BasicCoverageBlock
) -> &mut BasicCoverageBlockData
{
435 &mut self.basic_coverage_blocks
[bcb
]
439 fn format_counter(&self, counter_kind
: &CoverageKind
) -> String
{
440 self.coverage_counters
.debug_counters
.format_counter(counter_kind
)
444 fn inject_edge_counter_basic_block(
445 mir_body
: &mut mir
::Body
<'_
>,
449 let span
= mir_body
[from_bb
].terminator().source_info
.span
.shrink_to_hi();
450 let new_bb
= mir_body
.basic_blocks_mut().push(BasicBlockData
{
451 statements
: vec
![], // counter will be injected here
452 terminator
: Some(Terminator
{
453 source_info
: SourceInfo
::outermost(span
),
454 kind
: TerminatorKind
::Goto { target: to_bb }
,
458 let edge_ref
= mir_body
[from_bb
]
461 .find(|successor
| **successor
== to_bb
)
462 .expect("from_bb should have a successor for to_bb");
468 mir_body
: &mut mir
::Body
<'_
>,
469 counter_kind
: CoverageKind
,
471 some_code_region
: Option
<CodeRegion
>,
474 " injecting statement {:?} for {:?} at code region: {:?}",
475 counter_kind
, bb
, some_code_region
477 let data
= &mut mir_body
[bb
];
478 let source_info
= data
.terminator().source_info
;
479 let statement
= Statement
{
481 kind
: StatementKind
::Coverage(Box
::new(Coverage
{
483 code_region
: some_code_region
,
486 data
.statements
.insert(0, statement
);
489 // Non-code expressions are injected into the coverage map, without generating executable code.
490 fn inject_intermediate_expression(mir_body
: &mut mir
::Body
<'_
>, expression
: CoverageKind
) {
491 debug_assert
!(matches
!(expression
, CoverageKind
::Expression { .. }
));
492 debug
!(" injecting non-code expression {:?}", expression
);
493 let inject_in_bb
= mir
::START_BLOCK
;
494 let data
= &mut mir_body
[inject_in_bb
];
495 let source_info
= data
.terminator().source_info
;
496 let statement
= Statement
{
498 kind
: StatementKind
::Coverage(Box
::new(Coverage { kind: expression, code_region: None }
)),
500 data
.statements
.push(statement
);
503 /// Convert the Span into its file name, start line and column, and end line and column
505 source_map
: &SourceMap
,
507 source_file
: &Lrc
<SourceFile
>,
511 let (start_line
, mut start_col
) = source_file
.lookup_file_pos(span
.lo());
512 let (end_line
, end_col
) = if span
.hi() == span
.lo() {
513 let (end_line
, mut end_col
) = (start_line
, start_col
);
514 // Extend an empty span by one character so the region will be counted.
515 let CharPos(char_pos
) = start_col
;
516 if span
.hi() == body_span
.hi() {
517 start_col
= CharPos(char_pos
- 1);
519 end_col
= CharPos(char_pos
+ 1);
523 source_file
.lookup_file_pos(span
.hi())
525 let start_line
= source_map
.doctest_offset_line(&source_file
.name
, start_line
);
526 let end_line
= source_map
.doctest_offset_line(&source_file
.name
, end_line
);
529 start_line
: start_line
as u32,
530 start_col
: start_col
.to_u32() + 1,
531 end_line
: end_line
as u32,
532 end_col
: end_col
.to_u32() + 1,
539 ) -> (Option
<&rustc_hir
::FnSig
<'_
>>, &rustc_hir
::Body
<'_
>) {
540 // FIXME(#79625): Consider improving MIR to provide the information needed, to avoid going back
542 let hir_node
= tcx
.hir().get_if_local(def_id
).expect("expected DefId is local");
543 let fn_body_id
= hir
::map
::associated_body(hir_node
).expect("HIR node is a function with body");
544 (hir_node
.fn_sig(), tcx
.hir().body(fn_body_id
))
547 fn get_body_span
<'tcx
>(
549 hir_body
: &rustc_hir
::Body
<'tcx
>,
550 mir_body
: &mut mir
::Body
<'tcx
>,
552 let mut body_span
= hir_body
.value
.span
;
553 let def_id
= mir_body
.source
.def_id();
555 if tcx
.is_closure(def_id
) {
556 // If the MIR function is a closure, and if the closure body span
557 // starts from a macro, but it's content is not in that macro, try
558 // to find a non-macro callsite, and instrument the spans there
561 let expn_data
= body_span
.ctxt().outer_expn_data();
562 if expn_data
.is_root() {
565 if let ExpnKind
::Macro { .. }
= expn_data
.kind
{
566 body_span
= expn_data
.call_site
;
576 fn hash_mir_source
<'tcx
>(tcx
: TyCtxt
<'tcx
>, hir_body
: &'tcx rustc_hir
::Body
<'tcx
>) -> u64 {
577 // FIXME(cjgillot) Stop hashing HIR manually here.
578 let owner
= hir_body
.id().hir_id
.owner
;
579 tcx
.hir_owner_nodes(owner
).unwrap().hash_including_bodies
.to_smaller_hash()