]>
Commit | Line | Data |
---|---|---|
dfeec247 | 1 | use crate::{shim, util}; |
f9f354fc XL |
2 | use required_consts::RequiredConstsVisitor; |
3 | use rustc_data_structures::fx::FxHashSet; | |
dfeec247 | 4 | use rustc_hir as hir; |
f9f354fc | 5 | use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; |
dfeec247 XL |
6 | use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; |
7 | use rustc_index::vec::IndexVec; | |
f9f354fc XL |
8 | use rustc_middle::mir::visit::Visitor as _; |
9 | use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPhase, Promoted}; | |
ba9703b0 XL |
10 | use rustc_middle::ty::query::Providers; |
11 | use rustc_middle::ty::steal::Steal; | |
29967ef6 | 12 | use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; |
f9f354fc | 13 | use rustc_span::{Span, Symbol}; |
abe05a73 | 14 | use std::borrow::Cow; |
7cac9316 | 15 | |
dfeec247 | 16 | pub mod add_call_guards; |
ff7c6d11 | 17 | pub mod add_moves_for_packed_drops; |
dfeec247 | 18 | pub mod add_retag; |
1b1a35ee | 19 | pub mod check_const_item_mutation; |
e74abb32 | 20 | pub mod check_consts; |
f9f354fc | 21 | pub mod check_packed_ref; |
ea8adc8c | 22 | pub mod check_unsafety; |
dfeec247 XL |
23 | pub mod cleanup_post_borrowck; |
24 | pub mod const_prop; | |
29967ef6 | 25 | pub mod coverage; |
dfeec247 | 26 | pub mod deaggregator; |
1b1a35ee | 27 | pub mod dest_prop; |
dfeec247 | 28 | pub mod dump_mir; |
1b1a35ee | 29 | pub mod early_otherwise_branch; |
dfeec247 | 30 | pub mod elaborate_drops; |
29967ef6 | 31 | pub mod function_item_references; |
dfeec247 XL |
32 | pub mod generator; |
33 | pub mod inline; | |
34 | pub mod instcombine; | |
3dfed10e | 35 | pub mod match_branches; |
1b1a35ee | 36 | pub mod multiple_return_terminators; |
7453a54e | 37 | pub mod no_landing_pads; |
f9f354fc | 38 | pub mod nrvo; |
a7813a04 | 39 | pub mod promote_consts; |
ff7c6d11 | 40 | pub mod remove_noop_landing_pads; |
1b1a35ee | 41 | pub mod remove_unneeded_drops; |
f9f354fc | 42 | pub mod required_consts; |
dfeec247 XL |
43 | pub mod rustc_peek; |
44 | pub mod simplify; | |
45 | pub mod simplify_branches; | |
1b1a35ee | 46 | pub mod simplify_comparison_integral; |
dfeec247 | 47 | pub mod simplify_try; |
60c5eb7d | 48 | pub mod uninhabited_enum_branching; |
dfeec247 | 49 | pub mod unreachable_prop; |
f9f354fc | 50 | pub mod validate; |
7cac9316 | 51 | |
29967ef6 XL |
52 | pub use rustc_middle::mir::MirSource; |
53 | ||
f035d41b | 54 | pub(crate) fn provide(providers: &mut Providers) { |
ea8adc8c | 55 | self::check_unsafety::provide(providers); |
7cac9316 XL |
56 | *providers = Providers { |
57 | mir_keys, | |
58 | mir_const, | |
3dfed10e XL |
59 | mir_const_qualif: |tcx, def_id| { |
60 | let def_id = def_id.expect_local(); | |
61 | if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { | |
62 | tcx.mir_const_qualif_const_arg(def) | |
63 | } else { | |
64 | mir_const_qualif(tcx, ty::WithOptConstParam::unknown(def_id)) | |
65 | } | |
66 | }, | |
67 | mir_const_qualif_const_arg: |tcx, (did, param_did)| { | |
68 | mir_const_qualif(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) | |
69 | }, | |
70 | mir_promoted, | |
f035d41b | 71 | mir_drops_elaborated_and_const_checked, |
7cac9316 | 72 | optimized_mir, |
3dfed10e | 73 | optimized_mir_of_const_arg, |
7cac9316 | 74 | is_mir_available, |
3dfed10e XL |
75 | promoted_mir: |tcx, def_id| { |
76 | let def_id = def_id.expect_local(); | |
77 | if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { | |
78 | tcx.promoted_mir_of_const_arg(def) | |
79 | } else { | |
80 | promoted_mir(tcx, ty::WithOptConstParam::unknown(def_id)) | |
81 | } | |
82 | }, | |
83 | promoted_mir_of_const_arg: |tcx, (did, param_did)| { | |
84 | promoted_mir(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) | |
85 | }, | |
7cac9316 XL |
86 | ..*providers |
87 | }; | |
29967ef6 | 88 | coverage::query::provide(providers); |
7cac9316 XL |
89 | } |
90 | ||
416331ca | 91 | fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { |
f9f354fc | 92 | tcx.mir_keys(def_id.krate).contains(&def_id.expect_local()) |
7cac9316 XL |
93 | } |
94 | ||
9fa01778 | 95 | /// Finds the full set of `DefId`s within the current crate that have |
7cac9316 | 96 | /// MIR associated with them. |
f9f354fc | 97 | fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> FxHashSet<LocalDefId> { |
7cac9316 XL |
98 | assert_eq!(krate, LOCAL_CRATE); |
99 | ||
f9f354fc | 100 | let mut set = FxHashSet::default(); |
7cac9316 XL |
101 | |
102 | // All body-owners have MIR associated with them. | |
f9f354fc | 103 | set.extend(tcx.body_owners()); |
7cac9316 XL |
104 | |
105 | // Additionally, tuple struct/variant constructors have MIR, but | |
106 | // they don't have a BodyId, so we need to build them separately. | |
dc9dc135 XL |
107 | struct GatherCtors<'a, 'tcx> { |
108 | tcx: TyCtxt<'tcx>, | |
f9f354fc | 109 | set: &'a mut FxHashSet<LocalDefId>, |
7cac9316 XL |
110 | } |
111 | impl<'a, 'tcx> Visitor<'tcx> for GatherCtors<'a, 'tcx> { | |
dfeec247 XL |
112 | fn visit_variant_data( |
113 | &mut self, | |
114 | v: &'tcx hir::VariantData<'tcx>, | |
f9f354fc | 115 | _: Symbol, |
dfeec247 XL |
116 | _: &'tcx hir::Generics<'tcx>, |
117 | _: hir::HirId, | |
118 | _: Span, | |
119 | ) { | |
532ac7d7 | 120 | if let hir::VariantData::Tuple(_, hir_id) = *v { |
416331ca | 121 | self.set.insert(self.tcx.hir().local_def_id(hir_id)); |
7cac9316 XL |
122 | } |
123 | intravisit::walk_struct_def(self, v) | |
124 | } | |
ba9703b0 XL |
125 | type Map = intravisit::ErasedMap<'tcx>; |
126 | fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> { | |
7cac9316 XL |
127 | NestedVisitorMap::None |
128 | } | |
129 | } | |
dfeec247 XL |
130 | tcx.hir() |
131 | .krate() | |
132 | .visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }.as_deep_visitor()); | |
7cac9316 | 133 | |
f9f354fc | 134 | set |
7cac9316 XL |
135 | } |
136 | ||
abe05a73 XL |
137 | /// Generates a default name for the pass based on the name of the |
138 | /// type `T`. | |
139 | pub fn default_name<T: ?Sized>() -> Cow<'static, str> { | |
29967ef6 | 140 | let name = std::any::type_name::<T>(); |
74b04a01 | 141 | if let Some(tail) = name.rfind(':') { Cow::from(&name[tail + 1..]) } else { Cow::from(name) } |
abe05a73 XL |
142 | } |
143 | ||
144 | /// A streamlined trait that you can implement to create a pass; the | |
145 | /// pass will be named after the type, and it will consist of a main | |
146 | /// loop that goes over each available MIR and applies `run_pass`. | |
e1599b0c | 147 | pub trait MirPass<'tcx> { |
416331ca | 148 | fn name(&self) -> Cow<'_, str> { |
abe05a73 XL |
149 | default_name::<Self>() |
150 | } | |
151 | ||
29967ef6 | 152 | fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>); |
abe05a73 XL |
153 | } |
154 | ||
a1dfa0c6 | 155 | pub fn run_passes( |
dc9dc135 | 156 | tcx: TyCtxt<'tcx>, |
f9f354fc | 157 | body: &mut Body<'tcx>, |
a1dfa0c6 | 158 | mir_phase: MirPhase, |
f9f354fc | 159 | passes: &[&[&dyn MirPass<'tcx>]], |
a1dfa0c6 XL |
160 | ) { |
161 | let phase_index = mir_phase.phase_index(); | |
f9f354fc | 162 | let validate = tcx.sess.opts.debugging_opts.validate_mir; |
a1dfa0c6 | 163 | |
e1599b0c XL |
164 | if body.phase >= mir_phase { |
165 | return; | |
166 | } | |
a1dfa0c6 | 167 | |
f9f354fc | 168 | if validate { |
3dfed10e | 169 | validate::Validator { when: format!("input to phase {:?}", mir_phase), mir_phase } |
29967ef6 | 170 | .run_pass(tcx, body); |
f9f354fc XL |
171 | } |
172 | ||
e1599b0c XL |
173 | let mut index = 0; |
174 | let mut run_pass = |pass: &dyn MirPass<'tcx>| { | |
175 | let run_hooks = |body: &_, index, is_after| { | |
dfeec247 XL |
176 | dump_mir::on_mir_pass( |
177 | tcx, | |
178 | &format_args!("{:03}-{:03}", phase_index, index), | |
179 | &pass.name(), | |
dfeec247 XL |
180 | body, |
181 | is_after, | |
182 | ); | |
abe05a73 | 183 | }; |
e1599b0c | 184 | run_hooks(body, index, false); |
29967ef6 | 185 | pass.run_pass(tcx, body); |
e1599b0c | 186 | run_hooks(body, index, true); |
a1dfa0c6 | 187 | |
f9f354fc | 188 | if validate { |
3dfed10e XL |
189 | validate::Validator { |
190 | when: format!("after {} in phase {:?}", pass.name(), mir_phase), | |
191 | mir_phase, | |
192 | } | |
29967ef6 | 193 | .run_pass(tcx, body); |
f9f354fc XL |
194 | } |
195 | ||
e1599b0c | 196 | index += 1; |
abe05a73 XL |
197 | }; |
198 | ||
f9f354fc XL |
199 | for pass_group in passes { |
200 | for pass in *pass_group { | |
201 | run_pass(*pass); | |
202 | } | |
abe05a73 | 203 | } |
e1599b0c XL |
204 | |
205 | body.phase = mir_phase; | |
f9f354fc | 206 | |
3dfed10e XL |
207 | if mir_phase == MirPhase::Optimization { |
208 | validate::Validator { when: format!("end of phase {:?}", mir_phase), mir_phase } | |
29967ef6 | 209 | .run_pass(tcx, body); |
f9f354fc | 210 | } |
a1dfa0c6 | 211 | } |
abe05a73 | 212 | |
3dfed10e XL |
213 | fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> ConstQualifs { |
214 | let const_kind = tcx.hir().body_const_context(def.did); | |
60c5eb7d XL |
215 | |
216 | // No need to const-check a non-const `fn`. | |
217 | if const_kind.is_none() { | |
218 | return Default::default(); | |
219 | } | |
220 | ||
221 | // N.B., this `borrow()` is guaranteed to be valid (i.e., the value | |
3dfed10e | 222 | // cannot yet be stolen), because `mir_promoted()`, which steals |
60c5eb7d XL |
223 | // from `mir_const(), forces this query to execute before |
224 | // performing the steal. | |
3dfed10e | 225 | let body = &tcx.mir_const(def).borrow(); |
60c5eb7d XL |
226 | |
227 | if body.return_ty().references_error() { | |
228 | tcx.sess.delay_span_bug(body.span, "mir_const_qualif: MIR had errors"); | |
229 | return Default::default(); | |
230 | } | |
231 | ||
29967ef6 | 232 | let ccx = check_consts::ConstCx { body, tcx, const_kind, param_env: tcx.param_env(def.did) }; |
60c5eb7d | 233 | |
f9f354fc | 234 | let mut validator = check_consts::validation::Validator::new(&ccx); |
60c5eb7d XL |
235 | validator.check_body(); |
236 | ||
237 | // We return the qualifs in the return place for every MIR body, even though it is only used | |
238 | // when deciding to promote a reference to a `const` for now. | |
74b04a01 | 239 | validator.qualifs_in_return_place() |
60c5eb7d XL |
240 | } |
241 | ||
f9f354fc | 242 | /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts! |
3dfed10e XL |
243 | fn mir_const<'tcx>( |
244 | tcx: TyCtxt<'tcx>, | |
245 | def: ty::WithOptConstParam<LocalDefId>, | |
246 | ) -> &'tcx Steal<Body<'tcx>> { | |
247 | if let Some(def) = def.try_upgrade(tcx) { | |
248 | return tcx.mir_const(def); | |
249 | } | |
f9f354fc XL |
250 | |
251 | // Unsafety check uses the raw mir, so make sure it is run. | |
3dfed10e XL |
252 | if let Some(param_did) = def.const_param_did { |
253 | tcx.ensure().unsafety_check_result_for_const_arg((def.did, param_did)); | |
254 | } else { | |
255 | tcx.ensure().unsafety_check_result(def.did); | |
256 | } | |
ea8adc8c | 257 | |
3dfed10e | 258 | let mut body = tcx.mir_built(def).steal(); |
dfeec247 | 259 | |
29967ef6 | 260 | util::dump_mir(tcx, None, "mir_map", &0, &body, |_, _| Ok(())); |
dfeec247 XL |
261 | |
262 | run_passes( | |
263 | tcx, | |
264 | &mut body, | |
dfeec247 | 265 | MirPhase::Const, |
f9f354fc XL |
266 | &[&[ |
267 | // MIR-level lints. | |
268 | &check_packed_ref::CheckPackedRef, | |
1b1a35ee | 269 | &check_const_item_mutation::CheckConstItemMutation, |
29967ef6 | 270 | &function_item_references::FunctionItemReferences, |
dfeec247 XL |
271 | // What we need to do constant evaluation. |
272 | &simplify::SimplifyCfg::new("initial"), | |
273 | &rustc_peek::SanityCheck, | |
f9f354fc | 274 | ]], |
dfeec247 | 275 | ); |
dc9dc135 | 276 | tcx.alloc_steal_mir(body) |
7cac9316 XL |
277 | } |
278 | ||
3dfed10e | 279 | fn mir_promoted( |
e1599b0c | 280 | tcx: TyCtxt<'tcx>, |
3dfed10e XL |
281 | def: ty::WithOptConstParam<LocalDefId>, |
282 | ) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) { | |
283 | if let Some(def) = def.try_upgrade(tcx) { | |
284 | return tcx.mir_promoted(def); | |
285 | } | |
286 | ||
60c5eb7d XL |
287 | // Ensure that we compute the `mir_const_qualif` for constants at |
288 | // this point, before we steal the mir-const result. | |
3dfed10e XL |
289 | // Also this means promotion can rely on all const checks having been done. |
290 | let _ = tcx.mir_const_qualif_opt_const_arg(def); | |
29967ef6 | 291 | let _ = tcx.mir_abstract_const_opt_const_arg(def.to_global()); |
3dfed10e | 292 | let mut body = tcx.mir_const(def).steal(); |
f9f354fc XL |
293 | |
294 | let mut required_consts = Vec::new(); | |
295 | let mut required_consts_visitor = RequiredConstsVisitor::new(&mut required_consts); | |
296 | for (bb, bb_data) in traversal::reverse_postorder(&body) { | |
297 | required_consts_visitor.visit_basic_block_data(bb, bb_data); | |
298 | } | |
299 | body.required_consts = required_consts; | |
7cac9316 | 300 | |
60c5eb7d | 301 | let promote_pass = promote_consts::PromoteTemps::default(); |
3dfed10e XL |
302 | let promote: &[&dyn MirPass<'tcx>] = &[ |
303 | // What we need to run borrowck etc. | |
304 | &promote_pass, | |
305 | &simplify::SimplifyCfg::new("promote-consts"), | |
306 | ]; | |
307 | ||
308 | let opt_coverage: &[&dyn MirPass<'tcx>] = if tcx.sess.opts.debugging_opts.instrument_coverage { | |
29967ef6 | 309 | &[&coverage::InstrumentCoverage] |
3dfed10e XL |
310 | } else { |
311 | &[] | |
312 | }; | |
313 | ||
29967ef6 | 314 | run_passes(tcx, &mut body, MirPhase::ConstPromotion, &[promote, opt_coverage]); |
60c5eb7d XL |
315 | |
316 | let promoted = promote_pass.promoted_fragments.into_inner(); | |
e1599b0c | 317 | (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted)) |
7cac9316 XL |
318 | } |
319 | ||
f035d41b XL |
320 | fn mir_drops_elaborated_and_const_checked<'tcx>( |
321 | tcx: TyCtxt<'tcx>, | |
3dfed10e XL |
322 | def: ty::WithOptConstParam<LocalDefId>, |
323 | ) -> &'tcx Steal<Body<'tcx>> { | |
324 | if let Some(def) = def.try_upgrade(tcx) { | |
325 | return tcx.mir_drops_elaborated_and_const_checked(def); | |
326 | } | |
327 | ||
328 | // (Mir-)Borrowck uses `mir_promoted`, so we have to force it to | |
f035d41b | 329 | // execute before we can steal. |
3dfed10e XL |
330 | if let Some(param_did) = def.const_param_did { |
331 | tcx.ensure().mir_borrowck_const_arg((def.did, param_did)); | |
332 | } else { | |
333 | tcx.ensure().mir_borrowck(def.did); | |
334 | } | |
f035d41b | 335 | |
3dfed10e | 336 | let (body, _) = tcx.mir_promoted(def); |
f035d41b XL |
337 | let mut body = body.steal(); |
338 | ||
29967ef6 XL |
339 | run_post_borrowck_cleanup_passes(tcx, &mut body); |
340 | check_consts::post_drop_elaboration::check_live_drops(tcx, &body); | |
f035d41b XL |
341 | tcx.alloc_steal_mir(body) |
342 | } | |
343 | ||
344 | /// After this series of passes, no lifetime analysis based on borrowing can be done. | |
29967ef6 XL |
345 | fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
346 | debug!("post_borrowck_cleanup({:?})", body.source.def_id()); | |
f035d41b | 347 | |
f9f354fc XL |
348 | let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ |
349 | // Remove all things only needed by analysis | |
350 | &no_landing_pads::NoLandingPads::new(tcx), | |
351 | &simplify_branches::SimplifyBranches::new("initial"), | |
352 | &remove_noop_landing_pads::RemoveNoopLandingPads, | |
353 | &cleanup_post_borrowck::CleanupNonCodegenStatements, | |
354 | &simplify::SimplifyCfg::new("early-opt"), | |
355 | // These next passes must be executed together | |
356 | &add_call_guards::CriticalCallEdges, | |
357 | &elaborate_drops::ElaborateDrops, | |
358 | &no_landing_pads::NoLandingPads::new(tcx), | |
359 | // AddMovesForPackedDrops needs to run after drop | |
360 | // elaboration. | |
361 | &add_moves_for_packed_drops::AddMovesForPackedDrops, | |
362 | // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late, | |
363 | // but before optimizations begin. | |
364 | &add_retag::AddRetag, | |
365 | &simplify::SimplifyCfg::new("elaborate-drops"), | |
3dfed10e XL |
366 | // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening |
367 | // and it can help optimizations. | |
368 | &deaggregator::Deaggregator, | |
f9f354fc XL |
369 | ]; |
370 | ||
29967ef6 | 371 | run_passes(tcx, body, MirPhase::DropLowering, &[post_borrowck_cleanup]); |
f035d41b XL |
372 | } |
373 | ||
29967ef6 | 374 | fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
3dfed10e XL |
375 | let mir_opt_level = tcx.sess.opts.debugging_opts.mir_opt_level; |
376 | ||
377 | // Lowering generator control-flow and variables has to happen before we do anything else | |
378 | // to them. We run some optimizations before that, because they may be harder to do on the state | |
379 | // machine than on MIR with async primitives. | |
380 | let optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[ | |
f9f354fc XL |
381 | &unreachable_prop::UnreachablePropagation, |
382 | &uninhabited_enum_branching::UninhabitedEnumBranching, | |
383 | &simplify::SimplifyCfg::new("after-uninhabited-enum-branching"), | |
384 | &inline::Inline, | |
f9f354fc | 385 | &generator::StateTransform, |
3dfed10e XL |
386 | ]; |
387 | ||
388 | // Even if we don't do optimizations, we still have to lower generators for codegen. | |
389 | let no_optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[&generator::StateTransform]; | |
390 | ||
391 | // The main optimizations that we do on MIR. | |
392 | let optimizations: &[&dyn MirPass<'tcx>] = &[ | |
1b1a35ee | 393 | &remove_unneeded_drops::RemoveUnneededDrops, |
3dfed10e | 394 | &match_branches::MatchBranchSimplification, |
1b1a35ee XL |
395 | // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) |
396 | &multiple_return_terminators::MultipleReturnTerminators, | |
397 | &instcombine::InstCombine, | |
f9f354fc XL |
398 | &const_prop::ConstProp, |
399 | &simplify_branches::SimplifyBranches::new("after-const-prop"), | |
1b1a35ee XL |
400 | &early_otherwise_branch::EarlyOtherwiseBranch, |
401 | &simplify_comparison_integral::SimplifyComparisonIntegral, | |
f9f354fc XL |
402 | &simplify_try::SimplifyArmIdentity, |
403 | &simplify_try::SimplifyBranchSame, | |
1b1a35ee | 404 | &dest_prop::DestinationPropagation, |
29967ef6 | 405 | &simplify_branches::SimplifyBranches::new("final"), |
f9f354fc | 406 | &remove_noop_landing_pads::RemoveNoopLandingPads, |
f9f354fc XL |
407 | &simplify::SimplifyCfg::new("final"), |
408 | &nrvo::RenameReturnPlace, | |
409 | &simplify::SimplifyLocals, | |
1b1a35ee | 410 | &multiple_return_terminators::MultipleReturnTerminators, |
f9f354fc XL |
411 | ]; |
412 | ||
3dfed10e | 413 | // Optimizations to run even if mir optimizations have been disabled. |
f9f354fc | 414 | let no_optimizations: &[&dyn MirPass<'tcx>] = &[ |
f9f354fc XL |
415 | // FIXME(#70073): This pass is responsible for both optimization as well as some lints. |
416 | &const_prop::ConstProp, | |
f9f354fc XL |
417 | ]; |
418 | ||
3dfed10e | 419 | // Some cleanup necessary at least for LLVM and potentially other codegen backends. |
f9f354fc XL |
420 | let pre_codegen_cleanup: &[&dyn MirPass<'tcx>] = &[ |
421 | &add_call_guards::CriticalCallEdges, | |
422 | // Dump the end result for testing and debugging purposes. | |
423 | &dump_mir::Marker("PreCodegen"), | |
424 | ]; | |
425 | ||
3dfed10e XL |
426 | // End of pass declarations, now actually run the passes. |
427 | // Generator Lowering | |
428 | #[rustfmt::skip] | |
429 | run_passes( | |
430 | tcx, | |
431 | body, | |
3dfed10e XL |
432 | MirPhase::GeneratorLowering, |
433 | &[ | |
434 | if mir_opt_level > 0 { | |
435 | optimizations_with_generators | |
436 | } else { | |
437 | no_optimizations_with_generators | |
438 | } | |
439 | ], | |
440 | ); | |
f9f354fc | 441 | |
3dfed10e | 442 | // Main optimization passes |
f035d41b | 443 | #[rustfmt::skip] |
dfeec247 XL |
444 | run_passes( |
445 | tcx, | |
446 | body, | |
3dfed10e | 447 | MirPhase::Optimization, |
dfeec247 | 448 | &[ |
f9f354fc XL |
449 | if mir_opt_level > 0 { optimizations } else { no_optimizations }, |
450 | pre_codegen_cleanup, | |
dfeec247 XL |
451 | ], |
452 | ); | |
e1599b0c XL |
453 | } |
454 | ||
3dfed10e XL |
455 | fn optimized_mir<'tcx>(tcx: TyCtxt<'tcx>, did: DefId) -> &'tcx Body<'tcx> { |
456 | let did = did.expect_local(); | |
457 | if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) { | |
458 | tcx.optimized_mir_of_const_arg(def) | |
459 | } else { | |
460 | tcx.arena.alloc(inner_optimized_mir(tcx, ty::WithOptConstParam::unknown(did))) | |
461 | } | |
462 | } | |
463 | ||
464 | fn optimized_mir_of_const_arg<'tcx>( | |
465 | tcx: TyCtxt<'tcx>, | |
466 | (did, param_did): (LocalDefId, DefId), | |
467 | ) -> &'tcx Body<'tcx> { | |
468 | tcx.arena.alloc(inner_optimized_mir( | |
469 | tcx, | |
470 | ty::WithOptConstParam { did, const_param_did: Some(param_did) }, | |
471 | )) | |
472 | } | |
473 | ||
474 | fn inner_optimized_mir(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> { | |
475 | if tcx.is_constructor(def.did.to_def_id()) { | |
e1599b0c XL |
476 | // There's no reason to run all of the MIR passes on constructors when |
477 | // we can just output the MIR we want directly. This also saves const | |
478 | // qualification and borrow checking the trouble of special casing | |
479 | // constructors. | |
3dfed10e | 480 | return shim::build_adt_ctor(tcx, def.did.to_def_id()); |
e1599b0c XL |
481 | } |
482 | ||
3dfed10e | 483 | let mut body = tcx.mir_drops_elaborated_and_const_checked(def).steal(); |
29967ef6 | 484 | run_optimization_passes(tcx, &mut body); |
ba9703b0 XL |
485 | |
486 | debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); | |
487 | ||
f9f354fc | 488 | body |
7cac9316 | 489 | } |
416331ca | 490 | |
3dfed10e XL |
491 | fn promoted_mir<'tcx>( |
492 | tcx: TyCtxt<'tcx>, | |
493 | def: ty::WithOptConstParam<LocalDefId>, | |
494 | ) -> &'tcx IndexVec<Promoted, Body<'tcx>> { | |
495 | if tcx.is_constructor(def.did.to_def_id()) { | |
496 | return tcx.arena.alloc(IndexVec::new()); | |
e1599b0c XL |
497 | } |
498 | ||
3dfed10e XL |
499 | if let Some(param_did) = def.const_param_did { |
500 | tcx.ensure().mir_borrowck_const_arg((def.did, param_did)); | |
501 | } else { | |
502 | tcx.ensure().mir_borrowck(def.did); | |
503 | } | |
504 | let (_, promoted) = tcx.mir_promoted(def); | |
e1599b0c XL |
505 | let mut promoted = promoted.steal(); |
506 | ||
29967ef6 XL |
507 | for body in &mut promoted { |
508 | run_post_borrowck_cleanup_passes(tcx, body); | |
509 | run_optimization_passes(tcx, body); | |
e1599b0c XL |
510 | } |
511 | ||
ba9703b0 XL |
512 | debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); |
513 | ||
3dfed10e | 514 | tcx.arena.alloc(promoted) |
416331ca | 515 | } |