]>
Commit | Line | Data |
---|---|---|
dfeec247 | 1 | use crate::{shim, util}; |
f9f354fc XL |
2 | use required_consts::RequiredConstsVisitor; |
3 | use rustc_data_structures::fx::FxHashSet; | |
dfeec247 | 4 | use rustc_hir as hir; |
f9f354fc | 5 | use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; |
dfeec247 XL |
6 | use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; |
7 | use rustc_index::vec::IndexVec; | |
f9f354fc XL |
8 | use rustc_middle::mir::visit::Visitor as _; |
9 | use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPhase, Promoted}; | |
ba9703b0 XL |
10 | use rustc_middle::ty::query::Providers; |
11 | use rustc_middle::ty::steal::Steal; | |
3dfed10e | 12 | use rustc_middle::ty::{self, InstanceDef, TyCtxt, TypeFoldable}; |
f9f354fc | 13 | use rustc_span::{Span, Symbol}; |
abe05a73 | 14 | use std::borrow::Cow; |
7cac9316 | 15 | |
dfeec247 | 16 | pub mod add_call_guards; |
ff7c6d11 | 17 | pub mod add_moves_for_packed_drops; |
dfeec247 | 18 | pub mod add_retag; |
e74abb32 | 19 | pub mod check_consts; |
f9f354fc | 20 | pub mod check_packed_ref; |
ea8adc8c | 21 | pub mod check_unsafety; |
dfeec247 XL |
22 | pub mod cleanup_post_borrowck; |
23 | pub mod const_prop; | |
24 | pub mod copy_prop; | |
25 | pub mod deaggregator; | |
26 | pub mod dump_mir; | |
27 | pub mod elaborate_drops; | |
dfeec247 XL |
28 | pub mod generator; |
29 | pub mod inline; | |
30 | pub mod instcombine; | |
f035d41b | 31 | pub mod instrument_coverage; |
3dfed10e | 32 | pub mod match_branches; |
7453a54e | 33 | pub mod no_landing_pads; |
f9f354fc | 34 | pub mod nrvo; |
a7813a04 | 35 | pub mod promote_consts; |
0731742a | 36 | pub mod qualify_min_const_fn; |
ff7c6d11 | 37 | pub mod remove_noop_landing_pads; |
f9f354fc | 38 | pub mod required_consts; |
dfeec247 XL |
39 | pub mod rustc_peek; |
40 | pub mod simplify; | |
41 | pub mod simplify_branches; | |
42 | pub mod simplify_try; | |
60c5eb7d | 43 | pub mod uninhabited_enum_branching; |
dfeec247 | 44 | pub mod unreachable_prop; |
f9f354fc | 45 | pub mod validate; |
7cac9316 | 46 | |
f035d41b | 47 | pub(crate) fn provide(providers: &mut Providers) { |
ea8adc8c | 48 | self::check_unsafety::provide(providers); |
7cac9316 XL |
49 | *providers = Providers { |
50 | mir_keys, | |
51 | mir_const, | |
3dfed10e XL |
52 | mir_const_qualif: |tcx, def_id| { |
53 | let def_id = def_id.expect_local(); | |
54 | if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { | |
55 | tcx.mir_const_qualif_const_arg(def) | |
56 | } else { | |
57 | mir_const_qualif(tcx, ty::WithOptConstParam::unknown(def_id)) | |
58 | } | |
59 | }, | |
60 | mir_const_qualif_const_arg: |tcx, (did, param_did)| { | |
61 | mir_const_qualif(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) | |
62 | }, | |
63 | mir_promoted, | |
f035d41b | 64 | mir_drops_elaborated_and_const_checked, |
7cac9316 | 65 | optimized_mir, |
3dfed10e | 66 | optimized_mir_of_const_arg, |
7cac9316 | 67 | is_mir_available, |
3dfed10e XL |
68 | promoted_mir: |tcx, def_id| { |
69 | let def_id = def_id.expect_local(); | |
70 | if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { | |
71 | tcx.promoted_mir_of_const_arg(def) | |
72 | } else { | |
73 | promoted_mir(tcx, ty::WithOptConstParam::unknown(def_id)) | |
74 | } | |
75 | }, | |
76 | promoted_mir_of_const_arg: |tcx, (did, param_did)| { | |
77 | promoted_mir(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) | |
78 | }, | |
7cac9316 XL |
79 | ..*providers |
80 | }; | |
f035d41b | 81 | instrument_coverage::provide(providers); |
7cac9316 XL |
82 | } |
83 | ||
416331ca | 84 | fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { |
f9f354fc | 85 | tcx.mir_keys(def_id.krate).contains(&def_id.expect_local()) |
7cac9316 XL |
86 | } |
87 | ||
9fa01778 | 88 | /// Finds the full set of `DefId`s within the current crate that have |
7cac9316 | 89 | /// MIR associated with them. |
f9f354fc | 90 | fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> FxHashSet<LocalDefId> { |
7cac9316 XL |
91 | assert_eq!(krate, LOCAL_CRATE); |
92 | ||
f9f354fc | 93 | let mut set = FxHashSet::default(); |
7cac9316 XL |
94 | |
95 | // All body-owners have MIR associated with them. | |
f9f354fc | 96 | set.extend(tcx.body_owners()); |
7cac9316 XL |
97 | |
98 | // Additionally, tuple struct/variant constructors have MIR, but | |
99 | // they don't have a BodyId, so we need to build them separately. | |
dc9dc135 XL |
100 | struct GatherCtors<'a, 'tcx> { |
101 | tcx: TyCtxt<'tcx>, | |
f9f354fc | 102 | set: &'a mut FxHashSet<LocalDefId>, |
7cac9316 XL |
103 | } |
104 | impl<'a, 'tcx> Visitor<'tcx> for GatherCtors<'a, 'tcx> { | |
dfeec247 XL |
105 | fn visit_variant_data( |
106 | &mut self, | |
107 | v: &'tcx hir::VariantData<'tcx>, | |
f9f354fc | 108 | _: Symbol, |
dfeec247 XL |
109 | _: &'tcx hir::Generics<'tcx>, |
110 | _: hir::HirId, | |
111 | _: Span, | |
112 | ) { | |
532ac7d7 | 113 | if let hir::VariantData::Tuple(_, hir_id) = *v { |
416331ca | 114 | self.set.insert(self.tcx.hir().local_def_id(hir_id)); |
7cac9316 XL |
115 | } |
116 | intravisit::walk_struct_def(self, v) | |
117 | } | |
ba9703b0 XL |
118 | type Map = intravisit::ErasedMap<'tcx>; |
119 | fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> { | |
7cac9316 XL |
120 | NestedVisitorMap::None |
121 | } | |
122 | } | |
dfeec247 XL |
123 | tcx.hir() |
124 | .krate() | |
125 | .visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }.as_deep_visitor()); | |
7cac9316 | 126 | |
f9f354fc | 127 | set |
7cac9316 XL |
128 | } |
129 | ||
dc9dc135 | 130 | /// Where a specific `mir::Body` comes from. |
abe05a73 | 131 | #[derive(Debug, Copy, Clone)] |
9fa01778 XL |
132 | pub struct MirSource<'tcx> { |
133 | pub instance: InstanceDef<'tcx>, | |
abe05a73 XL |
134 | |
135 | /// If `Some`, this is a promoted rvalue within the parent function. | |
136 | pub promoted: Option<Promoted>, | |
137 | } | |
138 | ||
9fa01778 | 139 | impl<'tcx> MirSource<'tcx> { |
abe05a73 | 140 | pub fn item(def_id: DefId) -> Self { |
3dfed10e XL |
141 | MirSource { |
142 | instance: InstanceDef::Item(ty::WithOptConstParam::unknown(def_id)), | |
143 | promoted: None, | |
144 | } | |
145 | } | |
146 | ||
147 | pub fn with_opt_param(self) -> ty::WithOptConstParam<DefId> { | |
148 | self.instance.with_opt_param() | |
abe05a73 | 149 | } |
9fa01778 XL |
150 | |
151 | #[inline] | |
152 | pub fn def_id(&self) -> DefId { | |
153 | self.instance.def_id() | |
154 | } | |
abe05a73 XL |
155 | } |
156 | ||
157 | /// Generates a default name for the pass based on the name of the | |
158 | /// type `T`. | |
159 | pub fn default_name<T: ?Sized>() -> Cow<'static, str> { | |
416331ca | 160 | let name = ::std::any::type_name::<T>(); |
74b04a01 | 161 | if let Some(tail) = name.rfind(':') { Cow::from(&name[tail + 1..]) } else { Cow::from(name) } |
abe05a73 XL |
162 | } |
163 | ||
164 | /// A streamlined trait that you can implement to create a pass; the | |
165 | /// pass will be named after the type, and it will consist of a main | |
166 | /// loop that goes over each available MIR and applies `run_pass`. | |
e1599b0c | 167 | pub trait MirPass<'tcx> { |
416331ca | 168 | fn name(&self) -> Cow<'_, str> { |
abe05a73 XL |
169 | default_name::<Self>() |
170 | } | |
171 | ||
f9f354fc | 172 | fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut Body<'tcx>); |
abe05a73 XL |
173 | } |
174 | ||
a1dfa0c6 | 175 | pub fn run_passes( |
dc9dc135 | 176 | tcx: TyCtxt<'tcx>, |
f9f354fc | 177 | body: &mut Body<'tcx>, |
9fa01778 | 178 | instance: InstanceDef<'tcx>, |
e1599b0c | 179 | promoted: Option<Promoted>, |
a1dfa0c6 | 180 | mir_phase: MirPhase, |
f9f354fc | 181 | passes: &[&[&dyn MirPass<'tcx>]], |
a1dfa0c6 XL |
182 | ) { |
183 | let phase_index = mir_phase.phase_index(); | |
f9f354fc XL |
184 | let source = MirSource { instance, promoted }; |
185 | let validate = tcx.sess.opts.debugging_opts.validate_mir; | |
a1dfa0c6 | 186 | |
e1599b0c XL |
187 | if body.phase >= mir_phase { |
188 | return; | |
189 | } | |
a1dfa0c6 | 190 | |
f9f354fc | 191 | if validate { |
3dfed10e | 192 | validate::Validator { when: format!("input to phase {:?}", mir_phase), mir_phase } |
f9f354fc XL |
193 | .run_pass(tcx, source, body); |
194 | } | |
195 | ||
e1599b0c XL |
196 | let mut index = 0; |
197 | let mut run_pass = |pass: &dyn MirPass<'tcx>| { | |
198 | let run_hooks = |body: &_, index, is_after| { | |
dfeec247 XL |
199 | dump_mir::on_mir_pass( |
200 | tcx, | |
201 | &format_args!("{:03}-{:03}", phase_index, index), | |
202 | &pass.name(), | |
203 | source, | |
204 | body, | |
205 | is_after, | |
206 | ); | |
abe05a73 | 207 | }; |
e1599b0c XL |
208 | run_hooks(body, index, false); |
209 | pass.run_pass(tcx, source, body); | |
210 | run_hooks(body, index, true); | |
a1dfa0c6 | 211 | |
f9f354fc | 212 | if validate { |
3dfed10e XL |
213 | validate::Validator { |
214 | when: format!("after {} in phase {:?}", pass.name(), mir_phase), | |
215 | mir_phase, | |
216 | } | |
217 | .run_pass(tcx, source, body); | |
f9f354fc XL |
218 | } |
219 | ||
e1599b0c | 220 | index += 1; |
abe05a73 XL |
221 | }; |
222 | ||
f9f354fc XL |
223 | for pass_group in passes { |
224 | for pass in *pass_group { | |
225 | run_pass(*pass); | |
226 | } | |
abe05a73 | 227 | } |
e1599b0c XL |
228 | |
229 | body.phase = mir_phase; | |
f9f354fc | 230 | |
3dfed10e XL |
231 | if mir_phase == MirPhase::Optimization { |
232 | validate::Validator { when: format!("end of phase {:?}", mir_phase), mir_phase } | |
f9f354fc XL |
233 | .run_pass(tcx, source, body); |
234 | } | |
a1dfa0c6 | 235 | } |
abe05a73 | 236 | |
3dfed10e XL |
237 | fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> ConstQualifs { |
238 | let const_kind = tcx.hir().body_const_context(def.did); | |
60c5eb7d XL |
239 | |
240 | // No need to const-check a non-const `fn`. | |
241 | if const_kind.is_none() { | |
242 | return Default::default(); | |
243 | } | |
244 | ||
245 | // N.B., this `borrow()` is guaranteed to be valid (i.e., the value | |
3dfed10e | 246 | // cannot yet be stolen), because `mir_promoted()`, which steals |
60c5eb7d XL |
247 | // from `mir_const(), forces this query to execute before |
248 | // performing the steal. | |
3dfed10e | 249 | let body = &tcx.mir_const(def).borrow(); |
60c5eb7d XL |
250 | |
251 | if body.return_ty().references_error() { | |
252 | tcx.sess.delay_span_bug(body.span, "mir_const_qualif: MIR had errors"); | |
253 | return Default::default(); | |
254 | } | |
255 | ||
3dfed10e XL |
256 | let ccx = check_consts::ConstCx { |
257 | body, | |
258 | tcx, | |
259 | def_id: def.did, | |
260 | const_kind, | |
261 | param_env: tcx.param_env(def.did), | |
262 | }; | |
60c5eb7d | 263 | |
f9f354fc | 264 | let mut validator = check_consts::validation::Validator::new(&ccx); |
60c5eb7d XL |
265 | validator.check_body(); |
266 | ||
267 | // We return the qualifs in the return place for every MIR body, even though it is only used | |
268 | // when deciding to promote a reference to a `const` for now. | |
74b04a01 | 269 | validator.qualifs_in_return_place() |
60c5eb7d XL |
270 | } |
271 | ||
f9f354fc | 272 | /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts! |
3dfed10e XL |
273 | fn mir_const<'tcx>( |
274 | tcx: TyCtxt<'tcx>, | |
275 | def: ty::WithOptConstParam<LocalDefId>, | |
276 | ) -> &'tcx Steal<Body<'tcx>> { | |
277 | if let Some(def) = def.try_upgrade(tcx) { | |
278 | return tcx.mir_const(def); | |
279 | } | |
f9f354fc XL |
280 | |
281 | // Unsafety check uses the raw mir, so make sure it is run. | |
3dfed10e XL |
282 | if let Some(param_did) = def.const_param_did { |
283 | tcx.ensure().unsafety_check_result_for_const_arg((def.did, param_did)); | |
284 | } else { | |
285 | tcx.ensure().unsafety_check_result(def.did); | |
286 | } | |
ea8adc8c | 287 | |
3dfed10e | 288 | let mut body = tcx.mir_built(def).steal(); |
dfeec247 | 289 | |
3dfed10e XL |
290 | util::dump_mir( |
291 | tcx, | |
292 | None, | |
293 | "mir_map", | |
294 | &0, | |
295 | MirSource { instance: InstanceDef::Item(def.to_global()), promoted: None }, | |
296 | &body, | |
297 | |_, _| Ok(()), | |
298 | ); | |
dfeec247 XL |
299 | |
300 | run_passes( | |
301 | tcx, | |
302 | &mut body, | |
3dfed10e | 303 | InstanceDef::Item(def.to_global()), |
dfeec247 XL |
304 | None, |
305 | MirPhase::Const, | |
f9f354fc XL |
306 | &[&[ |
307 | // MIR-level lints. | |
308 | &check_packed_ref::CheckPackedRef, | |
dfeec247 XL |
309 | // What we need to do constant evaluation. |
310 | &simplify::SimplifyCfg::new("initial"), | |
311 | &rustc_peek::SanityCheck, | |
f9f354fc | 312 | ]], |
dfeec247 | 313 | ); |
dc9dc135 | 314 | tcx.alloc_steal_mir(body) |
7cac9316 XL |
315 | } |
316 | ||
3dfed10e | 317 | fn mir_promoted( |
e1599b0c | 318 | tcx: TyCtxt<'tcx>, |
3dfed10e XL |
319 | def: ty::WithOptConstParam<LocalDefId>, |
320 | ) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) { | |
321 | if let Some(def) = def.try_upgrade(tcx) { | |
322 | return tcx.mir_promoted(def); | |
323 | } | |
324 | ||
60c5eb7d XL |
325 | // Ensure that we compute the `mir_const_qualif` for constants at |
326 | // this point, before we steal the mir-const result. | |
3dfed10e XL |
327 | // Also this means promotion can rely on all const checks having been done. |
328 | let _ = tcx.mir_const_qualif_opt_const_arg(def); | |
f9f354fc | 329 | |
3dfed10e | 330 | let mut body = tcx.mir_const(def).steal(); |
f9f354fc XL |
331 | |
332 | let mut required_consts = Vec::new(); | |
333 | let mut required_consts_visitor = RequiredConstsVisitor::new(&mut required_consts); | |
334 | for (bb, bb_data) in traversal::reverse_postorder(&body) { | |
335 | required_consts_visitor.visit_basic_block_data(bb, bb_data); | |
336 | } | |
337 | body.required_consts = required_consts; | |
7cac9316 | 338 | |
60c5eb7d | 339 | let promote_pass = promote_consts::PromoteTemps::default(); |
3dfed10e XL |
340 | let promote: &[&dyn MirPass<'tcx>] = &[ |
341 | // What we need to run borrowck etc. | |
342 | &promote_pass, | |
343 | &simplify::SimplifyCfg::new("promote-consts"), | |
344 | ]; | |
345 | ||
346 | let opt_coverage: &[&dyn MirPass<'tcx>] = if tcx.sess.opts.debugging_opts.instrument_coverage { | |
347 | &[&instrument_coverage::InstrumentCoverage] | |
348 | } else { | |
349 | &[] | |
350 | }; | |
351 | ||
dfeec247 XL |
352 | run_passes( |
353 | tcx, | |
354 | &mut body, | |
3dfed10e | 355 | InstanceDef::Item(def.to_global()), |
dfeec247 | 356 | None, |
3dfed10e XL |
357 | MirPhase::ConstPromotion, |
358 | &[promote, opt_coverage], | |
dfeec247 | 359 | ); |
60c5eb7d XL |
360 | |
361 | let promoted = promote_pass.promoted_fragments.into_inner(); | |
e1599b0c | 362 | (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted)) |
7cac9316 XL |
363 | } |
364 | ||
f035d41b XL |
365 | fn mir_drops_elaborated_and_const_checked<'tcx>( |
366 | tcx: TyCtxt<'tcx>, | |
3dfed10e XL |
367 | def: ty::WithOptConstParam<LocalDefId>, |
368 | ) -> &'tcx Steal<Body<'tcx>> { | |
369 | if let Some(def) = def.try_upgrade(tcx) { | |
370 | return tcx.mir_drops_elaborated_and_const_checked(def); | |
371 | } | |
372 | ||
373 | // (Mir-)Borrowck uses `mir_promoted`, so we have to force it to | |
f035d41b | 374 | // execute before we can steal. |
3dfed10e XL |
375 | if let Some(param_did) = def.const_param_did { |
376 | tcx.ensure().mir_borrowck_const_arg((def.did, param_did)); | |
377 | } else { | |
378 | tcx.ensure().mir_borrowck(def.did); | |
379 | } | |
f035d41b | 380 | |
3dfed10e | 381 | let (body, _) = tcx.mir_promoted(def); |
f035d41b XL |
382 | let mut body = body.steal(); |
383 | ||
3dfed10e XL |
384 | run_post_borrowck_cleanup_passes(tcx, &mut body, def.did, None); |
385 | check_consts::post_drop_elaboration::check_live_drops(tcx, def.did, &body); | |
f035d41b XL |
386 | tcx.alloc_steal_mir(body) |
387 | } | |
388 | ||
389 | /// After this series of passes, no lifetime analysis based on borrowing can be done. | |
390 | fn run_post_borrowck_cleanup_passes<'tcx>( | |
e1599b0c | 391 | tcx: TyCtxt<'tcx>, |
f9f354fc XL |
392 | body: &mut Body<'tcx>, |
393 | def_id: LocalDefId, | |
e1599b0c XL |
394 | promoted: Option<Promoted>, |
395 | ) { | |
f035d41b XL |
396 | debug!("post_borrowck_cleanup({:?})", def_id); |
397 | ||
f9f354fc XL |
398 | let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ |
399 | // Remove all things only needed by analysis | |
400 | &no_landing_pads::NoLandingPads::new(tcx), | |
401 | &simplify_branches::SimplifyBranches::new("initial"), | |
402 | &remove_noop_landing_pads::RemoveNoopLandingPads, | |
403 | &cleanup_post_borrowck::CleanupNonCodegenStatements, | |
404 | &simplify::SimplifyCfg::new("early-opt"), | |
405 | // These next passes must be executed together | |
406 | &add_call_guards::CriticalCallEdges, | |
407 | &elaborate_drops::ElaborateDrops, | |
408 | &no_landing_pads::NoLandingPads::new(tcx), | |
409 | // AddMovesForPackedDrops needs to run after drop | |
410 | // elaboration. | |
411 | &add_moves_for_packed_drops::AddMovesForPackedDrops, | |
412 | // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late, | |
413 | // but before optimizations begin. | |
414 | &add_retag::AddRetag, | |
415 | &simplify::SimplifyCfg::new("elaborate-drops"), | |
3dfed10e XL |
416 | // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening |
417 | // and it can help optimizations. | |
418 | &deaggregator::Deaggregator, | |
f9f354fc XL |
419 | ]; |
420 | ||
f035d41b XL |
421 | run_passes( |
422 | tcx, | |
423 | body, | |
3dfed10e | 424 | InstanceDef::Item(ty::WithOptConstParam::unknown(def_id.to_def_id())), |
f035d41b | 425 | promoted, |
3dfed10e | 426 | MirPhase::DropLowering, |
f035d41b XL |
427 | &[post_borrowck_cleanup], |
428 | ); | |
429 | } | |
430 | ||
431 | fn run_optimization_passes<'tcx>( | |
432 | tcx: TyCtxt<'tcx>, | |
433 | body: &mut Body<'tcx>, | |
434 | def_id: LocalDefId, | |
435 | promoted: Option<Promoted>, | |
436 | ) { | |
3dfed10e XL |
437 | let mir_opt_level = tcx.sess.opts.debugging_opts.mir_opt_level; |
438 | ||
439 | // Lowering generator control-flow and variables has to happen before we do anything else | |
440 | // to them. We run some optimizations before that, because they may be harder to do on the state | |
441 | // machine than on MIR with async primitives. | |
442 | let optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[ | |
f9f354fc XL |
443 | &unreachable_prop::UnreachablePropagation, |
444 | &uninhabited_enum_branching::UninhabitedEnumBranching, | |
445 | &simplify::SimplifyCfg::new("after-uninhabited-enum-branching"), | |
446 | &inline::Inline, | |
f9f354fc | 447 | &generator::StateTransform, |
3dfed10e XL |
448 | ]; |
449 | ||
450 | // Even if we don't do optimizations, we still have to lower generators for codegen. | |
451 | let no_optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[&generator::StateTransform]; | |
452 | ||
453 | // The main optimizations that we do on MIR. | |
454 | let optimizations: &[&dyn MirPass<'tcx>] = &[ | |
f9f354fc | 455 | &instcombine::InstCombine, |
3dfed10e | 456 | &match_branches::MatchBranchSimplification, |
f9f354fc XL |
457 | &const_prop::ConstProp, |
458 | &simplify_branches::SimplifyBranches::new("after-const-prop"), | |
f9f354fc XL |
459 | &simplify_try::SimplifyArmIdentity, |
460 | &simplify_try::SimplifyBranchSame, | |
461 | ©_prop::CopyPropagation, | |
462 | &simplify_branches::SimplifyBranches::new("after-copy-prop"), | |
463 | &remove_noop_landing_pads::RemoveNoopLandingPads, | |
464 | &simplify::SimplifyCfg::new("after-remove-noop-landing-pads"), | |
465 | &simplify::SimplifyCfg::new("final"), | |
466 | &nrvo::RenameReturnPlace, | |
467 | &simplify::SimplifyLocals, | |
468 | ]; | |
469 | ||
3dfed10e | 470 | // Optimizations to run even if mir optimizations have been disabled. |
f9f354fc | 471 | let no_optimizations: &[&dyn MirPass<'tcx>] = &[ |
f9f354fc XL |
472 | // FIXME(#70073): This pass is responsible for both optimization as well as some lints. |
473 | &const_prop::ConstProp, | |
f9f354fc XL |
474 | ]; |
475 | ||
3dfed10e | 476 | // Some cleanup necessary at least for LLVM and potentially other codegen backends. |
f9f354fc XL |
477 | let pre_codegen_cleanup: &[&dyn MirPass<'tcx>] = &[ |
478 | &add_call_guards::CriticalCallEdges, | |
479 | // Dump the end result for testing and debugging purposes. | |
480 | &dump_mir::Marker("PreCodegen"), | |
481 | ]; | |
482 | ||
3dfed10e XL |
483 | // End of pass declarations, now actually run the passes. |
484 | // Generator Lowering | |
485 | #[rustfmt::skip] | |
486 | run_passes( | |
487 | tcx, | |
488 | body, | |
489 | InstanceDef::Item(ty::WithOptConstParam::unknown(def_id.to_def_id())), | |
490 | promoted, | |
491 | MirPhase::GeneratorLowering, | |
492 | &[ | |
493 | if mir_opt_level > 0 { | |
494 | optimizations_with_generators | |
495 | } else { | |
496 | no_optimizations_with_generators | |
497 | } | |
498 | ], | |
499 | ); | |
f9f354fc | 500 | |
3dfed10e | 501 | // Main optimization passes |
f035d41b | 502 | #[rustfmt::skip] |
dfeec247 XL |
503 | run_passes( |
504 | tcx, | |
505 | body, | |
3dfed10e | 506 | InstanceDef::Item(ty::WithOptConstParam::unknown(def_id.to_def_id())), |
dfeec247 | 507 | promoted, |
3dfed10e | 508 | MirPhase::Optimization, |
dfeec247 | 509 | &[ |
f9f354fc XL |
510 | if mir_opt_level > 0 { optimizations } else { no_optimizations }, |
511 | pre_codegen_cleanup, | |
dfeec247 XL |
512 | ], |
513 | ); | |
e1599b0c XL |
514 | } |
515 | ||
3dfed10e XL |
516 | fn optimized_mir<'tcx>(tcx: TyCtxt<'tcx>, did: DefId) -> &'tcx Body<'tcx> { |
517 | let did = did.expect_local(); | |
518 | if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) { | |
519 | tcx.optimized_mir_of_const_arg(def) | |
520 | } else { | |
521 | tcx.arena.alloc(inner_optimized_mir(tcx, ty::WithOptConstParam::unknown(did))) | |
522 | } | |
523 | } | |
524 | ||
525 | fn optimized_mir_of_const_arg<'tcx>( | |
526 | tcx: TyCtxt<'tcx>, | |
527 | (did, param_did): (LocalDefId, DefId), | |
528 | ) -> &'tcx Body<'tcx> { | |
529 | tcx.arena.alloc(inner_optimized_mir( | |
530 | tcx, | |
531 | ty::WithOptConstParam { did, const_param_did: Some(param_did) }, | |
532 | )) | |
533 | } | |
534 | ||
535 | fn inner_optimized_mir(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> { | |
536 | if tcx.is_constructor(def.did.to_def_id()) { | |
e1599b0c XL |
537 | // There's no reason to run all of the MIR passes on constructors when |
538 | // we can just output the MIR we want directly. This also saves const | |
539 | // qualification and borrow checking the trouble of special casing | |
540 | // constructors. | |
3dfed10e | 541 | return shim::build_adt_ctor(tcx, def.did.to_def_id()); |
e1599b0c XL |
542 | } |
543 | ||
3dfed10e XL |
544 | let mut body = tcx.mir_drops_elaborated_and_const_checked(def).steal(); |
545 | run_optimization_passes(tcx, &mut body, def.did, None); | |
ba9703b0 XL |
546 | |
547 | debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); | |
548 | ||
f9f354fc | 549 | body |
7cac9316 | 550 | } |
416331ca | 551 | |
3dfed10e XL |
552 | fn promoted_mir<'tcx>( |
553 | tcx: TyCtxt<'tcx>, | |
554 | def: ty::WithOptConstParam<LocalDefId>, | |
555 | ) -> &'tcx IndexVec<Promoted, Body<'tcx>> { | |
556 | if tcx.is_constructor(def.did.to_def_id()) { | |
557 | return tcx.arena.alloc(IndexVec::new()); | |
e1599b0c XL |
558 | } |
559 | ||
3dfed10e XL |
560 | if let Some(param_did) = def.const_param_did { |
561 | tcx.ensure().mir_borrowck_const_arg((def.did, param_did)); | |
562 | } else { | |
563 | tcx.ensure().mir_borrowck(def.did); | |
564 | } | |
565 | let (_, promoted) = tcx.mir_promoted(def); | |
e1599b0c XL |
566 | let mut promoted = promoted.steal(); |
567 | ||
568 | for (p, mut body) in promoted.iter_enumerated_mut() { | |
3dfed10e XL |
569 | run_post_borrowck_cleanup_passes(tcx, &mut body, def.did, Some(p)); |
570 | run_optimization_passes(tcx, &mut body, def.did, Some(p)); | |
e1599b0c XL |
571 | } |
572 | ||
ba9703b0 XL |
573 | debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); |
574 | ||
3dfed10e | 575 | tcx.arena.alloc(promoted) |
416331ca | 576 | } |