]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | #![allow(rustc::potential_query_instability)] |
c295e0f8 | 2 | #![feature(box_patterns)] |
5e7ed085 | 3 | #![feature(let_chains)] |
3c0e092e | 4 | #![feature(let_else)] |
c295e0f8 XL |
5 | #![feature(map_try_insert)] |
6 | #![feature(min_specialization)] | |
c295e0f8 | 7 | #![feature(never_type)] |
5e7ed085 FG |
8 | #![feature(once_cell)] |
9 | #![feature(option_get_or_insert_default)] | |
c295e0f8 XL |
10 | #![feature(trusted_step)] |
11 | #![feature(try_blocks)] | |
923072b8 | 12 | #![feature(yeet_expr)] |
c295e0f8 XL |
13 | #![recursion_limit = "256"] |
14 | ||
15 | #[macro_use] | |
16 | extern crate tracing; | |
17 | #[macro_use] | |
18 | extern crate rustc_middle; | |
19 | ||
f9f354fc | 20 | use required_consts::RequiredConstsVisitor; |
c295e0f8 | 21 | use rustc_const_eval::util; |
5099ac24 | 22 | use rustc_data_structures::fx::FxIndexSet; |
fc512014 | 23 | use rustc_data_structures::steal::Steal; |
dfeec247 | 24 | use rustc_hir as hir; |
17df50a5 | 25 | use rustc_hir::def_id::{DefId, LocalDefId}; |
5099ac24 | 26 | use rustc_hir::intravisit::{self, Visitor}; |
dfeec247 | 27 | use rustc_index::vec::IndexVec; |
f9f354fc | 28 | use rustc_middle::mir::visit::Visitor as _; |
a2a8927a | 29 | use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted}; |
ba9703b0 | 30 | use rustc_middle::ty::query::Providers; |
29967ef6 | 31 | use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; |
f9f354fc | 32 | use rustc_span::{Span, Symbol}; |
c295e0f8 | 33 | |
a2a8927a XL |
34 | #[macro_use] |
35 | mod pass_manager; | |
36 | ||
37 | use pass_manager::{self as pm, Lint, MirLint, WithMinOptLevel}; | |
38 | ||
c295e0f8 XL |
39 | mod abort_unwinding_calls; |
40 | mod add_call_guards; | |
41 | mod add_moves_for_packed_drops; | |
42 | mod add_retag; | |
43 | mod check_const_item_mutation; | |
44 | mod check_packed_ref; | |
ea8adc8c | 45 | pub mod check_unsafety; |
5099ac24 FG |
46 | // This pass is public to allow external drivers to perform MIR cleanup |
47 | pub mod cleanup_post_borrowck; | |
c295e0f8 XL |
48 | mod const_debuginfo; |
49 | mod const_goto; | |
50 | mod const_prop; | |
5e7ed085 | 51 | mod const_prop_lint; |
c295e0f8 | 52 | mod coverage; |
923072b8 | 53 | mod dead_store_elimination; |
c295e0f8 XL |
54 | mod deaggregator; |
55 | mod deduplicate_blocks; | |
04454e1e | 56 | mod deref_separator; |
c295e0f8 | 57 | mod dest_prop; |
dfeec247 | 58 | pub mod dump_mir; |
c295e0f8 | 59 | mod early_otherwise_branch; |
923072b8 | 60 | mod elaborate_box_derefs; |
c295e0f8 XL |
61 | mod elaborate_drops; |
62 | mod function_item_references; | |
63 | mod generator; | |
64 | mod inline; | |
65 | mod instcombine; | |
66 | mod lower_intrinsics; | |
67 | mod lower_slice_len; | |
a2a8927a | 68 | mod marker; |
c295e0f8 XL |
69 | mod match_branches; |
70 | mod multiple_return_terminators; | |
71 | mod normalize_array_len; | |
72 | mod nrvo; | |
5099ac24 FG |
73 | // This pass is public to allow external drivers to perform MIR cleanup |
74 | pub mod remove_false_edges; | |
c295e0f8 XL |
75 | mod remove_noop_landing_pads; |
76 | mod remove_storage_markers; | |
a2a8927a | 77 | mod remove_uninit_drops; |
c295e0f8 XL |
78 | mod remove_unneeded_drops; |
79 | mod remove_zsts; | |
80 | mod required_consts; | |
3c0e092e | 81 | mod reveal_all; |
c295e0f8 XL |
82 | mod separate_const_switch; |
83 | mod shim; | |
5099ac24 FG |
84 | // This pass is public to allow external drivers to perform MIR cleanup |
85 | pub mod simplify; | |
c295e0f8 XL |
86 | mod simplify_branches; |
87 | mod simplify_comparison_integral; | |
88 | mod simplify_try; | |
89 | mod uninhabited_enum_branching; | |
90 | mod unreachable_prop; | |
91 | ||
a2a8927a | 92 | use rustc_const_eval::transform::check_consts::{self, ConstCx}; |
c295e0f8 XL |
93 | use rustc_const_eval::transform::promote_consts; |
94 | use rustc_const_eval::transform::validate; | |
c295e0f8 XL |
95 | use rustc_mir_dataflow::rustc_peek; |
96 | ||
97 | pub fn provide(providers: &mut Providers) { | |
98 | check_unsafety::provide(providers); | |
99 | check_packed_ref::provide(providers); | |
100 | coverage::query::provide(providers); | |
101 | shim::provide(providers); | |
7cac9316 XL |
102 | *providers = Providers { |
103 | mir_keys, | |
104 | mir_const, | |
3dfed10e XL |
105 | mir_const_qualif: |tcx, def_id| { |
106 | let def_id = def_id.expect_local(); | |
107 | if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { | |
108 | tcx.mir_const_qualif_const_arg(def) | |
109 | } else { | |
110 | mir_const_qualif(tcx, ty::WithOptConstParam::unknown(def_id)) | |
111 | } | |
112 | }, | |
113 | mir_const_qualif_const_arg: |tcx, (did, param_did)| { | |
114 | mir_const_qualif(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) | |
115 | }, | |
116 | mir_promoted, | |
f035d41b | 117 | mir_drops_elaborated_and_const_checked, |
5869c6ff XL |
118 | mir_for_ctfe, |
119 | mir_for_ctfe_of_const_arg, | |
7cac9316 XL |
120 | optimized_mir, |
121 | is_mir_available, | |
5869c6ff | 122 | is_ctfe_mir_available: |tcx, did| is_mir_available(tcx, did), |
c295e0f8 XL |
123 | mir_callgraph_reachable: inline::cycle::mir_callgraph_reachable, |
124 | mir_inliner_callees: inline::cycle::mir_inliner_callees, | |
3dfed10e XL |
125 | promoted_mir: |tcx, def_id| { |
126 | let def_id = def_id.expect_local(); | |
127 | if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { | |
128 | tcx.promoted_mir_of_const_arg(def) | |
129 | } else { | |
130 | promoted_mir(tcx, ty::WithOptConstParam::unknown(def_id)) | |
131 | } | |
132 | }, | |
133 | promoted_mir_of_const_arg: |tcx, (did, param_did)| { | |
134 | promoted_mir(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) | |
135 | }, | |
7cac9316 XL |
136 | ..*providers |
137 | }; | |
138 | } | |
139 | ||
416331ca | 140 | fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { |
17df50a5 XL |
141 | let def_id = def_id.expect_local(); |
142 | tcx.mir_keys(()).contains(&def_id) | |
7cac9316 XL |
143 | } |
144 | ||
9fa01778 | 145 | /// Finds the full set of `DefId`s within the current crate that have |
7cac9316 | 146 | /// MIR associated with them. |
5099ac24 FG |
147 | fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { |
148 | let mut set = FxIndexSet::default(); | |
7cac9316 XL |
149 | |
150 | // All body-owners have MIR associated with them. | |
c295e0f8 | 151 | set.extend(tcx.hir().body_owners()); |
7cac9316 XL |
152 | |
153 | // Additionally, tuple struct/variant constructors have MIR, but | |
154 | // they don't have a BodyId, so we need to build them separately. | |
dc9dc135 XL |
155 | struct GatherCtors<'a, 'tcx> { |
156 | tcx: TyCtxt<'tcx>, | |
5099ac24 | 157 | set: &'a mut FxIndexSet<LocalDefId>, |
7cac9316 | 158 | } |
a2a8927a | 159 | impl<'tcx> Visitor<'tcx> for GatherCtors<'_, 'tcx> { |
dfeec247 XL |
160 | fn visit_variant_data( |
161 | &mut self, | |
162 | v: &'tcx hir::VariantData<'tcx>, | |
f9f354fc | 163 | _: Symbol, |
dfeec247 XL |
164 | _: &'tcx hir::Generics<'tcx>, |
165 | _: hir::HirId, | |
166 | _: Span, | |
167 | ) { | |
532ac7d7 | 168 | if let hir::VariantData::Tuple(_, hir_id) = *v { |
416331ca | 169 | self.set.insert(self.tcx.hir().local_def_id(hir_id)); |
7cac9316 XL |
170 | } |
171 | intravisit::walk_struct_def(self, v) | |
172 | } | |
7cac9316 | 173 | } |
923072b8 | 174 | tcx.hir().deep_visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }); |
7cac9316 | 175 | |
f9f354fc | 176 | set |
7cac9316 XL |
177 | } |
178 | ||
3dfed10e XL |
179 | fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> ConstQualifs { |
180 | let const_kind = tcx.hir().body_const_context(def.did); | |
60c5eb7d XL |
181 | |
182 | // No need to const-check a non-const `fn`. | |
183 | if const_kind.is_none() { | |
184 | return Default::default(); | |
185 | } | |
186 | ||
187 | // N.B., this `borrow()` is guaranteed to be valid (i.e., the value | |
3dfed10e | 188 | // cannot yet be stolen), because `mir_promoted()`, which steals |
60c5eb7d XL |
189 | // from `mir_const(), forces this query to execute before |
190 | // performing the steal. | |
3dfed10e | 191 | let body = &tcx.mir_const(def).borrow(); |
60c5eb7d XL |
192 | |
193 | if body.return_ty().references_error() { | |
194 | tcx.sess.delay_span_bug(body.span, "mir_const_qualif: MIR had errors"); | |
195 | return Default::default(); | |
196 | } | |
197 | ||
29967ef6 | 198 | let ccx = check_consts::ConstCx { body, tcx, const_kind, param_env: tcx.param_env(def.did) }; |
60c5eb7d | 199 | |
94222f64 | 200 | let mut validator = check_consts::check::Checker::new(&ccx); |
60c5eb7d XL |
201 | validator.check_body(); |
202 | ||
203 | // We return the qualifs in the return place for every MIR body, even though it is only used | |
204 | // when deciding to promote a reference to a `const` for now. | |
74b04a01 | 205 | validator.qualifs_in_return_place() |
60c5eb7d XL |
206 | } |
207 | ||
f9f354fc | 208 | /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts! |
3dfed10e XL |
209 | fn mir_const<'tcx>( |
210 | tcx: TyCtxt<'tcx>, | |
211 | def: ty::WithOptConstParam<LocalDefId>, | |
212 | ) -> &'tcx Steal<Body<'tcx>> { | |
213 | if let Some(def) = def.try_upgrade(tcx) { | |
214 | return tcx.mir_const(def); | |
215 | } | |
f9f354fc XL |
216 | |
217 | // Unsafety check uses the raw mir, so make sure it is run. | |
94222f64 XL |
218 | if !tcx.sess.opts.debugging_opts.thir_unsafeck { |
219 | if let Some(param_did) = def.const_param_did { | |
220 | tcx.ensure().unsafety_check_result_for_const_arg((def.did, param_did)); | |
221 | } else { | |
222 | tcx.ensure().unsafety_check_result(def.did); | |
223 | } | |
3dfed10e | 224 | } |
ea8adc8c | 225 | |
3dfed10e | 226 | let mut body = tcx.mir_built(def).steal(); |
dfeec247 | 227 | |
c295e0f8 | 228 | rustc_middle::mir::dump_mir(tcx, None, "mir_map", &0, &body, |_, _| Ok(())); |
dfeec247 | 229 | |
a2a8927a | 230 | pm::run_passes( |
dfeec247 XL |
231 | tcx, |
232 | &mut body, | |
a2a8927a | 233 | &[ |
f9f354fc | 234 | // MIR-level lints. |
a2a8927a XL |
235 | &Lint(check_packed_ref::CheckPackedRef), |
236 | &Lint(check_const_item_mutation::CheckConstItemMutation), | |
237 | &Lint(function_item_references::FunctionItemReferences), | |
dfeec247 XL |
238 | // What we need to do constant evaluation. |
239 | &simplify::SimplifyCfg::new("initial"), | |
a2a8927a XL |
240 | &rustc_peek::SanityCheck, // Just a lint |
241 | &marker::PhaseChange(MirPhase::Const), | |
242 | ], | |
dfeec247 | 243 | ); |
dc9dc135 | 244 | tcx.alloc_steal_mir(body) |
7cac9316 XL |
245 | } |
246 | ||
5869c6ff | 247 | /// Compute the main MIR body and the list of MIR bodies of the promoteds. |
a2a8927a | 248 | fn mir_promoted<'tcx>( |
e1599b0c | 249 | tcx: TyCtxt<'tcx>, |
3dfed10e XL |
250 | def: ty::WithOptConstParam<LocalDefId>, |
251 | ) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) { | |
252 | if let Some(def) = def.try_upgrade(tcx) { | |
253 | return tcx.mir_promoted(def); | |
254 | } | |
255 | ||
60c5eb7d XL |
256 | // Ensure that we compute the `mir_const_qualif` for constants at |
257 | // this point, before we steal the mir-const result. | |
3dfed10e | 258 | // Also this means promotion can rely on all const checks having been done. |
5099ac24 | 259 | let const_qualifs = tcx.mir_const_qualif_opt_const_arg(def); |
3dfed10e | 260 | let mut body = tcx.mir_const(def).steal(); |
5099ac24 FG |
261 | if let Some(error_reported) = const_qualifs.tainted_by_errors { |
262 | body.tainted_by_errors = Some(error_reported); | |
263 | } | |
f9f354fc XL |
264 | |
265 | let mut required_consts = Vec::new(); | |
266 | let mut required_consts_visitor = RequiredConstsVisitor::new(&mut required_consts); | |
267 | for (bb, bb_data) in traversal::reverse_postorder(&body) { | |
268 | required_consts_visitor.visit_basic_block_data(bb, bb_data); | |
269 | } | |
270 | body.required_consts = required_consts; | |
7cac9316 | 271 | |
a2a8927a | 272 | // What we need to run borrowck etc. |
60c5eb7d | 273 | let promote_pass = promote_consts::PromoteTemps::default(); |
a2a8927a XL |
274 | pm::run_passes( |
275 | tcx, | |
276 | &mut body, | |
277 | &[ | |
278 | &promote_pass, | |
279 | &simplify::SimplifyCfg::new("promote-consts"), | |
280 | &coverage::InstrumentCoverage, | |
281 | ], | |
282 | ); | |
60c5eb7d XL |
283 | |
284 | let promoted = promote_pass.promoted_fragments.into_inner(); | |
e1599b0c | 285 | (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted)) |
7cac9316 XL |
286 | } |
287 | ||
5869c6ff XL |
288 | /// Compute the MIR that is used during CTFE (and thus has no optimizations run on it) |
289 | fn mir_for_ctfe<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx Body<'tcx> { | |
290 | let did = def_id.expect_local(); | |
291 | if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) { | |
292 | tcx.mir_for_ctfe_of_const_arg(def) | |
293 | } else { | |
294 | tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(did))) | |
295 | } | |
296 | } | |
297 | ||
298 | /// Same as `mir_for_ctfe`, but used to get the MIR of a const generic parameter. | |
299 | /// The docs on `WithOptConstParam` explain this a bit more, but the TLDR is that | |
300 | /// we'd get cycle errors with `mir_for_ctfe`, because typeck would need to typeck | |
301 | /// the const parameter while type checking the main body, which in turn would try | |
302 | /// to type check the main body again. | |
303 | fn mir_for_ctfe_of_const_arg<'tcx>( | |
304 | tcx: TyCtxt<'tcx>, | |
305 | (did, param_did): (LocalDefId, DefId), | |
306 | ) -> &'tcx Body<'tcx> { | |
307 | tcx.arena.alloc(inner_mir_for_ctfe( | |
308 | tcx, | |
309 | ty::WithOptConstParam { did, const_param_did: Some(param_did) }, | |
310 | )) | |
311 | } | |
312 | ||
313 | fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> { | |
314 | // FIXME: don't duplicate this between the optimized_mir/mir_for_ctfe queries | |
315 | if tcx.is_constructor(def.did.to_def_id()) { | |
316 | // There's no reason to run all of the MIR passes on constructors when | |
317 | // we can just output the MIR we want directly. This also saves const | |
318 | // qualification and borrow checking the trouble of special casing | |
319 | // constructors. | |
320 | return shim::build_adt_ctor(tcx, def.did.to_def_id()); | |
321 | } | |
322 | ||
323 | let context = tcx | |
324 | .hir() | |
325 | .body_const_context(def.did) | |
326 | .expect("mir_for_ctfe should not be used for runtime functions"); | |
327 | ||
328 | let mut body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); | |
329 | ||
330 | match context { | |
331 | // Do not const prop functions, either they get executed at runtime or exported to metadata, | |
332 | // so we run const prop on them, or they don't, in which case we const evaluate some control | |
333 | // flow paths of the function and any errors in those paths will get emitted as const eval | |
334 | // errors. | |
335 | hir::ConstContext::ConstFn => {} | |
336 | // Static items always get evaluated, so we can just let const eval see if any erroneous | |
337 | // control flow paths get executed. | |
338 | hir::ConstContext::Static(_) => {} | |
339 | // Associated constants get const prop run so we detect common failure situations in the | |
340 | // crate that defined the constant. | |
341 | // Technically we want to not run on regular const items, but oli-obk doesn't know how to | |
342 | // conveniently detect that at this point without looking at the HIR. | |
343 | hir::ConstContext::Const => { | |
a2a8927a | 344 | pm::run_passes( |
5869c6ff XL |
345 | tcx, |
346 | &mut body, | |
5e7ed085 | 347 | &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimized)], |
5869c6ff XL |
348 | ); |
349 | } | |
350 | } | |
351 | ||
5099ac24 | 352 | debug_assert!(!body.has_free_regions(), "Free regions in MIR for CTFE"); |
5869c6ff XL |
353 | |
354 | body | |
355 | } | |
356 | ||
357 | /// Obtain just the main MIR (no promoteds) and run some cleanups on it. This also runs | |
358 | /// mir borrowck *before* doing so in order to ensure that borrowck can be run and doesn't | |
359 | /// end up missing the source MIR due to stealing happening. | |
f035d41b XL |
360 | fn mir_drops_elaborated_and_const_checked<'tcx>( |
361 | tcx: TyCtxt<'tcx>, | |
3dfed10e XL |
362 | def: ty::WithOptConstParam<LocalDefId>, |
363 | ) -> &'tcx Steal<Body<'tcx>> { | |
364 | if let Some(def) = def.try_upgrade(tcx) { | |
365 | return tcx.mir_drops_elaborated_and_const_checked(def); | |
366 | } | |
367 | ||
5099ac24 | 368 | let mir_borrowck = tcx.mir_borrowck_opt_const_arg(def); |
f035d41b | 369 | |
04454e1e | 370 | let is_fn_like = tcx.def_kind(def.did).is_fn_like(); |
5869c6ff XL |
371 | if is_fn_like { |
372 | let did = def.did.to_def_id(); | |
373 | let def = ty::WithOptConstParam::unknown(did); | |
374 | ||
375 | // Do not compute the mir call graph without said call graph actually being used. | |
a2a8927a | 376 | if inline::Inline.is_enabled(&tcx.sess) { |
5869c6ff XL |
377 | let _ = tcx.mir_inliner_callees(ty::InstanceDef::Item(def)); |
378 | } | |
379 | } | |
380 | ||
3dfed10e | 381 | let (body, _) = tcx.mir_promoted(def); |
f035d41b | 382 | let mut body = body.steal(); |
5099ac24 FG |
383 | if let Some(error_reported) = mir_borrowck.tainted_by_errors { |
384 | body.tainted_by_errors = Some(error_reported); | |
385 | } | |
f035d41b | 386 | |
a2a8927a XL |
387 | // IMPORTANT |
388 | pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]); | |
389 | ||
390 | // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled. | |
391 | if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) { | |
392 | pm::run_passes( | |
393 | tcx, | |
394 | &mut body, | |
395 | &[ | |
396 | &simplify::SimplifyCfg::new("remove-false-edges"), | |
397 | &remove_uninit_drops::RemoveUninitDrops, | |
398 | ], | |
399 | ); | |
400 | check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint | |
401 | } | |
402 | ||
29967ef6 | 403 | run_post_borrowck_cleanup_passes(tcx, &mut body); |
5e7ed085 | 404 | assert!(body.phase == MirPhase::Deaggregated); |
f035d41b XL |
405 | tcx.alloc_steal_mir(body) |
406 | } | |
407 | ||
408 | /// After this series of passes, no lifetime analysis based on borrowing can be done. | |
29967ef6 XL |
409 | fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
410 | debug!("post_borrowck_cleanup({:?})", body.source.def_id()); | |
f035d41b | 411 | |
f9f354fc XL |
412 | let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ |
413 | // Remove all things only needed by analysis | |
a2a8927a | 414 | &simplify_branches::SimplifyConstCondition::new("initial"), |
f9f354fc XL |
415 | &remove_noop_landing_pads::RemoveNoopLandingPads, |
416 | &cleanup_post_borrowck::CleanupNonCodegenStatements, | |
417 | &simplify::SimplifyCfg::new("early-opt"), | |
418 | // These next passes must be executed together | |
419 | &add_call_guards::CriticalCallEdges, | |
420 | &elaborate_drops::ElaborateDrops, | |
94222f64 XL |
421 | // This will remove extraneous landing pads which are no longer |
422 | // necessary as well as well as forcing any call in a non-unwinding | |
423 | // function calling a possibly-unwinding function to abort the process. | |
424 | &abort_unwinding_calls::AbortUnwindingCalls, | |
f9f354fc XL |
425 | // AddMovesForPackedDrops needs to run after drop |
426 | // elaboration. | |
427 | &add_moves_for_packed_drops::AddMovesForPackedDrops, | |
428 | // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late, | |
429 | // but before optimizations begin. | |
04454e1e | 430 | &deref_separator::Derefer, |
923072b8 | 431 | &elaborate_box_derefs::ElaborateBoxDerefs, |
f9f354fc | 432 | &add_retag::AddRetag, |
fc512014 | 433 | &lower_intrinsics::LowerIntrinsics, |
f9f354fc | 434 | &simplify::SimplifyCfg::new("elaborate-drops"), |
3dfed10e XL |
435 | // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening |
436 | // and it can help optimizations. | |
437 | &deaggregator::Deaggregator, | |
5e7ed085 | 438 | &Lint(const_prop_lint::ConstProp), |
f9f354fc XL |
439 | ]; |
440 | ||
a2a8927a | 441 | pm::run_passes(tcx, body, post_borrowck_cleanup); |
f035d41b XL |
442 | } |
443 | ||
29967ef6 | 444 | fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
a2a8927a XL |
445 | fn o1<T>(x: T) -> WithMinOptLevel<T> { |
446 | WithMinOptLevel(1, x) | |
447 | } | |
3dfed10e XL |
448 | |
449 | // Lowering generator control-flow and variables has to happen before we do anything else | |
450 | // to them. We run some optimizations before that, because they may be harder to do on the state | |
451 | // machine than on MIR with async primitives. | |
a2a8927a | 452 | pm::run_passes( |
3dfed10e XL |
453 | tcx, |
454 | body, | |
3dfed10e | 455 | &[ |
a2a8927a XL |
456 | &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. |
457 | &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first | |
458 | &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering | |
459 | &unreachable_prop::UnreachablePropagation, | |
460 | &uninhabited_enum_branching::UninhabitedEnumBranching, | |
461 | &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")), | |
462 | &inline::Inline, | |
463 | &generator::StateTransform, | |
3dfed10e XL |
464 | ], |
465 | ); | |
f9f354fc | 466 | |
5e7ed085 | 467 | assert!(body.phase == MirPhase::GeneratorsLowered); |
a2a8927a XL |
468 | |
469 | // The main optimizations that we do on MIR. | |
470 | pm::run_passes( | |
dfeec247 XL |
471 | tcx, |
472 | body, | |
dfeec247 | 473 | &[ |
a2a8927a XL |
474 | &remove_storage_markers::RemoveStorageMarkers, |
475 | &remove_zsts::RemoveZsts, | |
476 | &const_goto::ConstGoto, | |
477 | &remove_unneeded_drops::RemoveUnneededDrops, | |
478 | &match_branches::MatchBranchSimplification, | |
479 | // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) | |
480 | &multiple_return_terminators::MultipleReturnTerminators, | |
481 | &instcombine::InstCombine, | |
482 | &separate_const_switch::SeparateConstSwitch, | |
483 | // | |
484 | // FIXME(#70073): This pass is responsible for both optimization as well as some lints. | |
485 | &const_prop::ConstProp, | |
486 | // | |
487 | // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0. | |
923072b8 | 488 | &const_debuginfo::ConstDebugInfo, |
a2a8927a XL |
489 | &o1(simplify_branches::SimplifyConstCondition::new("after-const-prop")), |
490 | &early_otherwise_branch::EarlyOtherwiseBranch, | |
491 | &simplify_comparison_integral::SimplifyComparisonIntegral, | |
492 | &simplify_try::SimplifyArmIdentity, | |
493 | &simplify_try::SimplifyBranchSame, | |
923072b8 | 494 | &dead_store_elimination::DeadStoreElimination, |
a2a8927a XL |
495 | &dest_prop::DestinationPropagation, |
496 | &o1(simplify_branches::SimplifyConstCondition::new("final")), | |
497 | &o1(remove_noop_landing_pads::RemoveNoopLandingPads), | |
498 | &o1(simplify::SimplifyCfg::new("final")), | |
499 | &nrvo::RenameReturnPlace, | |
a2a8927a XL |
500 | &simplify::SimplifyLocals, |
501 | &multiple_return_terminators::MultipleReturnTerminators, | |
502 | &deduplicate_blocks::DeduplicateBlocks, | |
503 | // Some cleanup necessary at least for LLVM and potentially other codegen backends. | |
504 | &add_call_guards::CriticalCallEdges, | |
5e7ed085 | 505 | &marker::PhaseChange(MirPhase::Optimized), |
a2a8927a XL |
506 | // Dump the end result for testing and debugging purposes. |
507 | &dump_mir::Marker("PreCodegen"), | |
dfeec247 XL |
508 | ], |
509 | ); | |
e1599b0c XL |
510 | } |
511 | ||
5869c6ff | 512 | /// Optimize the MIR and prepare it for codegen. |
3dfed10e XL |
513 | fn optimized_mir<'tcx>(tcx: TyCtxt<'tcx>, did: DefId) -> &'tcx Body<'tcx> { |
514 | let did = did.expect_local(); | |
5869c6ff XL |
515 | assert_eq!(ty::WithOptConstParam::try_lookup(did, tcx), None); |
516 | tcx.arena.alloc(inner_optimized_mir(tcx, did)) | |
3dfed10e XL |
517 | } |
518 | ||
5869c6ff XL |
519 | fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { |
520 | if tcx.is_constructor(did.to_def_id()) { | |
e1599b0c XL |
521 | // There's no reason to run all of the MIR passes on constructors when |
522 | // we can just output the MIR we want directly. This also saves const | |
523 | // qualification and borrow checking the trouble of special casing | |
524 | // constructors. | |
5869c6ff | 525 | return shim::build_adt_ctor(tcx, did.to_def_id()); |
e1599b0c XL |
526 | } |
527 | ||
5869c6ff XL |
528 | match tcx.hir().body_const_context(did) { |
529 | // Run the `mir_for_ctfe` query, which depends on `mir_drops_elaborated_and_const_checked` | |
530 | // which we are going to steal below. Thus we need to run `mir_for_ctfe` first, so it | |
531 | // computes and caches its result. | |
532 | Some(hir::ConstContext::ConstFn) => tcx.ensure().mir_for_ctfe(did), | |
533 | None => {} | |
534 | Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), | |
535 | } | |
923072b8 | 536 | debug!("about to call mir_drops_elaborated..."); |
5869c6ff XL |
537 | let mut body = |
538 | tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); | |
923072b8 | 539 | debug!("body: {:#?}", body); |
29967ef6 | 540 | run_optimization_passes(tcx, &mut body); |
ba9703b0 | 541 | |
5099ac24 | 542 | debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); |
ba9703b0 | 543 | |
f9f354fc | 544 | body |
7cac9316 | 545 | } |
416331ca | 546 | |
5869c6ff XL |
547 | /// Fetch all the promoteds of an item and prepare their MIR bodies to be ready for |
548 | /// constant evaluation once all substitutions become known. | |
3dfed10e XL |
549 | fn promoted_mir<'tcx>( |
550 | tcx: TyCtxt<'tcx>, | |
551 | def: ty::WithOptConstParam<LocalDefId>, | |
552 | ) -> &'tcx IndexVec<Promoted, Body<'tcx>> { | |
553 | if tcx.is_constructor(def.did.to_def_id()) { | |
554 | return tcx.arena.alloc(IndexVec::new()); | |
e1599b0c XL |
555 | } |
556 | ||
5099ac24 FG |
557 | let tainted_by_errors = tcx.mir_borrowck_opt_const_arg(def).tainted_by_errors; |
558 | let mut promoted = tcx.mir_promoted(def).1.steal(); | |
e1599b0c | 559 | |
29967ef6 | 560 | for body in &mut promoted { |
5099ac24 FG |
561 | if let Some(error_reported) = tainted_by_errors { |
562 | body.tainted_by_errors = Some(error_reported); | |
563 | } | |
29967ef6 | 564 | run_post_borrowck_cleanup_passes(tcx, body); |
e1599b0c XL |
565 | } |
566 | ||
5099ac24 | 567 | debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); |
ba9703b0 | 568 | |
3dfed10e | 569 | tcx.arena.alloc(promoted) |
416331ca | 570 | } |