]>
Commit | Line | Data |
---|---|---|
0531ce1d XL |
1 | //! Propagates constants for early reporting of statically known |
2 | //! assertion failures | |
3 | ||
dc9dc135 XL |
4 | use std::cell::Cell; |
5 | ||
487cf647 FG |
6 | use either::Right; |
7 | ||
9c376795 | 8 | use rustc_const_eval::const_eval::CheckAlignment; |
f035d41b | 9 | use rustc_data_structures::fx::FxHashSet; |
ba9703b0 | 10 | use rustc_hir::def::DefKind; |
f9f354fc | 11 | use rustc_index::bit_set::BitSet; |
ba9703b0 | 12 | use rustc_index::vec::IndexVec; |
ba9703b0 | 13 | use rustc_middle::mir::visit::{ |
dfeec247 XL |
14 | MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor, |
15 | }; | |
9ffffee4 | 16 | use rustc_middle::mir::*; |
c295e0f8 | 17 | use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; |
2b03887a | 18 | use rustc_middle::ty::InternalSubsts; |
9ffffee4 | 19 | use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisitableExt}; |
ba9703b0 | 20 | use rustc_span::{def_id::DefId, Span}; |
9c376795 | 21 | use rustc_target::abi::{self, Align, HasDataLayout, Size, TargetDataLayout}; |
064997fb | 22 | use rustc_target::spec::abi::Abi as CallAbi; |
ba9703b0 | 23 | use rustc_trait_selection::traits; |
0531ce1d | 24 | |
c295e0f8 | 25 | use crate::MirPass; |
c295e0f8 | 26 | use rustc_const_eval::interpret::{ |
5e7ed085 | 27 | self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame, |
064997fb | 28 | ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, PlaceTy, |
f2b60f7d | 29 | Pointer, Scalar, StackPopCleanup, StackPopUnwind, |
a1dfa0c6 | 30 | }; |
a1dfa0c6 | 31 | |
3dfed10e XL |
32 | /// The maximum number of bytes that we'll allocate space for a local or the return value. |
33 | /// Needed for #66397, because otherwise we eval into large places and that can cause OOM or just | |
34 | /// Severely regress performance. | |
e74abb32 XL |
35 | const MAX_ALLOC_LIMIT: u64 = 1024; |
36 | ||
ba9703b0 XL |
37 | /// Macro for machine-specific `InterpError` without allocation. |
38 | /// (These will never be shown to the user, but they help diagnose ICEs.) | |
39 | macro_rules! throw_machine_stop_str { | |
40 | ($($tt:tt)*) => {{ | |
41 | // We make a new local type for it. The type itself does not carry any information, | |
42 | // but its vtable (for the `MachineStopType` trait) does. | |
43 | struct Zst; | |
f9f354fc XL |
44 | // Printing this type shows the desired string. |
45 | impl std::fmt::Display for Zst { | |
ba9703b0 XL |
46 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
47 | write!(f, $($tt)*) | |
48 | } | |
49 | } | |
50 | impl rustc_middle::mir::interpret::MachineStopType for Zst {} | |
51 | throw_machine_stop!(Zst) | |
52 | }}; | |
53 | } | |
54 | ||
0531ce1d XL |
55 | pub struct ConstProp; |
56 | ||
e1599b0c | 57 | impl<'tcx> MirPass<'tcx> for ConstProp { |
064997fb FG |
58 | fn is_enabled(&self, sess: &rustc_session::Session) -> bool { |
59 | sess.mir_opt_level() >= 1 | |
a2a8927a XL |
60 | } |
61 | ||
923072b8 | 62 | #[instrument(skip(self, tcx), level = "debug")] |
29967ef6 | 63 | fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
0531ce1d | 64 | // will be evaluated by miri and produce its errors there |
29967ef6 | 65 | if body.source.promoted.is_some() { |
0531ce1d XL |
66 | return; |
67 | } | |
a1dfa0c6 | 68 | |
29967ef6 | 69 | let def_id = body.source.def_id().expect_local(); |
04454e1e FG |
70 | let def_kind = tcx.def_kind(def_id); |
71 | let is_fn_like = def_kind.is_fn_like(); | |
72 | let is_assoc_const = def_kind == DefKind::AssocConst; | |
a1dfa0c6 XL |
73 | |
74 | // Only run const prop on functions, methods, closures and associated constants | |
dfeec247 | 75 | if !is_fn_like && !is_assoc_const { |
a1dfa0c6 | 76 | // skip anon_const/statics/consts because they'll be evaluated by miri anyway |
29967ef6 | 77 | trace!("ConstProp skipped for {:?}", def_id); |
dfeec247 | 78 | return; |
0531ce1d | 79 | } |
a1dfa0c6 | 80 | |
9ffffee4 | 81 | let is_generator = tcx.type_of(def_id.to_def_id()).subst_identity().is_generator(); |
e74abb32 XL |
82 | // FIXME(welseywiser) const prop doesn't work on generators because of query cycles |
83 | // computing their layout. | |
84 | if is_generator { | |
29967ef6 | 85 | trace!("ConstProp skipped for generator {:?}", def_id); |
dfeec247 XL |
86 | return; |
87 | } | |
88 | ||
89 | // Check if it's even possible to satisfy the 'where' clauses | |
90 | // for this item. | |
91 | // This branch will never be taken for any normal function. | |
92 | // However, it's possible to `#!feature(trivial_bounds)]` to write | |
93 | // a function with impossible to satisfy clauses, e.g.: | |
94 | // `fn foo() where String: Copy {}` | |
95 | // | |
96 | // We don't usually need to worry about this kind of case, | |
97 | // since we would get a compilation error if the user tried | |
98 | // to call it. However, since we can do const propagation | |
99 | // even without any calls to the function, we need to make | |
100 | // sure that it even makes sense to try to evaluate the body. | |
101 | // If there are unsatisfiable where clauses, then all bets are | |
102 | // off, and we just give up. | |
103 | // | |
104 | // We manually filter the predicates, skipping anything that's not | |
105 | // "global". We are in a potentially generic context | |
106 | // (e.g. we are evaluating a function without substituting generic | |
107 | // parameters, so this filtering serves two purposes: | |
108 | // | |
109 | // 1. We skip evaluating any predicates that we would | |
110 | // never be able prove are unsatisfiable (e.g. `<T as Foo>` | |
111 | // 2. We avoid trying to normalize predicates involving generic | |
112 | // parameters (e.g. `<T as Foo>::MyItem`). This can confuse | |
113 | // the normalization code (leading to cycle errors), since | |
114 | // it's usually never invoked in this way. | |
115 | let predicates = tcx | |
29967ef6 | 116 | .predicates_of(def_id.to_def_id()) |
dfeec247 XL |
117 | .predicates |
118 | .iter() | |
5099ac24 | 119 | .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None }); |
3dfed10e | 120 | if traits::impossible_predicates( |
dfeec247 | 121 | tcx, |
ba9703b0 | 122 | traits::elaborate_predicates(tcx, predicates).map(|o| o.predicate).collect(), |
dfeec247 | 123 | ) { |
29967ef6 | 124 | trace!("ConstProp skipped for {:?}: found unsatisfiable predicates", def_id); |
dfeec247 | 125 | return; |
e74abb32 XL |
126 | } |
127 | ||
29967ef6 | 128 | trace!("ConstProp starting for {:?}", def_id); |
0531ce1d | 129 | |
dfeec247 | 130 | let dummy_body = &Body::new( |
29967ef6 | 131 | body.source, |
f2b60f7d | 132 | (*body.basic_blocks).clone(), |
dfeec247 XL |
133 | body.source_scopes.clone(), |
134 | body.local_decls.clone(), | |
135 | Default::default(), | |
136 | body.arg_count, | |
137 | Default::default(), | |
5869c6ff | 138 | body.span, |
6a06907d | 139 | body.generator_kind(), |
5099ac24 | 140 | body.tainted_by_errors, |
dfeec247 | 141 | ); |
dc9dc135 | 142 | |
0531ce1d XL |
143 | // FIXME(oli-obk, eddyb) Optimize locals (or even local paths) to hold |
144 | // constants, instead of just checking for const-folding succeeding. | |
94222f64 | 145 | // That would require a uniform one-def no-mutation analysis |
0531ce1d | 146 | // and RPO (or recursing when needing the value of a local). |
29967ef6 | 147 | let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); |
dc9dc135 | 148 | optimization_finder.visit_body(body); |
0531ce1d | 149 | |
29967ef6 | 150 | trace!("ConstProp done for {:?}", def_id); |
0531ce1d XL |
151 | } |
152 | } | |
153 | ||
064997fb | 154 | pub struct ConstPropMachine<'mir, 'tcx> { |
ba9703b0 | 155 | /// The virtual call stack. |
136023e0 | 156 | stack: Vec<Frame<'mir, 'tcx>>, |
f035d41b | 157 | /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end. |
064997fb | 158 | pub written_only_inside_own_block_locals: FxHashSet<Local>, |
f035d41b | 159 | /// Locals that need to be cleared after every block terminates. |
064997fb FG |
160 | pub only_propagate_inside_block_locals: BitSet<Local>, |
161 | pub can_const_prop: IndexVec<Local, ConstPropMode>, | |
ba9703b0 XL |
162 | } |
163 | ||
a2a8927a | 164 | impl ConstPropMachine<'_, '_> { |
064997fb | 165 | pub fn new( |
3dfed10e XL |
166 | only_propagate_inside_block_locals: BitSet<Local>, |
167 | can_const_prop: IndexVec<Local, ConstPropMode>, | |
168 | ) -> Self { | |
f035d41b XL |
169 | Self { |
170 | stack: Vec::new(), | |
171 | written_only_inside_own_block_locals: Default::default(), | |
172 | only_propagate_inside_block_locals, | |
3dfed10e | 173 | can_const_prop, |
f035d41b | 174 | } |
ba9703b0 XL |
175 | } |
176 | } | |
e74abb32 | 177 | |
ba9703b0 | 178 | impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { |
f9f354fc | 179 | compile_time_machine!(<'mir, 'tcx>); |
136023e0 | 180 | const PANIC_ON_ALLOC_FAIL: bool = true; // all allocations are small (see `MAX_ALLOC_LIMIT`) |
e74abb32 | 181 | |
fc512014 XL |
182 | type MemoryKind = !; |
183 | ||
f2b60f7d | 184 | #[inline(always)] |
9c376795 | 185 | fn enforce_alignment(_ecx: &InterpCx<'mir, 'tcx, Self>) -> CheckAlignment { |
f2b60f7d FG |
186 | // We do not check for alignment to avoid having to carry an `Align` |
187 | // in `ConstValue::ByRef`. | |
9c376795 | 188 | CheckAlignment::No |
f2b60f7d FG |
189 | } |
190 | ||
191 | #[inline(always)] | |
192 | fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { | |
193 | false // for now, we don't enforce validity | |
194 | } | |
9c376795 FG |
195 | fn alignment_check_failed( |
196 | ecx: &InterpCx<'mir, 'tcx, Self>, | |
197 | _has: Align, | |
198 | _required: Align, | |
199 | _check: CheckAlignment, | |
200 | ) -> InterpResult<'tcx, ()> { | |
201 | span_bug!( | |
202 | ecx.cur_span(), | |
203 | "`alignment_check_failed` called when no alignment check requested" | |
204 | ) | |
205 | } | |
f2b60f7d | 206 | |
5869c6ff XL |
207 | fn load_mir( |
208 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
209 | _instance: ty::InstanceDef<'tcx>, | |
210 | ) -> InterpResult<'tcx, &'tcx Body<'tcx>> { | |
211 | throw_machine_stop_str!("calling functions isn't supported in ConstProp") | |
212 | } | |
213 | ||
60c5eb7d | 214 | fn find_mir_or_eval_fn( |
e74abb32 XL |
215 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, |
216 | _instance: ty::Instance<'tcx>, | |
064997fb | 217 | _abi: CallAbi, |
e74abb32 | 218 | _args: &[OpTy<'tcx>], |
923072b8 FG |
219 | _destination: &PlaceTy<'tcx>, |
220 | _target: Option<BasicBlock>, | |
17df50a5 | 221 | _unwind: StackPopUnwind, |
a2a8927a | 222 | ) -> InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> { |
e74abb32 XL |
223 | Ok(None) |
224 | } | |
225 | ||
e74abb32 XL |
226 | fn call_intrinsic( |
227 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
e74abb32 XL |
228 | _instance: ty::Instance<'tcx>, |
229 | _args: &[OpTy<'tcx>], | |
923072b8 FG |
230 | _destination: &PlaceTy<'tcx>, |
231 | _target: Option<BasicBlock>, | |
17df50a5 | 232 | _unwind: StackPopUnwind, |
60c5eb7d | 233 | ) -> InterpResult<'tcx> { |
ba9703b0 | 234 | throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp") |
60c5eb7d XL |
235 | } |
236 | ||
237 | fn assert_panic( | |
238 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
ba9703b0 XL |
239 | _msg: &rustc_middle::mir::AssertMessage<'tcx>, |
240 | _unwind: Option<rustc_middle::mir::BasicBlock>, | |
e74abb32 | 241 | ) -> InterpResult<'tcx> { |
ba9703b0 | 242 | bug!("panics terminators are not evaluated in ConstProp") |
e74abb32 XL |
243 | } |
244 | ||
e74abb32 XL |
245 | fn binary_ptr_op( |
246 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
247 | _bin_op: BinOp, | |
6a06907d XL |
248 | _left: &ImmTy<'tcx>, |
249 | _right: &ImmTy<'tcx>, | |
e74abb32 XL |
250 | ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { |
251 | // We can't do this because aliasing of memory can differ between const eval and llvm | |
ba9703b0 | 252 | throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") |
e74abb32 XL |
253 | } |
254 | ||
f035d41b XL |
255 | fn access_local_mut<'a>( |
256 | ecx: &'a mut InterpCx<'mir, 'tcx, Self>, | |
257 | frame: usize, | |
258 | local: Local, | |
064997fb | 259 | ) -> InterpResult<'tcx, &'a mut interpret::Operand<Self::Provenance>> { |
3dfed10e XL |
260 | if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation { |
261 | throw_machine_stop_str!("tried to write to a local that is marked as not propagatable") | |
262 | } | |
f035d41b | 263 | if frame == 0 && ecx.machine.only_propagate_inside_block_locals.contains(local) { |
3dfed10e XL |
264 | trace!( |
265 | "mutating local {:?} which is restricted to its block. \ | |
266 | Will remove it from const-prop after block is finished.", | |
267 | local | |
268 | ); | |
f035d41b XL |
269 | ecx.machine.written_only_inside_own_block_locals.insert(local); |
270 | } | |
271 | ecx.machine.stack[frame].locals[local].access_mut() | |
272 | } | |
273 | ||
ba9703b0 | 274 | fn before_access_global( |
04454e1e FG |
275 | _tcx: TyCtxt<'tcx>, |
276 | _machine: &Self, | |
ba9703b0 | 277 | _alloc_id: AllocId, |
487cf647 | 278 | alloc: ConstAllocation<'tcx>, |
ba9703b0 XL |
279 | _static_def_id: Option<DefId>, |
280 | is_write: bool, | |
e74abb32 | 281 | ) -> InterpResult<'tcx> { |
ba9703b0 XL |
282 | if is_write { |
283 | throw_machine_stop_str!("can't write to global"); | |
284 | } | |
285 | // If the static allocation is mutable, then we can't const prop it as its content | |
286 | // might be different at runtime. | |
9ffffee4 | 287 | if alloc.inner().mutability.is_mut() { |
ba9703b0 | 288 | throw_machine_stop_str!("can't access mutable globals in ConstProp"); |
e74abb32 XL |
289 | } |
290 | ||
291 | Ok(()) | |
292 | } | |
293 | ||
04454e1e FG |
294 | #[inline(always)] |
295 | fn expose_ptr( | |
296 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
297 | _ptr: Pointer<AllocId>, | |
298 | ) -> InterpResult<'tcx> { | |
299 | throw_machine_stop_str!("exposing pointers isn't supported in ConstProp") | |
300 | } | |
301 | ||
3dfed10e XL |
302 | #[inline(always)] |
303 | fn init_frame_extra( | |
304 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
305 | frame: Frame<'mir, 'tcx>, | |
306 | ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { | |
307 | Ok(frame) | |
308 | } | |
309 | ||
ba9703b0 | 310 | #[inline(always)] |
a2a8927a | 311 | fn stack<'a>( |
ba9703b0 | 312 | ecx: &'a InterpCx<'mir, 'tcx, Self>, |
064997fb | 313 | ) -> &'a [Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] { |
ba9703b0 XL |
314 | &ecx.machine.stack |
315 | } | |
316 | ||
317 | #[inline(always)] | |
a2a8927a | 318 | fn stack_mut<'a>( |
ba9703b0 | 319 | ecx: &'a mut InterpCx<'mir, 'tcx, Self>, |
064997fb | 320 | ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>> { |
ba9703b0 | 321 | &mut ecx.machine.stack |
e74abb32 | 322 | } |
e74abb32 XL |
323 | } |
324 | ||
0531ce1d | 325 | /// Finds optimization opportunities on the MIR. |
dc9dc135 | 326 | struct ConstPropagator<'mir, 'tcx> { |
ba9703b0 | 327 | ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, |
dc9dc135 | 328 | tcx: TyCtxt<'tcx>, |
0531ce1d | 329 | param_env: ParamEnv<'tcx>, |
04454e1e | 330 | local_decls: &'mir IndexVec<Local, LocalDecl<'tcx>>, |
dfeec247 XL |
331 | // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store |
332 | // the last known `SourceInfo` here and just keep revisiting it. | |
333 | source_info: Option<SourceInfo>, | |
0531ce1d XL |
334 | } |
335 | ||
a2a8927a | 336 | impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> { |
c295e0f8 | 337 | type LayoutOfResult = Result<TyAndLayout<'tcx>, LayoutError<'tcx>>; |
0531ce1d | 338 | |
c295e0f8 XL |
339 | #[inline] |
340 | fn handle_layout_err(&self, err: LayoutError<'tcx>, _: Span, _: Ty<'tcx>) -> LayoutError<'tcx> { | |
341 | err | |
0531ce1d XL |
342 | } |
343 | } | |
344 | ||
a2a8927a | 345 | impl HasDataLayout for ConstPropagator<'_, '_> { |
0531ce1d XL |
346 | #[inline] |
347 | fn data_layout(&self) -> &TargetDataLayout { | |
348 | &self.tcx.data_layout | |
349 | } | |
350 | } | |
351 | ||
a2a8927a | 352 | impl<'tcx> ty::layout::HasTyCtxt<'tcx> for ConstPropagator<'_, 'tcx> { |
0531ce1d | 353 | #[inline] |
dc9dc135 | 354 | fn tcx(&self) -> TyCtxt<'tcx> { |
0531ce1d XL |
355 | self.tcx |
356 | } | |
357 | } | |
358 | ||
a2a8927a | 359 | impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { |
94222f64 XL |
360 | #[inline] |
361 | fn param_env(&self) -> ty::ParamEnv<'tcx> { | |
362 | self.param_env | |
363 | } | |
364 | } | |
365 | ||
dc9dc135 | 366 | impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { |
0531ce1d | 367 | fn new( |
f9f354fc | 368 | body: &Body<'tcx>, |
dc9dc135 | 369 | dummy_body: &'mir Body<'tcx>, |
dc9dc135 | 370 | tcx: TyCtxt<'tcx>, |
dc9dc135 | 371 | ) -> ConstPropagator<'mir, 'tcx> { |
29967ef6 | 372 | let def_id = body.source.def_id(); |
dfeec247 | 373 | let substs = &InternalSubsts::identity_for_item(tcx, def_id); |
3dfed10e | 374 | let param_env = tcx.param_env_reveal_all_normalized(def_id); |
dfeec247 | 375 | |
3dfed10e | 376 | let can_const_prop = CanConstProp::check(tcx, param_env, body); |
f035d41b XL |
377 | let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len()); |
378 | for (l, mode) in can_const_prop.iter_enumerated() { | |
379 | if *mode == ConstPropMode::OnlyInsideOwnBlock { | |
380 | only_propagate_inside_block_locals.insert(l); | |
381 | } | |
382 | } | |
383 | let mut ecx = InterpCx::new( | |
384 | tcx, | |
04454e1e | 385 | tcx.def_span(def_id), |
f035d41b | 386 | param_env, |
3dfed10e | 387 | ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), |
f035d41b | 388 | ); |
dc9dc135 | 389 | |
923072b8 | 390 | let ret_layout = ecx |
064997fb | 391 | .layout_of(body.bound_return_ty().subst(tcx, substs)) |
dfeec247 | 392 | .ok() |
923072b8 | 393 | // Don't bother allocating memory for large values. |
064997fb FG |
394 | // I don't know how return types can seem to be unsized but this happens in the |
395 | // `type/type-unsatisfiable.rs` test. | |
396 | .filter(|ret_layout| { | |
487cf647 | 397 | ret_layout.is_sized() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) |
064997fb | 398 | }) |
923072b8 FG |
399 | .unwrap_or_else(|| ecx.layout_of(tcx.types.unit).unwrap()); |
400 | ||
401 | let ret = ecx | |
402 | .allocate(ret_layout, MemoryKind::Stack) | |
403 | .expect("couldn't perform small allocation") | |
404 | .into(); | |
60c5eb7d | 405 | |
dc9dc135 | 406 | ecx.push_stack_frame( |
60c5eb7d | 407 | Instance::new(def_id, substs), |
dc9dc135 | 408 | dummy_body, |
923072b8 | 409 | &ret, |
a2a8927a | 410 | StackPopCleanup::Root { cleanup: false }, |
dfeec247 XL |
411 | ) |
412 | .expect("failed to push initial stack frame"); | |
48663c56 | 413 | |
0531ce1d | 414 | ConstPropagator { |
94b46f34 | 415 | ecx, |
0531ce1d | 416 | tcx, |
0531ce1d | 417 | param_env, |
04454e1e | 418 | local_decls: &dummy_body.local_decls, |
dfeec247 | 419 | source_info: None, |
0531ce1d XL |
420 | } |
421 | } | |
422 | ||
f9f354fc | 423 | fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { |
f035d41b | 424 | let op = match self.ecx.eval_place_to_op(place, None) { |
f2b60f7d FG |
425 | Ok(op) => { |
426 | if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) { | |
427 | // Make sure nobody accidentally uses this value. | |
428 | return None; | |
429 | } | |
430 | op | |
431 | } | |
f035d41b XL |
432 | Err(e) => { |
433 | trace!("get_const failed: {}", e); | |
434 | return None; | |
435 | } | |
436 | }; | |
60c5eb7d | 437 | |
f9f354fc XL |
438 | // Try to read the local as an immediate so that if it is representable as a scalar, we can |
439 | // handle it as such, but otherwise, just return the value as is. | |
f2b60f7d | 440 | Some(match self.ecx.read_immediate_raw(&op) { |
487cf647 | 441 | Ok(Right(imm)) => imm.into(), |
f9f354fc | 442 | _ => op, |
f035d41b | 443 | }) |
dc9dc135 XL |
444 | } |
445 | ||
f9f354fc XL |
446 | /// Remove `local` from the pool of `Locals`. Allows writing to them, |
447 | /// but not reading from them anymore. | |
448 | fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { | |
064997fb FG |
449 | ecx.frame_mut().locals[local] = LocalState { |
450 | value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)), | |
451 | layout: Cell::new(None), | |
452 | }; | |
dfeec247 XL |
453 | } |
454 | ||
f9f354fc | 455 | /// Returns the value, if any, of evaluating `c`. |
5e7ed085 | 456 | fn eval_constant(&mut self, c: &Constant<'tcx>) -> Option<OpTy<'tcx>> { |
dfeec247 | 457 | // FIXME we need to revisit this for #67176 |
5099ac24 | 458 | if c.needs_subst() { |
dfeec247 XL |
459 | return None; |
460 | } | |
461 | ||
487cf647 FG |
462 | // No span, we don't want errors to be shown. |
463 | self.ecx.eval_mir_constant(&c.literal, None, None).ok() | |
8faf50e0 XL |
464 | } |
465 | ||
f9f354fc | 466 | /// Returns the value, if any, of evaluating `place`. |
ba9703b0 | 467 | fn eval_place(&mut self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { |
dc9dc135 | 468 | trace!("eval_place(place={:?})", place); |
9ffffee4 | 469 | self.ecx.eval_place_to_op(place, None).ok() |
0531ce1d XL |
470 | } |
471 | ||
f9f354fc XL |
472 | /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant` |
473 | /// or `eval_place`, depending on the variant of `Operand` used. | |
5e7ed085 | 474 | fn eval_operand(&mut self, op: &Operand<'tcx>) -> Option<OpTy<'tcx>> { |
0531ce1d | 475 | match *op { |
5e7ed085 | 476 | Operand::Constant(ref c) => self.eval_constant(c), |
ba9703b0 | 477 | Operand::Move(place) | Operand::Copy(place) => self.eval_place(place), |
0531ce1d XL |
478 | } |
479 | } | |
480 | ||
3dfed10e XL |
481 | fn propagate_operand(&mut self, operand: &mut Operand<'tcx>) { |
482 | match *operand { | |
483 | Operand::Copy(l) | Operand::Move(l) => { | |
5e7ed085 FG |
484 | if let Some(value) = self.get_const(l) && self.should_const_prop(&value) { |
485 | // FIXME(felix91gr): this code only handles `Scalar` cases. | |
486 | // For now, we're not handling `ScalarPair` cases because | |
487 | // doing so here would require a lot of code duplication. | |
488 | // We should hopefully generalize `Operand` handling into a fn, | |
489 | // and use it to do const-prop here and everywhere else | |
490 | // where it makes sense. | |
491 | if let interpret::Operand::Immediate(interpret::Immediate::Scalar( | |
f2b60f7d | 492 | scalar, |
5e7ed085 FG |
493 | )) = *value |
494 | { | |
495 | *operand = self.operand_from_scalar( | |
496 | scalar, | |
497 | value.layout.ty, | |
498 | self.source_info.unwrap().span, | |
499 | ); | |
3dfed10e XL |
500 | } |
501 | } | |
502 | } | |
503 | Operand::Constant(_) => (), | |
504 | } | |
505 | } | |
506 | ||
5e7ed085 | 507 | fn const_prop(&mut self, rvalue: &Rvalue<'tcx>, place: Place<'tcx>) -> Option<()> { |
e74abb32 XL |
508 | // Perform any special handling for specific Rvalue types. |
509 | // Generally, checks here fall into one of two categories: | |
510 | // 1. Additional checking to provide useful lints to the user | |
511 | // - In this case, we will do some validation and then fall through to the | |
512 | // end of the function which evals the assignment. | |
513 | // 2. Working around bugs in other parts of the compiler | |
514 | // - In this case, we'll return `None` from this function to stop evaluation. | |
515 | match rvalue { | |
dfeec247 | 516 | // Do not try creating references (#67862) |
f035d41b XL |
517 | Rvalue::AddressOf(_, place) | Rvalue::Ref(_, _, place) => { |
518 | trace!("skipping AddressOf | Ref for {:?}", place); | |
519 | ||
520 | // This may be creating mutable references or immutable references to cells. | |
521 | // If that happens, the pointed to value could be mutated via that reference. | |
522 | // Since we aren't tracking references, the const propagator loses track of what | |
523 | // value the local has right now. | |
524 | // Thus, all locals that have their reference taken | |
525 | // must not take part in propagation. | |
526 | Self::remove_const(&mut self.ecx, place.local); | |
e74abb32 | 527 | |
dfeec247 | 528 | return None; |
e74abb32 | 529 | } |
f035d41b XL |
530 | Rvalue::ThreadLocalRef(def_id) => { |
531 | trace!("skipping ThreadLocalRef({:?})", def_id); | |
e74abb32 | 532 | |
f035d41b XL |
533 | return None; |
534 | } | |
535 | ||
536 | // There's no other checking to do at this time. | |
537 | Rvalue::Aggregate(..) | |
538 | | Rvalue::Use(..) | |
064997fb | 539 | | Rvalue::CopyForDeref(..) |
f035d41b XL |
540 | | Rvalue::Repeat(..) |
541 | | Rvalue::Len(..) | |
542 | | Rvalue::Cast(..) | |
c295e0f8 | 543 | | Rvalue::ShallowInitBox(..) |
f035d41b | 544 | | Rvalue::Discriminant(..) |
9ffffee4 FG |
545 | | Rvalue::NullaryOp(..) |
546 | | Rvalue::UnaryOp(..) | |
547 | | Rvalue::BinaryOp(..) | |
548 | | Rvalue::CheckedBinaryOp(..) => {} | |
0531ce1d | 549 | } |
e74abb32 | 550 | |
f9f354fc | 551 | // FIXME we need to revisit this for #67176 |
5099ac24 | 552 | if rvalue.needs_subst() { |
f9f354fc XL |
553 | return None; |
554 | } | |
f2b60f7d FG |
555 | if !rvalue |
556 | .ty(&self.ecx.frame().body.local_decls, *self.ecx.tcx) | |
2b03887a | 557 | .is_sized(*self.ecx.tcx, self.param_env) |
f2b60f7d FG |
558 | { |
559 | // the interpreter doesn't support unsized locals (only unsized arguments), | |
560 | // but rustc does (in a kinda broken way), so we have to skip them here | |
561 | return None; | |
562 | } | |
f9f354fc | 563 | |
9c376795 | 564 | self.eval_rvalue_with_identities(rvalue, place) |
3dfed10e XL |
565 | } |
566 | ||
5e7ed085 | 567 | // Attempt to use algebraic identities to eliminate constant expressions |
3dfed10e XL |
568 | fn eval_rvalue_with_identities( |
569 | &mut self, | |
570 | rvalue: &Rvalue<'tcx>, | |
571 | place: Place<'tcx>, | |
572 | ) -> Option<()> { | |
9ffffee4 | 573 | match rvalue { |
a2a8927a XL |
574 | Rvalue::BinaryOp(op, box (left, right)) |
575 | | Rvalue::CheckedBinaryOp(op, box (left, right)) => { | |
9ffffee4 | 576 | let l = self.ecx.eval_operand(left, None).and_then(|x| self.ecx.read_immediate(&x)); |
f2b60f7d | 577 | let r = |
9ffffee4 | 578 | self.ecx.eval_operand(right, None).and_then(|x| self.ecx.read_immediate(&x)); |
a2a8927a XL |
579 | |
580 | let const_arg = match (l, r) { | |
f2b60f7d | 581 | (Ok(x), Err(_)) | (Err(_), Ok(x)) => x, // exactly one side is known |
9ffffee4 FG |
582 | (Err(_), Err(_)) => return None, // neither side is known |
583 | (Ok(_), Ok(_)) => return self.ecx.eval_rvalue_into_place(rvalue, place).ok(), // both sides are known | |
a2a8927a XL |
584 | }; |
585 | ||
064997fb FG |
586 | if !matches!(const_arg.layout.abi, abi::Abi::Scalar(..)) { |
587 | // We cannot handle Scalar Pair stuff. | |
f2b60f7d | 588 | // No point in calling `eval_rvalue_into_place`, since only one side is known |
9ffffee4 | 589 | return None; |
064997fb FG |
590 | } |
591 | ||
9ffffee4 FG |
592 | let arg_value = const_arg.to_scalar().to_bits(const_arg.layout.size).ok()?; |
593 | let dest = self.ecx.eval_place(place).ok()?; | |
a2a8927a XL |
594 | |
595 | match op { | |
9ffffee4 FG |
596 | BinOp::BitAnd if arg_value == 0 => { |
597 | self.ecx.write_immediate(*const_arg, &dest).ok() | |
598 | } | |
a2a8927a XL |
599 | BinOp::BitOr |
600 | if arg_value == const_arg.layout.size.truncate(u128::MAX) | |
601 | || (const_arg.layout.ty.is_bool() && arg_value == 1) => | |
602 | { | |
9ffffee4 | 603 | self.ecx.write_immediate(*const_arg, &dest).ok() |
a2a8927a XL |
604 | } |
605 | BinOp::Mul if const_arg.layout.ty.is_integral() && arg_value == 0 => { | |
606 | if let Rvalue::CheckedBinaryOp(_, _) = rvalue { | |
607 | let val = Immediate::ScalarPair( | |
9c376795 FG |
608 | const_arg.to_scalar(), |
609 | Scalar::from_bool(false), | |
a2a8927a | 610 | ); |
9ffffee4 | 611 | self.ecx.write_immediate(val, &dest).ok() |
a2a8927a | 612 | } else { |
9ffffee4 | 613 | self.ecx.write_immediate(*const_arg, &dest).ok() |
3dfed10e XL |
614 | } |
615 | } | |
9ffffee4 | 616 | _ => None, |
3dfed10e XL |
617 | } |
618 | } | |
9ffffee4 FG |
619 | _ => self.ecx.eval_rvalue_into_place(rvalue, place).ok(), |
620 | } | |
0531ce1d | 621 | } |
48663c56 | 622 | |
f9f354fc | 623 | /// Creates a new `Operand::Constant` from a `Scalar` value |
48663c56 | 624 | fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>, span: Span) -> Operand<'tcx> { |
dfeec247 XL |
625 | Operand::Constant(Box::new(Constant { |
626 | span, | |
627 | user_ty: None, | |
923072b8 | 628 | literal: ConstantKind::from_scalar(self.tcx, scalar, ty), |
dfeec247 | 629 | })) |
48663c56 XL |
630 | } |
631 | ||
dc9dc135 XL |
632 | fn replace_with_const( |
633 | &mut self, | |
634 | rval: &mut Rvalue<'tcx>, | |
6a06907d | 635 | value: &OpTy<'tcx>, |
dc9dc135 XL |
636 | source_info: SourceInfo, |
637 | ) { | |
f9f354fc | 638 | if let Rvalue::Use(Operand::Constant(c)) = rval { |
6a06907d | 639 | match c.literal { |
923072b8 | 640 | ConstantKind::Ty(c) if matches!(c.kind(), ConstKind::Unevaluated(..)) => {} |
6a06907d XL |
641 | _ => { |
642 | trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c); | |
643 | return; | |
644 | } | |
f9f354fc XL |
645 | } |
646 | } | |
647 | ||
fc512014 | 648 | trace!("attempting to replace {:?} with {:?}", rval, value); |
74b04a01 | 649 | if let Err(e) = self.ecx.const_validate_operand( |
48663c56 XL |
650 | value, |
651 | vec![], | |
dc9dc135 | 652 | // FIXME: is ref tracking too expensive? |
29967ef6 | 653 | // FIXME: what is the point of ref tracking if we do not even check the tracked refs? |
74b04a01 | 654 | &mut interpret::RefTracking::empty(), |
29967ef6 | 655 | CtfeValidationMode::Regular, |
dc9dc135 XL |
656 | ) { |
657 | trace!("validation error, attempt failed: {:?}", e); | |
658 | return; | |
659 | } | |
660 | ||
04454e1e | 661 | // FIXME> figure out what to do when read_immediate_raw fails |
9ffffee4 | 662 | let imm = self.ecx.read_immediate_raw(value).ok(); |
48663c56 | 663 | |
487cf647 | 664 | if let Some(Right(imm)) = imm { |
dc9dc135 | 665 | match *imm { |
f2b60f7d | 666 | interpret::Immediate::Scalar(scalar) => { |
dfeec247 XL |
667 | *rval = Rvalue::Use(self.operand_from_scalar( |
668 | scalar, | |
669 | value.layout.ty, | |
670 | source_info.span, | |
671 | )); | |
672 | } | |
f2b60f7d | 673 | Immediate::ScalarPair(..) => { |
3dfed10e XL |
674 | // Found a value represented as a pair. For now only do const-prop if the type |
675 | // of `rvalue` is also a tuple with two scalars. | |
676 | // FIXME: enable the general case stated above ^. | |
5099ac24 | 677 | let ty = value.layout.ty; |
60c5eb7d | 678 | // Only do it for tuples |
5e7ed085 | 679 | if let ty::Tuple(types) = ty.kind() { |
60c5eb7d | 680 | // Only do it if tuple is also a pair with two scalars |
5e7ed085 | 681 | if let [ty1, ty2] = types[..] { |
9ffffee4 FG |
682 | let ty_is_scalar = |ty| { |
683 | self.ecx.layout_of(ty).ok().map(|layout| layout.abi.is_scalar()) | |
684 | == Some(true) | |
685 | }; | |
686 | let alloc = if ty_is_scalar(ty1) && ty_is_scalar(ty2) { | |
687 | let alloc = self | |
688 | .ecx | |
689 | .intern_with_temp_alloc(value.layout, |ecx, dest| { | |
690 | ecx.write_immediate(*imm, dest) | |
691 | }) | |
692 | .unwrap(); | |
693 | Some(alloc) | |
694 | } else { | |
695 | None | |
696 | }; | |
697 | ||
698 | if let Some(alloc) = alloc { | |
3dfed10e XL |
699 | // Assign entire constant in a single statement. |
700 | // We can't use aggregates, as we run after the aggregate-lowering `MirPhase`. | |
923072b8 FG |
701 | let const_val = ConstValue::ByRef { alloc, offset: Size::ZERO }; |
702 | let literal = ConstantKind::Val(const_val, ty); | |
3dfed10e XL |
703 | *rval = Rvalue::Use(Operand::Constant(Box::new(Constant { |
704 | span: source_info.span, | |
705 | user_ty: None, | |
923072b8 | 706 | literal, |
3dfed10e | 707 | }))); |
60c5eb7d XL |
708 | } |
709 | } | |
48663c56 | 710 | } |
dfeec247 | 711 | } |
3dfed10e XL |
712 | // Scalars or scalar pairs that contain undef values are assumed to not have |
713 | // successfully evaluated and are thus not propagated. | |
dfeec247 | 714 | _ => {} |
48663c56 XL |
715 | } |
716 | } | |
717 | } | |
718 | ||
f9f354fc | 719 | /// Returns `true` if and only if this `op` should be const-propagated into. |
6a06907d | 720 | fn should_const_prop(&mut self, op: &OpTy<'tcx>) -> bool { |
fc512014 XL |
721 | if !self.tcx.consider_optimizing(|| format!("ConstantPropagation - OpTy: {:?}", op)) { |
722 | return false; | |
723 | } | |
724 | ||
6a06907d | 725 | match **op { |
f2b60f7d FG |
726 | interpret::Operand::Immediate(Immediate::Scalar(s)) => s.try_to_int().is_ok(), |
727 | interpret::Operand::Immediate(Immediate::ScalarPair(l, r)) => { | |
728 | l.try_to_int().is_ok() && r.try_to_int().is_ok() | |
dfeec247 | 729 | } |
dfeec247 | 730 | _ => false, |
60c5eb7d | 731 | } |
48663c56 | 732 | } |
0531ce1d XL |
733 | } |
734 | ||
dfeec247 XL |
735 | /// The mode that `ConstProp` is allowed to run in for a given `Local`. |
736 | #[derive(Clone, Copy, Debug, PartialEq)] | |
064997fb | 737 | pub enum ConstPropMode { |
dfeec247 XL |
738 | /// The `Local` can be propagated into and reads of this `Local` can also be propagated. |
739 | FullConstProp, | |
f9f354fc XL |
740 | /// The `Local` can only be propagated into and from its own block. |
741 | OnlyInsideOwnBlock, | |
dfeec247 XL |
742 | /// The `Local` can be propagated into but reads cannot be propagated. |
743 | OnlyPropagateInto, | |
f035d41b XL |
744 | /// The `Local` cannot be part of propagation at all. Any statement |
745 | /// referencing it either for reading or writing will not get propagated. | |
dfeec247 XL |
746 | NoPropagation, |
747 | } | |
748 | ||
064997fb | 749 | pub struct CanConstProp { |
dfeec247 | 750 | can_const_prop: IndexVec<Local, ConstPropMode>, |
f9f354fc XL |
751 | // False at the beginning. Once set, no more assignments are allowed to that local. |
752 | found_assignment: BitSet<Local>, | |
753 | // Cache of locals' information | |
754 | local_kinds: IndexVec<Local, LocalKind>, | |
0531ce1d XL |
755 | } |
756 | ||
757 | impl CanConstProp { | |
f9f354fc | 758 | /// Returns true if `local` can be propagated |
064997fb | 759 | pub fn check<'tcx>( |
3dfed10e XL |
760 | tcx: TyCtxt<'tcx>, |
761 | param_env: ParamEnv<'tcx>, | |
762 | body: &Body<'tcx>, | |
763 | ) -> IndexVec<Local, ConstPropMode> { | |
0531ce1d | 764 | let mut cpv = CanConstProp { |
dfeec247 | 765 | can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls), |
f9f354fc XL |
766 | found_assignment: BitSet::new_empty(body.local_decls.len()), |
767 | local_kinds: IndexVec::from_fn_n( | |
768 | |local| body.local_kind(local), | |
769 | body.local_decls.len(), | |
770 | ), | |
0531ce1d XL |
771 | }; |
772 | for (local, val) in cpv.can_const_prop.iter_enumerated_mut() { | |
3dfed10e XL |
773 | let ty = body.local_decls[local].ty; |
774 | match tcx.layout_of(param_env.and(ty)) { | |
775 | Ok(layout) if layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) => {} | |
776 | // Either the layout fails to compute, then we can't use this local anyway | |
777 | // or the local is too large, then we don't want to. | |
778 | _ => { | |
779 | *val = ConstPropMode::NoPropagation; | |
780 | continue; | |
781 | } | |
782 | } | |
f9f354fc XL |
783 | // Cannot use args at all |
784 | // Cannot use locals because if x < y { y - x } else { x - y } would | |
0531ce1d XL |
785 | // lint for x != y |
786 | // FIXME(oli-obk): lint variables until they are used in a condition | |
787 | // FIXME(oli-obk): lint if return value is constant | |
f9f354fc | 788 | if cpv.local_kinds[local] == LocalKind::Arg { |
dfeec247 | 789 | *val = ConstPropMode::OnlyPropagateInto; |
f9f354fc XL |
790 | trace!( |
791 | "local {:?} can't be const propagated because it's a function argument", | |
792 | local | |
793 | ); | |
794 | } else if cpv.local_kinds[local] == LocalKind::Var { | |
795 | *val = ConstPropMode::OnlyInsideOwnBlock; | |
796 | trace!( | |
797 | "local {:?} will only be propagated inside its block, because it's a user variable", | |
798 | local | |
799 | ); | |
dc9dc135 | 800 | } |
0531ce1d | 801 | } |
ba9703b0 | 802 | cpv.visit_body(&body); |
0531ce1d XL |
803 | cpv.can_const_prop |
804 | } | |
805 | } | |
806 | ||
a2a8927a | 807 | impl Visitor<'_> for CanConstProp { |
064997fb | 808 | fn visit_local(&mut self, local: Local, context: PlaceContext, _: Location) { |
ba9703b0 | 809 | use rustc_middle::mir::visit::PlaceContext::*; |
0531ce1d | 810 | match context { |
f9f354fc XL |
811 | // Projections are fine, because `&mut foo.x` will be caught by |
812 | // `MutatingUseContext::Borrow` elsewhere. | |
813 | MutatingUse(MutatingUseContext::Projection) | |
814 | // These are just stores, where the storing is not propagatable, but there may be later | |
815 | // mutations of the same local via `Store` | |
816 | | MutatingUse(MutatingUseContext::Call) | |
a2a8927a | 817 | | MutatingUse(MutatingUseContext::AsmOutput) |
04454e1e | 818 | | MutatingUse(MutatingUseContext::Deinit) |
f9f354fc | 819 | // Actual store that can possibly even propagate a value |
04454e1e FG |
820 | | MutatingUse(MutatingUseContext::Store) |
821 | | MutatingUse(MutatingUseContext::SetDiscriminant) => { | |
f9f354fc XL |
822 | if !self.found_assignment.insert(local) { |
823 | match &mut self.can_const_prop[local] { | |
824 | // If the local can only get propagated in its own block, then we don't have | |
825 | // to worry about multiple assignments, as we'll nuke the const state at the | |
826 | // end of the block anyway, and inside the block we overwrite previous | |
827 | // states as applicable. | |
828 | ConstPropMode::OnlyInsideOwnBlock => {} | |
f035d41b XL |
829 | ConstPropMode::NoPropagation => {} |
830 | ConstPropMode::OnlyPropagateInto => {} | |
831 | other @ ConstPropMode::FullConstProp => { | |
f9f354fc | 832 | trace!( |
3dfed10e XL |
833 | "local {:?} can't be propagated because of multiple assignments. Previous state: {:?}", |
834 | local, other, | |
f9f354fc | 835 | ); |
3dfed10e | 836 | *other = ConstPropMode::OnlyInsideOwnBlock; |
f9f354fc XL |
837 | } |
838 | } | |
dfeec247 XL |
839 | } |
840 | } | |
0531ce1d | 841 | // Reading constants is allowed an arbitrary number of times |
dfeec247 XL |
842 | NonMutatingUse(NonMutatingUseContext::Copy) |
843 | | NonMutatingUse(NonMutatingUseContext::Move) | |
844 | | NonMutatingUse(NonMutatingUseContext::Inspect) | |
845 | | NonMutatingUse(NonMutatingUseContext::Projection) | |
dfeec247 | 846 | | NonUse(_) => {} |
f9f354fc XL |
847 | |
848 | // These could be propagated with a smarter analysis or just some careful thinking about | |
849 | // whether they'd be fine right now. | |
5099ac24 | 850 | MutatingUse(MutatingUseContext::Yield) |
f9f354fc XL |
851 | | MutatingUse(MutatingUseContext::Drop) |
852 | | MutatingUse(MutatingUseContext::Retag) | |
853 | // These can't ever be propagated under any scheme, as we can't reason about indirect | |
854 | // mutation. | |
855 | | NonMutatingUse(NonMutatingUseContext::SharedBorrow) | |
856 | | NonMutatingUse(NonMutatingUseContext::ShallowBorrow) | |
857 | | NonMutatingUse(NonMutatingUseContext::UniqueBorrow) | |
858 | | NonMutatingUse(NonMutatingUseContext::AddressOf) | |
859 | | MutatingUse(MutatingUseContext::Borrow) | |
860 | | MutatingUse(MutatingUseContext::AddressOf) => { | |
dc9dc135 | 861 | trace!("local {:?} can't be propagaged because it's used: {:?}", local, context); |
dfeec247 XL |
862 | self.can_const_prop[local] = ConstPropMode::NoPropagation; |
863 | } | |
0531ce1d XL |
864 | } |
865 | } | |
866 | } | |
867 | ||
a2a8927a | 868 | impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { |
e74abb32 XL |
869 | fn tcx(&self) -> TyCtxt<'tcx> { |
870 | self.tcx | |
871 | } | |
872 | ||
f9f354fc | 873 | fn visit_body(&mut self, body: &mut Body<'tcx>) { |
f2b60f7d | 874 | for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { |
f9f354fc XL |
875 | self.visit_basic_block_data(bb, data); |
876 | } | |
877 | } | |
878 | ||
3dfed10e XL |
879 | fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { |
880 | self.super_operand(operand, location); | |
881 | ||
6a06907d | 882 | // Only const prop copies and moves on `mir_opt_level=3` as doing so |
1b1a35ee | 883 | // currently slightly increases compile time in some cases. |
6a06907d | 884 | if self.tcx.sess.mir_opt_level() >= 3 { |
3dfed10e XL |
885 | self.propagate_operand(operand) |
886 | } | |
887 | } | |
888 | ||
dfeec247 | 889 | fn visit_constant(&mut self, constant: &mut Constant<'tcx>, location: Location) { |
0531ce1d XL |
890 | trace!("visit_constant: {:?}", constant); |
891 | self.super_constant(constant, location); | |
5e7ed085 | 892 | self.eval_constant(constant); |
0531ce1d XL |
893 | } |
894 | ||
dfeec247 | 895 | fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { |
0531ce1d | 896 | trace!("visit_statement: {:?}", statement); |
dfeec247 XL |
897 | let source_info = statement.source_info; |
898 | self.source_info = Some(source_info); | |
9ffffee4 FG |
899 | match statement.kind { |
900 | StatementKind::Assign(box (place, ref mut rval)) => { | |
901 | let can_const_prop = self.ecx.machine.can_const_prop[place.local]; | |
902 | if let Some(()) = self.const_prop(rval, place) { | |
903 | // This will return None if the above `const_prop` invocation only "wrote" a | |
904 | // type whose creation requires no write. E.g. a generator whose initial state | |
905 | // consists solely of uninitialized memory (so it doesn't capture any locals). | |
906 | if let Some(ref value) = self.get_const(place) && self.should_const_prop(value) { | |
907 | trace!("replacing {:?} with {:?}", rval, value); | |
908 | self.replace_with_const(rval, value, source_info); | |
909 | if can_const_prop == ConstPropMode::FullConstProp | |
910 | || can_const_prop == ConstPropMode::OnlyInsideOwnBlock | |
911 | { | |
912 | trace!("propagated into {:?}", place); | |
913 | } | |
dfeec247 | 914 | } |
9ffffee4 FG |
915 | match can_const_prop { |
916 | ConstPropMode::OnlyInsideOwnBlock => { | |
917 | trace!( | |
918 | "found local restricted to its block. \ | |
f035d41b | 919 | Will remove it from const-prop after block is finished. Local: {:?}", |
9ffffee4 FG |
920 | place.local |
921 | ); | |
8faf50e0 | 922 | } |
9ffffee4 FG |
923 | ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { |
924 | trace!("can't propagate into {:?}", place); | |
925 | if place.local != RETURN_PLACE { | |
3dfed10e XL |
926 | Self::remove_const(&mut self.ecx, place.local); |
927 | } | |
928 | } | |
9ffffee4 | 929 | ConstPropMode::FullConstProp => {} |
3dfed10e | 930 | } |
9ffffee4 FG |
931 | } else { |
932 | // Const prop failed, so erase the destination, ensuring that whatever happens | |
933 | // from here on, does not know about the previous value. | |
934 | // This is important in case we have | |
935 | // ```rust | |
936 | // let mut x = 42; | |
937 | // x = SOME_MUTABLE_STATIC; | |
938 | // // x must now be uninit | |
939 | // ``` | |
940 | // FIXME: we overzealously erase the entire local, because that's easier to | |
941 | // implement. | |
942 | trace!( | |
943 | "propagation into {:?} failed. | |
944 | Nuking the entire site from orbit, it's the only way to be sure", | |
945 | place, | |
946 | ); | |
947 | Self::remove_const(&mut self.ecx, place.local); | |
3dfed10e | 948 | } |
9ffffee4 FG |
949 | } |
950 | StatementKind::SetDiscriminant { ref place, .. } => { | |
951 | match self.ecx.machine.can_const_prop[place.local] { | |
952 | ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { | |
953 | if self.ecx.statement(statement).is_ok() { | |
954 | trace!("propped discriminant into {:?}", place); | |
e74abb32 | 955 | } else { |
9ffffee4 FG |
956 | Self::remove_const(&mut self.ecx, place.local); |
957 | } | |
958 | } | |
959 | ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { | |
960 | Self::remove_const(&mut self.ecx, place.local); | |
961 | } | |
e74abb32 | 962 | } |
e74abb32 | 963 | } |
9ffffee4 FG |
964 | StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { |
965 | let frame = self.ecx.frame_mut(); | |
966 | frame.locals[local].value = if let StatementKind::StorageLive(_) = statement.kind { | |
967 | LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)) | |
968 | } else { | |
969 | LocalValue::Dead | |
970 | }; | |
971 | } | |
972 | _ => {} | |
0531ce1d | 973 | } |
e74abb32 | 974 | |
48663c56 | 975 | self.super_statement(statement, location); |
0531ce1d XL |
976 | } |
977 | ||
dfeec247 | 978 | fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { |
48663c56 | 979 | let source_info = terminator.source_info; |
dfeec247 XL |
980 | self.source_info = Some(source_info); |
981 | self.super_terminator(terminator, location); | |
9ffffee4 | 982 | |
48663c56 | 983 | match &mut terminator.kind { |
5e7ed085 | 984 | TerminatorKind::Assert { expected, ref mut cond, .. } => { |
9ffffee4 FG |
985 | if let Some(ref value) = self.eval_operand(&cond) |
986 | && let Ok(value_const) = self.ecx.read_scalar(&value) | |
987 | && self.should_const_prop(value) | |
988 | { | |
48663c56 | 989 | trace!("assertion on {:?} should be {:?}", value, expected); |
9ffffee4 FG |
990 | *cond = self.operand_from_scalar( |
991 | value_const, | |
992 | self.tcx.types.bool, | |
993 | source_info.span, | |
994 | ); | |
0531ce1d | 995 | } |
dfeec247 | 996 | } |
3dfed10e XL |
997 | TerminatorKind::SwitchInt { ref mut discr, .. } => { |
998 | // FIXME: This is currently redundant with `visit_operand`, but sadly | |
999 | // always visiting operands currently causes a perf regression in LLVM codegen, so | |
6a06907d | 1000 | // `visit_operand` currently only runs for propagates places for `mir_opt_level=4`. |
3dfed10e | 1001 | self.propagate_operand(discr) |
dfeec247 | 1002 | } |
3dfed10e | 1003 | // None of these have Operands to const-propagate. |
dfeec247 XL |
1004 | TerminatorKind::Goto { .. } |
1005 | | TerminatorKind::Resume | |
1006 | | TerminatorKind::Abort | |
1007 | | TerminatorKind::Return | |
1008 | | TerminatorKind::Unreachable | |
1009 | | TerminatorKind::Drop { .. } | |
1010 | | TerminatorKind::DropAndReplace { .. } | |
1011 | | TerminatorKind::Yield { .. } | |
1012 | | TerminatorKind::GeneratorDrop | |
f035d41b | 1013 | | TerminatorKind::FalseEdge { .. } |
f9f354fc XL |
1014 | | TerminatorKind::FalseUnwind { .. } |
1015 | | TerminatorKind::InlineAsm { .. } => {} | |
3dfed10e XL |
1016 | // Every argument in our function calls have already been propagated in `visit_operand`. |
1017 | // | |
1b1a35ee | 1018 | // NOTE: because LLVM codegen gives slight performance regressions with it, so this is |
6a06907d | 1019 | // gated on `mir_opt_level=3`. |
3dfed10e | 1020 | TerminatorKind::Call { .. } => {} |
f9f354fc | 1021 | } |
9ffffee4 FG |
1022 | } |
1023 | ||
1024 | fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { | |
1025 | self.super_basic_block_data(block, data); | |
f035d41b XL |
1026 | |
1027 | // We remove all Locals which are restricted in propagation to their containing blocks and | |
1028 | // which were modified in the current block. | |
3dfed10e | 1029 | // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. |
f035d41b XL |
1030 | let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); |
1031 | for &local in locals.iter() { | |
f9f354fc | 1032 | Self::remove_const(&mut self.ecx, local); |
0531ce1d | 1033 | } |
f035d41b XL |
1034 | locals.clear(); |
1035 | // Put it back so we reuse the heap of the storage | |
1036 | self.ecx.machine.written_only_inside_own_block_locals = locals; | |
1037 | if cfg!(debug_assertions) { | |
1038 | // Ensure we are correctly erasing locals with the non-debug-assert logic. | |
1039 | for local in self.ecx.machine.only_propagate_inside_block_locals.iter() { | |
1040 | assert!( | |
1041 | self.get_const(local.into()).is_none() | |
1042 | || self | |
1043 | .layout_of(self.local_decls[local].ty) | |
1044 | .map_or(true, |layout| layout.is_zst()) | |
1045 | ) | |
1046 | } | |
1047 | } | |
0531ce1d XL |
1048 | } |
1049 | } |