]>
Commit | Line | Data |
---|---|---|
0531ce1d XL |
1 | //! Propagates constants for early reporting of statically known |
2 | //! assertion failures | |
3 | ||
dc9dc135 XL |
4 | use std::cell::Cell; |
5 | ||
3dfed10e | 6 | use rustc_ast::Mutability; |
f035d41b | 7 | use rustc_data_structures::fx::FxHashSet; |
ba9703b0 XL |
8 | use rustc_hir::def::DefKind; |
9 | use rustc_hir::HirId; | |
f9f354fc | 10 | use rustc_index::bit_set::BitSet; |
ba9703b0 | 11 | use rustc_index::vec::IndexVec; |
ba9703b0 | 12 | use rustc_middle::mir::visit::{ |
dfeec247 XL |
13 | MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor, |
14 | }; | |
ba9703b0 | 15 | use rustc_middle::mir::{ |
cdc7bbd5 XL |
16 | AssertKind, BasicBlock, BinOp, Body, Constant, ConstantKind, Local, LocalDecl, LocalKind, |
17 | Location, Operand, Place, Rvalue, SourceInfo, SourceScope, SourceScopeData, Statement, | |
18 | StatementKind, Terminator, TerminatorKind, UnOp, RETURN_PLACE, | |
48663c56 | 19 | }; |
ba9703b0 XL |
20 | use rustc_middle::ty::layout::{HasTyCtxt, LayoutError, TyAndLayout}; |
21 | use rustc_middle::ty::subst::{InternalSubsts, Subst}; | |
29967ef6 XL |
22 | use rustc_middle::ty::{ |
23 | self, ConstInt, ConstKind, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeFoldable, | |
24 | }; | |
ba9703b0 XL |
25 | use rustc_session::lint; |
26 | use rustc_span::{def_id::DefId, Span}; | |
27 | use rustc_target::abi::{HasDataLayout, LayoutOf, Size, TargetDataLayout}; | |
5869c6ff | 28 | use rustc_target::spec::abi::Abi; |
ba9703b0 | 29 | use rustc_trait_selection::traits; |
0531ce1d | 30 | |
3dfed10e | 31 | use crate::const_eval::ConstEvalErr; |
dc9dc135 | 32 | use crate::interpret::{ |
29967ef6 | 33 | self, compile_time_machine, AllocId, Allocation, ConstValue, CtfeValidationMode, Frame, ImmTy, |
136023e0 XL |
34 | Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, MemoryKind, OpTy, |
35 | Operand as InterpOperand, PlaceTy, Scalar, ScalarMaybeUninit, StackPopCleanup, StackPopUnwind, | |
a1dfa0c6 | 36 | }; |
29967ef6 | 37 | use crate::transform::MirPass; |
a1dfa0c6 | 38 | |
3dfed10e XL |
39 | /// The maximum number of bytes that we'll allocate space for a local or the return value. |
40 | /// Needed for #66397, because otherwise we eval into large places and that can cause OOM or just | |
41 | /// Severely regress performance. | |
e74abb32 XL |
42 | const MAX_ALLOC_LIMIT: u64 = 1024; |
43 | ||
ba9703b0 XL |
44 | /// Macro for machine-specific `InterpError` without allocation. |
45 | /// (These will never be shown to the user, but they help diagnose ICEs.) | |
46 | macro_rules! throw_machine_stop_str { | |
47 | ($($tt:tt)*) => {{ | |
48 | // We make a new local type for it. The type itself does not carry any information, | |
49 | // but its vtable (for the `MachineStopType` trait) does. | |
50 | struct Zst; | |
f9f354fc XL |
51 | // Printing this type shows the desired string. |
52 | impl std::fmt::Display for Zst { | |
ba9703b0 XL |
53 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
54 | write!(f, $($tt)*) | |
55 | } | |
56 | } | |
57 | impl rustc_middle::mir::interpret::MachineStopType for Zst {} | |
58 | throw_machine_stop!(Zst) | |
59 | }}; | |
60 | } | |
61 | ||
0531ce1d XL |
62 | pub struct ConstProp; |
63 | ||
e1599b0c | 64 | impl<'tcx> MirPass<'tcx> for ConstProp { |
29967ef6 | 65 | fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
0531ce1d | 66 | // will be evaluated by miri and produce its errors there |
29967ef6 | 67 | if body.source.promoted.is_some() { |
0531ce1d XL |
68 | return; |
69 | } | |
a1dfa0c6 | 70 | |
ba9703b0 | 71 | use rustc_middle::hir::map::blocks::FnLikeNode; |
29967ef6 XL |
72 | let def_id = body.source.def_id().expect_local(); |
73 | let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); | |
a1dfa0c6 | 74 | |
dc9dc135 | 75 | let is_fn_like = FnLikeNode::from_node(tcx.hir().get(hir_id)).is_some(); |
29967ef6 | 76 | let is_assoc_const = tcx.def_kind(def_id.to_def_id()) == DefKind::AssocConst; |
a1dfa0c6 XL |
77 | |
78 | // Only run const prop on functions, methods, closures and associated constants | |
dfeec247 | 79 | if !is_fn_like && !is_assoc_const { |
a1dfa0c6 | 80 | // skip anon_const/statics/consts because they'll be evaluated by miri anyway |
29967ef6 | 81 | trace!("ConstProp skipped for {:?}", def_id); |
dfeec247 | 82 | return; |
0531ce1d | 83 | } |
a1dfa0c6 | 84 | |
29967ef6 | 85 | let is_generator = tcx.type_of(def_id.to_def_id()).is_generator(); |
e74abb32 XL |
86 | // FIXME(welseywiser) const prop doesn't work on generators because of query cycles |
87 | // computing their layout. | |
88 | if is_generator { | |
29967ef6 | 89 | trace!("ConstProp skipped for generator {:?}", def_id); |
dfeec247 XL |
90 | return; |
91 | } | |
92 | ||
93 | // Check if it's even possible to satisfy the 'where' clauses | |
94 | // for this item. | |
95 | // This branch will never be taken for any normal function. | |
96 | // However, it's possible to `#!feature(trivial_bounds)]` to write | |
97 | // a function with impossible to satisfy clauses, e.g.: | |
98 | // `fn foo() where String: Copy {}` | |
99 | // | |
100 | // We don't usually need to worry about this kind of case, | |
101 | // since we would get a compilation error if the user tried | |
102 | // to call it. However, since we can do const propagation | |
103 | // even without any calls to the function, we need to make | |
104 | // sure that it even makes sense to try to evaluate the body. | |
105 | // If there are unsatisfiable where clauses, then all bets are | |
106 | // off, and we just give up. | |
107 | // | |
108 | // We manually filter the predicates, skipping anything that's not | |
109 | // "global". We are in a potentially generic context | |
110 | // (e.g. we are evaluating a function without substituting generic | |
111 | // parameters, so this filtering serves two purposes: | |
112 | // | |
113 | // 1. We skip evaluating any predicates that we would | |
114 | // never be able prove are unsatisfiable (e.g. `<T as Foo>` | |
115 | // 2. We avoid trying to normalize predicates involving generic | |
116 | // parameters (e.g. `<T as Foo>::MyItem`). This can confuse | |
117 | // the normalization code (leading to cycle errors), since | |
118 | // it's usually never invoked in this way. | |
119 | let predicates = tcx | |
29967ef6 | 120 | .predicates_of(def_id.to_def_id()) |
dfeec247 XL |
121 | .predicates |
122 | .iter() | |
ba9703b0 | 123 | .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None }); |
3dfed10e | 124 | if traits::impossible_predicates( |
dfeec247 | 125 | tcx, |
ba9703b0 | 126 | traits::elaborate_predicates(tcx, predicates).map(|o| o.predicate).collect(), |
dfeec247 | 127 | ) { |
29967ef6 | 128 | trace!("ConstProp skipped for {:?}: found unsatisfiable predicates", def_id); |
dfeec247 | 129 | return; |
e74abb32 XL |
130 | } |
131 | ||
29967ef6 | 132 | trace!("ConstProp starting for {:?}", def_id); |
0531ce1d | 133 | |
dfeec247 | 134 | let dummy_body = &Body::new( |
29967ef6 | 135 | body.source, |
dfeec247 XL |
136 | body.basic_blocks().clone(), |
137 | body.source_scopes.clone(), | |
138 | body.local_decls.clone(), | |
139 | Default::default(), | |
140 | body.arg_count, | |
141 | Default::default(), | |
5869c6ff | 142 | body.span, |
6a06907d | 143 | body.generator_kind(), |
dfeec247 | 144 | ); |
dc9dc135 | 145 | |
0531ce1d XL |
146 | // FIXME(oli-obk, eddyb) Optimize locals (or even local paths) to hold |
147 | // constants, instead of just checking for const-folding succeeding. | |
148 | // That would require an uniform one-def no-mutation analysis | |
149 | // and RPO (or recursing when needing the value of a local). | |
29967ef6 | 150 | let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); |
dc9dc135 | 151 | optimization_finder.visit_body(body); |
0531ce1d | 152 | |
29967ef6 | 153 | trace!("ConstProp done for {:?}", def_id); |
0531ce1d XL |
154 | } |
155 | } | |
156 | ||
ba9703b0 XL |
157 | struct ConstPropMachine<'mir, 'tcx> { |
158 | /// The virtual call stack. | |
136023e0 | 159 | stack: Vec<Frame<'mir, 'tcx>>, |
f035d41b XL |
160 | /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end. |
161 | written_only_inside_own_block_locals: FxHashSet<Local>, | |
162 | /// Locals that need to be cleared after every block terminates. | |
163 | only_propagate_inside_block_locals: BitSet<Local>, | |
3dfed10e | 164 | can_const_prop: IndexVec<Local, ConstPropMode>, |
ba9703b0 XL |
165 | } |
166 | ||
167 | impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> { | |
3dfed10e XL |
168 | fn new( |
169 | only_propagate_inside_block_locals: BitSet<Local>, | |
170 | can_const_prop: IndexVec<Local, ConstPropMode>, | |
171 | ) -> Self { | |
f035d41b XL |
172 | Self { |
173 | stack: Vec::new(), | |
174 | written_only_inside_own_block_locals: Default::default(), | |
175 | only_propagate_inside_block_locals, | |
3dfed10e | 176 | can_const_prop, |
f035d41b | 177 | } |
ba9703b0 XL |
178 | } |
179 | } | |
e74abb32 | 180 | |
ba9703b0 | 181 | impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { |
f9f354fc | 182 | compile_time_machine!(<'mir, 'tcx>); |
136023e0 | 183 | const PANIC_ON_ALLOC_FAIL: bool = true; // all allocations are small (see `MAX_ALLOC_LIMIT`) |
e74abb32 | 184 | |
fc512014 XL |
185 | type MemoryKind = !; |
186 | ||
e74abb32 | 187 | type MemoryExtra = (); |
e74abb32 | 188 | |
5869c6ff XL |
189 | fn load_mir( |
190 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
191 | _instance: ty::InstanceDef<'tcx>, | |
192 | ) -> InterpResult<'tcx, &'tcx Body<'tcx>> { | |
193 | throw_machine_stop_str!("calling functions isn't supported in ConstProp") | |
194 | } | |
195 | ||
60c5eb7d | 196 | fn find_mir_or_eval_fn( |
e74abb32 XL |
197 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, |
198 | _instance: ty::Instance<'tcx>, | |
5869c6ff | 199 | _abi: Abi, |
e74abb32 | 200 | _args: &[OpTy<'tcx>], |
6a06907d | 201 | _ret: Option<(&PlaceTy<'tcx>, BasicBlock)>, |
17df50a5 | 202 | _unwind: StackPopUnwind, |
e74abb32 XL |
203 | ) -> InterpResult<'tcx, Option<&'mir Body<'tcx>>> { |
204 | Ok(None) | |
205 | } | |
206 | ||
e74abb32 XL |
207 | fn call_intrinsic( |
208 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
e74abb32 XL |
209 | _instance: ty::Instance<'tcx>, |
210 | _args: &[OpTy<'tcx>], | |
6a06907d | 211 | _ret: Option<(&PlaceTy<'tcx>, BasicBlock)>, |
17df50a5 | 212 | _unwind: StackPopUnwind, |
60c5eb7d | 213 | ) -> InterpResult<'tcx> { |
ba9703b0 | 214 | throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp") |
60c5eb7d XL |
215 | } |
216 | ||
217 | fn assert_panic( | |
218 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
ba9703b0 XL |
219 | _msg: &rustc_middle::mir::AssertMessage<'tcx>, |
220 | _unwind: Option<rustc_middle::mir::BasicBlock>, | |
e74abb32 | 221 | ) -> InterpResult<'tcx> { |
ba9703b0 | 222 | bug!("panics terminators are not evaluated in ConstProp") |
e74abb32 XL |
223 | } |
224 | ||
e74abb32 XL |
225 | fn binary_ptr_op( |
226 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
227 | _bin_op: BinOp, | |
6a06907d XL |
228 | _left: &ImmTy<'tcx>, |
229 | _right: &ImmTy<'tcx>, | |
e74abb32 XL |
230 | ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { |
231 | // We can't do this because aliasing of memory can differ between const eval and llvm | |
ba9703b0 | 232 | throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") |
e74abb32 XL |
233 | } |
234 | ||
e74abb32 XL |
235 | fn box_alloc( |
236 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
6a06907d | 237 | _dest: &PlaceTy<'tcx>, |
e74abb32 | 238 | ) -> InterpResult<'tcx> { |
ba9703b0 | 239 | throw_machine_stop_str!("can't const prop heap allocations") |
e74abb32 XL |
240 | } |
241 | ||
242 | fn access_local( | |
243 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
244 | frame: &Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>, | |
245 | local: Local, | |
246 | ) -> InterpResult<'tcx, InterpOperand<Self::PointerTag>> { | |
247 | let l = &frame.locals[local]; | |
248 | ||
249 | if l.value == LocalValue::Uninitialized { | |
ba9703b0 | 250 | throw_machine_stop_str!("tried to access an uninitialized local") |
e74abb32 XL |
251 | } |
252 | ||
253 | l.access() | |
254 | } | |
255 | ||
f035d41b XL |
256 | fn access_local_mut<'a>( |
257 | ecx: &'a mut InterpCx<'mir, 'tcx, Self>, | |
258 | frame: usize, | |
259 | local: Local, | |
260 | ) -> InterpResult<'tcx, Result<&'a mut LocalValue<Self::PointerTag>, MemPlace<Self::PointerTag>>> | |
261 | { | |
3dfed10e XL |
262 | if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation { |
263 | throw_machine_stop_str!("tried to write to a local that is marked as not propagatable") | |
264 | } | |
f035d41b | 265 | if frame == 0 && ecx.machine.only_propagate_inside_block_locals.contains(local) { |
3dfed10e XL |
266 | trace!( |
267 | "mutating local {:?} which is restricted to its block. \ | |
268 | Will remove it from const-prop after block is finished.", | |
269 | local | |
270 | ); | |
f035d41b XL |
271 | ecx.machine.written_only_inside_own_block_locals.insert(local); |
272 | } | |
273 | ecx.machine.stack[frame].locals[local].access_mut() | |
274 | } | |
275 | ||
ba9703b0 | 276 | fn before_access_global( |
dfeec247 | 277 | _memory_extra: &(), |
ba9703b0 | 278 | _alloc_id: AllocId, |
e74abb32 | 279 | allocation: &Allocation<Self::PointerTag, Self::AllocExtra>, |
ba9703b0 XL |
280 | _static_def_id: Option<DefId>, |
281 | is_write: bool, | |
e74abb32 | 282 | ) -> InterpResult<'tcx> { |
ba9703b0 XL |
283 | if is_write { |
284 | throw_machine_stop_str!("can't write to global"); | |
285 | } | |
286 | // If the static allocation is mutable, then we can't const prop it as its content | |
287 | // might be different at runtime. | |
288 | if allocation.mutability == Mutability::Mut { | |
289 | throw_machine_stop_str!("can't access mutable globals in ConstProp"); | |
e74abb32 XL |
290 | } |
291 | ||
292 | Ok(()) | |
293 | } | |
294 | ||
3dfed10e XL |
295 | #[inline(always)] |
296 | fn init_frame_extra( | |
297 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
298 | frame: Frame<'mir, 'tcx>, | |
299 | ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { | |
300 | Ok(frame) | |
301 | } | |
302 | ||
ba9703b0 XL |
303 | #[inline(always)] |
304 | fn stack( | |
305 | ecx: &'a InterpCx<'mir, 'tcx, Self>, | |
306 | ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { | |
307 | &ecx.machine.stack | |
308 | } | |
309 | ||
310 | #[inline(always)] | |
311 | fn stack_mut( | |
312 | ecx: &'a mut InterpCx<'mir, 'tcx, Self>, | |
313 | ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { | |
314 | &mut ecx.machine.stack | |
e74abb32 | 315 | } |
e74abb32 XL |
316 | } |
317 | ||
0531ce1d | 318 | /// Finds optimization opportunities on the MIR. |
dc9dc135 | 319 | struct ConstPropagator<'mir, 'tcx> { |
ba9703b0 | 320 | ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, |
dc9dc135 | 321 | tcx: TyCtxt<'tcx>, |
0531ce1d | 322 | param_env: ParamEnv<'tcx>, |
60c5eb7d XL |
323 | // FIXME(eddyb) avoid cloning these two fields more than once, |
324 | // by accessing them through `ecx` instead. | |
29967ef6 | 325 | source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>, |
48663c56 | 326 | local_decls: IndexVec<Local, LocalDecl<'tcx>>, |
dfeec247 XL |
327 | // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store |
328 | // the last known `SourceInfo` here and just keep revisiting it. | |
329 | source_info: Option<SourceInfo>, | |
0531ce1d XL |
330 | } |
331 | ||
dc9dc135 | 332 | impl<'mir, 'tcx> LayoutOf for ConstPropagator<'mir, 'tcx> { |
48663c56 | 333 | type Ty = Ty<'tcx>; |
ba9703b0 | 334 | type TyAndLayout = Result<TyAndLayout<'tcx>, LayoutError<'tcx>>; |
0531ce1d | 335 | |
ba9703b0 | 336 | fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyAndLayout { |
0531ce1d XL |
337 | self.tcx.layout_of(self.param_env.and(ty)) |
338 | } | |
339 | } | |
340 | ||
dc9dc135 | 341 | impl<'mir, 'tcx> HasDataLayout for ConstPropagator<'mir, 'tcx> { |
0531ce1d XL |
342 | #[inline] |
343 | fn data_layout(&self) -> &TargetDataLayout { | |
344 | &self.tcx.data_layout | |
345 | } | |
346 | } | |
347 | ||
dc9dc135 | 348 | impl<'mir, 'tcx> HasTyCtxt<'tcx> for ConstPropagator<'mir, 'tcx> { |
0531ce1d | 349 | #[inline] |
dc9dc135 | 350 | fn tcx(&self) -> TyCtxt<'tcx> { |
0531ce1d XL |
351 | self.tcx |
352 | } | |
353 | } | |
354 | ||
dc9dc135 | 355 | impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { |
0531ce1d | 356 | fn new( |
f9f354fc | 357 | body: &Body<'tcx>, |
dc9dc135 | 358 | dummy_body: &'mir Body<'tcx>, |
dc9dc135 | 359 | tcx: TyCtxt<'tcx>, |
dc9dc135 | 360 | ) -> ConstPropagator<'mir, 'tcx> { |
29967ef6 | 361 | let def_id = body.source.def_id(); |
dfeec247 | 362 | let substs = &InternalSubsts::identity_for_item(tcx, def_id); |
3dfed10e | 363 | let param_env = tcx.param_env_reveal_all_normalized(def_id); |
dfeec247 | 364 | |
dc9dc135 | 365 | let span = tcx.def_span(def_id); |
3dfed10e XL |
366 | // FIXME: `CanConstProp::check` computes the layout of all locals, return those layouts |
367 | // so we can write them to `ecx.frame_mut().locals.layout, reducing the duplication in | |
368 | // `layout_of` query invocations. | |
369 | let can_const_prop = CanConstProp::check(tcx, param_env, body); | |
f035d41b XL |
370 | let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len()); |
371 | for (l, mode) in can_const_prop.iter_enumerated() { | |
372 | if *mode == ConstPropMode::OnlyInsideOwnBlock { | |
373 | only_propagate_inside_block_locals.insert(l); | |
374 | } | |
375 | } | |
376 | let mut ecx = InterpCx::new( | |
377 | tcx, | |
378 | span, | |
379 | param_env, | |
3dfed10e | 380 | ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), |
f035d41b XL |
381 | (), |
382 | ); | |
dc9dc135 | 383 | |
dfeec247 XL |
384 | let ret = ecx |
385 | .layout_of(body.return_ty().subst(tcx, substs)) | |
386 | .ok() | |
387 | // Don't bother allocating memory for ZST types which have no values | |
388 | // or for large values. | |
389 | .filter(|ret_layout| { | |
390 | !ret_layout.is_zst() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) | |
391 | }) | |
136023e0 XL |
392 | .map(|ret_layout| { |
393 | ecx.allocate(ret_layout, MemoryKind::Stack) | |
394 | .expect("couldn't perform small allocation") | |
395 | .into() | |
396 | }); | |
60c5eb7d | 397 | |
dc9dc135 | 398 | ecx.push_stack_frame( |
60c5eb7d | 399 | Instance::new(def_id, substs), |
dc9dc135 | 400 | dummy_body, |
6a06907d | 401 | ret.as_ref(), |
dfeec247 XL |
402 | StackPopCleanup::None { cleanup: false }, |
403 | ) | |
404 | .expect("failed to push initial stack frame"); | |
48663c56 | 405 | |
0531ce1d | 406 | ConstPropagator { |
94b46f34 | 407 | ecx, |
0531ce1d | 408 | tcx, |
0531ce1d | 409 | param_env, |
60c5eb7d XL |
410 | // FIXME(eddyb) avoid cloning these two fields more than once, |
411 | // by accessing them through `ecx` instead. | |
412 | source_scopes: body.source_scopes.clone(), | |
dc9dc135 XL |
413 | //FIXME(wesleywiser) we can't steal this because `Visitor::super_visit_body()` needs it |
414 | local_decls: body.local_decls.clone(), | |
dfeec247 | 415 | source_info: None, |
0531ce1d XL |
416 | } |
417 | } | |
418 | ||
f9f354fc | 419 | fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { |
f035d41b XL |
420 | let op = match self.ecx.eval_place_to_op(place, None) { |
421 | Ok(op) => op, | |
422 | Err(e) => { | |
423 | trace!("get_const failed: {}", e); | |
424 | return None; | |
425 | } | |
426 | }; | |
60c5eb7d | 427 | |
f9f354fc XL |
428 | // Try to read the local as an immediate so that if it is representable as a scalar, we can |
429 | // handle it as such, but otherwise, just return the value as is. | |
6a06907d | 430 | Some(match self.ecx.try_read_immediate(&op) { |
f035d41b | 431 | Ok(Ok(imm)) => imm.into(), |
f9f354fc | 432 | _ => op, |
f035d41b | 433 | }) |
dc9dc135 XL |
434 | } |
435 | ||
f9f354fc XL |
436 | /// Remove `local` from the pool of `Locals`. Allows writing to them, |
437 | /// but not reading from them anymore. | |
438 | fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { | |
439 | ecx.frame_mut().locals[local] = | |
dfeec247 | 440 | LocalState { value: LocalValue::Uninitialized, layout: Cell::new(None) }; |
dc9dc135 XL |
441 | } |
442 | ||
dfeec247 | 443 | fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> { |
cdc7bbd5 | 444 | source_info.scope.lint_root(&self.source_scopes) |
dfeec247 XL |
445 | } |
446 | ||
74b04a01 | 447 | fn use_ecx<F, T>(&mut self, f: F) -> Option<T> |
94b46f34 | 448 | where |
dc9dc135 | 449 | F: FnOnce(&mut Self) -> InterpResult<'tcx, T>, |
94b46f34 | 450 | { |
ba9703b0 | 451 | match f(self) { |
94b46f34 | 452 | Ok(val) => Some(val), |
8faf50e0 | 453 | Err(error) => { |
3dfed10e | 454 | trace!("InterpCx operation failed: {:?}", error); |
74b04a01 XL |
455 | // Some errors shouldn't come up because creating them causes |
456 | // an allocation, which we should avoid. When that happens, | |
457 | // dedicated error variants should be introduced instead. | |
ba9703b0 | 458 | assert!( |
6a06907d XL |
459 | !error.kind().formatted_string(), |
460 | "const-prop encountered formatting error: {}", | |
74b04a01 XL |
461 | error |
462 | ); | |
94b46f34 | 463 | None |
dfeec247 | 464 | } |
ba9703b0 | 465 | } |
94b46f34 XL |
466 | } |
467 | ||
f9f354fc | 468 | /// Returns the value, if any, of evaluating `c`. |
dfeec247 | 469 | fn eval_constant(&mut self, c: &Constant<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { |
dfeec247 XL |
470 | // FIXME we need to revisit this for #67176 |
471 | if c.needs_subst() { | |
472 | return None; | |
473 | } | |
474 | ||
6a06907d | 475 | match self.ecx.mir_const_to_op(&c.literal, None) { |
dfeec247 | 476 | Ok(op) => Some(op), |
8faf50e0 | 477 | Err(error) => { |
f035d41b | 478 | let tcx = self.ecx.tcx.at(c.span); |
3dfed10e | 479 | let err = ConstEvalErr::new(&self.ecx, error, Some(c.span)); |
dfeec247 | 480 | if let Some(lint_root) = self.lint_root(source_info) { |
6a06907d XL |
481 | let lint_only = match c.literal { |
482 | ConstantKind::Ty(ct) => match ct.val { | |
483 | // Promoteds must lint and not error as the user didn't ask for them | |
cdc7bbd5 XL |
484 | ConstKind::Unevaluated(ty::Unevaluated { |
485 | def: _, | |
486 | substs: _, | |
487 | promoted: Some(_), | |
488 | }) => true, | |
6a06907d XL |
489 | // Out of backwards compatibility we cannot report hard errors in unused |
490 | // generic functions using associated constants of the generic parameters. | |
491 | _ => c.literal.needs_subst(), | |
492 | }, | |
493 | ConstantKind::Val(_, ty) => ty.needs_subst(), | |
dfeec247 XL |
494 | }; |
495 | if lint_only { | |
496 | // Out of backwards compatibility we cannot report hard errors in unused | |
497 | // generic functions using associated constants of the generic parameters. | |
f035d41b | 498 | err.report_as_lint(tcx, "erroneous constant used", lint_root, Some(c.span)); |
dfeec247 | 499 | } else { |
f035d41b | 500 | err.report_as_error(tcx, "erroneous constant used"); |
dfeec247 XL |
501 | } |
502 | } else { | |
f035d41b | 503 | err.report_as_error(tcx, "erroneous constant used"); |
dfeec247 | 504 | } |
8faf50e0 | 505 | None |
dfeec247 | 506 | } |
8faf50e0 XL |
507 | } |
508 | } | |
509 | ||
f9f354fc | 510 | /// Returns the value, if any, of evaluating `place`. |
ba9703b0 | 511 | fn eval_place(&mut self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { |
dc9dc135 | 512 | trace!("eval_place(place={:?})", place); |
74b04a01 | 513 | self.use_ecx(|this| this.ecx.eval_place_to_op(place, None)) |
0531ce1d XL |
514 | } |
515 | ||
f9f354fc XL |
516 | /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant` |
517 | /// or `eval_place`, depending on the variant of `Operand` used. | |
dfeec247 | 518 | fn eval_operand(&mut self, op: &Operand<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { |
0531ce1d | 519 | match *op { |
dfeec247 | 520 | Operand::Constant(ref c) => self.eval_constant(c, source_info), |
ba9703b0 | 521 | Operand::Move(place) | Operand::Copy(place) => self.eval_place(place), |
0531ce1d XL |
522 | } |
523 | } | |
524 | ||
74b04a01 XL |
525 | fn report_assert_as_lint( |
526 | &self, | |
527 | lint: &'static lint::Lint, | |
528 | source_info: SourceInfo, | |
529 | message: &'static str, | |
f9652781 | 530 | panic: AssertKind<impl std::fmt::Debug>, |
136023e0 XL |
531 | ) { |
532 | if let Some(lint_root) = self.lint_root(source_info) { | |
533 | self.tcx.struct_span_lint_hir(lint, lint_root, source_info.span, |lint| { | |
534 | let mut err = lint.build(message); | |
535 | err.span_label(source_info.span, format!("{:?}", panic)); | |
536 | err.emit() | |
537 | }); | |
538 | } | |
74b04a01 | 539 | } |
dfeec247 | 540 | |
74b04a01 XL |
541 | fn check_unary_op( |
542 | &mut self, | |
543 | op: UnOp, | |
544 | arg: &Operand<'tcx>, | |
545 | source_info: SourceInfo, | |
546 | ) -> Option<()> { | |
f035d41b | 547 | if let (val, true) = self.use_ecx(|this| { |
6a06907d XL |
548 | let val = this.ecx.read_immediate(&this.ecx.eval_operand(arg, None)?)?; |
549 | let (_res, overflow, _ty) = this.ecx.overflowing_unary_op(op, &val)?; | |
f035d41b | 550 | Ok((val, overflow)) |
74b04a01 XL |
551 | })? { |
552 | // `AssertKind` only has an `OverflowNeg` variant, so make sure that is | |
553 | // appropriate to use. | |
554 | assert_eq!(op, UnOp::Neg, "Neg is the only UnOp that can overflow"); | |
555 | self.report_assert_as_lint( | |
556 | lint::builtin::ARITHMETIC_OVERFLOW, | |
557 | source_info, | |
558 | "this arithmetic operation will overflow", | |
f035d41b | 559 | AssertKind::OverflowNeg(val.to_const_int()), |
136023e0 XL |
560 | ); |
561 | return None; | |
74b04a01 | 562 | } |
dfeec247 XL |
563 | |
564 | Some(()) | |
565 | } | |
566 | ||
567 | fn check_binary_op( | |
568 | &mut self, | |
569 | op: BinOp, | |
570 | left: &Operand<'tcx>, | |
571 | right: &Operand<'tcx>, | |
572 | source_info: SourceInfo, | |
dfeec247 | 573 | ) -> Option<()> { |
6a06907d XL |
574 | let r = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?)); |
575 | let l = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?)); | |
74b04a01 | 576 | // Check for exceeding shifts *even if* we cannot evaluate the LHS. |
dfeec247 | 577 | if op == BinOp::Shr || op == BinOp::Shl { |
3dfed10e | 578 | let r = r?; |
74b04a01 XL |
579 | // We need the type of the LHS. We cannot use `place_layout` as that is the type |
580 | // of the result, which for checked binops is not the same! | |
581 | let left_ty = left.ty(&self.local_decls, self.tcx); | |
f035d41b | 582 | let left_size = self.ecx.layout_of(left_ty).ok()?.size; |
dfeec247 | 583 | let right_size = r.layout.size; |
74b04a01 | 584 | let r_bits = r.to_scalar().ok(); |
136023e0 | 585 | let r_bits = r_bits.and_then(|r| r.to_bits(right_size).ok()); |
f035d41b XL |
586 | if r_bits.map_or(false, |b| b >= left_size.bits() as u128) { |
587 | debug!("check_binary_op: reporting assert for {:?}", source_info); | |
74b04a01 XL |
588 | self.report_assert_as_lint( |
589 | lint::builtin::ARITHMETIC_OVERFLOW, | |
590 | source_info, | |
591 | "this arithmetic operation will overflow", | |
f035d41b XL |
592 | AssertKind::Overflow( |
593 | op, | |
594 | match l { | |
595 | Some(l) => l.to_const_int(), | |
596 | // Invent a dummy value, the diagnostic ignores it anyway | |
597 | None => ConstInt::new( | |
29967ef6 | 598 | ScalarInt::try_from_uint(1_u8, left_size).unwrap(), |
f035d41b XL |
599 | left_ty.is_signed(), |
600 | left_ty.is_ptr_sized_integral(), | |
601 | ), | |
602 | }, | |
603 | r.to_const_int(), | |
604 | ), | |
136023e0 XL |
605 | ); |
606 | return None; | |
dfeec247 XL |
607 | } |
608 | } | |
609 | ||
6a06907d | 610 | if let (Some(l), Some(r)) = (&l, &r) { |
3dfed10e XL |
611 | // The remaining operators are handled through `overflowing_binary_op`. |
612 | if self.use_ecx(|this| { | |
613 | let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, l, r)?; | |
614 | Ok(overflow) | |
615 | })? { | |
616 | self.report_assert_as_lint( | |
617 | lint::builtin::ARITHMETIC_OVERFLOW, | |
618 | source_info, | |
619 | "this arithmetic operation will overflow", | |
620 | AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()), | |
136023e0 XL |
621 | ); |
622 | return None; | |
3dfed10e | 623 | } |
dfeec247 | 624 | } |
dfeec247 XL |
625 | Some(()) |
626 | } | |
627 | ||
3dfed10e XL |
628 | fn propagate_operand(&mut self, operand: &mut Operand<'tcx>) { |
629 | match *operand { | |
630 | Operand::Copy(l) | Operand::Move(l) => { | |
631 | if let Some(value) = self.get_const(l) { | |
6a06907d | 632 | if self.should_const_prop(&value) { |
3dfed10e XL |
633 | // FIXME(felix91gr): this code only handles `Scalar` cases. |
634 | // For now, we're not handling `ScalarPair` cases because | |
635 | // doing so here would require a lot of code duplication. | |
636 | // We should hopefully generalize `Operand` handling into a fn, | |
637 | // and use it to do const-prop here and everywhere else | |
638 | // where it makes sense. | |
639 | if let interpret::Operand::Immediate(interpret::Immediate::Scalar( | |
640 | ScalarMaybeUninit::Scalar(scalar), | |
641 | )) = *value | |
642 | { | |
643 | *operand = self.operand_from_scalar( | |
644 | scalar, | |
645 | value.layout.ty, | |
646 | self.source_info.unwrap().span, | |
647 | ); | |
648 | } | |
649 | } | |
650 | } | |
651 | } | |
652 | Operand::Constant(_) => (), | |
653 | } | |
654 | } | |
655 | ||
0531ce1d XL |
656 | fn const_prop( |
657 | &mut self, | |
658 | rvalue: &Rvalue<'tcx>, | |
0531ce1d | 659 | source_info: SourceInfo, |
ba9703b0 | 660 | place: Place<'tcx>, |
e74abb32 | 661 | ) -> Option<()> { |
e74abb32 XL |
662 | // Perform any special handling for specific Rvalue types. |
663 | // Generally, checks here fall into one of two categories: | |
664 | // 1. Additional checking to provide useful lints to the user | |
665 | // - In this case, we will do some validation and then fall through to the | |
666 | // end of the function which evals the assignment. | |
667 | // 2. Working around bugs in other parts of the compiler | |
668 | // - In this case, we'll return `None` from this function to stop evaluation. | |
669 | match rvalue { | |
74b04a01 XL |
670 | // Additional checking: give lints to the user if an overflow would occur. |
671 | // We do this here and not in the `Assert` terminator as that terminator is | |
672 | // only sometimes emitted (overflow checks can be disabled), but we want to always | |
673 | // lint. | |
674 | Rvalue::UnaryOp(op, arg) => { | |
675 | trace!("checking UnaryOp(op = {:?}, arg = {:?})", op, arg); | |
676 | self.check_unary_op(*op, arg, source_info)?; | |
0531ce1d | 677 | } |
6a06907d | 678 | Rvalue::BinaryOp(op, box (left, right)) => { |
e74abb32 | 679 | trace!("checking BinaryOp(op = {:?}, left = {:?}, right = {:?})", op, left, right); |
74b04a01 XL |
680 | self.check_binary_op(*op, left, right, source_info)?; |
681 | } | |
6a06907d | 682 | Rvalue::CheckedBinaryOp(op, box (left, right)) => { |
74b04a01 XL |
683 | trace!( |
684 | "checking CheckedBinaryOp(op = {:?}, left = {:?}, right = {:?})", | |
685 | op, | |
686 | left, | |
687 | right | |
688 | ); | |
689 | self.check_binary_op(*op, left, right, source_info)?; | |
e74abb32 XL |
690 | } |
691 | ||
dfeec247 | 692 | // Do not try creating references (#67862) |
f035d41b XL |
693 | Rvalue::AddressOf(_, place) | Rvalue::Ref(_, _, place) => { |
694 | trace!("skipping AddressOf | Ref for {:?}", place); | |
695 | ||
696 | // This may be creating mutable references or immutable references to cells. | |
697 | // If that happens, the pointed to value could be mutated via that reference. | |
698 | // Since we aren't tracking references, the const propagator loses track of what | |
699 | // value the local has right now. | |
700 | // Thus, all locals that have their reference taken | |
701 | // must not take part in propagation. | |
702 | Self::remove_const(&mut self.ecx, place.local); | |
e74abb32 | 703 | |
dfeec247 | 704 | return None; |
e74abb32 | 705 | } |
f035d41b XL |
706 | Rvalue::ThreadLocalRef(def_id) => { |
707 | trace!("skipping ThreadLocalRef({:?})", def_id); | |
e74abb32 | 708 | |
f035d41b XL |
709 | return None; |
710 | } | |
711 | ||
712 | // There's no other checking to do at this time. | |
713 | Rvalue::Aggregate(..) | |
714 | | Rvalue::Use(..) | |
715 | | Rvalue::Repeat(..) | |
716 | | Rvalue::Len(..) | |
717 | | Rvalue::Cast(..) | |
718 | | Rvalue::Discriminant(..) | |
719 | | Rvalue::NullaryOp(..) => {} | |
0531ce1d | 720 | } |
e74abb32 | 721 | |
f9f354fc XL |
722 | // FIXME we need to revisit this for #67176 |
723 | if rvalue.needs_subst() { | |
724 | return None; | |
725 | } | |
726 | ||
6a06907d | 727 | if self.tcx.sess.mir_opt_level() >= 4 { |
3dfed10e XL |
728 | self.eval_rvalue_with_identities(rvalue, place) |
729 | } else { | |
730 | self.use_ecx(|this| this.ecx.eval_rvalue_into_place(rvalue, place)) | |
731 | } | |
732 | } | |
733 | ||
734 | // Attempt to use albegraic identities to eliminate constant expressions | |
735 | fn eval_rvalue_with_identities( | |
736 | &mut self, | |
737 | rvalue: &Rvalue<'tcx>, | |
738 | place: Place<'tcx>, | |
739 | ) -> Option<()> { | |
74b04a01 | 740 | self.use_ecx(|this| { |
3dfed10e | 741 | match rvalue { |
6a06907d XL |
742 | Rvalue::BinaryOp(op, box (left, right)) |
743 | | Rvalue::CheckedBinaryOp(op, box (left, right)) => { | |
3dfed10e XL |
744 | let l = this.ecx.eval_operand(left, None); |
745 | let r = this.ecx.eval_operand(right, None); | |
746 | ||
747 | let const_arg = match (l, r) { | |
6a06907d | 748 | (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?, |
3dfed10e XL |
749 | (Err(e), Err(_)) => return Err(e), |
750 | (Ok(_), Ok(_)) => { | |
751 | this.ecx.eval_rvalue_into_place(rvalue, place)?; | |
752 | return Ok(()); | |
753 | } | |
754 | }; | |
755 | ||
136023e0 | 756 | let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?; |
3dfed10e XL |
757 | let dest = this.ecx.eval_place(place)?; |
758 | ||
759 | match op { | |
760 | BinOp::BitAnd => { | |
761 | if arg_value == 0 { | |
6a06907d | 762 | this.ecx.write_immediate(*const_arg, &dest)?; |
3dfed10e XL |
763 | } |
764 | } | |
765 | BinOp::BitOr => { | |
29967ef6 | 766 | if arg_value == const_arg.layout.size.truncate(u128::MAX) |
3dfed10e XL |
767 | || (const_arg.layout.ty.is_bool() && arg_value == 1) |
768 | { | |
6a06907d | 769 | this.ecx.write_immediate(*const_arg, &dest)?; |
3dfed10e XL |
770 | } |
771 | } | |
772 | BinOp::Mul => { | |
773 | if const_arg.layout.ty.is_integral() && arg_value == 0 { | |
6a06907d | 774 | if let Rvalue::CheckedBinaryOp(_, _) = rvalue { |
3dfed10e XL |
775 | let val = Immediate::ScalarPair( |
776 | const_arg.to_scalar()?.into(), | |
777 | Scalar::from_bool(false).into(), | |
778 | ); | |
6a06907d | 779 | this.ecx.write_immediate(val, &dest)?; |
3dfed10e | 780 | } else { |
6a06907d | 781 | this.ecx.write_immediate(*const_arg, &dest)?; |
3dfed10e XL |
782 | } |
783 | } | |
784 | } | |
785 | _ => { | |
786 | this.ecx.eval_rvalue_into_place(rvalue, place)?; | |
787 | } | |
788 | } | |
789 | } | |
790 | _ => { | |
791 | this.ecx.eval_rvalue_into_place(rvalue, place)?; | |
792 | } | |
793 | } | |
794 | ||
e74abb32 XL |
795 | Ok(()) |
796 | }) | |
0531ce1d | 797 | } |
48663c56 | 798 | |
f9f354fc | 799 | /// Creates a new `Operand::Constant` from a `Scalar` value |
48663c56 | 800 | fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>, span: Span) -> Operand<'tcx> { |
dfeec247 XL |
801 | Operand::Constant(Box::new(Constant { |
802 | span, | |
803 | user_ty: None, | |
6a06907d | 804 | literal: ty::Const::from_scalar(self.tcx, scalar, ty).into(), |
dfeec247 | 805 | })) |
48663c56 XL |
806 | } |
807 | ||
dc9dc135 XL |
808 | fn replace_with_const( |
809 | &mut self, | |
810 | rval: &mut Rvalue<'tcx>, | |
6a06907d | 811 | value: &OpTy<'tcx>, |
dc9dc135 XL |
812 | source_info: SourceInfo, |
813 | ) { | |
f9f354fc | 814 | if let Rvalue::Use(Operand::Constant(c)) = rval { |
6a06907d XL |
815 | match c.literal { |
816 | ConstantKind::Ty(c) if matches!(c.val, ConstKind::Unevaluated(..)) => {} | |
817 | _ => { | |
818 | trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c); | |
819 | return; | |
820 | } | |
f9f354fc XL |
821 | } |
822 | } | |
823 | ||
fc512014 | 824 | trace!("attempting to replace {:?} with {:?}", rval, value); |
74b04a01 | 825 | if let Err(e) = self.ecx.const_validate_operand( |
48663c56 XL |
826 | value, |
827 | vec![], | |
dc9dc135 | 828 | // FIXME: is ref tracking too expensive? |
29967ef6 | 829 | // FIXME: what is the point of ref tracking if we do not even check the tracked refs? |
74b04a01 | 830 | &mut interpret::RefTracking::empty(), |
29967ef6 | 831 | CtfeValidationMode::Regular, |
dc9dc135 XL |
832 | ) { |
833 | trace!("validation error, attempt failed: {:?}", e); | |
834 | return; | |
835 | } | |
836 | ||
74b04a01 XL |
837 | // FIXME> figure out what to do when try_read_immediate fails |
838 | let imm = self.use_ecx(|this| this.ecx.try_read_immediate(value)); | |
48663c56 | 839 | |
dc9dc135 XL |
840 | if let Some(Ok(imm)) = imm { |
841 | match *imm { | |
f9f354fc | 842 | interpret::Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar)) => { |
dfeec247 XL |
843 | *rval = Rvalue::Use(self.operand_from_scalar( |
844 | scalar, | |
845 | value.layout.ty, | |
846 | source_info.span, | |
847 | )); | |
848 | } | |
48663c56 | 849 | Immediate::ScalarPair( |
3dfed10e XL |
850 | ScalarMaybeUninit::Scalar(_), |
851 | ScalarMaybeUninit::Scalar(_), | |
48663c56 | 852 | ) => { |
3dfed10e XL |
853 | // Found a value represented as a pair. For now only do const-prop if the type |
854 | // of `rvalue` is also a tuple with two scalars. | |
855 | // FIXME: enable the general case stated above ^. | |
856 | let ty = &value.layout.ty; | |
60c5eb7d | 857 | // Only do it for tuples |
1b1a35ee | 858 | if let ty::Tuple(substs) = ty.kind() { |
60c5eb7d XL |
859 | // Only do it if tuple is also a pair with two scalars |
860 | if substs.len() == 2 { | |
3dfed10e | 861 | let alloc = self.use_ecx(|this| { |
60c5eb7d XL |
862 | let ty1 = substs[0].expect_ty(); |
863 | let ty2 = substs[1].expect_ty(); | |
864 | let ty_is_scalar = |ty| { | |
ba9703b0 | 865 | this.ecx.layout_of(ty).ok().map(|layout| layout.abi.is_scalar()) |
60c5eb7d XL |
866 | == Some(true) |
867 | }; | |
868 | if ty_is_scalar(ty1) && ty_is_scalar(ty2) { | |
3dfed10e XL |
869 | let alloc = this |
870 | .ecx | |
871 | .intern_with_temp_alloc(value.layout, |ecx, dest| { | |
136023e0 | 872 | ecx.write_immediate(*imm, dest) |
3dfed10e XL |
873 | }) |
874 | .unwrap(); | |
875 | Ok(Some(alloc)) | |
60c5eb7d XL |
876 | } else { |
877 | Ok(None) | |
878 | } | |
879 | }); | |
880 | ||
3dfed10e XL |
881 | if let Some(Some(alloc)) = alloc { |
882 | // Assign entire constant in a single statement. | |
883 | // We can't use aggregates, as we run after the aggregate-lowering `MirPhase`. | |
884 | *rval = Rvalue::Use(Operand::Constant(Box::new(Constant { | |
885 | span: source_info.span, | |
886 | user_ty: None, | |
6a06907d XL |
887 | literal: self |
888 | .ecx | |
889 | .tcx | |
890 | .mk_const(ty::Const { | |
891 | ty, | |
892 | val: ty::ConstKind::Value(ConstValue::ByRef { | |
893 | alloc, | |
894 | offset: Size::ZERO, | |
895 | }), | |
896 | }) | |
897 | .into(), | |
3dfed10e | 898 | }))); |
60c5eb7d XL |
899 | } |
900 | } | |
48663c56 | 901 | } |
dfeec247 | 902 | } |
3dfed10e XL |
903 | // Scalars or scalar pairs that contain undef values are assumed to not have |
904 | // successfully evaluated and are thus not propagated. | |
dfeec247 | 905 | _ => {} |
48663c56 XL |
906 | } |
907 | } | |
908 | } | |
909 | ||
f9f354fc | 910 | /// Returns `true` if and only if this `op` should be const-propagated into. |
6a06907d XL |
911 | fn should_const_prop(&mut self, op: &OpTy<'tcx>) -> bool { |
912 | let mir_opt_level = self.tcx.sess.mir_opt_level(); | |
60c5eb7d XL |
913 | |
914 | if mir_opt_level == 0 { | |
915 | return false; | |
916 | } | |
917 | ||
fc512014 XL |
918 | if !self.tcx.consider_optimizing(|| format!("ConstantPropagation - OpTy: {:?}", op)) { |
919 | return false; | |
920 | } | |
921 | ||
6a06907d | 922 | match **op { |
f9f354fc | 923 | interpret::Operand::Immediate(Immediate::Scalar(ScalarMaybeUninit::Scalar(s))) => { |
136023e0 | 924 | s.try_to_int().is_ok() |
dfeec247 XL |
925 | } |
926 | interpret::Operand::Immediate(Immediate::ScalarPair( | |
f9f354fc XL |
927 | ScalarMaybeUninit::Scalar(l), |
928 | ScalarMaybeUninit::Scalar(r), | |
136023e0 | 929 | )) => l.try_to_int().is_ok() && r.try_to_int().is_ok(), |
dfeec247 | 930 | _ => false, |
60c5eb7d | 931 | } |
48663c56 | 932 | } |
0531ce1d XL |
933 | } |
934 | ||
dfeec247 XL |
935 | /// The mode that `ConstProp` is allowed to run in for a given `Local`. |
936 | #[derive(Clone, Copy, Debug, PartialEq)] | |
937 | enum ConstPropMode { | |
938 | /// The `Local` can be propagated into and reads of this `Local` can also be propagated. | |
939 | FullConstProp, | |
f9f354fc XL |
940 | /// The `Local` can only be propagated into and from its own block. |
941 | OnlyInsideOwnBlock, | |
dfeec247 XL |
942 | /// The `Local` can be propagated into but reads cannot be propagated. |
943 | OnlyPropagateInto, | |
f035d41b XL |
944 | /// The `Local` cannot be part of propagation at all. Any statement |
945 | /// referencing it either for reading or writing will not get propagated. | |
dfeec247 XL |
946 | NoPropagation, |
947 | } | |
948 | ||
0531ce1d | 949 | struct CanConstProp { |
dfeec247 | 950 | can_const_prop: IndexVec<Local, ConstPropMode>, |
f9f354fc XL |
951 | // False at the beginning. Once set, no more assignments are allowed to that local. |
952 | found_assignment: BitSet<Local>, | |
953 | // Cache of locals' information | |
954 | local_kinds: IndexVec<Local, LocalKind>, | |
0531ce1d XL |
955 | } |
956 | ||
957 | impl CanConstProp { | |
f9f354fc | 958 | /// Returns true if `local` can be propagated |
3dfed10e XL |
959 | fn check( |
960 | tcx: TyCtxt<'tcx>, | |
961 | param_env: ParamEnv<'tcx>, | |
962 | body: &Body<'tcx>, | |
963 | ) -> IndexVec<Local, ConstPropMode> { | |
0531ce1d | 964 | let mut cpv = CanConstProp { |
dfeec247 | 965 | can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls), |
f9f354fc XL |
966 | found_assignment: BitSet::new_empty(body.local_decls.len()), |
967 | local_kinds: IndexVec::from_fn_n( | |
968 | |local| body.local_kind(local), | |
969 | body.local_decls.len(), | |
970 | ), | |
0531ce1d XL |
971 | }; |
972 | for (local, val) in cpv.can_const_prop.iter_enumerated_mut() { | |
3dfed10e XL |
973 | let ty = body.local_decls[local].ty; |
974 | match tcx.layout_of(param_env.and(ty)) { | |
975 | Ok(layout) if layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) => {} | |
976 | // Either the layout fails to compute, then we can't use this local anyway | |
977 | // or the local is too large, then we don't want to. | |
978 | _ => { | |
979 | *val = ConstPropMode::NoPropagation; | |
980 | continue; | |
981 | } | |
982 | } | |
f9f354fc XL |
983 | // Cannot use args at all |
984 | // Cannot use locals because if x < y { y - x } else { x - y } would | |
0531ce1d XL |
985 | // lint for x != y |
986 | // FIXME(oli-obk): lint variables until they are used in a condition | |
987 | // FIXME(oli-obk): lint if return value is constant | |
f9f354fc | 988 | if cpv.local_kinds[local] == LocalKind::Arg { |
dfeec247 | 989 | *val = ConstPropMode::OnlyPropagateInto; |
f9f354fc XL |
990 | trace!( |
991 | "local {:?} can't be const propagated because it's a function argument", | |
992 | local | |
993 | ); | |
994 | } else if cpv.local_kinds[local] == LocalKind::Var { | |
995 | *val = ConstPropMode::OnlyInsideOwnBlock; | |
996 | trace!( | |
997 | "local {:?} will only be propagated inside its block, because it's a user variable", | |
998 | local | |
999 | ); | |
dc9dc135 | 1000 | } |
0531ce1d | 1001 | } |
ba9703b0 | 1002 | cpv.visit_body(&body); |
0531ce1d XL |
1003 | cpv.can_const_prop |
1004 | } | |
1005 | } | |
1006 | ||
1007 | impl<'tcx> Visitor<'tcx> for CanConstProp { | |
dfeec247 | 1008 | fn visit_local(&mut self, &local: &Local, context: PlaceContext, _: Location) { |
ba9703b0 | 1009 | use rustc_middle::mir::visit::PlaceContext::*; |
0531ce1d | 1010 | match context { |
f9f354fc XL |
1011 | // Projections are fine, because `&mut foo.x` will be caught by |
1012 | // `MutatingUseContext::Borrow` elsewhere. | |
1013 | MutatingUse(MutatingUseContext::Projection) | |
1014 | // These are just stores, where the storing is not propagatable, but there may be later | |
1015 | // mutations of the same local via `Store` | |
1016 | | MutatingUse(MutatingUseContext::Call) | |
1017 | // Actual store that can possibly even propagate a value | |
1018 | | MutatingUse(MutatingUseContext::Store) => { | |
1019 | if !self.found_assignment.insert(local) { | |
1020 | match &mut self.can_const_prop[local] { | |
1021 | // If the local can only get propagated in its own block, then we don't have | |
1022 | // to worry about multiple assignments, as we'll nuke the const state at the | |
1023 | // end of the block anyway, and inside the block we overwrite previous | |
1024 | // states as applicable. | |
1025 | ConstPropMode::OnlyInsideOwnBlock => {} | |
f035d41b XL |
1026 | ConstPropMode::NoPropagation => {} |
1027 | ConstPropMode::OnlyPropagateInto => {} | |
1028 | other @ ConstPropMode::FullConstProp => { | |
f9f354fc | 1029 | trace!( |
3dfed10e XL |
1030 | "local {:?} can't be propagated because of multiple assignments. Previous state: {:?}", |
1031 | local, other, | |
f9f354fc | 1032 | ); |
3dfed10e | 1033 | *other = ConstPropMode::OnlyInsideOwnBlock; |
f9f354fc XL |
1034 | } |
1035 | } | |
dfeec247 XL |
1036 | } |
1037 | } | |
0531ce1d | 1038 | // Reading constants is allowed an arbitrary number of times |
dfeec247 XL |
1039 | NonMutatingUse(NonMutatingUseContext::Copy) |
1040 | | NonMutatingUse(NonMutatingUseContext::Move) | |
1041 | | NonMutatingUse(NonMutatingUseContext::Inspect) | |
1042 | | NonMutatingUse(NonMutatingUseContext::Projection) | |
dfeec247 | 1043 | | NonUse(_) => {} |
f9f354fc XL |
1044 | |
1045 | // These could be propagated with a smarter analysis or just some careful thinking about | |
1046 | // whether they'd be fine right now. | |
1047 | MutatingUse(MutatingUseContext::AsmOutput) | |
1048 | | MutatingUse(MutatingUseContext::Yield) | |
1049 | | MutatingUse(MutatingUseContext::Drop) | |
1050 | | MutatingUse(MutatingUseContext::Retag) | |
1051 | // These can't ever be propagated under any scheme, as we can't reason about indirect | |
1052 | // mutation. | |
1053 | | NonMutatingUse(NonMutatingUseContext::SharedBorrow) | |
1054 | | NonMutatingUse(NonMutatingUseContext::ShallowBorrow) | |
1055 | | NonMutatingUse(NonMutatingUseContext::UniqueBorrow) | |
1056 | | NonMutatingUse(NonMutatingUseContext::AddressOf) | |
1057 | | MutatingUse(MutatingUseContext::Borrow) | |
1058 | | MutatingUse(MutatingUseContext::AddressOf) => { | |
dc9dc135 | 1059 | trace!("local {:?} can't be propagaged because it's used: {:?}", local, context); |
dfeec247 XL |
1060 | self.can_const_prop[local] = ConstPropMode::NoPropagation; |
1061 | } | |
0531ce1d XL |
1062 | } |
1063 | } | |
1064 | } | |
1065 | ||
dc9dc135 | 1066 | impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> { |
e74abb32 XL |
1067 | fn tcx(&self) -> TyCtxt<'tcx> { |
1068 | self.tcx | |
1069 | } | |
1070 | ||
f9f354fc XL |
1071 | fn visit_body(&mut self, body: &mut Body<'tcx>) { |
1072 | for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { | |
1073 | self.visit_basic_block_data(bb, data); | |
1074 | } | |
1075 | } | |
1076 | ||
3dfed10e XL |
1077 | fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { |
1078 | self.super_operand(operand, location); | |
1079 | ||
6a06907d | 1080 | // Only const prop copies and moves on `mir_opt_level=3` as doing so |
1b1a35ee | 1081 | // currently slightly increases compile time in some cases. |
6a06907d | 1082 | if self.tcx.sess.mir_opt_level() >= 3 { |
3dfed10e XL |
1083 | self.propagate_operand(operand) |
1084 | } | |
1085 | } | |
1086 | ||
dfeec247 | 1087 | fn visit_constant(&mut self, constant: &mut Constant<'tcx>, location: Location) { |
0531ce1d XL |
1088 | trace!("visit_constant: {:?}", constant); |
1089 | self.super_constant(constant, location); | |
dfeec247 | 1090 | self.eval_constant(constant, self.source_info.unwrap()); |
0531ce1d XL |
1091 | } |
1092 | ||
dfeec247 | 1093 | fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { |
0531ce1d | 1094 | trace!("visit_statement: {:?}", statement); |
dfeec247 XL |
1095 | let source_info = statement.source_info; |
1096 | self.source_info = Some(source_info); | |
ba9703b0 | 1097 | if let StatementKind::Assign(box (place, ref mut rval)) = statement.kind { |
3dfed10e XL |
1098 | let can_const_prop = self.ecx.machine.can_const_prop[place.local]; |
1099 | if let Some(()) = self.const_prop(rval, source_info, place) { | |
1100 | // This will return None if the above `const_prop` invocation only "wrote" a | |
1101 | // type whose creation requires no write. E.g. a generator whose initial state | |
1102 | // consists solely of uninitialized memory (so it doesn't capture any locals). | |
6a06907d | 1103 | if let Some(ref value) = self.get_const(place) { |
3dfed10e XL |
1104 | if self.should_const_prop(value) { |
1105 | trace!("replacing {:?} with {:?}", rval, value); | |
1106 | self.replace_with_const(rval, value, source_info); | |
1107 | if can_const_prop == ConstPropMode::FullConstProp | |
1108 | || can_const_prop == ConstPropMode::OnlyInsideOwnBlock | |
1109 | { | |
1110 | trace!("propagated into {:?}", place); | |
dfeec247 XL |
1111 | } |
1112 | } | |
3dfed10e XL |
1113 | } |
1114 | match can_const_prop { | |
1115 | ConstPropMode::OnlyInsideOwnBlock => { | |
1116 | trace!( | |
1117 | "found local restricted to its block. \ | |
f035d41b | 1118 | Will remove it from const-prop after block is finished. Local: {:?}", |
3dfed10e XL |
1119 | place.local |
1120 | ); | |
1121 | } | |
1122 | ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { | |
1123 | trace!("can't propagate into {:?}", place); | |
1124 | if place.local != RETURN_PLACE { | |
1125 | Self::remove_const(&mut self.ecx, place.local); | |
8faf50e0 | 1126 | } |
0531ce1d | 1127 | } |
3dfed10e | 1128 | ConstPropMode::FullConstProp => {} |
0531ce1d | 1129 | } |
f035d41b | 1130 | } else { |
3dfed10e XL |
1131 | // Const prop failed, so erase the destination, ensuring that whatever happens |
1132 | // from here on, does not know about the previous value. | |
1133 | // This is important in case we have | |
1134 | // ```rust | |
1135 | // let mut x = 42; | |
1136 | // x = SOME_MUTABLE_STATIC; | |
1137 | // // x must now be uninit | |
1138 | // ``` | |
1139 | // FIXME: we overzealously erase the entire local, because that's easier to | |
1140 | // implement. | |
f035d41b | 1141 | trace!( |
3dfed10e XL |
1142 | "propagation into {:?} failed. |
1143 | Nuking the entire site from orbit, it's the only way to be sure", | |
f035d41b XL |
1144 | place, |
1145 | ); | |
1146 | Self::remove_const(&mut self.ecx, place.local); | |
0531ce1d | 1147 | } |
e74abb32 XL |
1148 | } else { |
1149 | match statement.kind { | |
3dfed10e XL |
1150 | StatementKind::SetDiscriminant { ref place, .. } => { |
1151 | match self.ecx.machine.can_const_prop[place.local] { | |
1152 | ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { | |
1153 | if self.use_ecx(|this| this.ecx.statement(statement)).is_some() { | |
1154 | trace!("propped discriminant into {:?}", place); | |
1155 | } else { | |
1156 | Self::remove_const(&mut self.ecx, place.local); | |
1157 | } | |
1158 | } | |
1159 | ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { | |
1160 | Self::remove_const(&mut self.ecx, place.local); | |
1161 | } | |
1162 | } | |
1163 | } | |
dfeec247 | 1164 | StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { |
e74abb32 XL |
1165 | let frame = self.ecx.frame_mut(); |
1166 | frame.locals[local].value = | |
1167 | if let StatementKind::StorageLive(_) = statement.kind { | |
1168 | LocalValue::Uninitialized | |
1169 | } else { | |
1170 | LocalValue::Dead | |
1171 | }; | |
1172 | } | |
1173 | _ => {} | |
1174 | } | |
0531ce1d | 1175 | } |
e74abb32 | 1176 | |
48663c56 | 1177 | self.super_statement(statement, location); |
0531ce1d XL |
1178 | } |
1179 | ||
dfeec247 | 1180 | fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { |
48663c56 | 1181 | let source_info = terminator.source_info; |
dfeec247 XL |
1182 | self.source_info = Some(source_info); |
1183 | self.super_terminator(terminator, location); | |
48663c56 | 1184 | match &mut terminator.kind { |
416331ca | 1185 | TerminatorKind::Assert { expected, ref msg, ref mut cond, .. } => { |
6a06907d | 1186 | if let Some(ref value) = self.eval_operand(&cond, source_info) { |
48663c56 | 1187 | trace!("assertion on {:?} should be {:?}", value, expected); |
f9f354fc | 1188 | let expected = ScalarMaybeUninit::from(Scalar::from_bool(*expected)); |
6a06907d | 1189 | let value_const = self.ecx.read_scalar(&value).unwrap(); |
48663c56 | 1190 | if expected != value_const { |
f9652781 XL |
1191 | enum DbgVal<T> { |
1192 | Val(T), | |
1193 | Underscore, | |
1194 | } | |
1195 | impl<T: std::fmt::Debug> std::fmt::Debug for DbgVal<T> { | |
1196 | fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | |
1197 | match self { | |
1198 | Self::Val(val) => val.fmt(fmt), | |
1199 | Self::Underscore => fmt.write_str("_"), | |
1200 | } | |
1201 | } | |
1202 | } | |
f035d41b | 1203 | let mut eval_to_int = |op| { |
f9652781 XL |
1204 | // This can be `None` if the lhs wasn't const propagated and we just |
1205 | // triggered the assert on the value of the rhs. | |
136023e0 XL |
1206 | self.eval_operand(op, source_info).map_or(DbgVal::Underscore, |op| { |
1207 | DbgVal::Val(self.ecx.read_immediate(&op).unwrap().to_const_int()) | |
1208 | }) | |
f035d41b XL |
1209 | }; |
1210 | let msg = match msg { | |
1211 | AssertKind::DivisionByZero(op) => { | |
1212 | Some(AssertKind::DivisionByZero(eval_to_int(op))) | |
1213 | } | |
1214 | AssertKind::RemainderByZero(op) => { | |
1215 | Some(AssertKind::RemainderByZero(eval_to_int(op))) | |
1216 | } | |
1217 | AssertKind::BoundsCheck { ref len, ref index } => { | |
1218 | let len = eval_to_int(len); | |
1219 | let index = eval_to_int(index); | |
1220 | Some(AssertKind::BoundsCheck { len, index }) | |
1221 | } | |
1222 | // Overflow is are already covered by checks on the binary operators. | |
1223 | AssertKind::Overflow(..) | AssertKind::OverflowNeg(_) => None, | |
1224 | // Need proper const propagator for these. | |
1225 | _ => None, | |
1226 | }; | |
f9f354fc | 1227 | // Poison all places this operand references so that further code |
48663c56 XL |
1228 | // doesn't use the invalid value |
1229 | match cond { | |
1230 | Operand::Move(ref place) | Operand::Copy(ref place) => { | |
f9f354fc | 1231 | Self::remove_const(&mut self.ecx, place.local); |
dfeec247 | 1232 | } |
48663c56 XL |
1233 | Operand::Constant(_) => {} |
1234 | } | |
f035d41b XL |
1235 | if let Some(msg) = msg { |
1236 | self.report_assert_as_lint( | |
1237 | lint::builtin::UNCONDITIONAL_PANIC, | |
1238 | source_info, | |
1239 | "this operation will panic at runtime", | |
1240 | msg, | |
1241 | ); | |
1242 | } | |
48663c56 | 1243 | } else { |
60c5eb7d | 1244 | if self.should_const_prop(value) { |
f9f354fc | 1245 | if let ScalarMaybeUninit::Scalar(scalar) = value_const { |
48663c56 XL |
1246 | *cond = self.operand_from_scalar( |
1247 | scalar, | |
1248 | self.tcx.types.bool, | |
1249 | source_info.span, | |
1250 | ); | |
0531ce1d | 1251 | } |
48663c56 | 1252 | } |
0531ce1d | 1253 | } |
0531ce1d | 1254 | } |
dfeec247 | 1255 | } |
3dfed10e XL |
1256 | TerminatorKind::SwitchInt { ref mut discr, .. } => { |
1257 | // FIXME: This is currently redundant with `visit_operand`, but sadly | |
1258 | // always visiting operands currently causes a perf regression in LLVM codegen, so | |
6a06907d | 1259 | // `visit_operand` currently only runs for propagates places for `mir_opt_level=4`. |
3dfed10e | 1260 | self.propagate_operand(discr) |
dfeec247 | 1261 | } |
3dfed10e | 1262 | // None of these have Operands to const-propagate. |
dfeec247 XL |
1263 | TerminatorKind::Goto { .. } |
1264 | | TerminatorKind::Resume | |
1265 | | TerminatorKind::Abort | |
1266 | | TerminatorKind::Return | |
1267 | | TerminatorKind::Unreachable | |
1268 | | TerminatorKind::Drop { .. } | |
1269 | | TerminatorKind::DropAndReplace { .. } | |
1270 | | TerminatorKind::Yield { .. } | |
1271 | | TerminatorKind::GeneratorDrop | |
f035d41b | 1272 | | TerminatorKind::FalseEdge { .. } |
f9f354fc XL |
1273 | | TerminatorKind::FalseUnwind { .. } |
1274 | | TerminatorKind::InlineAsm { .. } => {} | |
3dfed10e XL |
1275 | // Every argument in our function calls have already been propagated in `visit_operand`. |
1276 | // | |
1b1a35ee | 1277 | // NOTE: because LLVM codegen gives slight performance regressions with it, so this is |
6a06907d | 1278 | // gated on `mir_opt_level=3`. |
3dfed10e | 1279 | TerminatorKind::Call { .. } => {} |
f9f354fc | 1280 | } |
f035d41b XL |
1281 | |
1282 | // We remove all Locals which are restricted in propagation to their containing blocks and | |
1283 | // which were modified in the current block. | |
3dfed10e | 1284 | // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. |
f035d41b XL |
1285 | let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); |
1286 | for &local in locals.iter() { | |
f9f354fc | 1287 | Self::remove_const(&mut self.ecx, local); |
0531ce1d | 1288 | } |
f035d41b XL |
1289 | locals.clear(); |
1290 | // Put it back so we reuse the heap of the storage | |
1291 | self.ecx.machine.written_only_inside_own_block_locals = locals; | |
1292 | if cfg!(debug_assertions) { | |
1293 | // Ensure we are correctly erasing locals with the non-debug-assert logic. | |
1294 | for local in self.ecx.machine.only_propagate_inside_block_locals.iter() { | |
1295 | assert!( | |
1296 | self.get_const(local.into()).is_none() | |
1297 | || self | |
1298 | .layout_of(self.local_decls[local].ty) | |
1299 | .map_or(true, |layout| layout.is_zst()) | |
1300 | ) | |
1301 | } | |
1302 | } | |
0531ce1d XL |
1303 | } |
1304 | } |