]>
Commit | Line | Data |
---|---|---|
0531ce1d XL |
1 | //! Propagates constants for early reporting of statically known |
2 | //! assertion failures | |
3 | ||
dc9dc135 XL |
4 | use std::cell::Cell; |
5 | ||
3dfed10e | 6 | use rustc_ast::Mutability; |
f035d41b | 7 | use rustc_data_structures::fx::FxHashSet; |
ba9703b0 XL |
8 | use rustc_hir::def::DefKind; |
9 | use rustc_hir::HirId; | |
f9f354fc | 10 | use rustc_index::bit_set::BitSet; |
ba9703b0 | 11 | use rustc_index::vec::IndexVec; |
ba9703b0 | 12 | use rustc_middle::mir::visit::{ |
dfeec247 XL |
13 | MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor, |
14 | }; | |
ba9703b0 | 15 | use rustc_middle::mir::{ |
3dfed10e XL |
16 | AssertKind, BasicBlock, BinOp, Body, ClearCrossCrate, Constant, Local, LocalDecl, LocalKind, |
17 | Location, Operand, Place, Rvalue, SourceInfo, SourceScope, SourceScopeData, Statement, | |
18 | StatementKind, Terminator, TerminatorKind, UnOp, RETURN_PLACE, | |
48663c56 | 19 | }; |
ba9703b0 XL |
20 | use rustc_middle::ty::layout::{HasTyCtxt, LayoutError, TyAndLayout}; |
21 | use rustc_middle::ty::subst::{InternalSubsts, Subst}; | |
29967ef6 XL |
22 | use rustc_middle::ty::{ |
23 | self, ConstInt, ConstKind, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeFoldable, | |
24 | }; | |
ba9703b0 XL |
25 | use rustc_session::lint; |
26 | use rustc_span::{def_id::DefId, Span}; | |
27 | use rustc_target::abi::{HasDataLayout, LayoutOf, Size, TargetDataLayout}; | |
28 | use rustc_trait_selection::traits; | |
0531ce1d | 29 | |
3dfed10e | 30 | use crate::const_eval::ConstEvalErr; |
dc9dc135 | 31 | use crate::interpret::{ |
29967ef6 XL |
32 | self, compile_time_machine, AllocId, Allocation, ConstValue, CtfeValidationMode, Frame, ImmTy, |
33 | Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, Memory, MemoryKind, OpTy, | |
34 | Operand as InterpOperand, PlaceTy, Pointer, Scalar, ScalarMaybeUninit, StackPopCleanup, | |
a1dfa0c6 | 35 | }; |
29967ef6 | 36 | use crate::transform::MirPass; |
a1dfa0c6 | 37 | |
3dfed10e XL |
38 | /// The maximum number of bytes that we'll allocate space for a local or the return value. |
39 | /// Needed for #66397, because otherwise we eval into large places and that can cause OOM or just | |
40 | /// Severely regress performance. | |
e74abb32 XL |
41 | const MAX_ALLOC_LIMIT: u64 = 1024; |
42 | ||
ba9703b0 XL |
43 | /// Macro for machine-specific `InterpError` without allocation. |
44 | /// (These will never be shown to the user, but they help diagnose ICEs.) | |
45 | macro_rules! throw_machine_stop_str { | |
46 | ($($tt:tt)*) => {{ | |
47 | // We make a new local type for it. The type itself does not carry any information, | |
48 | // but its vtable (for the `MachineStopType` trait) does. | |
49 | struct Zst; | |
f9f354fc XL |
50 | // Printing this type shows the desired string. |
51 | impl std::fmt::Display for Zst { | |
ba9703b0 XL |
52 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
53 | write!(f, $($tt)*) | |
54 | } | |
55 | } | |
56 | impl rustc_middle::mir::interpret::MachineStopType for Zst {} | |
57 | throw_machine_stop!(Zst) | |
58 | }}; | |
59 | } | |
60 | ||
0531ce1d XL |
61 | pub struct ConstProp; |
62 | ||
e1599b0c | 63 | impl<'tcx> MirPass<'tcx> for ConstProp { |
29967ef6 | 64 | fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
0531ce1d | 65 | // will be evaluated by miri and produce its errors there |
29967ef6 | 66 | if body.source.promoted.is_some() { |
0531ce1d XL |
67 | return; |
68 | } | |
a1dfa0c6 | 69 | |
ba9703b0 | 70 | use rustc_middle::hir::map::blocks::FnLikeNode; |
29967ef6 XL |
71 | let def_id = body.source.def_id().expect_local(); |
72 | let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); | |
a1dfa0c6 | 73 | |
dc9dc135 | 74 | let is_fn_like = FnLikeNode::from_node(tcx.hir().get(hir_id)).is_some(); |
29967ef6 | 75 | let is_assoc_const = tcx.def_kind(def_id.to_def_id()) == DefKind::AssocConst; |
a1dfa0c6 XL |
76 | |
77 | // Only run const prop on functions, methods, closures and associated constants | |
dfeec247 | 78 | if !is_fn_like && !is_assoc_const { |
a1dfa0c6 | 79 | // skip anon_const/statics/consts because they'll be evaluated by miri anyway |
29967ef6 | 80 | trace!("ConstProp skipped for {:?}", def_id); |
dfeec247 | 81 | return; |
0531ce1d | 82 | } |
a1dfa0c6 | 83 | |
29967ef6 | 84 | let is_generator = tcx.type_of(def_id.to_def_id()).is_generator(); |
e74abb32 XL |
85 | // FIXME(welseywiser) const prop doesn't work on generators because of query cycles |
86 | // computing their layout. | |
87 | if is_generator { | |
29967ef6 | 88 | trace!("ConstProp skipped for generator {:?}", def_id); |
dfeec247 XL |
89 | return; |
90 | } | |
91 | ||
92 | // Check if it's even possible to satisfy the 'where' clauses | |
93 | // for this item. | |
94 | // This branch will never be taken for any normal function. | |
95 | // However, it's possible to `#!feature(trivial_bounds)]` to write | |
96 | // a function with impossible to satisfy clauses, e.g.: | |
97 | // `fn foo() where String: Copy {}` | |
98 | // | |
99 | // We don't usually need to worry about this kind of case, | |
100 | // since we would get a compilation error if the user tried | |
101 | // to call it. However, since we can do const propagation | |
102 | // even without any calls to the function, we need to make | |
103 | // sure that it even makes sense to try to evaluate the body. | |
104 | // If there are unsatisfiable where clauses, then all bets are | |
105 | // off, and we just give up. | |
106 | // | |
107 | // We manually filter the predicates, skipping anything that's not | |
108 | // "global". We are in a potentially generic context | |
109 | // (e.g. we are evaluating a function without substituting generic | |
110 | // parameters, so this filtering serves two purposes: | |
111 | // | |
112 | // 1. We skip evaluating any predicates that we would | |
113 | // never be able prove are unsatisfiable (e.g. `<T as Foo>` | |
114 | // 2. We avoid trying to normalize predicates involving generic | |
115 | // parameters (e.g. `<T as Foo>::MyItem`). This can confuse | |
116 | // the normalization code (leading to cycle errors), since | |
117 | // it's usually never invoked in this way. | |
118 | let predicates = tcx | |
29967ef6 | 119 | .predicates_of(def_id.to_def_id()) |
dfeec247 XL |
120 | .predicates |
121 | .iter() | |
ba9703b0 | 122 | .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None }); |
3dfed10e | 123 | if traits::impossible_predicates( |
dfeec247 | 124 | tcx, |
ba9703b0 | 125 | traits::elaborate_predicates(tcx, predicates).map(|o| o.predicate).collect(), |
dfeec247 | 126 | ) { |
29967ef6 | 127 | trace!("ConstProp skipped for {:?}: found unsatisfiable predicates", def_id); |
dfeec247 | 128 | return; |
e74abb32 XL |
129 | } |
130 | ||
29967ef6 | 131 | trace!("ConstProp starting for {:?}", def_id); |
0531ce1d | 132 | |
dfeec247 | 133 | let dummy_body = &Body::new( |
29967ef6 | 134 | body.source, |
dfeec247 XL |
135 | body.basic_blocks().clone(), |
136 | body.source_scopes.clone(), | |
137 | body.local_decls.clone(), | |
138 | Default::default(), | |
139 | body.arg_count, | |
140 | Default::default(), | |
29967ef6 | 141 | tcx.def_span(def_id), |
dfeec247 XL |
142 | body.generator_kind, |
143 | ); | |
dc9dc135 | 144 | |
0531ce1d XL |
145 | // FIXME(oli-obk, eddyb) Optimize locals (or even local paths) to hold |
146 | // constants, instead of just checking for const-folding succeeding. | |
147 | // That would require an uniform one-def no-mutation analysis | |
148 | // and RPO (or recursing when needing the value of a local). | |
29967ef6 | 149 | let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); |
dc9dc135 | 150 | optimization_finder.visit_body(body); |
0531ce1d | 151 | |
29967ef6 | 152 | trace!("ConstProp done for {:?}", def_id); |
0531ce1d XL |
153 | } |
154 | } | |
155 | ||
ba9703b0 XL |
156 | struct ConstPropMachine<'mir, 'tcx> { |
157 | /// The virtual call stack. | |
158 | stack: Vec<Frame<'mir, 'tcx, (), ()>>, | |
f035d41b XL |
159 | /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end. |
160 | written_only_inside_own_block_locals: FxHashSet<Local>, | |
161 | /// Locals that need to be cleared after every block terminates. | |
162 | only_propagate_inside_block_locals: BitSet<Local>, | |
3dfed10e | 163 | can_const_prop: IndexVec<Local, ConstPropMode>, |
ba9703b0 XL |
164 | } |
165 | ||
166 | impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> { | |
3dfed10e XL |
167 | fn new( |
168 | only_propagate_inside_block_locals: BitSet<Local>, | |
169 | can_const_prop: IndexVec<Local, ConstPropMode>, | |
170 | ) -> Self { | |
f035d41b XL |
171 | Self { |
172 | stack: Vec::new(), | |
173 | written_only_inside_own_block_locals: Default::default(), | |
174 | only_propagate_inside_block_locals, | |
3dfed10e | 175 | can_const_prop, |
f035d41b | 176 | } |
ba9703b0 XL |
177 | } |
178 | } | |
e74abb32 | 179 | |
ba9703b0 | 180 | impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { |
f9f354fc | 181 | compile_time_machine!(<'mir, 'tcx>); |
e74abb32 | 182 | |
fc512014 XL |
183 | type MemoryKind = !; |
184 | ||
e74abb32 | 185 | type MemoryExtra = (); |
e74abb32 | 186 | |
60c5eb7d | 187 | fn find_mir_or_eval_fn( |
e74abb32 XL |
188 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, |
189 | _instance: ty::Instance<'tcx>, | |
190 | _args: &[OpTy<'tcx>], | |
60c5eb7d XL |
191 | _ret: Option<(PlaceTy<'tcx>, BasicBlock)>, |
192 | _unwind: Option<BasicBlock>, | |
e74abb32 XL |
193 | ) -> InterpResult<'tcx, Option<&'mir Body<'tcx>>> { |
194 | Ok(None) | |
195 | } | |
196 | ||
e74abb32 XL |
197 | fn call_intrinsic( |
198 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
e74abb32 XL |
199 | _instance: ty::Instance<'tcx>, |
200 | _args: &[OpTy<'tcx>], | |
60c5eb7d | 201 | _ret: Option<(PlaceTy<'tcx>, BasicBlock)>, |
dfeec247 | 202 | _unwind: Option<BasicBlock>, |
60c5eb7d | 203 | ) -> InterpResult<'tcx> { |
ba9703b0 | 204 | throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp") |
60c5eb7d XL |
205 | } |
206 | ||
207 | fn assert_panic( | |
208 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
ba9703b0 XL |
209 | _msg: &rustc_middle::mir::AssertMessage<'tcx>, |
210 | _unwind: Option<rustc_middle::mir::BasicBlock>, | |
e74abb32 | 211 | ) -> InterpResult<'tcx> { |
ba9703b0 | 212 | bug!("panics terminators are not evaluated in ConstProp") |
e74abb32 XL |
213 | } |
214 | ||
dfeec247 | 215 | fn ptr_to_int(_mem: &Memory<'mir, 'tcx, Self>, _ptr: Pointer) -> InterpResult<'tcx, u64> { |
ba9703b0 | 216 | throw_unsup!(ReadPointerAsBytes) |
e74abb32 XL |
217 | } |
218 | ||
219 | fn binary_ptr_op( | |
220 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
221 | _bin_op: BinOp, | |
222 | _left: ImmTy<'tcx>, | |
223 | _right: ImmTy<'tcx>, | |
224 | ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { | |
225 | // We can't do this because aliasing of memory can differ between const eval and llvm | |
ba9703b0 | 226 | throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") |
e74abb32 XL |
227 | } |
228 | ||
e74abb32 XL |
229 | fn box_alloc( |
230 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
231 | _dest: PlaceTy<'tcx>, | |
232 | ) -> InterpResult<'tcx> { | |
ba9703b0 | 233 | throw_machine_stop_str!("can't const prop heap allocations") |
e74abb32 XL |
234 | } |
235 | ||
236 | fn access_local( | |
237 | _ecx: &InterpCx<'mir, 'tcx, Self>, | |
238 | frame: &Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>, | |
239 | local: Local, | |
240 | ) -> InterpResult<'tcx, InterpOperand<Self::PointerTag>> { | |
241 | let l = &frame.locals[local]; | |
242 | ||
243 | if l.value == LocalValue::Uninitialized { | |
ba9703b0 | 244 | throw_machine_stop_str!("tried to access an uninitialized local") |
e74abb32 XL |
245 | } |
246 | ||
247 | l.access() | |
248 | } | |
249 | ||
f035d41b XL |
250 | fn access_local_mut<'a>( |
251 | ecx: &'a mut InterpCx<'mir, 'tcx, Self>, | |
252 | frame: usize, | |
253 | local: Local, | |
254 | ) -> InterpResult<'tcx, Result<&'a mut LocalValue<Self::PointerTag>, MemPlace<Self::PointerTag>>> | |
255 | { | |
3dfed10e XL |
256 | if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation { |
257 | throw_machine_stop_str!("tried to write to a local that is marked as not propagatable") | |
258 | } | |
f035d41b | 259 | if frame == 0 && ecx.machine.only_propagate_inside_block_locals.contains(local) { |
3dfed10e XL |
260 | trace!( |
261 | "mutating local {:?} which is restricted to its block. \ | |
262 | Will remove it from const-prop after block is finished.", | |
263 | local | |
264 | ); | |
f035d41b XL |
265 | ecx.machine.written_only_inside_own_block_locals.insert(local); |
266 | } | |
267 | ecx.machine.stack[frame].locals[local].access_mut() | |
268 | } | |
269 | ||
ba9703b0 | 270 | fn before_access_global( |
dfeec247 | 271 | _memory_extra: &(), |
ba9703b0 | 272 | _alloc_id: AllocId, |
e74abb32 | 273 | allocation: &Allocation<Self::PointerTag, Self::AllocExtra>, |
ba9703b0 XL |
274 | _static_def_id: Option<DefId>, |
275 | is_write: bool, | |
e74abb32 | 276 | ) -> InterpResult<'tcx> { |
ba9703b0 XL |
277 | if is_write { |
278 | throw_machine_stop_str!("can't write to global"); | |
279 | } | |
280 | // If the static allocation is mutable, then we can't const prop it as its content | |
281 | // might be different at runtime. | |
282 | if allocation.mutability == Mutability::Mut { | |
283 | throw_machine_stop_str!("can't access mutable globals in ConstProp"); | |
e74abb32 XL |
284 | } |
285 | ||
286 | Ok(()) | |
287 | } | |
288 | ||
3dfed10e XL |
289 | #[inline(always)] |
290 | fn init_frame_extra( | |
291 | _ecx: &mut InterpCx<'mir, 'tcx, Self>, | |
292 | frame: Frame<'mir, 'tcx>, | |
293 | ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { | |
294 | Ok(frame) | |
295 | } | |
296 | ||
ba9703b0 XL |
297 | #[inline(always)] |
298 | fn stack( | |
299 | ecx: &'a InterpCx<'mir, 'tcx, Self>, | |
300 | ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { | |
301 | &ecx.machine.stack | |
302 | } | |
303 | ||
304 | #[inline(always)] | |
305 | fn stack_mut( | |
306 | ecx: &'a mut InterpCx<'mir, 'tcx, Self>, | |
307 | ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { | |
308 | &mut ecx.machine.stack | |
e74abb32 | 309 | } |
e74abb32 XL |
310 | } |
311 | ||
0531ce1d | 312 | /// Finds optimization opportunities on the MIR. |
dc9dc135 | 313 | struct ConstPropagator<'mir, 'tcx> { |
ba9703b0 | 314 | ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, |
dc9dc135 | 315 | tcx: TyCtxt<'tcx>, |
0531ce1d | 316 | param_env: ParamEnv<'tcx>, |
60c5eb7d XL |
317 | // FIXME(eddyb) avoid cloning these two fields more than once, |
318 | // by accessing them through `ecx` instead. | |
29967ef6 | 319 | source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>, |
48663c56 | 320 | local_decls: IndexVec<Local, LocalDecl<'tcx>>, |
dfeec247 XL |
321 | // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store |
322 | // the last known `SourceInfo` here and just keep revisiting it. | |
323 | source_info: Option<SourceInfo>, | |
0531ce1d XL |
324 | } |
325 | ||
dc9dc135 | 326 | impl<'mir, 'tcx> LayoutOf for ConstPropagator<'mir, 'tcx> { |
48663c56 | 327 | type Ty = Ty<'tcx>; |
ba9703b0 | 328 | type TyAndLayout = Result<TyAndLayout<'tcx>, LayoutError<'tcx>>; |
0531ce1d | 329 | |
ba9703b0 | 330 | fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyAndLayout { |
0531ce1d XL |
331 | self.tcx.layout_of(self.param_env.and(ty)) |
332 | } | |
333 | } | |
334 | ||
dc9dc135 | 335 | impl<'mir, 'tcx> HasDataLayout for ConstPropagator<'mir, 'tcx> { |
0531ce1d XL |
336 | #[inline] |
337 | fn data_layout(&self) -> &TargetDataLayout { | |
338 | &self.tcx.data_layout | |
339 | } | |
340 | } | |
341 | ||
dc9dc135 | 342 | impl<'mir, 'tcx> HasTyCtxt<'tcx> for ConstPropagator<'mir, 'tcx> { |
0531ce1d | 343 | #[inline] |
dc9dc135 | 344 | fn tcx(&self) -> TyCtxt<'tcx> { |
0531ce1d XL |
345 | self.tcx |
346 | } | |
347 | } | |
348 | ||
dc9dc135 | 349 | impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { |
0531ce1d | 350 | fn new( |
f9f354fc | 351 | body: &Body<'tcx>, |
dc9dc135 | 352 | dummy_body: &'mir Body<'tcx>, |
dc9dc135 | 353 | tcx: TyCtxt<'tcx>, |
dc9dc135 | 354 | ) -> ConstPropagator<'mir, 'tcx> { |
29967ef6 | 355 | let def_id = body.source.def_id(); |
dfeec247 | 356 | let substs = &InternalSubsts::identity_for_item(tcx, def_id); |
3dfed10e | 357 | let param_env = tcx.param_env_reveal_all_normalized(def_id); |
dfeec247 | 358 | |
dc9dc135 | 359 | let span = tcx.def_span(def_id); |
3dfed10e XL |
360 | // FIXME: `CanConstProp::check` computes the layout of all locals, return those layouts |
361 | // so we can write them to `ecx.frame_mut().locals.layout, reducing the duplication in | |
362 | // `layout_of` query invocations. | |
363 | let can_const_prop = CanConstProp::check(tcx, param_env, body); | |
f035d41b XL |
364 | let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len()); |
365 | for (l, mode) in can_const_prop.iter_enumerated() { | |
366 | if *mode == ConstPropMode::OnlyInsideOwnBlock { | |
367 | only_propagate_inside_block_locals.insert(l); | |
368 | } | |
369 | } | |
370 | let mut ecx = InterpCx::new( | |
371 | tcx, | |
372 | span, | |
373 | param_env, | |
3dfed10e | 374 | ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), |
f035d41b XL |
375 | (), |
376 | ); | |
dc9dc135 | 377 | |
dfeec247 XL |
378 | let ret = ecx |
379 | .layout_of(body.return_ty().subst(tcx, substs)) | |
380 | .ok() | |
381 | // Don't bother allocating memory for ZST types which have no values | |
382 | // or for large values. | |
383 | .filter(|ret_layout| { | |
384 | !ret_layout.is_zst() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) | |
385 | }) | |
386 | .map(|ret_layout| ecx.allocate(ret_layout, MemoryKind::Stack)); | |
60c5eb7d | 387 | |
dc9dc135 | 388 | ecx.push_stack_frame( |
60c5eb7d | 389 | Instance::new(def_id, substs), |
dc9dc135 | 390 | dummy_body, |
60c5eb7d | 391 | ret.map(Into::into), |
dfeec247 XL |
392 | StackPopCleanup::None { cleanup: false }, |
393 | ) | |
394 | .expect("failed to push initial stack frame"); | |
48663c56 | 395 | |
0531ce1d | 396 | ConstPropagator { |
94b46f34 | 397 | ecx, |
0531ce1d | 398 | tcx, |
0531ce1d | 399 | param_env, |
60c5eb7d XL |
400 | // FIXME(eddyb) avoid cloning these two fields more than once, |
401 | // by accessing them through `ecx` instead. | |
402 | source_scopes: body.source_scopes.clone(), | |
dc9dc135 XL |
403 | //FIXME(wesleywiser) we can't steal this because `Visitor::super_visit_body()` needs it |
404 | local_decls: body.local_decls.clone(), | |
dfeec247 | 405 | source_info: None, |
0531ce1d XL |
406 | } |
407 | } | |
408 | ||
f9f354fc | 409 | fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { |
f035d41b XL |
410 | let op = match self.ecx.eval_place_to_op(place, None) { |
411 | Ok(op) => op, | |
412 | Err(e) => { | |
413 | trace!("get_const failed: {}", e); | |
414 | return None; | |
415 | } | |
416 | }; | |
60c5eb7d | 417 | |
f9f354fc XL |
418 | // Try to read the local as an immediate so that if it is representable as a scalar, we can |
419 | // handle it as such, but otherwise, just return the value as is. | |
f035d41b XL |
420 | Some(match self.ecx.try_read_immediate(op) { |
421 | Ok(Ok(imm)) => imm.into(), | |
f9f354fc | 422 | _ => op, |
f035d41b | 423 | }) |
dc9dc135 XL |
424 | } |
425 | ||
f9f354fc XL |
426 | /// Remove `local` from the pool of `Locals`. Allows writing to them, |
427 | /// but not reading from them anymore. | |
428 | fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { | |
429 | ecx.frame_mut().locals[local] = | |
dfeec247 | 430 | LocalState { value: LocalValue::Uninitialized, layout: Cell::new(None) }; |
dc9dc135 XL |
431 | } |
432 | ||
dfeec247 XL |
433 | fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> { |
434 | match &self.source_scopes[source_info.scope].local_data { | |
435 | ClearCrossCrate::Set(data) => Some(data.lint_root), | |
436 | ClearCrossCrate::Clear => None, | |
437 | } | |
438 | } | |
439 | ||
74b04a01 | 440 | fn use_ecx<F, T>(&mut self, f: F) -> Option<T> |
94b46f34 | 441 | where |
dc9dc135 | 442 | F: FnOnce(&mut Self) -> InterpResult<'tcx, T>, |
94b46f34 | 443 | { |
ba9703b0 | 444 | match f(self) { |
94b46f34 | 445 | Ok(val) => Some(val), |
8faf50e0 | 446 | Err(error) => { |
3dfed10e | 447 | trace!("InterpCx operation failed: {:?}", error); |
74b04a01 XL |
448 | // Some errors shouldn't come up because creating them causes |
449 | // an allocation, which we should avoid. When that happens, | |
450 | // dedicated error variants should be introduced instead. | |
ba9703b0 | 451 | assert!( |
74b04a01 XL |
452 | !error.kind.allocates(), |
453 | "const-prop encountered allocating error: {}", | |
454 | error | |
455 | ); | |
94b46f34 | 456 | None |
dfeec247 | 457 | } |
ba9703b0 | 458 | } |
94b46f34 XL |
459 | } |
460 | ||
f9f354fc | 461 | /// Returns the value, if any, of evaluating `c`. |
dfeec247 | 462 | fn eval_constant(&mut self, c: &Constant<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { |
dfeec247 XL |
463 | // FIXME we need to revisit this for #67176 |
464 | if c.needs_subst() { | |
465 | return None; | |
466 | } | |
467 | ||
3dfed10e | 468 | match self.ecx.const_to_op(c.literal, None) { |
dfeec247 | 469 | Ok(op) => Some(op), |
8faf50e0 | 470 | Err(error) => { |
f035d41b | 471 | let tcx = self.ecx.tcx.at(c.span); |
3dfed10e | 472 | let err = ConstEvalErr::new(&self.ecx, error, Some(c.span)); |
dfeec247 XL |
473 | if let Some(lint_root) = self.lint_root(source_info) { |
474 | let lint_only = match c.literal.val { | |
475 | // Promoteds must lint and not error as the user didn't ask for them | |
476 | ConstKind::Unevaluated(_, _, Some(_)) => true, | |
477 | // Out of backwards compatibility we cannot report hard errors in unused | |
478 | // generic functions using associated constants of the generic parameters. | |
479 | _ => c.literal.needs_subst(), | |
480 | }; | |
481 | if lint_only { | |
482 | // Out of backwards compatibility we cannot report hard errors in unused | |
483 | // generic functions using associated constants of the generic parameters. | |
f035d41b | 484 | err.report_as_lint(tcx, "erroneous constant used", lint_root, Some(c.span)); |
dfeec247 | 485 | } else { |
f035d41b | 486 | err.report_as_error(tcx, "erroneous constant used"); |
dfeec247 XL |
487 | } |
488 | } else { | |
f035d41b | 489 | err.report_as_error(tcx, "erroneous constant used"); |
dfeec247 | 490 | } |
8faf50e0 | 491 | None |
dfeec247 | 492 | } |
8faf50e0 XL |
493 | } |
494 | } | |
495 | ||
f9f354fc | 496 | /// Returns the value, if any, of evaluating `place`. |
ba9703b0 | 497 | fn eval_place(&mut self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { |
dc9dc135 | 498 | trace!("eval_place(place={:?})", place); |
74b04a01 | 499 | self.use_ecx(|this| this.ecx.eval_place_to_op(place, None)) |
0531ce1d XL |
500 | } |
501 | ||
f9f354fc XL |
502 | /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant` |
503 | /// or `eval_place`, depending on the variant of `Operand` used. | |
dfeec247 | 504 | fn eval_operand(&mut self, op: &Operand<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { |
0531ce1d | 505 | match *op { |
dfeec247 | 506 | Operand::Constant(ref c) => self.eval_constant(c, source_info), |
ba9703b0 | 507 | Operand::Move(place) | Operand::Copy(place) => self.eval_place(place), |
0531ce1d XL |
508 | } |
509 | } | |
510 | ||
74b04a01 XL |
511 | fn report_assert_as_lint( |
512 | &self, | |
513 | lint: &'static lint::Lint, | |
514 | source_info: SourceInfo, | |
515 | message: &'static str, | |
f9652781 | 516 | panic: AssertKind<impl std::fmt::Debug>, |
74b04a01 XL |
517 | ) -> Option<()> { |
518 | let lint_root = self.lint_root(source_info)?; | |
519 | self.tcx.struct_span_lint_hir(lint, lint_root, source_info.span, |lint| { | |
520 | let mut err = lint.build(message); | |
521 | err.span_label(source_info.span, format!("{:?}", panic)); | |
522 | err.emit() | |
523 | }); | |
ba9703b0 | 524 | None |
74b04a01 | 525 | } |
dfeec247 | 526 | |
74b04a01 XL |
527 | fn check_unary_op( |
528 | &mut self, | |
529 | op: UnOp, | |
530 | arg: &Operand<'tcx>, | |
531 | source_info: SourceInfo, | |
532 | ) -> Option<()> { | |
f035d41b | 533 | if let (val, true) = self.use_ecx(|this| { |
74b04a01 XL |
534 | let val = this.ecx.read_immediate(this.ecx.eval_operand(arg, None)?)?; |
535 | let (_res, overflow, _ty) = this.ecx.overflowing_unary_op(op, val)?; | |
f035d41b | 536 | Ok((val, overflow)) |
74b04a01 XL |
537 | })? { |
538 | // `AssertKind` only has an `OverflowNeg` variant, so make sure that is | |
539 | // appropriate to use. | |
540 | assert_eq!(op, UnOp::Neg, "Neg is the only UnOp that can overflow"); | |
541 | self.report_assert_as_lint( | |
542 | lint::builtin::ARITHMETIC_OVERFLOW, | |
543 | source_info, | |
544 | "this arithmetic operation will overflow", | |
f035d41b | 545 | AssertKind::OverflowNeg(val.to_const_int()), |
74b04a01 XL |
546 | )?; |
547 | } | |
dfeec247 XL |
548 | |
549 | Some(()) | |
550 | } | |
551 | ||
552 | fn check_binary_op( | |
553 | &mut self, | |
554 | op: BinOp, | |
555 | left: &Operand<'tcx>, | |
556 | right: &Operand<'tcx>, | |
557 | source_info: SourceInfo, | |
dfeec247 | 558 | ) -> Option<()> { |
3dfed10e | 559 | let r = self.use_ecx(|this| this.ecx.read_immediate(this.ecx.eval_operand(right, None)?)); |
f035d41b | 560 | let l = self.use_ecx(|this| this.ecx.read_immediate(this.ecx.eval_operand(left, None)?)); |
74b04a01 | 561 | // Check for exceeding shifts *even if* we cannot evaluate the LHS. |
dfeec247 | 562 | if op == BinOp::Shr || op == BinOp::Shl { |
3dfed10e | 563 | let r = r?; |
74b04a01 XL |
564 | // We need the type of the LHS. We cannot use `place_layout` as that is the type |
565 | // of the result, which for checked binops is not the same! | |
566 | let left_ty = left.ty(&self.local_decls, self.tcx); | |
f035d41b | 567 | let left_size = self.ecx.layout_of(left_ty).ok()?.size; |
dfeec247 | 568 | let right_size = r.layout.size; |
74b04a01 XL |
569 | let r_bits = r.to_scalar().ok(); |
570 | // This is basically `force_bits`. | |
571 | let r_bits = r_bits.and_then(|r| r.to_bits_or_ptr(right_size, &self.tcx).ok()); | |
f035d41b XL |
572 | if r_bits.map_or(false, |b| b >= left_size.bits() as u128) { |
573 | debug!("check_binary_op: reporting assert for {:?}", source_info); | |
74b04a01 XL |
574 | self.report_assert_as_lint( |
575 | lint::builtin::ARITHMETIC_OVERFLOW, | |
576 | source_info, | |
577 | "this arithmetic operation will overflow", | |
f035d41b XL |
578 | AssertKind::Overflow( |
579 | op, | |
580 | match l { | |
581 | Some(l) => l.to_const_int(), | |
582 | // Invent a dummy value, the diagnostic ignores it anyway | |
583 | None => ConstInt::new( | |
29967ef6 | 584 | ScalarInt::try_from_uint(1_u8, left_size).unwrap(), |
f035d41b XL |
585 | left_ty.is_signed(), |
586 | left_ty.is_ptr_sized_integral(), | |
587 | ), | |
588 | }, | |
589 | r.to_const_int(), | |
590 | ), | |
74b04a01 | 591 | )?; |
dfeec247 XL |
592 | } |
593 | } | |
594 | ||
3dfed10e XL |
595 | if let (Some(l), Some(r)) = (l, r) { |
596 | // The remaining operators are handled through `overflowing_binary_op`. | |
597 | if self.use_ecx(|this| { | |
598 | let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, l, r)?; | |
599 | Ok(overflow) | |
600 | })? { | |
601 | self.report_assert_as_lint( | |
602 | lint::builtin::ARITHMETIC_OVERFLOW, | |
603 | source_info, | |
604 | "this arithmetic operation will overflow", | |
605 | AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()), | |
606 | )?; | |
607 | } | |
dfeec247 | 608 | } |
dfeec247 XL |
609 | Some(()) |
610 | } | |
611 | ||
3dfed10e XL |
612 | fn propagate_operand(&mut self, operand: &mut Operand<'tcx>) { |
613 | match *operand { | |
614 | Operand::Copy(l) | Operand::Move(l) => { | |
615 | if let Some(value) = self.get_const(l) { | |
616 | if self.should_const_prop(value) { | |
617 | // FIXME(felix91gr): this code only handles `Scalar` cases. | |
618 | // For now, we're not handling `ScalarPair` cases because | |
619 | // doing so here would require a lot of code duplication. | |
620 | // We should hopefully generalize `Operand` handling into a fn, | |
621 | // and use it to do const-prop here and everywhere else | |
622 | // where it makes sense. | |
623 | if let interpret::Operand::Immediate(interpret::Immediate::Scalar( | |
624 | ScalarMaybeUninit::Scalar(scalar), | |
625 | )) = *value | |
626 | { | |
627 | *operand = self.operand_from_scalar( | |
628 | scalar, | |
629 | value.layout.ty, | |
630 | self.source_info.unwrap().span, | |
631 | ); | |
632 | } | |
633 | } | |
634 | } | |
635 | } | |
636 | Operand::Constant(_) => (), | |
637 | } | |
638 | } | |
639 | ||
0531ce1d XL |
640 | fn const_prop( |
641 | &mut self, | |
642 | rvalue: &Rvalue<'tcx>, | |
0531ce1d | 643 | source_info: SourceInfo, |
ba9703b0 | 644 | place: Place<'tcx>, |
e74abb32 | 645 | ) -> Option<()> { |
e74abb32 XL |
646 | // Perform any special handling for specific Rvalue types. |
647 | // Generally, checks here fall into one of two categories: | |
648 | // 1. Additional checking to provide useful lints to the user | |
649 | // - In this case, we will do some validation and then fall through to the | |
650 | // end of the function which evals the assignment. | |
651 | // 2. Working around bugs in other parts of the compiler | |
652 | // - In this case, we'll return `None` from this function to stop evaluation. | |
653 | match rvalue { | |
74b04a01 XL |
654 | // Additional checking: give lints to the user if an overflow would occur. |
655 | // We do this here and not in the `Assert` terminator as that terminator is | |
656 | // only sometimes emitted (overflow checks can be disabled), but we want to always | |
657 | // lint. | |
658 | Rvalue::UnaryOp(op, arg) => { | |
659 | trace!("checking UnaryOp(op = {:?}, arg = {:?})", op, arg); | |
660 | self.check_unary_op(*op, arg, source_info)?; | |
0531ce1d | 661 | } |
e74abb32 XL |
662 | Rvalue::BinaryOp(op, left, right) => { |
663 | trace!("checking BinaryOp(op = {:?}, left = {:?}, right = {:?})", op, left, right); | |
74b04a01 XL |
664 | self.check_binary_op(*op, left, right, source_info)?; |
665 | } | |
666 | Rvalue::CheckedBinaryOp(op, left, right) => { | |
667 | trace!( | |
668 | "checking CheckedBinaryOp(op = {:?}, left = {:?}, right = {:?})", | |
669 | op, | |
670 | left, | |
671 | right | |
672 | ); | |
673 | self.check_binary_op(*op, left, right, source_info)?; | |
e74abb32 XL |
674 | } |
675 | ||
dfeec247 | 676 | // Do not try creating references (#67862) |
f035d41b XL |
677 | Rvalue::AddressOf(_, place) | Rvalue::Ref(_, _, place) => { |
678 | trace!("skipping AddressOf | Ref for {:?}", place); | |
679 | ||
680 | // This may be creating mutable references or immutable references to cells. | |
681 | // If that happens, the pointed to value could be mutated via that reference. | |
682 | // Since we aren't tracking references, the const propagator loses track of what | |
683 | // value the local has right now. | |
684 | // Thus, all locals that have their reference taken | |
685 | // must not take part in propagation. | |
686 | Self::remove_const(&mut self.ecx, place.local); | |
e74abb32 | 687 | |
dfeec247 | 688 | return None; |
e74abb32 | 689 | } |
f035d41b XL |
690 | Rvalue::ThreadLocalRef(def_id) => { |
691 | trace!("skipping ThreadLocalRef({:?})", def_id); | |
e74abb32 | 692 | |
f035d41b XL |
693 | return None; |
694 | } | |
695 | ||
696 | // There's no other checking to do at this time. | |
697 | Rvalue::Aggregate(..) | |
698 | | Rvalue::Use(..) | |
699 | | Rvalue::Repeat(..) | |
700 | | Rvalue::Len(..) | |
701 | | Rvalue::Cast(..) | |
702 | | Rvalue::Discriminant(..) | |
703 | | Rvalue::NullaryOp(..) => {} | |
0531ce1d | 704 | } |
e74abb32 | 705 | |
f9f354fc XL |
706 | // FIXME we need to revisit this for #67176 |
707 | if rvalue.needs_subst() { | |
708 | return None; | |
709 | } | |
710 | ||
3dfed10e XL |
711 | if self.tcx.sess.opts.debugging_opts.mir_opt_level >= 3 { |
712 | self.eval_rvalue_with_identities(rvalue, place) | |
713 | } else { | |
714 | self.use_ecx(|this| this.ecx.eval_rvalue_into_place(rvalue, place)) | |
715 | } | |
716 | } | |
717 | ||
718 | // Attempt to use albegraic identities to eliminate constant expressions | |
719 | fn eval_rvalue_with_identities( | |
720 | &mut self, | |
721 | rvalue: &Rvalue<'tcx>, | |
722 | place: Place<'tcx>, | |
723 | ) -> Option<()> { | |
74b04a01 | 724 | self.use_ecx(|this| { |
3dfed10e XL |
725 | match rvalue { |
726 | Rvalue::BinaryOp(op, left, right) | Rvalue::CheckedBinaryOp(op, left, right) => { | |
727 | let l = this.ecx.eval_operand(left, None); | |
728 | let r = this.ecx.eval_operand(right, None); | |
729 | ||
730 | let const_arg = match (l, r) { | |
731 | (Ok(x), Err(_)) | (Err(_), Ok(x)) => this.ecx.read_immediate(x)?, | |
732 | (Err(e), Err(_)) => return Err(e), | |
733 | (Ok(_), Ok(_)) => { | |
734 | this.ecx.eval_rvalue_into_place(rvalue, place)?; | |
735 | return Ok(()); | |
736 | } | |
737 | }; | |
738 | ||
739 | let arg_value = | |
740 | this.ecx.force_bits(const_arg.to_scalar()?, const_arg.layout.size)?; | |
741 | let dest = this.ecx.eval_place(place)?; | |
742 | ||
743 | match op { | |
744 | BinOp::BitAnd => { | |
745 | if arg_value == 0 { | |
746 | this.ecx.write_immediate(*const_arg, dest)?; | |
747 | } | |
748 | } | |
749 | BinOp::BitOr => { | |
29967ef6 | 750 | if arg_value == const_arg.layout.size.truncate(u128::MAX) |
3dfed10e XL |
751 | || (const_arg.layout.ty.is_bool() && arg_value == 1) |
752 | { | |
753 | this.ecx.write_immediate(*const_arg, dest)?; | |
754 | } | |
755 | } | |
756 | BinOp::Mul => { | |
757 | if const_arg.layout.ty.is_integral() && arg_value == 0 { | |
758 | if let Rvalue::CheckedBinaryOp(_, _, _) = rvalue { | |
759 | let val = Immediate::ScalarPair( | |
760 | const_arg.to_scalar()?.into(), | |
761 | Scalar::from_bool(false).into(), | |
762 | ); | |
763 | this.ecx.write_immediate(val, dest)?; | |
764 | } else { | |
765 | this.ecx.write_immediate(*const_arg, dest)?; | |
766 | } | |
767 | } | |
768 | } | |
769 | _ => { | |
770 | this.ecx.eval_rvalue_into_place(rvalue, place)?; | |
771 | } | |
772 | } | |
773 | } | |
774 | _ => { | |
775 | this.ecx.eval_rvalue_into_place(rvalue, place)?; | |
776 | } | |
777 | } | |
778 | ||
e74abb32 XL |
779 | Ok(()) |
780 | }) | |
0531ce1d | 781 | } |
48663c56 | 782 | |
f9f354fc | 783 | /// Creates a new `Operand::Constant` from a `Scalar` value |
48663c56 | 784 | fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>, span: Span) -> Operand<'tcx> { |
dfeec247 XL |
785 | Operand::Constant(Box::new(Constant { |
786 | span, | |
787 | user_ty: None, | |
f9f354fc | 788 | literal: ty::Const::from_scalar(self.tcx, scalar, ty), |
dfeec247 | 789 | })) |
48663c56 XL |
790 | } |
791 | ||
dc9dc135 XL |
792 | fn replace_with_const( |
793 | &mut self, | |
794 | rval: &mut Rvalue<'tcx>, | |
dfeec247 | 795 | value: OpTy<'tcx>, |
dc9dc135 XL |
796 | source_info: SourceInfo, |
797 | ) { | |
f9f354fc XL |
798 | if let Rvalue::Use(Operand::Constant(c)) = rval { |
799 | if !matches!(c.literal.val, ConstKind::Unevaluated(..)) { | |
800 | trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c); | |
801 | return; | |
802 | } | |
803 | } | |
804 | ||
fc512014 | 805 | trace!("attempting to replace {:?} with {:?}", rval, value); |
74b04a01 | 806 | if let Err(e) = self.ecx.const_validate_operand( |
48663c56 XL |
807 | value, |
808 | vec![], | |
dc9dc135 | 809 | // FIXME: is ref tracking too expensive? |
29967ef6 | 810 | // FIXME: what is the point of ref tracking if we do not even check the tracked refs? |
74b04a01 | 811 | &mut interpret::RefTracking::empty(), |
29967ef6 | 812 | CtfeValidationMode::Regular, |
dc9dc135 XL |
813 | ) { |
814 | trace!("validation error, attempt failed: {:?}", e); | |
815 | return; | |
816 | } | |
817 | ||
74b04a01 XL |
818 | // FIXME> figure out what to do when try_read_immediate fails |
819 | let imm = self.use_ecx(|this| this.ecx.try_read_immediate(value)); | |
48663c56 | 820 | |
dc9dc135 XL |
821 | if let Some(Ok(imm)) = imm { |
822 | match *imm { | |
f9f354fc | 823 | interpret::Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar)) => { |
dfeec247 XL |
824 | *rval = Rvalue::Use(self.operand_from_scalar( |
825 | scalar, | |
826 | value.layout.ty, | |
827 | source_info.span, | |
828 | )); | |
829 | } | |
48663c56 | 830 | Immediate::ScalarPair( |
3dfed10e XL |
831 | ScalarMaybeUninit::Scalar(_), |
832 | ScalarMaybeUninit::Scalar(_), | |
48663c56 | 833 | ) => { |
3dfed10e XL |
834 | // Found a value represented as a pair. For now only do const-prop if the type |
835 | // of `rvalue` is also a tuple with two scalars. | |
836 | // FIXME: enable the general case stated above ^. | |
837 | let ty = &value.layout.ty; | |
60c5eb7d | 838 | // Only do it for tuples |
1b1a35ee | 839 | if let ty::Tuple(substs) = ty.kind() { |
60c5eb7d XL |
840 | // Only do it if tuple is also a pair with two scalars |
841 | if substs.len() == 2 { | |
3dfed10e | 842 | let alloc = self.use_ecx(|this| { |
60c5eb7d XL |
843 | let ty1 = substs[0].expect_ty(); |
844 | let ty2 = substs[1].expect_ty(); | |
845 | let ty_is_scalar = |ty| { | |
ba9703b0 | 846 | this.ecx.layout_of(ty).ok().map(|layout| layout.abi.is_scalar()) |
60c5eb7d XL |
847 | == Some(true) |
848 | }; | |
849 | if ty_is_scalar(ty1) && ty_is_scalar(ty2) { | |
3dfed10e XL |
850 | let alloc = this |
851 | .ecx | |
852 | .intern_with_temp_alloc(value.layout, |ecx, dest| { | |
853 | ecx.write_immediate_to_mplace(*imm, dest) | |
854 | }) | |
855 | .unwrap(); | |
856 | Ok(Some(alloc)) | |
60c5eb7d XL |
857 | } else { |
858 | Ok(None) | |
859 | } | |
860 | }); | |
861 | ||
3dfed10e XL |
862 | if let Some(Some(alloc)) = alloc { |
863 | // Assign entire constant in a single statement. | |
864 | // We can't use aggregates, as we run after the aggregate-lowering `MirPhase`. | |
865 | *rval = Rvalue::Use(Operand::Constant(Box::new(Constant { | |
866 | span: source_info.span, | |
867 | user_ty: None, | |
868 | literal: self.ecx.tcx.mk_const(ty::Const { | |
869 | ty, | |
870 | val: ty::ConstKind::Value(ConstValue::ByRef { | |
871 | alloc, | |
872 | offset: Size::ZERO, | |
873 | }), | |
874 | }), | |
875 | }))); | |
60c5eb7d XL |
876 | } |
877 | } | |
48663c56 | 878 | } |
dfeec247 | 879 | } |
3dfed10e XL |
880 | // Scalars or scalar pairs that contain undef values are assumed to not have |
881 | // successfully evaluated and are thus not propagated. | |
dfeec247 | 882 | _ => {} |
48663c56 XL |
883 | } |
884 | } | |
885 | } | |
886 | ||
f9f354fc | 887 | /// Returns `true` if and only if this `op` should be const-propagated into. |
60c5eb7d XL |
888 | fn should_const_prop(&mut self, op: OpTy<'tcx>) -> bool { |
889 | let mir_opt_level = self.tcx.sess.opts.debugging_opts.mir_opt_level; | |
890 | ||
891 | if mir_opt_level == 0 { | |
892 | return false; | |
893 | } | |
894 | ||
fc512014 XL |
895 | if !self.tcx.consider_optimizing(|| format!("ConstantPropagation - OpTy: {:?}", op)) { |
896 | return false; | |
897 | } | |
898 | ||
60c5eb7d | 899 | match *op { |
f9f354fc | 900 | interpret::Operand::Immediate(Immediate::Scalar(ScalarMaybeUninit::Scalar(s))) => { |
dfeec247 XL |
901 | s.is_bits() |
902 | } | |
903 | interpret::Operand::Immediate(Immediate::ScalarPair( | |
f9f354fc XL |
904 | ScalarMaybeUninit::Scalar(l), |
905 | ScalarMaybeUninit::Scalar(r), | |
dfeec247 | 906 | )) => l.is_bits() && r.is_bits(), |
dfeec247 | 907 | _ => false, |
60c5eb7d | 908 | } |
48663c56 | 909 | } |
0531ce1d XL |
910 | } |
911 | ||
dfeec247 XL |
912 | /// The mode that `ConstProp` is allowed to run in for a given `Local`. |
913 | #[derive(Clone, Copy, Debug, PartialEq)] | |
914 | enum ConstPropMode { | |
915 | /// The `Local` can be propagated into and reads of this `Local` can also be propagated. | |
916 | FullConstProp, | |
f9f354fc XL |
917 | /// The `Local` can only be propagated into and from its own block. |
918 | OnlyInsideOwnBlock, | |
dfeec247 XL |
919 | /// The `Local` can be propagated into but reads cannot be propagated. |
920 | OnlyPropagateInto, | |
f035d41b XL |
921 | /// The `Local` cannot be part of propagation at all. Any statement |
922 | /// referencing it either for reading or writing will not get propagated. | |
dfeec247 XL |
923 | NoPropagation, |
924 | } | |
925 | ||
0531ce1d | 926 | struct CanConstProp { |
dfeec247 | 927 | can_const_prop: IndexVec<Local, ConstPropMode>, |
f9f354fc XL |
928 | // False at the beginning. Once set, no more assignments are allowed to that local. |
929 | found_assignment: BitSet<Local>, | |
930 | // Cache of locals' information | |
931 | local_kinds: IndexVec<Local, LocalKind>, | |
0531ce1d XL |
932 | } |
933 | ||
934 | impl CanConstProp { | |
f9f354fc | 935 | /// Returns true if `local` can be propagated |
3dfed10e XL |
936 | fn check( |
937 | tcx: TyCtxt<'tcx>, | |
938 | param_env: ParamEnv<'tcx>, | |
939 | body: &Body<'tcx>, | |
940 | ) -> IndexVec<Local, ConstPropMode> { | |
0531ce1d | 941 | let mut cpv = CanConstProp { |
dfeec247 | 942 | can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls), |
f9f354fc XL |
943 | found_assignment: BitSet::new_empty(body.local_decls.len()), |
944 | local_kinds: IndexVec::from_fn_n( | |
945 | |local| body.local_kind(local), | |
946 | body.local_decls.len(), | |
947 | ), | |
0531ce1d XL |
948 | }; |
949 | for (local, val) in cpv.can_const_prop.iter_enumerated_mut() { | |
3dfed10e XL |
950 | let ty = body.local_decls[local].ty; |
951 | match tcx.layout_of(param_env.and(ty)) { | |
952 | Ok(layout) if layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) => {} | |
953 | // Either the layout fails to compute, then we can't use this local anyway | |
954 | // or the local is too large, then we don't want to. | |
955 | _ => { | |
956 | *val = ConstPropMode::NoPropagation; | |
957 | continue; | |
958 | } | |
959 | } | |
f9f354fc XL |
960 | // Cannot use args at all |
961 | // Cannot use locals because if x < y { y - x } else { x - y } would | |
0531ce1d XL |
962 | // lint for x != y |
963 | // FIXME(oli-obk): lint variables until they are used in a condition | |
964 | // FIXME(oli-obk): lint if return value is constant | |
f9f354fc | 965 | if cpv.local_kinds[local] == LocalKind::Arg { |
dfeec247 | 966 | *val = ConstPropMode::OnlyPropagateInto; |
f9f354fc XL |
967 | trace!( |
968 | "local {:?} can't be const propagated because it's a function argument", | |
969 | local | |
970 | ); | |
971 | } else if cpv.local_kinds[local] == LocalKind::Var { | |
972 | *val = ConstPropMode::OnlyInsideOwnBlock; | |
973 | trace!( | |
974 | "local {:?} will only be propagated inside its block, because it's a user variable", | |
975 | local | |
976 | ); | |
dc9dc135 | 977 | } |
0531ce1d | 978 | } |
ba9703b0 | 979 | cpv.visit_body(&body); |
0531ce1d XL |
980 | cpv.can_const_prop |
981 | } | |
982 | } | |
983 | ||
984 | impl<'tcx> Visitor<'tcx> for CanConstProp { | |
dfeec247 | 985 | fn visit_local(&mut self, &local: &Local, context: PlaceContext, _: Location) { |
ba9703b0 | 986 | use rustc_middle::mir::visit::PlaceContext::*; |
0531ce1d | 987 | match context { |
f9f354fc XL |
988 | // Projections are fine, because `&mut foo.x` will be caught by |
989 | // `MutatingUseContext::Borrow` elsewhere. | |
990 | MutatingUse(MutatingUseContext::Projection) | |
991 | // These are just stores, where the storing is not propagatable, but there may be later | |
992 | // mutations of the same local via `Store` | |
993 | | MutatingUse(MutatingUseContext::Call) | |
994 | // Actual store that can possibly even propagate a value | |
995 | | MutatingUse(MutatingUseContext::Store) => { | |
996 | if !self.found_assignment.insert(local) { | |
997 | match &mut self.can_const_prop[local] { | |
998 | // If the local can only get propagated in its own block, then we don't have | |
999 | // to worry about multiple assignments, as we'll nuke the const state at the | |
1000 | // end of the block anyway, and inside the block we overwrite previous | |
1001 | // states as applicable. | |
1002 | ConstPropMode::OnlyInsideOwnBlock => {} | |
f035d41b XL |
1003 | ConstPropMode::NoPropagation => {} |
1004 | ConstPropMode::OnlyPropagateInto => {} | |
1005 | other @ ConstPropMode::FullConstProp => { | |
f9f354fc | 1006 | trace!( |
3dfed10e XL |
1007 | "local {:?} can't be propagated because of multiple assignments. Previous state: {:?}", |
1008 | local, other, | |
f9f354fc | 1009 | ); |
3dfed10e | 1010 | *other = ConstPropMode::OnlyInsideOwnBlock; |
f9f354fc XL |
1011 | } |
1012 | } | |
dfeec247 XL |
1013 | } |
1014 | } | |
0531ce1d | 1015 | // Reading constants is allowed an arbitrary number of times |
dfeec247 XL |
1016 | NonMutatingUse(NonMutatingUseContext::Copy) |
1017 | | NonMutatingUse(NonMutatingUseContext::Move) | |
1018 | | NonMutatingUse(NonMutatingUseContext::Inspect) | |
1019 | | NonMutatingUse(NonMutatingUseContext::Projection) | |
dfeec247 | 1020 | | NonUse(_) => {} |
f9f354fc XL |
1021 | |
1022 | // These could be propagated with a smarter analysis or just some careful thinking about | |
1023 | // whether they'd be fine right now. | |
1024 | MutatingUse(MutatingUseContext::AsmOutput) | |
1025 | | MutatingUse(MutatingUseContext::Yield) | |
1026 | | MutatingUse(MutatingUseContext::Drop) | |
1027 | | MutatingUse(MutatingUseContext::Retag) | |
1028 | // These can't ever be propagated under any scheme, as we can't reason about indirect | |
1029 | // mutation. | |
1030 | | NonMutatingUse(NonMutatingUseContext::SharedBorrow) | |
1031 | | NonMutatingUse(NonMutatingUseContext::ShallowBorrow) | |
1032 | | NonMutatingUse(NonMutatingUseContext::UniqueBorrow) | |
1033 | | NonMutatingUse(NonMutatingUseContext::AddressOf) | |
1034 | | MutatingUse(MutatingUseContext::Borrow) | |
1035 | | MutatingUse(MutatingUseContext::AddressOf) => { | |
dc9dc135 | 1036 | trace!("local {:?} can't be propagaged because it's used: {:?}", local, context); |
dfeec247 XL |
1037 | self.can_const_prop[local] = ConstPropMode::NoPropagation; |
1038 | } | |
0531ce1d XL |
1039 | } |
1040 | } | |
1041 | } | |
1042 | ||
dc9dc135 | 1043 | impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> { |
e74abb32 XL |
1044 | fn tcx(&self) -> TyCtxt<'tcx> { |
1045 | self.tcx | |
1046 | } | |
1047 | ||
f9f354fc XL |
1048 | fn visit_body(&mut self, body: &mut Body<'tcx>) { |
1049 | for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { | |
1050 | self.visit_basic_block_data(bb, data); | |
1051 | } | |
1052 | } | |
1053 | ||
3dfed10e XL |
1054 | fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { |
1055 | self.super_operand(operand, location); | |
1056 | ||
1b1a35ee XL |
1057 | // Only const prop copies and moves on `mir_opt_level=2` as doing so |
1058 | // currently slightly increases compile time in some cases. | |
1059 | if self.tcx.sess.opts.debugging_opts.mir_opt_level >= 2 { | |
3dfed10e XL |
1060 | self.propagate_operand(operand) |
1061 | } | |
1062 | } | |
1063 | ||
dfeec247 | 1064 | fn visit_constant(&mut self, constant: &mut Constant<'tcx>, location: Location) { |
0531ce1d XL |
1065 | trace!("visit_constant: {:?}", constant); |
1066 | self.super_constant(constant, location); | |
dfeec247 | 1067 | self.eval_constant(constant, self.source_info.unwrap()); |
0531ce1d XL |
1068 | } |
1069 | ||
dfeec247 | 1070 | fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { |
0531ce1d | 1071 | trace!("visit_statement: {:?}", statement); |
dfeec247 XL |
1072 | let source_info = statement.source_info; |
1073 | self.source_info = Some(source_info); | |
ba9703b0 | 1074 | if let StatementKind::Assign(box (place, ref mut rval)) = statement.kind { |
3dfed10e XL |
1075 | let can_const_prop = self.ecx.machine.can_const_prop[place.local]; |
1076 | if let Some(()) = self.const_prop(rval, source_info, place) { | |
1077 | // This will return None if the above `const_prop` invocation only "wrote" a | |
1078 | // type whose creation requires no write. E.g. a generator whose initial state | |
1079 | // consists solely of uninitialized memory (so it doesn't capture any locals). | |
1080 | if let Some(value) = self.get_const(place) { | |
1081 | if self.should_const_prop(value) { | |
1082 | trace!("replacing {:?} with {:?}", rval, value); | |
1083 | self.replace_with_const(rval, value, source_info); | |
1084 | if can_const_prop == ConstPropMode::FullConstProp | |
1085 | || can_const_prop == ConstPropMode::OnlyInsideOwnBlock | |
1086 | { | |
1087 | trace!("propagated into {:?}", place); | |
dfeec247 XL |
1088 | } |
1089 | } | |
3dfed10e XL |
1090 | } |
1091 | match can_const_prop { | |
1092 | ConstPropMode::OnlyInsideOwnBlock => { | |
1093 | trace!( | |
1094 | "found local restricted to its block. \ | |
f035d41b | 1095 | Will remove it from const-prop after block is finished. Local: {:?}", |
3dfed10e XL |
1096 | place.local |
1097 | ); | |
1098 | } | |
1099 | ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { | |
1100 | trace!("can't propagate into {:?}", place); | |
1101 | if place.local != RETURN_PLACE { | |
1102 | Self::remove_const(&mut self.ecx, place.local); | |
8faf50e0 | 1103 | } |
0531ce1d | 1104 | } |
3dfed10e | 1105 | ConstPropMode::FullConstProp => {} |
0531ce1d | 1106 | } |
f035d41b | 1107 | } else { |
3dfed10e XL |
1108 | // Const prop failed, so erase the destination, ensuring that whatever happens |
1109 | // from here on, does not know about the previous value. | |
1110 | // This is important in case we have | |
1111 | // ```rust | |
1112 | // let mut x = 42; | |
1113 | // x = SOME_MUTABLE_STATIC; | |
1114 | // // x must now be uninit | |
1115 | // ``` | |
1116 | // FIXME: we overzealously erase the entire local, because that's easier to | |
1117 | // implement. | |
f035d41b | 1118 | trace!( |
3dfed10e XL |
1119 | "propagation into {:?} failed. |
1120 | Nuking the entire site from orbit, it's the only way to be sure", | |
f035d41b XL |
1121 | place, |
1122 | ); | |
1123 | Self::remove_const(&mut self.ecx, place.local); | |
0531ce1d | 1124 | } |
e74abb32 XL |
1125 | } else { |
1126 | match statement.kind { | |
3dfed10e XL |
1127 | StatementKind::SetDiscriminant { ref place, .. } => { |
1128 | match self.ecx.machine.can_const_prop[place.local] { | |
1129 | ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { | |
1130 | if self.use_ecx(|this| this.ecx.statement(statement)).is_some() { | |
1131 | trace!("propped discriminant into {:?}", place); | |
1132 | } else { | |
1133 | Self::remove_const(&mut self.ecx, place.local); | |
1134 | } | |
1135 | } | |
1136 | ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { | |
1137 | Self::remove_const(&mut self.ecx, place.local); | |
1138 | } | |
1139 | } | |
1140 | } | |
dfeec247 | 1141 | StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { |
e74abb32 XL |
1142 | let frame = self.ecx.frame_mut(); |
1143 | frame.locals[local].value = | |
1144 | if let StatementKind::StorageLive(_) = statement.kind { | |
1145 | LocalValue::Uninitialized | |
1146 | } else { | |
1147 | LocalValue::Dead | |
1148 | }; | |
1149 | } | |
1150 | _ => {} | |
1151 | } | |
0531ce1d | 1152 | } |
e74abb32 | 1153 | |
48663c56 | 1154 | self.super_statement(statement, location); |
0531ce1d XL |
1155 | } |
1156 | ||
dfeec247 | 1157 | fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { |
48663c56 | 1158 | let source_info = terminator.source_info; |
dfeec247 XL |
1159 | self.source_info = Some(source_info); |
1160 | self.super_terminator(terminator, location); | |
48663c56 | 1161 | match &mut terminator.kind { |
416331ca | 1162 | TerminatorKind::Assert { expected, ref msg, ref mut cond, .. } => { |
48663c56 XL |
1163 | if let Some(value) = self.eval_operand(&cond, source_info) { |
1164 | trace!("assertion on {:?} should be {:?}", value, expected); | |
f9f354fc | 1165 | let expected = ScalarMaybeUninit::from(Scalar::from_bool(*expected)); |
48663c56 XL |
1166 | let value_const = self.ecx.read_scalar(value).unwrap(); |
1167 | if expected != value_const { | |
f9652781 XL |
1168 | enum DbgVal<T> { |
1169 | Val(T), | |
1170 | Underscore, | |
1171 | } | |
1172 | impl<T: std::fmt::Debug> std::fmt::Debug for DbgVal<T> { | |
1173 | fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | |
1174 | match self { | |
1175 | Self::Val(val) => val.fmt(fmt), | |
1176 | Self::Underscore => fmt.write_str("_"), | |
1177 | } | |
1178 | } | |
1179 | } | |
f035d41b | 1180 | let mut eval_to_int = |op| { |
f9652781 XL |
1181 | // This can be `None` if the lhs wasn't const propagated and we just |
1182 | // triggered the assert on the value of the rhs. | |
1183 | match self.eval_operand(op, source_info) { | |
1184 | Some(op) => { | |
1185 | DbgVal::Val(self.ecx.read_immediate(op).unwrap().to_const_int()) | |
1186 | } | |
1187 | None => DbgVal::Underscore, | |
1188 | } | |
f035d41b XL |
1189 | }; |
1190 | let msg = match msg { | |
1191 | AssertKind::DivisionByZero(op) => { | |
1192 | Some(AssertKind::DivisionByZero(eval_to_int(op))) | |
1193 | } | |
1194 | AssertKind::RemainderByZero(op) => { | |
1195 | Some(AssertKind::RemainderByZero(eval_to_int(op))) | |
1196 | } | |
1197 | AssertKind::BoundsCheck { ref len, ref index } => { | |
1198 | let len = eval_to_int(len); | |
1199 | let index = eval_to_int(index); | |
1200 | Some(AssertKind::BoundsCheck { len, index }) | |
1201 | } | |
1202 | // Overflow is are already covered by checks on the binary operators. | |
1203 | AssertKind::Overflow(..) | AssertKind::OverflowNeg(_) => None, | |
1204 | // Need proper const propagator for these. | |
1205 | _ => None, | |
1206 | }; | |
f9f354fc | 1207 | // Poison all places this operand references so that further code |
48663c56 XL |
1208 | // doesn't use the invalid value |
1209 | match cond { | |
1210 | Operand::Move(ref place) | Operand::Copy(ref place) => { | |
f9f354fc | 1211 | Self::remove_const(&mut self.ecx, place.local); |
dfeec247 | 1212 | } |
48663c56 XL |
1213 | Operand::Constant(_) => {} |
1214 | } | |
f035d41b XL |
1215 | if let Some(msg) = msg { |
1216 | self.report_assert_as_lint( | |
1217 | lint::builtin::UNCONDITIONAL_PANIC, | |
1218 | source_info, | |
1219 | "this operation will panic at runtime", | |
1220 | msg, | |
1221 | ); | |
1222 | } | |
48663c56 | 1223 | } else { |
60c5eb7d | 1224 | if self.should_const_prop(value) { |
f9f354fc | 1225 | if let ScalarMaybeUninit::Scalar(scalar) = value_const { |
48663c56 XL |
1226 | *cond = self.operand_from_scalar( |
1227 | scalar, | |
1228 | self.tcx.types.bool, | |
1229 | source_info.span, | |
1230 | ); | |
0531ce1d | 1231 | } |
48663c56 | 1232 | } |
0531ce1d | 1233 | } |
0531ce1d | 1234 | } |
dfeec247 | 1235 | } |
3dfed10e XL |
1236 | TerminatorKind::SwitchInt { ref mut discr, .. } => { |
1237 | // FIXME: This is currently redundant with `visit_operand`, but sadly | |
1238 | // always visiting operands currently causes a perf regression in LLVM codegen, so | |
1239 | // `visit_operand` currently only runs for propagates places for `mir_opt_level=3`. | |
1240 | self.propagate_operand(discr) | |
dfeec247 | 1241 | } |
3dfed10e | 1242 | // None of these have Operands to const-propagate. |
dfeec247 XL |
1243 | TerminatorKind::Goto { .. } |
1244 | | TerminatorKind::Resume | |
1245 | | TerminatorKind::Abort | |
1246 | | TerminatorKind::Return | |
1247 | | TerminatorKind::Unreachable | |
1248 | | TerminatorKind::Drop { .. } | |
1249 | | TerminatorKind::DropAndReplace { .. } | |
1250 | | TerminatorKind::Yield { .. } | |
1251 | | TerminatorKind::GeneratorDrop | |
f035d41b | 1252 | | TerminatorKind::FalseEdge { .. } |
f9f354fc XL |
1253 | | TerminatorKind::FalseUnwind { .. } |
1254 | | TerminatorKind::InlineAsm { .. } => {} | |
3dfed10e XL |
1255 | // Every argument in our function calls have already been propagated in `visit_operand`. |
1256 | // | |
1b1a35ee XL |
1257 | // NOTE: because LLVM codegen gives slight performance regressions with it, so this is |
1258 | // gated on `mir_opt_level=2`. | |
3dfed10e | 1259 | TerminatorKind::Call { .. } => {} |
f9f354fc | 1260 | } |
f035d41b XL |
1261 | |
1262 | // We remove all Locals which are restricted in propagation to their containing blocks and | |
1263 | // which were modified in the current block. | |
3dfed10e | 1264 | // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. |
f035d41b XL |
1265 | let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); |
1266 | for &local in locals.iter() { | |
f9f354fc | 1267 | Self::remove_const(&mut self.ecx, local); |
0531ce1d | 1268 | } |
f035d41b XL |
1269 | locals.clear(); |
1270 | // Put it back so we reuse the heap of the storage | |
1271 | self.ecx.machine.written_only_inside_own_block_locals = locals; | |
1272 | if cfg!(debug_assertions) { | |
1273 | // Ensure we are correctly erasing locals with the non-debug-assert logic. | |
1274 | for local in self.ecx.machine.only_propagate_inside_block_locals.iter() { | |
1275 | assert!( | |
1276 | self.get_const(local.into()).is_none() | |
1277 | || self | |
1278 | .layout_of(self.local_decls[local].ty) | |
1279 | .map_or(true, |layout| layout.is_zst()) | |
1280 | ) | |
1281 | } | |
1282 | } | |
0531ce1d XL |
1283 | } |
1284 | } |