1 //! This module specifies the type based interner for constants.
3 //! After a const evaluation has computed a value, before we destroy the const evaluator's session
4 //! memory, we need to extract all memory allocations to the global memory pool so they stay around.
6 use super::validity
::RefTracking
;
7 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
9 use rustc_middle
::mir
::interpret
::InterpResult
;
10 use rustc_middle
::ty
::{self, query::TyCtxtAt, Ty}
;
12 use rustc_ast
::ast
::Mutability
;
14 use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, Scalar, ValueVisitor}
;
16 pub trait CompileTimeMachine
<'mir
, 'tcx
> = Machine
<
24 MemoryMap
= FxHashMap
<AllocId
, (MemoryKind
<!>, Allocation
)>,
27 struct InternVisitor
<'rt
, 'mir
, 'tcx
, M
: CompileTimeMachine
<'mir
, 'tcx
>> {
28 /// The ectx from which we intern.
29 ecx
: &'rt
mut InterpCx
<'mir
, 'tcx
, M
>,
30 /// Previously encountered safe references.
31 ref_tracking
: &'rt
mut RefTracking
<(MPlaceTy
<'tcx
>, InternMode
)>,
32 /// A list of all encountered allocations. After type-based interning, we traverse this list to
33 /// also intern allocations that are only referenced by a raw pointer or inside a union.
34 leftover_allocations
: &'rt
mut FxHashSet
<AllocId
>,
35 /// The root kind of the value that we're looking at. This field is never mutated and only used
36 /// for sanity assertions that will ICE when `const_qualif` screws up.
38 /// This field stores whether we are *currently* inside an `UnsafeCell`. This can affect
39 /// the intern mode of references we encounter.
40 inside_unsafe_cell
: bool
,
42 /// This flag is to avoid triggering UnsafeCells are not allowed behind references in constants
44 /// It's a copy of `mir::Body`'s ignore_interior_mut_in_const_validation field
45 ignore_interior_mut_in_const
: bool
,
48 #[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
50 /// A static and its current mutability. Below shared references inside a `static mut`,
51 /// this is *immutable*, and below mutable references inside an `UnsafeCell`, this
53 Static(hir
::Mutability
),
54 /// The "base value" of a const, which can have `UnsafeCell` (as in `const FOO: Cell<i32>`),
55 /// but that interior mutability is simply ignored.
57 /// The "inner values" of a const with references, where `UnsafeCell` is an error.
61 /// Signalling data structure to ensure we don't recurse
62 /// into the memory of other constants or statics
65 fn mutable_memory_in_const(tcx
: TyCtxtAt
<'_
>, kind
: &str) {
66 // FIXME: show this in validation instead so we can point at where in the value the error is?
67 tcx
.sess
.span_err(tcx
.span
, &format
!("mutable memory ({}) is not allowed in constant", kind
));
70 /// Intern an allocation without looking at its children.
71 /// `mode` is the mode of the environment where we found this pointer.
72 /// `mutablity` is the mutability of the place to be interned; even if that says
73 /// `immutable` things might become mutable if `ty` is not frozen.
74 /// `ty` can be `None` if there is no potential interior mutability
75 /// to account for (e.g. for vtables).
76 fn intern_shallow
<'rt
, 'mir
, 'tcx
, M
: CompileTimeMachine
<'mir
, 'tcx
>>(
77 ecx
: &'rt
mut InterpCx
<'mir
, 'tcx
, M
>,
78 leftover_allocations
: &'rt
mut FxHashSet
<AllocId
>,
82 ) -> Option
<IsStaticOrFn
> {
83 trace
!("intern_shallow {:?} with {:?}", alloc_id
, mode
);
86 let (kind
, mut alloc
) = match ecx
.memory
.alloc_map
.remove(&alloc_id
) {
89 // Pointer not found in local memory map. It is either a pointer to the global
91 // If the pointer is dangling (neither in local nor global memory), we leave it
92 // to validation to error -- it has the much better error messages, pointing out where
93 // in the value the dangling reference lies.
94 // The `delay_span_bug` ensures that we don't forget such a check in validation.
95 if tcx
.get_global_alloc(alloc_id
).is_none() {
96 tcx
.sess
.delay_span_bug(ecx
.tcx
.span
, "tried to intern dangling pointer");
98 // treat dangling pointers like other statics
99 // just to stop trying to recurse into them
100 return Some(IsStaticOrFn
);
103 // This match is just a canary for future changes to `MemoryKind`, which most likely need
104 // changes in this function.
106 MemoryKind
::Stack
| MemoryKind
::Vtable
| MemoryKind
::CallerLocation
=> {}
108 // Set allocation mutability as appropriate. This is used by LLVM to put things into
109 // read-only memory, and also by Miri when evaluating other globals that
111 if let InternMode
::Static(mutability
) = mode
{
112 // For this, we need to take into account `UnsafeCell`. When `ty` is `None`, we assume
113 // no interior mutability.
114 let frozen
= ty
.map_or(true, |ty
| ty
.is_freeze(ecx
.tcx
, ecx
.param_env
));
115 // For statics, allocation mutability is the combination of the place mutability and
116 // the type mutability.
117 // The entire allocation needs to be mutable if it contains an `UnsafeCell` anywhere.
118 let immutable
= mutability
== Mutability
::Not
&& frozen
;
120 alloc
.mutability
= Mutability
::Not
;
122 // Just making sure we are not "upgrading" an immutable allocation to mutable.
123 assert_eq
!(alloc
.mutability
, Mutability
::Mut
);
126 // No matter what, *constants are never mutable*. Mutating them is UB.
127 // See const_eval::machine::MemoryExtra::can_access_statics for why
128 // immutability is so important.
130 // There are no sensible checks we can do here; grep for `mutable_memory_in_const` to
131 // find the checks we are doing elsewhere to avoid even getting here for memory
132 // that "wants" to be mutable.
133 alloc
.mutability
= Mutability
::Not
;
135 // link the alloc id to the actual allocation
136 let alloc
= tcx
.intern_const_alloc(alloc
);
137 leftover_allocations
.extend(alloc
.relocations().iter().map(|&(_
, ((), reloc
))| reloc
));
138 tcx
.set_alloc_id_memory(alloc_id
, alloc
);
142 impl<'rt
, 'mir
, 'tcx
, M
: CompileTimeMachine
<'mir
, 'tcx
>> InternVisitor
<'rt
, 'mir
, 'tcx
, M
> {
147 ty
: Option
<Ty
<'tcx
>>,
148 ) -> Option
<IsStaticOrFn
> {
149 intern_shallow(self.ecx
, self.leftover_allocations
, alloc_id
, mode
, ty
)
153 impl<'rt
, 'mir
, 'tcx
: 'mir
, M
: CompileTimeMachine
<'mir
, 'tcx
>> ValueVisitor
<'mir
, 'tcx
, M
>
154 for InternVisitor
<'rt
, 'mir
, 'tcx
, M
>
156 type V
= MPlaceTy
<'tcx
>;
159 fn ecx(&self) -> &InterpCx
<'mir
, 'tcx
, M
> {
165 mplace
: MPlaceTy
<'tcx
>,
166 fields
: impl Iterator
<Item
= InterpResult
<'tcx
, Self::V
>>,
167 ) -> InterpResult
<'tcx
> {
168 if let Some(def
) = mplace
.layout
.ty
.ty_adt_def() {
169 if Some(def
.did
) == self.ecx
.tcx
.lang_items().unsafe_cell_type() {
170 if self.mode
== InternMode
::ConstInner
&& !self.ignore_interior_mut_in_const
{
171 // We do not actually make this memory mutable. But in case the user
172 // *expected* it to be mutable, make sure we error. This is just a
173 // sanity check to prevent users from accidentally exploiting the UB
174 // they caused. It also helps us to find cases where const-checking
175 // failed to prevent an `UnsafeCell` (but as `ignore_interior_mut_in_const`
176 // shows that part is not airtight).
177 mutable_memory_in_const(self.ecx
.tcx
, "`UnsafeCell`");
179 // We are crossing over an `UnsafeCell`, we can mutate again. This means that
180 // References we encounter inside here are interned as pointing to mutable
182 // Remember the `old` value to handle nested `UnsafeCell`.
183 let old
= std
::mem
::replace(&mut self.inside_unsafe_cell
, true);
184 let walked
= self.walk_aggregate(mplace
, fields
);
185 self.inside_unsafe_cell
= old
;
189 self.walk_aggregate(mplace
, fields
)
192 fn visit_value(&mut self, mplace
: MPlaceTy
<'tcx
>) -> InterpResult
<'tcx
> {
193 // Handle Reference types, as these are the only relocations supported by const eval.
194 // Raw pointers (and boxes) are handled by the `leftover_relocations` logic.
195 let tcx
= self.ecx
.tcx
;
196 let ty
= mplace
.layout
.ty
;
197 if let ty
::Ref(_
, referenced_ty
, ref_mutability
) = ty
.kind
{
198 let value
= self.ecx
.read_immediate(mplace
.into())?
;
199 let mplace
= self.ecx
.ref_to_mplace(value
)?
;
200 assert_eq
!(mplace
.layout
.ty
, referenced_ty
);
201 // Handle trait object vtables.
202 if let ty
::Dynamic(..) =
203 tcx
.struct_tail_erasing_lifetimes(referenced_ty
, self.ecx
.param_env
).kind
205 // Validation will error (with a better message) on an invalid vtable pointer
206 // so we can safely not do anything if this is not a real pointer.
207 if let Scalar
::Ptr(vtable
) = mplace
.meta
.unwrap_meta() {
208 // Explicitly choose const mode here, since vtables are immutable, even
209 // if the reference of the fat pointer is mutable.
210 self.intern_shallow(vtable
.alloc_id
, InternMode
::ConstInner
, None
);
212 // Let validation show the error message, but make sure it *does* error.
214 .delay_span_bug(tcx
.span
, "vtables pointers cannot be integer pointers");
217 // Check if we have encountered this pointer+layout combination before.
218 // Only recurse for allocation-backed pointers.
219 if let Scalar
::Ptr(ptr
) = mplace
.ptr
{
220 // Compute the mode with which we intern this.
221 let ref_mode
= match self.mode
{
222 InternMode
::Static(mutbl
) => {
223 // In statics, merge outer mutability with reference mutability and
224 // take into account whether we are in an `UnsafeCell`.
226 // The only way a mutable reference actually works as a mutable reference is
227 // by being in a `static mut` directly or behind another mutable reference.
228 // If there's an immutable reference or we are inside a `static`, then our
229 // mutable reference is equivalent to an immutable one. As an example:
230 // `&&mut Foo` is semantically equivalent to `&&Foo`
231 match ref_mutability
{
232 _
if self.inside_unsafe_cell
=> {
233 // Inside an `UnsafeCell` is like inside a `static mut`, the "outer"
234 // mutability does not matter.
235 InternMode
::Static(ref_mutability
)
238 // A shared reference, things become immutable.
239 // We do *not* consier `freeze` here -- that is done more precisely
240 // when traversing the referenced data (by tracking `UnsafeCell`).
241 InternMode
::Static(Mutability
::Not
)
244 // Mutable reference.
245 InternMode
::Static(mutbl
)
249 InternMode
::ConstBase
| InternMode
::ConstInner
=> {
250 // Ignore `UnsafeCell`, everything is immutable. Do some sanity checking
251 // for mutable references that we encounter -- they must all be ZST.
252 // This helps to prevent users from accidentally exploiting UB that they
253 // caused (by somehow getting a mutable reference in a `const`).
254 if ref_mutability
== Mutability
::Mut
{
255 match referenced_ty
.kind
{
256 ty
::Array(_
, n
) if n
.eval_usize(*tcx
, self.ecx
.param_env
) == 0 => {}
258 if mplace
.meta
.unwrap_meta().to_machine_usize(self.ecx
)?
260 _
=> mutable_memory_in_const(tcx
, "`&mut`"),
263 // A shared reference. We cannot check `freeze` here due to references
264 // like `&dyn Trait` that are actually immutable. We do check for
265 // concrete `UnsafeCell` when traversing the pointee though (if it is
266 // a new allocation, not yet interned).
268 // Go on with the "inner" rules.
269 InternMode
::ConstInner
272 match self.intern_shallow(ptr
.alloc_id
, ref_mode
, Some(referenced_ty
)) {
273 // No need to recurse, these are interned already and statics may have
274 // cycles, so we don't want to recurse there
275 Some(IsStaticOrFn
) => {}
276 // intern everything referenced by this value. The mutability is taken from the
277 // reference. It is checked above that mutable references only happen in
279 None
=> self.ref_tracking
.track((mplace
, ref_mode
), || ()),
284 // Not a reference -- proceed recursively.
285 self.walk_value(mplace
)
290 #[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
291 pub enum InternKind
{
292 /// The `mutability` of the static, ignoring the type which may have interior mutability.
293 Static(hir
::Mutability
),
298 /// Intern `ret` and everything it references.
300 /// This *cannot raise an interpreter error*. Doing so is left to validation, which
301 /// tracks where in the value we are and thus can show much better error messages.
302 /// Any errors here would anyway be turned into `const_err` lints, whereas validation failures
304 pub fn intern_const_alloc_recursive
<M
: CompileTimeMachine
<'mir
, 'tcx
>>(
305 ecx
: &mut InterpCx
<'mir
, 'tcx
, M
>,
306 intern_kind
: InternKind
,
308 ignore_interior_mut_in_const
: bool
,
313 let base_intern_mode
= match intern_kind
{
314 InternKind
::Static(mutbl
) => InternMode
::Static(mutbl
),
315 // FIXME: what about array lengths, array initializers?
316 InternKind
::Constant
| InternKind
::Promoted
=> InternMode
::ConstBase
,
319 // Type based interning.
320 // `ref_tracking` tracks typed references we have already interned and still need to crawl for
321 // more typed information inside them.
322 // `leftover_allocations` collects *all* allocations we see, because some might not
323 // be available in a typed way. They get interned at the end.
324 let mut ref_tracking
= RefTracking
::empty();
325 let leftover_allocations
= &mut FxHashSet
::default();
327 // start with the outermost allocation
330 leftover_allocations
,
331 // The outermost allocation must exist, because we allocated it with
332 // `Memory::allocate`.
333 ret
.ptr
.assert_ptr().alloc_id
,
338 ref_tracking
.track((ret
, base_intern_mode
), || ());
340 while let Some(((mplace
, mode
), _
)) = ref_tracking
.todo
.pop() {
341 let res
= InternVisitor
{
342 ref_tracking
: &mut ref_tracking
,
345 leftover_allocations
,
346 ignore_interior_mut_in_const
,
347 inside_unsafe_cell
: false,
349 .visit_value(mplace
);
350 // We deliberately *ignore* interpreter errors here. When there is a problem, the remaining
351 // references are "leftover"-interned, and later validation will show a proper error
352 // and point at the right part of the value causing the problem.
356 ecx
.tcx
.sess
.delay_span_bug(
359 "error during interning should later cause validation failure: {}",
363 // Some errors shouldn't come up because creating them causes
364 // an allocation, which we should avoid. When that happens,
365 // dedicated error variants should be introduced instead.
367 !error
.kind
.allocates(),
368 "interning encountered allocating error: {}",
375 // Intern the rest of the allocations as mutable. These might be inside unions, padding, raw
376 // pointers, ... So we can't intern them according to their type rules
378 let mut todo
: Vec
<_
> = leftover_allocations
.iter().cloned().collect();
379 while let Some(alloc_id
) = todo
.pop() {
380 if let Some((_
, mut alloc
)) = ecx
.memory
.alloc_map
.remove(&alloc_id
) {
381 // We can't call the `intern_shallow` method here, as its logic is tailored to safe
382 // references and a `leftover_allocations` set (where we only have a todo-list here).
383 // So we hand-roll the interning logic here again.
385 // Statics may contain mutable allocations even behind relocations.
386 // Even for immutable statics it would be ok to have mutable allocations behind
387 // raw pointers, e.g. for `static FOO: *const AtomicUsize = &AtomicUsize::new(42)`.
388 InternKind
::Static(_
) => {}
389 // Raw pointers in promoteds may only point to immutable things so we mark
390 // everything as immutable.
391 // It is UB to mutate through a raw pointer obtained via an immutable reference:
392 // Since all references and pointers inside a promoted must by their very definition
393 // be created from an immutable reference (and promotion also excludes interior
394 // mutability), mutating through them would be UB.
395 // There's no way we can check whether the user is using raw pointers correctly,
396 // so all we can do is mark this as immutable here.
397 InternKind
::Promoted
=> {
398 // See const_eval::machine::MemoryExtra::can_access_statics for why
399 // immutability is so important.
400 alloc
.mutability
= Mutability
::Not
;
402 InternKind
::Constant
=> {
403 // If it's a constant, we should not have any "leftovers" as everything
404 // is tracked by const-checking.
405 // FIXME: downgrade this to a warning? It rejects some legitimate consts,
406 // such as `const CONST_RAW: *const Vec<i32> = &Vec::new() as *const _;`.
409 .span_err(ecx
.tcx
.span
, "untyped pointers are not allowed in constant");
410 // For better errors later, mark the allocation as immutable.
411 alloc
.mutability
= Mutability
::Not
;
414 let alloc
= tcx
.intern_const_alloc(alloc
);
415 tcx
.set_alloc_id_memory(alloc_id
, alloc
);
416 for &(_
, ((), reloc
)) in alloc
.relocations().iter() {
417 if leftover_allocations
.insert(reloc
) {
421 } else if ecx
.memory
.dead_alloc_map
.contains_key(&alloc_id
) {
422 // Codegen does not like dangling pointers, and generally `tcx` assumes that
423 // all allocations referenced anywhere actually exist. So, make sure we error here.
424 ecx
.tcx
.sess
.span_err(ecx
.tcx
.span
, "encountered dangling pointer in final constant");
425 } else if ecx
.tcx
.get_global_alloc(alloc_id
).is_none() {
426 // We have hit an `AllocId` that is neither in local or global memory and isn't
427 // marked as dangling by local memory. That should be impossible.
428 span_bug
!(ecx
.tcx
.span
, "encountered unknown alloc id {:?}", alloc_id
);