1 use super::place
::PlaceRef
;
2 use super::{FunctionCx, LocalRef}
;
10 use rustc_middle
::mir
::interpret
::{ConstValue, Pointer, Scalar}
;
11 use rustc_middle
::ty
::layout
::TyAndLayout
;
12 use rustc_middle
::ty
::Ty
;
13 use rustc_target
::abi
::{Abi, Align, LayoutOf, Size}
;
17 /// The representation of a Rust value. The enum variant is in fact
18 /// uniquely determined by the value's type, but is kept as a
20 #[derive(Copy, Clone, Debug)]
21 pub enum OperandValue
<V
> {
22 /// A reference to the actual operand. The data is guaranteed
23 /// to be valid for the operand's lifetime.
24 /// The second value, if any, is the extra data (vtable or length)
25 /// which indicates that it refers to an unsized rvalue.
26 Ref(V
, Option
<V
>, Align
),
27 /// A single LLVM value.
29 /// A pair of immediate LLVM values. Used by fat pointers too.
33 /// An `OperandRef` is an "SSA" reference to a Rust value, along with
36 /// NOTE: unless you know a value's type exactly, you should not
37 /// generate LLVM opcodes acting on it and instead act via methods,
38 /// to avoid nasty edge cases. In particular, using `Builder::store`
39 /// directly is sure to cause problems -- use `OperandRef::store`
41 #[derive(Copy, Clone)]
42 pub struct OperandRef
<'tcx
, V
> {
44 pub val
: OperandValue
<V
>,
46 // The layout of value, based on its Rust type.
47 pub layout
: TyAndLayout
<'tcx
>,
50 impl<V
: CodegenObject
> fmt
::Debug
for OperandRef
<'tcx
, V
> {
51 fn fmt(&self, f
: &mut fmt
::Formatter
<'_
>) -> fmt
::Result
{
52 write
!(f
, "OperandRef({:?} @ {:?})", self.val
, self.layout
)
56 impl<'a
, 'tcx
, V
: CodegenObject
> OperandRef
<'tcx
, V
> {
57 pub fn new_zst
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
59 layout
: TyAndLayout
<'tcx
>,
60 ) -> OperandRef
<'tcx
, V
> {
61 assert
!(layout
.is_zst());
63 val
: OperandValue
::Immediate(bx
.const_undef(bx
.immediate_backend_type(layout
))),
68 pub fn from_const
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
70 val
: ConstValue
<'tcx
>,
73 let layout
= bx
.layout_of(ty
);
76 return OperandRef
::new_zst(bx
, layout
);
80 ConstValue
::Scalar(x
) => {
81 let scalar
= match layout
.abi
{
82 Abi
::Scalar(ref x
) => x
,
83 _
=> bug
!("from_const: invalid ByVal layout: {:#?}", layout
),
85 let llval
= bx
.scalar_to_backend(x
, scalar
, bx
.immediate_backend_type(layout
));
86 OperandValue
::Immediate(llval
)
88 ConstValue
::Slice { data, start, end }
=> {
89 let a_scalar
= match layout
.abi
{
90 Abi
::ScalarPair(ref a
, _
) => a
,
91 _
=> bug
!("from_const: invalid ScalarPair layout: {:#?}", layout
),
93 let a
= Scalar
::from_pointer(
94 Pointer
::new(bx
.tcx().create_memory_alloc(data
), Size
::from_bytes(start
)),
97 let a_llval
= bx
.scalar_to_backend(
100 bx
.scalar_pair_element_backend_type(layout
, 0, true),
102 let b_llval
= bx
.const_usize((end
- start
) as u64);
103 OperandValue
::Pair(a_llval
, b_llval
)
105 ConstValue
::ByRef { alloc, offset }
=> {
106 return bx
.load_operand(bx
.from_const_alloc(layout
, alloc
, offset
));
110 OperandRef { val, layout }
113 /// Asserts that this operand refers to a scalar and returns
114 /// a reference to its value.
115 pub fn immediate(self) -> V
{
117 OperandValue
::Immediate(s
) => s
,
118 _
=> bug
!("not immediate: {:?}", self),
122 pub fn deref
<Cx
: LayoutTypeMethods
<'tcx
>>(self, cx
: &Cx
) -> PlaceRef
<'tcx
, V
> {
123 let projected_ty
= self
127 .unwrap_or_else(|| bug
!("deref of non-pointer {:?}", self))
129 let (llptr
, llextra
) = match self.val
{
130 OperandValue
::Immediate(llptr
) => (llptr
, None
),
131 OperandValue
::Pair(llptr
, llextra
) => (llptr
, Some(llextra
)),
132 OperandValue
::Ref(..) => bug
!("Deref of by-Ref operand {:?}", self),
134 let layout
= cx
.layout_of(projected_ty
);
135 PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi }
138 /// If this operand is a `Pair`, we return an aggregate with the two values.
139 /// For other cases, see `immediate`.
140 pub fn immediate_or_packed_pair
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
144 if let OperandValue
::Pair(a
, b
) = self.val
{
145 let llty
= bx
.cx().backend_type(self.layout
);
146 debug
!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty
);
147 // Reconstruct the immediate aggregate.
148 let mut llpair
= bx
.cx().const_undef(llty
);
149 let imm_a
= bx
.from_immediate(a
);
150 let imm_b
= bx
.from_immediate(b
);
151 llpair
= bx
.insert_value(llpair
, imm_a
, 0);
152 llpair
= bx
.insert_value(llpair
, imm_b
, 1);
159 /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
160 pub fn from_immediate_or_packed_pair
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
163 layout
: TyAndLayout
<'tcx
>,
165 let val
= if let Abi
::ScalarPair(ref a
, ref b
) = layout
.abi
{
166 debug
!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval
, layout
);
168 // Deconstruct the immediate aggregate.
169 let a_llval
= bx
.extract_value(llval
, 0);
170 let a_llval
= bx
.to_immediate_scalar(a_llval
, a
);
171 let b_llval
= bx
.extract_value(llval
, 1);
172 let b_llval
= bx
.to_immediate_scalar(b_llval
, b
);
173 OperandValue
::Pair(a_llval
, b_llval
)
175 OperandValue
::Immediate(llval
)
177 OperandRef { val, layout }
180 pub fn extract_field
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
185 let field
= self.layout
.field(bx
.cx(), i
);
186 let offset
= self.layout
.fields
.offset(i
);
188 let mut val
= match (self.val
, &self.layout
.abi
) {
189 // If the field is ZST, it has no data.
190 _
if field
.is_zst() => {
191 return OperandRef
::new_zst(bx
, field
);
194 // Newtype of a scalar, scalar pair or vector.
195 (OperandValue
::Immediate(_
) | OperandValue
::Pair(..), _
)
196 if field
.size
== self.layout
.size
=>
198 assert_eq
!(offset
.bytes(), 0);
202 // Extract a scalar component from a pair.
203 (OperandValue
::Pair(a_llval
, b_llval
), &Abi
::ScalarPair(ref a
, ref b
)) => {
204 if offset
.bytes() == 0 {
205 assert_eq
!(field
.size
, a
.value
.size(bx
.cx()));
206 OperandValue
::Immediate(a_llval
)
208 assert_eq
!(offset
, a
.value
.size(bx
.cx()).align_to(b
.value
.align(bx
.cx()).abi
));
209 assert_eq
!(field
.size
, b
.value
.size(bx
.cx()));
210 OperandValue
::Immediate(b_llval
)
214 // `#[repr(simd)]` types are also immediate.
215 (OperandValue
::Immediate(llval
), &Abi
::Vector { .. }
) => {
216 OperandValue
::Immediate(bx
.extract_element(llval
, bx
.cx().const_usize(i
as u64)))
219 _
=> bug
!("OperandRef::extract_field({:?}): not applicable", self),
222 match (&mut val
, &field
.abi
) {
223 (OperandValue
::Immediate(llval
), _
) => {
224 // Bools in union fields needs to be truncated.
225 *llval
= bx
.to_immediate(*llval
, field
);
226 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
227 *llval
= bx
.bitcast(*llval
, bx
.cx().immediate_backend_type(field
));
229 (OperandValue
::Pair(a
, b
), Abi
::ScalarPair(a_abi
, b_abi
)) => {
230 // Bools in union fields needs to be truncated.
231 *a
= bx
.to_immediate_scalar(*a
, a_abi
);
232 *b
= bx
.to_immediate_scalar(*b
, b_abi
);
233 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
234 *a
= bx
.bitcast(*a
, bx
.cx().scalar_pair_element_backend_type(field
, 0, true));
235 *b
= bx
.bitcast(*b
, bx
.cx().scalar_pair_element_backend_type(field
, 1, true));
237 (OperandValue
::Pair(..), _
) => bug
!(),
238 (OperandValue
::Ref(..), _
) => bug
!(),
241 OperandRef { val, layout: field }
245 impl<'a
, 'tcx
, V
: CodegenObject
> OperandValue
<V
> {
246 pub fn store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
249 dest
: PlaceRef
<'tcx
, V
>,
251 self.store_with_flags(bx
, dest
, MemFlags
::empty());
254 pub fn volatile_store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
257 dest
: PlaceRef
<'tcx
, V
>,
259 self.store_with_flags(bx
, dest
, MemFlags
::VOLATILE
);
262 pub fn unaligned_volatile_store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
265 dest
: PlaceRef
<'tcx
, V
>,
267 self.store_with_flags(bx
, dest
, MemFlags
::VOLATILE
| MemFlags
::UNALIGNED
);
270 pub fn nontemporal_store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
273 dest
: PlaceRef
<'tcx
, V
>,
275 self.store_with_flags(bx
, dest
, MemFlags
::NONTEMPORAL
);
278 fn store_with_flags
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
281 dest
: PlaceRef
<'tcx
, V
>,
284 debug
!("OperandRef::store: operand={:?}, dest={:?}", self, dest
);
285 // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
286 // value is through `undef`, and store itself is useless.
287 if dest
.layout
.is_zst() {
291 OperandValue
::Ref(r
, None
, source_align
) => {
292 if flags
.contains(MemFlags
::NONTEMPORAL
) {
293 // HACK(nox): This is inefficient but there is no nontemporal memcpy.
294 let ty
= bx
.backend_type(dest
.layout
);
295 let ptr
= bx
.pointercast(r
, bx
.type_ptr_to(ty
));
296 let val
= bx
.load(ty
, ptr
, source_align
);
297 bx
.store_with_flags(val
, dest
.llval
, dest
.align
, flags
);
300 base
::memcpy_ty(bx
, dest
.llval
, dest
.align
, r
, source_align
, dest
.layout
, flags
)
302 OperandValue
::Ref(_
, Some(_
), _
) => {
303 bug
!("cannot directly store unsized values");
305 OperandValue
::Immediate(s
) => {
306 let val
= bx
.from_immediate(s
);
307 bx
.store_with_flags(val
, dest
.llval
, dest
.align
, flags
);
309 OperandValue
::Pair(a
, b
) => {
310 let (a_scalar
, b_scalar
) = match dest
.layout
.abi
{
311 Abi
::ScalarPair(ref a
, ref b
) => (a
, b
),
312 _
=> bug
!("store_with_flags: invalid ScalarPair layout: {:#?}", dest
.layout
),
314 let b_offset
= a_scalar
.value
.size(bx
).align_to(b_scalar
.value
.align(bx
).abi
);
316 let llptr
= bx
.struct_gep(dest
.llval
, 0);
317 let val
= bx
.from_immediate(a
);
318 let align
= dest
.align
;
319 bx
.store_with_flags(val
, llptr
, align
, flags
);
321 let llptr
= bx
.struct_gep(dest
.llval
, 1);
322 let val
= bx
.from_immediate(b
);
323 let align
= dest
.align
.restrict_for_offset(b_offset
);
324 bx
.store_with_flags(val
, llptr
, align
, flags
);
329 pub fn store_unsized
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
332 indirect_dest
: PlaceRef
<'tcx
, V
>,
334 debug
!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest
);
335 let flags
= MemFlags
::empty();
337 // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
338 let unsized_ty
= indirect_dest
342 .unwrap_or_else(|| bug
!("indirect_dest has non-pointer type: {:?}", indirect_dest
))
345 let (llptr
, llextra
) = if let OperandValue
::Ref(llptr
, Some(llextra
), _
) = self {
348 bug
!("store_unsized called with a sized value")
351 // FIXME: choose an appropriate alignment, or use dynamic align somehow
352 let max_align
= Align
::from_bits(128).unwrap();
353 let min_align
= Align
::from_bits(8).unwrap();
355 // Allocate an appropriate region on the stack, and copy the value into it
356 let (llsize
, _
) = glue
::size_and_align_of_dst(bx
, unsized_ty
, Some(llextra
));
357 let lldst
= bx
.array_alloca(bx
.cx().type_i8(), llsize
, max_align
);
358 bx
.memcpy(lldst
, max_align
, llptr
, min_align
, llsize
, flags
);
360 // Store the allocated region and the extra to the indirect place.
361 let indirect_operand
= OperandValue
::Pair(lldst
, llextra
);
362 indirect_operand
.store(bx
, indirect_dest
);
366 impl<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>> FunctionCx
<'a
, 'tcx
, Bx
> {
367 fn maybe_codegen_consume_direct(
370 place_ref
: mir
::PlaceRef
<'tcx
>,
371 ) -> Option
<OperandRef
<'tcx
, Bx
::Value
>> {
372 debug
!("maybe_codegen_consume_direct(place_ref={:?})", place_ref
);
374 match self.locals
[place_ref
.local
] {
375 LocalRef
::Operand(Some(mut o
)) => {
376 // Moves out of scalar and scalar pair fields are trivial.
377 for elem
in place_ref
.projection
.iter() {
379 mir
::ProjectionElem
::Field(ref f
, _
) => {
380 o
= o
.extract_field(bx
, f
.index());
382 mir
::ProjectionElem
::Index(_
)
383 | mir
::ProjectionElem
::ConstantIndex { .. }
=> {
384 // ZSTs don't require any actual memory access.
385 // FIXME(eddyb) deduplicate this with the identical
386 // checks in `codegen_consume` and `extract_field`.
387 let elem
= o
.layout
.field(bx
.cx(), 0);
389 o
= OperandRef
::new_zst(bx
, elem
);
400 LocalRef
::Operand(None
) => {
401 bug
!("use of {:?} before def", place_ref
);
403 LocalRef
::Place(..) | LocalRef
::UnsizedPlace(..) => {
404 // watch out for locals that do not have an
405 // alloca; they are handled somewhat differently
411 pub fn codegen_consume(
414 place_ref
: mir
::PlaceRef
<'tcx
>,
415 ) -> OperandRef
<'tcx
, Bx
::Value
> {
416 debug
!("codegen_consume(place_ref={:?})", place_ref
);
418 let ty
= self.monomorphized_place_ty(place_ref
);
419 let layout
= bx
.cx().layout_of(ty
);
421 // ZSTs don't require any actual memory access.
423 return OperandRef
::new_zst(bx
, layout
);
426 if let Some(o
) = self.maybe_codegen_consume_direct(bx
, place_ref
) {
430 // for most places, to consume them we just load them
431 // out from their home
432 let place
= self.codegen_place(bx
, place_ref
);
433 bx
.load_operand(place
)
436 pub fn codegen_operand(
439 operand
: &mir
::Operand
<'tcx
>,
440 ) -> OperandRef
<'tcx
, Bx
::Value
> {
441 debug
!("codegen_operand(operand={:?})", operand
);
444 mir
::Operand
::Copy(ref place
) | mir
::Operand
::Move(ref place
) => {
445 self.codegen_consume(bx
, place
.as_ref())
448 mir
::Operand
::Constant(ref constant
) => {
449 // This cannot fail because we checked all required_consts in advance.
450 self.eval_mir_constant_to_operand(bx
, constant
).unwrap_or_else(|_err
| {
451 span_bug
!(constant
.span
, "erroneous constant not captured by required_consts")